@syncular/client 0.0.1 → 0.0.2-126

package/src/mutations.ts

@@ -14,11 +14,19 @@
  */
 
 import type {
+  ColumnCodecDialect,
+  ColumnCodecSource,
   SyncOperation,
   SyncPushRequest,
   SyncPushResponse,
   SyncTransport,
 } from '@syncular/core';
+import {
+  applyCodecsToDbRow,
+  isRecord,
+  randomId,
+  toTableColumnCodecs,
+} from '@syncular/core';
 import type { Insertable, Kysely, Transaction, Updateable } from 'kysely';
 import { sql } from 'kysely';
 import { enqueueOutboxCommit } from './outbox';
@@ -138,21 +146,6 @@ export type MutationsApi<DB, CommitOptions = unknown> = {
   [T in KnownTableKey<DB>]: TableMutations<DB, T>;
 };
 
-function randomId(): string {
-  if (
-    typeof crypto !== 'undefined' &&
-    typeof crypto.randomUUID === 'function'
-  ) {
-    return crypto.randomUUID();
-  }
-  // Very small fallback; good enough for tests.
-  return `${Date.now()}-${Math.random().toString(16).slice(2)}`;
-}
-
-function isRecord(value: unknown): value is Record<string, unknown> {
-  return typeof value === 'object' && value !== null && !Array.isArray(value);
-}
-
 function sanitizePayload(
   payload: Record<string, unknown>,
   args: { omit: string[] }
@@ -268,6 +261,32 @@ async function dynamicInsert<T>(
   `.execute(trx);
 }
 
+async function dynamicUpsert<T>(
+  trx: Transaction<T>,
+  table: string,
+  idColumn: string,
+  id: string,
+  values: Record<string, unknown>
+): Promise<void> {
+  validateTableName(table);
+  validateColumnName(idColumn);
+
+  // Check if the row already exists
+  const existing = await sql`
+    select 1 from ${sql.table(table)}
+    where ${sql.ref(idColumn)} = ${sql.val(id)}
+    limit 1
+  `.execute(trx);
+
+  if (existing.rows.length > 0) {
+    // Row exists: just update the provided columns
+    await dynamicUpdate(trx, table, idColumn, id, values);
+  } else {
+    // Row doesn't exist: insert with all provided columns + id
+    await dynamicInsert(trx, table, { ...values, [idColumn]: id });
+  }
+}
+
 async function dynamicUpdate<T>(
   trx: Transaction<T>,
   table: string,
@@ -381,6 +400,8 @@ export interface OutboxCommitConfig<DB extends SyncClientDb> {
   idColumn?: string;
   versionColumn?: string | null;
   omitColumns?: string[];
+  columnCodecs?: ColumnCodecSource;
+  codecDialect?: ColumnCodecDialect;
 }
 
 export function createOutboxCommit<DB extends SyncClientDb>(
@@ -389,6 +410,7 @@ export function createOutboxCommit<DB extends SyncClientDb>(
   const idColumn = config.idColumn ?? 'id';
   const versionColumn = config.versionColumn ?? 'server_version';
   const omitColumns = config.omitColumns ?? [];
+  const codecDialect = config.codecDialect ?? 'sqlite';
 
   return async (fn) => {
     const operations: SyncOperation[] = [];
@@ -402,6 +424,38 @@ export function createOutboxCommit<DB extends SyncClientDb>(
       .transaction()
       .execute(async (trx) => {
         const txTableCache = new Map<string, any>();
+        const tableCodecCache = new Map<
+          string,
+          Map<string, ReturnType<typeof toTableColumnCodecs>>
+        >();
+        const resolveTableCodecs = (
+          table: string,
+          row: Record<string, unknown>
+        ) => {
+          const columnCodecs = config.columnCodecs;
+          if (!columnCodecs) return {};
+          const columns = Object.keys(row);
+          if (columns.length === 0) return {};
+
+          let tableCache = tableCodecCache.get(table);
+          if (!tableCache) {
+            tableCache = new Map<
+              string,
+              ReturnType<typeof toTableColumnCodecs>
+            >();
+            tableCodecCache.set(table, tableCache);
+          }
+
+          const cacheKey = columns.slice().sort().join('\u0000');
+          const cached = tableCache.get(cacheKey);
+          if (cached) return cached;
+
+          const resolved = toTableColumnCodecs(table, columnCodecs, columns, {
+            dialect: codecDialect,
+          });
+          tableCache.set(cacheKey, resolved);
+          return resolved;
+        };
 
         const makeTxTable = (table: string) => {
           const cached = txTableCache.get(table);
@@ -415,8 +469,13 @@ export function createOutboxCommit<DB extends SyncClientDb>(
                 typeof rawId === 'string' && rawId ? rawId : randomId();
 
               const row = { ...raw, [idColumn]: id };
+              const dbRow = applyCodecsToDbRow(
+                row,
+                resolveTableCodecs(table, row),
+                codecDialect
+              );
 
-              await dynamicInsert(trx, table, row);
+              await dynamicInsert(trx, table, dbRow);
 
               const payload = sanitizePayload(row, {
                 omit: [
@@ -451,8 +510,16 @@ export function createOutboxCommit<DB extends SyncClientDb>(
                 toInsert.push({ ...raw, [idColumn]: id });
               }
 
-              if (toInsert.length > 0) {
-                await dynamicInsert(trx, table, toInsert);
+              const dbRows = toInsert.map((row) =>
+                applyCodecsToDbRow(
+                  row,
+                  resolveTableCodecs(table, row),
+                  codecDialect
+                )
+              );
+
+              if (dbRows.length > 0) {
+                await dynamicInsert(trx, table, dbRows);
               }
 
               for (let i = 0; i < toInsert.length; i++) {
@@ -492,8 +559,13 @@ export function createOutboxCommit<DB extends SyncClientDb>(
 
               const hasExplicitBaseVersion =
                 !!opts && hasOwn(opts, 'baseVersion');
+              const dbPatch = applyCodecsToDbRow(
+                sanitized,
+                resolveTableCodecs(table, sanitized),
+                codecDialect
+              );
 
-              await dynamicUpdate(trx, table, idColumn, id, sanitized);
+              await dynamicUpdate(trx, table, idColumn, id, dbPatch);
 
               const baseVersion = hasExplicitBaseVersion
                 ? (opts!.baseVersion ?? null)
@@ -547,7 +619,46 @@ export function createOutboxCommit<DB extends SyncClientDb>(
             },
 
             async upsert(id, patch, opts) {
-              await tableApi.update(id, patch, opts);
+              const rawPatch = isRecord(patch) ? patch : {};
+              const sanitized = sanitizePayload(rawPatch, {
+                omit: [
+                  idColumn,
+                  ...(versionColumn ? [versionColumn] : []),
+                  ...omitColumns,
+                ],
+              });
+
+              const hasExplicitBaseVersion =
+                !!opts && hasOwn(opts, 'baseVersion');
+              const dbPatch = applyCodecsToDbRow(
+                sanitized,
+                resolveTableCodecs(table, sanitized),
+                codecDialect
+              );
+
+              await dynamicUpsert(trx, table, idColumn, id, dbPatch);
+
+              const baseVersion = hasExplicitBaseVersion
+                ? (opts!.baseVersion ?? null)
+                : versionColumn
+                  ? await readBaseVersion({
+                      trx: trx,
+                      table,
+                      rowId: id,
+                      idColumn,
+                      versionColumn,
+                    })
+                  : null;
+
+              operations.push({
+                table: table,
+                row_id: id,
+                op: 'upsert',
+                payload: sanitized,
+                base_version: coerceBaseVersion(baseVersion),
+              });
+
+              localMutations.push({ table, rowId: id, op: 'upsert' });
             },
           };
 
@@ -814,7 +925,18 @@ export function createPushCommit<DB = AnyDb>(
       }
     }
 
-    const rawResponse = await config.transport.push(requestToSend);
+    const combined = await config.transport.sync({
+      clientId: requestToSend.clientId,
+      push: {
+        clientCommitId: requestToSend.clientCommitId,
+        operations: requestToSend.operations,
+        schemaVersion: requestToSend.schemaVersion,
+      },
+    });
+    if (!combined.push) {
+      throw new Error('Server returned no push response');
+    }
+    const rawResponse = combined.push;
 
     let response = rawResponse;
     if (sortedPlugins.length > 0) {

package/src/outbox.ts

@@ -3,6 +3,7 @@
  */
 
 import type { SyncOperation } from '@syncular/core';
+import { isRecord, randomId } from '@syncular/core';
 import type { Kysely } from 'kysely';
 import { sql } from 'kysely';
 import type { OutboxCommitStatus, SyncClientDb } from './schema';
@@ -22,21 +23,6 @@ export interface OutboxCommit {
   schema_version: number;
 }
 
-function randomId(): string {
-  if (
-    typeof crypto !== 'undefined' &&
-    typeof crypto.randomUUID === 'function'
-  ) {
-    return crypto.randomUUID();
-  }
-  // Very small fallback; good enough for tests.
-  return `${Date.now()}-${Math.random().toString(16).slice(2)}`;
-}
-
-function isRecord(value: unknown): value is Record<string, unknown> {
-  return typeof value === 'object' && value !== null && !Array.isArray(value);
-}
-
 function isSyncOperation(value: unknown): value is SyncOperation {
   if (!isRecord(value)) return false;
   if (typeof value.table !== 'string') return false;

package/src/plugins/incrementing-version.ts

@@ -50,7 +50,7 @@ function touchLru(
  * - Operations are pushed in commit order for a given client (outbox ordering).
  *
  * This plugin:
- * - Tracks the "next expected server version" per (shape, table, row_id) based on
+ * - Tracks the "next expected server version" per (table, row_id) based on
  *   successfully applied pushes.
  * - Rewrites outgoing `base_version` to that expected version when it is higher
  *   than the caller-provided value, preventing "self-conflicts" on hot rows.
@@ -122,7 +122,7 @@ export function createIncrementingVersionPlugin(
             maxTrackedRows
           );
         } else {
-          // Insert case: most shapes start at version 1.
+          // Insert case: most tables start at version 1.
           touchLru(nextExpectedBaseVersionByRow, key, 1, maxTrackedRows);
         }
       }

package/src/pull-engine.test.ts

@@ -0,0 +1,147 @@
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
+import { gzipSync } from 'node:zlib';
+import {
+  encodeSnapshotRows,
+  type SyncPullResponse,
+  type SyncTransport,
+} from '@syncular/core';
+import { type Kysely, sql } from 'kysely';
+import { createBunSqliteDb } from '../../dialect-bun-sqlite/src';
+import { createClientHandler } from './handlers/create-handler';
+import { ClientTableRegistry } from './handlers/registry';
+import { ensureClientSyncSchema } from './migrate';
+import { applyPullResponse, buildPullRequest } from './pull-engine';
+import type { SyncClientDb } from './schema';
+
+interface ItemsTable {
+  id: string;
+  name: string;
+}
+
+interface TestDb extends SyncClientDb {
+  items: ItemsTable;
+}
+
+function createStreamFromBytes(
+  bytes: Uint8Array,
+  chunkSize = 1024
+): ReadableStream<Uint8Array> {
+  return new ReadableStream<Uint8Array>({
+    start(controller) {
+      for (let index = 0; index < bytes.length; index += chunkSize) {
+        controller.enqueue(bytes.subarray(index, index + chunkSize));
+      }
+      controller.close();
+    },
+  });
+}
+
+describe('applyPullResponse chunk streaming', () => {
+  let db: Kysely<TestDb>;
+
+  beforeEach(async () => {
+    db = createBunSqliteDb<TestDb>({ path: ':memory:' });
+    await ensureClientSyncSchema(db);
+    await db.schema
+      .createTable('items')
+      .addColumn('id', 'text', (col) => col.primaryKey())
+      .addColumn('name', 'text', (col) => col.notNull())
+      .execute();
+  });
+
+  afterEach(async () => {
+    await db.destroy();
+  });
+
+  it('applies chunked bootstrap snapshots using streaming transport', async () => {
+    const rows = Array.from({ length: 5000 }, (_, index) => ({
+      id: `${index + 1}`,
+      name: `Item ${index + 1}`,
+    }));
+    const encoded = encodeSnapshotRows(rows);
+    const compressed = new Uint8Array(gzipSync(encoded));
+
+    let streamFetchCount = 0;
+    const transport: SyncTransport = {
+      async sync() {
+        return {};
+      },
+      async fetchSnapshotChunk() {
+        throw new Error('fetchSnapshotChunk should not be used');
+      },
+      async fetchSnapshotChunkStream() {
+        streamFetchCount += 1;
+        return createStreamFromBytes(compressed, 257);
+      },
+    };
+
+    const handlers = new ClientTableRegistry<TestDb>().register(
+      createClientHandler({
+        table: 'items',
+        scopes: ['items:{id}'],
+      })
+    );
+
+    const options = {
+      clientId: 'client-1',
+      subscriptions: [
+        {
+          id: 'items-sub',
+          table: 'items',
+          scopes: {},
+        },
+      ],
+      stateId: 'default',
+    };
+
+    const pullState = await buildPullRequest(db, options);
+
+    const response: SyncPullResponse = {
+      ok: true,
+      subscriptions: [
+        {
+          id: 'items-sub',
+          status: 'active',
+          scopes: {},
+          bootstrap: true,
+          bootstrapState: null,
+          nextCursor: 1,
+          commits: [],
+          snapshots: [
+            {
+              table: 'items',
+              rows: [],
+              chunks: [
+                {
+                  id: 'chunk-1',
+                  byteLength: compressed.length,
+                  sha256: '',
+                  encoding: 'json-row-frame-v1',
+                  compression: 'gzip',
+                },
+              ],
+              isFirstPage: true,
+              isLastPage: true,
+            },
+          ],
+        },
+      ],
+    };
+
+    await applyPullResponse(
+      db,
+      transport,
+      handlers,
+      options,
+      pullState,
+      response
+    );
+
+    const countResult = await sql<{ count: number }>`
+      select count(*) as count
+      from ${sql.table('items')}
+    `.execute(db);
+    expect(Number(countResult.rows[0]?.count ?? 0)).toBe(rows.length);
+    expect(streamFetchCount).toBe(1);
+  });
+});