@syncular/client 0.0.1 → 0.0.2-127

package/src/client.test.ts

@@ -0,0 +1,369 @@
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
+import type { SyncTransport } from '@syncular/core';
+import type { Kysely } from 'kysely';
+import { sql } from 'kysely';
+import { createBunSqliteDb } from '../../dialect-bun-sqlite/src';
+import { ensureClientBlobSchema } from './blobs/migrate';
+import { Client, type ClientBlobStorage } from './client';
+import { SyncEngine } from './engine/SyncEngine';
+import { ClientTableRegistry } from './handlers/registry';
+import { ensureClientSyncSchema } from './migrate';
+import type { SyncClientDb } from './schema';
+
+interface TasksTable {
+  id: string;
+  title: string;
+  server_version: number;
+}
+
+interface TestDb extends SyncClientDb {
+  tasks: TasksTable;
+}
+
+const noopTransport: SyncTransport = {
+  async sync() {
+    return {};
+  },
+  async fetchSnapshotChunk() {
+    return new Uint8Array();
+  },
+};
+
+function createMemoryBlobStorage(): ClientBlobStorage {
+  const memory = new Map<string, Uint8Array>();
+  return {
+    async write(hash, data) {
+      if (data instanceof ReadableStream) {
+        const reader = data.getReader();
+        const chunks: Uint8Array[] = [];
+        let total = 0;
+        while (true) {
+          const chunk = await reader.read();
+          if (chunk.done) break;
+          chunks.push(chunk.value);
+          total += chunk.value.length;
+        }
+        const combined = new Uint8Array(total);
+        let offset = 0;
+        for (const chunk of chunks) {
+          combined.set(chunk, offset);
+          offset += chunk.length;
+        }
+        memory.set(hash, combined);
+        return;
+      }
+      memory.set(hash, new Uint8Array(data));
+    },
+    async read(hash) {
+      const data = memory.get(hash);
+      return data ? new Uint8Array(data) : null;
+    },
+    async delete(hash) {
+      memory.delete(hash);
+    },
+    async exists(hash) {
+      return memory.has(hash);
+    },
+  };
+}
+
+describe('Client conflict events', () => {
+  let db: Kysely<TestDb>;
+  let client: Client<TestDb>;
+  let engine: SyncEngine<TestDb>;
+
+  async function seedConflict(id: string): Promise<void> {
+    const now = Date.now();
+    await sql`
+      insert into ${sql.table('sync_outbox_commits')} (
+        ${sql.ref('id')},
+        ${sql.ref('client_commit_id')},
+        ${sql.ref('status')},
+        ${sql.ref('operations_json')},
+        ${sql.ref('last_response_json')},
+        ${sql.ref('error')},
+        ${sql.ref('created_at')},
+        ${sql.ref('updated_at')},
+        ${sql.ref('attempt_count')},
+        ${sql.ref('acked_commit_seq')},
+        ${sql.ref('schema_version')}
+      ) values (
+        ${'outbox-1'},
+        ${'commit-1'},
+        ${'failed'},
+        ${JSON.stringify([
+          {
+            table: 'tasks',
+            row_id: 't1',
+            op: 'upsert',
+            payload: { id: 't1', title: 'local', server_version: 1 },
+          },
+        ])},
+        ${null},
+        ${'conflict'},
+        ${now},
+        ${now},
+        ${1},
+        ${null},
+        ${1}
+      )
+    `.execute(db);
+
+    await sql`
+      insert into ${sql.table('sync_conflicts')} (
+        ${sql.ref('id')},
+        ${sql.ref('outbox_commit_id')},
+        ${sql.ref('client_commit_id')},
+        ${sql.ref('op_index')},
+        ${sql.ref('result_status')},
+        ${sql.ref('message')},
+        ${sql.ref('code')},
+        ${sql.ref('server_version')},
+        ${sql.ref('server_row_json')},
+        ${sql.ref('created_at')},
+        ${sql.ref('resolved_at')},
+        ${sql.ref('resolution')}
+      ) values (
+        ${id},
+        ${'outbox-1'},
+        ${'commit-1'},
+        ${0},
+        ${'conflict'},
+        ${'server conflict'},
+        ${'CONFLICT'},
+        ${2},
+        ${JSON.stringify({ id: 't1', title: 'server', server_version: 2 })},
+        ${now},
+        ${null},
+        ${null}
+      )
+    `.execute(db);
+  }
+
+  async function runConflictCheck(
+    clientInstance: Client<TestDb>
+  ): Promise<void> {
+    const checker = Reflect.get(clientInstance, 'checkForNewConflicts');
+    if (typeof checker !== 'function') {
+      throw new Error('Expected checkForNewConflicts to be callable');
+    }
+    await checker.call(clientInstance);
+  }
+
+  beforeEach(async () => {
+    db = createBunSqliteDb<TestDb>({ path: ':memory:' });
+    await ensureClientSyncSchema(db);
+    await db.schema
+      .createTable('tasks')
+      .addColumn('id', 'text', (col) => col.primaryKey())
+      .addColumn('title', 'text', (col) => col.notNull())
+      .addColumn('server_version', 'integer', (col) =>
+        col.notNull().defaultTo(0)
+      )
+      .execute();
+
+    const handlers = new ClientTableRegistry<TestDb>();
+    client = new Client<TestDb>({
+      db,
+      transport: noopTransport,
+      tableHandlers: handlers,
+      clientId: 'client-1',
+      actorId: 'u1',
+      subscriptions: [],
+    });
+
+    engine = new SyncEngine<TestDb>({
+      db,
+      transport: noopTransport,
+      handlers: handlers,
+      actorId: 'u1',
+      clientId: 'client-1',
+      subscriptions: [],
+    });
+    Reflect.set(client, 'engine', engine);
+  });
+
+  afterEach(async () => {
+    client.destroy();
+    await db.destroy();
+  });
+
+  it('emits conflict:resolved with the resolved conflict payload', async () => {
+    await seedConflict('conflict-1');
+
+    const resolvedEvents: Array<{ id: string }> = [];
+    client.on('conflict:resolved', (conflict) => {
+      resolvedEvents.push({ id: conflict.id });
+    });
+
+    await client.resolveConflict('conflict-1', { strategy: 'keep-local' });
+
+    expect(resolvedEvents).toEqual([{ id: 'conflict-1' }]);
+
+    const resolvedRow = await sql<{ resolved_at: number | null }>`
+      select ${sql.ref('resolved_at')}
+      from ${sql.table('sync_conflicts')}
+      where ${sql.ref('id')} = ${'conflict-1'}
+      limit 1
+    `.execute(db);
+    expect(resolvedRow.rows[0]?.resolved_at).not.toBeNull();
+  });
+
+  it('emits conflict:new only once per unresolved conflict id', async () => {
+    await seedConflict('conflict-1');
+
+    const newEvents: string[] = [];
+    client.on('conflict:new', (conflict) => {
+      newEvents.push(conflict.id);
+    });
+
+    await runConflictCheck(client);
+    await runConflictCheck(client);
+
+    expect(newEvents).toEqual(['conflict-1']);
+  });
+});
+
+describe('Client blob upload queue recovery', () => {
+  let db: Kysely<TestDb>;
+  let client: Client<TestDb>;
+  let initiateCalls = 0;
+
+  async function insertBlobOutboxRow(input: {
+    hash: string;
+    status: string;
+    attemptCount: number;
+    updatedAt: number;
+  }): Promise<void> {
+    const now = Date.now();
+    await sql`
+      insert into ${sql.table('sync_blob_outbox')} (
+        ${sql.join([
+          sql.ref('hash'),
+          sql.ref('size'),
+          sql.ref('mime_type'),
+          sql.ref('body'),
+          sql.ref('encrypted'),
+          sql.ref('key_id'),
+          sql.ref('status'),
+          sql.ref('attempt_count'),
+          sql.ref('error'),
+          sql.ref('created_at'),
+          sql.ref('updated_at'),
+        ])}
+      ) values (
+        ${sql.join([
+          sql.val(input.hash),
+          sql.val(3),
+          sql.val('application/octet-stream'),
+          sql.val(new Uint8Array([1, 2, 3])),
+          sql.val(0),
+          sql.val(null),
+          sql.val(input.status),
+          sql.val(input.attemptCount),
+          sql.val(null),
+          sql.val(now),
+          sql.val(input.updatedAt),
+        ])}
+      )
+    `.execute(db);
+  }
+
+  beforeEach(async () => {
+    db = createBunSqliteDb<TestDb>({ path: ':memory:' });
+    await ensureClientSyncSchema(db);
+    await ensureClientBlobSchema(db);
+    initiateCalls = 0;
+
+    const handlers = new ClientTableRegistry<TestDb>();
+    const transport: SyncTransport = {
+      ...noopTransport,
+      blobs: {
+        async initiateUpload() {
+          initiateCalls++;
+          return { exists: true };
+        },
+        async completeUpload() {
+          return { ok: true };
+        },
+        async getDownloadUrl() {
+          return {
+            url: 'https://example.invalid/blob',
+            expiresAt: new Date(0).toISOString(),
+          };
+        },
+      },
+    };
+
+    client = new Client<TestDb>({
+      db,
+      transport,
+      tableHandlers: handlers,
+      blobStorage: createMemoryBlobStorage(),
+      clientId: 'client-1',
+      actorId: 'u1',
+      subscriptions: [],
+    });
+  });
+
+  afterEach(async () => {
+    client.destroy();
+    await db.destroy();
+  });
+
+  it('requeues stale uploading rows and uploads them on the next queue run', async () => {
+    await insertBlobOutboxRow({
+      hash: 'sha256:stale-upload',
+      status: 'uploading',
+      attemptCount: 0,
+      updatedAt: Date.now() - 31_000,
+    });
+
+    const result = await client.blobs!.processUploadQueue();
+
+    expect(result.uploaded).toBe(1);
+    expect(result.failed).toBe(0);
+    expect(initiateCalls).toBe(1);
+
+    const remaining = await sql<{ count: number | bigint }>`
+      select count(${sql.ref('hash')}) as count
+      from ${sql.table('sync_blob_outbox')}
+      where ${sql.ref('hash')} = ${'sha256:stale-upload'}
+    `.execute(db);
+    expect(Number(remaining.rows[0]?.count ?? 0)).toBe(0);
+  });
+
+  it('marks stale uploading rows as failed after max retries', async () => {
+    await insertBlobOutboxRow({
+      hash: 'sha256:stale-failed',
+      status: 'uploading',
+      attemptCount: 2,
+      updatedAt: Date.now() - 31_000,
+    });
+
+    await client.blobs!.processUploadQueue();
+
+    expect(initiateCalls).toBe(0);
+
+    const rowResult = await sql<{
+      status: string;
+      attempt_count: number;
+      error: string | null;
+    }>`
+      select
+        ${sql.ref('status')} as status,
+        ${sql.ref('attempt_count')} as attempt_count,
+        ${sql.ref('error')} as error
+      from ${sql.table('sync_blob_outbox')}
+      where ${sql.ref('hash')} = ${'sha256:stale-failed'}
+      limit 1
+    `.execute(db);
+    const row = rowResult.rows[0];
+    if (!row) {
+      throw new Error('Expected stale failed row to remain in outbox');
+    }
+    expect(row.status).toBe('failed');
+    expect(row.attempt_count).toBe(3);
+    expect(row.error).toContain('Upload timed out while in uploading state');
+  });
+});

package/src/client.ts

@@ -9,7 +9,12 @@
  * - Conflict handling with events
  */
 
-import type { BlobRef, SyncTransport } from '@syncular/core';
+import type {
+  BlobRef,
+  ColumnCodecDialect,
+  ColumnCodecSource,
+  SyncTransport,
+} from '@syncular/core';
 import type { Kysely } from 'kysely';
 import { sql } from 'kysely';
 import { ensureClientBlobSchema } from './blobs/migrate';
@@ -85,7 +90,7 @@ export interface ClientOptions<DB extends SyncClientDb> {
   /** Subscriptions to sync */
   subscriptions: Array<{
     id: string;
-    shape: string;
+    table: string;
     scopes?: Record<string, string | string[]>;
     params?: Record<string, unknown>;
   }>;
@@ -113,6 +118,12 @@ export interface ClientOptions<DB extends SyncClientDb> {
 
   /** Optional: Columns to omit from sync */
   omitColumns?: string[];
+
+  /** Optional: Column codec resolver */
+  columnCodecs?: ColumnCodecSource;
+
+  /** Optional: Codec dialect override (default: 'sqlite') */
+  codecDialect?: ColumnCodecDialect;
 }
 
 export interface ClientState {
@@ -251,7 +262,7 @@ type ClientEventHandler<E extends ClientEventType> = (
  *   tableHandlers,
  *   clientId: 'device-123',
  *   actorId: 'user-456',
- *   subscriptions: [{ id: 'tasks', shape: 'tasks', scopes: { user_id: 'user-456' } }],
+ *   subscriptions: [{ id: 'tasks', table: 'tasks', scopes: { user_id: 'user-456' } }],
  * });
  *
  * await client.start();
@@ -268,6 +279,7 @@ export class Client<DB extends SyncClientDb = SyncClientDb> {
   private engine: SyncEngine<DB> | null = null;
   private started = false;
   private destroyed = false;
+  private emittedConflictIds = new Set<string>();
   private eventListeners = new Map<
     ClientEventType,
     Set<ClientEventHandler<any>>
@@ -295,6 +307,8 @@ export class Client<DB extends SyncClientDb = SyncClientDb> {
       idColumn: options.idColumn ?? 'id',
       versionColumn: options.versionColumn ?? 'server_version',
       omitColumns: options.omitColumns ?? [],
+      columnCodecs: options.columnCodecs,
+      codecDialect: options.codecDialect,
     });
     this.mutations = createMutationsApi(commitFn) as MutationsApi<DB>;
 
@@ -352,12 +366,12 @@ export class Client<DB extends SyncClientDb = SyncClientDb> {
     this.engine = new SyncEngine({
       db: this.options.db,
       transport: this.options.transport,
-      shapes: this.options.tableHandlers,
+      handlers: this.options.tableHandlers,
       clientId: this.options.clientId,
       actorId: this.options.actorId,
       subscriptions: this.options.subscriptions.map((s) => ({
         id: s.id,
-        shape: s.shape,
+        table: s.table,
         scopes: s.scopes ?? {},
         params: s.params ?? {},
       })),
@@ -415,7 +429,7 @@ export class Client<DB extends SyncClientDb = SyncClientDb> {
   updateSubscriptions(
     subscriptions: Array<{
       id: string;
-      shape: string;
+      table: string;
       scopes?: Record<string, string | string[]>;
       params?: Record<string, unknown>;
     }>
@@ -425,7 +439,7 @@ export class Client<DB extends SyncClientDb = SyncClientDb> {
       this.engine.updateSubscriptions(
         subscriptions.map((s) => ({
           id: s.id,
-          shape: s.shape,
+          table: s.table,
           scopes: s.scopes ?? {},
           params: s.params ?? {},
         }))
@@ -438,13 +452,13 @@ export class Client<DB extends SyncClientDb = SyncClientDb> {
    */
   getSubscriptions(): Array<{
     id: string;
-    shape: string;
+    table: string;
     scopes: Record<string, string | string[]>;
     params: Record<string, unknown>;
   }> {
     return this.options.subscriptions.map((s) => ({
       id: s.id,
-      shape: s.shape,
+      table: s.table,
       scopes: s.scopes ?? {},
       params: s.params ?? {},
     }));
@@ -545,6 +559,8 @@ export class Client<DB extends SyncClientDb = SyncClientDb> {
     resolution: ConflictResolution
   ): Promise<void> {
     const { resolveConflict } = await import('./conflicts');
+    const pendingBeforeResolve = await this.getConflicts();
+    const resolvedConflict = pendingBeforeResolve.find((c) => c.id === id);
 
     // For 'keep-local' and 'keep-server', we just mark it resolved
     // For 'custom', we would need to apply the payload - but that requires
@@ -556,11 +572,9 @@ export class Client<DB extends SyncClientDb = SyncClientDb> {
 
     await resolveConflict(this.options.db, { id, resolution: resolutionStr });
 
-    // Get the conflict for the event
-    const conflicts = await this.getConflicts();
-    const resolved = conflicts.find((c) => c.id === id);
-    if (resolved) {
-      this.emit('conflict:resolved', resolved);
+    this.emittedConflictIds.delete(id);
+    if (resolvedConflict) {
+      this.emit('conflict:resolved', resolvedConflict);
     }
   }
 
@@ -806,7 +820,19 @@ export class Client<DB extends SyncClientDb = SyncClientDb> {
 
   private async checkForNewConflicts(): Promise<void> {
     const conflicts = await this.getConflicts();
+    const activeIds = new Set(conflicts.map((conflict) => conflict.id));
+
+    for (const id of this.emittedConflictIds) {
+      if (!activeIds.has(id)) {
+        this.emittedConflictIds.delete(id);
+      }
+    }
+
     for (const conflict of conflicts) {
+      if (this.emittedConflictIds.has(conflict.id)) {
+        continue;
+      }
+      this.emittedConflictIds.add(conflict.id);
       this.emit('conflict:new', conflict);
     }
   }
@@ -841,6 +867,8 @@ export class Client<DB extends SyncClientDb = SyncClientDb> {
   ): BlobClient {
     const db = this.options.db;
     const blobs = transport.blobs!;
+    const staleUploadingTimeoutMs = 30_000;
+    const maxUploadRetries = 3;
 
     return {
       async store(data, options) {
@@ -1019,33 +1047,78 @@ export class Client<DB extends SyncClientDb = SyncClientDb> {
       async processUploadQueue() {
         let uploaded = 0;
         let failed = 0;
+        const now = Date.now();
+        const staleThreshold = now - staleUploadingTimeoutMs;
+
+        await sql`
+          update ${sql.table('sync_blob_outbox')}
+          set
+            ${sql.ref('status')} = ${sql.val('failed')},
+            ${sql.ref('attempt_count')} = ${sql.ref('attempt_count')} + ${sql.val(
+              1
+            )},
+            ${sql.ref('error')} = ${sql.val(
+              'Upload timed out while in uploading state'
+            )},
+            ${sql.ref('updated_at')} = ${sql.val(now)}
+          where ${sql.ref('status')} = ${sql.val('uploading')}
+            and ${sql.ref('updated_at')} < ${sql.val(staleThreshold)}
+            and ${sql.ref('attempt_count')} + ${sql.val(1)} >= ${sql.val(
+              maxUploadRetries
+            )}
+        `.execute(db);
+
+        await sql`
+          update ${sql.table('sync_blob_outbox')}
+          set
+            ${sql.ref('status')} = ${sql.val('pending')},
+            ${sql.ref('attempt_count')} = ${sql.ref('attempt_count')} + ${sql.val(
+              1
+            )},
+            ${sql.ref('error')} = ${sql.val(
+              'Upload timed out while in uploading state; retrying'
+            )},
+            ${sql.ref('updated_at')} = ${sql.val(now)}
+          where ${sql.ref('status')} = ${sql.val('uploading')}
+            and ${sql.ref('updated_at')} < ${sql.val(staleThreshold)}
+            and ${sql.ref('attempt_count')} + ${sql.val(1)} < ${sql.val(
+              maxUploadRetries
+            )}
+        `.execute(db);
 
         const pendingResult = await sql<{
           hash: string;
           size: number;
           mime_type: string;
           body: Uint8Array | null;
+          attempt_count: number;
         }>`
           select
             ${sql.ref('hash')},
             ${sql.ref('size')},
             ${sql.ref('mime_type')},
-            ${sql.ref('body')}
+            ${sql.ref('body')},
+            ${sql.ref('attempt_count')}
           from ${sql.table('sync_blob_outbox')}
           where ${sql.ref('status')} = ${sql.val('pending')}
+            and ${sql.ref('attempt_count')} < ${sql.val(maxUploadRetries)}
           limit ${sql.val(10)}
         `.execute(db);
         const pending = pendingResult.rows;
 
         for (const item of pending) {
+          const nextAttemptCount = item.attempt_count + 1;
           try {
             // Mark as uploading
             await sql`
               update ${sql.table('sync_blob_outbox')}
               set
                 ${sql.ref('status')} = ${sql.val('uploading')},
+                ${sql.ref('attempt_count')} = ${sql.val(nextAttemptCount)},
+                ${sql.ref('error')} = ${sql.val(null)},
                 ${sql.ref('updated_at')} = ${sql.val(Date.now())}
               where ${sql.ref('hash')} = ${sql.val(item.hash)}
+                and ${sql.ref('status')} = ${sql.val('pending')}
             `.execute(db);
 
             // Initiate upload
@@ -1071,7 +1144,12 @@ export class Client<DB extends SyncClientDb = SyncClientDb> {
               }
 
               // Complete
-              await blobs.completeUpload(item.hash);
+              const completeResult = await blobs.completeUpload(item.hash);
+              if (!completeResult.ok) {
+                throw new Error(
+                  completeResult.error ?? 'Failed to complete blob upload'
+                );
+              }
             }
 
             // Mark as complete
@@ -1082,22 +1160,23 @@ export class Client<DB extends SyncClientDb = SyncClientDb> {
 
             uploaded++;
           } catch (err) {
-            // Mark as failed
+            const nextStatus =
+              nextAttemptCount >= maxUploadRetries ? 'failed' : 'pending';
+
             await sql`
               update ${sql.table('sync_blob_outbox')}
               set
-                ${sql.ref('status')} = ${sql.val('failed')},
+                ${sql.ref('status')} = ${sql.val(nextStatus)},
                 ${sql.ref('error')} = ${sql.val(
                   err instanceof Error ? err.message : 'Unknown error'
                 )},
-                ${sql.ref('attempt_count')} = ${sql.ref('attempt_count')} + ${sql.val(
-                  1
-                )},
                 ${sql.ref('updated_at')} = ${sql.val(Date.now())}
               where ${sql.ref('hash')} = ${sql.val(item.hash)}
             `.execute(db);
 
-            failed++;
+            if (nextStatus === 'failed') {
+              failed++;
+            }
           }
         }
 

package/src/conflicts.ts

@@ -3,20 +3,11 @@
  */
 
 import type { SyncOperationResult, SyncPushResponse } from '@syncular/core';
+import { randomId } from '@syncular/core';
 import type { Kysely } from 'kysely';
 import { sql } from 'kysely';
 import type { SyncClientDb } from './schema';
 
-function randomId(): string {
-  if (
-    typeof crypto !== 'undefined' &&
-    typeof crypto.randomUUID === 'function'
-  ) {
-    return crypto.randomUUID();
-  }
-  return `${Date.now()}-${Math.random().toString(16).slice(2)}`;
-}
-
 function messageFromResult(
   r: Extract<SyncOperationResult, { status: 'conflict' | 'error' }>
 ): {