@syncular/client 0.15.47 → 0.16.1

package/src/client.ts

@@ -119,6 +119,8 @@ import {
   dropOutboxCommitsInScope,
   encodeOutboxCommit,
   listOutbox,
+  iterateOutbox,
+  countOutbox,
   listOutboxBeforeImages,
   type OutboxBeforeImage,
   type OutboxCommit,
@@ -513,6 +515,8 @@ function emptySummary(pushed: number): MutableSummary {
   };
 }
 
+const LOG_EPOCH_META_KEY = 'logEpoch';
+
 function isFinalPushResult(frame: PushResultFrame): boolean {
   return (
     frame.status !== 'rejected' ||
@@ -525,6 +529,37 @@ function isFinalPushResult(frame: PushResultFrame): boolean {
   );
 }
 
+/** Canonical client reads, shared by synchronous cores and promise hosts. */
+export type ClientSnapshotMethods = Pick<
+  SyncClient,
+  | 'querySnapshot'
+  | 'statusSnapshot'
+  | 'diagnosticsSnapshot'
+  | 'conflicts'
+  | 'rejections'
+  | 'commitOutcome'
+  | 'commitOutcomes'
+  | 'resolveCommitOutcome'
+>;
+
+/** Project a method contract across an asynchronous host boundary. */
+export type PromiseMethods<Methods> = {
+  [Key in keyof Methods]: Methods[Key] extends (
+    ...args: infer Args
+  ) => infer Result
+    ? (...args: Args) => Promise<Awaited<Result>>
+    : never;
+};
+
+/** A reader can execute locally or cross a worker/native boundary. */
+export type ClientSnapshotReader = {
+  [Key in keyof ClientSnapshotMethods]: (
+    ...args: Parameters<ClientSnapshotMethods[Key]>
+  ) =>
+    | ReturnType<ClientSnapshotMethods[Key]>
+    | Promise<ReturnType<ClientSnapshotMethods[Key]>>;
+};
+
 export class SyncClient {
   readonly #config: SyncClientConfig;
   readonly #db: ClientDatabase;
@@ -579,6 +614,10 @@ export class SyncClient {
   readonly #invalidation = new InvalidationEmitter();
   /** §8.6: subscribable presence-change listeners (twin of onPresence). */
   readonly #presenceListeners = new Set<(scopeKey: string) => void>();
+  readonly #syncNeededListeners = new Set<
+    (reason: 'startup' | 'hello' | WakeReason) => void
+  >();
+  readonly #syncIntentListeners = new Set<(intent: SyncIntent) => void>();
   readonly #diagnostics = new ClientDiagnosticsEmitter();
   #diagnosticsDeferralDepth = 0;
   #diagnosticsPending = false;
@@ -701,12 +740,12 @@ export class SyncClient {
     // an application-issued sync() call.
     const startupWork =
       this.#schemaFloor === undefined &&
-      (listOutbox(this.#db).length > 0 ||
+      (countOutbox(this.#db) > 0 ||
         subscriptions.some((sub) => sub.status === 'active'));
     if (startupWork && this.#securityLifecycle === 'active') {
       this.#needsPull = true;
-      this.#config.onSyncNeeded?.('startup');
-      this.#config.onSyncIntent?.({ kind: 'interactive' });
+      this.#emitSyncNeeded('startup');
+      this.#emitSyncIntent({ kind: 'interactive' });
     }
     // Console introspection is a no-op outside a dev page.
     this.#devtoolsUnregister = registerDevtools({
@@ -716,10 +755,10 @@ export class SyncClient {
       role: () => 'direct',
       outbox: async () => this.pendingCommits().length,
       subscriptions: async () => this.subscriptions(),
-      conflicts: async () => this.conflicts.length,
-      rejections: async () => this.rejections.length,
-      syncNeeded: async () => this.syncNeeded,
-      upgrading: async () => this.upgrading,
+      conflicts: async () => this.conflicts().length,
+      rejections: async () => this.rejections().length,
+      syncNeeded: async () => this.statusSnapshot().syncNeeded,
+      upgrading: async () => this.statusSnapshot().upgrading,
       onInvalidate: (listener) => this.onInvalidate(listener),
     });
     this.#emitDiagnostics();
@@ -776,6 +815,29 @@ export class SyncClient {
     this.#replayOutbox();
   }
 
+  /** §2.1 reset after the server reports a different log continuity. */
+  #runLogEpochReset(logEpoch: string): string[] {
+    const subscriptions = loadSubscriptions(this.#db);
+    const pending = listOutbox(this.#db);
+    this.#setUpgrading(true);
+    this.#applyBatch((batch) => {
+      this.#db.transaction(() => {
+        dropAndRecreateSyncedTables(this.#db, this.#schema);
+        resetSubscriptionsForBump(this.#db);
+        setMeta(this.#db, LOG_EPOCH_META_KEY, logEpoch);
+        for (const commit of pending) {
+          this.#applyOperationsLocally(commit.operations, batch);
+        }
+      });
+      for (const table of this.#schema.tables.values()) batch.table(table.name);
+    });
+    this.#localResetEpoch += 1;
+    this.#setSyncNeeded(true);
+    this.#emitSyncNeeded('startup');
+    this.#emitSyncIntent({ kind: 'interactive' });
+    return subscriptions.map((subscription) => subscription.id);
+  }
+
   #setUpgrading(upgrading: boolean): void {
     if (this.#upgrading === upgrading) return;
     this.#applyBatch((batch) => {
@@ -803,6 +865,7 @@ export class SyncClient {
   }
 
   async close(): Promise<void> {
+    this.#emitSyncIntent({ kind: 'none' });
     this.#devtoolsUnregister?.();
     this.#devtoolsUnregister = undefined;
     this.disconnectRealtime();
@@ -810,10 +873,42 @@ export class SyncClient {
     await this.#lease?.release();
     this.#lease = undefined;
     this.#started = false;
+    this.#syncNeededListeners.clear();
+    this.#syncIntentListeners.clear();
+  }
+
+  #emitSyncNeeded(reason: 'startup' | 'hello' | WakeReason): void {
+    try {
+      this.#config.onSyncNeeded?.(reason);
+    } catch {
+      // An observer cannot alter sync correctness.
+    }
+    for (const listener of this.#syncNeededListeners) {
+      try {
+        listener(reason);
+      } catch {
+        // An observer cannot alter sync correctness.
+      }
+    }
+  }
+
+  #emitSyncIntent(intent: SyncIntent): void {
+    try {
+      this.#config.onSyncIntent?.(intent);
+    } catch {
+      // An observer cannot alter sync correctness.
+    }
+    for (const listener of this.#syncIntentListeners) {
+      try {
+        listener(intent);
+      } catch {
+        // An observer cannot alter sync correctness.
+      }
+    }
   }
 
   /** Current fail-closed local-replica security state. */
-  get securityLifecycle(): SecurityLifecycle {
+  securityLifecycle(): SecurityLifecycle {
     return this.#securityLifecycle;
   }
 
@@ -865,12 +960,12 @@ export class SyncClient {
     this.#securityLifecycle = 'active';
     const startupWork =
       this.#schemaFloor === undefined &&
-      (listOutbox(this.#db).length > 0 ||
+      (countOutbox(this.#db) > 0 ||
         loadSubscriptions(this.#db).some((sub) => sub.status === 'active'));
     if (startupWork) {
       this.#setSyncNeeded(true);
-      this.#config.onSyncNeeded?.('startup');
-      this.#config.onSyncIntent?.({ kind: 'interactive' });
+      this.#emitSyncNeeded('startup');
+      this.#emitSyncIntent({ kind: 'interactive' });
     }
     this.#emitDiagnostics();
   }
@@ -982,6 +1077,24 @@ export class SyncClient {
     return this.#changes.on(listener);
   }
 
+  /** Subscribe to host wake signals raised by startup and realtime. */
+  onSyncNeeded(
+    listener: (reason: 'startup' | 'hello' | WakeReason) => void,
+  ): () => void {
+    this.#syncNeededListeners.add(listener);
+    return () => {
+      this.#syncNeededListeners.delete(listener);
+    };
+  }
+
+  /** Subscribe to exact core-owned scheduling instructions. */
+  onSyncIntent(listener: (intent: SyncIntent) => void): () => void {
+    this.#syncIntentListeners.add(listener);
+    return () => {
+      this.#syncIntentListeners.delete(listener);
+    };
+  }
+
   /** Subscribe to complete, privacy-safe diagnostic snapshots. */
   onDiagnostics(listener: ClientDiagnosticsListener): () => void {
     return this.#diagnostics.on(listener);
@@ -1114,7 +1227,7 @@ export class SyncClient {
       replica: {
         localRevision: getLocalRevision(this.#db).toString(),
         syncNeeded: this.#needsPull,
-        pendingOutbox: listOutbox(this.#db).length,
+        pendingOutbox: countOutbox(this.#db),
       },
       lease: leaseState,
       subscriptions: allSubscriptions.slice(
@@ -1230,7 +1343,7 @@ export class SyncClient {
   #statusSnapshot(outboxCount?: number): SyncStatusSnapshot {
     return {
       currentSchemaVersion: this.#config.schema.version,
-      outbox: outboxCount ?? listOutbox(this.#db).length,
+      outbox: outboxCount ?? countOutbox(this.#db),
       upgrading: this.#upgrading,
       leaseState: this.#leaseState,
       schemaFloor: this.#schemaFloor,
@@ -1526,12 +1639,12 @@ export class SyncClient {
     await transport.upload(blobId, bytes, mediaType);
   }
 
-  get conflicts(): readonly ConflictRecord[] {
+  conflicts(): readonly ConflictRecord[] {
     this.#requireActive();
     return this.#conflicts;
   }
 
-  get rejections(): readonly RejectionRecord[] {
+  rejections(): readonly RejectionRecord[] {
     this.#requireActive();
     return this.#rejections;
   }
@@ -1622,30 +1735,6 @@ export class SyncClient {
     });
   }
 
-  /** Non-undefined once the server declared a schema floor (§1.6). */
-  get schemaFloor(): SchemaFloor | undefined {
-    return this.#schemaFloor;
-  }
-
-  /**
-   * §7.4.5: true while a schema-bump reset + first re-bootstrap is in
-   * flight — the app's "upgrading…" cue. Clears when the first post-reset
-   * bootstrap round reaches idle (every subscription past its fresh
-   * bootstrap).
-   */
-  get upgrading(): boolean {
-    return this.#upgrading;
-  }
-
-  /**
-   * §7.3.5: the current auth-lease state (opaque). Undefined until a
-   * `LEASE` frame arrives. `errorCode` is set when a round was rejected
-   * with a request-level lease code — syncing on the lease has stopped.
-   */
-  get leaseState(): LeaseState | undefined {
-    return this.#leaseState;
-  }
-
   /** §7.3.5: remaining lease validity in ms (`expiresAtMs − now`), or
    * `undefined` if no lease is held. Negative once expired. */
   leaseRemainingMs(now: number = this.#now()): number | undefined {
@@ -1658,11 +1747,6 @@ export class SyncClient {
     return this.#schemaFloor !== undefined;
   }
 
-  /** §8: a hello/wake-up asked for a pull that has not run yet. */
-  get syncNeeded(): boolean {
-    return this.#needsPull;
-  }
-
   /**
    * §8.6 presence on a scope key: the current peers present there (a map
    * of `actorId clientId` → peer). Empty for a key with no present peers.
@@ -1781,12 +1865,17 @@ export class SyncClient {
       cursor: -1,
       status: 'active',
     });
+    this.#setSyncNeeded(true);
+    this.#emitSyncIntent({ kind: 'interactive' });
     this.#emitDiagnostics();
   }
 
   unsubscribe(id: string): void {
     this.#requireActive();
+    if (getSubscription(this.#db, id) === undefined) return;
     deleteSubscription(this.#db, id);
+    this.#setSyncNeeded(true);
+    this.#emitSyncIntent({ kind: 'interactive' });
     this.#emitDiagnostics();
   }
 
@@ -1851,7 +1940,9 @@ export class SyncClient {
               status: 'active',
             });
           });
+          this.#needsPull = true;
           batch.window(baseKey, base.table, unit);
+          batch.status();
         });
         changed = true;
         widened = true;
@@ -1867,6 +1958,9 @@ export class SyncClient {
     const effects: CommandEffects = {
       sync: changed || widened ? { kind: 'interactive' } : { kind: 'none' },
     };
+    if (effects.sync.kind === 'interactive') {
+      this.#emitSyncIntent(effects.sync);
+    }
     return { value: undefined, effects };
   }
 
@@ -1934,6 +2028,8 @@ export class SyncClient {
       });
       batch.scopeMap(table, effective);
       batch.window(baseKey, table.name, unit);
+      this.#needsPull = true;
+      batch.status();
     });
   }
 
@@ -2043,7 +2139,9 @@ export class SyncClient {
         this.#applyOperationsLocally(operations, batch);
         batch.status();
       });
+      this.#needsPull = true;
     });
+    this.#emitSyncIntent({ kind: 'interactive' });
     return clientCommitId;
   }
 
@@ -2320,8 +2418,8 @@ export class SyncClient {
     this.#localResetEpoch += 1;
 
     if (!priorUpgrading) this.#config.onUpgrading?.(true);
-    this.#config.onSyncNeeded?.('startup');
-    this.#config.onSyncIntent?.({ kind: 'interactive' });
+    this.#emitSyncNeeded('startup');
+    this.#emitSyncIntent({ kind: 'interactive' });
     return {
       alreadyApplied: false,
       retainedCommits: pending.length,
@@ -2408,12 +2506,19 @@ export class SyncClient {
     outbox: OutboxCommit[];
     deferred: number;
   }> {
-    const pending = listOutbox(this.#db);
+    // Pin before the first encryption await: mutations can append while a
+    // round is encoding, and belong to the next request.
+    const bounds = this.#db.query(
+      'SELECT COUNT(*) AS count, MAX(seq) AS last_seq FROM _syncular_outbox',
+    )[0]!;
+    const pendingCount = bounds.count as number;
+    const throughSeq = (bounds.last_seq as number | null) ?? 0;
     const pushFrames: RequestFrame[] = [];
     const outbox: OutboxCommit[] = [];
     let deferred = 0;
     let ops = 0;
-    for (const commit of pending) {
+    let processed = 0;
+    for (const commit of iterateOutbox(this.#db, throughSeq)) {
       // §6.1 splitBatch: whole commits in commit order, stopping before the
       // per-request operation cap. A first commit that alone exceeds the cap
       // is sent alone — the server rejects it loudly rather than the queue
@@ -2422,9 +2527,10 @@ export class SyncClient {
         outbox.length > 0 &&
         ops + commit.operations.length > MAX_OPS_PER_REQUEST
       ) {
-        deferred += 1;
-        continue;
+        deferred = pendingCount - processed;
+        break;
       }
+      processed += 1;
       try {
         pushFrames.push(
           // §5.11: encrypted columns are encrypted at this encode-at-send
@@ -2585,9 +2691,14 @@ export class SyncClient {
     // survive it — the reference server keeps no replay buffer (§8.2).
     this.#setSyncNeeded(false);
     try {
+      const logEpoch = getMeta(this.#db, LOG_EPOCH_META_KEY);
       // §5.9.7 B4: upload pending blobs BEFORE pushing rows that reference
       // them, so the server-side existence check (§6.6) passes.
-      if (this.#hasBlobs && this.#config.blobs !== undefined) {
+      if (
+        logEpoch !== undefined &&
+        this.#hasBlobs &&
+        this.#config.blobs !== undefined
+      ) {
         await this.flushBlobUploads();
       }
       // §7.4.4: encode the outbox with the CURRENT codec; a commit that
@@ -2596,7 +2707,9 @@ export class SyncClient {
       // the queue. `pushFrames` and `outbox` stay index-aligned for result
       // mapping.
       const { pushFrames, outbox, deferred } =
-        await this.#encodeOutboxForPush();
+        logEpoch === undefined
+          ? { pushFrames: [], outbox: [], deferred: 0 }
+          : await this.#encodeOutboxForPush();
       // Captured together with the subscription state below: the response
       // apply persists SUB_END cursors only while this epoch is current.
       const resetEpoch = this.#localResetEpoch;
@@ -2609,6 +2722,7 @@ export class SyncClient {
           type: 'REQ_HEADER',
           clientId: this.#clientId,
           schemaVersion: this.#schema.version,
+          ...(logEpoch !== undefined ? { logEpoch } : {}),
         },
         ...pushFrames,
         {
@@ -2686,11 +2800,7 @@ export class SyncClient {
           delayMs: this.#retryDelayMs,
         };
         this.#retryDelayMs = Math.min(this.#retryDelayMs * 2, 30_000);
-        try {
-          this.#config.onSyncIntent?.(intent);
-        } catch {
-          // An observer cannot alter sync correctness.
-        }
+        this.#emitSyncIntent(intent);
       }
       throw error;
     } finally {
@@ -2712,7 +2822,8 @@ export class SyncClient {
         last.segmentRowsApplied === 0 &&
         last.bootstrapping.length === 0 &&
         last.resets.length === 0 &&
-        (last.deferredCommits ?? 0) === 0
+        (last.deferredCommits ?? 0) === 0 &&
+        !this.#needsPull
       ) {
         return last;
       }
@@ -2916,14 +3027,14 @@ export class SyncClient {
     if (event.event === 'hello') {
       if (event.data.requiresSync) {
         this.#setSyncNeeded(true);
-        this.#config.onSyncNeeded?.('hello');
+        this.#emitSyncNeeded('hello');
       }
       return;
     }
     if (event.event === 'sync') {
       // §8.3: any wake-up means "run a pull soon", never data.
       this.#setSyncNeeded(true);
-      this.#config.onSyncNeeded?.(event.data.reason);
+      this.#emitSyncNeeded(event.data.reason);
       return;
     }
     if (event.event === 'presence') {
@@ -2995,7 +3106,7 @@ export class SyncClient {
       } catch {
         // A delta that cannot be applied is recovered by a pull (§8.3).
         this.#setSyncNeeded(true);
-        this.#config.onSyncNeeded?.('catchup-required');
+        this.#emitSyncNeeded('catchup-required');
       }
     });
   }
@@ -3057,6 +3168,16 @@ export class SyncClient {
     if (header?.type !== 'RESP_HEADER') {
       throw new ClientSyncError('sync.invalid_request', 'missing RESP_HEADER');
     }
+    if (
+      message.wireVersion < 2 ||
+      header.logEpoch === undefined ||
+      header.resetRequired === undefined
+    ) {
+      throw new ClientSyncError(
+        'client.invalid_host_response',
+        'the server response does not carry wire version 2 log-epoch state',
+      );
+    }
     if (header.requiredSchemaVersion !== undefined) {
       // §1.6 schema floor: nothing else was processed — stop syncing and
       // surface the upgrade requirement. A live-round floor always stops:
@@ -3078,6 +3199,26 @@ export class SyncClient {
         schemaFloor,
       };
     }
+    const currentLogEpoch = getMeta(this.#db, LOG_EPOCH_META_KEY);
+    if (header.resetRequired) {
+      if (mode !== 'pull' || message.frames.length !== 1) {
+        throw new ClientSyncError(
+          'client.invalid_host_response',
+          'a log-epoch reset response must contain only RESP_HEADER',
+        );
+      }
+      return {
+        ...summary,
+        resets: this.#runLogEpochReset(header.logEpoch),
+        bootstrapping: [],
+      };
+    }
+    if (currentLogEpoch === undefined || currentLogEpoch !== header.logEpoch) {
+      throw new ClientSyncError(
+        'client.invalid_host_response',
+        'the server changed logEpoch without requiring a reset',
+      );
+    }
 
     let section: OpenSection | undefined;
     let errorFrame: ClientSyncError | undefined;
@@ -3101,8 +3242,7 @@ export class SyncClient {
             });
             break;
           case 'PUSH_RESULT': {
-            let outboxCount =
-              responseOutboxCount ?? listOutbox(this.#db).length;
+            let outboxCount = responseOutboxCount ?? countOutbox(this.#db);
             this.#applyBatch(
               (batch) => {
                 const drained = this.#handlePushResult(

package/src/index.ts

@@ -35,6 +35,7 @@ export * from './realtime-supervisor';
 export * from './schema';
 export * from './sql-tag';
 export * from './state';
+export * from './sync-scheduler';
 export * from './transport';
 export * from './window';
 export * from './worker-host';

package/src/outbox.ts

@@ -11,7 +11,7 @@ import {
   type PushOperation,
   type ScopeMap,
 } from '@syncular/core';
-import type { ClientDatabase } from './database';
+import type { ClientDatabase, SqlRow } from './database';
 import type { EncryptionConfig } from './encryption';
 import { ClientSyncError } from './errors';
 import {
@@ -85,19 +85,50 @@ export function appendOutboxCommit(
   }
 }
 
-/** Pending commits in FIFO creation order (§7.1). */
+/** Pending commits in FIFO creation order (§7.1). Full reads serve replay and the public listing. */
 export function listOutbox(db: ClientDatabase): OutboxCommit[] {
   return db
     .query(
       `SELECT seq, client_commit_id, created_at_ms, operations
        FROM _syncular_outbox ORDER BY seq ASC`,
     )
-    .map((row) => ({
-      seq: row.seq as number,
-      clientCommitId: row.client_commit_id as string,
-      createdAtMs: row.created_at_ms as number,
-      operations: JSON.parse(row.operations as string) as OutboxOperation[],
-    }));
+    .map(decodeOutboxRow);
+}
+
+function decodeOutboxRow(row: SqlRow): OutboxCommit {
+  return {
+    seq: row.seq as number,
+    clientCommitId: row.client_commit_id as string,
+    createdAtMs: row.created_at_ms as number,
+    operations: JSON.parse(row.operations as string) as OutboxOperation[],
+  };
+}
+
+/** Keyset pages bound staging; laziness decodes only commits consumed by the encoder. */
+export function* iterateOutbox(
+  db: ClientDatabase,
+  throughSeq: number,
+): Generator<OutboxCommit> {
+  let afterSeq = 0;
+  while (afterSeq < throughSeq) {
+    const rows = db.query(
+      `SELECT seq, client_commit_id, created_at_ms, operations FROM _syncular_outbox
+       WHERE seq > ? AND seq <= ? ORDER BY seq ASC LIMIT 32`,
+      [afterSeq, throughSeq],
+    );
+    if (rows.length === 0) return;
+    for (const row of rows) {
+      const commit = decodeOutboxRow(row);
+      afterSeq = commit.seq;
+      yield commit;
+    }
+  }
+}
+
+/** Routine status reads never load operation bodies. */
+export function countOutbox(db: ClientDatabase): number {
+  return db.query('SELECT COUNT(*) AS count FROM _syncular_outbox')[0]!
+    .count as number;
 }
 
 export function deleteOutboxCommit(