@syncular/server 0.39.2 → 0.41.0

package/src/d1-storage.ts

@@ -1,7 +1,7 @@
 import { validateCommitPruneQuery } from './prune';
 import { StorageQueryError } from './storage-errors';
 import type { CommitPruneQuery, CommitPruneResult } from './storage';
-import { PARTITION_TOUCH_INTERVAL_MS } from './storage';
+import { clientRoundState, PARTITION_TOUCH_INTERVAL_MS } from './storage';
 /**
  * Cloudflare D1 server storage for Workers deployments.
  *
@@ -78,6 +78,7 @@ import {
   type StoredColumnLayout,
   scanRowPageSql,
   schemaDdl,
+  serverIndexDdl,
   SQLITE_TABLE_INDEXES_SQL,
   type SqliteTableIndexRow,
   sqliteIndexSignatures,
@@ -104,6 +105,7 @@ import {
   type SqliteRowRecord,
   serializePushResult,
   sqliteDdlStatements,
+  SYNC_CLIENT_ROUND_COLUMNS,
   toStoredChange,
   toStoredRow,
 } from './sqlite-dialect';
@@ -114,6 +116,7 @@ import type {
   ClientCursorInfo,
   ClientPushResult,
   ClientRecord,
+  ClientRecordWriteOptions,
   ClientSubscription,
   CommitChangeSummary,
   CommitDetail,
@@ -999,6 +1002,15 @@ export class D1ServerStorage implements ServerStorage {
         'ALTER TABLE sync_clients ADD COLUMN wire_version INTEGER NOT NULL DEFAULT 1;',
       );
     }
+    for (const [name, definition] of Object.entries(
+      SYNC_CLIENT_ROUND_COLUMNS,
+    )) {
+      if (!results.some((column) => column.name === name)) {
+        await this.#db.exec(
+          `ALTER TABLE sync_clients ADD COLUMN ${name} ${definition};`,
+        );
+      }
+    }
   }
 
   /** Resolve a table's compiled schema; row operations require `ensureSchema`. */
@@ -1129,6 +1141,44 @@ export class D1ServerStorage implements ServerStorage {
           'sqlite',
         );
       }
+      // A database from before the round-state columns gains them in place.
+      statementsExecuted++;
+      const clientColumns = await this.#db
+        .prepare('PRAGMA table_info("sync_clients")')
+        .all<{ name: string }>();
+      const missingRoundColumns = Object.entries(
+        SYNC_CLIENT_ROUND_COLUMNS,
+      ).filter(
+        ([name]) =>
+          !clientColumns.results.some((column) => column.name === name),
+      );
+      if (missingRoundColumns.length > 0) {
+        await batch(
+          missingRoundColumns.map(([name, definition]) =>
+            this.#db.prepare(
+              `ALTER TABLE sync_clients ADD COLUMN ${name} ${definition}`,
+            ),
+          ),
+        );
+      }
+      // Server-only indexes reconcile at every open of a database already at
+      // this version: they never reach a client, so no version bump is needed.
+      const indexes = new Map<string, ReadonlyMap<string, string>>();
+      for (const table of tables) {
+        statementsExecuted++;
+        const { results } = await this.#db
+          .prepare(SQLITE_TABLE_INDEXES_SQL)
+          .bind(table.name)
+          .all<SqliteTableIndexRow>();
+        indexes.set(
+          table.name,
+          new Map(Object.entries(sqliteIndexSignatures(results))),
+        );
+      }
+      const reconcile = serverIndexDdl(schema, indexes);
+      if (reconcile.length > 0) {
+        await batch(reconcile.map((sql) => this.#db.prepare(sql)));
+      }
       this.#tables = schema.tables;
       this.#schemaVersion = schema.version;
       return { complete: true, statementsExecuted };
@@ -1248,19 +1298,20 @@ export class D1ServerStorage implements ServerStorage {
         const columns = await this.#db
           .prepare('PRAGMA table_info("sync_clients")')
           .all<{ name: string }>();
-        const needsWireVersion = !columns.results.some(
-          (column) => column.name === 'wire_version',
+        const missingColumns = Object.entries({
+          wire_version: 'INTEGER NOT NULL DEFAULT 1',
+          ...SYNC_CLIENT_ROUND_COLUMNS,
+        }).filter(
+          ([name]) => !columns.results.some((column) => column.name === name),
         );
         if (
           !(await save(
             { ...state, phase: 'prepare', offset: 0 },
-            needsWireVersion
-              ? [
-                  this.#db.prepare(
-                    'ALTER TABLE sync_clients ADD COLUMN wire_version INTEGER NOT NULL DEFAULT 1',
-                  ),
-                ]
-              : [],
+            missingColumns.map(([name, definition]) =>
+              this.#db.prepare(
+                `ALTER TABLE sync_clients ADD COLUMN ${name} ${definition}`,
+              ),
+            ),
           ))
         )
           break;
@@ -2333,7 +2384,7 @@ export class D1ServerStorage implements ServerStorage {
   ): Promise<ClientRecord | undefined> {
     const record = await this.#db
       .prepare(
-        'SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=? AND client_id=?',
+        'SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms, last_error_code, consecutive_failures FROM sync_clients WHERE partition=? AND client_id=?',
       )
       .bind(partition, clientId)
       .first<{
@@ -2343,6 +2394,9 @@ export class D1ServerStorage implements ServerStorage {
         cursor: number;
         subscriptions: string;
         updated_at_ms: number;
+        last_round_at_ms: number | null;
+        last_error_code: string | null;
+        consecutive_failures: number;
       }>();
     if (record === null) return undefined;
     return {
@@ -2352,25 +2406,39 @@ export class D1ServerStorage implements ServerStorage {
       cursor: record.cursor,
       updatedAtMs: record.updated_at_ms,
       subscriptions: JSON.parse(record.subscriptions) as ClientSubscription[],
+      ...clientRoundState(
+        record.last_round_at_ms,
+        record.last_error_code,
+        record.consecutive_failures,
+      ),
     };
   }
 
   async putClientRecord(
     partition: string,
     record: ClientRecord,
+    options?: ClientRecordWriteOptions,
   ): Promise<void> {
     // Update the existing record in place, so the table keeps one row per
     // (partition, client_id). Every non-key field is written from the new
     // record; a delete-and-insert would drop and recreate the row instead.
+    // `keepRoundOutcome` leaves the round-state columns alone (§1.5, §4.5).
     await this.#db
       .prepare(
-        `INSERT INTO sync_clients(partition, client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms) VALUES (?,?,?,?,?,?,?)
+        `INSERT INTO sync_clients(partition, client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms) VALUES (?,?,?,?,?,?,?,?)
          ON CONFLICT(partition, client_id) DO UPDATE SET
            actor_id=excluded.actor_id,
            wire_version=excluded.wire_version,
            cursor=excluded.cursor,
            subscriptions=excluded.subscriptions,
-           updated_at_ms=excluded.updated_at_ms`,
+           updated_at_ms=excluded.updated_at_ms,
+           last_round_at_ms=excluded.last_round_at_ms${
+             options?.keepRoundOutcome === true
+               ? ''
+               : `,
+           last_error_code=NULL,
+           consecutive_failures=0`
+           }`,
       )
       .bind(
         partition,
@@ -2380,7 +2448,26 @@ export class D1ServerStorage implements ServerStorage {
         record.cursor,
         JSON.stringify(record.subscriptions),
         record.updatedAtMs,
+        record.updatedAtMs,
+      )
+      .run();
+  }
+
+  async recordClientRoundFailure(
+    partition: string,
+    clientId: string,
+    actorId: string,
+    errorCode: string,
+    atMs: number,
+  ): Promise<void> {
+    await this.#db
+      .prepare(
+        `UPDATE sync_clients
+         SET last_round_at_ms=?, last_error_code=?,
+             consecutive_failures=consecutive_failures+1
+         WHERE partition=? AND client_id=? AND actor_id=?`,
       )
+      .bind(atMs, errorCode, partition, clientId, actorId)
       .run();
   }
 
@@ -2537,7 +2624,7 @@ export class D1ServerStorage implements ServerStorage {
   async listClientRecords(partition: string): Promise<ClientRecord[]> {
     const { results } = await this.#db
       .prepare(
-        'SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=? ORDER BY updated_at_ms DESC',
+        'SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms, last_error_code, consecutive_failures FROM sync_clients WHERE partition=? ORDER BY updated_at_ms DESC',
       )
       .bind(partition)
       .all<{
@@ -2547,6 +2634,9 @@ export class D1ServerStorage implements ServerStorage {
         cursor: number;
         subscriptions: string;
         updated_at_ms: number;
+        last_round_at_ms: number | null;
+        last_error_code: string | null;
+        consecutive_failures: number;
       }>();
     return results.map((record) => ({
       clientId: record.client_id,
@@ -2555,6 +2645,11 @@ export class D1ServerStorage implements ServerStorage {
       cursor: record.cursor,
       updatedAtMs: record.updated_at_ms,
       subscriptions: JSON.parse(record.subscriptions) as ClientSubscription[],
+      ...clientRoundState(
+        record.last_round_at_ms,
+        record.last_error_code,
+        record.consecutive_failures,
+      ),
     }));
   }
 

package/src/handler.ts

@@ -19,6 +19,7 @@ import {
   type ReqHeaderFrame,
   type RequestMessage,
   type ResponseFrame,
+  type RoundFailureFrame,
   type ScopeMap,
   type SubscriptionFrame,
 } from '@syncular/core';
@@ -30,7 +31,7 @@ import {
   RESOLVER_OUTAGE,
   touchAuthenticatedPartition,
 } from './context';
-import { SyncError, syncError } from './errors';
+import { reportError, SyncError, syncError } from './errors';
 import {
   emitEvent,
   type PullSubscriptionSummary,
@@ -70,6 +71,8 @@ interface RequestPlan {
   readonly logEpoch: string;
   readonly epochReset: boolean;
   readonly header: ReqHeaderFrame;
+  /** §1.5: the code this request reports for its last unseen round failure. */
+  readonly roundFailure: RoundFailureFrame | undefined;
   readonly pushes: readonly PushCommitFrame[];
   readonly pull: PullHeaderFrame | undefined;
   readonly subscriptions: readonly SubscriptionPlan[];
@@ -212,10 +215,12 @@ async function planRequest(
     throw syncError('sync.invalid_request', 'missing REQ_HEADER');
   }
   const pushes: PushCommitFrame[] = [];
+  let roundFailure: RoundFailureFrame | undefined;
   let pull: PullHeaderFrame | undefined;
   const subFrames: SubscriptionFrame[] = [];
   for (const frame of request.frames) {
-    if (frame.type === 'PUSH_COMMIT') pushes.push(frame);
+    if (frame.type === 'ROUND_FAILURE') roundFailure = frame;
+    else if (frame.type === 'PUSH_COMMIT') pushes.push(frame);
     else if (frame.type === 'PULL_HEADER') pull = frame;
     else if (frame.type === 'SUBSCRIPTION') subFrames.push(frame);
   }
@@ -241,6 +246,7 @@ async function planRequest(
       logEpoch: registry.logEpoch,
       epochReset,
       header,
+      roundFailure,
       pushes,
       pull,
       subscriptions: [],
@@ -256,6 +262,7 @@ async function planRequest(
       logEpoch: registry.logEpoch,
       epochReset: true,
       header,
+      roundFailure,
       pushes: [],
       pull: undefined,
       subscriptions: [],
@@ -362,6 +369,7 @@ async function planRequest(
     logEpoch: registry.logEpoch,
     epochReset: false,
     header,
+    roundFailure,
     pushes,
     pull,
     subscriptions,
@@ -709,14 +717,21 @@ async function* streamResponse(
         : [...(previous?.subscriptions ?? [])];
     const cursor =
       cursors.length > 0 ? Math.min(...cursors) : (previous?.cursor ?? -1);
-    await ctx.storage.putClientRecord(ctx.partition, {
-      clientId: plan.header.clientId,
-      actorId: ctx.actorId,
-      wireVersion: plan.wireVersion,
-      cursor,
-      updatedAtMs: clockOf(ctx)(),
-      subscriptions,
-    });
+    await ctx.storage.putClientRecord(
+      ctx.partition,
+      {
+        clientId: plan.header.clientId,
+        actorId: ctx.actorId,
+        wireVersion: plan.wireVersion,
+        cursor,
+        updatedAtMs: clockOf(ctx)(),
+        subscriptions,
+      },
+      // §4.5: a round that carried a `ROUND_FAILURE` report completes without
+      // clearing the reported code and streak; the next request without the
+      // frame clears them as a plain completed round does.
+      plan.roundFailure !== undefined ? { keepRoundOutcome: true } : undefined,
+    );
     if (
       ctx.realtime?.notifySubscriptions !== undefined &&
       JSON.stringify(
@@ -734,6 +749,7 @@ async function* streamResponse(
         report.outcome = 'error';
         report.errorCode = error.code;
       }
+      await recordRoundFailure(ctx, plan.header.clientId, error);
       // §1.6: in-band ERROR, then END, nothing else.
       yield encodeResponseFrame(
         {
@@ -765,7 +781,14 @@ export async function createSyncResponseStream(
   ctx: SyncRequestContext,
 ): Promise<AsyncIterable<Uint8Array>> {
   const sink = ctx.events;
-  if (sink === undefined) return createStreamCore(bytes, ctx, undefined);
+  if (sink === undefined) {
+    try {
+      return await createStreamCore(bytes, ctx, undefined);
+    } catch (error) {
+      await recordRoundFailure(ctx, requestClientId(bytes), error);
+      throw error;
+    }
+  }
   // Instrumented path: measure with the ctx clock, count bytes at the
   // seam, emit exactly one `request.handled` per request. Every event of
   // this request carries one `requestId`.
@@ -779,6 +802,7 @@ export async function createSyncResponseStream(
   try {
     return await createStreamCore(bytes, requestCtx, events, startedAtMs);
   } catch (error) {
+    const clientId = requestClientId(bytes);
     emitEvent(events, {
       type: 'request.handled',
       kind: 'sync',
@@ -789,11 +813,78 @@ export async function createSyncResponseStream(
       bytesIn: bytes.length,
       bytesOut: 0,
       outcome: 'rejected',
+      ...(clientId !== undefined ? { clientId } : {}),
       errorCode: error instanceof SyncError ? error.code : 'internal',
       pushCommits: 0,
       pulled: false,
       subscriptions: 0,
     });
+    await recordRoundFailure(ctx, clientId, error);
+    throw error;
+  }
+}
+
+/** The `REQ_HEADER` clientId of a request that failed before planning. */
+function requestClientId(bytes: Uint8Array): string | undefined {
+  try {
+    const message = decodeMessage(bytes);
+    const header = message.frames[0];
+    return message.msgKind === 'request' && header?.type === 'REQ_HEADER'
+      ? header.clientId
+      : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+/**
+ * Record a failed round on the client's stored record so an operator can see
+ * a stuck client and why. A storage failure here is reported through
+ * `onError` and never replaces the round's own outcome.
+ */
+async function recordRoundFailure(
+  ctx: SyncRequestContext,
+  clientId: string | undefined,
+  error: unknown,
+): Promise<void> {
+  await recordRoundFailureCode(
+    ctx,
+    clientId,
+    error instanceof SyncError ? error.code : 'sync.internal_error',
+  );
+}
+
+/** As `recordRoundFailure`, for a code that is not a thrown catalog error
+ * (a `ROUND_FAILURE` report, §1.5). */
+async function recordRoundFailureCode(
+  ctx: SyncRequestContext,
+  clientId: string | undefined,
+  code: string,
+): Promise<void> {
+  if (clientId === undefined) return;
+  try {
+    await ctx.storage.recordClientRoundFailure(
+      ctx.partition,
+      clientId,
+      ctx.actorId,
+      code,
+      clockOf(ctx)(),
+    );
+  } catch (failure) {
+    reportError(failure, ctx.onError, 'sync');
+  }
+}
+
+/** Record a failure thrown while the response streams (after planning). */
+async function* recordedStream(
+  stream: AsyncIterable<Uint8Array>,
+  ctx: SyncRequestContext,
+  clientId: string,
+): AsyncGenerator<Uint8Array> {
+  try {
+    yield* stream;
+  } catch (error) {
+    await recordRoundFailure(ctx, clientId, error);
     throw error;
   }
 }
@@ -855,6 +946,16 @@ async function createStreamCore(
   );
   const served = selected ?? schema;
   const plan = await planRequest(request, ctx, served, registry);
+  // §1.5/§4.5: record the reported round failure on the existing record
+  // before the round runs (it never creates one, never touches another
+  // actor's record). The completion of this round leaves it in place.
+  if (plan.roundFailure !== undefined) {
+    await recordRoundFailureCode(
+      ctx,
+      plan.header.clientId,
+      plan.roundFailure.code,
+    );
+  }
   const report: RequestReport = { outcome: 'ok' };
   // RFC 0007 read-verify-refuse for the streamed read path: the generator
   // builds the whole response (its data reads) before yielding the first
@@ -883,15 +984,21 @@ async function createStreamCore(
     if (changed !== undefined) throw serveNotReadyError(changed);
     for (const chunk of buffered) yield chunk;
   };
-  if (events === undefined) return verified();
-  return instrumentedStream(
-    verified(),
+  if (events === undefined) {
+    return recordedStream(verified(), ctx, plan.header.clientId);
+  }
+  return recordedStream(
+    instrumentedStream(
+      verified(),
+      ctx,
+      events,
+      plan,
+      report,
+      bytes.length,
+      startedAtMs,
+    ),
     ctx,
-    events,
-    plan,
-    report,
-    bytes.length,
-    startedAtMs,
+    plan.header.clientId,
   );
 }