@syncular/server 0.39.1 → 0.40.0

package/src/d1-storage.ts

@@ -1,7 +1,7 @@
 import { validateCommitPruneQuery } from './prune';
 import { StorageQueryError } from './storage-errors';
 import type { CommitPruneQuery, CommitPruneResult } from './storage';
-import { PARTITION_TOUCH_INTERVAL_MS } from './storage';
+import { clientRoundState, PARTITION_TOUCH_INTERVAL_MS } from './storage';
 /**
  * Cloudflare D1 server storage for Workers deployments.
  *
@@ -78,6 +78,7 @@ import {
   type StoredColumnLayout,
   scanRowPageSql,
   schemaDdl,
+  serverIndexDdl,
   SQLITE_TABLE_INDEXES_SQL,
   type SqliteTableIndexRow,
   sqliteIndexSignatures,
@@ -104,6 +105,7 @@ import {
   type SqliteRowRecord,
   serializePushResult,
   sqliteDdlStatements,
+  SYNC_CLIENT_ROUND_COLUMNS,
   toStoredChange,
   toStoredRow,
 } from './sqlite-dialect';
@@ -999,6 +1001,15 @@ export class D1ServerStorage implements ServerStorage {
         'ALTER TABLE sync_clients ADD COLUMN wire_version INTEGER NOT NULL DEFAULT 1;',
       );
     }
+    for (const [name, definition] of Object.entries(
+      SYNC_CLIENT_ROUND_COLUMNS,
+    )) {
+      if (!results.some((column) => column.name === name)) {
+        await this.#db.exec(
+          `ALTER TABLE sync_clients ADD COLUMN ${name} ${definition};`,
+        );
+      }
+    }
   }
 
   /** Resolve a table's compiled schema; row operations require `ensureSchema`. */
@@ -1129,6 +1140,44 @@ export class D1ServerStorage implements ServerStorage {
           'sqlite',
         );
       }
+      // A database from before the round-state columns gains them in place.
+      statementsExecuted++;
+      const clientColumns = await this.#db
+        .prepare('PRAGMA table_info("sync_clients")')
+        .all<{ name: string }>();
+      const missingRoundColumns = Object.entries(
+        SYNC_CLIENT_ROUND_COLUMNS,
+      ).filter(
+        ([name]) =>
+          !clientColumns.results.some((column) => column.name === name),
+      );
+      if (missingRoundColumns.length > 0) {
+        await batch(
+          missingRoundColumns.map(([name, definition]) =>
+            this.#db.prepare(
+              `ALTER TABLE sync_clients ADD COLUMN ${name} ${definition}`,
+            ),
+          ),
+        );
+      }
+      // Server-only indexes reconcile at every open of a database already at
+      // this version: they never reach a client, so no version bump is needed.
+      const indexes = new Map<string, ReadonlyMap<string, string>>();
+      for (const table of tables) {
+        statementsExecuted++;
+        const { results } = await this.#db
+          .prepare(SQLITE_TABLE_INDEXES_SQL)
+          .bind(table.name)
+          .all<SqliteTableIndexRow>();
+        indexes.set(
+          table.name,
+          new Map(Object.entries(sqliteIndexSignatures(results))),
+        );
+      }
+      const reconcile = serverIndexDdl(schema, indexes);
+      if (reconcile.length > 0) {
+        await batch(reconcile.map((sql) => this.#db.prepare(sql)));
+      }
       this.#tables = schema.tables;
       this.#schemaVersion = schema.version;
       return { complete: true, statementsExecuted };
@@ -1248,19 +1297,20 @@ export class D1ServerStorage implements ServerStorage {
         const columns = await this.#db
           .prepare('PRAGMA table_info("sync_clients")')
           .all<{ name: string }>();
-        const needsWireVersion = !columns.results.some(
-          (column) => column.name === 'wire_version',
+        const missingColumns = Object.entries({
+          wire_version: 'INTEGER NOT NULL DEFAULT 1',
+          ...SYNC_CLIENT_ROUND_COLUMNS,
+        }).filter(
+          ([name]) => !columns.results.some((column) => column.name === name),
         );
         if (
           !(await save(
             { ...state, phase: 'prepare', offset: 0 },
-            needsWireVersion
-              ? [
-                  this.#db.prepare(
-                    'ALTER TABLE sync_clients ADD COLUMN wire_version INTEGER NOT NULL DEFAULT 1',
-                  ),
-                ]
-              : [],
+            missingColumns.map(([name, definition]) =>
+              this.#db.prepare(
+                `ALTER TABLE sync_clients ADD COLUMN ${name} ${definition}`,
+              ),
+            ),
           ))
         )
           break;
@@ -2333,7 +2383,7 @@ export class D1ServerStorage implements ServerStorage {
   ): Promise<ClientRecord | undefined> {
     const record = await this.#db
       .prepare(
-        'SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=? AND client_id=?',
+        'SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms, last_error_code, consecutive_failures FROM sync_clients WHERE partition=? AND client_id=?',
       )
       .bind(partition, clientId)
       .first<{
@@ -2343,6 +2393,9 @@ export class D1ServerStorage implements ServerStorage {
         cursor: number;
         subscriptions: string;
         updated_at_ms: number;
+        last_round_at_ms: number | null;
+        last_error_code: string | null;
+        consecutive_failures: number;
       }>();
     if (record === null) return undefined;
     return {
@@ -2352,6 +2405,11 @@ export class D1ServerStorage implements ServerStorage {
       cursor: record.cursor,
       updatedAtMs: record.updated_at_ms,
       subscriptions: JSON.parse(record.subscriptions) as ClientSubscription[],
+      ...clientRoundState(
+        record.last_round_at_ms,
+        record.last_error_code,
+        record.consecutive_failures,
+      ),
     };
   }
 
@@ -2364,13 +2422,16 @@ export class D1ServerStorage implements ServerStorage {
     // record; a delete-and-insert would drop and recreate the row instead.
     await this.#db
       .prepare(
-        `INSERT INTO sync_clients(partition, client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms) VALUES (?,?,?,?,?,?,?)
+        `INSERT INTO sync_clients(partition, client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms) VALUES (?,?,?,?,?,?,?,?)
          ON CONFLICT(partition, client_id) DO UPDATE SET
            actor_id=excluded.actor_id,
            wire_version=excluded.wire_version,
            cursor=excluded.cursor,
            subscriptions=excluded.subscriptions,
-           updated_at_ms=excluded.updated_at_ms`,
+           updated_at_ms=excluded.updated_at_ms,
+           last_round_at_ms=excluded.last_round_at_ms,
+           last_error_code=NULL,
+           consecutive_failures=0`,
       )
       .bind(
         partition,
@@ -2380,7 +2441,26 @@ export class D1ServerStorage implements ServerStorage {
         record.cursor,
         JSON.stringify(record.subscriptions),
         record.updatedAtMs,
+        record.updatedAtMs,
+      )
+      .run();
+  }
+
+  async recordClientRoundFailure(
+    partition: string,
+    clientId: string,
+    actorId: string,
+    errorCode: string,
+    atMs: number,
+  ): Promise<void> {
+    await this.#db
+      .prepare(
+        `UPDATE sync_clients
+         SET last_round_at_ms=?, last_error_code=?,
+             consecutive_failures=consecutive_failures+1
+         WHERE partition=? AND client_id=? AND actor_id=?`,
       )
+      .bind(atMs, errorCode, partition, clientId, actorId)
       .run();
   }
 
@@ -2537,7 +2617,7 @@ export class D1ServerStorage implements ServerStorage {
   async listClientRecords(partition: string): Promise<ClientRecord[]> {
     const { results } = await this.#db
       .prepare(
-        'SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=? ORDER BY updated_at_ms DESC',
+        'SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms, last_error_code, consecutive_failures FROM sync_clients WHERE partition=? ORDER BY updated_at_ms DESC',
       )
       .bind(partition)
       .all<{
@@ -2547,6 +2627,9 @@ export class D1ServerStorage implements ServerStorage {
         cursor: number;
         subscriptions: string;
         updated_at_ms: number;
+        last_round_at_ms: number | null;
+        last_error_code: string | null;
+        consecutive_failures: number;
       }>();
     return results.map((record) => ({
       clientId: record.client_id,
@@ -2555,6 +2638,11 @@ export class D1ServerStorage implements ServerStorage {
       cursor: record.cursor,
       updatedAtMs: record.updated_at_ms,
       subscriptions: JSON.parse(record.subscriptions) as ClientSubscription[],
+      ...clientRoundState(
+        record.last_round_at_ms,
+        record.last_error_code,
+        record.consecutive_failures,
+      ),
     }));
   }
 

package/src/handler.ts

@@ -30,7 +30,7 @@ import {
   RESOLVER_OUTAGE,
   touchAuthenticatedPartition,
 } from './context';
-import { SyncError, syncError } from './errors';
+import { reportError, SyncError, syncError } from './errors';
 import {
   emitEvent,
   type PullSubscriptionSummary,
@@ -734,6 +734,7 @@ async function* streamResponse(
         report.outcome = 'error';
         report.errorCode = error.code;
       }
+      await recordRoundFailure(ctx, plan.header.clientId, error);
       // §1.6: in-band ERROR, then END, nothing else.
       yield encodeResponseFrame(
         {
@@ -765,7 +766,14 @@ export async function createSyncResponseStream(
   ctx: SyncRequestContext,
 ): Promise<AsyncIterable<Uint8Array>> {
   const sink = ctx.events;
-  if (sink === undefined) return createStreamCore(bytes, ctx, undefined);
+  if (sink === undefined) {
+    try {
+      return await createStreamCore(bytes, ctx, undefined);
+    } catch (error) {
+      await recordRoundFailure(ctx, requestClientId(bytes), error);
+      throw error;
+    }
+  }
   // Instrumented path: measure with the ctx clock, count bytes at the
   // seam, emit exactly one `request.handled` per request. Every event of
   // this request carries one `requestId`.
@@ -779,6 +787,7 @@ export async function createSyncResponseStream(
   try {
     return await createStreamCore(bytes, requestCtx, events, startedAtMs);
   } catch (error) {
+    const clientId = requestClientId(bytes);
     emitEvent(events, {
       type: 'request.handled',
       kind: 'sync',
@@ -789,11 +798,64 @@ export async function createSyncResponseStream(
       bytesIn: bytes.length,
       bytesOut: 0,
       outcome: 'rejected',
+      ...(clientId !== undefined ? { clientId } : {}),
       errorCode: error instanceof SyncError ? error.code : 'internal',
       pushCommits: 0,
       pulled: false,
       subscriptions: 0,
     });
+    await recordRoundFailure(ctx, clientId, error);
+    throw error;
+  }
+}
+
+/** The `REQ_HEADER` clientId of a request that failed before planning. */
+function requestClientId(bytes: Uint8Array): string | undefined {
+  try {
+    const message = decodeMessage(bytes);
+    const header = message.frames[0];
+    return message.msgKind === 'request' && header?.type === 'REQ_HEADER'
+      ? header.clientId
+      : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+/**
+ * Record a failed round on the client's stored record so an operator can see
+ * a stuck client and why. A storage failure here is reported through
+ * `onError` and never replaces the round's own outcome.
+ */
+async function recordRoundFailure(
+  ctx: SyncRequestContext,
+  clientId: string | undefined,
+  error: unknown,
+): Promise<void> {
+  if (clientId === undefined) return;
+  try {
+    await ctx.storage.recordClientRoundFailure(
+      ctx.partition,
+      clientId,
+      ctx.actorId,
+      error instanceof SyncError ? error.code : 'sync.internal_error',
+      clockOf(ctx)(),
+    );
+  } catch (failure) {
+    reportError(failure, ctx.onError, 'sync');
+  }
+}
+
+/** Record a failure thrown while the response streams (after planning). */
+async function* recordedStream(
+  stream: AsyncIterable<Uint8Array>,
+  ctx: SyncRequestContext,
+  clientId: string,
+): AsyncGenerator<Uint8Array> {
+  try {
+    yield* stream;
+  } catch (error) {
+    await recordRoundFailure(ctx, clientId, error);
     throw error;
   }
 }
@@ -883,15 +945,21 @@ async function createStreamCore(
     if (changed !== undefined) throw serveNotReadyError(changed);
     for (const chunk of buffered) yield chunk;
   };
-  if (events === undefined) return verified();
-  return instrumentedStream(
-    verified(),
+  if (events === undefined) {
+    return recordedStream(verified(), ctx, plan.header.clientId);
+  }
+  return recordedStream(
+    instrumentedStream(
+      verified(),
+      ctx,
+      events,
+      plan,
+      report,
+      bytes.length,
+      startedAtMs,
+    ),
     ctx,
-    events,
-    plan,
-    report,
-    bytes.length,
-    startedAtMs,
+    plan.header.clientId,
   );
 }
 

package/src/postgres-fanout.ts

@@ -29,6 +29,14 @@
  * The payload carries `commitSeq` purely for observability/ordering in logs;
  * the wake itself is seq-agnostic (a session pulls whatever it is behind).
  *
+ * ## Listener gaps
+ *
+ * Postgres drops notifications for a session that is not listening, so a
+ * LISTEN connection that dies and reconnects (server restart, failover, a
+ * serverless compute suspending) misses every commit in between. The
+ * connection reports each LISTEN that takes effect through `onListen`; every
+ * report after the first wakes all local sessions with `catchup-required`.
+ *
  * pglite cannot exercise cross-connection NOTIFY, so the integration
  * (`postgres-fanout.integration.test.ts`) is env-gated on `SYNCULAR_PG_URL`
  * and skips cleanly; the payload encode/parse is unit-tested hermetically.
@@ -84,21 +92,27 @@ export function parseFanoutPayload(text: string): FanoutPayload | undefined {
 /** The hub surface the fanout wakes (a `RealtimeHub` satisfies this). */
 export interface FanoutWakeTarget {
   wake(partition: string, reason: WakeReason): void;
+  wakeAll(reason: WakeReason): void;
 }
 
 /**
  * A raw notification connection. Production wires a driver's LISTEN support:
+ *   - Bun.sql and postgres.js: `sql.listen(channel, handler, onListen)`; both
+ *     reconnect a dropped listener and call `onListen` again;
  *   - node-postgres: a dedicated `Client` with `client.on('notification', …)`
- *     after `LISTEN syncular_commit`;
- *   - Bun.sql: `sql.listen(channel, handler)`.
+ *     after `LISTEN syncular_commit`, reconnected by the host.
  * The `notify` side can reuse the main pool (`SELECT pg_notify($1,$2)`), but
- * LISTEN needs its own long-lived connection.
+ * LISTEN needs its own long-lived, unpooled connection.
  */
 export interface PgNotificationConnection {
-  /** Register a payload handler for the channel and begin listening. */
+  /**
+   * Register a payload handler for the channel and begin listening. Call
+   * `onListen` every time LISTEN takes effect, including after a reconnect.
+   */
   listen(
     channel: string,
     handler: (payload: string) => void,
+    onListen: () => void,
   ): Promise<void> | void;
   /** Send a notification on the channel. */
   notify(channel: string, payload: string): Promise<void>;
@@ -122,11 +136,20 @@ export class PostgresFanout {
   async install(hub: FanoutWakeTarget): Promise<void> {
     if (this.#installed) throw new Error('fanout already installed');
     this.#installed = true;
-    await this.#conn.listen(FANOUT_CHANNEL, (payload) => {
-      const parsed = parseFanoutPayload(payload);
-      if (parsed === undefined) return; // ignore malformed frames
-      hub.wake(parsed.partition, FANOUT_WAKE_REASON);
-    });
+    let listening = false;
+    await this.#conn.listen(
+      FANOUT_CHANNEL,
+      (payload) => {
+        const parsed = parseFanoutPayload(payload);
+        if (parsed === undefined) return; // ignore malformed frames
+        hub.wake(parsed.partition, FANOUT_WAKE_REASON);
+      },
+      () => {
+        // A repeated LISTEN follows a gap whose notifications are lost.
+        if (listening) hub.wakeAll(FANOUT_WAKE_REASON);
+        listening = true;
+      },
+    );
   }
 
   /** Fan an applied commit out to the other instances. */