@syncular/server 0.39.1 → 0.40.0

package/README.md

@@ -135,7 +135,10 @@ the complete [backup and restore runbook](https://syncular.dev/server-operations
 A `SyncError` answers with its catalog code and status. By default, any other exception an
 adapter catches answers `sync.internal_error` and is handed to
 `SyncServerConfig.onError` exactly once; a throwing observer does not change
-the response. The same config accepts an optional synchronous `mapError` hook:
+the response. A `SyncError` never reaches `onError`: observe a catalog failure
+through the `request.handled` event (`outcome`, `errorCode`, `clientId`) or the
+client's stored `lastErrorCode` / `consecutiveFailures` / `lastRoundAtMs`, which
+`ServerStorage.recordClientRoundFailure` updates on every failed round. The same config accepts an optional synchronous `mapError` hook:
 after `onError` observes the original, `mapError` may return a catalog
 `SyncError` (with structured `details`, for example `retryAfterMs`) that the
 adapter answers instead. Return `undefined` to keep `sync.internal_error`. A
@@ -1292,15 +1295,13 @@ install no fanout at all.
 ```ts
 import { PostgresFanout, type PgNotificationConnection } from '@syncular/server';
 
-// node-postgres: a dedicated Client for LISTEN + the pool for NOTIFY.
+// Bun.sql over the direct (unpooled) URL: LISTEN needs a session-mode
+// connection, and Bun.sql reconnects a dropped listener and re-runs onListen.
+const sql = new Bun.SQL(process.env.DATABASE_URL_DIRECT!);
 const conn: PgNotificationConnection = {
-  async listen(channel, handler) {
-    const client = await pool.connect(); // long-lived, NOT released
-    client.on('notification', (m) => m.payload && handler(m.payload));
-    await client.query(`LISTEN ${channel}`);
-  },
-  notify: (channel, payload) =>
-    pool.query('SELECT pg_notify($1, $2)', [channel, payload]).then(() => {}),
+  listen: (channel, handler, onListen) =>
+    sql.listen(channel, handler, onListen).then(() => {}),
+  notify: (channel, payload) => sql.notify(channel, payload).then(() => {}),
 };
 const fanout = new PostgresFanout(conn);
 await fanout.install(hub); // start the LISTEN loop
@@ -1308,10 +1309,16 @@ await fanout.install(hub); // start the LISTEN loop
 await fanout.notifyCommit(partition, commitSeq);
 ```
 
+Notifications sent while a listener session is down are lost. Every
+`onListen` after the first therefore wakes all local sessions with
+`catchup-required` (`hub.wakeAll`). A node-postgres adapter reconnects its
+dedicated `Client` on `end` and calls `onListen` after each `LISTEN`.
+
 pglite is single-connection and cannot exercise cross-connection NOTIFY,
 so the fanout integration test is env-gated on `SYNCULAR_PG_URL` (it wires
-Bun.sql as a worked example) and skips cleanly; the payload encode/parse
-and wake wiring are unit-tested hermetically.
+Bun.sql as a worked example and terminates the listener backend to cover the
+reconnect path) and skips cleanly; the payload encode/parse and wake wiring
+are unit-tested hermetically.
 
 ### Bench lane
 

package/dist/admin.d.ts

@@ -51,6 +51,12 @@ export interface AdminClient {
     readonly connected?: boolean;
     /** When the client's oldest open socket opened. */
     readonly connectedSinceMs?: number;
+    /** When its last sync round finished, successfully or not. */
+    readonly lastRoundAtMs?: number;
+    /** Catalog code of its last round; present only while that round failed. */
+    readonly lastErrorCode?: string;
+    /** Rounds failed in a row since its last success; absent at zero. */
+    readonly consecutiveFailures?: number;
 }
 export interface AdminListClientsOptions {
     /** Default 200. */

package/dist/admin.js

@@ -926,6 +926,15 @@ export class SyncularAdmin {
                     ? { connected: connectedSinceMs !== undefined }
                     : {}),
                 ...(connectedSinceMs !== undefined ? { connectedSinceMs } : {}),
+                ...(record.lastRoundAtMs !== undefined
+                    ? { lastRoundAtMs: record.lastRoundAtMs }
+                    : {}),
+                ...(record.lastErrorCode !== undefined
+                    ? { lastErrorCode: record.lastErrorCode }
+                    : {}),
+                ...(record.consecutiveFailures !== undefined
+                    ? { consecutiveFailures: record.consecutiveFailures }
+                    : {}),
             };
         };
     }

package/dist/d1-storage.d.ts

@@ -124,6 +124,7 @@ export declare class D1ServerStorage implements ServerStorage {
     scanRowsByIndex(partition: string, query: IndexRowScanQuery): Promise<StoredRow[]>;
     getClientRecord(partition: string, clientId: string): Promise<ClientRecord | undefined>;
     putClientRecord(partition: string, record: ClientRecord): Promise<void>;
+    recordClientRoundFailure(partition: string, clientId: string, actorId: string, errorCode: string, atMs: number): Promise<void>;
     advanceClientCursor(partition: string, clientId: string, actorId: string, logEpoch: string, cursor: number, updatedAtMs: number): Promise<void>;
     updateClientCursor(partition: string, clientId: string, cursor: number, updatedAtMs: number): Promise<void>;
     getActiveClientCursorFloor(partition: string, cutoffMs: number): Promise<number | null>;

package/dist/d1-storage.js

@@ -1,7 +1,7 @@
 var _a;
 import { validateCommitPruneQuery } from './prune.js';
 import { StorageQueryError } from './storage-errors.js';
-import { PARTITION_TOUCH_INTERVAL_MS } from './storage.js';
+import { clientRoundState, PARTITION_TOUCH_INTERVAL_MS } from './storage.js';
 /**
  * Cloudflare D1 server storage for Workers deployments.
  *
@@ -46,9 +46,9 @@ import { PARTITION_TOUCH_INTERVAL_MS } from './storage.js';
 import { decodeRow } from '@syncular/core';
 import { bindAuthoritativePartition, prepareAuthoritativeQuery, } from './authoritative-query.js';
 import { syncError } from './errors.js';
-import { assertAppendOnlyMigration, assertPhysicalColumns, assertStoredLayouts, commitWindowPageSql, LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL, deleteRowSql, deleteSqliteRowScopesSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, scanRowPageSql, schemaDdl, SQLITE_TABLE_INDEXES_SQL, sqliteIndexSignatures, selectRowScopesSql, selectRowSql, selectRowsForRewriteSql, tableColumnNames, toSqlValue, upsertSql, upsertValues, } from './relational-rows.js';
+import { assertAppendOnlyMigration, assertPhysicalColumns, assertStoredLayouts, commitWindowPageSql, LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL, deleteRowSql, deleteSqliteRowScopesSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, scanRowPageSql, schemaDdl, serverIndexDdl, SQLITE_TABLE_INDEXES_SQL, sqliteIndexSignatures, selectRowScopesSql, selectRowSql, selectRowsForRewriteSql, tableColumnNames, toSqlValue, upsertSql, upsertValues, } from './relational-rows.js';
 import { matchesEffective } from './scopes.js';
-import { asUint8Array, collectCommitWindowPage, deserializePushResult, placeholders, ERASE_SCOPE_VALUE_HISTORY_SQLITE_SQL, eraseScopeValueHistoryParams, LIST_SCOPE_VALUE_ROWS_SQLITE_SQL, serializePushResult, sqliteDdlStatements, toStoredChange, toStoredRow, } from './sqlite-dialect.js';
+import { asUint8Array, collectCommitWindowPage, deserializePushResult, placeholders, ERASE_SCOPE_VALUE_HISTORY_SQLITE_SQL, eraseScopeValueHistoryParams, LIST_SCOPE_VALUE_ROWS_SQLITE_SQL, serializePushResult, sqliteDdlStatements, SYNC_CLIENT_ROUND_COLUMNS, toStoredChange, toStoredRow, } from './sqlite-dialect.js';
 import { isD1ConstraintError, StorageConstraintError } from './storage-errors.js';
 import { assertScopeIndexedScan, resolveIndexRowScan } from './storage-query.js';
 function toStoredReaction(record) {
@@ -617,6 +617,11 @@ export class D1ServerStorage {
         if (!results.some((column) => column.name === 'wire_version')) {
             await this.#db.exec('ALTER TABLE sync_clients ADD COLUMN wire_version INTEGER NOT NULL DEFAULT 1;');
         }
+        for (const [name, definition] of Object.entries(SYNC_CLIENT_ROUND_COLUMNS)) {
+            if (!results.some((column) => column.name === name)) {
+                await this.#db.exec(`ALTER TABLE sync_clients ADD COLUMN ${name} ${definition};`);
+            }
+        }
     }
     /** Resolve a table's compiled schema; row operations require `ensureSchema`. */
     table(name) {
@@ -713,6 +718,30 @@ export class D1ServerStorage {
                     primaryKeyPosition: column.pk,
                 })), 'sqlite');
             }
+            // A database from before the round-state columns gains them in place.
+            statementsExecuted++;
+            const clientColumns = await this.#db
+                .prepare('PRAGMA table_info("sync_clients")')
+                .all();
+            const missingRoundColumns = Object.entries(SYNC_CLIENT_ROUND_COLUMNS).filter(([name]) => !clientColumns.results.some((column) => column.name === name));
+            if (missingRoundColumns.length > 0) {
+                await batch(missingRoundColumns.map(([name, definition]) => this.#db.prepare(`ALTER TABLE sync_clients ADD COLUMN ${name} ${definition}`)));
+            }
+            // Server-only indexes reconcile at every open of a database already at
+            // this version: they never reach a client, so no version bump is needed.
+            const indexes = new Map();
+            for (const table of tables) {
+                statementsExecuted++;
+                const { results } = await this.#db
+                    .prepare(SQLITE_TABLE_INDEXES_SQL)
+                    .bind(table.name)
+                    .all();
+                indexes.set(table.name, new Map(Object.entries(sqliteIndexSignatures(results))));
+            }
+            const reconcile = serverIndexDdl(schema, indexes);
+            if (reconcile.length > 0) {
+                await batch(reconcile.map((sql) => this.#db.prepare(sql)));
+            }
             this.#tables = schema.tables;
             this.#schemaVersion = schema.version;
             return { complete: true, statementsExecuted };
@@ -816,12 +845,11 @@ export class D1ServerStorage {
                 const columns = await this.#db
                     .prepare('PRAGMA table_info("sync_clients")')
                     .all();
-                const needsWireVersion = !columns.results.some((column) => column.name === 'wire_version');
-                if (!(await save({ ...state, phase: 'prepare', offset: 0 }, needsWireVersion
-                    ? [
-                        this.#db.prepare('ALTER TABLE sync_clients ADD COLUMN wire_version INTEGER NOT NULL DEFAULT 1'),
-                    ]
-                    : [])))
+                const missingColumns = Object.entries({
+                    wire_version: 'INTEGER NOT NULL DEFAULT 1',
+                    ...SYNC_CLIENT_ROUND_COLUMNS,
+                }).filter(([name]) => !columns.results.some((column) => column.name === name));
+                if (!(await save({ ...state, phase: 'prepare', offset: 0 }, missingColumns.map(([name, definition]) => this.#db.prepare(`ALTER TABLE sync_clients ADD COLUMN ${name} ${definition}`)))))
                     break;
             }
             else if (state.phase === 'prepare') {
@@ -1562,7 +1590,7 @@ export class D1ServerStorage {
     }
     async getClientRecord(partition, clientId) {
         const record = await this.#db
-            .prepare('SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=? AND client_id=?')
+            .prepare('SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms, last_error_code, consecutive_failures FROM sync_clients WHERE partition=? AND client_id=?')
             .bind(partition, clientId)
             .first();
         if (record === null)
@@ -1574,6 +1602,7 @@ export class D1ServerStorage {
             cursor: record.cursor,
             updatedAtMs: record.updated_at_ms,
             subscriptions: JSON.parse(record.subscriptions),
+            ...clientRoundState(record.last_round_at_ms, record.last_error_code, record.consecutive_failures),
         };
     }
     async putClientRecord(partition, record) {
@@ -1581,14 +1610,26 @@ export class D1ServerStorage {
         // (partition, client_id). Every non-key field is written from the new
         // record; a delete-and-insert would drop and recreate the row instead.
         await this.#db
-            .prepare(`INSERT INTO sync_clients(partition, client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms) VALUES (?,?,?,?,?,?,?)
+            .prepare(`INSERT INTO sync_clients(partition, client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms) VALUES (?,?,?,?,?,?,?,?)
          ON CONFLICT(partition, client_id) DO UPDATE SET
            actor_id=excluded.actor_id,
            wire_version=excluded.wire_version,
            cursor=excluded.cursor,
            subscriptions=excluded.subscriptions,
-           updated_at_ms=excluded.updated_at_ms`)
-            .bind(partition, record.clientId, record.actorId, record.wireVersion, record.cursor, JSON.stringify(record.subscriptions), record.updatedAtMs)
+           updated_at_ms=excluded.updated_at_ms,
+           last_round_at_ms=excluded.last_round_at_ms,
+           last_error_code=NULL,
+           consecutive_failures=0`)
+            .bind(partition, record.clientId, record.actorId, record.wireVersion, record.cursor, JSON.stringify(record.subscriptions), record.updatedAtMs, record.updatedAtMs)
+            .run();
+    }
+    async recordClientRoundFailure(partition, clientId, actorId, errorCode, atMs) {
+        await this.#db
+            .prepare(`UPDATE sync_clients
+         SET last_round_at_ms=?, last_error_code=?,
+             consecutive_failures=consecutive_failures+1
+         WHERE partition=? AND client_id=? AND actor_id=?`)
+            .bind(atMs, errorCode, partition, clientId, actorId)
             .run();
     }
     async advanceClientCursor(partition, clientId, actorId, logEpoch, cursor, updatedAtMs) {
@@ -1686,7 +1727,7 @@ export class D1ServerStorage {
     // -- admin/console read surface --------------------------------------------
     async listClientRecords(partition) {
         const { results } = await this.#db
-            .prepare('SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=? ORDER BY updated_at_ms DESC')
+            .prepare('SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms, last_error_code, consecutive_failures FROM sync_clients WHERE partition=? ORDER BY updated_at_ms DESC')
             .bind(partition)
             .all();
         return results.map((record) => ({
@@ -1696,6 +1737,7 @@ export class D1ServerStorage {
             cursor: record.cursor,
             updatedAtMs: record.updated_at_ms,
             subscriptions: JSON.parse(record.subscriptions),
+            ...clientRoundState(record.last_round_at_ms, record.last_error_code, record.consecutive_failures),
         }));
     }
     async listCommitMetadata(partition, query) {

package/dist/handler.js

@@ -11,7 +11,7 @@
  */
 import { DecodeError, decodeMessage, } from '@syncular/core';
 import { clockOf, limitsOf, REMOTE_COMMAND_CLIENT_ID_PREFIX, RESOLVER_OUTAGE, touchAuthenticatedPartition, } from './context.js';
-import { SyncError, syncError } from './errors.js';
+import { reportError, SyncError, syncError } from './errors.js';
 import { emitEvent, } from './events.js';
 import { END_FRAME_BYTES, encodeResponseFrame, responseEnvelopeHeader, } from './frame-bytes.js';
 import { ACCEPT_EXTERNAL_ROWS, ACCEPT_INLINE_ROWS, clampPullLimits, prepareSections, preparedSection, } from './pull.js';
@@ -495,6 +495,7 @@ async function* streamResponse(plan, ctx, schema, report) {
                 report.outcome = 'error';
                 report.errorCode = error.code;
             }
+            await recordRoundFailure(ctx, plan.header.clientId, error);
             // §1.6: in-band ERROR, then END, nothing else.
             yield encodeResponseFrame({
                 type: 'ERROR',
@@ -519,8 +520,15 @@ async function* streamResponse(plan, ctx, schema, report) {
  */
 export async function createSyncResponseStream(bytes, ctx) {
     const sink = ctx.events;
-    if (sink === undefined)
-        return createStreamCore(bytes, ctx, undefined);
+    if (sink === undefined) {
+        try {
+            return await createStreamCore(bytes, ctx, undefined);
+        }
+        catch (error) {
+            await recordRoundFailure(ctx, requestClientId(bytes), error);
+            throw error;
+        }
+    }
     // Instrumented path: measure with the ctx clock, count bytes at the
     // seam, emit exactly one `request.handled` per request. Every event of
     // this request carries one `requestId`.
@@ -535,6 +543,7 @@ export async function createSyncResponseStream(bytes, ctx) {
         return await createStreamCore(bytes, requestCtx, events, startedAtMs);
     }
     catch (error) {
+        const clientId = requestClientId(bytes);
         emitEvent(events, {
             type: 'request.handled',
             kind: 'sync',
@@ -545,11 +554,51 @@ export async function createSyncResponseStream(bytes, ctx) {
             bytesIn: bytes.length,
             bytesOut: 0,
             outcome: 'rejected',
+            ...(clientId !== undefined ? { clientId } : {}),
             errorCode: error instanceof SyncError ? error.code : 'internal',
             pushCommits: 0,
             pulled: false,
             subscriptions: 0,
         });
+        await recordRoundFailure(ctx, clientId, error);
+        throw error;
+    }
+}
+/** The `REQ_HEADER` clientId of a request that failed before planning. */
+function requestClientId(bytes) {
+    try {
+        const message = decodeMessage(bytes);
+        const header = message.frames[0];
+        return message.msgKind === 'request' && header?.type === 'REQ_HEADER'
+            ? header.clientId
+            : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * Record a failed round on the client's stored record so an operator can see
+ * a stuck client and why. A storage failure here is reported through
+ * `onError` and never replaces the round's own outcome.
+ */
+async function recordRoundFailure(ctx, clientId, error) {
+    if (clientId === undefined)
+        return;
+    try {
+        await ctx.storage.recordClientRoundFailure(ctx.partition, clientId, ctx.actorId, error instanceof SyncError ? error.code : 'sync.internal_error', clockOf(ctx)());
+    }
+    catch (failure) {
+        reportError(failure, ctx.onError, 'sync');
+    }
+}
+/** Record a failure thrown while the response streams (after planning). */
+async function* recordedStream(stream, ctx, clientId) {
+    try {
+        yield* stream;
+    }
+    catch (error) {
+        await recordRoundFailure(ctx, clientId, error);
         throw error;
     }
 }
@@ -623,9 +672,10 @@ async function createStreamCore(bytes, ctx, events, startedAtMs = 0) {
         for (const chunk of buffered)
             yield chunk;
     };
-    if (events === undefined)
-        return verified();
-    return instrumentedStream(verified(), ctx, events, plan, report, bytes.length, startedAtMs);
+    if (events === undefined) {
+        return recordedStream(verified(), ctx, plan.header.clientId);
+    }
+    return recordedStream(instrumentedStream(verified(), ctx, events, plan, report, bytes.length, startedAtMs), ctx, plan.header.clientId);
 }
 /** Counts response bytes and emits `request.handled` when the stream
  * finishes — including a thrown host failure mid-stream (fail loud). */

package/dist/postgres-fanout.d.ts

@@ -29,6 +29,14 @@
  * The payload carries `commitSeq` purely for observability/ordering in logs;
  * the wake itself is seq-agnostic (a session pulls whatever it is behind).
  *
+ * ## Listener gaps
+ *
+ * Postgres drops notifications for a session that is not listening, so a
+ * LISTEN connection that dies and reconnects (server restart, failover, a
+ * serverless compute suspending) misses every commit in between. The
+ * connection reports each LISTEN that takes effect through `onListen`; every
+ * report after the first wakes all local sessions with `catchup-required`.
+ *
  * pglite cannot exercise cross-connection NOTIFY, so the integration
  * (`postgres-fanout.integration.test.ts`) is env-gated on `SYNCULAR_PG_URL`
  * and skips cleanly; the payload encode/parse is unit-tested hermetically.
@@ -51,18 +59,23 @@ export declare function parseFanoutPayload(text: string): FanoutPayload | undefi
 /** The hub surface the fanout wakes (a `RealtimeHub` satisfies this). */
 export interface FanoutWakeTarget {
     wake(partition: string, reason: WakeReason): void;
+    wakeAll(reason: WakeReason): void;
 }
 /**
  * A raw notification connection. Production wires a driver's LISTEN support:
+ *   - Bun.sql and postgres.js: `sql.listen(channel, handler, onListen)`; both
+ *     reconnect a dropped listener and call `onListen` again;
  *   - node-postgres: a dedicated `Client` with `client.on('notification', …)`
- *     after `LISTEN syncular_commit`;
- *   - Bun.sql: `sql.listen(channel, handler)`.
+ *     after `LISTEN syncular_commit`, reconnected by the host.
  * The `notify` side can reuse the main pool (`SELECT pg_notify($1,$2)`), but
- * LISTEN needs its own long-lived connection.
+ * LISTEN needs its own long-lived, unpooled connection.
  */
 export interface PgNotificationConnection {
-    /** Register a payload handler for the channel and begin listening. */
-    listen(channel: string, handler: (payload: string) => void): Promise<void> | void;
+    /**
+     * Register a payload handler for the channel and begin listening. Call
+     * `onListen` every time LISTEN takes effect, including after a reconnect.
+     */
+    listen(channel: string, handler: (payload: string) => void, onListen: () => void): Promise<void> | void;
     /** Send a notification on the channel. */
     notify(channel: string, payload: string): Promise<void>;
     close?(): Promise<void>;

package/dist/postgres-fanout.js

@@ -54,11 +54,17 @@ export class PostgresFanout {
         if (this.#installed)
             throw new Error('fanout already installed');
         this.#installed = true;
+        let listening = false;
         await this.#conn.listen(FANOUT_CHANNEL, (payload) => {
             const parsed = parseFanoutPayload(payload);
             if (parsed === undefined)
                 return; // ignore malformed frames
             hub.wake(parsed.partition, FANOUT_WAKE_REASON);
+        }, () => {
+            // A repeated LISTEN follows a gap whose notifications are lost.
+            if (listening)
+                hub.wakeAll(FANOUT_WAKE_REASON);
+            listening = true;
         });
     }
     /** Fan an applied commit out to the other instances. */

package/dist/postgres-storage.d.ts

@@ -69,7 +69,7 @@ import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, CheckpointDec
  *     set, §5.9.5) as an index range, never a scan. `postgres-explain
  *     .test.ts` asserts an `Index` node here too.
  */
-export declare const POSTGRES_DDL = "\nCREATE TABLE IF NOT EXISTS sync_partitions(\n  partition TEXT PRIMARY KEY,\n  max_commit_seq BIGINT NOT NULL DEFAULT 0,\n  horizon_seq BIGINT NOT NULL DEFAULT 0\n);\nCREATE TABLE IF NOT EXISTS sync_partition_registry(\n  partition TEXT PRIMARY KEY,\n  log_epoch TEXT NOT NULL,\n  epoch_required BOOLEAN NOT NULL DEFAULT FALSE,\n  last_authenticated_at_ms BIGINT NOT NULL\n);\nCREATE TABLE IF NOT EXISTS sync_row_scopes(\n  partition TEXT NOT NULL, tbl TEXT NOT NULL,\n  var TEXT NOT NULL, value TEXT NOT NULL, row_id TEXT NOT NULL,\n  PRIMARY KEY(partition, tbl, var, value, row_id)\n);\nCREATE INDEX IF NOT EXISTS sync_row_scopes_by_row\n  ON sync_row_scopes(partition, tbl, row_id);\nCREATE TABLE IF NOT EXISTS sync_commits(\n  partition TEXT NOT NULL, commit_seq BIGINT NOT NULL,\n  client_id TEXT NOT NULL, client_commit_id TEXT NOT NULL,\n  actor_id TEXT NOT NULL, created_at_ms BIGINT NOT NULL,\n  writer_version BIGINT,\n  PRIMARY KEY(partition, commit_seq)\n);\nCREATE INDEX IF NOT EXISTS sync_commits_by_time\n  ON sync_commits(partition, created_at_ms);\nCREATE TABLE IF NOT EXISTS sync_changes(\n  partition TEXT NOT NULL, commit_seq BIGINT NOT NULL, idx INTEGER NOT NULL,\n  tbl TEXT NOT NULL, row_id TEXT NOT NULL, op SMALLINT NOT NULL,\n  row_version BIGINT, scopes JSONB NOT NULL, payload BYTEA,\n  PRIMARY KEY(partition, commit_seq, idx)\n);\nCREATE INDEX IF NOT EXISTS sync_changes_by_table\n  ON sync_changes(partition, commit_seq, tbl, idx);\nCREATE TABLE IF NOT EXISTS sync_change_scopes(\n  partition TEXT NOT NULL, tbl TEXT NOT NULL,\n  var TEXT NOT NULL, value TEXT NOT NULL, commit_seq BIGINT NOT NULL,\n  PRIMARY KEY(partition, tbl, var, value, commit_seq)\n);\nCREATE TABLE IF NOT EXISTS sync_push_results(\n  partition TEXT NOT NULL, client_id TEXT NOT NULL,\n  client_commit_id TEXT NOT NULL, result JSONB NOT NULL,\n  PRIMARY KEY(partition, client_id, client_commit_id)\n);\nCREATE TABLE IF NOT EXISTS sync_reactions(\n  partition TEXT NOT NULL, idempotency_key TEXT NOT NULL,\n  type TEXT NOT NULL, version INTEGER NOT NULL, payload JSONB NOT NULL,\n  source_client_id TEXT NOT NULL, source_client_commit_id TEXT NOT NULL,\n  source_commit_seq BIGINT NOT NULL, created_at_ms BIGINT NOT NULL,\n  available_at_ms BIGINT NOT NULL, status TEXT NOT NULL,\n  attempts INTEGER NOT NULL, max_attempts INTEGER NOT NULL,\n  lease_owner TEXT, lease_expires_at_ms BIGINT, completed_at_ms BIGINT,\n  last_failure JSONB,\n  PRIMARY KEY(partition, idempotency_key),\n  CHECK(status IN ('pending', 'leased', 'completed', 'dead-letter'))\n);\nCREATE INDEX IF NOT EXISTS sync_reactions_due\n  ON sync_reactions(partition, status, available_at_ms, created_at_ms, idempotency_key);\nCREATE INDEX IF NOT EXISTS sync_reactions_lease\n  ON sync_reactions(partition, status, lease_expires_at_ms);\nCREATE INDEX IF NOT EXISTS sync_reactions_completed\n  ON sync_reactions(partition, status, completed_at_ms, idempotency_key);\nCREATE INDEX IF NOT EXISTS sync_reactions_dead_letter\n  ON sync_reactions(partition, status, available_at_ms, idempotency_key);\nCREATE TABLE IF NOT EXISTS sync_clients(\n  partition TEXT NOT NULL, client_id TEXT NOT NULL, actor_id TEXT NOT NULL,\n  wire_version INTEGER NOT NULL DEFAULT 1,\n  cursor BIGINT NOT NULL, subscriptions JSONB NOT NULL,\n  updated_at_ms BIGINT NOT NULL,\n  PRIMARY KEY(partition, client_id)\n);\nALTER TABLE sync_clients\n  ADD COLUMN IF NOT EXISTS wire_version INTEGER NOT NULL DEFAULT 1;\nCREATE TABLE IF NOT EXISTS sync_blob_refs(\n  partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,\n  blob_id TEXT NOT NULL,\n  PRIMARY KEY(partition, tbl, row_id, blob_id)\n);\nCREATE INDEX IF NOT EXISTS sync_blob_refs_by_blob\n  ON sync_blob_refs(partition, blob_id);\nCREATE TABLE IF NOT EXISTS sync_tombstones(\n  partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,\n  commit_seq BIGINT NOT NULL,\n  PRIMARY KEY(partition, tbl, row_id)\n);\nCREATE TABLE IF NOT EXISTS sync_backfill_checkpoints(\n  partition TEXT NOT NULL, name TEXT NOT NULL,\n  schema_version INTEGER NOT NULL,\n  state TEXT NOT NULL CHECK(state IN ('declared','backfilling','activated')),\n  watermark BIGINT NOT NULL DEFAULT 0,\n  owner_epoch BIGINT NOT NULL DEFAULT 0,\n  observed_rows BIGINT NOT NULL DEFAULT 0,\n  updated_at_ms BIGINT NOT NULL,\n  PRIMARY KEY(partition, name)\n);\nCREATE TABLE IF NOT EXISTS sync_writer_fence(\n  partition TEXT PRIMARY KEY,\n  required_writer_version INTEGER NOT NULL\n);\n";
+export declare const POSTGRES_DDL = "\nCREATE TABLE IF NOT EXISTS sync_partitions(\n  partition TEXT PRIMARY KEY,\n  max_commit_seq BIGINT NOT NULL DEFAULT 0,\n  horizon_seq BIGINT NOT NULL DEFAULT 0\n);\nCREATE TABLE IF NOT EXISTS sync_partition_registry(\n  partition TEXT PRIMARY KEY,\n  log_epoch TEXT NOT NULL,\n  epoch_required BOOLEAN NOT NULL DEFAULT FALSE,\n  last_authenticated_at_ms BIGINT NOT NULL\n);\nCREATE TABLE IF NOT EXISTS sync_row_scopes(\n  partition TEXT NOT NULL, tbl TEXT NOT NULL,\n  var TEXT NOT NULL, value TEXT NOT NULL, row_id TEXT NOT NULL,\n  PRIMARY KEY(partition, tbl, var, value, row_id)\n);\nCREATE INDEX IF NOT EXISTS sync_row_scopes_by_row\n  ON sync_row_scopes(partition, tbl, row_id);\nCREATE TABLE IF NOT EXISTS sync_commits(\n  partition TEXT NOT NULL, commit_seq BIGINT NOT NULL,\n  client_id TEXT NOT NULL, client_commit_id TEXT NOT NULL,\n  actor_id TEXT NOT NULL, created_at_ms BIGINT NOT NULL,\n  writer_version BIGINT,\n  PRIMARY KEY(partition, commit_seq)\n);\nCREATE INDEX IF NOT EXISTS sync_commits_by_time\n  ON sync_commits(partition, created_at_ms);\nCREATE TABLE IF NOT EXISTS sync_changes(\n  partition TEXT NOT NULL, commit_seq BIGINT NOT NULL, idx INTEGER NOT NULL,\n  tbl TEXT NOT NULL, row_id TEXT NOT NULL, op SMALLINT NOT NULL,\n  row_version BIGINT, scopes JSONB NOT NULL, payload BYTEA,\n  PRIMARY KEY(partition, commit_seq, idx)\n);\nCREATE INDEX IF NOT EXISTS sync_changes_by_table\n  ON sync_changes(partition, commit_seq, tbl, idx);\nCREATE TABLE IF NOT EXISTS sync_change_scopes(\n  partition TEXT NOT NULL, tbl TEXT NOT NULL,\n  var TEXT NOT NULL, value TEXT NOT NULL, commit_seq BIGINT NOT NULL,\n  PRIMARY KEY(partition, tbl, var, value, commit_seq)\n);\nCREATE TABLE IF NOT EXISTS sync_push_results(\n  partition TEXT NOT NULL, client_id TEXT NOT NULL,\n  client_commit_id TEXT NOT NULL, result JSONB NOT NULL,\n  PRIMARY KEY(partition, client_id, client_commit_id)\n);\nCREATE TABLE IF NOT EXISTS sync_reactions(\n  partition TEXT NOT NULL, idempotency_key TEXT NOT NULL,\n  type TEXT NOT NULL, version INTEGER NOT NULL, payload JSONB NOT NULL,\n  source_client_id TEXT NOT NULL, source_client_commit_id TEXT NOT NULL,\n  source_commit_seq BIGINT NOT NULL, created_at_ms BIGINT NOT NULL,\n  available_at_ms BIGINT NOT NULL, status TEXT NOT NULL,\n  attempts INTEGER NOT NULL, max_attempts INTEGER NOT NULL,\n  lease_owner TEXT, lease_expires_at_ms BIGINT, completed_at_ms BIGINT,\n  last_failure JSONB,\n  PRIMARY KEY(partition, idempotency_key),\n  CHECK(status IN ('pending', 'leased', 'completed', 'dead-letter'))\n);\nCREATE INDEX IF NOT EXISTS sync_reactions_due\n  ON sync_reactions(partition, status, available_at_ms, created_at_ms, idempotency_key);\nCREATE INDEX IF NOT EXISTS sync_reactions_lease\n  ON sync_reactions(partition, status, lease_expires_at_ms);\nCREATE INDEX IF NOT EXISTS sync_reactions_completed\n  ON sync_reactions(partition, status, completed_at_ms, idempotency_key);\nCREATE INDEX IF NOT EXISTS sync_reactions_dead_letter\n  ON sync_reactions(partition, status, available_at_ms, idempotency_key);\nCREATE TABLE IF NOT EXISTS sync_clients(\n  partition TEXT NOT NULL, client_id TEXT NOT NULL, actor_id TEXT NOT NULL,\n  wire_version INTEGER NOT NULL DEFAULT 1,\n  cursor BIGINT NOT NULL, subscriptions JSONB NOT NULL,\n  updated_at_ms BIGINT NOT NULL,\n  last_round_at_ms BIGINT, last_error_code TEXT,\n  consecutive_failures INTEGER NOT NULL DEFAULT 0,\n  PRIMARY KEY(partition, client_id)\n);\nALTER TABLE sync_clients\n  ADD COLUMN IF NOT EXISTS wire_version INTEGER NOT NULL DEFAULT 1;\nALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS last_round_at_ms BIGINT;\nALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS last_error_code TEXT;\nALTER TABLE sync_clients\n  ADD COLUMN IF NOT EXISTS consecutive_failures INTEGER NOT NULL DEFAULT 0;\nCREATE TABLE IF NOT EXISTS sync_blob_refs(\n  partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,\n  blob_id TEXT NOT NULL,\n  PRIMARY KEY(partition, tbl, row_id, blob_id)\n);\nCREATE INDEX IF NOT EXISTS sync_blob_refs_by_blob\n  ON sync_blob_refs(partition, blob_id);\nCREATE TABLE IF NOT EXISTS sync_tombstones(\n  partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,\n  commit_seq BIGINT NOT NULL,\n  PRIMARY KEY(partition, tbl, row_id)\n);\nCREATE TABLE IF NOT EXISTS sync_backfill_checkpoints(\n  partition TEXT NOT NULL, name TEXT NOT NULL,\n  schema_version INTEGER NOT NULL,\n  state TEXT NOT NULL CHECK(state IN ('declared','backfilling','activated')),\n  watermark BIGINT NOT NULL DEFAULT 0,\n  owner_epoch BIGINT NOT NULL DEFAULT 0,\n  observed_rows BIGINT NOT NULL DEFAULT 0,\n  updated_at_ms BIGINT NOT NULL,\n  PRIMARY KEY(partition, name)\n);\nCREATE TABLE IF NOT EXISTS sync_writer_fence(\n  partition TEXT PRIMARY KEY,\n  required_writer_version INTEGER NOT NULL\n);\n";
 export declare class PostgresServerStorage implements ServerStorage {
     #private;
     constructor(exec: PgExecutor);
@@ -130,6 +130,7 @@ export declare class PostgresServerStorage implements ServerStorage {
     scanRowsByIndex(partition: string, query: IndexRowScanQuery): Promise<StoredRow[]>;
     getClientRecord(partition: string, clientId: string): Promise<ClientRecord | undefined>;
     putClientRecord(partition: string, record: ClientRecord): Promise<void>;
+    recordClientRoundFailure(partition: string, clientId: string, actorId: string, errorCode: string, atMs: number): Promise<void>;
     advanceClientCursor(partition: string, clientId: string, actorId: string, logEpoch: string, cursor: number, updatedAtMs: number): Promise<void>;
     updateClientCursor(partition: string, clientId: string, cursor: number, updatedAtMs: number): Promise<void>;
     getActiveClientCursorFloor(partition: string, cutoffMs: number): Promise<number | null>;