@syncular/server 0.39.2 → 0.40.0

package/dist/sqlite-dialect.d.ts

@@ -36,7 +36,13 @@ import type { ScopeValueHistoryQuery, StoredChange, StoredCommit, StoredPushResu
  * inverted index. Declared here so every SQLite-family storage (and D1,
  * which applies this same DDL) keeps the same plan shape as Postgres.
  */
-export declare const SQLITE_DDL = "\nCREATE TABLE IF NOT EXISTS sync_partitions(\n  partition TEXT PRIMARY KEY,\n  max_commit_seq INTEGER NOT NULL DEFAULT 0,\n  horizon_seq INTEGER NOT NULL DEFAULT 0\n);\nCREATE TABLE IF NOT EXISTS sync_partition_registry(\n  partition TEXT PRIMARY KEY,\n  log_epoch TEXT NOT NULL,\n  epoch_required INTEGER NOT NULL DEFAULT 0,\n  last_authenticated_at_ms INTEGER NOT NULL\n);\nCREATE TABLE IF NOT EXISTS sync_row_scopes(\n  partition TEXT NOT NULL, tbl TEXT NOT NULL,\n  var TEXT NOT NULL, value TEXT NOT NULL, row_id TEXT NOT NULL,\n  PRIMARY KEY(partition, tbl, var, value, row_id)\n);\nCREATE INDEX IF NOT EXISTS sync_row_scopes_by_row\n  ON sync_row_scopes(partition, tbl, row_id);\nCREATE TABLE IF NOT EXISTS sync_commits(\n  partition TEXT NOT NULL, commit_seq INTEGER NOT NULL,\n  client_id TEXT NOT NULL, client_commit_id TEXT NOT NULL,\n  actor_id TEXT NOT NULL, created_at_ms INTEGER NOT NULL,\n  writer_version INTEGER,\n  PRIMARY KEY(partition, commit_seq)\n);\nCREATE INDEX IF NOT EXISTS sync_commits_by_time\n  ON sync_commits(partition, created_at_ms);\nCREATE TABLE IF NOT EXISTS sync_changes(\n  partition TEXT NOT NULL, commit_seq INTEGER NOT NULL, idx INTEGER NOT NULL,\n  tbl TEXT NOT NULL, row_id TEXT NOT NULL, op INTEGER NOT NULL,\n  row_version INTEGER, scopes TEXT NOT NULL, payload BLOB,\n  PRIMARY KEY(partition, commit_seq, idx)\n);\nCREATE INDEX IF NOT EXISTS sync_changes_by_table\n  ON sync_changes(partition, commit_seq, tbl, idx);\nCREATE TABLE IF NOT EXISTS sync_change_scopes(\n  partition TEXT NOT NULL, tbl TEXT NOT NULL,\n  var TEXT NOT NULL, value TEXT NOT NULL, commit_seq INTEGER NOT NULL,\n  PRIMARY KEY(partition, tbl, var, value, commit_seq)\n);\nCREATE TABLE IF NOT EXISTS sync_push_results(\n  partition TEXT NOT NULL, client_id TEXT NOT NULL,\n  client_commit_id TEXT NOT NULL, result TEXT NOT NULL,\n  PRIMARY KEY(partition, client_id, client_commit_id)\n);\nCREATE TABLE IF NOT EXISTS sync_reactions(\n  partition TEXT NOT NULL, idempotency_key TEXT NOT NULL,\n  type TEXT NOT NULL, version INTEGER NOT NULL, payload TEXT NOT NULL,\n  source_client_id TEXT NOT NULL, source_client_commit_id TEXT NOT NULL,\n  source_commit_seq INTEGER NOT NULL, created_at_ms INTEGER NOT NULL,\n  available_at_ms INTEGER NOT NULL, status TEXT NOT NULL,\n  attempts INTEGER NOT NULL, max_attempts INTEGER NOT NULL,\n  lease_owner TEXT, lease_expires_at_ms INTEGER, completed_at_ms INTEGER,\n  last_failure TEXT,\n  PRIMARY KEY(partition, idempotency_key),\n  CHECK(status IN ('pending', 'leased', 'completed', 'dead-letter'))\n);\nCREATE INDEX IF NOT EXISTS sync_reactions_due\n  ON sync_reactions(partition, status, available_at_ms, created_at_ms, idempotency_key);\nCREATE INDEX IF NOT EXISTS sync_reactions_lease\n  ON sync_reactions(partition, status, lease_expires_at_ms);\nCREATE INDEX IF NOT EXISTS sync_reactions_completed\n  ON sync_reactions(partition, status, completed_at_ms, idempotency_key);\nCREATE INDEX IF NOT EXISTS sync_reactions_dead_letter\n  ON sync_reactions(partition, status, available_at_ms, idempotency_key);\nCREATE TABLE IF NOT EXISTS sync_clients(\n  partition TEXT NOT NULL, client_id TEXT NOT NULL, actor_id TEXT NOT NULL,\n  wire_version INTEGER NOT NULL DEFAULT 1,\n  cursor INTEGER NOT NULL, subscriptions TEXT NOT NULL,\n  updated_at_ms INTEGER NOT NULL,\n  PRIMARY KEY(partition, client_id)\n);\nCREATE TABLE IF NOT EXISTS sync_blob_refs(\n  partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,\n  blob_id TEXT NOT NULL,\n  PRIMARY KEY(partition, tbl, row_id, blob_id)\n);\nCREATE INDEX IF NOT EXISTS sync_blob_refs_by_blob\n  ON sync_blob_refs(partition, blob_id);\nCREATE TABLE IF NOT EXISTS sync_tombstones(\n  partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,\n  commit_seq INTEGER NOT NULL,\n  PRIMARY KEY(partition, tbl, row_id)\n);\nCREATE TABLE IF NOT EXISTS sync_backfill_checkpoints(\n  partition TEXT NOT NULL, name TEXT NOT NULL,\n  schema_version INTEGER NOT NULL,\n  state TEXT NOT NULL CHECK(state IN ('declared','backfilling','activated')),\n  watermark INTEGER NOT NULL DEFAULT 0,\n  owner_epoch INTEGER NOT NULL DEFAULT 0,\n  observed_rows INTEGER NOT NULL DEFAULT 0,\n  updated_at_ms INTEGER NOT NULL,\n  PRIMARY KEY(partition, name)\n);\nCREATE TABLE IF NOT EXISTS sync_writer_fence(\n  partition TEXT PRIMARY KEY,\n  required_writer_version INTEGER NOT NULL\n);\n";
+export declare const SQLITE_DDL = "\nCREATE TABLE IF NOT EXISTS sync_partitions(\n  partition TEXT PRIMARY KEY,\n  max_commit_seq INTEGER NOT NULL DEFAULT 0,\n  horizon_seq INTEGER NOT NULL DEFAULT 0\n);\nCREATE TABLE IF NOT EXISTS sync_partition_registry(\n  partition TEXT PRIMARY KEY,\n  log_epoch TEXT NOT NULL,\n  epoch_required INTEGER NOT NULL DEFAULT 0,\n  last_authenticated_at_ms INTEGER NOT NULL\n);\nCREATE TABLE IF NOT EXISTS sync_row_scopes(\n  partition TEXT NOT NULL, tbl TEXT NOT NULL,\n  var TEXT NOT NULL, value TEXT NOT NULL, row_id TEXT NOT NULL,\n  PRIMARY KEY(partition, tbl, var, value, row_id)\n);\nCREATE INDEX IF NOT EXISTS sync_row_scopes_by_row\n  ON sync_row_scopes(partition, tbl, row_id);\nCREATE TABLE IF NOT EXISTS sync_commits(\n  partition TEXT NOT NULL, commit_seq INTEGER NOT NULL,\n  client_id TEXT NOT NULL, client_commit_id TEXT NOT NULL,\n  actor_id TEXT NOT NULL, created_at_ms INTEGER NOT NULL,\n  writer_version INTEGER,\n  PRIMARY KEY(partition, commit_seq)\n);\nCREATE INDEX IF NOT EXISTS sync_commits_by_time\n  ON sync_commits(partition, created_at_ms);\nCREATE TABLE IF NOT EXISTS sync_changes(\n  partition TEXT NOT NULL, commit_seq INTEGER NOT NULL, idx INTEGER NOT NULL,\n  tbl TEXT NOT NULL, row_id TEXT NOT NULL, op INTEGER NOT NULL,\n  row_version INTEGER, scopes TEXT NOT NULL, payload BLOB,\n  PRIMARY KEY(partition, commit_seq, idx)\n);\nCREATE INDEX IF NOT EXISTS sync_changes_by_table\n  ON sync_changes(partition, commit_seq, tbl, idx);\nCREATE TABLE IF NOT EXISTS sync_change_scopes(\n  partition TEXT NOT NULL, tbl TEXT NOT NULL,\n  var TEXT NOT NULL, value TEXT NOT NULL, commit_seq INTEGER NOT NULL,\n  PRIMARY KEY(partition, tbl, var, value, commit_seq)\n);\nCREATE TABLE IF NOT EXISTS sync_push_results(\n  partition TEXT NOT NULL, client_id TEXT NOT NULL,\n  client_commit_id TEXT NOT NULL, result TEXT NOT NULL,\n  PRIMARY KEY(partition, client_id, client_commit_id)\n);\nCREATE TABLE IF NOT EXISTS sync_reactions(\n  partition TEXT NOT NULL, idempotency_key TEXT NOT NULL,\n  type TEXT NOT NULL, version INTEGER NOT NULL, payload TEXT NOT NULL,\n  source_client_id TEXT NOT NULL, source_client_commit_id TEXT NOT NULL,\n  source_commit_seq INTEGER NOT NULL, created_at_ms INTEGER NOT NULL,\n  available_at_ms INTEGER NOT NULL, status TEXT NOT NULL,\n  attempts INTEGER NOT NULL, max_attempts INTEGER NOT NULL,\n  lease_owner TEXT, lease_expires_at_ms INTEGER, completed_at_ms INTEGER,\n  last_failure TEXT,\n  PRIMARY KEY(partition, idempotency_key),\n  CHECK(status IN ('pending', 'leased', 'completed', 'dead-letter'))\n);\nCREATE INDEX IF NOT EXISTS sync_reactions_due\n  ON sync_reactions(partition, status, available_at_ms, created_at_ms, idempotency_key);\nCREATE INDEX IF NOT EXISTS sync_reactions_lease\n  ON sync_reactions(partition, status, lease_expires_at_ms);\nCREATE INDEX IF NOT EXISTS sync_reactions_completed\n  ON sync_reactions(partition, status, completed_at_ms, idempotency_key);\nCREATE INDEX IF NOT EXISTS sync_reactions_dead_letter\n  ON sync_reactions(partition, status, available_at_ms, idempotency_key);\nCREATE TABLE IF NOT EXISTS sync_clients(\n  partition TEXT NOT NULL, client_id TEXT NOT NULL, actor_id TEXT NOT NULL,\n  wire_version INTEGER NOT NULL DEFAULT 1,\n  cursor INTEGER NOT NULL, subscriptions TEXT NOT NULL,\n  updated_at_ms INTEGER NOT NULL,\n  last_round_at_ms INTEGER, last_error_code TEXT,\n  consecutive_failures INTEGER NOT NULL DEFAULT 0,\n  PRIMARY KEY(partition, client_id)\n);\nCREATE TABLE IF NOT EXISTS sync_blob_refs(\n  partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,\n  blob_id TEXT NOT NULL,\n  PRIMARY KEY(partition, tbl, row_id, blob_id)\n);\nCREATE INDEX IF NOT EXISTS sync_blob_refs_by_blob\n  ON sync_blob_refs(partition, blob_id);\nCREATE TABLE IF NOT EXISTS sync_tombstones(\n  partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,\n  commit_seq INTEGER NOT NULL,\n  PRIMARY KEY(partition, tbl, row_id)\n);\nCREATE TABLE IF NOT EXISTS sync_backfill_checkpoints(\n  partition TEXT NOT NULL, name TEXT NOT NULL,\n  schema_version INTEGER NOT NULL,\n  state TEXT NOT NULL CHECK(state IN ('declared','backfilling','activated')),\n  watermark INTEGER NOT NULL DEFAULT 0,\n  owner_epoch INTEGER NOT NULL DEFAULT 0,\n  observed_rows INTEGER NOT NULL DEFAULT 0,\n  updated_at_ms INTEGER NOT NULL,\n  PRIMARY KEY(partition, name)\n);\nCREATE TABLE IF NOT EXISTS sync_writer_fence(\n  partition TEXT PRIMARY KEY,\n  required_writer_version INTEGER NOT NULL\n);\n";
+/** Columns added to `sync_clients` after its first release, by column name. */
+export declare const SYNC_CLIENT_ROUND_COLUMNS: {
+    readonly last_round_at_ms: "INTEGER";
+    readonly last_error_code: "TEXT";
+    readonly consecutive_failures: "INTEGER NOT NULL DEFAULT 0";
+};
 /** Split the DDL into individual statements (D1 applies them one by one). */
 export declare function sqliteDdlStatements(): string[];
 /** §4.6.1: rowIds of one table bound to one scope value, budgeted. */

package/dist/sqlite-dialect.js

@@ -89,6 +89,8 @@ CREATE TABLE IF NOT EXISTS sync_clients(
   wire_version INTEGER NOT NULL DEFAULT 1,
   cursor INTEGER NOT NULL, subscriptions TEXT NOT NULL,
   updated_at_ms INTEGER NOT NULL,
+  last_round_at_ms INTEGER, last_error_code TEXT,
+  consecutive_failures INTEGER NOT NULL DEFAULT 0,
   PRIMARY KEY(partition, client_id)
 );
 CREATE TABLE IF NOT EXISTS sync_blob_refs(
@@ -118,6 +120,12 @@ CREATE TABLE IF NOT EXISTS sync_writer_fence(
   required_writer_version INTEGER NOT NULL
 );
 `;
+/** Columns added to `sync_clients` after its first release, by column name. */
+export const SYNC_CLIENT_ROUND_COLUMNS = {
+    last_round_at_ms: 'INTEGER',
+    last_error_code: 'TEXT',
+    consecutive_failures: 'INTEGER NOT NULL DEFAULT 0',
+};
 /** Split the DDL into individual statements (D1 applies them one by one). */
 export function sqliteDdlStatements() {
     return SQLITE_DDL.split(';')

package/dist/sqlite-storage.d.ts

@@ -57,6 +57,7 @@ export declare class SqliteServerStorage implements ServerStorage {
     scanRowsByIndex(partition: string, query: IndexRowScanQuery): Promise<StoredRow[]>;
     getClientRecord(partition: string, clientId: string): Promise<ClientRecord | undefined>;
     putClientRecord(partition: string, record: ClientRecord): Promise<void>;
+    recordClientRoundFailure(partition: string, clientId: string, actorId: string, errorCode: string, atMs: number): Promise<void>;
     advanceClientCursor(partition: string, clientId: string, actorId: string, logEpoch: string, cursor: number, updatedAtMs: number): Promise<void>;
     updateClientCursor(partition: string, clientId: string, cursor: number, updatedAtMs: number): Promise<void>;
     getActiveClientCursorFloor(partition: string, cutoffMs: number): Promise<number | null>;

package/dist/sqlite-storage.js

@@ -1,7 +1,7 @@
 import { retainRollbackFailure, rollbackAfterError } from './storage-errors.js';
 import { validateCommitPruneQuery } from './prune.js';
 import { StorageQueryError } from './storage-errors.js';
-import { PARTITION_TOUCH_INTERVAL_MS, serveGateRefusal } from './storage.js';
+import { clientRoundState, PARTITION_TOUCH_INTERVAL_MS, serveGateRefusal, } from './storage.js';
 import { serveNotReadyError } from './readiness.js';
 /**
  * SQLite server storage over the shared synchronous driver.
@@ -13,9 +13,9 @@ import { serveNotReadyError } from './readiness.js';
  */
 import { bindAuthoritativePartition, prepareAuthoritativeQuery, } from './authoritative-query.js';
 import { syncError } from './errors.js';
-import { assertPhysicalColumns, assertStoredLayouts, commitWindowPageSql, LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL, deleteRowSql, deleteSqliteRowScopesSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, scanRowPageSql, schemaDdl, SQLITE_TABLE_INDEXES_SQL, sqliteIndexSignatures, selectRowScopesSql, selectRowSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
+import { assertPhysicalColumns, assertStoredLayouts, commitWindowPageSql, LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL, deleteRowSql, deleteSqliteRowScopesSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, scanRowPageSql, schemaDdl, serverIndexDdl, SQLITE_TABLE_INDEXES_SQL, sqliteIndexSignatures, selectRowScopesSql, selectRowSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
 import { matchesEffective } from './scopes.js';
-import { collectCommitWindowPage, deserializePushResult, ERASE_SCOPE_VALUE_HISTORY_SQLITE_SQL, eraseScopeValueHistoryParams, LIST_SCOPE_VALUE_ROWS_SQLITE_SQL, placeholders, SQLITE_DDL, serializePushResult, toStoredChange, toStoredRow, } from './sqlite-dialect.js';
+import { collectCommitWindowPage, deserializePushResult, ERASE_SCOPE_VALUE_HISTORY_SQLITE_SQL, eraseScopeValueHistoryParams, LIST_SCOPE_VALUE_ROWS_SQLITE_SQL, placeholders, SQLITE_DDL, SYNC_CLIENT_ROUND_COLUMNS, serializePushResult, toStoredChange, toStoredRow, } from './sqlite-dialect.js';
 import { SqliteAdapterRequiredError, } from './sqlite-driver.js';
 import { isSqliteConstraintError, StorageConstraintError, } from './storage-errors.js';
 import { assertScopeIndexedScan, resolveIndexRowScan } from './storage-query.js';
@@ -285,6 +285,11 @@ export class SqliteServerStorage {
         if (!clientColumns.some((column) => column.name === 'wire_version')) {
             this.db.exec('ALTER TABLE sync_clients ADD COLUMN wire_version INTEGER NOT NULL DEFAULT 1');
         }
+        for (const [name, definition] of Object.entries(SYNC_CLIENT_ROUND_COLUMNS)) {
+            if (!clientColumns.some((column) => column.name === name)) {
+                this.db.exec(`ALTER TABLE sync_clients ADD COLUMN ${name} ${definition}`);
+            }
+        }
         const commitColumns = this.db
             .query('PRAGMA table_info("sync_commits")')
             .all();
@@ -357,6 +362,29 @@ END`);
                     primaryKeyPosition: column.pk,
                 })), 'sqlite');
             }
+            // Server-only indexes reconcile at every open of a database already at
+            // this version: they never reach a client, so no version bump is needed.
+            const reconcile = serverIndexDdl(schema, new Map([...schema.tables.values()].map((table) => [
+                table.name,
+                new Map(Object.entries(sqliteIndexSignatures(this.db
+                    .query(SQLITE_TABLE_INDEXES_SQL)
+                    .all(table.name)))),
+            ])));
+            if (reconcile.length > 0) {
+                await this.#serializeWrite(() => {
+                    this.db.exec('BEGIN IMMEDIATE');
+                    try {
+                        for (const statement of reconcile)
+                            this.db.exec(statement);
+                        this.db.exec('COMMIT');
+                    }
+                    catch (error) {
+                        rollbackAfterError(() => {
+                            this.db.exec('ROLLBACK');
+                        }, error);
+                    }
+                });
+            }
         }
         if (marker === null || marker.schema_version < schema.version) {
             // Introspect existing app tables, then apply the migration subset
@@ -1133,7 +1161,7 @@ END`);
     }
     async getClientRecord(partition, clientId) {
         const record = this.db
-            .query('SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=? AND client_id=?')
+            .query('SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms, last_error_code, consecutive_failures FROM sync_clients WHERE partition=? AND client_id=?')
             .get(partition, clientId);
         if (record === null)
             return undefined;
@@ -1144,12 +1172,21 @@ END`);
             cursor: record.cursor,
             updatedAtMs: record.updated_at_ms,
             subscriptions: JSON.parse(record.subscriptions),
+            ...clientRoundState(record.last_round_at_ms, record.last_error_code, record.consecutive_failures),
         };
     }
     async putClientRecord(partition, record) {
         this.db
-            .query('INSERT OR REPLACE INTO sync_clients(partition, client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms) VALUES (?,?,?,?,?,?,?)')
-            .run(partition, record.clientId, record.actorId, record.wireVersion, record.cursor, JSON.stringify(record.subscriptions), record.updatedAtMs);
+            .query('INSERT OR REPLACE INTO sync_clients(partition, client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms) VALUES (?,?,?,?,?,?,?,?)')
+            .run(partition, record.clientId, record.actorId, record.wireVersion, record.cursor, JSON.stringify(record.subscriptions), record.updatedAtMs, record.updatedAtMs);
+    }
+    async recordClientRoundFailure(partition, clientId, actorId, errorCode, atMs) {
+        this.db
+            .query(`UPDATE sync_clients
+         SET last_round_at_ms=?, last_error_code=?,
+             consecutive_failures=consecutive_failures+1
+         WHERE partition=? AND client_id=? AND actor_id=?`)
+            .run(atMs, errorCode, partition, clientId, actorId);
     }
     async advanceClientCursor(partition, clientId, actorId, logEpoch, cursor, updatedAtMs) {
         this.db
@@ -1241,7 +1278,7 @@ END`);
     // -- admin/console read surface --------------------------------------------
     async listClientRecords(partition) {
         const records = this.db
-            .query('SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=? ORDER BY updated_at_ms DESC')
+            .query('SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms, last_error_code, consecutive_failures FROM sync_clients WHERE partition=? ORDER BY updated_at_ms DESC')
             .all(partition);
         return records.map((record) => ({
             clientId: record.client_id,
@@ -1250,6 +1287,7 @@ END`);
             cursor: record.cursor,
             updatedAtMs: record.updated_at_ms,
             subscriptions: JSON.parse(record.subscriptions),
+            ...clientRoundState(record.last_round_at_ms, record.last_error_code, record.consecutive_failures),
         }));
     }
     async listCommitMetadata(partition, query) {

package/dist/storage.d.ts

@@ -235,7 +235,15 @@ export interface ClientRecord {
     readonly cursor: number;
     readonly updatedAtMs: number;
     readonly subscriptions: readonly ClientSubscription[];
-}
+    /** When this client's last sync round finished, successfully or not. */
+    readonly lastRoundAtMs?: number;
+    /** Code of the last round when it failed; absent after a successful round. */
+    readonly lastErrorCode?: string;
+    /** Rounds that failed in a row since the last successful round. */
+    readonly consecutiveFailures?: number;
+}
+/** The round-state fields of a stored `sync_clients` row. */
+export declare function clientRoundState(lastRoundAtMs: number | null, lastErrorCode: string | null, consecutiveFailures: number): Pick<ClientRecord, 'lastRoundAtMs' | 'lastErrorCode' | 'consecutiveFailures'>;
 export interface CommitWindowQuery {
     readonly table: string;
     /**
@@ -680,7 +688,18 @@ export interface ServerStorage {
      */
     scanRowsByIndex?(partition: string, query: IndexRowScanQuery): Promise<StoredRow[]>;
     getClientRecord(partition: string, clientId: string): Promise<ClientRecord | undefined>;
+    /**
+     * Write the record for a completed round. The round-state fields reset: the
+     * stored last round is a success at `updatedAtMs` with no failures.
+     */
     putClientRecord(partition: string, record: ClientRecord): Promise<void>;
+    /**
+     * Record a failed round on an existing record: set `lastRoundAtMs` and
+     * `lastErrorCode`, and increment `consecutiveFailures`. Requires a matching
+     * actor; a missing record or another actor's record is unchanged, and
+     * `updatedAtMs` (the pruning liveness signal) is never touched.
+     */
+    recordClientRoundFailure(partition: string, clientId: string, actorId: string, errorCode: string, atMs: number): Promise<void>;
     /**
      * Atomically advance an existing record's cursor and activity timestamp.
      * Preserve registration fields; require matching actor and current log epoch.

package/dist/storage.js

@@ -38,6 +38,14 @@ export function serveGateTokenChanged(before, after) {
     }
     return undefined;
 }
+/** The round-state fields of a stored `sync_clients` row. */
+export function clientRoundState(lastRoundAtMs, lastErrorCode, consecutiveFailures) {
+    return {
+        ...(lastRoundAtMs !== null ? { lastRoundAtMs } : {}),
+        ...(lastErrorCode !== null ? { lastErrorCode } : {}),
+        ...(consecutiveFailures > 0 ? { consecutiveFailures } : {}),
+    };
+}
 /** Durable partition identity refreshed after host authentication (§2.1). */
 /** Longest age of `lastAuthenticatedAtMs` that `touchPartition` leaves unwritten. */
 export const PARTITION_TOUCH_INTERVAL_MS = 60_000;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@syncular/server",
-  "version": "0.39.2",
+  "version": "0.40.0",
   "description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
   "license": "Apache-2.0",
   "author": "Benjamin Kniffler",
@@ -68,7 +68,7 @@
     "verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
   },
   "dependencies": {
-    "@syncular/core": "0.39.2"
+    "@syncular/core": "0.40.0"
   },
   "devDependencies": {
     "@electric-sql/pglite": "^0.5.4"

package/src/admin.ts

@@ -90,6 +90,12 @@ export interface AdminClient {
   readonly connected?: boolean;
   /** When the client's oldest open socket opened. */
   readonly connectedSinceMs?: number;
+  /** When its last sync round finished, successfully or not. */
+  readonly lastRoundAtMs?: number;
+  /** Catalog code of its last round; present only while that round failed. */
+  readonly lastErrorCode?: string;
+  /** Rounds failed in a row since its last success; absent at zero. */
+  readonly consecutiveFailures?: number;
 }
 
 export interface AdminListClientsOptions {
@@ -1633,6 +1639,15 @@ export class SyncularAdmin {
           ? { connected: connectedSinceMs !== undefined }
           : {}),
         ...(connectedSinceMs !== undefined ? { connectedSinceMs } : {}),
+        ...(record.lastRoundAtMs !== undefined
+          ? { lastRoundAtMs: record.lastRoundAtMs }
+          : {}),
+        ...(record.lastErrorCode !== undefined
+          ? { lastErrorCode: record.lastErrorCode }
+          : {}),
+        ...(record.consecutiveFailures !== undefined
+          ? { consecutiveFailures: record.consecutiveFailures }
+          : {}),
       };
     };
   }

package/src/d1-storage.ts

@@ -1,7 +1,7 @@
 import { validateCommitPruneQuery } from './prune';
 import { StorageQueryError } from './storage-errors';
 import type { CommitPruneQuery, CommitPruneResult } from './storage';
-import { PARTITION_TOUCH_INTERVAL_MS } from './storage';
+import { clientRoundState, PARTITION_TOUCH_INTERVAL_MS } from './storage';
 /**
  * Cloudflare D1 server storage for Workers deployments.
  *
@@ -78,6 +78,7 @@ import {
   type StoredColumnLayout,
   scanRowPageSql,
   schemaDdl,
+  serverIndexDdl,
   SQLITE_TABLE_INDEXES_SQL,
   type SqliteTableIndexRow,
   sqliteIndexSignatures,
@@ -104,6 +105,7 @@ import {
   type SqliteRowRecord,
   serializePushResult,
   sqliteDdlStatements,
+  SYNC_CLIENT_ROUND_COLUMNS,
   toStoredChange,
   toStoredRow,
 } from './sqlite-dialect';
@@ -999,6 +1001,15 @@ export class D1ServerStorage implements ServerStorage {
         'ALTER TABLE sync_clients ADD COLUMN wire_version INTEGER NOT NULL DEFAULT 1;',
       );
     }
+    for (const [name, definition] of Object.entries(
+      SYNC_CLIENT_ROUND_COLUMNS,
+    )) {
+      if (!results.some((column) => column.name === name)) {
+        await this.#db.exec(
+          `ALTER TABLE sync_clients ADD COLUMN ${name} ${definition};`,
+        );
+      }
+    }
   }
 
   /** Resolve a table's compiled schema; row operations require `ensureSchema`. */
@@ -1129,6 +1140,44 @@ export class D1ServerStorage implements ServerStorage {
           'sqlite',
         );
       }
+      // A database from before the round-state columns gains them in place.
+      statementsExecuted++;
+      const clientColumns = await this.#db
+        .prepare('PRAGMA table_info("sync_clients")')
+        .all<{ name: string }>();
+      const missingRoundColumns = Object.entries(
+        SYNC_CLIENT_ROUND_COLUMNS,
+      ).filter(
+        ([name]) =>
+          !clientColumns.results.some((column) => column.name === name),
+      );
+      if (missingRoundColumns.length > 0) {
+        await batch(
+          missingRoundColumns.map(([name, definition]) =>
+            this.#db.prepare(
+              `ALTER TABLE sync_clients ADD COLUMN ${name} ${definition}`,
+            ),
+          ),
+        );
+      }
+      // Server-only indexes reconcile at every open of a database already at
+      // this version: they never reach a client, so no version bump is needed.
+      const indexes = new Map<string, ReadonlyMap<string, string>>();
+      for (const table of tables) {
+        statementsExecuted++;
+        const { results } = await this.#db
+          .prepare(SQLITE_TABLE_INDEXES_SQL)
+          .bind(table.name)
+          .all<SqliteTableIndexRow>();
+        indexes.set(
+          table.name,
+          new Map(Object.entries(sqliteIndexSignatures(results))),
+        );
+      }
+      const reconcile = serverIndexDdl(schema, indexes);
+      if (reconcile.length > 0) {
+        await batch(reconcile.map((sql) => this.#db.prepare(sql)));
+      }
       this.#tables = schema.tables;
       this.#schemaVersion = schema.version;
       return { complete: true, statementsExecuted };
@@ -1248,19 +1297,20 @@ export class D1ServerStorage implements ServerStorage {
         const columns = await this.#db
           .prepare('PRAGMA table_info("sync_clients")')
           .all<{ name: string }>();
-        const needsWireVersion = !columns.results.some(
-          (column) => column.name === 'wire_version',
+        const missingColumns = Object.entries({
+          wire_version: 'INTEGER NOT NULL DEFAULT 1',
+          ...SYNC_CLIENT_ROUND_COLUMNS,
+        }).filter(
+          ([name]) => !columns.results.some((column) => column.name === name),
         );
         if (
           !(await save(
             { ...state, phase: 'prepare', offset: 0 },
-            needsWireVersion
-              ? [
-                  this.#db.prepare(
-                    'ALTER TABLE sync_clients ADD COLUMN wire_version INTEGER NOT NULL DEFAULT 1',
-                  ),
-                ]
-              : [],
+            missingColumns.map(([name, definition]) =>
+              this.#db.prepare(
+                `ALTER TABLE sync_clients ADD COLUMN ${name} ${definition}`,
+              ),
+            ),
           ))
         )
           break;
@@ -2333,7 +2383,7 @@ export class D1ServerStorage implements ServerStorage {
   ): Promise<ClientRecord | undefined> {
     const record = await this.#db
       .prepare(
-        'SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=? AND client_id=?',
+        'SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms, last_error_code, consecutive_failures FROM sync_clients WHERE partition=? AND client_id=?',
       )
       .bind(partition, clientId)
       .first<{
@@ -2343,6 +2393,9 @@ export class D1ServerStorage implements ServerStorage {
         cursor: number;
         subscriptions: string;
         updated_at_ms: number;
+        last_round_at_ms: number | null;
+        last_error_code: string | null;
+        consecutive_failures: number;
       }>();
     if (record === null) return undefined;
     return {
@@ -2352,6 +2405,11 @@ export class D1ServerStorage implements ServerStorage {
       cursor: record.cursor,
       updatedAtMs: record.updated_at_ms,
       subscriptions: JSON.parse(record.subscriptions) as ClientSubscription[],
+      ...clientRoundState(
+        record.last_round_at_ms,
+        record.last_error_code,
+        record.consecutive_failures,
+      ),
     };
   }
 
@@ -2364,13 +2422,16 @@ export class D1ServerStorage implements ServerStorage {
     // record; a delete-and-insert would drop and recreate the row instead.
     await this.#db
       .prepare(
-        `INSERT INTO sync_clients(partition, client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms) VALUES (?,?,?,?,?,?,?)
+        `INSERT INTO sync_clients(partition, client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms) VALUES (?,?,?,?,?,?,?,?)
          ON CONFLICT(partition, client_id) DO UPDATE SET
            actor_id=excluded.actor_id,
            wire_version=excluded.wire_version,
            cursor=excluded.cursor,
            subscriptions=excluded.subscriptions,
-           updated_at_ms=excluded.updated_at_ms`,
+           updated_at_ms=excluded.updated_at_ms,
+           last_round_at_ms=excluded.last_round_at_ms,
+           last_error_code=NULL,
+           consecutive_failures=0`,
       )
       .bind(
         partition,
@@ -2380,7 +2441,26 @@ export class D1ServerStorage implements ServerStorage {
         record.cursor,
         JSON.stringify(record.subscriptions),
         record.updatedAtMs,
+        record.updatedAtMs,
+      )
+      .run();
+  }
+
+  async recordClientRoundFailure(
+    partition: string,
+    clientId: string,
+    actorId: string,
+    errorCode: string,
+    atMs: number,
+  ): Promise<void> {
+    await this.#db
+      .prepare(
+        `UPDATE sync_clients
+         SET last_round_at_ms=?, last_error_code=?,
+             consecutive_failures=consecutive_failures+1
+         WHERE partition=? AND client_id=? AND actor_id=?`,
       )
+      .bind(atMs, errorCode, partition, clientId, actorId)
       .run();
   }
 
@@ -2537,7 +2617,7 @@ export class D1ServerStorage implements ServerStorage {
   async listClientRecords(partition: string): Promise<ClientRecord[]> {
     const { results } = await this.#db
       .prepare(
-        'SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=? ORDER BY updated_at_ms DESC',
+        'SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms, last_error_code, consecutive_failures FROM sync_clients WHERE partition=? ORDER BY updated_at_ms DESC',
       )
       .bind(partition)
       .all<{
@@ -2547,6 +2627,9 @@ export class D1ServerStorage implements ServerStorage {
         cursor: number;
         subscriptions: string;
         updated_at_ms: number;
+        last_round_at_ms: number | null;
+        last_error_code: string | null;
+        consecutive_failures: number;
       }>();
     return results.map((record) => ({
       clientId: record.client_id,
@@ -2555,6 +2638,11 @@ export class D1ServerStorage implements ServerStorage {
       cursor: record.cursor,
       updatedAtMs: record.updated_at_ms,
       subscriptions: JSON.parse(record.subscriptions) as ClientSubscription[],
+      ...clientRoundState(
+        record.last_round_at_ms,
+        record.last_error_code,
+        record.consecutive_failures,
+      ),
     }));
   }
 

package/src/handler.ts

@@ -30,7 +30,7 @@ import {
   RESOLVER_OUTAGE,
   touchAuthenticatedPartition,
 } from './context';
-import { SyncError, syncError } from './errors';
+import { reportError, SyncError, syncError } from './errors';
 import {
   emitEvent,
   type PullSubscriptionSummary,
@@ -734,6 +734,7 @@ async function* streamResponse(
         report.outcome = 'error';
         report.errorCode = error.code;
       }
+      await recordRoundFailure(ctx, plan.header.clientId, error);
       // §1.6: in-band ERROR, then END, nothing else.
       yield encodeResponseFrame(
         {
@@ -765,7 +766,14 @@ export async function createSyncResponseStream(
   ctx: SyncRequestContext,
 ): Promise<AsyncIterable<Uint8Array>> {
   const sink = ctx.events;
-  if (sink === undefined) return createStreamCore(bytes, ctx, undefined);
+  if (sink === undefined) {
+    try {
+      return await createStreamCore(bytes, ctx, undefined);
+    } catch (error) {
+      await recordRoundFailure(ctx, requestClientId(bytes), error);
+      throw error;
+    }
+  }
   // Instrumented path: measure with the ctx clock, count bytes at the
   // seam, emit exactly one `request.handled` per request. Every event of
   // this request carries one `requestId`.
@@ -779,6 +787,7 @@ export async function createSyncResponseStream(
   try {
     return await createStreamCore(bytes, requestCtx, events, startedAtMs);
   } catch (error) {
+    const clientId = requestClientId(bytes);
     emitEvent(events, {
       type: 'request.handled',
       kind: 'sync',
@@ -789,11 +798,64 @@ export async function createSyncResponseStream(
       bytesIn: bytes.length,
       bytesOut: 0,
       outcome: 'rejected',
+      ...(clientId !== undefined ? { clientId } : {}),
       errorCode: error instanceof SyncError ? error.code : 'internal',
       pushCommits: 0,
       pulled: false,
       subscriptions: 0,
     });
+    await recordRoundFailure(ctx, clientId, error);
+    throw error;
+  }
+}
+
+/** The `REQ_HEADER` clientId of a request that failed before planning. */
+function requestClientId(bytes: Uint8Array): string | undefined {
+  try {
+    const message = decodeMessage(bytes);
+    const header = message.frames[0];
+    return message.msgKind === 'request' && header?.type === 'REQ_HEADER'
+      ? header.clientId
+      : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+/**
+ * Record a failed round on the client's stored record so an operator can see
+ * a stuck client and why. A storage failure here is reported through
+ * `onError` and never replaces the round's own outcome.
+ */
+async function recordRoundFailure(
+  ctx: SyncRequestContext,
+  clientId: string | undefined,
+  error: unknown,
+): Promise<void> {
+  if (clientId === undefined) return;
+  try {
+    await ctx.storage.recordClientRoundFailure(
+      ctx.partition,
+      clientId,
+      ctx.actorId,
+      error instanceof SyncError ? error.code : 'sync.internal_error',
+      clockOf(ctx)(),
+    );
+  } catch (failure) {
+    reportError(failure, ctx.onError, 'sync');
+  }
+}
+
+/** Record a failure thrown while the response streams (after planning). */
+async function* recordedStream(
+  stream: AsyncIterable<Uint8Array>,
+  ctx: SyncRequestContext,
+  clientId: string,
+): AsyncGenerator<Uint8Array> {
+  try {
+    yield* stream;
+  } catch (error) {
+    await recordRoundFailure(ctx, clientId, error);
     throw error;
   }
 }
@@ -883,15 +945,21 @@ async function createStreamCore(
     if (changed !== undefined) throw serveNotReadyError(changed);
     for (const chunk of buffered) yield chunk;
   };
-  if (events === undefined) return verified();
-  return instrumentedStream(
-    verified(),
+  if (events === undefined) {
+    return recordedStream(verified(), ctx, plan.header.clientId);
+  }
+  return recordedStream(
+    instrumentedStream(
+      verified(),
+      ctx,
+      events,
+      plan,
+      report,
+      bytes.length,
+      startedAtMs,
+    ),
     ctx,
-    events,
-    plan,
-    report,
-    bytes.length,
-    startedAtMs,
+    plan.header.clientId,
   );
 }