@syncular/server 0.39.1 → 0.40.0

package/dist/postgres-storage.js

@@ -1,11 +1,11 @@
 import { validateCommitPruneQuery } from './prune.js';
 import { StorageQueryError } from './storage-errors.js';
-import { PARTITION_TOUCH_INTERVAL_MS, serveGateRefusal } from './storage.js';
+import { clientRoundState, PARTITION_TOUCH_INTERVAL_MS, serveGateRefusal, } from './storage.js';
 import { serveNotReadyError } from './readiness.js';
 import { bindAuthoritativePartition, postgresPlaceholders, prepareAuthoritativeQuery, } from './authoritative-query.js';
 import { syncError } from './errors.js';
 import { asBytes, asNumber, } from './pg-executor.js';
-import { assertPhysicalColumns, assertStoredLayouts, COMMIT_WINDOW_BATCH_SQL, LATEST_SCOPED_CHANGE_BATCH_SQL, deleteRowSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, tableColumnNames, retiredTableNames, rewritePlan, columnSqlType, rewriteValues, SCHEMA_META_DDL_POSTGRES, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, scanRowBatchSql, scanRowPageSql, indexSignature, schemaDdl, selectRowScopesSql, selectRowSql, prefetchRowsSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
+import { assertPhysicalColumns, assertStoredLayouts, COMMIT_WINDOW_BATCH_SQL, LATEST_SCOPED_CHANGE_BATCH_SQL, deleteRowSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, tableColumnNames, retiredTableNames, rewritePlan, columnSqlType, rewriteValues, SCHEMA_META_DDL_POSTGRES, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, scanRowBatchSql, scanRowPageSql, indexSignature, schemaDdl, serverIndexDdl, selectRowScopesSql, selectRowSql, prefetchRowsSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
 import { matchesEffective, sha256Hex } from './scopes.js';
 import { isPostgresConstraintError, StorageConstraintError, } from './storage-errors.js';
 import { assertScopeIndexedScan, resolveIndexRowScan } from './storage-query.js';
@@ -109,10 +109,16 @@ CREATE TABLE IF NOT EXISTS sync_clients(
   wire_version INTEGER NOT NULL DEFAULT 1,
   cursor BIGINT NOT NULL, subscriptions JSONB NOT NULL,
   updated_at_ms BIGINT NOT NULL,
+  last_round_at_ms BIGINT, last_error_code TEXT,
+  consecutive_failures INTEGER NOT NULL DEFAULT 0,
   PRIMARY KEY(partition, client_id)
 );
 ALTER TABLE sync_clients
   ADD COLUMN IF NOT EXISTS wire_version INTEGER NOT NULL DEFAULT 1;
+ALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS last_round_at_ms BIGINT;
+ALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS last_error_code TEXT;
+ALTER TABLE sync_clients
+  ADD COLUMN IF NOT EXISTS consecutive_failures INTEGER NOT NULL DEFAULT 0;
 CREATE TABLE IF NOT EXISTS sync_blob_refs(
   partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,
   blob_id TEXT NOT NULL,
@@ -169,6 +175,56 @@ async function installCheckpointOn(client, partition, name, schemaVersion, nowMs
          sync_writer_fence.required_writer_version,
          EXCLUDED.required_writer_version)`, [partition, schemaVersion]);
 }
+/**
+ * A scalar subquery over the table names in the JSON array bound to
+ * `namesParam`: the free-standing indexes of each table as a JSON array of
+ * `{tbl, name, unique, partial, columns}`. Parity with the SQLite/D1
+ * `origin === 'c'` filter: constraint-owned indexes (PRIMARY KEY, UNIQUE
+ * constraints) can only be removed through their constraint, so DROP INDEX on
+ * them would abort the transaction. Tables resolve through `to_regclass`,
+ * which follows the session's `search_path` like the row queries do.
+ */
+function indexesSubquery(namesParam) {
+    return `(SELECT coalesce(json_agg(json_build_object(
+                  'tbl', tables.name,
+                  'name', index_class.relname,
+                  'unique', index_meta.indisunique,
+                  'partial', index_meta.indpred IS NOT NULL,
+                  'columns', ARRAY(
+                    SELECT attribute.attname::text
+                      FROM unnest(index_meta.indkey::int2[])
+                           WITH ORDINALITY AS key(attnum, ord)
+                      LEFT JOIN pg_catalog.pg_attribute AS attribute
+                        ON attribute.attrelid = index_meta.indrelid
+                       AND attribute.attnum = key.attnum
+                     WHERE key.ord <= index_meta.indnkeyatts
+                     ORDER BY key.ord))), '[]'::json)::text
+           FROM jsonb_array_elements_text(${namesParam}::text::jsonb) AS tables(name)
+           JOIN pg_catalog.pg_index AS index_meta
+             ON index_meta.indrelid = to_regclass(tables.name)
+           JOIN pg_catalog.pg_class AS index_class
+             ON index_class.oid = index_meta.indexrelid
+          WHERE NOT index_meta.indisprimary
+            AND NOT EXISTS (
+              SELECT 1
+                FROM pg_catalog.pg_constraint AS owning_constraint
+               WHERE owning_constraint.conindid = index_meta.indexrelid
+            ))`;
+}
+/** Table name to index name to {@link indexSignature}, from {@link indexesSubquery} JSON. */
+function parseIndexSignatures(json) {
+    const byTable = new Map();
+    for (const index of JSON.parse(json)) {
+        const indexes = byTable.get(index.tbl) ?? new Map();
+        indexes.set(index.name, indexSignature(index.columns, index.unique, index.partial));
+        byTable.set(index.tbl, indexes);
+    }
+    return byTable;
+}
+async function readIndexSignatures(client, tableNames) {
+    const { rows } = await client.query(`SELECT ${indexesSubquery('$1')} AS indexes`, [JSON.stringify(tableNames)]);
+    return parseIndexSignatures(rows[0]?.indexes ?? '[]');
+}
 /** RFC 0007 serve gate on either the pool or a transaction's client. */
 async function readServeGateOn(client, partition, runningSchemaVersion) {
     // One statement: every request reads the gate twice (entry and verify),
@@ -1069,7 +1125,8 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
                  JOIN pg_attribute attribute
                    ON attribute.attrelid = to_regclass(tables.name)
                 WHERE attribute.attnum > 0
-                  AND NOT attribute.attisdropped) AS columns
+                  AND NOT attribute.attisdropped) AS columns,
+              ${indexesSubquery('$2')} AS indexes
          FROM (SELECT 1) AS one
          LEFT JOIN sync_schema_meta meta ON meta.id=1`, [schema.version, JSON.stringify([...schema.tables.keys()])]);
         const marker = probe[0];
@@ -1105,6 +1162,19 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
                     primaryKeyPosition: column.pk ?? 0,
                 })), 'postgres');
             }
+            // Server-only indexes reconcile at every open of a database already at
+            // this version: they never reach a client, so no version bump is needed.
+            // The unlocked read finds the common no-op case; the diff is recomputed
+            // under the migration barrier before any DDL runs.
+            const reconcile = serverIndexDdl(schema, parseIndexSignatures(marker.indexes));
+            if (reconcile.length > 0) {
+                await this.#exec.transaction(async (client) => {
+                    await client.query('LOCK TABLE sync_partitions IN EXCLUSIVE MODE');
+                    for (const statement of serverIndexDdl(schema, await readIndexSignatures(client, [...schema.tables.keys()]))) {
+                        await client.query(statement);
+                    }
+                });
+            }
         }
         if (stored === undefined || stored < schema.version) {
             // Introspect existing app tables, apply the migration subset
@@ -1145,48 +1215,15 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
                 const retiredTables = retiredTableNames(schema, layouts);
                 const existing = new Map();
                 const existingIndexes = new Map();
+                const tableIndexes = await readIndexSignatures(client, [
+                    ...schema.tables.keys(),
+                ]);
                 for (const table of schema.tables.values()) {
                     const { rows } = await client.query(`SELECT column_name FROM information_schema.columns
              WHERE table_schema = current_schema() AND table_name = $1`, [table.name]);
                     if (rows.length > 0) {
                         existing.set(table.name, new Set(rows.map((r) => r.column_name)));
-                        // Parity with the SQLite/D1 `origin === 'c'` filter: only
-                        // free-standing indexes enter the rebuild set. Constraint-owned
-                        // indexes (PRIMARY KEY, UNIQUE constraints) can only be removed
-                        // through their constraint, so DROP INDEX on them would abort
-                        // the migration transaction.
-                        const indexes = await client.query(`SELECT index_class.relname AS index_name,
-                      index_meta.indisunique AS is_unique,
-                      index_meta.indpred IS NOT NULL AS is_partial,
-                      ARRAY(
-                        SELECT attribute.attname::text
-                          FROM unnest(index_meta.indkey::int2[])
-                               WITH ORDINALITY AS key(attnum, ord)
-                          LEFT JOIN pg_catalog.pg_attribute AS attribute
-                            ON attribute.attrelid = table_class.oid
-                           AND attribute.attnum = key.attnum
-                         WHERE key.ord <= index_meta.indnkeyatts
-                         ORDER BY key.ord
-                      ) AS columns
-                 FROM pg_catalog.pg_class AS table_class
-                 JOIN pg_catalog.pg_namespace AS namespace
-                   ON namespace.oid = table_class.relnamespace
-                 JOIN pg_catalog.pg_index AS index_meta
-                   ON index_meta.indrelid = table_class.oid
-                 JOIN pg_catalog.pg_class AS index_class
-                   ON index_class.oid = index_meta.indexrelid
-                WHERE namespace.nspname = current_schema()
-                  AND table_class.relname = $1
-                  AND NOT index_meta.indisprimary
-                  AND NOT EXISTS (
-                    SELECT 1
-                      FROM pg_catalog.pg_constraint AS owning_constraint
-                     WHERE owning_constraint.conindid = index_meta.indexrelid
-                  )`, [table.name]);
-                        existingIndexes.set(table.name, new Map(indexes.rows.map((index) => [
-                            index.index_name,
-                            indexSignature(index.columns, index.is_unique, index.is_partial),
-                        ])));
+                        existingIndexes.set(table.name, tableIndexes.get(table.name) ?? new Map());
                     }
                 }
                 if (retiredTables.length > 0) {
@@ -1774,7 +1811,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
         return scanRowsByIndexOn(this.#exec, this.table(query.table), partition, query);
     }
     async getClientRecord(partition, clientId) {
-        const { rows } = await this.#exec.query('SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=$1 AND client_id=$2', [partition, clientId]);
+        const { rows } = await this.#exec.query('SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms, last_error_code, consecutive_failures FROM sync_clients WHERE partition=$1 AND client_id=$2', [partition, clientId]);
         const record = rows[0];
         if (record === undefined)
             return undefined;
@@ -1785,16 +1822,21 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
             cursor: asNumber(record.cursor),
             updatedAtMs: asNumber(record.updated_at_ms),
             subscriptions: asJson(record.subscriptions),
+            ...clientRoundState(record.last_round_at_ms === null
+                ? null
+                : asNumber(record.last_round_at_ms), record.last_error_code, asNumber(record.consecutive_failures)),
         };
     }
     async putClientRecord(partition, record) {
-        await this.#exec.query(`INSERT INTO sync_clients(partition, client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms)
-       VALUES ($1,$2,$3,$4,$5,$6,$7)
+        await this.#exec.query(`INSERT INTO sync_clients(partition, client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms)
+       VALUES ($1,$2,$3,$4,$5,$6,$7,$7)
        ON CONFLICT (partition, client_id) DO UPDATE
          SET actor_id=EXCLUDED.actor_id, wire_version=EXCLUDED.wire_version,
              cursor=EXCLUDED.cursor,
              subscriptions=EXCLUDED.subscriptions,
-             updated_at_ms=EXCLUDED.updated_at_ms`, [
+             updated_at_ms=EXCLUDED.updated_at_ms,
+             last_round_at_ms=EXCLUDED.last_round_at_ms,
+             last_error_code=NULL, consecutive_failures=0`, [
             partition,
             record.clientId,
             record.actorId,
@@ -1804,6 +1846,12 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
             record.updatedAtMs,
         ]);
     }
+    async recordClientRoundFailure(partition, clientId, actorId, errorCode, atMs) {
+        await this.#exec.query(`UPDATE sync_clients
+         SET last_round_at_ms=$1, last_error_code=$2,
+             consecutive_failures=consecutive_failures+1
+       WHERE partition=$3 AND client_id=$4 AND actor_id=$5`, [atMs, errorCode, partition, clientId, actorId]);
+    }
     async advanceClientCursor(partition, clientId, actorId, logEpoch, cursor, updatedAtMs) {
         await this.#exec.query(`UPDATE sync_clients
          SET cursor=GREATEST(cursor, $1), updated_at_ms=GREATEST(updated_at_ms, $2)
@@ -1895,7 +1943,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
     }
     // -- admin/console read surface --------------------------------------------
     async listClientRecords(partition) {
-        const { rows } = await this.#exec.query('SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=$1 ORDER BY updated_at_ms DESC', [partition]);
+        const { rows } = await this.#exec.query('SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms, last_error_code, consecutive_failures FROM sync_clients WHERE partition=$1 ORDER BY updated_at_ms DESC', [partition]);
         return rows.map((r) => ({
             clientId: r.client_id,
             actorId: r.actor_id,
@@ -1905,6 +1953,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
             subscriptions: (typeof r.subscriptions === 'string'
                 ? JSON.parse(r.subscriptions)
                 : r.subscriptions),
+            ...clientRoundState(r.last_round_at_ms === null ? null : asNumber(r.last_round_at_ms), r.last_error_code, asNumber(r.consecutive_failures)),
         }));
     }
     async listCommitMetadata(partition, query) {

package/dist/realtime.d.ts

@@ -257,6 +257,8 @@ export declare class RealtimeHub {
     refreshScopes(partition: string, actorId?: string): Promise<void>;
     /** Broadcast a wake-up (host-initiated resync, schema rollover, §8.3). */
     wake(partition: string, reason: WakeReason): void;
+    /** Wake every session on every partition (fanout listener gap, §8.3). */
+    wakeAll(reason: WakeReason): void;
 }
 export declare function createRealtimeHub(config: RealtimeHubConfig): RealtimeHub;
 export {};

package/dist/realtime.js

@@ -1012,6 +1012,11 @@ export class RealtimeHub {
             session.sendWake(reason);
         }
     }
+    /** Wake every session on every partition (fanout listener gap, §8.3). */
+    wakeAll(reason) {
+        for (const session of this.#sessions)
+            session.sendWake(reason);
+    }
 }
 export function createRealtimeHub(config) {
     return new RealtimeHub(config);

package/dist/relational-rows.d.ts

@@ -404,3 +404,12 @@ export declare function rewritePlan(table: CompiledTable, oldLayout: readonly St
  * names and survive every bump.
  */
 export declare function schemaDdl(schema: CompiledSchema, existingColumnsByTable: ReadonlyMap<string, ReadonlySet<string>>, dialect: RelationalDialect, existingIndexesByTable?: ReadonlyMap<string, ReadonlyMap<string, string>>): string[];
+/**
+ * DDL that reconciles the server-only indexes of a database already at
+ * `schema.version`, so declaring or changing a `serverIndexes` entry needs no
+ * schema-version bump (server indexes never reach a client). It creates
+ * missing server indexes and drops every Syncular-owned index the schema no
+ * longer names. Declared `indexes` belong to the version bump and are never
+ * created or rebuilt here.
+ */
+export declare function serverIndexDdl(schema: CompiledSchema, existingIndexesByTable: ReadonlyMap<string, ReadonlyMap<string, string>>): string[];

package/dist/relational-rows.js

@@ -969,3 +969,33 @@ export function schemaDdl(schema, existingColumnsByTable, dialect, existingIndex
     // receiving table re-creates it.
     return [...drops, ...creates];
 }
+/**
+ * DDL that reconciles the server-only indexes of a database already at
+ * `schema.version`, so declaring or changing a `serverIndexes` entry needs no
+ * schema-version bump (server indexes never reach a client). It creates
+ * missing server indexes and drops every Syncular-owned index the schema no
+ * longer names. Declared `indexes` belong to the version bump and are never
+ * created or rebuilt here.
+ */
+export function serverIndexDdl(schema, existingIndexesByTable) {
+    const drops = [];
+    const creates = [];
+    for (const table of schema.tables.values()) {
+        const existing = existingIndexesByTable.get(table.name);
+        if (existing === undefined)
+            continue;
+        const desired = desiredIndexes(table);
+        for (const indexName of existing.keys()) {
+            if (indexName.startsWith(SYNC_INDEX_PREFIX) &&
+                !desired.some((index) => index.name === indexName)) {
+                drops.push(dropIndexDdl(indexName));
+            }
+        }
+        // `desiredIndexes` lists the declared indexes first, then the server ones.
+        for (const index of desired.slice(table.indexes.length)) {
+            if (!existing.has(index.name))
+                creates.push(index.ddl);
+        }
+    }
+    return [...drops, ...creates];
+}

package/dist/sqlite-dialect.d.ts

@@ -36,7 +36,13 @@ import type { ScopeValueHistoryQuery, StoredChange, StoredCommit, StoredPushResu
  * inverted index. Declared here so every SQLite-family storage (and D1,
  * which applies this same DDL) keeps the same plan shape as Postgres.
  */
-export declare const SQLITE_DDL = "\nCREATE TABLE IF NOT EXISTS sync_partitions(\n  partition TEXT PRIMARY KEY,\n  max_commit_seq INTEGER NOT NULL DEFAULT 0,\n  horizon_seq INTEGER NOT NULL DEFAULT 0\n);\nCREATE TABLE IF NOT EXISTS sync_partition_registry(\n  partition TEXT PRIMARY KEY,\n  log_epoch TEXT NOT NULL,\n  epoch_required INTEGER NOT NULL DEFAULT 0,\n  last_authenticated_at_ms INTEGER NOT NULL\n);\nCREATE TABLE IF NOT EXISTS sync_row_scopes(\n  partition TEXT NOT NULL, tbl TEXT NOT NULL,\n  var TEXT NOT NULL, value TEXT NOT NULL, row_id TEXT NOT NULL,\n  PRIMARY KEY(partition, tbl, var, value, row_id)\n);\nCREATE INDEX IF NOT EXISTS sync_row_scopes_by_row\n  ON sync_row_scopes(partition, tbl, row_id);\nCREATE TABLE IF NOT EXISTS sync_commits(\n  partition TEXT NOT NULL, commit_seq INTEGER NOT NULL,\n  client_id TEXT NOT NULL, client_commit_id TEXT NOT NULL,\n  actor_id TEXT NOT NULL, created_at_ms INTEGER NOT NULL,\n  writer_version INTEGER,\n  PRIMARY KEY(partition, commit_seq)\n);\nCREATE INDEX IF NOT EXISTS sync_commits_by_time\n  ON sync_commits(partition, created_at_ms);\nCREATE TABLE IF NOT EXISTS sync_changes(\n  partition TEXT NOT NULL, commit_seq INTEGER NOT NULL, idx INTEGER NOT NULL,\n  tbl TEXT NOT NULL, row_id TEXT NOT NULL, op INTEGER NOT NULL,\n  row_version INTEGER, scopes TEXT NOT NULL, payload BLOB,\n  PRIMARY KEY(partition, commit_seq, idx)\n);\nCREATE INDEX IF NOT EXISTS sync_changes_by_table\n  ON sync_changes(partition, commit_seq, tbl, idx);\nCREATE TABLE IF NOT EXISTS sync_change_scopes(\n  partition TEXT NOT NULL, tbl TEXT NOT NULL,\n  var TEXT NOT NULL, value TEXT NOT NULL, commit_seq INTEGER NOT NULL,\n  PRIMARY KEY(partition, tbl, var, value, commit_seq)\n);\nCREATE TABLE IF NOT EXISTS sync_push_results(\n  partition TEXT NOT NULL, client_id TEXT NOT NULL,\n  client_commit_id TEXT NOT NULL, result TEXT NOT NULL,\n  PRIMARY KEY(partition, client_id, client_commit_id)\n);\nCREATE TABLE IF NOT EXISTS sync_reactions(\n  partition TEXT NOT NULL, idempotency_key TEXT NOT NULL,\n  type TEXT NOT NULL, version INTEGER NOT NULL, payload TEXT NOT NULL,\n  source_client_id TEXT NOT NULL, source_client_commit_id TEXT NOT NULL,\n  source_commit_seq INTEGER NOT NULL, created_at_ms INTEGER NOT NULL,\n  available_at_ms INTEGER NOT NULL, status TEXT NOT NULL,\n  attempts INTEGER NOT NULL, max_attempts INTEGER NOT NULL,\n  lease_owner TEXT, lease_expires_at_ms INTEGER, completed_at_ms INTEGER,\n  last_failure TEXT,\n  PRIMARY KEY(partition, idempotency_key),\n  CHECK(status IN ('pending', 'leased', 'completed', 'dead-letter'))\n);\nCREATE INDEX IF NOT EXISTS sync_reactions_due\n  ON sync_reactions(partition, status, available_at_ms, created_at_ms, idempotency_key);\nCREATE INDEX IF NOT EXISTS sync_reactions_lease\n  ON sync_reactions(partition, status, lease_expires_at_ms);\nCREATE INDEX IF NOT EXISTS sync_reactions_completed\n  ON sync_reactions(partition, status, completed_at_ms, idempotency_key);\nCREATE INDEX IF NOT EXISTS sync_reactions_dead_letter\n  ON sync_reactions(partition, status, available_at_ms, idempotency_key);\nCREATE TABLE IF NOT EXISTS sync_clients(\n  partition TEXT NOT NULL, client_id TEXT NOT NULL, actor_id TEXT NOT NULL,\n  wire_version INTEGER NOT NULL DEFAULT 1,\n  cursor INTEGER NOT NULL, subscriptions TEXT NOT NULL,\n  updated_at_ms INTEGER NOT NULL,\n  PRIMARY KEY(partition, client_id)\n);\nCREATE TABLE IF NOT EXISTS sync_blob_refs(\n  partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,\n  blob_id TEXT NOT NULL,\n  PRIMARY KEY(partition, tbl, row_id, blob_id)\n);\nCREATE INDEX IF NOT EXISTS sync_blob_refs_by_blob\n  ON sync_blob_refs(partition, blob_id);\nCREATE TABLE IF NOT EXISTS sync_tombstones(\n  partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,\n  commit_seq INTEGER NOT NULL,\n  PRIMARY KEY(partition, tbl, row_id)\n);\nCREATE TABLE IF NOT EXISTS sync_backfill_checkpoints(\n  partition TEXT NOT NULL, name TEXT NOT NULL,\n  schema_version INTEGER NOT NULL,\n  state TEXT NOT NULL CHECK(state IN ('declared','backfilling','activated')),\n  watermark INTEGER NOT NULL DEFAULT 0,\n  owner_epoch INTEGER NOT NULL DEFAULT 0,\n  observed_rows INTEGER NOT NULL DEFAULT 0,\n  updated_at_ms INTEGER NOT NULL,\n  PRIMARY KEY(partition, name)\n);\nCREATE TABLE IF NOT EXISTS sync_writer_fence(\n  partition TEXT PRIMARY KEY,\n  required_writer_version INTEGER NOT NULL\n);\n";
+export declare const SQLITE_DDL = "\nCREATE TABLE IF NOT EXISTS sync_partitions(\n  partition TEXT PRIMARY KEY,\n  max_commit_seq INTEGER NOT NULL DEFAULT 0,\n  horizon_seq INTEGER NOT NULL DEFAULT 0\n);\nCREATE TABLE IF NOT EXISTS sync_partition_registry(\n  partition TEXT PRIMARY KEY,\n  log_epoch TEXT NOT NULL,\n  epoch_required INTEGER NOT NULL DEFAULT 0,\n  last_authenticated_at_ms INTEGER NOT NULL\n);\nCREATE TABLE IF NOT EXISTS sync_row_scopes(\n  partition TEXT NOT NULL, tbl TEXT NOT NULL,\n  var TEXT NOT NULL, value TEXT NOT NULL, row_id TEXT NOT NULL,\n  PRIMARY KEY(partition, tbl, var, value, row_id)\n);\nCREATE INDEX IF NOT EXISTS sync_row_scopes_by_row\n  ON sync_row_scopes(partition, tbl, row_id);\nCREATE TABLE IF NOT EXISTS sync_commits(\n  partition TEXT NOT NULL, commit_seq INTEGER NOT NULL,\n  client_id TEXT NOT NULL, client_commit_id TEXT NOT NULL,\n  actor_id TEXT NOT NULL, created_at_ms INTEGER NOT NULL,\n  writer_version INTEGER,\n  PRIMARY KEY(partition, commit_seq)\n);\nCREATE INDEX IF NOT EXISTS sync_commits_by_time\n  ON sync_commits(partition, created_at_ms);\nCREATE TABLE IF NOT EXISTS sync_changes(\n  partition TEXT NOT NULL, commit_seq INTEGER NOT NULL, idx INTEGER NOT NULL,\n  tbl TEXT NOT NULL, row_id TEXT NOT NULL, op INTEGER NOT NULL,\n  row_version INTEGER, scopes TEXT NOT NULL, payload BLOB,\n  PRIMARY KEY(partition, commit_seq, idx)\n);\nCREATE INDEX IF NOT EXISTS sync_changes_by_table\n  ON sync_changes(partition, commit_seq, tbl, idx);\nCREATE TABLE IF NOT EXISTS sync_change_scopes(\n  partition TEXT NOT NULL, tbl TEXT NOT NULL,\n  var TEXT NOT NULL, value TEXT NOT NULL, commit_seq INTEGER NOT NULL,\n  PRIMARY KEY(partition, tbl, var, value, commit_seq)\n);\nCREATE TABLE IF NOT EXISTS sync_push_results(\n  partition TEXT NOT NULL, client_id TEXT NOT NULL,\n  client_commit_id TEXT NOT NULL, result TEXT NOT NULL,\n  PRIMARY KEY(partition, client_id, client_commit_id)\n);\nCREATE TABLE IF NOT EXISTS sync_reactions(\n  partition TEXT NOT NULL, idempotency_key TEXT NOT NULL,\n  type TEXT NOT NULL, version INTEGER NOT NULL, payload TEXT NOT NULL,\n  source_client_id TEXT NOT NULL, source_client_commit_id TEXT NOT NULL,\n  source_commit_seq INTEGER NOT NULL, created_at_ms INTEGER NOT NULL,\n  available_at_ms INTEGER NOT NULL, status TEXT NOT NULL,\n  attempts INTEGER NOT NULL, max_attempts INTEGER NOT NULL,\n  lease_owner TEXT, lease_expires_at_ms INTEGER, completed_at_ms INTEGER,\n  last_failure TEXT,\n  PRIMARY KEY(partition, idempotency_key),\n  CHECK(status IN ('pending', 'leased', 'completed', 'dead-letter'))\n);\nCREATE INDEX IF NOT EXISTS sync_reactions_due\n  ON sync_reactions(partition, status, available_at_ms, created_at_ms, idempotency_key);\nCREATE INDEX IF NOT EXISTS sync_reactions_lease\n  ON sync_reactions(partition, status, lease_expires_at_ms);\nCREATE INDEX IF NOT EXISTS sync_reactions_completed\n  ON sync_reactions(partition, status, completed_at_ms, idempotency_key);\nCREATE INDEX IF NOT EXISTS sync_reactions_dead_letter\n  ON sync_reactions(partition, status, available_at_ms, idempotency_key);\nCREATE TABLE IF NOT EXISTS sync_clients(\n  partition TEXT NOT NULL, client_id TEXT NOT NULL, actor_id TEXT NOT NULL,\n  wire_version INTEGER NOT NULL DEFAULT 1,\n  cursor INTEGER NOT NULL, subscriptions TEXT NOT NULL,\n  updated_at_ms INTEGER NOT NULL,\n  last_round_at_ms INTEGER, last_error_code TEXT,\n  consecutive_failures INTEGER NOT NULL DEFAULT 0,\n  PRIMARY KEY(partition, client_id)\n);\nCREATE TABLE IF NOT EXISTS sync_blob_refs(\n  partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,\n  blob_id TEXT NOT NULL,\n  PRIMARY KEY(partition, tbl, row_id, blob_id)\n);\nCREATE INDEX IF NOT EXISTS sync_blob_refs_by_blob\n  ON sync_blob_refs(partition, blob_id);\nCREATE TABLE IF NOT EXISTS sync_tombstones(\n  partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,\n  commit_seq INTEGER NOT NULL,\n  PRIMARY KEY(partition, tbl, row_id)\n);\nCREATE TABLE IF NOT EXISTS sync_backfill_checkpoints(\n  partition TEXT NOT NULL, name TEXT NOT NULL,\n  schema_version INTEGER NOT NULL,\n  state TEXT NOT NULL CHECK(state IN ('declared','backfilling','activated')),\n  watermark INTEGER NOT NULL DEFAULT 0,\n  owner_epoch INTEGER NOT NULL DEFAULT 0,\n  observed_rows INTEGER NOT NULL DEFAULT 0,\n  updated_at_ms INTEGER NOT NULL,\n  PRIMARY KEY(partition, name)\n);\nCREATE TABLE IF NOT EXISTS sync_writer_fence(\n  partition TEXT PRIMARY KEY,\n  required_writer_version INTEGER NOT NULL\n);\n";
+/** Columns added to `sync_clients` after its first release, by column name. */
+export declare const SYNC_CLIENT_ROUND_COLUMNS: {
+    readonly last_round_at_ms: "INTEGER";
+    readonly last_error_code: "TEXT";
+    readonly consecutive_failures: "INTEGER NOT NULL DEFAULT 0";
+};
 /** Split the DDL into individual statements (D1 applies them one by one). */
 export declare function sqliteDdlStatements(): string[];
 /** §4.6.1: rowIds of one table bound to one scope value, budgeted. */

package/dist/sqlite-dialect.js

@@ -89,6 +89,8 @@ CREATE TABLE IF NOT EXISTS sync_clients(
   wire_version INTEGER NOT NULL DEFAULT 1,
   cursor INTEGER NOT NULL, subscriptions TEXT NOT NULL,
   updated_at_ms INTEGER NOT NULL,
+  last_round_at_ms INTEGER, last_error_code TEXT,
+  consecutive_failures INTEGER NOT NULL DEFAULT 0,
   PRIMARY KEY(partition, client_id)
 );
 CREATE TABLE IF NOT EXISTS sync_blob_refs(
@@ -118,6 +120,12 @@ CREATE TABLE IF NOT EXISTS sync_writer_fence(
   required_writer_version INTEGER NOT NULL
 );
 `;
+/** Columns added to `sync_clients` after its first release, by column name. */
+export const SYNC_CLIENT_ROUND_COLUMNS = {
+    last_round_at_ms: 'INTEGER',
+    last_error_code: 'TEXT',
+    consecutive_failures: 'INTEGER NOT NULL DEFAULT 0',
+};
 /** Split the DDL into individual statements (D1 applies them one by one). */
 export function sqliteDdlStatements() {
     return SQLITE_DDL.split(';')

package/dist/sqlite-storage.d.ts

@@ -57,6 +57,7 @@ export declare class SqliteServerStorage implements ServerStorage {
     scanRowsByIndex(partition: string, query: IndexRowScanQuery): Promise<StoredRow[]>;
     getClientRecord(partition: string, clientId: string): Promise<ClientRecord | undefined>;
     putClientRecord(partition: string, record: ClientRecord): Promise<void>;
+    recordClientRoundFailure(partition: string, clientId: string, actorId: string, errorCode: string, atMs: number): Promise<void>;
     advanceClientCursor(partition: string, clientId: string, actorId: string, logEpoch: string, cursor: number, updatedAtMs: number): Promise<void>;
     updateClientCursor(partition: string, clientId: string, cursor: number, updatedAtMs: number): Promise<void>;
     getActiveClientCursorFloor(partition: string, cutoffMs: number): Promise<number | null>;

package/dist/sqlite-storage.js

@@ -1,7 +1,7 @@
 import { retainRollbackFailure, rollbackAfterError } from './storage-errors.js';
 import { validateCommitPruneQuery } from './prune.js';
 import { StorageQueryError } from './storage-errors.js';
-import { PARTITION_TOUCH_INTERVAL_MS, serveGateRefusal } from './storage.js';
+import { clientRoundState, PARTITION_TOUCH_INTERVAL_MS, serveGateRefusal, } from './storage.js';
 import { serveNotReadyError } from './readiness.js';
 /**
  * SQLite server storage over the shared synchronous driver.
@@ -13,9 +13,9 @@ import { serveNotReadyError } from './readiness.js';
  */
 import { bindAuthoritativePartition, prepareAuthoritativeQuery, } from './authoritative-query.js';
 import { syncError } from './errors.js';
-import { assertPhysicalColumns, assertStoredLayouts, commitWindowPageSql, LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL, deleteRowSql, deleteSqliteRowScopesSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, scanRowPageSql, schemaDdl, SQLITE_TABLE_INDEXES_SQL, sqliteIndexSignatures, selectRowScopesSql, selectRowSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
+import { assertPhysicalColumns, assertStoredLayouts, commitWindowPageSql, LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL, deleteRowSql, deleteSqliteRowScopesSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, scanRowPageSql, schemaDdl, serverIndexDdl, SQLITE_TABLE_INDEXES_SQL, sqliteIndexSignatures, selectRowScopesSql, selectRowSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
 import { matchesEffective } from './scopes.js';
-import { collectCommitWindowPage, deserializePushResult, ERASE_SCOPE_VALUE_HISTORY_SQLITE_SQL, eraseScopeValueHistoryParams, LIST_SCOPE_VALUE_ROWS_SQLITE_SQL, placeholders, SQLITE_DDL, serializePushResult, toStoredChange, toStoredRow, } from './sqlite-dialect.js';
+import { collectCommitWindowPage, deserializePushResult, ERASE_SCOPE_VALUE_HISTORY_SQLITE_SQL, eraseScopeValueHistoryParams, LIST_SCOPE_VALUE_ROWS_SQLITE_SQL, placeholders, SQLITE_DDL, SYNC_CLIENT_ROUND_COLUMNS, serializePushResult, toStoredChange, toStoredRow, } from './sqlite-dialect.js';
 import { SqliteAdapterRequiredError, } from './sqlite-driver.js';
 import { isSqliteConstraintError, StorageConstraintError, } from './storage-errors.js';
 import { assertScopeIndexedScan, resolveIndexRowScan } from './storage-query.js';
@@ -285,6 +285,11 @@ export class SqliteServerStorage {
         if (!clientColumns.some((column) => column.name === 'wire_version')) {
             this.db.exec('ALTER TABLE sync_clients ADD COLUMN wire_version INTEGER NOT NULL DEFAULT 1');
         }
+        for (const [name, definition] of Object.entries(SYNC_CLIENT_ROUND_COLUMNS)) {
+            if (!clientColumns.some((column) => column.name === name)) {
+                this.db.exec(`ALTER TABLE sync_clients ADD COLUMN ${name} ${definition}`);
+            }
+        }
         const commitColumns = this.db
             .query('PRAGMA table_info("sync_commits")')
             .all();
@@ -357,6 +362,29 @@ END`);
                     primaryKeyPosition: column.pk,
                 })), 'sqlite');
             }
+            // Server-only indexes reconcile at every open of a database already at
+            // this version: they never reach a client, so no version bump is needed.
+            const reconcile = serverIndexDdl(schema, new Map([...schema.tables.values()].map((table) => [
+                table.name,
+                new Map(Object.entries(sqliteIndexSignatures(this.db
+                    .query(SQLITE_TABLE_INDEXES_SQL)
+                    .all(table.name)))),
+            ])));
+            if (reconcile.length > 0) {
+                await this.#serializeWrite(() => {
+                    this.db.exec('BEGIN IMMEDIATE');
+                    try {
+                        for (const statement of reconcile)
+                            this.db.exec(statement);
+                        this.db.exec('COMMIT');
+                    }
+                    catch (error) {
+                        rollbackAfterError(() => {
+                            this.db.exec('ROLLBACK');
+                        }, error);
+                    }
+                });
+            }
         }
         if (marker === null || marker.schema_version < schema.version) {
             // Introspect existing app tables, then apply the migration subset
@@ -1133,7 +1161,7 @@ END`);
     }
     async getClientRecord(partition, clientId) {
         const record = this.db
-            .query('SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=? AND client_id=?')
+            .query('SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms, last_error_code, consecutive_failures FROM sync_clients WHERE partition=? AND client_id=?')
             .get(partition, clientId);
         if (record === null)
             return undefined;
@@ -1144,12 +1172,21 @@ END`);
             cursor: record.cursor,
             updatedAtMs: record.updated_at_ms,
             subscriptions: JSON.parse(record.subscriptions),
+            ...clientRoundState(record.last_round_at_ms, record.last_error_code, record.consecutive_failures),
         };
     }
     async putClientRecord(partition, record) {
         this.db
-            .query('INSERT OR REPLACE INTO sync_clients(partition, client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms) VALUES (?,?,?,?,?,?,?)')
-            .run(partition, record.clientId, record.actorId, record.wireVersion, record.cursor, JSON.stringify(record.subscriptions), record.updatedAtMs);
+            .query('INSERT OR REPLACE INTO sync_clients(partition, client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms) VALUES (?,?,?,?,?,?,?,?)')
+            .run(partition, record.clientId, record.actorId, record.wireVersion, record.cursor, JSON.stringify(record.subscriptions), record.updatedAtMs, record.updatedAtMs);
+    }
+    async recordClientRoundFailure(partition, clientId, actorId, errorCode, atMs) {
+        this.db
+            .query(`UPDATE sync_clients
+         SET last_round_at_ms=?, last_error_code=?,
+             consecutive_failures=consecutive_failures+1
+         WHERE partition=? AND client_id=? AND actor_id=?`)
+            .run(atMs, errorCode, partition, clientId, actorId);
     }
     async advanceClientCursor(partition, clientId, actorId, logEpoch, cursor, updatedAtMs) {
         this.db
@@ -1241,7 +1278,7 @@ END`);
     // -- admin/console read surface --------------------------------------------
     async listClientRecords(partition) {
         const records = this.db
-            .query('SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=? ORDER BY updated_at_ms DESC')
+            .query('SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms, last_error_code, consecutive_failures FROM sync_clients WHERE partition=? ORDER BY updated_at_ms DESC')
             .all(partition);
         return records.map((record) => ({
             clientId: record.client_id,
@@ -1250,6 +1287,7 @@ END`);
             cursor: record.cursor,
             updatedAtMs: record.updated_at_ms,
             subscriptions: JSON.parse(record.subscriptions),
+            ...clientRoundState(record.last_round_at_ms, record.last_error_code, record.consecutive_failures),
         }));
     }
     async listCommitMetadata(partition, query) {

package/dist/storage.d.ts

@@ -235,7 +235,15 @@ export interface ClientRecord {
     readonly cursor: number;
     readonly updatedAtMs: number;
     readonly subscriptions: readonly ClientSubscription[];
-}
+    /** When this client's last sync round finished, successfully or not. */
+    readonly lastRoundAtMs?: number;
+    /** Code of the last round when it failed; absent after a successful round. */
+    readonly lastErrorCode?: string;
+    /** Rounds that failed in a row since the last successful round. */
+    readonly consecutiveFailures?: number;
+}
+/** The round-state fields of a stored `sync_clients` row. */
+export declare function clientRoundState(lastRoundAtMs: number | null, lastErrorCode: string | null, consecutiveFailures: number): Pick<ClientRecord, 'lastRoundAtMs' | 'lastErrorCode' | 'consecutiveFailures'>;
 export interface CommitWindowQuery {
     readonly table: string;
     /**
@@ -680,7 +688,18 @@ export interface ServerStorage {
      */
     scanRowsByIndex?(partition: string, query: IndexRowScanQuery): Promise<StoredRow[]>;
     getClientRecord(partition: string, clientId: string): Promise<ClientRecord | undefined>;
+    /**
+     * Write the record for a completed round. The round-state fields reset: the
+     * stored last round is a success at `updatedAtMs` with no failures.
+     */
     putClientRecord(partition: string, record: ClientRecord): Promise<void>;
+    /**
+     * Record a failed round on an existing record: set `lastRoundAtMs` and
+     * `lastErrorCode`, and increment `consecutiveFailures`. Requires a matching
+     * actor; a missing record or another actor's record is unchanged, and
+     * `updatedAtMs` (the pruning liveness signal) is never touched.
+     */
+    recordClientRoundFailure(partition: string, clientId: string, actorId: string, errorCode: string, atMs: number): Promise<void>;
     /**
      * Atomically advance an existing record's cursor and activity timestamp.
      * Preserve registration fields; require matching actor and current log epoch.

package/dist/storage.js

@@ -38,6 +38,14 @@ export function serveGateTokenChanged(before, after) {
     }
     return undefined;
 }
+/** The round-state fields of a stored `sync_clients` row. */
+export function clientRoundState(lastRoundAtMs, lastErrorCode, consecutiveFailures) {
+    return {
+        ...(lastRoundAtMs !== null ? { lastRoundAtMs } : {}),
+        ...(lastErrorCode !== null ? { lastErrorCode } : {}),
+        ...(consecutiveFailures > 0 ? { consecutiveFailures } : {}),
+    };
+}
 /** Durable partition identity refreshed after host authentication (§2.1). */
 /** Longest age of `lastAuthenticatedAtMs` that `touchPartition` leaves unwritten. */
 export const PARTITION_TOUCH_INTERVAL_MS = 60_000;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@syncular/server",
-  "version": "0.39.1",
+  "version": "0.40.0",
   "description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
   "license": "Apache-2.0",
   "author": "Benjamin Kniffler",
@@ -68,7 +68,7 @@
     "verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
   },
   "dependencies": {
-    "@syncular/core": "0.39.1"
+    "@syncular/core": "0.40.0"
   },
   "devDependencies": {
     "@electric-sql/pglite": "^0.5.4"

package/src/admin.ts

@@ -90,6 +90,12 @@ export interface AdminClient {
   readonly connected?: boolean;
   /** When the client's oldest open socket opened. */
   readonly connectedSinceMs?: number;
+  /** When its last sync round finished, successfully or not. */
+  readonly lastRoundAtMs?: number;
+  /** Catalog code of its last round; present only while that round failed. */
+  readonly lastErrorCode?: string;
+  /** Rounds failed in a row since its last success; absent at zero. */
+  readonly consecutiveFailures?: number;
 }
 
 export interface AdminListClientsOptions {
@@ -1633,6 +1639,15 @@ export class SyncularAdmin {
           ? { connected: connectedSinceMs !== undefined }
           : {}),
         ...(connectedSinceMs !== undefined ? { connectedSinceMs } : {}),
+        ...(record.lastRoundAtMs !== undefined
+          ? { lastRoundAtMs: record.lastRoundAtMs }
+          : {}),
+        ...(record.lastErrorCode !== undefined
+          ? { lastErrorCode: record.lastErrorCode }
+          : {}),
+        ...(record.consecutiveFailures !== undefined
+          ? { consecutiveFailures: record.consecutiveFailures }
+          : {}),
       };
     };
   }