@syncular/server 0.21.0 → 0.22.0

package/dist/validate.d.ts

@@ -95,7 +95,19 @@ export interface ValidateCommitInput {
  * staged, but before commit-log/idempotency append and transaction commit.
  */
 export type CommitValidator = (input: ValidateCommitInput) => void | Promise<void>;
-/** Ambient context a validator may consult (§6.7). */
+/**
+ * Ambient context a validator may consult (§6.7).
+ *
+ * A validator runs inside the push transaction. On a single-connection
+ * executor (SQLite, PGlite) the storage serializes authoritative queries
+ * behind that transaction, so a validator that calls `queryAuthoritative`
+ * waits for the transaction it is already inside and the push never
+ * completes. On a pool-backed executor (`pg`, `Bun.sql`) the query takes a
+ * second connection and returns committed state without the candidate write.
+ * Either way a rule that must read rows belongs in the whole-commit
+ * `CommitValidator`, whose reader runs get and scan operations on the same
+ * transaction (§6.8).
+ */
 export interface ValidateContext {
     /** Host-authenticated actor (§1.1) performing the write. */
     readonly actorId: string;
@@ -110,6 +122,13 @@ export interface ValidateContext {
  * any other error is treated as a rejection with a generic
  * `sync.constraint_violation` code (a validator SHOULD throw
  * `ValidationRejection` to control the code). MUST NOT mutate the row.
+ *
+ * The hook runs inside the push transaction and cannot read candidate rows
+ * from an authoritative query: on a single-connection executor
+ * `queryAuthoritative` serializes behind that transaction and the push never
+ * completes, and on a pool-backed executor it reads committed state without
+ * the candidate write. Read rows with the whole-commit `CommitValidator`
+ * reader instead (§6.8).
  */
 export type Validator = (op: ValidateOperation, ctx: ValidateContext) => void | Promise<void>;
 /** Table name → validator (§6.7). Absent tables are unvalidated. */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@syncular/server",
-  "version": "0.21.0",
+  "version": "0.22.0",
   "description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
   "license": "Apache-2.0",
   "author": "Benjamin Kniffler",
@@ -68,7 +68,7 @@
     "verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
   },
   "dependencies": {
-    "@syncular/core": "0.21.0"
+    "@syncular/core": "0.22.0"
   },
   "devDependencies": {
     "@electric-sql/pglite": "^0.5.4"

package/src/pg-executor.ts

@@ -41,6 +41,14 @@ export type PgRow = Record<string, unknown>;
  * `params` use Postgres positional placeholders (`$1`, `$2`, ...). `text`
  * is a single statement (no multi-statement `;`-joined batches — those are
  * driver-specific and defeat parameterization).
+ *
+ * Syncular never overlaps statements on one `PgQueryable`. A client pinned
+ * to a connection by `PgExecutor.transaction` gets every statement chained
+ * on one FIFO by `PostgresServerStorage`, so a host validator that starts
+ * independent reads concurrently still reaches the driver one statement at
+ * a time. An adapter does not need to queue calls itself, and must not
+ * accept two in flight: node-postgres warns about that today and rejects it
+ * in node-postgres 9.
  */
 export interface PgQueryable {
   query<Row = PgRow>(
@@ -58,7 +66,9 @@ export interface PgExecutor extends PgQueryable {
   /**
    * Run `fn` inside a single Postgres transaction on one pinned connection.
    * The executor issues `BEGIN` before `fn`, `COMMIT` on resolve, and
-   * `ROLLBACK` on throw (rethrowing the original error).
+   * `ROLLBACK` on throw (rethrowing the original error). The caller
+   * serializes the statements `fn` issues on the provided client (see
+   * `PgQueryable`), so the driver never sees two of them in flight.
    */
   transaction<T>(fn: (client: PgQueryable) => Promise<T>): Promise<T>;
   /** Release any pooled resources. Optional (pglite/Bun.sql). */

package/src/postgres-storage.ts

@@ -52,6 +52,7 @@ import {
   type PgQueryable,
 } from './pg-executor';
 import {
+  assertPhysicalColumns,
   commitWindowPageSql,
   deleteRowSql,
   dropTableDdl,
@@ -122,7 +123,14 @@ import { assertScopeIndexedScan, resolveIndexRowScan } from './storage-query';
  *   - `sync_row_scopes_pk (partition, tbl, var, value, row_id)` — same shape
  *     for `scanRows`, ordered by `row_id`.
  * Both are the PRIMARY KEY, so they are the clustering/covering index for
- * their table. No secondary index is needed for the hot path.
+ * their table.
+ *
+ * `sync_row_scopes_by_row (partition, tbl, row_id)` is the one secondary
+ * scope index: `writeRowOn` and `deleteRow` replace a row's scope entries
+ * with a `(partition, tbl, row_id)` predicate, and `row_id` is the LAST
+ * column of the inverted PRIMARY KEY, so without this index that predicate
+ * scans the table's whole scope range on every row written (a bulk import
+ * costs one such scan per row).
  *
  * Blob reference index (§5.9.4) — parity with the SQLite dialect's
  * `sync_blob_refs`:
@@ -151,6 +159,8 @@ CREATE TABLE IF NOT EXISTS sync_row_scopes(
   var TEXT NOT NULL, value TEXT NOT NULL, row_id TEXT NOT NULL,
   PRIMARY KEY(partition, tbl, var, value, row_id)
 );
+CREATE INDEX IF NOT EXISTS sync_row_scopes_by_row
+  ON sync_row_scopes(partition, tbl, row_id);
 CREATE TABLE IF NOT EXISTS sync_commits(
   partition TEXT NOT NULL, commit_seq BIGINT NOT NULL,
   client_id TEXT NOT NULL, client_commit_id TEXT NOT NULL,
@@ -601,7 +611,23 @@ class PostgresTransaction implements StorageTransaction {
     resolve: () => void,
     reject: (error: unknown) => void,
   ) {
-    this.#client = client;
+    // One pinned connection carries one statement at a time: node-postgres
+    // warns about a concurrently issued second query today and rejects it in
+    // node-postgres 9. A whole-commit validator that fans independent reads
+    // out with Promise.all must not interleave them, so every statement
+    // issued through this transaction is chained on one FIFO — the same
+    // shape as `SqliteServerStorage.queryAuthoritative`.
+    let tail: Promise<void> = Promise.resolve();
+    this.#client = {
+      query: <Row>(text: string, params?: readonly unknown[]) => {
+        const run = tail.then(() => client.query<Row>(text, params));
+        tail = run.then(
+          () => undefined,
+          () => undefined,
+        );
+        return run;
+      },
+    };
     this.#partition = partition;
     this.#resolveTable = resolveTable;
     this.#resolve = resolve;
@@ -977,6 +1003,68 @@ export class PostgresServerStorage implements ServerStorage {
         `stored schema version ${stored} is newer than the configured schema (${schema.version}) — refusing to run an older server against a migrated database`,
       );
     }
+    if (stored !== undefined && stored === schema.version) {
+      // Version equality is not layout equality: a marker written by another
+      // build at the same version describes rows the running codec cannot
+      // decode. Compare the stored layouts instead of trusting the number.
+      const storedLayouts = parseLayouts(
+        typeof marker.rows[0]?.layouts === 'string'
+          ? marker.rows[0].layouts
+          : undefined,
+      );
+      const configuredLayouts = parseLayouts(layoutsOf(schema));
+      let mismatch: string | undefined;
+      for (const [tableName, columns] of Object.entries(configuredLayouts)) {
+        const storedTable = storedLayouts[tableName];
+        const columnCount = Math.max(columns.length, storedTable?.length ?? 0);
+        for (let index = 0; index < columnCount; index++) {
+          const expected = columns[index];
+          const actual = storedTable?.[index];
+          if (
+            expected !== undefined &&
+            actual !== undefined &&
+            actual.name === expected.name &&
+            actual.type === expected.type &&
+            actual.nullable === expected.nullable
+          ) {
+            continue;
+          }
+          mismatch = `table ${JSON.stringify(tableName)} column ${JSON.stringify(expected?.name ?? actual?.name ?? '')}`;
+          break;
+        }
+        if (mismatch !== undefined) break;
+      }
+      if (mismatch === undefined) {
+        for (const tableName of Object.keys(storedLayouts)) {
+          if (!(tableName in configuredLayouts)) {
+            mismatch = `table ${JSON.stringify(tableName)}`;
+            break;
+          }
+        }
+      }
+      if (mismatch !== undefined) {
+        throw new Error(
+          `stored schema layouts disagree with the configured schema at version ${schema.version} (${mismatch}) — refusing to serve a database whose stored rows the running code cannot decode`,
+        );
+      }
+      // The persisted layouts describe the codec's app columns only: read the
+      // physical tables so a same-version database missing a
+      // storage-internal column is refused at startup instead of failing at
+      // the first write. Resolve through `to_regclass`, which follows the
+      // session's `search_path` exactly as the unqualified row queries do —
+      // `current_schema()` names only the first existing schema, so it would
+      // reject a table that resolves later in the path.
+      for (const table of schema.tables.values()) {
+        const { rows } = await this.#exec.query<{ column_name: string }>(
+          `SELECT attribute.attname AS column_name
+           FROM pg_attribute attribute
+           WHERE attribute.attrelid = to_regclass($1)
+             AND attribute.attnum > 0 AND NOT attribute.attisdropped`,
+          [table.name],
+        );
+        assertPhysicalColumns(table, new Set(rows.map((r) => r.column_name)));
+      }
+    }
     if (stored === undefined || stored < schema.version) {
       // Introspect existing app tables, apply the migration subset
       // (CREATE TABLE / ADD COLUMN / rebuild indexes), then rewrite stored

package/src/pull.ts

@@ -180,6 +180,7 @@ async function signedUrlFields(
 
 function segmentRefFrame(
   record: SegmentRecord,
+  scopeDigest: string,
   extra: { url?: string; urlExpiresAtMs?: number },
 ): ResponseFrame {
   return {
@@ -190,7 +191,11 @@ function segmentRefFrame(
     byteLength: record.byteLength,
     rowCount: record.rowCount,
     asOfCommitSeq: record.asOfCommitSeq,
-    scopeDigest: record.scopeDigest,
+    // The frame reports the digest THIS subscription is authorized under. A
+    // stored entry can carry several digests when identical content was
+    // published under different scopes (§5.1), and the record's primary
+    // digest is not necessarily the caller's (§5.5).
+    scopeDigest,
     ...(record.rowCursor !== null ? { rowCursor: record.rowCursor } : {}),
     ...(record.nextRowCursor !== null
       ? { nextRowCursor: record.nextRowCursor }
@@ -244,6 +249,7 @@ async function* sqliteImageSegment(
     });
     yield segmentRefFrame(
       existing,
+      digest,
       await signedUrlFields(ctx, limits, existing.segmentId, digest, now),
     );
     return true;
@@ -342,6 +348,7 @@ async function* sqliteImageSegment(
   });
   yield segmentRefFrame(
     record,
+    digest,
     await signedUrlFields(
       ctx,
       limits,
@@ -458,6 +465,7 @@ async function* bootstrapSegments(
       });
       yield segmentRefFrame(
         record,
+        digest,
         await signedUrlFields(ctx, limits, record.segmentId, digest, now),
       );
     }

package/src/realtime.ts

@@ -3,7 +3,10 @@
  *
  * A `RealtimeSession` wraps a connected socket's send/receive callbacks:
  * the host feeds inbound text frames to `handleMessage`, binary frames to
- * `handleBinary`, and wires `session.close()` to socket close. Initial
+ * `handleBinary`, and wires `session.close()` to socket close. `handleMessage`
+ * queues the ack's cursor write without awaiting it, so `drain()` is the
+ * awaitable boundary a host uses before closing its storage or ending a
+ * hibernatable event (§8.2). Initial
  * subscription registration comes from the client's most recent pull
  * (§8.1, loaded from the client record); a sync round completed on the
  * connection replaces it at round end (§8.7). Deltas are complete SSP2
@@ -253,6 +256,14 @@ export class RealtimeSession {
       timer?: ReturnType<typeof setTimeout>;
     }
   >();
+  /** Queued control-plane writes (ack cursor persistence, §8.2). */
+  #controlTail: Promise<void> = Promise.resolve();
+  /** First control-plane failure not yet surfaced by `drain()`. */
+  #controlFailure: unknown = undefined;
+  /** Issue counter for registration refreshes (§8.7). */
+  #registrationGeneration = 0;
+  /** Generation of the most recent refresh whose result was applied. */
+  #registrationApplied = 0;
 
   constructor(
     hub: RealtimeHub,
@@ -313,7 +324,21 @@ export class RealtimeSession {
     this.lastKnownSeq = Math.max(this.lastKnownSeq, this.cursor);
     if (this.cursor >= this.lastKnownSeq) this.wakePending = false;
     // §8.2: acks update the client cursor record without an HTTP pull.
-    void this.#persistCursor();
+    this.#persistCursor();
+  }
+
+  /**
+   * Resolve once every queued control-plane write has settled (§8.2). A host
+   * awaits this before it closes its storage or ends a hibernatable event, so
+   * an ack's cursor write is not abandoned mid-flight. A persistence failure
+   * surfaces here instead of vanishing.
+   */
+  drain(): Promise<void> {
+    return this.#controlTail.then(() => {
+      const failure = this.#controlFailure;
+      this.#controlFailure = undefined;
+      if (failure !== undefined) throw failure;
+    });
   }
 
   /** §8.6.2 inbound presence publish/leave from the client. */
@@ -590,7 +615,7 @@ export class RealtimeSession {
       finishRound();
       // §8.7 registration at round end: reload the persisted list — it
       // only advances on success, so failed rounds change nothing.
-      await this.#refreshRegistrations();
+      await this.refreshRegistrations(false);
     }
   }
 
@@ -601,22 +626,42 @@ export class RealtimeSession {
     await this.#send(tagged);
   }
 
-  async #refreshRegistrations(): Promise<void> {
-    // §8.6.3: a registration change re-derives the presence grant — capture
-    // the keys before, reconcile after (leaves on lost keys, snapshots on
-    // gained keys).
-    const previousKeys = this.#currentScopeKeys();
+  /**
+   * Reload this connection's registrations and reconcile presence (§8.7).
+   * The round-end refresh and the host-initiated `RealtimeHub.refreshScopes`
+   * both route through here under one per-session generation, so a load that
+   * resolves late can never overwrite a newer result: a slow resolver may
+   * return scopes a later refresh already revoked, and the later refresh
+   * keeps the connection's current grants.
+   *
+   * `failClosed` selects the failure semantics. The round-end refresh keeps
+   * the previous registrations so a failed round changes nothing. The host
+   * refresh empties them so a revoked or unreadable grant stops receiving
+   * deltas immediately; it also survives a newer round-end refresh that
+   * failed, because that round applied nothing.
+   */
+  async refreshRegistrations(failClosed: boolean): Promise<void> {
+    const generation = ++this.#registrationGeneration;
+    let registrations: Registration[];
     try {
-      this.registrations = await this.#hub.loadRegistrations(
+      registrations = await this.#hub.loadRegistrations(
         this.partition,
         this.actorId,
         this.clientId,
       );
     } catch {
-      // Fail closed: an unreadable record or resolver failure leaves the
-      // previous registrations in place; the next round repairs it.
-      return;
+      if (!failClosed) return;
+      // Fail closed (host refresh): drop every grant on the connection.
+      registrations = [];
     }
+    // A newer refresh already applied its result; this late one is stale.
+    if (generation < this.#registrationApplied) return;
+    this.#registrationApplied = generation;
+    // §8.6.3: a registration change re-derives the presence grant — capture
+    // the keys before, reconcile after (leaves on lost keys, snapshots on
+    // gained keys).
+    const previousKeys = this.#currentScopeKeys();
+    this.registrations = registrations;
     this.reconcilePresence(previousKeys);
   }
 
@@ -628,8 +673,11 @@ export class RealtimeSession {
     this.#hub.disconnect(this);
   }
 
-  async #persistCursor(): Promise<void> {
-    try {
+  #persistCursor(): void {
+    // §8.2: start the write now and keep it on the control tail so `drain()`
+    // can await it. Two acks may write concurrently: storage folds them with
+    // `MAX(cursor, ?)`, so neither order loses the later cursor.
+    const write = (async () => {
       await this.#storage.advanceClientCursor(
         this.partition,
         this.clientId,
@@ -638,9 +686,15 @@ export class RealtimeSession {
         this.cursor,
         this.#clock(),
       );
-    } catch {
-      // Cursor persistence is best-effort; the next pull repairs it.
-    }
+    })();
+    write.catch((error: unknown) => {
+      // Best-effort for the connection (the next pull repairs the cursor);
+      // `drain()` carries the first failure to the host.
+      this.#controlFailure ??= error;
+    });
+    this.#controlTail = Promise.allSettled([this.#controlTail, write]).then(
+      () => undefined,
+    );
   }
 
   /** Fire-and-forget send for control text and deltas; a host send
@@ -1106,6 +1160,30 @@ export class RealtimeHub {
     }
   }
 
+  /**
+   * Host-initiated, fail-closed registration refresh (§8.7). A host calls this
+   * after it changes an actor's membership or connection state, so the
+   * partition's connected sessions stop using grants the host already revoked
+   * without waiting for the client to happen to run a round.
+   *
+   * This differs deliberately from the round-end refresh
+   * (`RealtimeSession.refreshRegistrations`), which keeps the previous
+   * registrations when the client record or resolver cannot be read so a
+   * failed round changes nothing. Here an unresolvable session is emptied:
+   * a revoked or unreadable grant must stop receiving deltas rather than live
+   * on until the next round. Presence is reconciled exactly as at round end.
+   */
+  async refreshScopes(partition: string, actorId?: string): Promise<void> {
+    for (const session of [...this.#sessions]) {
+      if (session.partition !== partition) continue;
+      if (actorId !== undefined && session.actorId !== actorId) continue;
+      // Fail closed (see the doc above): an unreadable client record drops
+      // every grant on the connection. The session's generation guard keeps
+      // this revocation from being overwritten by a slower round-end reload.
+      await session.refreshRegistrations(true);
+    }
+  }
+
   /** Broadcast a wake-up (host-initiated resync, schema rollover, §8.3). */
   wake(partition: string, reason: WakeReason): void {
     for (const session of this.#sessions) {

package/src/relational-rows.ts

@@ -191,6 +191,42 @@ export function tableColumnNames(table: CompiledTable): string[] {
   ];
 }
 
+/**
+ * The physical column set the row codec and the scope index read for one
+ * configured table. The persisted marker's layouts describe the codec's app
+ * columns only and its version number is the schema's, not the database's, so
+ * a same-version database that is missing a storage-internal meta column (a
+ * version-only bump whose ALTER never ran) still has to be refused: compare
+ * the physical table and fail closed without writing a migration.
+ *
+ * `existingColumns` must be resolved the way the unqualified row queries
+ * resolve the table — on PostgreSQL, through the session `search_path`, not
+ * `current_schema()`. A search path whose first schema is not where the
+ * configured tables live is outside the supported contract: DDL is created in
+ * that first schema, so `ensureSchema` already writes shadow tables there.
+ *
+ * Scope: this checks that the columns EXIST. Stored types, nullability, and
+ * non-column storage internals (the tombstone bookkeeping) stay outside it,
+ * because a wrong type or nullability is a codec-compatibility question the
+ * persisted layouts answer for app columns.
+ */
+export function assertPhysicalColumns(
+  table: CompiledTable,
+  existingColumns: ReadonlySet<string>,
+): void {
+  if (existingColumns.size === 0) {
+    throw new Error(
+      `table ${JSON.stringify(table.name)} is missing from the database at the stored schema version — refusing to serve a database whose rows the running code cannot read`,
+    );
+  }
+  for (const name of tableColumnNames(table)) {
+    if (existingColumns.has(name)) continue;
+    throw new Error(
+      `table ${JSON.stringify(table.name)} is missing column ${JSON.stringify(name)} at the stored schema version — refusing to serve a database whose rows the running code cannot read`,
+    );
+  }
+}
+
 /** CREATE TABLE IF NOT EXISTS for one app table. */
 export function createTableDdl(
   table: CompiledTable,