@syncular/server 0.16.1 → 0.17.0

package/src/postgres-storage.ts

@@ -230,6 +230,13 @@ async function lockPartitionOn(
   client: PgQueryable,
   partition: string,
 ): Promise<void> {
+  const locked = await client.query(
+    'SELECT max_commit_seq FROM sync_partitions WHERE partition=$1 FOR UPDATE',
+    [partition],
+  );
+  if (locked.rows.length > 0) return;
+  // The first writer initializes the partition. Concurrent initializers may
+  // wait on this insert, so acquire the row lock again before applying writes.
   await client.query(
     `INSERT INTO sync_partitions(partition, max_commit_seq) VALUES ($1, 0)
     ON CONFLICT (partition) DO NOTHING`,
@@ -753,32 +760,40 @@ class PostgresTransaction implements StorageTransaction {
     // Allocate the next dense commitSeq under a per-partition row lock: the
     // UPDATE … RETURNING serializes concurrent pushes to this partition and
     // never leaves a gap on rollback (see the file header).
-    const { rows } = await q.query<{ max_commit_seq: unknown }>(
-      `INSERT INTO sync_partitions(partition, max_commit_seq) VALUES ($1, 1)
-       ON CONFLICT (partition) DO UPDATE
-         SET max_commit_seq = sync_partitions.max_commit_seq + 1
-       RETURNING max_commit_seq`,
-      [p],
-    );
-    const commitSeq = asNumber(rows[0]?.max_commit_seq);
-    await q.query(
-      `INSERT INTO sync_commits(partition, commit_seq, client_id, client_commit_id, actor_id, created_at_ms)
-       VALUES ($1,$2,$3,$4,$5,$6)`,
+    const { rows } = await q.query<{ commit_seq: unknown }>(
+      `WITH allocated AS (
+         INSERT INTO sync_partitions(partition, max_commit_seq) VALUES ($1, 1)
+         ON CONFLICT (partition) DO UPDATE
+           SET max_commit_seq = sync_partitions.max_commit_seq + 1
+         RETURNING max_commit_seq
+       )
+       INSERT INTO sync_commits(partition, commit_seq, client_id, client_commit_id, actor_id, created_at_ms)
+       SELECT $1, max_commit_seq, $2, $3, $4, $5 FROM allocated
+       RETURNING commit_seq`,
       [
         p,
-        commitSeq,
         commit.clientId,
         commit.clientCommitId,
         commit.actorId,
         commit.createdAtMs,
       ],
     );
+    const commitSeq = asNumber(rows[0]?.commit_seq);
     for (let idx = 0; idx < commit.changes.length; idx++) {
       const change = commit.changes[idx];
       if (change === undefined) continue;
+      // Bind serialized scopes as text before parsing JSONB. Drivers that
+      // encode JSONB parameters would otherwise store this string as a scalar.
       await q.query(
-        `INSERT INTO sync_changes(partition, commit_seq, idx, tbl, row_id, op, row_version, scopes, payload)
-         VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)`,
+        `WITH inserted AS (
+           INSERT INTO sync_changes(partition, commit_seq, idx, tbl, row_id, op, row_version, scopes, payload)
+           VALUES ($1,$2,$3,$4,$5,$6,$7,$8::text::jsonb,$9)
+           RETURNING partition, tbl, commit_seq, scopes
+         )
+         INSERT INTO sync_change_scopes(partition, tbl, var, value, commit_seq)
+         SELECT inserted.partition, inserted.tbl, scope.key, scope.value, inserted.commit_seq
+         FROM inserted CROSS JOIN LATERAL jsonb_each_text(inserted.scopes) AS scope
+         ON CONFLICT DO NOTHING`,
         [
           p,
           commitSeq,
@@ -791,13 +806,6 @@ class PostgresTransaction implements StorageTransaction {
           change.payload ?? null,
         ],
       );
-      for (const [variable, value] of Object.entries(change.scopes)) {
-        await q.query(
-          `INSERT INTO sync_change_scopes(partition, tbl, var, value, commit_seq)
-           VALUES ($1,$2,$3,$4,$5) ON CONFLICT DO NOTHING`,
-          [p, change.table, variable, value, commitSeq],
-        );
-      }
     }
     return commitSeq;
   }
@@ -986,6 +994,9 @@ export class PostgresServerStorage implements ServerStorage {
           await client.query('DELETE FROM sync_row_scopes WHERE tbl=$1', [
             tableName,
           ]);
+          await client.query('DELETE FROM sync_blob_refs WHERE tbl=$1', [
+            tableName,
+          ]);
           await client.query(dropTableDdl(tableName));
         }
         for (const statement of schemaDdl(
@@ -1676,6 +1687,38 @@ export class PostgresServerStorage implements ServerStorage {
     );
   }
 
+  async advanceClientCursor(
+    partition: string,
+    clientId: string,
+    actorId: string,
+    logEpoch: string,
+    cursor: number,
+    updatedAtMs: number,
+  ): Promise<void> {
+    await this.#exec.query(
+      `UPDATE sync_clients
+         SET cursor=GREATEST(cursor, $1), updated_at_ms=GREATEST(updated_at_ms, $2)
+         WHERE partition=$3 AND client_id=$4 AND actor_id=$5
+           AND EXISTS (SELECT 1 FROM sync_partition_registry
+                       WHERE partition=sync_clients.partition AND log_epoch=$6)`,
+      [cursor, updatedAtMs, partition, clientId, actorId, logEpoch],
+    );
+  }
+
+  async updateClientCursor(
+    partition: string,
+    clientId: string,
+    cursor: number,
+    updatedAtMs: number,
+  ): Promise<void> {
+    await this.#exec.query(
+      `UPDATE sync_clients
+       SET cursor=GREATEST(cursor, $3), updated_at_ms=GREATEST(updated_at_ms, $4)
+       WHERE partition=$1 AND client_id=$2`,
+      [partition, clientId, cursor, updatedAtMs],
+    );
+  }
+
   async getActiveClientCursorFloor(
     partition: string,
     cutoffMs: number,

package/src/pull.ts

@@ -501,6 +501,28 @@ export async function* subscriptionSection(
 
   const token = parseBootstrapToken(sub.bootstrapState, sub.table);
 
+  let commits: StoredCommit[] = [];
+  if (
+    token === undefined &&
+    sub.cursor >= 0 &&
+    sub.cursor >= horizonSeq &&
+    sub.cursor <= maxSeq
+  ) {
+    commits = await ctx.storage.readCommitWindow(ctx.partition, {
+      table: sub.table,
+      scopeFilter: plan.effective,
+      afterSeq: sub.cursor,
+      throughSeq: maxSeq,
+      limitChanges: limits.limitCommits + 1,
+    });
+    // Validate continuity before committing to an active section. A prune
+    // during a paged read can otherwise make an incomplete window look empty.
+    horizonSeq = Math.max(
+      horizonSeq,
+      await ctx.storage.getHorizonSeq(ctx.partition),
+    );
+  }
+
   // §4.6: a cursor behind the horizon (and not resuming a bootstrap)
   // cannot compute deltas — answer `reset` and echo the cursor.
   if (token === undefined && sub.cursor >= 0 && sub.cursor < horizonSeq) {
@@ -574,13 +596,6 @@ export async function* subscriptionSection(
     effectiveScopes: plan.effective,
     bootstrap: false,
   };
-  const commits = await ctx.storage.readCommitWindow(ctx.partition, {
-    table: sub.table,
-    scopeFilter: plan.effective,
-    afterSeq: sub.cursor,
-    throughSeq: maxSeq,
-    limitChanges: limits.limitCommits + 1,
-  });
   let delivered = 0;
   let deliveredCommits = 0;
   let lastDeliveredSeq = sub.cursor;

package/src/realtime.ts

@@ -307,6 +307,10 @@ export class RealtimeSession {
     const cursor = (parsed as { cursor?: unknown }).cursor;
     if (typeof cursor !== 'number' || !Number.isSafeInteger(cursor)) return;
     this.cursor = Math.max(this.cursor, cursor);
+    // The client may have caught up through another binding (§8.4). Its ack
+    // proves those commits no longer need socket notifications, so the next
+    // notification must be adjacent to the acknowledged cursor.
+    this.lastKnownSeq = Math.max(this.lastKnownSeq, this.cursor);
     if (this.cursor >= this.lastKnownSeq) this.wakePending = false;
     // §8.2: acks update the client cursor record without an HTTP pull.
     void this.#persistCursor();
@@ -626,16 +630,14 @@ export class RealtimeSession {
 
   async #persistCursor(): Promise<void> {
     try {
-      const record = await this.#storage.getClientRecord(
+      await this.#storage.advanceClientCursor(
         this.partition,
         this.clientId,
+        this.actorId,
+        this.logEpoch,
+        this.cursor,
+        this.#clock(),
       );
-      if (record === undefined) return;
-      await this.#storage.putClientRecord(this.partition, {
-        ...record,
-        cursor: Math.max(record.cursor, this.cursor),
-        updatedAtMs: this.#clock(),
-      });
     } catch {
       // Cursor persistence is best-effort; the next pull repairs it.
     }
@@ -688,8 +690,9 @@ export class RealtimeSession {
     }
   }
 
-  /** Called by the hub for every applied commit, in commitSeq order. */
+  /** Called by the hub for every applied commit notification. */
   deliverCommit(commit: StoredCommit): void {
+    const contiguous = commit.commitSeq === this.lastKnownSeq + 1;
     this.lastKnownSeq = Math.max(this.lastKnownSeq, commit.commitSeq);
     const sections: Array<{
       registration: Registration;
@@ -716,6 +719,14 @@ export class RealtimeSession {
         );
       if (changes.length > 0) sections.push({ registration, changes });
     }
+    if (!contiguous) {
+      // Pushes allocate commitSeq under the partition lock but notify after
+      // commit, so racing notifications may arrive out of order. A gap,
+      // duplicate, or regression cannot be a delta: the client would apply
+      // and acknowledge a cursor that does not prove the intervening log.
+      this.sendWake('catchup-required');
+      return;
+    }
     if (sections.length === 0) return;
     if (this.#activeRound !== undefined) {
       // §8.7 interleaving: no 0x00 messages while a response stream is
@@ -771,7 +782,7 @@ export class RealtimeSession {
     tagged[0] = REALTIME_TAG_DELTA;
     tagged.set(bytes, 1);
     this.#sendSafe(tagged);
-    this.cursor = commit.commitSeq;
+    this.cursor = Math.max(this.cursor, commit.commitSeq);
     const events = this.#events;
     if (events !== undefined) {
       emitEvent(events, {

package/src/relational-rows.ts

@@ -205,7 +205,7 @@ export function physicalIndexName(declaredName: string): string {
 
 /**
  * CREATE INDEX IF NOT EXISTS for the table's user-declared indexes. These use
- * the same declared names and columns the client materializes. Cross-table
+ * the declared columns with a leading server partition column. Cross-table
  * index-name uniqueness is the user's schema concern, as it is client-side.
  * Server-side the physical name carries the {@link SYNC_INDEX_PREFIX}
  * ownership marker.
@@ -215,7 +215,7 @@ export function createIndexDdl(table: CompiledTable): string[] {
   if (!table.materialize) return [];
   return table.indexes.map((index) => {
     const unique = index.unique ? 'UNIQUE ' : '';
-    const columns = index.columns
+    const columns = [SYNC_PARTITION_COLUMN, ...index.columns]
       .map((column) => quoteIdent(column))
       .join(', ');
     return `CREATE ${unique}INDEX IF NOT EXISTS ${quoteIdent(physicalIndexName(index.name))} ON ${quoteIdent(table.name)} (${columns})`;
@@ -477,6 +477,22 @@ export function deleteRowSql(
   return `DELETE FROM ${quoteIdent(table.name)} WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=${p[0]} AND ${quoteIdent(SYNC_ROW_ID_COLUMN)}=${p[1]}`;
 }
 
+/**
+ * SQLite: remove the old row's exact scope keys before replacing or deleting the row.
+ * Params: [partition, table, rowId, partition, rowId]. The caller keeps this
+ * statement and the row write in the same transaction (or D1 atomic batch).
+ */
+export function deleteSqliteRowScopesSql(table: CompiledTable): string {
+  return `DELETE FROM sync_row_scopes
+    WHERE partition=? AND tbl=? AND row_id=?
+      AND (var, value) IN (
+        SELECT key, value FROM json_each((
+          SELECT ${quoteIdent(SYNC_SCOPES_COLUMN)} FROM ${quoteIdent(table.name)}
+          WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=? AND ${quoteIdent(SYNC_ROW_ID_COLUMN)}=?
+        ))
+      )`;
+}
+
 /**
  * The schema-version marker table gates DDL work. `ensureSchema` compares the
  * stored version and skips

package/src/sqlite-storage.ts

@@ -17,6 +17,7 @@ import { syncError } from './errors';
 import {
   commitWindowPageSql,
   deleteRowSql,
+  deleteSqliteRowScopesSql,
   dropTableDdl,
   indexRowPageStatement,
   layoutsOf,
@@ -228,13 +229,15 @@ class SqliteTransaction implements StorageTransaction {
   async deleteRow(table: string, rowId: string): Promise<void> {
     this.#assertOpen();
     const db = this.#storage.db;
-    db.query(deleteRowSql(this.#storage.table(table), 'sqlite')).run(
+    const compiled = this.#storage.table(table);
+    db.query(deleteSqliteRowScopesSql(compiled)).run(
+      this.#partition,
+      table,
+      rowId,
       this.#partition,
       rowId,
     );
-    db.query(
-      'DELETE FROM sync_row_scopes WHERE partition=? AND tbl=? AND row_id=?',
-    ).run(this.#partition, table, rowId);
+    db.query(deleteRowSql(compiled, 'sqlite')).run(this.#partition, rowId);
     // §5.9.4: a deleted row references no blobs.
     db.query(
       'DELETE FROM sync_blob_refs WHERE partition=? AND tbl=? AND row_id=?',
@@ -485,6 +488,9 @@ export class SqliteServerStorage implements ServerStorage {
           this.db
             .query('DELETE FROM sync_row_scopes WHERE tbl=?')
             .run(tableName);
+          this.db
+            .query('DELETE FROM sync_blob_refs WHERE tbl=?')
+            .run(tableName);
           this.db.exec(dropTableDdl(tableName));
         }
         for (const statement of schemaDdl(
@@ -683,6 +689,9 @@ export class SqliteServerStorage implements ServerStorage {
   /** Internal: write a row + refresh its scope-index entries. */
   writeRow(partition: string, table: string, row: StoredRow): void {
     const compiled = this.table(table);
+    this.db
+      .query(deleteSqliteRowScopesSql(compiled))
+      .run(partition, table, row.rowId, partition, row.rowId);
     this.db
       .query(upsertSql(compiled, 'sqlite'))
       .run(
@@ -694,11 +703,6 @@ export class SqliteServerStorage implements ServerStorage {
           | null
         )[]),
       );
-    this.db
-      .query(
-        'DELETE FROM sync_row_scopes WHERE partition=? AND tbl=? AND row_id=?',
-      )
-      .run(partition, table, row.rowId);
     for (const [variable, value] of Object.entries(row.scopes)) {
       this.db
         .query(
@@ -1272,6 +1276,38 @@ export class SqliteServerStorage implements ServerStorage {
       );
   }
 
+  async advanceClientCursor(
+    partition: string,
+    clientId: string,
+    actorId: string,
+    logEpoch: string,
+    cursor: number,
+    updatedAtMs: number,
+  ): Promise<void> {
+    this.db
+      .query(`UPDATE sync_clients
+         SET cursor=MAX(cursor, ?), updated_at_ms=MAX(updated_at_ms, ?)
+         WHERE partition=? AND client_id=? AND actor_id=?
+           AND EXISTS (SELECT 1 FROM sync_partition_registry
+                       WHERE partition=sync_clients.partition AND log_epoch=?)`)
+      .run(cursor, updatedAtMs, partition, clientId, actorId, logEpoch);
+  }
+
+  async updateClientCursor(
+    partition: string,
+    clientId: string,
+    cursor: number,
+    updatedAtMs: number,
+  ): Promise<void> {
+    this.db
+      .query(
+        `UPDATE sync_clients
+         SET cursor=MAX(cursor, ?), updated_at_ms=MAX(updated_at_ms, ?)
+         WHERE partition=? AND client_id=?`,
+      )
+      .run(cursor, updatedAtMs, partition, clientId);
+  }
+
   async getActiveClientCursorFloor(
     partition: string,
     cutoffMs: number,

package/src/storage-errors.ts

@@ -15,6 +15,10 @@ export class StorageConstraintError extends Error {
 
 /** Stable, privacy-safe failures for trusted server storage queries. */
 export type StorageQueryErrorCode =
+  | 'sync.storage.schema_migration_pending'
+  | 'sync.storage.schema_migration_conflict'
+  | 'sync.storage.schema_changed'
+  | 'sync.storage.invalid_migration_budget'
   | 'sync.storage.scan_requires_scope'
   | 'sync.storage.index_not_found'
   | 'sync.storage.index_not_materialized'
@@ -26,6 +30,14 @@ export type StorageQueryErrorCode =
 
 const STORAGE_QUERY_MESSAGES: Readonly<Record<StorageQueryErrorCode, string>> =
   {
+    'sync.storage.schema_migration_pending':
+      'schema migration requires another invocation',
+    'sync.storage.schema_migration_conflict':
+      'another schema migration target is already pending',
+    'sync.storage.schema_changed':
+      'storage schema is not ready for this operation',
+    'sync.storage.invalid_migration_budget':
+      'migration statement budget must be an integer from 10 through 1000',
     'sync.storage.prune_epoch_mismatch':
       'partition log epoch changed; recompute retention inputs',
     'sync.storage.partition_unregistered':

package/src/storage.ts

@@ -560,6 +560,27 @@ export interface ServerStorage {
     clientId: string,
   ): Promise<ClientRecord | undefined>;
   putClientRecord(partition: string, record: ClientRecord): Promise<void>;
+  /**
+   * Atomically advance an existing record's cursor and activity timestamp.
+   * Preserve registration fields; require matching actor and current log epoch.
+   * Missing records and mismatched identities are unchanged (§8.2).
+   */
+  advanceClientCursor(
+    partition: string,
+    clientId: string,
+    actorId: string,
+    logEpoch: string,
+    cursor: number,
+    updatedAtMs: number,
+  ): Promise<void>;
+  /** Advance an existing client's ACK cursor and timestamp atomically,
+   * preserving actor, wire version, and subscriptions. Missing records stay absent. */
+  updateClientCursor(
+    partition: string,
+    clientId: string,
+    cursor: number,
+    updatedAtMs: number,
+  ): Promise<void>;
   /** Minimum cursor with updatedAtMs >= cutoff; null when none are active. */
   getActiveClientCursorFloor(
     partition: string,