@syncular/server 0.21.1 → 0.22.0

package/src/sqlite-segment-store.ts

@@ -3,12 +3,19 @@
  */
 import {
   DEFAULT_SEGMENT_TTL_MS,
+  mergeSegmentRecord,
+  publicationKey,
+  publicationRecord,
+  publicationsForRecord,
   type SegmentFindKey,
   type SegmentMetadata,
+  type SegmentPublication,
   type SegmentRecord,
+  segmentBytesEqual,
   type SegmentStore,
   type SegmentStoreStats,
   segmentIdFor,
+  selectSegmentPublication,
 } from './segment-store';
 import {
   SqliteAdapterRequiredError,
@@ -40,6 +47,33 @@ export class SqliteSegmentStore implements SegmentStore {
         expires_at_ms INTEGER NOT NULL, bytes BLOB NOT NULL
       );
     `);
+    // One row per scope digest a content address was published under (§5.1):
+    // identical bytes from two scopes share a `segment_id` and must both stay
+    // downloadable (§5.5).
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS sync_segment_scopes(
+        segment_id TEXT NOT NULL, scope_digest TEXT NOT NULL,
+        PRIMARY KEY (segment_id, scope_digest)
+      );
+    `);
+    // One row per publication of a content address (§5.1): the same bytes
+    // under a different partition, epoch, table, or pin are distinct grants
+    // and must each stay downloadable. Rows written before this table existed
+    // materialize from the top-level record plus `sync_segment_scopes` on
+    // read, so an in-flight 0.21 object loses no publication.
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS sync_segment_publications(
+        segment_id TEXT NOT NULL, pub_key TEXT NOT NULL,
+        partition TEXT NOT NULL, log_epoch TEXT NOT NULL,
+        tbl TEXT NOT NULL, schema_version INTEGER NOT NULL,
+        media_type TEXT NOT NULL, scope_digest TEXT NOT NULL,
+        as_of_commit_seq INTEGER NOT NULL, row_count INTEGER NOT NULL,
+        row_cursor TEXT, next_row_cursor TEXT,
+        byte_length INTEGER NOT NULL, created_at_ms INTEGER NOT NULL,
+        expires_at_ms INTEGER NOT NULL,
+        PRIMARY KEY (segment_id, pub_key)
+      );
+    `);
     const columns = this.db
       .query<{ name: string }, []>('PRAGMA table_info(sync_segments)')
       .all();
@@ -48,6 +82,64 @@ export class SqliteSegmentStore implements SegmentStore {
         "ALTER TABLE sync_segments ADD COLUMN log_epoch TEXT NOT NULL DEFAULT ''",
       );
     }
+    // Backfill digests recorded before the side table existed, so an
+    // upgraded store keeps `find`/download working for cached segments.
+    this.db.exec(
+      `INSERT OR IGNORE INTO sync_segment_scopes(segment_id, scope_digest)
+       SELECT segment_id, scope_digest FROM sync_segments`,
+    );
+  }
+
+  /** Every digest recorded for a content address, primary first. */
+  #scopeDigestsFor(segmentId: string, primary: string): string[] {
+    const rows = this.db
+      .query<{ scope_digest: string }, [string]>(
+        'SELECT scope_digest FROM sync_segment_scopes WHERE segment_id=?',
+      )
+      .all(segmentId);
+    const digests = rows.map((row) => row.scope_digest);
+    return [primary, ...digests.filter((digest) => digest !== primary)];
+  }
+
+  /** Stored publications for a content address, oldest first. */
+  #publicationsFor(segmentId: string): SegmentPublication[] {
+    const rows = this.db
+      .query<
+        {
+          partition: string;
+          log_epoch: string;
+          tbl: string;
+          schema_version: number;
+          media_type: string;
+          scope_digest: string;
+          as_of_commit_seq: number;
+          row_count: number;
+          row_cursor: string | null;
+          next_row_cursor: string | null;
+          byte_length: number;
+          created_at_ms: number;
+          expires_at_ms: number;
+        },
+        [string]
+      >(
+        'SELECT * FROM sync_segment_publications WHERE segment_id=? ORDER BY rowid',
+      )
+      .all(segmentId);
+    return rows.map((row) => ({
+      partition: row.partition,
+      logEpoch: row.log_epoch,
+      table: row.tbl,
+      schemaVersion: row.schema_version,
+      mediaType: row.media_type === 'sqlite' ? 'sqlite' : 'rows',
+      scopeDigest: row.scope_digest,
+      asOfCommitSeq: row.as_of_commit_seq,
+      rowCount: row.row_count,
+      rowCursor: row.row_cursor,
+      nextRowCursor: row.next_row_cursor,
+      byteLength: row.byte_length,
+      createdAtMs: row.created_at_ms,
+      expiresAtMs: row.expires_at_ms,
+    }));
   }
 
   async put(
@@ -56,38 +148,99 @@ export class SqliteSegmentStore implements SegmentStore {
     nowMs: number,
   ): Promise<SegmentRecord> {
     const segmentId = await segmentIdFor(bytes);
-    const record: SegmentRecord = {
-      ...metadata,
+    const existing = await this.get(segmentId);
+    if (existing !== undefined && !segmentBytesEqual(existing.bytes, bytes)) {
+      throw new Error(
+        `SqliteSegmentStore: content-address collision at ${segmentId} (§5.1)`,
+      );
+    }
+    const record = mergeSegmentRecord(
+      existing?.record,
+      metadata,
       segmentId,
-      byteLength: bytes.length,
-      createdAtMs: nowMs,
-      expiresAtMs: nowMs + this.#ttlMs,
-    };
-    this.db
-      .query(
-        `INSERT OR REPLACE INTO sync_segments(
-          segment_id, partition, log_epoch, tbl, schema_version, media_type,
-          scope_digest, as_of_commit_seq, row_count, row_cursor,
-          next_row_cursor, byte_length, created_at_ms, expires_at_ms, bytes
-        ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
-      )
-      .run(
-        record.segmentId,
-        record.partition,
-        record.logEpoch,
-        record.table,
-        record.schemaVersion,
-        record.mediaType,
-        record.scopeDigest,
-        record.asOfCommitSeq,
-        record.rowCount,
-        record.rowCursor,
-        record.nextRowCursor,
-        record.byteLength,
-        record.createdAtMs,
-        record.expiresAtMs,
-        bytes,
+      bytes.length,
+      nowMs,
+      this.#ttlMs,
+    );
+    // The bytes are proven identical to the stored ones, so replacing the
+    // row with the merged record (metadata, digest set, merged times) is the
+    // merge: the other digests live in `sync_segment_scopes` below. The
+    // publication set is rewritten from the merged record in one transaction
+    // so a crash cannot leave provenance half-applied.
+    this.db.exec('BEGIN IMMEDIATE');
+    try {
+      this.db
+        .query(
+          `INSERT OR REPLACE INTO sync_segments(
+            segment_id, partition, log_epoch, tbl, schema_version, media_type,
+            scope_digest, as_of_commit_seq, row_count, row_cursor,
+            next_row_cursor, byte_length, created_at_ms, expires_at_ms, bytes
+          ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
+        )
+        .run(
+          record.segmentId,
+          record.partition,
+          record.logEpoch,
+          record.table,
+          record.schemaVersion,
+          record.mediaType,
+          record.scopeDigest,
+          record.asOfCommitSeq,
+          record.rowCount,
+          record.rowCursor,
+          record.nextRowCursor,
+          record.byteLength,
+          record.createdAtMs,
+          record.expiresAtMs,
+          bytes,
+        );
+      // Rebuild the digest side table from the merged record so a pruned
+      // (expired) publication's digest disappears from the compatibility
+      // view instead of lingering as a stale scope row.
+      this.db
+        .query('DELETE FROM sync_segment_scopes WHERE segment_id=?')
+        .run(record.segmentId);
+      const insertScope = this.db.query(
+        `INSERT OR IGNORE INTO sync_segment_scopes(segment_id, scope_digest)
+         VALUES (?,?)`,
+      );
+      for (const digest of record.scopeDigests) {
+        insertScope.run(record.segmentId, digest);
+      }
+      this.db
+        .query('DELETE FROM sync_segment_publications WHERE segment_id=?')
+        .run(record.segmentId);
+      const insertPublication = this.db.query(
+        `INSERT INTO sync_segment_publications(
+           segment_id, pub_key, partition, log_epoch, tbl, schema_version,
+           media_type, scope_digest, as_of_commit_seq, row_count, row_cursor,
+           next_row_cursor, byte_length, created_at_ms, expires_at_ms
+         ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
       );
+      for (const publication of record.publications) {
+        insertPublication.run(
+          record.segmentId,
+          publicationKey(publication),
+          publication.partition,
+          publication.logEpoch,
+          publication.table,
+          publication.schemaVersion,
+          publication.mediaType,
+          publication.scopeDigest,
+          publication.asOfCommitSeq,
+          publication.rowCount,
+          publication.rowCursor,
+          publication.nextRowCursor,
+          publication.byteLength,
+          publication.createdAtMs,
+          publication.expiresAtMs,
+        );
+      }
+      this.db.exec('COMMIT');
+    } catch (error) {
+      this.db.exec('ROLLBACK');
+      throw error;
+    }
     return record;
   }
 
@@ -117,22 +270,33 @@ export class SqliteSegmentStore implements SegmentStore {
       >('SELECT * FROM sync_segments WHERE segment_id=?')
       .get(segmentId);
     if (row === null) return undefined;
+    const scopeDigests = this.#scopeDigestsFor(
+      row.segment_id,
+      row.scope_digest,
+    );
+    const top = {
+      segmentId: row.segment_id,
+      partition: row.partition,
+      logEpoch: row.log_epoch,
+      table: row.tbl,
+      schemaVersion: row.schema_version,
+      mediaType:
+        row.media_type === 'sqlite' ? ('sqlite' as const) : ('rows' as const),
+      scopeDigest: row.scope_digest,
+      scopeDigests,
+      asOfCommitSeq: row.as_of_commit_seq,
+      rowCount: row.row_count,
+      rowCursor: row.row_cursor,
+      nextRowCursor: row.next_row_cursor,
+      byteLength: row.byte_length,
+      createdAtMs: row.created_at_ms,
+      expiresAtMs: row.expires_at_ms,
+    };
+    const stored = this.#publicationsFor(row.segment_id);
     return {
       record: {
-        segmentId: row.segment_id,
-        partition: row.partition,
-        logEpoch: row.log_epoch,
-        table: row.tbl,
-        schemaVersion: row.schema_version,
-        mediaType: row.media_type === 'sqlite' ? 'sqlite' : 'rows',
-        scopeDigest: row.scope_digest,
-        asOfCommitSeq: row.as_of_commit_seq,
-        rowCount: row.row_count,
-        rowCursor: row.row_cursor,
-        nextRowCursor: row.next_row_cursor,
-        byteLength: row.byte_length,
-        createdAtMs: row.created_at_ms,
-        expiresAtMs: row.expires_at_ms,
+        ...top,
+        publications: stored.length > 0 ? stored : publicationsForRecord(top),
       },
       bytes: new Uint8Array(row.bytes),
     };
@@ -142,21 +306,14 @@ export class SqliteSegmentStore implements SegmentStore {
     key: SegmentFindKey,
     nowMs: number,
   ): Promise<SegmentRecord | undefined> {
-    const row = this.db
+    // A publication row is the provenance index; a record written before the
+    // table existed falls back to the legacy top-level/scope join below.
+    const publication = this.db
       .query<
-        {
-          segment_id: string;
-          row_count: number;
-          next_row_cursor: string | null;
-          byte_length: number;
-          created_at_ms: number;
-          expires_at_ms: number;
-        },
+        { segment_id: string },
         [string, string, string, number, string, string, number, number]
       >(
-        `SELECT segment_id, row_count, next_row_cursor, byte_length,
-                created_at_ms, expires_at_ms
-         FROM sync_segments
+        `SELECT segment_id FROM sync_segment_publications
          WHERE partition=? AND log_epoch=? AND tbl=? AND schema_version=? AND media_type=?
            AND scope_digest=? AND as_of_commit_seq=? AND row_cursor IS NULL
            AND expires_at_ms > ?
@@ -172,9 +329,34 @@ export class SqliteSegmentStore implements SegmentStore {
         key.asOfCommitSeq,
         nowMs,
       );
-    if (row === null) return undefined;
-    return {
-      segmentId: row.segment_id,
+    const legacy = this.db
+      .query<
+        { segment_id: string },
+        [string, string, string, number, string, string, number, number]
+      >(
+        `SELECT s.segment_id
+         FROM sync_segments s
+         JOIN sync_segment_scopes sc ON sc.segment_id = s.segment_id
+         WHERE s.partition=? AND s.log_epoch=? AND s.tbl=? AND s.schema_version=? AND s.media_type=?
+           AND sc.scope_digest=? AND s.as_of_commit_seq=? AND s.row_cursor IS NULL
+           AND s.expires_at_ms > ?
+         LIMIT 1`,
+      )
+      .get(
+        key.partition,
+        key.logEpoch,
+        key.table,
+        key.schemaVersion,
+        key.mediaType,
+        key.scopeDigest,
+        key.asOfCommitSeq,
+        nowMs,
+      );
+    const segmentId = publication?.segment_id ?? legacy?.segment_id;
+    if (segmentId === undefined) return undefined;
+    const entry = await this.get(segmentId);
+    if (entry === undefined) return undefined;
+    const match = selectSegmentPublication(entry.record, {
       partition: key.partition,
       logEpoch: key.logEpoch,
       table: key.table,
@@ -182,13 +364,11 @@ export class SqliteSegmentStore implements SegmentStore {
       mediaType: key.mediaType,
       scopeDigest: key.scopeDigest,
       asOfCommitSeq: key.asOfCommitSeq,
-      rowCount: row.row_count,
       rowCursor: null,
-      nextRowCursor: row.next_row_cursor,
-      byteLength: row.byte_length,
-      createdAtMs: row.created_at_ms,
-      expiresAtMs: row.expires_at_ms,
-    };
+      nowMs,
+    });
+    if (match === undefined) return undefined;
+    return publicationRecord(entry.record, match);
   }
 
   async stats(): Promise<SegmentStoreStats> {

package/src/sqlite-storage.ts

@@ -15,6 +15,7 @@ import {
 } from './authoritative-query';
 import { syncError } from './errors';
 import {
+  assertPhysicalColumns,
   commitWindowPageSql,
   deleteRowSql,
   deleteSqliteRowScopesSql,
@@ -481,6 +482,60 @@ export class SqliteServerStorage implements ServerStorage {
         `stored schema version ${marker.schema_version} is newer than the configured schema (${schema.version}) — refusing to run an older server against a migrated database`,
       );
     }
+    if (marker !== null && marker.schema_version === schema.version) {
+      // Version equality is not layout equality: a marker written by another
+      // build at the same version describes rows the running codec cannot
+      // decode. Compare the stored layouts instead of trusting the number.
+      const storedLayouts = parseLayouts(marker.layouts);
+      const configuredLayouts = parseLayouts(layoutsOf(schema));
+      let mismatch: string | undefined;
+      for (const [tableName, columns] of Object.entries(configuredLayouts)) {
+        const stored = storedLayouts[tableName];
+        const columnCount = Math.max(columns.length, stored?.length ?? 0);
+        for (let index = 0; index < columnCount; index++) {
+          const expected = columns[index];
+          const actual = stored?.[index];
+          if (
+            expected !== undefined &&
+            actual !== undefined &&
+            actual.name === expected.name &&
+            actual.type === expected.type &&
+            actual.nullable === expected.nullable
+          ) {
+            continue;
+          }
+          mismatch = `table ${JSON.stringify(tableName)} column ${JSON.stringify(expected?.name ?? actual?.name ?? '')}`;
+          break;
+        }
+        if (mismatch !== undefined) break;
+      }
+      if (mismatch === undefined) {
+        for (const tableName of Object.keys(storedLayouts)) {
+          if (!(tableName in configuredLayouts)) {
+            mismatch = `table ${JSON.stringify(tableName)}`;
+            break;
+          }
+        }
+      }
+      if (mismatch !== undefined) {
+        throw new Error(
+          `stored schema layouts disagree with the configured schema at version ${schema.version} (${mismatch}) — refusing to serve a database whose stored rows the running code cannot decode`,
+        );
+      }
+      // The persisted layouts describe the codec's app columns only: read the
+      // physical tables so a same-version database missing a
+      // storage-internal column is refused at startup instead of failing at
+      // the first write.
+      for (const table of schema.tables.values()) {
+        const escapedTableName = table.name.replaceAll('"', '""');
+        const columns = this.db
+          .query<{ name: string }, []>(
+            `PRAGMA table_info("${escapedTableName}")`,
+          )
+          .all();
+        assertPhysicalColumns(table, new Set(columns.map((c) => c.name)));
+      }
+    }
     if (marker === null || marker.schema_version < schema.version) {
       // Introspect existing app tables, then apply the migration subset
       // (CREATE TABLE / ADD COLUMN / rebuild indexes) to reach `schema`, then

package/src/storage.ts

@@ -458,6 +458,19 @@ export interface ServerStorage {
   /** Read continuity without refreshing authenticated activity. */
   getPartitionLogEpoch(partition: string): Promise<string | undefined>;
 
+  /**
+   * Open a storage transaction. This is the raw primitive, not a command
+   * boundary: it acquires NO partition write lock. In-tree SQLite begins
+   * `IMMEDIATE`, so a hand-rolled command on SQLite happens to be serialized
+   * against a concurrent push; PostgreSQL's `BEGIN` takes no row lock, so the
+   * same command races a push on Postgres and a SQLite-only test conceals it.
+   *
+   * A trusted command must therefore run through `registerRemoteCommand`,
+   * which builds after `lockPartitionForPush` and the idempotency recheck and
+   * shares the committed transaction, or call `lockPartitionForPush` before
+   * its first authority, version, or replay read. SSP2 request handling does
+   * this for you; only host-authored commands choose.
+   */
   begin(partition: string): Promise<StorageTransaction>;
 
   getMaxCommitSeq(partition: string): Promise<number>;
@@ -560,6 +573,18 @@ export interface ServerStorage {
    * Optional registered-query capability. The implementation MUST replace
    * every generated app-table relation with a partition-filtered relation and
    * return rows plus maxCommitSeq from one consistent database snapshot.
+   *
+   * The storage serializes this call behind an open transaction on a
+   * single-connection executor (SQLite, PGlite). A validator runs inside the
+   * push transaction, so on that executor a validator that calls
+   * `queryAuthoritative` waits on the transaction it is already inside and
+   * the push never completes. A pool-backed executor (`pg`, `Bun.sql`) hands
+   * the call a second connection instead: the validator does not deadlock,
+   * but it reads committed state without the candidate write. On either
+   * executor read candidate state through the whole-commit `CommitValidator`
+   * reader (§6.8). A call made from any other context waits for the open
+   * transaction to finish on a single-connection executor, and runs
+   * concurrently on a pool-backed one.
    */
   queryAuthoritative?(
     partition: string,

package/src/validate.ts

@@ -123,7 +123,19 @@ export type CommitValidator = (
   input: ValidateCommitInput,
 ) => void | Promise<void>;
 
-/** Ambient context a validator may consult (§6.7). */
+/**
+ * Ambient context a validator may consult (§6.7).
+ *
+ * A validator runs inside the push transaction. On a single-connection
+ * executor (SQLite, PGlite) the storage serializes authoritative queries
+ * behind that transaction, so a validator that calls `queryAuthoritative`
+ * waits for the transaction it is already inside and the push never
+ * completes. On a pool-backed executor (`pg`, `Bun.sql`) the query takes a
+ * second connection and returns committed state without the candidate write.
+ * Either way a rule that must read rows belongs in the whole-commit
+ * `CommitValidator`, whose reader runs get and scan operations on the same
+ * transaction (§6.8).
+ */
 export interface ValidateContext {
   /** Host-authenticated actor (§1.1) performing the write. */
   readonly actorId: string;
@@ -139,6 +151,13 @@ export interface ValidateContext {
  * any other error is treated as a rejection with a generic
  * `sync.constraint_violation` code (a validator SHOULD throw
  * `ValidationRejection` to control the code). MUST NOT mutate the row.
+ *
+ * The hook runs inside the push transaction and cannot read candidate rows
+ * from an authoritative query: on a single-connection executor
+ * `queryAuthoritative` serializes behind that transaction and the push never
+ * completes, and on a pool-backed executor it reads committed state without
+ * the candidate write. Read rows with the whole-commit `CommitValidator`
+ * reader instead (§6.8).
  */
 export type Validator = (
   op: ValidateOperation,