@syncular/server 0.22.0 → 0.24.0

package/src/push.ts

@@ -52,7 +52,10 @@ import type {
   StoredPushResult,
   StoredRow,
 } from './storage';
+import { serveGateRefusal } from './storage';
 import { StorageConstraintError } from './storage-errors';
+import { transactionQuery } from './storage-query';
+import { serveNotReadyError } from './readiness';
 import type {
   CommitValidationReader,
   CommitValidator,
@@ -184,6 +187,7 @@ async function runValidator(
   values: readonly RowValue[] | undefined,
   storedValues: readonly RowValue[] | undefined,
   opIndex: number,
+  tx: StorageTransaction,
   partition: string,
   actorId: string,
 ): Promise<TerminatingOutcome | undefined> {
@@ -204,7 +208,11 @@ async function runValidator(
             ? toValidateRow(table.columns, storedValues)
             : undefined,
       },
-      { actorId, partition },
+      {
+        actorId,
+        partition,
+        queryAuthoritative: (query) => transactionQuery(tx, query),
+      },
     );
   } catch (error) {
     if (error instanceof ValidationRejection) {
@@ -361,6 +369,7 @@ async function applyOperation(
       undefined,
       storedValues,
       opIndex,
+      tx,
       partition,
       actorId,
     );
@@ -551,6 +560,7 @@ async function applyOperation(
       values,
       storedValues,
       opIndex,
+      tx,
       partition,
       actorId,
     );
@@ -681,6 +691,7 @@ async function applyOperation(
     values,
     undefined,
     opIndex,
+    tx,
     partition,
     actorId,
   );
@@ -834,6 +845,7 @@ function commitValidationReader(
         serverVersion: stored.serverVersion,
       }));
     },
+    queryAuthoritative: (query) => transactionQuery(tx, query),
   };
 }
 
@@ -1082,6 +1094,7 @@ async function enforceReferences(
             undefined,
             storedValues,
             item.originOpIndex,
+            tx,
             partition,
             actorId,
           );
@@ -1115,6 +1128,7 @@ async function enforceReferences(
             values,
             storedValues,
             item.originOpIndex,
+            tx,
             partition,
             actorId,
           );
@@ -1356,6 +1370,15 @@ export async function processPushOperationsWithTrace(
       );
     }
     await lockPartitionForPush();
+    // RFC 0007: evaluate the gate on this transaction's own connection while
+    // the partition lock is held, so the migration check and the write share
+    // one transaction and no migration can interleave in between.
+    const gate = await tx.readServeGate(schema.version);
+    const refusal = serveGateRefusal(gate, schema.version, ctx.checkpoints);
+    if (refusal !== undefined) {
+      await tx.rollback();
+      throw serveNotReadyError(refusal);
+    }
     // The optimistic lookup above may have raced another delivery. Re-check
     // only after acquiring partition serialization and before any operation
     // read, validation, merge, or staged write. The re-check runs on the
@@ -1551,6 +1574,13 @@ export async function processPushOperationsWithTrace(
       }
     }
     if (ctx.realtime !== undefined && changes.length > 0) {
+      // RFC 0007 fanout decision: the notification carries the durable commit
+      // entry itself (this commit's `changes`), not a projection read or a pull
+      // window, so fanning it out before the request's buffered read-verify can
+      // never serve a mixed result. Each peer's next pull is gated at the
+      // storage seam; a refused mixed push+pull is safe because the push is
+      // durable and replayable under its commit id. This is commit-entry
+      // durability, not gate inheritance.
       await ctx.realtime.notifyCommit(partition, {
         commitSeq,
         createdAtMs,

package/src/readiness.ts

@@ -7,12 +7,17 @@
  * a structured failure that cannot be confused with request authentication.
  */
 import type { SyncServerConfig } from './context';
+import { type SyncError, syncError } from './errors';
 import { type CompiledSchema, compileSchema } from './schema';
+import type { ServeGateRefusal } from './storage';
 
 export const SYNC_SERVER_READINESS_ERROR_CODE =
   'sync.schema_not_ready' as const;
 
-export type SyncServerReadinessPhase = 'schema_compile' | 'storage_migration';
+export type SyncServerReadinessPhase =
+  | 'schema_compile'
+  | 'storage_migration'
+  | 'backfill_checkpoint';
 
 export class SyncServerReadinessError extends Error {
   override readonly name = 'SyncServerReadinessError';
@@ -42,7 +47,7 @@ export class SyncServerReadinessError extends Error {
  * `cause` locally for the table/column or storage diagnostic.
  */
 export async function ensureSyncServerReady(
-  config: Pick<SyncServerConfig, 'schema' | 'storage'>,
+  config: Pick<SyncServerConfig, 'schema' | 'storage' | 'checkpoints'>,
 ): Promise<void> {
   let compiled: CompiledSchema;
   try {
@@ -55,12 +60,42 @@ export async function ensureSyncServerReady(
     });
   }
   try {
-    await config.storage.ensureSchema(compiled);
+    await config.storage.ensureSchema(compiled, config.checkpoints);
   } catch (cause) {
+    // RFC 0007: a declared-but-incomplete checkpoint is a barrier refusal, not
+    // an ordinary migration failure — hosts distinguish it by `phase` while the
+    // client-visible identity stays `sync.schema_not_ready`.
+    const phase: SyncServerReadinessPhase =
+      (cause as { code?: unknown } | null | undefined)?.code ===
+      'sync.storage.checkpoint_incomplete'
+        ? 'backfill_checkpoint'
+        : 'storage_migration';
     throw new SyncServerReadinessError({
-      phase: 'storage_migration',
+      phase,
       schemaVersion: config.schema.version,
       cause,
     });
   }
 }
+
+/**
+ * RFC 0007: refuse a serve path that observed a closed gate. The projection
+ * name travels in structured details, never in the message.
+ */
+export function serveNotReadyError(refusal: ServeGateRefusal): SyncError {
+  return syncError(
+    SYNC_SERVER_READINESS_ERROR_CODE,
+    'server schema is not ready for this request',
+    refusal.kind === 'checkpoint'
+      ? JSON.stringify({
+          partition: refusal.partition,
+          projection: refusal.name,
+        })
+      : refusal.kind === 'epoch'
+        ? JSON.stringify({ logEpochChanged: true })
+        : JSON.stringify({
+            storedSchemaVersion: refusal.stored,
+            runningSchemaVersion: refusal.running,
+          }),
+  );
+}

package/src/realtime.ts

@@ -600,7 +600,28 @@ export class RealtimeSession {
       // begin its next round as soon as END *arrives* (§8.7 rule 3), so
       // in-flight must end with the send, not with generator cleanup.
       const iterator = stream[Symbol.asyncIterator]();
-      let step = await iterator.next();
+      let step: IteratorResult<Uint8Array>;
+      try {
+        step = await iterator.next();
+      } catch (error) {
+        // The buffered read-verify refusal is thrown on the generator's first
+        // `next()`, before any chunk is yielded (the generator builds the whole
+        // response, then verifies). Nothing has escaped, so it is delivered as
+        // the same §8.4 ERROR frame HTTP returns rather than a silent close. A
+        // throw after the first chunk still takes the §8.7 violation path.
+        if (error instanceof DecodeError || error instanceof SyncError) {
+          const sync =
+            error instanceof SyncError
+              ? error
+              : syncError(error.code, error.message);
+          finishRound();
+          await this.#sendRoundChunk(
+            errorResponseBytes(sync, this.wireVersion, this.logEpoch),
+          );
+          return;
+        }
+        throw error;
+      }
       while (!step.done) {
         const chunk = step.value;
         step = await iterator.next();

package/src/relational-rows.ts

@@ -55,6 +55,7 @@ import {
 } from '@syncular/core';
 import type { CompiledSchema, CompiledTable, IndexSchema } from './schema';
 import type { StoredRow } from './storage';
+import { StorageQueryError } from './storage-errors';
 
 export type RelationalDialect = 'sqlite' | 'postgres';
 
@@ -192,38 +193,112 @@ export function tableColumnNames(table: CompiledTable): string[] {
 }
 
 /**
- * The physical column set the row codec and the scope index read for one
- * configured table. The persisted marker's layouts describe the codec's app
+ * The storage-internal columns of every synced table, in `createTableDdl`
+ * order (app columns sit between the row id and the version). The same-version
+ * physical check compares a table against exactly these declarations.
+ */
+function syncMetaColumns(dialect: RelationalDialect): readonly {
+  readonly name: string;
+  readonly type: string;
+  readonly notNull: boolean;
+}[] {
+  const payloadType = dialect === 'postgres' ? 'BYTEA' : 'BLOB';
+  return [
+    { name: SYNC_PARTITION_COLUMN, type: 'TEXT', notNull: true },
+    { name: SYNC_ROW_ID_COLUMN, type: 'TEXT', notNull: true },
+    {
+      name: SYNC_VERSION_COLUMN,
+      type: dialect === 'postgres' ? 'BIGINT' : 'INTEGER',
+      notNull: true,
+    },
+    {
+      name: SYNC_SCOPES_COLUMN,
+      type: dialect === 'postgres' ? 'JSONB' : 'TEXT',
+      notNull: true,
+    },
+    { name: SYNC_PAYLOAD_COLUMN, type: payloadType, notNull: true },
+    { name: SYNC_COLUMN_VERSIONS_COLUMN, type: payloadType, notNull: false },
+  ];
+}
+
+/** One physical column of a synced table, read from the database catalog. */
+export interface PhysicalColumn {
+  readonly name: string;
+  /** Declared type (SQLite) or `format_type` (PostgreSQL); compared case-insensitively. */
+  readonly type: string;
+  readonly notNull: boolean;
+  /** 1-based position in the primary key; 0 when the column is not in it. */
+  readonly primaryKeyPosition: number;
+}
+
+/**
+ * Refuse a same-version database whose physical table the row store cannot
+ * read or write. The persisted marker's layouts describe the codec's app
  * columns only and its version number is the schema's, not the database's, so
- * a same-version database that is missing a storage-internal meta column (a
- * version-only bump whose ALTER never ran) still has to be refused: compare
- * the physical table and fail closed without writing a migration.
+ * a same-version database whose storage-internal meta columns drifted (a
+ * version-only bump whose ALTER never ran, or a table written by another
+ * build) is only visible in the catalog. Every configured column must exist;
+ * the meta columns must also carry the type, nullability, and primary-key
+ * position `createTableDdl` declares. App-column types and nullability stay
+ * with the stored layouts: migration-added app columns are nullable by design.
+ * Fails closed without writing a migration.
  *
- * `existingColumns` must be resolved the way the unqualified row queries
- * resolve the table — on PostgreSQL, through the session `search_path`, not
+ * `columns` must be resolved the way the unqualified row queries resolve the
+ * table: on PostgreSQL, through the session `search_path`, not
  * `current_schema()`. A search path whose first schema is not where the
  * configured tables live is outside the supported contract: DDL is created in
  * that first schema, so `ensureSchema` already writes shadow tables there.
  *
- * Scope: this checks that the columns EXIST. Stored types, nullability, and
- * non-column storage internals (the tombstone bookkeeping) stay outside it,
- * because a wrong type or nullability is a codec-compatibility question the
- * persisted layouts answer for app columns.
+ * Core storage tables (`sync_tombstones`, `sync_commits`, ...) are outside
+ * this check: every open re-applies their `CREATE TABLE IF NOT EXISTS` DDL
+ * and column additions, independently of the version marker.
  */
 export function assertPhysicalColumns(
   table: CompiledTable,
-  existingColumns: ReadonlySet<string>,
+  columns: readonly PhysicalColumn[],
+  dialect: RelationalDialect,
 ): void {
-  if (existingColumns.size === 0) {
-    throw new Error(
-      `table ${JSON.stringify(table.name)} is missing from the database at the stored schema version — refusing to serve a database whose rows the running code cannot read`,
-    );
-  }
+  const refuse = (details: Record<string, string>): never => {
+    throw new StorageQueryError('sync.storage.physical_layout_mismatch', {
+      table: table.name,
+      ...details,
+    });
+  };
+  if (columns.length === 0) refuse({ reason: 'missing_table' });
+  const byName = new Map(columns.map((column) => [column.name, column]));
   for (const name of tableColumnNames(table)) {
-    if (existingColumns.has(name)) continue;
-    throw new Error(
-      `table ${JSON.stringify(table.name)} is missing column ${JSON.stringify(name)} at the stored schema version — refusing to serve a database whose rows the running code cannot read`,
-    );
+    if (!byName.has(name)) refuse({ column: name, reason: 'missing_column' });
+  }
+  for (const expected of syncMetaColumns(dialect)) {
+    const actual = byName.get(expected.name);
+    if (actual === undefined) continue;
+    if (actual.type.toUpperCase() !== expected.type) {
+      refuse({
+        column: expected.name,
+        reason: 'type',
+        expected: expected.type,
+        actual: actual.type,
+      });
+    }
+    if (actual.notNull !== expected.notNull) {
+      refuse({
+        column: expected.name,
+        reason: 'nullability',
+        expected: expected.notNull ? 'not_null' : 'nullable',
+        actual: actual.notNull ? 'not_null' : 'nullable',
+      });
+    }
+  }
+  for (const column of columns) {
+    const expected =
+      column.name === SYNC_PARTITION_COLUMN
+        ? 1
+        : column.name === SYNC_ROW_ID_COLUMN
+          ? 2
+          : 0;
+    if (column.primaryKeyPosition !== expected) {
+      refuse({ column: column.name, reason: 'primary_key' });
+    }
   }
 }
 
@@ -232,22 +307,20 @@ export function createTableDdl(
   table: CompiledTable,
   dialect: RelationalDialect,
 ): string {
-  const versionType = dialect === 'postgres' ? 'BIGINT' : 'INTEGER';
-  const scopesType = dialect === 'postgres' ? 'JSONB' : 'TEXT';
-  const payloadType = dialect === 'postgres' ? 'BYTEA' : 'BLOB';
+  const [partition, rowId, ...trailing] = syncMetaColumns(dialect).map(
+    (column) =>
+      `${quoteIdent(column.name)} ${column.type}${column.notNull ? ' NOT NULL' : ''}`,
+  );
   const defs = [
-    `${quoteIdent(SYNC_PARTITION_COLUMN)} TEXT NOT NULL`,
-    `${quoteIdent(SYNC_ROW_ID_COLUMN)} TEXT NOT NULL`,
+    partition,
+    rowId,
     ...(table.materialize
       ? table.columns.map((column) => {
           const notNull = column.nullable ? '' : ' NOT NULL';
           return `${quoteIdent(column.name)} ${columnSqlType(column, dialect)}${notNull}`;
         })
       : []),
-    `${quoteIdent(SYNC_VERSION_COLUMN)} ${versionType} NOT NULL`,
-    `${quoteIdent(SYNC_SCOPES_COLUMN)} ${scopesType} NOT NULL`,
-    `${quoteIdent(SYNC_PAYLOAD_COLUMN)} ${payloadType} NOT NULL`,
-    `${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} ${payloadType}`,
+    ...trailing,
     `PRIMARY KEY (${quoteIdent(SYNC_PARTITION_COLUMN)}, ${quoteIdent(SYNC_ROW_ID_COLUMN)})`,
   ];
   return `CREATE TABLE IF NOT EXISTS ${quoteIdent(table.name)} (${defs.join(', ')})`;
@@ -660,6 +733,48 @@ export function parseLayouts(json: string | null | undefined): StoredLayouts {
   return JSON.parse(json) as StoredLayouts;
 }
 
+/**
+ * Version equality is not layout equality: a marker written by another build
+ * at the same version describes rows the running codec cannot decode. Refuse
+ * when the stored layouts disagree with the configured schema, naming the
+ * first table (and column) that differs.
+ */
+export function assertStoredLayouts(
+  schema: CompiledSchema,
+  storedJson: string | null | undefined,
+): void {
+  const storedLayouts = parseLayouts(storedJson);
+  const configuredLayouts = parseLayouts(layoutsOf(schema));
+  for (const [tableName, columns] of Object.entries(configuredLayouts)) {
+    const stored = storedLayouts[tableName];
+    const columnCount = Math.max(columns.length, stored?.length ?? 0);
+    for (let index = 0; index < columnCount; index++) {
+      const expected = columns[index];
+      const actual = stored?.[index];
+      if (
+        expected !== undefined &&
+        actual !== undefined &&
+        actual.name === expected.name &&
+        actual.type === expected.type &&
+        actual.nullable === expected.nullable
+      ) {
+        continue;
+      }
+      throw new StorageQueryError('sync.storage.stored_layout_mismatch', {
+        table: tableName,
+        column: expected?.name ?? actual?.name ?? '',
+      });
+    }
+  }
+  for (const tableName of Object.keys(storedLayouts)) {
+    if (!(tableName in configuredLayouts)) {
+      throw new StorageQueryError('sync.storage.stored_layout_mismatch', {
+        table: tableName,
+      });
+    }
+  }
+}
+
 /**
  * Tables present in the stored layout but absent from the configured head
  * schema. A schema-version bump retires their relational current-row tables;

package/src/segment-download.ts

@@ -196,6 +196,22 @@ async function downloadSegment(
   }
   const publication = candidates[candidates.length - 1]!;
 
+  // RFC 0007: a descriptor minted before this partition declared a backfill
+  // checkpoint is not a complete window once the projection becomes
+  // checkpointed. SPEC §9 mints segments per schema version, so refuse and
+  // let the client re-pull for a fresh descriptor.
+  const checkpoints = await ctx.storage.readCheckpoints(ctx.partition);
+  if (
+    checkpoints.some(
+      (checkpoint) => checkpoint.schemaVersion > publication.schemaVersion,
+    )
+  ) {
+    throw syncError(
+      'sync.segment_expired',
+      'segment predates the partition backfill checkpoint — re-pull (§5.1)',
+    );
+  }
+
   return {
     // The returned record is projected onto the selected publication, so its
     // `scopeDigests`/`publications` compatibility view cannot union a grant

package/src/sqlite-dialect.ts

@@ -66,6 +66,7 @@ CREATE TABLE IF NOT EXISTS sync_commits(
   partition TEXT NOT NULL, commit_seq INTEGER NOT NULL,
   client_id TEXT NOT NULL, client_commit_id TEXT NOT NULL,
   actor_id TEXT NOT NULL, created_at_ms INTEGER NOT NULL,
+  writer_version INTEGER,
   PRIMARY KEY(partition, commit_seq)
 );
 CREATE INDEX IF NOT EXISTS sync_commits_by_time
@@ -127,6 +128,20 @@ CREATE TABLE IF NOT EXISTS sync_tombstones(
   commit_seq INTEGER NOT NULL,
   PRIMARY KEY(partition, tbl, row_id)
 );
+CREATE TABLE IF NOT EXISTS sync_backfill_checkpoints(
+  partition TEXT NOT NULL, name TEXT NOT NULL,
+  schema_version INTEGER NOT NULL,
+  state TEXT NOT NULL CHECK(state IN ('declared','backfilling','activated')),
+  watermark INTEGER NOT NULL DEFAULT 0,
+  owner_epoch INTEGER NOT NULL DEFAULT 0,
+  observed_rows INTEGER NOT NULL DEFAULT 0,
+  updated_at_ms INTEGER NOT NULL,
+  PRIMARY KEY(partition, name)
+);
+CREATE TABLE IF NOT EXISTS sync_writer_fence(
+  partition TEXT PRIMARY KEY,
+  required_writer_version INTEGER NOT NULL
+);
 `;
 
 /** Split the DDL into individual statements (D1 applies them one by one). */