@syncular/server 0.22.0 → 0.24.0

package/src/storage-errors.ts

@@ -26,7 +26,15 @@ export type StorageQueryErrorCode =
   | 'sync.storage.invalid_limit'
   | 'sync.storage.prune_epoch_mismatch'
   | 'sync.storage.partition_unregistered'
-  | 'sync.storage.invalid_prune_cursor';
+  | 'sync.storage.invalid_prune_cursor'
+  | 'sync.storage.checkpoint_not_declared'
+  | 'sync.storage.checkpoint_unsupported'
+  | 'sync.storage.checkpoint_fence_missing'
+  | 'sync.storage.checkpoint_incomplete'
+  | 'sync.storage.stored_layout_mismatch'
+  | 'sync.storage.physical_layout_mismatch'
+  | 'sync.storage.transaction_query_unsupported'
+  | 'sync.storage.query_over_staged_writes';
 
 const STORAGE_QUERY_MESSAGES: Readonly<Record<StorageQueryErrorCode, string>> =
   {
@@ -44,6 +52,22 @@ const STORAGE_QUERY_MESSAGES: Readonly<Record<StorageQueryErrorCode, string>> =
       'pruning requires a registered partition',
     'sync.storage.invalid_prune_cursor':
       'pruning requires a non-negative safe integer cursor and a non-empty log epoch',
+    'sync.storage.checkpoint_not_declared':
+      'checkpoint claim requires a declared, not yet activated row',
+    'sync.storage.checkpoint_unsupported':
+      'this storage backend cannot declare backfill checkpoints',
+    'sync.storage.checkpoint_fence_missing':
+      'checkpoint activation requires the writer fence raised at declaration',
+    'sync.storage.checkpoint_incomplete':
+      'a declared backfill checkpoint is not activated and this caller did not declare it',
+    'sync.storage.stored_layout_mismatch':
+      'stored schema layouts disagree with the configured schema at the same version',
+    'sync.storage.physical_layout_mismatch':
+      'a synced table does not match the storage layout the running code reads and writes',
+    'sync.storage.transaction_query_unsupported':
+      'this storage transaction cannot run registered queries',
+    'sync.storage.query_over_staged_writes':
+      'registered query reads a table this transaction has buffered writes for',
     'sync.storage.scan_requires_scope':
       'scope-indexed row scans require at least one scope variable',
     'sync.storage.index_not_found':
@@ -58,15 +82,21 @@ const STORAGE_QUERY_MESSAGES: Readonly<Record<StorageQueryErrorCode, string>> =
 
 /**
  * Host-only query error. Messages never include identifiers, values, SQL,
- * paths, or row data; callers branch on `code`, never message text.
+ * paths, or row data; callers branch on `code`, never message text. The
+ * schema-readiness codes name the offending table and column in `details`.
  */
 export class StorageQueryError extends Error {
   override readonly name = 'StorageQueryError';
   readonly code: StorageQueryErrorCode;
+  readonly details: Readonly<Record<string, string>> | undefined;
 
-  constructor(code: StorageQueryErrorCode) {
+  constructor(
+    code: StorageQueryErrorCode,
+    details?: Readonly<Record<string, string>>,
+  ) {
     super(STORAGE_QUERY_MESSAGES[code]);
     this.code = code;
+    this.details = details;
   }
 }
 

package/src/storage-query.ts

@@ -1,6 +1,12 @@
 import type { ScopeMap } from '@syncular/core';
 import type { CompiledTable, IndexSchema } from './schema';
-import type { IndexRowScanQuery, RowScanQuery } from './storage';
+import type {
+  AuthoritativeQueryRequest,
+  AuthoritativeQueryResult,
+  IndexRowScanQuery,
+  RowScanQuery,
+  StorageTransaction,
+} from './storage';
 import { StorageQueryError } from './storage-errors';
 
 /**
@@ -51,3 +57,19 @@ export function resolveIndexRowScan(
   }
   return index;
 }
+
+/**
+ * §6.7/§6.8 transaction-bound registered query for every host hook that runs
+ * inside a push transaction. A storage transaction without the capability
+ * fails; the call never falls back to `ServerStorage.queryAuthoritative`,
+ * which reads outside the transaction.
+ */
+export async function transactionQuery(
+  tx: StorageTransaction,
+  query: AuthoritativeQueryRequest,
+): Promise<AuthoritativeQueryResult> {
+  if (tx.queryAuthoritative === undefined) {
+    throw new StorageQueryError('sync.storage.transaction_query_unsupported');
+  }
+  return tx.queryAuthoritative(query);
+}

package/src/storage.ts

@@ -31,6 +31,120 @@ export interface CommitPruneResult {
   readonly removedCommits: number;
 }
 
+/** RFC 0007 backfill lifecycle. `activated` is terminal within a schema version. */
+export type CheckpointState = 'declared' | 'backfilling' | 'activated';
+
+/**
+ * Host-owned checkpoint installation carried on `ensureSchema`: the schema
+ * bump, the `declared` row, and the raised writer fence commit together.
+ */
+export interface CheckpointDeclaration {
+  readonly partition: string;
+  readonly name: string;
+  readonly schemaVersion: number;
+}
+
+/** One host-storage backfill checkpoint row. Never a synced application row. */
+export interface StoredCheckpoint {
+  readonly partition: string;
+  readonly name: string;
+  readonly schemaVersion: number;
+  readonly state: CheckpointState;
+  /** Highest source `commit_seq` the projection was derived from. */
+  readonly watermark: number;
+  /** Monotonic claim counter; a batch is stale when it does not match. */
+  readonly ownerEpoch: number;
+  readonly observedRows: number;
+  readonly updatedAtMs: number;
+}
+
+/**
+ * RFC 0007 serve-gate token, read inside the transaction that produces the
+ * result. `storedSchemaVersion` is the published marker; `logEpoch` is the
+ * partition's log generation, so a restore that presents V -> other -> V is
+ * still distinguishable. `pending` names every incomplete declared checkpoint
+ * at the running schema version, across partitions.
+ */
+export interface ServeGate {
+  readonly storedSchemaVersion: number;
+  readonly logEpoch: string | undefined;
+  readonly pending: readonly {
+    readonly partition: string;
+    readonly name: string;
+  }[];
+}
+
+/** Why `serveGate` refuses, or `undefined` when this request may serve. */
+export type ServeGateRefusal =
+  | {
+      readonly kind: 'schema';
+      readonly stored: number;
+      readonly running: number;
+    }
+  | {
+      readonly kind: 'checkpoint';
+      readonly partition: string;
+      readonly name: string;
+    }
+  | {
+      /** The token changed between two reads: a restore rotated the epoch. */
+      readonly kind: 'epoch';
+    };
+
+/**
+ * Evaluate the RFC 0007 predicate against a gate token. A process that
+ * declares the pending checkpoint may serve while it backfills; any other
+ * process is refused, and a database newer than the running build is always
+ * refused.
+ */
+export function serveGateRefusal(
+  gate: ServeGate,
+  runningSchemaVersion: number,
+  checkpoints: readonly CheckpointDeclaration[] | undefined,
+): ServeGateRefusal | undefined {
+  if (gate.storedSchemaVersion > runningSchemaVersion) {
+    return {
+      kind: 'schema',
+      stored: gate.storedSchemaVersion,
+      running: runningSchemaVersion,
+    };
+  }
+  for (const pending of gate.pending) {
+    const declared = (checkpoints ?? []).some(
+      (declaration) =>
+        declaration.partition === pending.partition &&
+        declaration.name === pending.name,
+    );
+    if (!declared) {
+      return {
+        kind: 'checkpoint',
+        partition: pending.partition,
+        name: pending.name,
+      };
+    }
+  }
+  return undefined;
+}
+
+/**
+ * RFC 0007: compare two gate tokens read around a data read. The token binds
+ * the published schema version and the partition log epoch, so a restore that
+ * presents V → other → V is distinguishable. Returns a refusal when either
+ * changed across the read.
+ */
+export function serveGateTokenChanged(
+  before: ServeGate,
+  after: ServeGate,
+): ServeGateRefusal | undefined {
+  if (
+    before.storedSchemaVersion !== after.storedSchemaVersion ||
+    before.logEpoch !== after.logEpoch
+  ) {
+    return { kind: 'epoch' };
+  }
+  return undefined;
+}
+
 /** The current stored state of a synced row. */
 export interface StoredRow {
   readonly rowId: string;
@@ -359,6 +473,21 @@ export interface StorageTransaction {
    * read-your-own-writes semantics. It is not reachable from SSP2 requests.
    */
   scanRowsByIndex?(query: IndexRowScanQuery): Promise<StoredRow[]>;
+  /**
+   * Optional transaction-bound registered query (§6.7, §6.8): the relation
+   * rewrite of `ServerStorage.queryAuthoritative`, bound to this
+   * transaction's partition and executed on this transaction's connection.
+   * A hook inside the push transaction therefore reads the transaction's
+   * snapshot and never waits for a second connection. The push layer has
+   * already evaluated the serve gate on this transaction, so the call takes
+   * no checkpoint declarations. SQLite and PostgreSQL return the staged
+   * writes of this transaction. D1 buffers writes until commit and fails
+   * with `sync.storage.query_over_staged_writes` when the query reads a
+   * table this transaction has written.
+   */
+  queryAuthoritative?(
+    query: AuthoritativeQueryRequest,
+  ): Promise<AuthoritativeQueryResult>;
   /**
    * Serialize every push apply for this partition before any operation read,
    * validation, merge, or write. The push layer re-checks idempotency only
@@ -406,6 +535,27 @@ export interface StorageTransaction {
     clientCommitId: string,
     result: StoredPushResult,
   ): Promise<void>;
+  /**
+   * Advance this partition's checkpoint only while `owner_epoch` still
+   * matches, so the owner-epoch CAS rides inside the caller's transaction.
+   * The host calls this from `processPushOperationsWithTrace`'s operation
+   * builder: the backfill is an ordinary server-authoritative write, and this
+   * CAS commits with that write's row versions and commit-log append. False
+   * means abort and write nothing, rather than a throw for a superseded owner.
+   */
+  advanceCheckpoint(
+    name: string,
+    ownerEpoch: number,
+    watermark: number,
+    observedRows: number,
+    nowMs: number,
+  ): Promise<boolean>;
+  /**
+   * RFC 0007 serve gate on the caller's own connection, for write paths that
+   * already hold the partition lock: the gate and the write share one
+   * transaction, so a migration cannot interleave between them.
+   */
+  readServeGate(runningSchemaVersion: number): Promise<ServeGate>;
   /**
    * Blob reference index (§5.9.4) — ADDITIVE, optional. Set the blobIds a
    * row currently references (empty = clear), replacing any prior entries
@@ -438,7 +588,10 @@ export interface ServerStorage {
    * before binding a public port; protocol handlers still call this method
    * lazily as a defensive backstop.
    */
-  ensureSchema(schema: CompiledSchema): Promise<void>;
+  ensureSchema(
+    schema: CompiledSchema,
+    checkpoints?: readonly CheckpointDeclaration[],
+  ): Promise<void>;
 
   /** Create or refresh the authenticated partition registry row (§2.1). */
   touchPartition(
@@ -491,6 +644,84 @@ export interface ServerStorage {
     createdBeforeMs: number,
   ): Promise<number>;
 
+  /** Every declared backfill checkpoint for the partition, ordered by name. */
+  readCheckpoints(partition: string): Promise<StoredCheckpoint[]>;
+  /**
+   * RFC 0007 serve gate: read the published schema version, the partition's
+   * log epoch, and every incomplete declared checkpoint at
+   * `runningSchemaVersion`. Never memoized; the serve paths call it per
+   * request so an already-serving process notices a migration.
+   */
+  readServeGate(
+    partition: string,
+    runningSchemaVersion: number,
+  ): Promise<ServeGate>;
+  /**
+   * Atomically take ownership of a declared or backfilling checkpoint. Increments
+   * `owner_epoch` and moves `declared` to `backfilling`. Throws
+   * `sync.storage.checkpoint_not_declared` when no claimable row exists (absent,
+   * or already `activated`).
+   */
+  claimCheckpoint(
+    partition: string,
+    name: string,
+    schemaVersion: number,
+    nowMs: number,
+  ): Promise<StoredCheckpoint>;
+  /**
+   * Advance the watermark only while `owner_epoch` still matches. Returns false
+   * for a superseded owner; the stale case never throws.
+   */
+  advanceCheckpoint(
+    partition: string,
+    name: string,
+    ownerEpoch: number,
+    watermark: number,
+    observedRows: number,
+    nowMs: number,
+  ): Promise<boolean>;
+  /** Highest commitSeq carrying a change to one of `tables`; 0 when none. */
+  sourceCoverageSeq(
+    partition: string,
+    tables: readonly string[],
+  ): Promise<number>;
+  /**
+   * Whether any source change sits above `seq`. `unverifiable` when the pruning
+   * horizon has passed `seq`, so the history that would answer the question is
+   * gone; a pruned window is never reported as `clean`.
+   */
+  hasSourceChangesAbove(
+    partition: string,
+    tables: readonly string[],
+    seq: number,
+  ): Promise<'clean' | 'changed' | 'unverifiable'>;
+  /**
+   * Install the barrier: insert the checkpoint in `declared` and raise
+   * `required_writer_version` for the partition in one transaction. The fence
+   * is raised at declaration, before any backfill work, so old writers are
+   * rejected for the whole backfill window. Idempotent on an existing declared
+   * or backfilling row.
+   */
+  declareCheckpoint(
+    partition: string,
+    name: string,
+    schemaVersion: number,
+    nowMs: number,
+  ): Promise<StoredCheckpoint>;
+  /**
+   * Activate a declared checkpoint under the partition write lock. Refuses when
+   * the fence is not already raised at the checkpoint's schema version, when a
+   * source change sits above `watermark` (`stale`), or when the pruning horizon
+   * has passed `watermark` (`unverifiable`). `activated` is terminal.
+   */
+  activateCheckpoint(
+    partition: string,
+    name: string,
+    ownerEpoch: number,
+    watermark: number,
+    sources: readonly string[],
+    nowMs: number,
+  ): Promise<'activated' | 'stale' | 'unverifiable'>;
   getRow(
     partition: string,
     table: string,
@@ -574,21 +805,23 @@ export interface ServerStorage {
    * every generated app-table relation with a partition-filtered relation and
    * return rows plus maxCommitSeq from one consistent database snapshot.
    *
-   * The storage serializes this call behind an open transaction on a
-   * single-connection executor (SQLite, PGlite). A validator runs inside the
-   * push transaction, so on that executor a validator that calls
-   * `queryAuthoritative` waits on the transaction it is already inside and
-   * the push never completes. A pool-backed executor (`pg`, `Bun.sql`) hands
-   * the call a second connection instead: the validator does not deadlock,
-   * but it reads committed state without the candidate write. On either
-   * executor read candidate state through the whole-commit `CommitValidator`
-   * reader (§6.8). A call made from any other context waits for the open
-   * transaction to finish on a single-connection executor, and runs
-   * concurrently on a pool-backed one.
+   * Do not call this method from a hook that runs inside the push
+   * transaction (a row validator, the whole-commit validator, or a reaction
+   * planner). On a single-connection executor (SQLite, PGlite) the storage
+   * serializes this call behind the open transaction, so the hook waits on
+   * the transaction it is inside and the push never completes. A pool-backed
+   * executor (`pg`, `Bun.sql`) runs the call on a second connection: it reads
+   * committed state without the transaction's staged writes, and it waits
+   * forever when the push transactions hold every pool connection. Those
+   * hooks call the transaction-bound `queryAuthoritative` on
+   * `ValidateContext` or on the candidate-state reader instead (§6.7, §6.8).
+   * A call from any other context waits for the open transaction on a
+   * single-connection executor and runs concurrently on a pool-backed one.
    */
   queryAuthoritative?(
     partition: string,
     query: AuthoritativeQueryRequest,
+    checkpoints?: readonly CheckpointDeclaration[],
   ): Promise<AuthoritativeQueryResult>;
 
   /**

package/src/validate.ts

@@ -20,6 +20,10 @@ import {
   type RowValue,
   type ScopeMap,
 } from '@syncular/core';
+import type {
+  AuthoritativeQueryRequest,
+  AuthoritativeQueryResult,
+} from './storage';
 
 /**
  * §6.7 reserved code prefixes. A host validator code MUST NOT start with
@@ -104,6 +108,17 @@ export interface CommitValidationReader {
     rowId: string,
   ): Promise<CommitValidationRow | undefined>;
   scanRows(input: CommitValidationScanInput): Promise<CommitValidationRow[]>;
+  /**
+   * Run a generated registered query on the commit transaction (§6.8). The
+   * request has the shape `ServerStorage.queryAuthoritative` takes; the
+   * storage binds every relation to the commit's partition. SQLite and
+   * PostgreSQL return the final candidate state; D1 fails with
+   * `sync.storage.query_over_staged_writes` when the query reads a table the
+   * commit has written.
+   */
+  queryAuthoritative(
+    query: AuthoritativeQueryRequest,
+  ): Promise<AuthoritativeQueryResult>;
 }
 
 export interface ValidateCommitInput {
@@ -126,21 +141,31 @@ export type CommitValidator = (
 /**
  * Ambient context a validator may consult (§6.7).
  *
- * A validator runs inside the push transaction. On a single-connection
- * executor (SQLite, PGlite) the storage serializes authoritative queries
- * behind that transaction, so a validator that calls `queryAuthoritative`
- * waits for the transaction it is already inside and the push never
- * completes. On a pool-backed executor (`pg`, `Bun.sql`) the query takes a
- * second connection and returns committed state without the candidate write.
- * Either way a rule that must read rows belongs in the whole-commit
- * `CommitValidator`, whose reader runs get and scan operations on the same
- * transaction (§6.8).
+ * A validator runs inside the push transaction. It MUST NOT call
+ * `ServerStorage.queryAuthoritative`: on a single-connection executor
+ * (SQLite, PGlite) that call waits for the transaction the validator is
+ * inside and the push never completes, and on a pool-backed executor (`pg`,
+ * `Bun.sql`) it reads committed state without the staged writes, or waits
+ * forever once push transactions hold every pool connection. An authority
+ * read calls `queryAuthoritative` on this context instead.
  */
 export interface ValidateContext {
   /** Host-authenticated actor (§1.1) performing the write. */
   readonly actorId: string;
   /** The partition (§1.1) the commit targets. */
   readonly partition: string;
+  /**
+   * Run a generated registered query on the push transaction (§6.7). The
+   * request has the shape `ServerStorage.queryAuthoritative` takes; the
+   * storage binds every relation to the commit's partition. SQLite and
+   * PostgreSQL return the transaction's snapshot, including the operations
+   * staged before this one; D1 fails with
+   * `sync.storage.query_over_staged_writes` when the query reads a table the
+   * commit has written.
+   */
+  readonly queryAuthoritative: (
+    query: AuthoritativeQueryRequest,
+  ) => Promise<AuthoritativeQueryResult>;
 }
 
 /**
@@ -152,12 +177,12 @@ export interface ValidateContext {
  * `sync.constraint_violation` code (a validator SHOULD throw
  * `ValidationRejection` to control the code). MUST NOT mutate the row.
  *
- * The hook runs inside the push transaction and cannot read candidate rows
- * from an authoritative query: on a single-connection executor
- * `queryAuthoritative` serializes behind that transaction and the push never
- * completes, and on a pool-backed executor it reads committed state without
- * the candidate write. Read rows with the whole-commit `CommitValidator`
- * reader instead (§6.8).
+ * The hook runs inside the push transaction. An authority read uses
+ * `ctx.queryAuthoritative`, which executes on that transaction; calling
+ * `ServerStorage.queryAuthoritative` from here deadlocks on a
+ * single-connection executor and reads committed state on a pool-backed one.
+ * A rule over sibling operations belongs in the whole-commit
+ * `CommitValidator` (§6.8).
  */
 export type Validator = (
   op: ValidateOperation,