@syncular/server 0.22.0 → 0.24.0

package/dist/storage.d.ts

@@ -28,6 +28,72 @@ export interface CommitPruneResult {
     readonly horizonSeq: number;
     readonly removedCommits: number;
 }
+/** RFC 0007 backfill lifecycle. `activated` is terminal within a schema version. */
+export type CheckpointState = 'declared' | 'backfilling' | 'activated';
+/**
+ * Host-owned checkpoint installation carried on `ensureSchema`: the schema
+ * bump, the `declared` row, and the raised writer fence commit together.
+ */
+export interface CheckpointDeclaration {
+    readonly partition: string;
+    readonly name: string;
+    readonly schemaVersion: number;
+}
+/** One host-storage backfill checkpoint row. Never a synced application row. */
+export interface StoredCheckpoint {
+    readonly partition: string;
+    readonly name: string;
+    readonly schemaVersion: number;
+    readonly state: CheckpointState;
+    /** Highest source `commit_seq` the projection was derived from. */
+    readonly watermark: number;
+    /** Monotonic claim counter; a batch is stale when it does not match. */
+    readonly ownerEpoch: number;
+    readonly observedRows: number;
+    readonly updatedAtMs: number;
+}
+/**
+ * RFC 0007 serve-gate token, read inside the transaction that produces the
+ * result. `storedSchemaVersion` is the published marker; `logEpoch` is the
+ * partition's log generation, so a restore that presents V -> other -> V is
+ * still distinguishable. `pending` names every incomplete declared checkpoint
+ * at the running schema version, across partitions.
+ */
+export interface ServeGate {
+    readonly storedSchemaVersion: number;
+    readonly logEpoch: string | undefined;
+    readonly pending: readonly {
+        readonly partition: string;
+        readonly name: string;
+    }[];
+}
+/** Why `serveGate` refuses, or `undefined` when this request may serve. */
+export type ServeGateRefusal = {
+    readonly kind: 'schema';
+    readonly stored: number;
+    readonly running: number;
+} | {
+    readonly kind: 'checkpoint';
+    readonly partition: string;
+    readonly name: string;
+} | {
+    /** The token changed between two reads: a restore rotated the epoch. */
+    readonly kind: 'epoch';
+};
+/**
+ * Evaluate the RFC 0007 predicate against a gate token. A process that
+ * declares the pending checkpoint may serve while it backfills; any other
+ * process is refused, and a database newer than the running build is always
+ * refused.
+ */
+export declare function serveGateRefusal(gate: ServeGate, runningSchemaVersion: number, checkpoints: readonly CheckpointDeclaration[] | undefined): ServeGateRefusal | undefined;
+/**
+ * RFC 0007: compare two gate tokens read around a data read. The token binds
+ * the published schema version and the partition log epoch, so a restore that
+ * presents V → other → V is distinguishable. Returns a refusal when either
+ * changed across the read.
+ */
+export declare function serveGateTokenChanged(before: ServeGate, after: ServeGate): ServeGateRefusal | undefined;
 /** The current stored state of a synced row. */
 export interface StoredRow {
     readonly rowId: string;
@@ -315,6 +381,19 @@ export interface StorageTransaction {
      * read-your-own-writes semantics. It is not reachable from SSP2 requests.
      */
     scanRowsByIndex?(query: IndexRowScanQuery): Promise<StoredRow[]>;
+    /**
+     * Optional transaction-bound registered query (§6.7, §6.8): the relation
+     * rewrite of `ServerStorage.queryAuthoritative`, bound to this
+     * transaction's partition and executed on this transaction's connection.
+     * A hook inside the push transaction therefore reads the transaction's
+     * snapshot and never waits for a second connection. The push layer has
+     * already evaluated the serve gate on this transaction, so the call takes
+     * no checkpoint declarations. SQLite and PostgreSQL return the staged
+     * writes of this transaction. D1 buffers writes until commit and fails
+     * with `sync.storage.query_over_staged_writes` when the query reads a
+     * table this transaction has written.
+     */
+    queryAuthoritative?(query: AuthoritativeQueryRequest): Promise<AuthoritativeQueryResult>;
     /**
      * Serialize every push apply for this partition before any operation read,
      * validation, merge, or write. The push layer re-checks idempotency only
@@ -352,6 +431,21 @@ export interface StorageTransaction {
      * first writer wins; callers read the canonical value after commit.
      */
     putPushResult(clientId: string, clientCommitId: string, result: StoredPushResult): Promise<void>;
+    /**
+     * Advance this partition's checkpoint only while `owner_epoch` still
+     * matches, so the owner-epoch CAS rides inside the caller's transaction.
+     * The host calls this from `processPushOperationsWithTrace`'s operation
+     * builder: the backfill is an ordinary server-authoritative write, and this
+     * CAS commits with that write's row versions and commit-log append. False
+     * means abort and write nothing, rather than a throw for a superseded owner.
+     */
+    advanceCheckpoint(name: string, ownerEpoch: number, watermark: number, observedRows: number, nowMs: number): Promise<boolean>;
+    /**
+     * RFC 0007 serve gate on the caller's own connection, for write paths that
+     * already hold the partition lock: the gate and the write share one
+     * transaction, so a migration cannot interleave between them.
+     */
+    readServeGate(runningSchemaVersion: number): Promise<ServeGate>;
     /**
      * Blob reference index (§5.9.4) — ADDITIVE, optional. Set the blobIds a
      * row currently references (empty = clear), replacing any prior entries
@@ -379,7 +473,7 @@ export interface ServerStorage {
      * before binding a public port; protocol handlers still call this method
      * lazily as a defensive backstop.
      */
-    ensureSchema(schema: CompiledSchema): Promise<void>;
+    ensureSchema(schema: CompiledSchema, checkpoints?: readonly CheckpointDeclaration[]): Promise<void>;
     /** Create or refresh the authenticated partition registry row (§2.1). */
     touchPartition(partition: string, authenticatedAtMs: number, initialLogEpoch: string): Promise<PartitionRegistryEntry>;
     /** Rotate log continuity after restore and discard stale cursor records. */
@@ -413,6 +507,50 @@ export interface ServerStorage {
     pruneCommitsThrough(partition: string, query: CommitPruneQuery): Promise<CommitPruneResult>;
     /** Newest commitSeq created strictly before the timestamp; 0 if none. */
     getCommitSeqBefore(partition: string, createdBeforeMs: number): Promise<number>;
+    /** Every declared backfill checkpoint for the partition, ordered by name. */
+    readCheckpoints(partition: string): Promise<StoredCheckpoint[]>;
+    /**
+     * RFC 0007 serve gate: read the published schema version, the partition's
+     * log epoch, and every incomplete declared checkpoint at
+     * `runningSchemaVersion`. Never memoized; the serve paths call it per
+     * request so an already-serving process notices a migration.
+     */
+    readServeGate(partition: string, runningSchemaVersion: number): Promise<ServeGate>;
+    /**
+     * Atomically take ownership of a declared or backfilling checkpoint. Increments
+     * `owner_epoch` and moves `declared` to `backfilling`. Throws
+     * `sync.storage.checkpoint_not_declared` when no claimable row exists (absent,
+     * or already `activated`).
+     */
+    claimCheckpoint(partition: string, name: string, schemaVersion: number, nowMs: number): Promise<StoredCheckpoint>;
+    /**
+     * Advance the watermark only while `owner_epoch` still matches. Returns false
+     * for a superseded owner; the stale case never throws.
+     */
+    advanceCheckpoint(partition: string, name: string, ownerEpoch: number, watermark: number, observedRows: number, nowMs: number): Promise<boolean>;
+    /** Highest commitSeq carrying a change to one of `tables`; 0 when none. */
+    sourceCoverageSeq(partition: string, tables: readonly string[]): Promise<number>;
+    /**
+     * Whether any source change sits above `seq`. `unverifiable` when the pruning
+     * horizon has passed `seq`, so the history that would answer the question is
+     * gone; a pruned window is never reported as `clean`.
+     */
+    hasSourceChangesAbove(partition: string, tables: readonly string[], seq: number): Promise<'clean' | 'changed' | 'unverifiable'>;
+    /**
+     * Install the barrier: insert the checkpoint in `declared` and raise
+     * `required_writer_version` for the partition in one transaction. The fence
+     * is raised at declaration, before any backfill work, so old writers are
+     * rejected for the whole backfill window. Idempotent on an existing declared
+     * or backfilling row.
+     */
+    declareCheckpoint(partition: string, name: string, schemaVersion: number, nowMs: number): Promise<StoredCheckpoint>;
+    /**
+     * Activate a declared checkpoint under the partition write lock. Refuses when
+     * the fence is not already raised at the checkpoint's schema version, when a
+     * source change sits above `watermark` (`stale`), or when the pruning horizon
+     * has passed `watermark` (`unverifiable`). `activated` is terminal.
+     */
+    activateCheckpoint(partition: string, name: string, ownerEpoch: number, watermark: number, sources: readonly string[], nowMs: number): Promise<'activated' | 'stale' | 'unverifiable'>;
     getRow(partition: string, table: string, rowId: string): Promise<StoredRow | undefined>;
     /**
      * Idempotency lookup (§2.3). Throws `SyncError sync.idempotency_cache_miss`
@@ -450,19 +588,20 @@ export interface ServerStorage {
      * every generated app-table relation with a partition-filtered relation and
      * return rows plus maxCommitSeq from one consistent database snapshot.
      *
-     * The storage serializes this call behind an open transaction on a
-     * single-connection executor (SQLite, PGlite). A validator runs inside the
-     * push transaction, so on that executor a validator that calls
-     * `queryAuthoritative` waits on the transaction it is already inside and
-     * the push never completes. A pool-backed executor (`pg`, `Bun.sql`) hands
-     * the call a second connection instead: the validator does not deadlock,
-     * but it reads committed state without the candidate write. On either
-     * executor read candidate state through the whole-commit `CommitValidator`
-     * reader (§6.8). A call made from any other context waits for the open
-     * transaction to finish on a single-connection executor, and runs
-     * concurrently on a pool-backed one.
-     */
-    queryAuthoritative?(partition: string, query: AuthoritativeQueryRequest): Promise<AuthoritativeQueryResult>;
+     * Do not call this method from a hook that runs inside the push
+     * transaction (a row validator, the whole-commit validator, or a reaction
+     * planner). On a single-connection executor (SQLite, PGlite) the storage
+     * serializes this call behind the open transaction, so the hook waits on
+     * the transaction it is inside and the push never completes. A pool-backed
+     * executor (`pg`, `Bun.sql`) runs the call on a second connection: it reads
+     * committed state without the transaction's staged writes, and it waits
+     * forever when the push transactions hold every pool connection. Those
+     * hooks call the transaction-bound `queryAuthoritative` on
+     * `ValidateContext` or on the candidate-state reader instead (§6.7, §6.8).
+     * A call from any other context waits for the open transaction on a
+     * single-connection executor and runs concurrently on a pool-backed one.
+     */
+    queryAuthoritative?(partition: string, query: AuthoritativeQueryRequest, checkpoints?: readonly CheckpointDeclaration[]): Promise<AuthoritativeQueryResult>;
     /**
      * Optional trusted-host exact lookup through a declared relational index.
      * This capability is outside client scope/subscription authorization and

package/dist/storage.js

@@ -1 +1,40 @@
-export {};
+/**
+ * Evaluate the RFC 0007 predicate against a gate token. A process that
+ * declares the pending checkpoint may serve while it backfills; any other
+ * process is refused, and a database newer than the running build is always
+ * refused.
+ */
+export function serveGateRefusal(gate, runningSchemaVersion, checkpoints) {
+    if (gate.storedSchemaVersion > runningSchemaVersion) {
+        return {
+            kind: 'schema',
+            stored: gate.storedSchemaVersion,
+            running: runningSchemaVersion,
+        };
+    }
+    for (const pending of gate.pending) {
+        const declared = (checkpoints ?? []).some((declaration) => declaration.partition === pending.partition &&
+            declaration.name === pending.name);
+        if (!declared) {
+            return {
+                kind: 'checkpoint',
+                partition: pending.partition,
+                name: pending.name,
+            };
+        }
+    }
+    return undefined;
+}
+/**
+ * RFC 0007: compare two gate tokens read around a data read. The token binds
+ * the published schema version and the partition log epoch, so a restore that
+ * presents V → other → V is distinguishable. Returns a refusal when either
+ * changed across the read.
+ */
+export function serveGateTokenChanged(before, after) {
+    if (before.storedSchemaVersion !== after.storedSchemaVersion ||
+        before.logEpoch !== after.logEpoch) {
+        return { kind: 'epoch' };
+    }
+    return undefined;
+}

package/dist/validate.d.ts

@@ -14,6 +14,7 @@
  * object — zero cost, the events-seam discipline.
  */
 import { type RejectionDetails, type RowColumn, type RowValue, type ScopeMap } from '@syncular/core';
+import type { AuthoritativeQueryRequest, AuthoritativeQueryResult } from './storage.js';
 /**
  * §6.7 reserved code prefixes. A host validator code MUST NOT start with
  * any of these: they namespace the protocol's own error families (§10.2)
@@ -81,6 +82,15 @@ export interface CommitValidationScanInput {
 export interface CommitValidationReader {
     getRow(table: string, rowId: string): Promise<CommitValidationRow | undefined>;
     scanRows(input: CommitValidationScanInput): Promise<CommitValidationRow[]>;
+    /**
+     * Run a generated registered query on the commit transaction (§6.8). The
+     * request has the shape `ServerStorage.queryAuthoritative` takes; the
+     * storage binds every relation to the commit's partition. SQLite and
+     * PostgreSQL return the final candidate state; D1 fails with
+     * `sync.storage.query_over_staged_writes` when the query reads a table the
+     * commit has written.
+     */
+    queryAuthoritative(query: AuthoritativeQueryRequest): Promise<AuthoritativeQueryResult>;
 }
 export interface ValidateCommitInput {
     readonly clientId: string;
@@ -98,21 +108,29 @@ export type CommitValidator = (input: ValidateCommitInput) => void | Promise<voi
 /**
  * Ambient context a validator may consult (§6.7).
  *
- * A validator runs inside the push transaction. On a single-connection
- * executor (SQLite, PGlite) the storage serializes authoritative queries
- * behind that transaction, so a validator that calls `queryAuthoritative`
- * waits for the transaction it is already inside and the push never
- * completes. On a pool-backed executor (`pg`, `Bun.sql`) the query takes a
- * second connection and returns committed state without the candidate write.
- * Either way a rule that must read rows belongs in the whole-commit
- * `CommitValidator`, whose reader runs get and scan operations on the same
- * transaction (§6.8).
+ * A validator runs inside the push transaction. It MUST NOT call
+ * `ServerStorage.queryAuthoritative`: on a single-connection executor
+ * (SQLite, PGlite) that call waits for the transaction the validator is
+ * inside and the push never completes, and on a pool-backed executor (`pg`,
+ * `Bun.sql`) it reads committed state without the staged writes, or waits
+ * forever once push transactions hold every pool connection. An authority
+ * read calls `queryAuthoritative` on this context instead.
  */
 export interface ValidateContext {
     /** Host-authenticated actor (§1.1) performing the write. */
     readonly actorId: string;
     /** The partition (§1.1) the commit targets. */
     readonly partition: string;
+    /**
+     * Run a generated registered query on the push transaction (§6.7). The
+     * request has the shape `ServerStorage.queryAuthoritative` takes; the
+     * storage binds every relation to the commit's partition. SQLite and
+     * PostgreSQL return the transaction's snapshot, including the operations
+     * staged before this one; D1 fails with
+     * `sync.storage.query_over_staged_writes` when the query reads a table the
+     * commit has written.
+     */
+    readonly queryAuthoritative: (query: AuthoritativeQueryRequest) => Promise<AuthoritativeQueryResult>;
 }
 /**
  * A per-table validation hook (§6.7). Called once per push operation on
@@ -123,12 +141,12 @@ export interface ValidateContext {
  * `sync.constraint_violation` code (a validator SHOULD throw
  * `ValidationRejection` to control the code). MUST NOT mutate the row.
  *
- * The hook runs inside the push transaction and cannot read candidate rows
- * from an authoritative query: on a single-connection executor
- * `queryAuthoritative` serializes behind that transaction and the push never
- * completes, and on a pool-backed executor it reads committed state without
- * the candidate write. Read rows with the whole-commit `CommitValidator`
- * reader instead (§6.8).
+ * The hook runs inside the push transaction. An authority read uses
+ * `ctx.queryAuthoritative`, which executes on that transaction; calling
+ * `ServerStorage.queryAuthoritative` from here deadlocks on a
+ * single-connection executor and reads committed state on a pool-backed one.
+ * A rule over sibling operations belongs in the whole-commit
+ * `CommitValidator` (§6.8).
  */
 export type Validator = (op: ValidateOperation, ctx: ValidateContext) => void | Promise<void>;
 /** Table name → validator (§6.7). Absent tables are unvalidated. */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@syncular/server",
-  "version": "0.22.0",
+  "version": "0.24.0",
   "description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
   "license": "Apache-2.0",
   "author": "Benjamin Kniffler",
@@ -68,7 +68,7 @@
     "verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
   },
   "dependencies": {
-    "@syncular/core": "0.22.0"
+    "@syncular/core": "0.24.0"
   },
   "devDependencies": {
     "@electric-sql/pglite": "^0.5.4"

package/src/admin.ts

@@ -22,6 +22,7 @@ import { DEFAULT_RETENTION, type RetentionPolicy } from './prune';
 import { compileSchema, type ServerSchema } from './schema';
 import type { SegmentStore, SegmentStoreStats } from './segment-store';
 import type {
+  CheckpointDeclaration,
   ClientRecord,
   CommitMetadata,
   ReactionStatus,
@@ -157,6 +158,8 @@ export interface SyncularAdminOptions {
    * a storage instance that has not served a sync request yet.
    */
   readonly schema?: ServerSchema;
+  /** RFC 0007: declarations `inspectRow` carries into `ensureSchema`. */
+  readonly checkpoints?: readonly CheckpointDeclaration[];
   /** The event ring feeding the event tail. Absent ⇒ `events()` is empty. */
   readonly ring?: RingBufferEvents;
   readonly segments?: SegmentStore;
@@ -218,6 +221,7 @@ function toAdminClient(
 export class SyncularAdmin {
   readonly #storage: ServerStorage;
   readonly #schema?: ServerSchema;
+  readonly #checkpoints?: readonly CheckpointDeclaration[];
   readonly #ring?: RingBufferEvents;
   readonly #segments?: SegmentStore;
   readonly #blobs?: BlobStore;
@@ -228,6 +232,8 @@ export class SyncularAdmin {
   constructor(options: SyncularAdminOptions) {
     this.#storage = options.storage;
     if (options.schema !== undefined) this.#schema = options.schema;
+    if (options.checkpoints !== undefined)
+      this.#checkpoints = options.checkpoints;
     if (options.ring !== undefined) this.#ring = options.ring;
     if (options.segments !== undefined) this.#segments = options.segments;
     if (options.blobs !== undefined) this.#blobs = options.blobs;
@@ -248,6 +254,9 @@ export class SyncularAdmin {
     return new SyncularAdmin({
       storage: config.storage,
       schema: config.schema,
+      ...(config.checkpoints !== undefined
+        ? { checkpoints: config.checkpoints }
+        : {}),
       segments: config.segments,
       ...(config.blobs !== undefined ? { blobs: config.blobs } : {}),
       ...(config.leases !== undefined ? { leases: config.leases.store } : {}),
@@ -352,7 +361,10 @@ export class SyncularAdmin {
       'storage',
     );
     if (this.#schema !== undefined) {
-      await this.#storage.ensureSchema(compileSchema(this.#schema));
+      await this.#storage.ensureSchema(
+        compileSchema(this.#schema),
+        this.#checkpoints,
+      );
     }
     const row = await read(partition, table, rowId);
     if (row === undefined) {

package/src/blob-handlers.ts

@@ -233,7 +233,7 @@ async function downloadBlob(
   const schema = compileSchema(ctx.schema);
   // A blob download can hit a fresh storage instance before any sync
   // request has run (Workers): make sure the row tables exist first.
-  await ctx.storage.ensureSchema(schema);
+  await ctx.storage.ensureSchema(schema, ctx.checkpoints);
   const rows = await listRows.call(ctx.storage, ctx.partition, blobId);
   let authorized = false;
   for (const row of rows) {

package/src/context.ts

@@ -19,6 +19,7 @@ import type {
 } from './signed-url';
 import type { SqliteImageBuilder } from './sqlite-image';
 import type {
+  CheckpointDeclaration,
   PartitionRegistryEntry,
   ServerStorage,
   StoredCommit,
@@ -96,6 +97,13 @@ export interface RealtimeNotifier {
 export interface SyncServerConfig {
   readonly schema: ServerSchema;
   readonly storage: ServerStorage;
+  /**
+   * RFC 0007: the backfill checkpoints this host installs. Every readiness
+   * path carries this set into `ensureSchema`. A process that omits it
+   * declares nothing, and the storage refuses to serve a partition whose
+   * stored state shows a checkpoint that is not activated.
+   */
+  readonly checkpoints?: readonly CheckpointDeclaration[];
   readonly segments: SegmentStore;
   /**
    * Blob store (§5.9). Absent ⇒ blobs unsupported: a table with a