@syncular/server 0.22.0 → 0.24.0

package/src/d1-storage.ts

@@ -45,11 +45,14 @@ import type { CommitPruneQuery, CommitPruneResult } from './storage';
 import { decodeRow, type RowValue } from '@syncular/core';
 import {
   bindAuthoritativePartition,
+  type BoundAuthoritativeQuery,
   prepareAuthoritativeQuery,
 } from './authoritative-query';
 import { syncError } from './errors';
 import {
   assertAppendOnlyMigration,
+  assertPhysicalColumns,
+  assertStoredLayouts,
   commitWindowPageSql,
   deleteRowSql,
   deleteSqliteRowScopesSql,
@@ -81,6 +84,7 @@ import {
   asUint8Array,
   collectCommitWindowPage,
   deserializePushResult,
+  placeholders,
   type SqliteCommitWindowRecord,
   type SqliteRowRecord,
   serializePushResult,
@@ -90,6 +94,7 @@ import {
 import type {
   AuthoritativeQueryRequest,
   AuthoritativeQueryResult,
+  CheckpointDeclaration,
   ClientCursorInfo,
   ClientRecord,
   ClientSubscription,
@@ -111,8 +116,10 @@ import type {
   ScopeActivityQuery,
   ScopeCommitActivity,
   ServerStorage,
+  ServeGate,
   StorageTransaction,
   StoredCommit,
+  StoredCheckpoint,
   StoredPushResult,
   StoredReaction,
   StoredRow,
@@ -199,6 +206,49 @@ function relationalValuesEqual(left: RowValue, right: RowValue): boolean {
   return Object.is(left, right);
 }
 
+/** Run one partition-bound registered query and the cursor read as one batch. */
+async function queryAuthoritativeOn(
+  db: Pick<D1Database, 'prepare' | 'batch'>,
+  partition: string,
+  prepared: BoundAuthoritativeQuery,
+): Promise<AuthoritativeQueryResult> {
+  const results = await db.batch([
+    db.prepare(prepared.sql).bind(...prepared.params),
+    db
+      .prepare('SELECT max_commit_seq FROM sync_partitions WHERE partition=?')
+      .bind(partition),
+  ]);
+  const rowsResult = results[0];
+  const cursorResult = results[1];
+  if (
+    typeof rowsResult !== 'object' ||
+    rowsResult === null ||
+    !('results' in rowsResult) ||
+    !Array.isArray(rowsResult.results) ||
+    typeof cursorResult !== 'object' ||
+    cursorResult === null ||
+    !('results' in cursorResult) ||
+    !Array.isArray(cursorResult.results)
+  ) {
+    throw new Error('D1 registered query returned an invalid batch result');
+  }
+  const cursor = cursorResult.results[0];
+  const maxCommitSeq =
+    typeof cursor === 'object' &&
+    cursor !== null &&
+    'max_commit_seq' in cursor &&
+    typeof cursor.max_commit_seq === 'number'
+      ? cursor.max_commit_seq
+      : 0;
+  return {
+    rows: rowsResult.results.filter(
+      (row): row is Readonly<Record<string, unknown>> =>
+        typeof row === 'object' && row !== null,
+    ),
+    maxCommitSeq,
+  };
+}
+
 /** Read-your-own-writes overlay entry: a buffered upsert, or a deletion. */
 type PendingRow =
   | { readonly kind: 'row'; readonly row: StoredRow }
@@ -210,6 +260,10 @@ class D1Transaction implements StorageTransaction {
   readonly #db: Pick<D1Database, 'prepare' | 'batch'>;
   readonly #partition: string;
   readonly #resolveTable: (name: string) => CompiledTable;
+  /** Storage-owned relation rewrite, bound to this transaction's partition. */
+  readonly #prepareQuery: (
+    query: AuthoritativeQueryRequest,
+  ) => BoundAuthoritativeQuery;
   readonly #pushApplySerialized: boolean;
   readonly #buffer: BufferedStatement[] = [];
   #open = true;
@@ -238,11 +292,13 @@ class D1Transaction implements StorageTransaction {
     db: Pick<D1Database, 'prepare' | 'batch'>,
     partition: string,
     resolveTable: (name: string) => CompiledTable,
+    prepareQuery: (query: AuthoritativeQueryRequest) => BoundAuthoritativeQuery,
     pushApplySerialized: boolean,
   ) {
     this.#db = db;
     this.#partition = partition;
     this.#resolveTable = resolveTable;
+    this.#prepareQuery = prepareQuery;
     this.#pushApplySerialized = pushApplySerialized;
   }
 
@@ -435,6 +491,25 @@ class D1Transaction implements StorageTransaction {
       .slice(0, query.limit);
   }
 
+  async queryAuthoritative(
+    query: AuthoritativeQueryRequest,
+  ): Promise<AuthoritativeQueryResult> {
+    this.#assertOpen();
+    // D1 buffers writes until commit() and cannot overlay them on generated
+    // SQL, so a read over a written table would return committed state
+    // without this transaction's staged rows. Refuse it instead.
+    for (const key of this.#pending.keys()) {
+      if (query.tables.includes(key.slice(0, key.indexOf('\u0000')))) {
+        throw new StorageQueryError('sync.storage.query_over_staged_writes');
+      }
+    }
+    return queryAuthoritativeOn(
+      this.#db,
+      this.#partition,
+      this.#prepareQuery(query),
+    );
+  }
+
   async lockPartitionForPush(): Promise<void> {
     this.#assertOpen();
     if (!this.#pushApplySerialized) {
@@ -447,6 +522,31 @@ class D1Transaction implements StorageTransaction {
     this.#pushApplyCheckpoint = this.#buffer.length;
   }
 
+  advanceCheckpoint(
+    _name: string,
+    _ownerEpoch: number,
+    _watermark: number,
+    _observedRows: number,
+    _nowMs: number,
+  ): Promise<boolean> {
+    // D1 installs no fence and declares no checkpoints in this release.
+    return Promise.reject(
+      new StorageQueryError('sync.storage.checkpoint_unsupported'),
+    );
+  }
+
+  async readServeGate(_runningSchemaVersion: number): Promise<ServeGate> {
+    this.#assertOpen();
+    const marker = await this.#db
+      .prepare('SELECT schema_version FROM sync_schema_meta WHERE id=1')
+      .first<{ schema_version: number }>();
+    return {
+      storedSchemaVersion: marker?.schema_version ?? 0,
+      logEpoch: undefined,
+      pending: [],
+    };
+  }
+
   async commitRejectedPushResult(
     clientId: string,
     clientCommitId: string,
@@ -777,6 +877,27 @@ export interface D1ServerStorageOptions {
   readonly commitValidationSerialized?: boolean;
 }
 
+/**
+ * The core tables `D1ServerStorage` reads or writes outside schema migration.
+ * `migrateSchema` creates them in its core phase, which runs only when the
+ * version marker advances, so a same-version open refuses a database missing
+ * one. `sync_backfill_checkpoints` and `sync_writer_fence` stay out: D1
+ * declares no checkpoints and installs no fence.
+ */
+const D1_CORE_TABLES = [
+  'sync_partitions',
+  'sync_partition_registry',
+  'sync_row_scopes',
+  'sync_commits',
+  'sync_changes',
+  'sync_change_scopes',
+  'sync_push_results',
+  'sync_reactions',
+  'sync_clients',
+  'sync_blob_refs',
+  'sync_tombstones',
+] as const;
+
 interface D1MigrationState {
   phase: 'core' | 'prepare' | 'ddl' | 'rewrite';
   offset: number;
@@ -834,7 +955,15 @@ export class D1ServerStorage implements ServerStorage {
     return table;
   }
 
-  async ensureSchema(schema: CompiledSchema): Promise<void> {
+  async ensureSchema(
+    schema: CompiledSchema,
+    checkpoints?: readonly CheckpointDeclaration[],
+  ): Promise<void> {
+    if (checkpoints !== undefined && checkpoints.length > 0) {
+      // No database-side fence can be installed atomically with schema
+      // visibility here; refuse instead of accepting a weaker guarantee.
+      throw new StorageQueryError('sync.storage.checkpoint_unsupported');
+    }
     if (this.#schemaVersion === schema.version) {
       await this.#schemaDatabase().batch([]);
       return;
@@ -910,6 +1039,39 @@ export class D1ServerStorage implements ServerStorage {
       this.#db.prepare('SELECT * FROM sync_schema_migration WHERE id=1'),
     );
     if (claim === null && marker?.schema_version === schema.version) {
+      assertStoredLayouts(schema, marker.layouts);
+      statementsExecuted++;
+      const present = await this.#db
+        .prepare(
+          `SELECT name FROM sqlite_master WHERE type='table' AND name IN (${placeholders(D1_CORE_TABLES.length)})`,
+        )
+        .bind(...D1_CORE_TABLES)
+        .all<{ name: string }>();
+      const presentNames = new Set(present.results.map((row) => row.name));
+      for (const name of D1_CORE_TABLES) {
+        if (!presentNames.has(name)) {
+          throw new StorageQueryError('sync.storage.physical_layout_mismatch', {
+            table: name,
+            reason: 'missing_table',
+          });
+        }
+      }
+      for (const table of tables) {
+        statementsExecuted++;
+        const { results } = await this.#db
+          .prepare(`PRAGMA table_info(${quoteIdent(table.name)})`)
+          .all<{ name: string; type: string; notnull: number; pk: number }>();
+        assertPhysicalColumns(
+          table,
+          results.map((column) => ({
+            name: column.name,
+            type: column.type,
+            notNull: column.notnull === 1,
+            primaryKeyPosition: column.pk,
+          })),
+          'sqlite',
+        );
+      }
       this.#tables = schema.tables;
       this.#schemaVersion = schema.version;
       return { complete: true, statementsExecuted };
@@ -1412,6 +1574,7 @@ export class D1ServerStorage implements ServerStorage {
           throw new StorageQueryError('sync.storage.schema_changed');
         return table;
       },
+      (query) => this.#prepareAuthoritativeQuery(partition, query),
       this.#pushApplySerialized,
     );
   }
@@ -1427,14 +1590,29 @@ export class D1ServerStorage implements ServerStorage {
   async queryAuthoritative(
     partition: string,
     query: AuthoritativeQueryRequest,
+    _checkpoints?: readonly CheckpointDeclaration[],
   ): Promise<AuthoritativeQueryResult> {
+    // D1 declares no checkpoints. `#schemaDatabase` prepends a guard that
+    // re-checks the published marker inside this same batch, so condition 2
+    // is already evaluated with the query (§2.4 D1 schema readiness).
     const db = this.#schemaDatabase();
+    return queryAuthoritativeOn(
+      db,
+      partition,
+      this.#prepareAuthoritativeQuery(partition, query),
+    );
+  }
+
+  #prepareAuthoritativeQuery(
+    partition: string,
+    query: AuthoritativeQueryRequest,
+  ): BoundAuthoritativeQuery {
     if (this.#tables === undefined) {
       throw new Error(
         'ensureSchema(schema) must run before registered queries',
       );
     }
-    const prepared = bindAuthoritativePartition(
+    return bindAuthoritativePartition(
       prepareAuthoritativeQuery(
         query.plan,
         query.params,
@@ -1443,41 +1621,6 @@ export class D1ServerStorage implements ServerStorage {
       ),
       partition,
     );
-    const results = await db.batch([
-      db.prepare(prepared.sql).bind(...prepared.params),
-      db
-        .prepare('SELECT max_commit_seq FROM sync_partitions WHERE partition=?')
-        .bind(partition),
-    ]);
-    const rowsResult = results[0];
-    const cursorResult = results[1];
-    if (
-      typeof rowsResult !== 'object' ||
-      rowsResult === null ||
-      !('results' in rowsResult) ||
-      !Array.isArray(rowsResult.results) ||
-      typeof cursorResult !== 'object' ||
-      cursorResult === null ||
-      !('results' in cursorResult) ||
-      !Array.isArray(cursorResult.results)
-    ) {
-      throw new Error('D1 registered query returned an invalid batch result');
-    }
-    const cursor = cursorResult.results[0];
-    const maxCommitSeq =
-      typeof cursor === 'object' &&
-      cursor !== null &&
-      'max_commit_seq' in cursor &&
-      typeof cursor.max_commit_seq === 'number'
-        ? cursor.max_commit_seq
-        : 0;
-    return {
-      rows: rowsResult.results.filter(
-        (row): row is Readonly<Record<string, unknown>> =>
-          typeof row === 'object' && row !== null,
-      ),
-      maxCommitSeq,
-    };
   }
 
   async getPartitionLogEpoch(partition: string): Promise<string | undefined> {
@@ -1591,6 +1734,108 @@ export class D1ServerStorage implements ServerStorage {
     return row?.seq ?? 0;
   }
 
+  async readCheckpoints(_partition: string): Promise<StoredCheckpoint[]> {
+    // D1 supports no declared backfill checkpoints in this release: migrations
+    // run out of band and no database-side fence can be installed atomically
+    // with schema visibility. No checkpoint row can exist.
+    return [];
+  }
+
+  async readServeGate(
+    partition: string,
+    _runningSchemaVersion: number,
+  ): Promise<ServeGate> {
+    // D1 has no checkpoints. The published marker is the only gate input;
+    // `#schemaDatabase` already re-checks it per protected batch.
+    const marker = await this.#db
+      .prepare('SELECT schema_version FROM sync_schema_meta WHERE id=1')
+      .first<{ schema_version: number }>();
+    return {
+      storedSchemaVersion: marker?.schema_version ?? 0,
+      logEpoch: await this.getPartitionLogEpoch(partition),
+      pending: [],
+    };
+  }
+
+  async declareCheckpoint(
+    _partition: string,
+    _name: string,
+    _schemaVersion: number,
+    _nowMs: number,
+  ): Promise<StoredCheckpoint> {
+    throw new StorageQueryError('sync.storage.checkpoint_unsupported');
+  }
+
+  async activateCheckpoint(
+    _partition: string,
+    _name: string,
+    _ownerEpoch: number,
+    _watermark: number,
+    _sources: readonly string[],
+    _nowMs: number,
+  ): Promise<'activated' | 'stale' | 'unverifiable'> {
+    throw new StorageQueryError('sync.storage.checkpoint_unsupported');
+  }
+
+  async claimCheckpoint(
+    _partition: string,
+    _name: string,
+    _schemaVersion: number,
+    _nowMs: number,
+  ): Promise<StoredCheckpoint> {
+    throw new StorageQueryError('sync.storage.checkpoint_unsupported');
+  }
+
+  async advanceCheckpoint(
+    _partition: string,
+    _name: string,
+    _ownerEpoch: number,
+    _watermark: number,
+    _observedRows: number,
+    _nowMs: number,
+  ): Promise<boolean> {
+    throw new StorageQueryError('sync.storage.checkpoint_unsupported');
+  }
+
+  async sourceCoverageSeq(
+    partition: string,
+    tables: readonly string[],
+  ): Promise<number> {
+    if (tables.length === 0) return 0;
+    const tableParams = tables.map(() => '?').join(',');
+    const row = await this.#db
+      .prepare(
+        `SELECT max(commit_seq) AS seq FROM sync_changes
+          WHERE partition=? AND tbl IN (${tableParams})`,
+      )
+      .bind(partition, ...tables)
+      .first<{ seq: number | null }>();
+    return row?.seq ?? 0;
+  }
+
+  async hasSourceChangesAbove(
+    partition: string,
+    tables: readonly string[],
+    seq: number,
+  ): Promise<'clean' | 'changed' | 'unverifiable'> {
+    // Read the horizon before the window scan: a horizon past `seq` means the
+    // history that would answer the question is gone, so an empty scan proves
+    // nothing and must not be reported as `clean`.
+    const horizon = await this.getHorizonSeq(partition);
+    if (tables.length === 0) return 'clean';
+    const tableParams = tables.map(() => '?').join(',');
+    const hit = await this.#db
+      .prepare(
+        `SELECT 1 AS hit FROM sync_changes
+          WHERE partition=? AND tbl IN (${tableParams}) AND commit_seq>?
+          LIMIT 1`,
+      )
+      .bind(partition, ...tables, seq)
+      .first<{ hit: number }>();
+    if (hit !== null) return 'changed';
+    return horizon > seq ? 'unverifiable' : 'clean';
+  }
+
   async getRow(
     partition: string,
     table: string,

package/src/errors.ts

@@ -167,6 +167,16 @@ export const ERROR_CATALOG: Readonly<Record<string, ErrorCatalogEntry>> = {
     recommendedAction: 'retryLater',
     httpStatus: 500,
   },
+  // §2.4 on a non-D1 backend: the server refuses while a declared backfill
+  // checkpoint is incomplete or the database is newer than the running build.
+  // Formerly host-only; promoted to the §10.2 catalogue so an in-flight
+  // request can be refused with a distinguishable identity.
+  'sync.schema_not_ready': {
+    category: 'internal',
+    retryable: true,
+    recommendedAction: 'retryLater',
+    httpStatus: 503,
+  },
   'sync.too_many_operations': {
     category: 'invalid-request',
     retryable: false,

package/src/handler.ts

@@ -51,10 +51,17 @@ import {
   subscriptionSection,
 } from './pull';
 import { type ProcessedPushCommit, processPushCommitWithTrace } from './push';
+import { serveNotReadyError } from './readiness';
 import type { CompiledSchema } from './schema';
 import { compileSchema } from './schema';
 import { computeEffective, type ResolvedScopes } from './scopes';
-import type { ClientSubscription, PartitionRegistryEntry } from './storage';
+import { StorageQueryError } from './storage-errors';
+import {
+  type ClientSubscription,
+  type PartitionRegistryEntry,
+  serveGateRefusal,
+  serveGateTokenChanged,
+} from './storage';
 
 interface RequestPlan {
   readonly wireVersion: number;
@@ -709,13 +716,64 @@ async function createStreamCore(
   const schema = compileSchema(ctx.schema);
   // Relational row tables: create/
   // migrate on first contact; memoized per storage instance thereafter.
-  await ctx.storage.ensureSchema(schema);
+  try {
+    await ctx.storage.ensureSchema(schema, ctx.checkpoints);
+  } catch (error) {
+    if (
+      error instanceof StorageQueryError &&
+      error.code === 'sync.storage.checkpoint_incomplete'
+    ) {
+      const refusal = serveGateRefusal(
+        await ctx.storage.readServeGate(ctx.partition, schema.version),
+        schema.version,
+        ctx.checkpoints,
+      );
+      throw refusal !== undefined ? serveNotReadyError(refusal) : error;
+    }
+    throw error;
+  }
+  // RFC 0007 serve gate: re-read stored state on every request so an
+  // already-serving process notices a migration or a newly declared
+  // checkpoint. Read after `touchAuthenticatedPartition` establishes the
+  // partition's log epoch, so the entry token is the epoch this request
+  // serves under. Nothing has escaped at this point; the push transaction
+  // re-evaluates the gate under the partition lock.
   const registry = await touchAuthenticatedPartition(ctx);
+  const gate = await ctx.storage.readServeGate(ctx.partition, schema.version);
+  const refusal = serveGateRefusal(gate, schema.version, ctx.checkpoints);
+  if (refusal !== undefined) throw serveNotReadyError(refusal);
   const plan = await planRequest(request, ctx, schema, registry);
-  if (events === undefined) return streamResponse(plan, ctx, schema);
   const report: RequestReport = { outcome: 'ok' };
+  // RFC 0007 read-verify-refuse for the streamed read path: the generator
+  // builds the whole response (its data reads) before yielding the first
+  // frame, then compares the gate token across those reads. Nothing escapes
+  // before the verification, and the buffered size is bounded by the request's
+  // pull limits and the server's inline segment cap. Every request verifies,
+  // including a mixed push+pull: a write that already applied when the token
+  // changed is durable and replayable under the same commit id / idempotency
+  // key, so refusing the request is safe (proved in the serve-gate lane).
+  const verified = async function* (): AsyncGenerator<Uint8Array> {
+    const buffered: Uint8Array[] = [];
+    for await (const chunk of streamResponse(plan, ctx, schema, report)) {
+      buffered.push(chunk);
+    }
+    const after = await ctx.storage.readServeGate(
+      ctx.partition,
+      schema.version,
+    );
+    const changed = serveGateTokenChanged(gate, after);
+    const afterRefusal = serveGateRefusal(
+      after,
+      schema.version,
+      ctx.checkpoints,
+    );
+    if (afterRefusal !== undefined) throw serveNotReadyError(afterRefusal);
+    if (changed !== undefined) throw serveNotReadyError(changed);
+    for (const chunk of buffered) yield chunk;
+  };
+  if (events === undefined) return verified();
   return instrumentedStream(
-    streamResponse(plan, ctx, schema, report),
+    verified(),
     ctx,
     events,
     plan,

package/src/operations.ts

@@ -22,8 +22,13 @@ import { SyncError, syncError } from './errors';
 import { processPushOperationsWithTrace } from './push';
 import { compileSchema } from './schema';
 import { authorizeWrite, type ResolvedScopes } from './scopes';
-import type { AuthoritativeQueryValue } from './storage';
+import type {
+  AuthoritativeQueryRequest,
+  AuthoritativeQueryResult,
+  AuthoritativeQueryValue,
+} from './storage';
 import type { StorageTransaction } from './storage';
+import { transactionQuery } from './storage-query';
 import { toValidateRow, type ValidateRow } from './validate';
 
 export interface RemoteQueryDependency {
@@ -107,6 +112,14 @@ export interface RemoteCommandContext {
   readonly operationId: string;
   readonly requestId: string;
   getRow(table: string, rowId: string): Promise<ValidateRow | undefined>;
+  /**
+   * Run a generated registered query on the command's push transaction
+   * (§6.7). The storage binds every relation to the partition; unlike
+   * `getRow`, the rows are not filtered by the actor's scopes.
+   */
+  queryAuthoritative(
+    query: AuthoritativeQueryRequest,
+  ): Promise<AuthoritativeQueryResult>;
 }
 
 export interface RemoteCommandOptions<Input> {
@@ -415,22 +428,26 @@ export function registerRemoteQuery<Params>(
           'selected SQL has no generated relation plan; regenerate queries',
         );
       }
-      await ctx.storage.ensureSchema(schema);
+      await ctx.storage.ensureSchema(schema, ctx.checkpoints);
       const prefix = 'SELECT * FROM (';
       let result;
       try {
-        result = await ctx.storage.queryAuthoritative(ctx.partition, {
-          plan: {
-            sql: `${prefix}${selectedSql}) AS "_syncular_registered_query" LIMIT ?`,
-            relations: plan.relations.map((relation) => ({
-              ...relation,
-              start: relation.start + prefix.length,
-              end: relation.end + prefix.length,
-            })),
+        result = await ctx.storage.queryAuthoritative(
+          ctx.partition,
+          {
+            plan: {
+              sql: `${prefix}${selectedSql}) AS "_syncular_registered_query" LIMIT ?`,
+              relations: plan.relations.map((relation) => ({
+                ...relation,
+                start: relation.start + prefix.length,
+                end: relation.end + prefix.length,
+              })),
+            },
+            params: [...descriptor.bind(params), options.maxRows + 1],
+            tables: descriptor.tables,
           },
-          params: [...descriptor.bind(params), options.maxRows + 1],
-          tables: descriptor.tables,
-        });
+          ctx.checkpoints,
+        );
       } catch (error) {
         if (error instanceof SyncError) throw error;
         throw syncError(
@@ -566,6 +583,7 @@ function commandContext(
         decodeRow(table.columns, stored.payload),
       );
     },
+    queryAuthoritative: (query) => transactionQuery(tx, query),
   };
 }
 
@@ -603,7 +621,7 @@ export function registerRemoteCommand<Input>(
       }
       const resolved: ResolvedScopes = { ok: true, allowed };
       const schema = compileSchema(ctx.schema);
-      await ctx.storage.ensureSchema(schema);
+      await ctx.storage.ensureSchema(schema, ctx.checkpoints);
       const processed = await processPushOperationsWithTrace(
         ctx,
         schema,