@syncular/server 0.23.0 → 0.25.0

package/src/d1-storage.ts

@@ -45,11 +45,14 @@ import type { CommitPruneQuery, CommitPruneResult } from './storage';
 import { decodeRow, type RowValue } from '@syncular/core';
 import {
   bindAuthoritativePartition,
+  type BoundAuthoritativeQuery,
   prepareAuthoritativeQuery,
 } from './authoritative-query';
 import { syncError } from './errors';
 import {
   assertAppendOnlyMigration,
+  assertPhysicalColumns,
+  assertStoredLayouts,
   commitWindowPageSql,
   deleteRowSql,
   deleteSqliteRowScopesSql,
@@ -81,6 +84,7 @@ import {
   asUint8Array,
   collectCommitWindowPage,
   deserializePushResult,
+  placeholders,
   type SqliteCommitWindowRecord,
   type SqliteRowRecord,
   serializePushResult,
@@ -202,6 +206,49 @@ function relationalValuesEqual(left: RowValue, right: RowValue): boolean {
   return Object.is(left, right);
 }
 
+/** Run one partition-bound registered query and the cursor read as one batch. */
+async function queryAuthoritativeOn(
+  db: Pick<D1Database, 'prepare' | 'batch'>,
+  partition: string,
+  prepared: BoundAuthoritativeQuery,
+): Promise<AuthoritativeQueryResult> {
+  const results = await db.batch([
+    db.prepare(prepared.sql).bind(...prepared.params),
+    db
+      .prepare('SELECT max_commit_seq FROM sync_partitions WHERE partition=?')
+      .bind(partition),
+  ]);
+  const rowsResult = results[0];
+  const cursorResult = results[1];
+  if (
+    typeof rowsResult !== 'object' ||
+    rowsResult === null ||
+    !('results' in rowsResult) ||
+    !Array.isArray(rowsResult.results) ||
+    typeof cursorResult !== 'object' ||
+    cursorResult === null ||
+    !('results' in cursorResult) ||
+    !Array.isArray(cursorResult.results)
+  ) {
+    throw new Error('D1 registered query returned an invalid batch result');
+  }
+  const cursor = cursorResult.results[0];
+  const maxCommitSeq =
+    typeof cursor === 'object' &&
+    cursor !== null &&
+    'max_commit_seq' in cursor &&
+    typeof cursor.max_commit_seq === 'number'
+      ? cursor.max_commit_seq
+      : 0;
+  return {
+    rows: rowsResult.results.filter(
+      (row): row is Readonly<Record<string, unknown>> =>
+        typeof row === 'object' && row !== null,
+    ),
+    maxCommitSeq,
+  };
+}
+
 /** Read-your-own-writes overlay entry: a buffered upsert, or a deletion. */
 type PendingRow =
   | { readonly kind: 'row'; readonly row: StoredRow }
@@ -213,6 +260,10 @@ class D1Transaction implements StorageTransaction {
   readonly #db: Pick<D1Database, 'prepare' | 'batch'>;
   readonly #partition: string;
   readonly #resolveTable: (name: string) => CompiledTable;
+  /** Storage-owned relation rewrite, bound to this transaction's partition. */
+  readonly #prepareQuery: (
+    query: AuthoritativeQueryRequest,
+  ) => BoundAuthoritativeQuery;
   readonly #pushApplySerialized: boolean;
   readonly #buffer: BufferedStatement[] = [];
   #open = true;
@@ -241,11 +292,13 @@ class D1Transaction implements StorageTransaction {
     db: Pick<D1Database, 'prepare' | 'batch'>,
     partition: string,
     resolveTable: (name: string) => CompiledTable,
+    prepareQuery: (query: AuthoritativeQueryRequest) => BoundAuthoritativeQuery,
     pushApplySerialized: boolean,
   ) {
     this.#db = db;
     this.#partition = partition;
     this.#resolveTable = resolveTable;
+    this.#prepareQuery = prepareQuery;
     this.#pushApplySerialized = pushApplySerialized;
   }
 
@@ -438,6 +491,25 @@ class D1Transaction implements StorageTransaction {
       .slice(0, query.limit);
   }
 
+  async queryAuthoritative(
+    query: AuthoritativeQueryRequest,
+  ): Promise<AuthoritativeQueryResult> {
+    this.#assertOpen();
+    // D1 buffers writes until commit() and cannot overlay them on generated
+    // SQL, so a read over a written table would return committed state
+    // without this transaction's staged rows. Refuse it instead.
+    for (const key of this.#pending.keys()) {
+      if (query.tables.includes(key.slice(0, key.indexOf('\u0000')))) {
+        throw new StorageQueryError('sync.storage.query_over_staged_writes');
+      }
+    }
+    return queryAuthoritativeOn(
+      this.#db,
+      this.#partition,
+      this.#prepareQuery(query),
+    );
+  }
+
   async lockPartitionForPush(): Promise<void> {
     this.#assertOpen();
     if (!this.#pushApplySerialized) {
@@ -805,6 +877,27 @@ export interface D1ServerStorageOptions {
   readonly commitValidationSerialized?: boolean;
 }
 
+/**
+ * The core tables `D1ServerStorage` reads or writes outside schema migration.
+ * `migrateSchema` creates them in its core phase, which runs only when the
+ * version marker advances, so a same-version open refuses a database missing
+ * one. `sync_backfill_checkpoints` and `sync_writer_fence` stay out: D1
+ * declares no checkpoints and installs no fence.
+ */
+const D1_CORE_TABLES = [
+  'sync_partitions',
+  'sync_partition_registry',
+  'sync_row_scopes',
+  'sync_commits',
+  'sync_changes',
+  'sync_change_scopes',
+  'sync_push_results',
+  'sync_reactions',
+  'sync_clients',
+  'sync_blob_refs',
+  'sync_tombstones',
+] as const;
+
 interface D1MigrationState {
   phase: 'core' | 'prepare' | 'ddl' | 'rewrite';
   offset: number;
@@ -946,6 +1039,39 @@ export class D1ServerStorage implements ServerStorage {
       this.#db.prepare('SELECT * FROM sync_schema_migration WHERE id=1'),
     );
     if (claim === null && marker?.schema_version === schema.version) {
+      assertStoredLayouts(schema, marker.layouts);
+      statementsExecuted++;
+      const present = await this.#db
+        .prepare(
+          `SELECT name FROM sqlite_master WHERE type='table' AND name IN (${placeholders(D1_CORE_TABLES.length)})`,
+        )
+        .bind(...D1_CORE_TABLES)
+        .all<{ name: string }>();
+      const presentNames = new Set(present.results.map((row) => row.name));
+      for (const name of D1_CORE_TABLES) {
+        if (!presentNames.has(name)) {
+          throw new StorageQueryError('sync.storage.physical_layout_mismatch', {
+            table: name,
+            reason: 'missing_table',
+          });
+        }
+      }
+      for (const table of tables) {
+        statementsExecuted++;
+        const { results } = await this.#db
+          .prepare(`PRAGMA table_info(${quoteIdent(table.name)})`)
+          .all<{ name: string; type: string; notnull: number; pk: number }>();
+        assertPhysicalColumns(
+          table,
+          results.map((column) => ({
+            name: column.name,
+            type: column.type,
+            notNull: column.notnull === 1,
+            primaryKeyPosition: column.pk,
+          })),
+          'sqlite',
+        );
+      }
       this.#tables = schema.tables;
       this.#schemaVersion = schema.version;
       return { complete: true, statementsExecuted };
@@ -1448,6 +1574,7 @@ export class D1ServerStorage implements ServerStorage {
           throw new StorageQueryError('sync.storage.schema_changed');
         return table;
       },
+      (query) => this.#prepareAuthoritativeQuery(partition, query),
       this.#pushApplySerialized,
     );
   }
@@ -1469,12 +1596,23 @@ export class D1ServerStorage implements ServerStorage {
     // re-checks the published marker inside this same batch, so condition 2
     // is already evaluated with the query (§2.4 D1 schema readiness).
     const db = this.#schemaDatabase();
+    return queryAuthoritativeOn(
+      db,
+      partition,
+      this.#prepareAuthoritativeQuery(partition, query),
+    );
+  }
+
+  #prepareAuthoritativeQuery(
+    partition: string,
+    query: AuthoritativeQueryRequest,
+  ): BoundAuthoritativeQuery {
     if (this.#tables === undefined) {
       throw new Error(
         'ensureSchema(schema) must run before registered queries',
       );
     }
-    const prepared = bindAuthoritativePartition(
+    return bindAuthoritativePartition(
       prepareAuthoritativeQuery(
         query.plan,
         query.params,
@@ -1483,41 +1621,6 @@ export class D1ServerStorage implements ServerStorage {
       ),
       partition,
     );
-    const results = await db.batch([
-      db.prepare(prepared.sql).bind(...prepared.params),
-      db
-        .prepare('SELECT max_commit_seq FROM sync_partitions WHERE partition=?')
-        .bind(partition),
-    ]);
-    const rowsResult = results[0];
-    const cursorResult = results[1];
-    if (
-      typeof rowsResult !== 'object' ||
-      rowsResult === null ||
-      !('results' in rowsResult) ||
-      !Array.isArray(rowsResult.results) ||
-      typeof cursorResult !== 'object' ||
-      cursorResult === null ||
-      !('results' in cursorResult) ||
-      !Array.isArray(cursorResult.results)
-    ) {
-      throw new Error('D1 registered query returned an invalid batch result');
-    }
-    const cursor = cursorResult.results[0];
-    const maxCommitSeq =
-      typeof cursor === 'object' &&
-      cursor !== null &&
-      'max_commit_seq' in cursor &&
-      typeof cursor.max_commit_seq === 'number'
-        ? cursor.max_commit_seq
-        : 0;
-    return {
-      rows: rowsResult.results.filter(
-        (row): row is Readonly<Record<string, unknown>> =>
-          typeof row === 'object' && row !== null,
-      ),
-      maxCommitSeq,
-    };
   }
 
   async getPartitionLogEpoch(partition: string): Promise<string | undefined> {

package/src/operations.ts

@@ -22,8 +22,13 @@ import { SyncError, syncError } from './errors';
 import { processPushOperationsWithTrace } from './push';
 import { compileSchema } from './schema';
 import { authorizeWrite, type ResolvedScopes } from './scopes';
-import type { AuthoritativeQueryValue } from './storage';
+import type {
+  AuthoritativeQueryRequest,
+  AuthoritativeQueryResult,
+  AuthoritativeQueryValue,
+} from './storage';
 import type { StorageTransaction } from './storage';
+import { transactionQuery } from './storage-query';
 import { toValidateRow, type ValidateRow } from './validate';
 
 export interface RemoteQueryDependency {
@@ -107,6 +112,14 @@ export interface RemoteCommandContext {
   readonly operationId: string;
   readonly requestId: string;
   getRow(table: string, rowId: string): Promise<ValidateRow | undefined>;
+  /**
+   * Run a generated registered query on the command's push transaction
+   * (§6.7). The storage binds every relation to the partition; unlike
+   * `getRow`, the rows are not filtered by the actor's scopes.
+   */
+  queryAuthoritative(
+    query: AuthoritativeQueryRequest,
+  ): Promise<AuthoritativeQueryResult>;
 }
 
 export interface RemoteCommandOptions<Input> {
@@ -570,6 +583,7 @@ function commandContext(
         decodeRow(table.columns, stored.payload),
       );
     },
+    queryAuthoritative: (query) => transactionQuery(tx, query),
   };
 }
 

package/src/postgres-storage.ts

@@ -43,6 +43,7 @@ import { serveNotReadyError } from './readiness';
 import type { PushOperationResult } from '@syncular/core';
 import {
   bindAuthoritativePartition,
+  type BoundAuthoritativeQuery,
   postgresPlaceholders,
   prepareAuthoritativeQuery,
 } from './authoritative-query';
@@ -55,6 +56,7 @@ import {
 } from './pg-executor';
 import {
   assertPhysicalColumns,
+  assertStoredLayouts,
   commitWindowPageSql,
   deleteRowSql,
   dropTableDdl,
@@ -730,9 +732,36 @@ async function getPushResultOn(
   }
 }
 
+/** Run one partition-bound registered query and the cursor read on `client`. */
+async function queryAuthoritativeOn(
+  client: PgQueryable,
+  partition: string,
+  prepared: BoundAuthoritativeQuery,
+): Promise<AuthoritativeQueryResult> {
+  const result = await client.query<Readonly<Record<string, unknown>>>(
+    postgresPlaceholders(prepared.sql),
+    prepared.params,
+  );
+  const cursor = await client.query<{ max_commit_seq: unknown }>(
+    'SELECT max_commit_seq FROM sync_partitions WHERE partition=$1',
+    [partition],
+  );
+  return {
+    rows: result.rows,
+    maxCommitSeq:
+      cursor.rows[0] === undefined
+        ? 0
+        : asNumber(cursor.rows[0].max_commit_seq),
+  };
+}
+
 class PostgresTransaction implements StorageTransaction {
   #client: PgQueryable;
   #partition: string;
+  /** Storage-owned relation rewrite, bound to this transaction's partition. */
+  readonly #prepareQuery: (
+    query: AuthoritativeQueryRequest,
+  ) => BoundAuthoritativeQuery;
   /** Schema version this transaction writes; rides every appended commit row. */
   #writerVersion: number;
   #resolveTable: (name: string) => CompiledTable;
@@ -747,6 +776,7 @@ class PostgresTransaction implements StorageTransaction {
     partition: string,
     writerVersion: number,
     resolveTable: (name: string) => CompiledTable,
+    prepareQuery: (query: AuthoritativeQueryRequest) => BoundAuthoritativeQuery,
     resolve: () => void,
     reject: (error: unknown) => void,
   ) {
@@ -770,6 +800,7 @@ class PostgresTransaction implements StorageTransaction {
     this.#partition = partition;
     this.#writerVersion = writerVersion;
     this.#resolveTable = resolveTable;
+    this.#prepareQuery = prepareQuery;
     this.#resolve = resolve;
     this.#reject = reject;
   }
@@ -871,6 +902,19 @@ class PostgresTransaction implements StorageTransaction {
     );
   }
 
+  queryAuthoritative(
+    query: AuthoritativeQueryRequest,
+  ): Promise<AuthoritativeQueryResult> {
+    this.#assertOpen();
+    // Runs on this transaction's pinned client, so the read observes every
+    // staged row and never waits for a second pool connection.
+    return queryAuthoritativeOn(
+      this.#client,
+      this.#partition,
+      this.#prepareQuery(query),
+    );
+  }
+
   async lockPartitionForPush(): Promise<void> {
     this.#assertOpen();
     await lockPartitionOn(this.#client, this.#partition);
@@ -1229,65 +1273,47 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`);
       );
     }
     if (stored !== undefined && stored === schema.version) {
-      // Version equality is not layout equality: a marker written by another
-      // build at the same version describes rows the running codec cannot
-      // decode. Compare the stored layouts instead of trusting the number.
-      const storedLayouts = parseLayouts(
+      assertStoredLayouts(
+        schema,
         typeof marker.rows[0]?.layouts === 'string'
           ? marker.rows[0].layouts
           : undefined,
       );
-      const configuredLayouts = parseLayouts(layoutsOf(schema));
-      let mismatch: string | undefined;
-      for (const [tableName, columns] of Object.entries(configuredLayouts)) {
-        const storedTable = storedLayouts[tableName];
-        const columnCount = Math.max(columns.length, storedTable?.length ?? 0);
-        for (let index = 0; index < columnCount; index++) {
-          const expected = columns[index];
-          const actual = storedTable?.[index];
-          if (
-            expected !== undefined &&
-            actual !== undefined &&
-            actual.name === expected.name &&
-            actual.type === expected.type &&
-            actual.nullable === expected.nullable
-          ) {
-            continue;
-          }
-          mismatch = `table ${JSON.stringify(tableName)} column ${JSON.stringify(expected?.name ?? actual?.name ?? '')}`;
-          break;
-        }
-        if (mismatch !== undefined) break;
-      }
-      if (mismatch === undefined) {
-        for (const tableName of Object.keys(storedLayouts)) {
-          if (!(tableName in configuredLayouts)) {
-            mismatch = `table ${JSON.stringify(tableName)}`;
-            break;
-          }
-        }
-      }
-      if (mismatch !== undefined) {
-        throw new Error(
-          `stored schema layouts disagree with the configured schema at version ${schema.version} (${mismatch}) — refusing to serve a database whose stored rows the running code cannot decode`,
-        );
-      }
-      // The persisted layouts describe the codec's app columns only: read the
-      // physical tables so a same-version database missing a
-      // storage-internal column is refused at startup instead of failing at
-      // the first write. Resolve through `to_regclass`, which follows the
-      // session's `search_path` exactly as the unqualified row queries do —
+      // Resolve through `to_regclass`, which follows the session's
+      // `search_path` exactly as the unqualified row queries do:
       // `current_schema()` names only the first existing schema, so it would
       // reject a table that resolves later in the path.
       for (const table of schema.tables.values()) {
-        const { rows } = await this.#exec.query<{ column_name: string }>(
-          `SELECT attribute.attname AS column_name
+        const { rows } = await this.#exec.query<{
+          name: string;
+          type: string;
+          not_null: boolean;
+          pk: number | null;
+        }>(
+          `SELECT attribute.attname AS name,
+                  format_type(attribute.atttypid, attribute.atttypmod) AS type,
+                  attribute.attnotnull AS not_null,
+                  (SELECT key.position::int
+                     FROM pg_index pk,
+                          unnest(pk.indkey::int2[]) WITH ORDINALITY
+                            AS key(attnum, position)
+                    WHERE pk.indrelid = attribute.attrelid AND pk.indisprimary
+                      AND key.attnum = attribute.attnum) AS pk
            FROM pg_attribute attribute
            WHERE attribute.attrelid = to_regclass($1)
              AND attribute.attnum > 0 AND NOT attribute.attisdropped`,
           [table.name],
         );
-        assertPhysicalColumns(table, new Set(rows.map((r) => r.column_name)));
+        assertPhysicalColumns(
+          table,
+          rows.map((row) => ({
+            name: row.name,
+            type: row.type,
+            notNull: row.not_null,
+            primaryKeyPosition: row.pk ?? 0,
+          })),
+          'postgres',
+        );
       }
     }
     if (stored === undefined || stored < schema.version) {
@@ -1514,6 +1540,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`);
           partition,
           this.#schemaVersion ?? 0,
           (name) => this.table(name),
+          (query) => this.#prepareAuthoritativeQuery(partition, query),
           resolveScope,
           rejectScope,
         );
@@ -1538,17 +1565,16 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`);
     return rows[0] === undefined ? 0 : asNumber(rows[0].max_commit_seq);
   }
 
-  async queryAuthoritative(
+  #prepareAuthoritativeQuery(
     partition: string,
     query: AuthoritativeQueryRequest,
-    checkpoints?: readonly CheckpointDeclaration[],
-  ): Promise<AuthoritativeQueryResult> {
+  ): BoundAuthoritativeQuery {
     if (this.#tables === undefined) {
       throw new Error(
         'ensureSchema(schema) must run before registered queries',
       );
     }
-    const prepared = bindAuthoritativePartition(
+    return bindAuthoritativePartition(
       prepareAuthoritativeQuery(
         query.plan,
         query.params,
@@ -1557,6 +1583,14 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`);
       ),
       partition,
     );
+  }
+
+  async queryAuthoritative(
+    partition: string,
+    query: AuthoritativeQueryRequest,
+    checkpoints?: readonly CheckpointDeclaration[],
+  ): Promise<AuthoritativeQueryResult> {
+    const prepared = this.#prepareAuthoritativeQuery(partition, query);
     return this.#exec.transaction(async (client) => {
       await client.query(
         'SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY',
@@ -1569,21 +1603,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`);
       const gate = await readServeGateOn(client, partition, runningVersion);
       const refusal = serveGateRefusal(gate, runningVersion, checkpoints);
       if (refusal !== undefined) throw serveNotReadyError(refusal);
-      const result = await client.query<Readonly<Record<string, unknown>>>(
-        postgresPlaceholders(prepared.sql),
-        prepared.params,
-      );
-      const cursor = await client.query<{ max_commit_seq: unknown }>(
-        'SELECT max_commit_seq FROM sync_partitions WHERE partition=$1',
-        [partition],
-      );
-      return {
-        rows: result.rows,
-        maxCommitSeq:
-          cursor.rows[0] === undefined
-            ? 0
-            : asNumber(cursor.rows[0].max_commit_seq),
-      };
+      return queryAuthoritativeOn(client, partition, prepared);
     });
   }
 

package/src/push.ts

@@ -54,6 +54,7 @@ import type {
 } from './storage';
 import { serveGateRefusal } from './storage';
 import { StorageConstraintError } from './storage-errors';
+import { transactionQuery } from './storage-query';
 import { serveNotReadyError } from './readiness';
 import type {
   CommitValidationReader,
@@ -186,6 +187,7 @@ async function runValidator(
   values: readonly RowValue[] | undefined,
   storedValues: readonly RowValue[] | undefined,
   opIndex: number,
+  tx: StorageTransaction,
   partition: string,
   actorId: string,
 ): Promise<TerminatingOutcome | undefined> {
@@ -206,7 +208,11 @@ async function runValidator(
             ? toValidateRow(table.columns, storedValues)
             : undefined,
       },
-      { actorId, partition },
+      {
+        actorId,
+        partition,
+        queryAuthoritative: (query) => transactionQuery(tx, query),
+      },
     );
   } catch (error) {
     if (error instanceof ValidationRejection) {
@@ -363,6 +369,7 @@ async function applyOperation(
       undefined,
       storedValues,
       opIndex,
+      tx,
       partition,
       actorId,
     );
@@ -553,6 +560,7 @@ async function applyOperation(
       values,
       storedValues,
       opIndex,
+      tx,
       partition,
       actorId,
     );
@@ -683,6 +691,7 @@ async function applyOperation(
     values,
     undefined,
     opIndex,
+    tx,
     partition,
     actorId,
   );
@@ -836,6 +845,7 @@ function commitValidationReader(
         serverVersion: stored.serverVersion,
       }));
     },
+    queryAuthoritative: (query) => transactionQuery(tx, query),
   };
 }
 
@@ -1084,6 +1094,7 @@ async function enforceReferences(
             undefined,
             storedValues,
             item.originOpIndex,
+            tx,
             partition,
             actorId,
           );
@@ -1117,6 +1128,7 @@ async function enforceReferences(
             values,
             storedValues,
             item.originOpIndex,
+            tx,
             partition,
             actorId,
           );