@syncular/server 0.15.48 → 0.16.1

package/src/pull.ts

@@ -17,10 +17,16 @@ import { clockOf, limitsOf } from './context';
 import type { PullSegmentSummary } from './events';
 import type { CompiledSchema, CompiledTable } from './schema';
 import { scopeDigest } from './scopes';
-import type { SegmentRecord } from './segment-store';
+import type { SegmentRecord, SegmentStore } from './segment-store';
 import { issueSegmentUrl } from './signed-url';
 import type { SqliteImageBuilder } from './sqlite-image';
-import type { StoredCommit, StoredRow } from './storage';
+import type { ServerStorage, StoredCommit, StoredRow } from './storage';
+
+// One artifact build per owning storage pair and complete immutable identity.
+const imageBuilds = new WeakMap<
+  ServerStorage,
+  WeakMap<SegmentStore, Map<string, Promise<SegmentRecord>>>
+>();
 
 /**
  * Resolve the §5.3 image builder: the host-injected one if present, else the
@@ -260,57 +266,89 @@ async function* sqliteImageSegment(
   // support floor, not a fallback (§5.3: sqlite is an *accept*, not a demand).
   const buildImage = await resolveImageBuilder(ctx);
   if (buildImage === undefined) return false;
-  // The probe rows are the snapshot's first page — keep them and scan on
-  // from the probe's cursor instead of re-reading the whole prefix (the
-  // scan is keyset-ordered by rowId, so the concatenation is exactly the
-  // rows a single full scan would return).
-  const rows: StoredRow[] = [...probe];
-  let afterRowId: string | null = probe[probe.length - 1]?.rowId ?? null;
-  for (;;) {
-    const scanned: StoredRow[] = await storage.scanRows(partition, {
-      table: plan.table.name,
-      scopeFilter: plan.effective,
-      afterRowId,
-      limit: 50_000,
-    });
-    rows.push(...scanned);
-    const last = scanned[scanned.length - 1];
-    if (scanned.length < 50_000 || last === undefined) break;
-    afterRowId = last.rowId;
+  let stores = imageBuilds.get(storage);
+  if (stores === undefined) {
+    stores = new WeakMap();
+    imageBuilds.set(storage, stores);
   }
-  const bytes = buildImage({
-    table: plan.table,
+  let builds = stores.get(segments);
+  if (builds === undefined) {
+    builds = new Map();
+    stores.set(segments, builds);
+  }
+  const identity = {
+    partition,
+    logEpoch,
+    table: plan.table.name,
     schemaVersion: schema.version,
-    asOfCommitSeq: asOf,
+    mediaType: 'sqlite' as const,
     scopeDigest: digest,
-    rows,
-  });
-  const record = await segments.put(
-    {
-      partition,
-      logEpoch,
-      table: plan.table.name,
-      schemaVersion: schema.version,
-      mediaType: 'sqlite',
-      scopeDigest: digest,
-      asOfCommitSeq: asOf,
-      rowCount: rows.length,
-      rowCursor: null,
-      nextRowCursor: null,
-    },
-    bytes,
-    now,
-  );
+    asOfCommitSeq: asOf,
+  };
+  const key = JSON.stringify(identity);
+  let building = builds.get(key);
+  let builtHere = false;
+  if (building === undefined) {
+    building = (async () => {
+      // Another request can finish while this one's eligibility probe awaits.
+      const cached = await segments.find(identity, clockOf(ctx)());
+      if (cached !== undefined) return cached;
+      builtHere = true;
+      let rowCount = 0;
+      const bytes = await buildImage({
+        table: plan.table,
+        schemaVersion: schema.version,
+        asOfCommitSeq: asOf,
+        scopeDigest: digest,
+        rowBatches: (async function* () {
+          rowCount += probe.length;
+          yield probe;
+          let afterRowId = probe[probe.length - 1]!.rowId;
+          for (;;) {
+            const rows = await storage.scanRows(partition, {
+              table: plan.table.name,
+              scopeFilter: plan.effective,
+              afterRowId,
+              limit: 5_000,
+            });
+            rowCount += rows.length;
+            yield rows;
+            const last = rows[rows.length - 1];
+            if (rows.length < 5_000 || last === undefined) break;
+            afterRowId = last.rowId;
+          }
+        })(),
+      });
+      return segments.put(
+        { ...identity, rowCount, rowCursor: null, nextRowCursor: null },
+        bytes,
+        clockOf(ctx)(),
+      );
+    })();
+    builds.set(key, building);
+  }
+  let record: SegmentRecord;
+  try {
+    record = await building;
+  } finally {
+    if (builds.get(key) === building) builds.delete(key);
+  }
   trace?.segments.push({
     mediaType: 'sqlite',
     delivery: 'ref',
-    origin: 'built',
+    origin: builtHere ? 'built' : 'reused',
     bytes: record.byteLength,
     rows: record.rowCount,
   });
   yield segmentRefFrame(
     record,
-    await signedUrlFields(ctx, limits, record.segmentId, digest, now),
+    await signedUrlFields(
+      ctx,
+      limits,
+      record.segmentId,
+      digest,
+      clockOf(ctx)(),
+    ),
   );
   return true;
 }

package/src/push.ts

@@ -185,7 +185,7 @@ async function runValidator(
     return errorRecord(
       opIndex,
       'sync.constraint_violation',
-      `write validator for table ${JSON.stringify(table.name)} threw: ${error instanceof Error ? error.message : String(error)}`,
+      'write validator failed',
     );
   }
   return undefined;
@@ -220,7 +220,7 @@ async function mergeCrdtColumns(
       return errorRecord(
         opIndex,
         'sync.crdt_merge_failed',
-        `no CRDT merger registered for crdtType ${JSON.stringify(crdtType)} (§5.10.2)`,
+        'no CRDT merger registered',
       );
     }
     const storedRaw = storedValues?.[index];
@@ -228,11 +228,11 @@ async function mergeCrdtColumns(
     let merged: Uint8Array;
     try {
       merged = await merger(stored, incoming);
-    } catch (error) {
+    } catch {
       return errorRecord(
         opIndex,
         'sync.crdt_merge_failed',
-        `CRDT merger for ${JSON.stringify(crdtType)} threw: ${error instanceof Error ? error.message : String(error)}`,
+        'CRDT merger failed',
       );
     }
     values[index] = merged;
@@ -732,7 +732,7 @@ async function runCommitValidator(
     return errorRecord(
       operations[0]?.opIndex ?? 0,
       'sync.constraint_violation',
-      `whole-commit validator threw: ${error instanceof Error ? error.message : String(error)}`,
+      'whole-commit validator failed',
     );
   }
   return undefined;

package/src/sqlite-bun.ts

@@ -40,10 +40,10 @@ export class SqliteLeaseStore extends SharedSqliteLeaseStore {
   }
 }
 
-export const buildSqliteImage: SqliteImageBuilder = (input) => {
+export const buildSqliteImage: SqliteImageBuilder = async (input) => {
   const db = new BunSqliteDatabase();
   try {
-    writeSqliteImage(db, input);
+    await writeSqliteImage(db, input);
     return db.serialize();
   } finally {
     db.close();

package/src/sqlite-image.ts

@@ -57,7 +57,9 @@ export interface SqliteImageInput {
   readonly schemaVersion: number;
   readonly asOfCommitSeq: number;
   readonly scopeDigest: string;
-  readonly rows: readonly StoredRow[];
+  readonly rowBatches:
+    | AsyncIterable<readonly StoredRow[]>
+    | Iterable<readonly StoredRow[]>;
 }
 
 /**
@@ -68,14 +70,16 @@ export interface SqliteImageInput {
  * on the pull path. A Bun or Node host passes `buildSqliteImage`; a Workers
  * host omits it and serves the rows lane.
  */
-export type SqliteImageBuilder = (input: SqliteImageInput) => Uint8Array;
+export type SqliteImageBuilder = (
+  input: SqliteImageInput,
+) => Promise<Uint8Array>;
 
 /** Populate a §5.3 image database for a whole-table snapshot. */
-export function writeSqliteImage(
+export async function writeSqliteImage(
   db: SqliteDatabase,
   input: SqliteImageInput,
-): void {
-  const { table, rows } = input;
+): Promise<void> {
+  const { table, rowBatches } = input;
   const primaryKey = table.columns[table.primaryKeyIndex]?.name;
   const columnDefs = table.columns.map((column) => {
     const notNull = column.nullable ? '' : ' NOT NULL';
@@ -90,13 +94,6 @@ export function writeSqliteImage(
       "schemaVersion" INTEGER NOT NULL, "asOfCommitSeq" INTEGER NOT NULL,
       "scopeDigest" TEXT NOT NULL, "rowCount" INTEGER NOT NULL)`,
   );
-  db.query(`INSERT INTO ${IMAGE_METADATA_TABLE} VALUES (1, ?, ?, ?, ?, ?)`).run(
-    table.name,
-    input.schemaVersion,
-    input.asOfCommitSeq,
-    input.scopeDigest,
-    rows.length,
-  );
   const names = [
     ...table.columns.map((column) => quoteIdent(column.name)),
     quoteIdent(IMAGE_VERSION_COLUMN),
@@ -107,10 +104,24 @@ export function writeSqliteImage(
   );
   db.exec('BEGIN');
   try {
-    for (const row of rows) {
-      const values = decodeRow(table.columns, row.payload);
-      insert.run(...values.map(toSql), row.serverVersion);
+    let rowCount = 0;
+    for await (const rows of rowBatches) {
+      for (const row of rows) {
+        const values = decodeRow(table.columns, row.payload);
+        insert.run(...values.map(toSql), row.serverVersion);
+      }
+      rowCount += rows.length;
     }
+    db.query(
+      `INSERT INTO ${IMAGE_METADATA_TABLE} VALUES (1, ?, ?, ?, ?, ?)`,
+    ).run(
+      table.name,
+      input.schemaVersion,
+      input.asOfCommitSeq,
+      input.scopeDigest,
+      rowCount,
+    );
+
     db.exec('COMMIT');
   } catch (error) {
     db.exec('ROLLBACK');

package/src/sqlite-node.ts

@@ -43,13 +43,13 @@ export class SqliteLeaseStore extends SharedSqliteLeaseStore {
   }
 }
 
-export const buildSqliteImage: SqliteImageBuilder = (input) => {
+export const buildSqliteImage: SqliteImageBuilder = async (input) => {
   const directory = mkdtempSync(join(tmpdir(), 'syncular-server-image-'));
   const path = join(directory, 'segment.db');
   const db = new NodeSqliteDatabase(path);
   try {
     try {
-      writeSqliteImage(db, input);
+      await writeSqliteImage(db, input);
     } finally {
       db.close();
     }

package/src/sqlite-storage.ts

@@ -1,3 +1,6 @@
+import { validateCommitPruneQuery } from './prune';
+import { StorageQueryError } from './storage-errors';
+import type { CommitPruneQuery, CommitPruneResult } from './storage';
 /**
  * SQLite server storage over the shared synchronous driver.
  *
@@ -390,7 +393,7 @@ export class SqliteServerStorage implements ServerStorage {
   #tables: ReadonlyMap<string, CompiledTable> | undefined;
   #schemaVersion: number | undefined;
 
-  async #serializeReactionWrite<T>(operation: () => T): Promise<T> {
+  async #serializeWrite<T>(operation: () => T): Promise<T> {
     const previous = this.#transactionTail;
     let release!: () => void;
     this.#transactionTail = new Promise<void>((resolve) => {
@@ -559,7 +562,7 @@ export class SqliteServerStorage implements ServerStorage {
     authenticatedAtMs: number,
   ): Promise<PartitionRegistryEntry> {
     if (logEpoch.length === 0) throw new Error('log epoch must be non-empty');
-    return this.#serializeReactionWrite(() => {
+    return this.#serializeWrite(() => {
       this.db.exec('BEGIN IMMEDIATE');
       try {
         this.db
@@ -725,7 +728,7 @@ export class SqliteServerStorage implements ServerStorage {
     }
     const prepared = bindAuthoritativePartition(
       prepareAuthoritativeQuery(
-        query.sql,
+        query.plan,
         query.params,
         query.tables,
         this.#tables,
@@ -763,6 +766,14 @@ export class SqliteServerStorage implements ServerStorage {
     }
   }
 
+  async getPartitionLogEpoch(partition: string): Promise<string | undefined> {
+    return this.db
+      .query<{ log_epoch: string }, [string]>(
+        'SELECT log_epoch FROM sync_partition_registry WHERE partition=?',
+      )
+      .get(partition)?.log_epoch;
+  }
+
   async getHorizonSeq(partition: string): Promise<number> {
     const row = this.db
       .query<{ horizon_seq: number }, [string]>(
@@ -773,27 +784,62 @@ export class SqliteServerStorage implements ServerStorage {
   }
 
   async setHorizonSeq(partition: string, seq: number): Promise<void> {
-    this.db
-      .query('INSERT OR IGNORE INTO sync_partitions(partition) VALUES (?)')
-      .run(partition);
-    this.db
-      .query('UPDATE sync_partitions SET horizon_seq=? WHERE partition=?')
-      .run(seq, partition);
+    await this.#serializeWrite(() => {
+      this.db
+        .query(`INSERT INTO sync_partitions(partition, horizon_seq) VALUES (?,?)
+        ON CONFLICT(partition) DO UPDATE SET horizon_seq=max(horizon_seq,excluded.horizon_seq)`)
+        .run(partition, seq);
+    });
   }
 
-  async pruneCommitsThrough(partition: string, seq: number): Promise<number> {
-    const removed = this.db
-      .query('DELETE FROM sync_commits WHERE partition=? AND commit_seq<=?')
-      .run(partition, seq);
-    this.db
-      .query('DELETE FROM sync_changes WHERE partition=? AND commit_seq<=?')
-      .run(partition, seq);
-    this.db
-      .query(
-        'DELETE FROM sync_change_scopes WHERE partition=? AND commit_seq<=?',
-      )
-      .run(partition, seq);
-    return Number(removed.changes);
+  async pruneCommitsThrough(
+    partition: string,
+    query: CommitPruneQuery,
+  ): Promise<CommitPruneResult> {
+    validateCommitPruneQuery(query);
+    return this.#serializeWrite(() => {
+      this.db.exec('BEGIN IMMEDIATE');
+      try {
+        const epoch = this.db
+          .query<{ log_epoch: string }, [string]>(
+            'SELECT log_epoch FROM sync_partition_registry WHERE partition=?',
+          )
+          .get(partition)?.log_epoch;
+        if (epoch !== query.logEpoch)
+          throw new StorageQueryError('sync.storage.prune_epoch_mismatch');
+        const previousHorizonSeq =
+          this.db
+            .query<{ horizon_seq: number }, [string]>(
+              'SELECT horizon_seq FROM sync_partitions WHERE partition=?',
+            )
+            .get(partition)?.horizon_seq ?? 0;
+        const horizonSeq = Math.max(previousHorizonSeq, query.throughSeq);
+        this.db
+          .query(`INSERT INTO sync_partitions(partition, horizon_seq) VALUES (?,?)
+          ON CONFLICT(partition) DO UPDATE SET horizon_seq=excluded.horizon_seq`)
+          .run(partition, horizonSeq);
+        const removed = this.db
+          .query('DELETE FROM sync_commits WHERE partition=? AND commit_seq<=?')
+          .run(partition, horizonSeq);
+        this.db
+          .query('DELETE FROM sync_changes WHERE partition=? AND commit_seq<=?')
+          .run(partition, horizonSeq);
+        this.db
+          .query(
+            'DELETE FROM sync_change_scopes WHERE partition=? AND commit_seq<=?',
+          )
+          .run(partition, horizonSeq);
+        this.db.exec('COMMIT');
+        return {
+          previousHorizonSeq,
+          horizonSeq,
+          removedCommits: Number(removed.changes),
+        };
+      } catch (error) {
+        this.db.exec('ROLLBACK');
+        throw error;
+      }
+    });
   }
 
   async getCommitSeqBefore(
@@ -847,7 +893,7 @@ export class SqliteServerStorage implements ServerStorage {
     query: ReactionClaimQuery,
   ): Promise<StoredReaction[]> {
     if (query.types.length === 0 || query.limit <= 0) return [];
-    return this.#serializeReactionWrite(() => {
+    return this.#serializeWrite(() => {
       const typeParams = query.types.map(() => '?').join(',');
       const records = this.db
         .query<SqliteReactionRecord, (string | number)[]>(
@@ -895,7 +941,7 @@ export class SqliteServerStorage implements ServerStorage {
     leaseOwner: string,
     completedAtMs: number,
   ): Promise<boolean> {
-    return this.#serializeReactionWrite(() => {
+    return this.#serializeWrite(() => {
       const result = this.db
         .query(
           `UPDATE sync_reactions
@@ -915,7 +961,7 @@ export class SqliteServerStorage implements ServerStorage {
     leaseOwner: string,
     leaseExpiresAtMs: number,
   ): Promise<boolean> {
-    return this.#serializeReactionWrite(() => {
+    return this.#serializeWrite(() => {
       const result = this.db
         .query(
           `UPDATE sync_reactions SET lease_expires_at_ms=?
@@ -933,7 +979,7 @@ export class SqliteServerStorage implements ServerStorage {
     update: ReactionFailureUpdate,
   ): Promise<boolean> {
     const retry = update.retryAtMs !== undefined;
-    return this.#serializeReactionWrite(() => {
+    return this.#serializeWrite(() => {
       const result = this.db
         .query(
           `UPDATE sync_reactions
@@ -959,7 +1005,7 @@ export class SqliteServerStorage implements ServerStorage {
     idempotencyKey: string,
     nowMs: number,
   ): Promise<boolean> {
-    return this.#serializeReactionWrite(() => {
+    return this.#serializeWrite(() => {
       const result = this.db
         .query(
           `UPDATE sync_reactions
@@ -1014,7 +1060,7 @@ export class SqliteServerStorage implements ServerStorage {
     query: ReactionPruneQuery,
   ): Promise<PrunedReactionCounts> {
     if (query.limit <= 0) return { completed: 0, deadLetter: 0 };
-    return this.#serializeReactionWrite(() => {
+    return this.#serializeWrite(() => {
       const records = this.db
         .query<
           { status: 'completed' | 'dead-letter' },
@@ -1226,6 +1272,18 @@ export class SqliteServerStorage implements ServerStorage {
       );
   }
 
+  async getActiveClientCursorFloor(
+    partition: string,
+    cutoffMs: number,
+  ): Promise<number | null> {
+    const row = this.db
+      .query<{ cursor: number | null }, [string, number]>(
+        'SELECT MIN(cursor) AS cursor FROM sync_clients WHERE partition=? AND updated_at_ms>=?',
+      )
+      .get(partition, cutoffMs);
+    return row!.cursor;
+  }
+
   async listClientCursors(partition: string): Promise<ClientCursorInfo[]> {
     const records = this.db
       .query<

package/src/storage-errors.ts

@@ -19,10 +19,19 @@ export type StorageQueryErrorCode =
   | 'sync.storage.index_not_found'
   | 'sync.storage.index_not_materialized'
   | 'sync.storage.index_value_count_mismatch'
-  | 'sync.storage.invalid_limit';
+  | 'sync.storage.invalid_limit'
+  | 'sync.storage.prune_epoch_mismatch'
+  | 'sync.storage.partition_unregistered'
+  | 'sync.storage.invalid_prune_cursor';
 
 const STORAGE_QUERY_MESSAGES: Readonly<Record<StorageQueryErrorCode, string>> =
   {
+    'sync.storage.prune_epoch_mismatch':
+      'partition log epoch changed; recompute retention inputs',
+    'sync.storage.partition_unregistered':
+      'pruning requires a registered partition',
+    'sync.storage.invalid_prune_cursor':
+      'pruning requires a non-negative safe integer cursor and a non-empty log epoch',
     'sync.storage.scan_requires_scope':
       'scope-indexed row scans require at least one scope variable',
     'sync.storage.index_not_found':

package/src/storage.ts

@@ -1,3 +1,4 @@
+import type { AuthoritativeRelationPlan } from './authoritative-query';
 /**
  * Storage interface (defined by the SPEC's needs, implementation-agnostic).
  *
@@ -19,6 +20,17 @@
 import type { PushOperationResult, RowValue, ScopeMap } from '@syncular/core';
 import type { CompiledSchema } from './schema';
 
+export interface CommitPruneQuery {
+  readonly logEpoch: string;
+  readonly throughSeq: number;
+}
+
+export interface CommitPruneResult {
+  readonly previousHorizonSeq: number;
+  readonly horizonSeq: number;
+  readonly removedCommits: number;
+}
+
 /** The current stored state of a synced row. */
 export interface StoredRow {
   readonly rowId: string;
@@ -281,7 +293,7 @@ export type AuthoritativeQueryValue =
 
 export interface AuthoritativeQueryRequest {
   /** Generated, positional SQLite-family SQL. It never comes from the request. */
-  readonly sql: string;
+  readonly plan: AuthoritativeRelationPlan;
   readonly params: readonly AuthoritativeQueryValue[];
   /** Generated dependency set, used to validate and partition every relation. */
   readonly tables: readonly string[];
@@ -422,16 +434,23 @@ export interface ServerStorage {
   /** Registry entries ordered by partition for maintenance loops. */
   listPartitionRegistry(): Promise<PartitionRegistryEntry[]>;
 
+  /** Read continuity without refreshing authenticated activity. */
+  getPartitionLogEpoch(partition: string): Promise<string | undefined>;
+
   begin(partition: string): Promise<StorageTransaction>;
 
   getMaxCommitSeq(partition: string): Promise<number>;
   getHorizonSeq(partition: string): Promise<number>;
+  /** Monotonic within the current epoch; use atomic pruning for maintenance. */
   setHorizonSeq(partition: string, seq: number): Promise<void>;
   /**
-   * Deletes commits with `commitSeq <= seq` (log, changes, scope index).
-   * Returns the number of commits removed (ops observability).
+   * Atomically verifies the log epoch, advances the horizon monotonically,
+   * and removes log/change/scope records through the effective horizon.
    */
-  pruneCommitsThrough(partition: string, seq: number): Promise<number>;
+  pruneCommitsThrough(
+    partition: string,
+    query: CommitPruneQuery,
+  ): Promise<CommitPruneResult>;
   /** Newest commitSeq created strictly before the timestamp; 0 if none. */
   getCommitSeqBefore(
     partition: string,
@@ -541,7 +560,12 @@ export interface ServerStorage {
     clientId: string,
   ): Promise<ClientRecord | undefined>;
   putClientRecord(partition: string, record: ClientRecord): Promise<void>;
-  /** Cursor records feeding the §4.6 retention watermark. */
+  /** Minimum cursor with updatedAtMs >= cutoff; null when none are active. */
+  getActiveClientCursorFloor(
+    partition: string,
+    cutoffMs: number,
+  ): Promise<number | null>;
+  /** Cursor records for client listings and administrative counts. */
   listClientCursors(partition: string): Promise<ClientCursorInfo[]>;
 
   /**