@syncular/server 0.26.4 → 0.26.6

package/src/push.ts

@@ -696,8 +696,12 @@ async function applyOperation(
     actorId,
   );
   if (insertReject !== undefined) return insertReject;
-  // §5.2: an explicit insert recreates — the tombstone goes with it.
-  await tx.clearTombstone(op.table, op.rowId);
+  // §5.2: an explicit insert recreates — the tombstone goes with it. An
+  // unversioned insert already proved above, under the partition lock, that
+  // no tombstone exists (this commit writes its own only in `appendCommit`).
+  if (op.baseVersion !== undefined) {
+    await tx.clearTombstone(op.table, op.rowId);
+  }
   const newRow = {
     rowId: op.rowId,
     serverVersion: 1,
@@ -1415,6 +1419,18 @@ export async function processPushOperationsWithTrace(
     const stagedWrites: StagedWrite[] = [];
     let terminated: PushOperationResult | undefined;
     const operations = await buildOperations(tx);
+    if (tx.prefetchRows !== undefined) {
+      const targets = new Map<string, string[]>();
+      for (const op of operations) {
+        if (!schema.tables.has(op.table)) continue;
+        const ids = targets.get(op.table) ?? [];
+        ids.push(op.rowId);
+        targets.set(op.table, ids);
+      }
+      for (const [table, rowIds] of targets) {
+        await tx.prefetchRows(table, rowIds);
+      }
+    }
     for (let opIndex = 0; opIndex < operations.length; opIndex++) {
       const op = operations[opIndex];
       if (op === undefined) continue;

package/src/realtime.ts

@@ -39,7 +39,13 @@ import {
   RESOLVER_OUTAGE,
   touchAuthenticatedPartition,
 } from './context';
-import { SyncError, syncError } from './errors';
+import {
+  adapterSyncError,
+  reportError,
+  SyncError,
+  type SyncularErrorHandler,
+  syncError,
+} from './errors';
 import { emitEvent, type SyncularServerEvents } from './events';
 import { createSyncResponseStream } from './handler';
 import { type CompiledSchema, compileSchema } from './schema';
@@ -581,19 +587,18 @@ export class RealtimeSession {
         );
       } catch (error) {
         // §8.7 failures: what HTTP reports as status+JSON becomes a
-        // minimal RESP_HEADER/ERROR/END response stream on the socket.
-        if (error instanceof DecodeError || error instanceof SyncError) {
-          const sync =
-            error instanceof SyncError
-              ? error
-              : syncError(error.code, error.message);
-          finishRound(); // END is in this one chunk
-          await this.#sendRoundChunk(
-            errorResponseBytes(sync, this.wireVersion, this.logEpoch),
-          );
-          return;
-        }
-        throw error;
+        // minimal RESP_HEADER/ERROR/END response stream on the socket. An
+        // exception outside the catalog is reported and answers
+        // `sync.internal_error`, exactly like the HTTP adapter's 500.
+        const sync =
+          error instanceof DecodeError
+            ? syncError(error.code, error.message)
+            : adapterSyncError(error, this.#hub.onError, 'realtime');
+        finishRound(); // END is in this one chunk
+        await this.#sendRoundChunk(
+          errorResponseBytes(sync, this.wireVersion, this.logEpoch),
+        );
+        return;
       }
       // Fetch-ahead so the round stops being "in flight" the moment the
       // chunk carrying END is handed to send: the client may legally
@@ -604,23 +609,22 @@ export class RealtimeSession {
       try {
         step = await iterator.next();
       } catch (error) {
-        // The buffered read-verify refusal is thrown on the generator's first
-        // `next()`, before any chunk is yielded (the generator builds the whole
-        // response, then verifies). Nothing has escaped, so it is delivered as
-        // the same §8.4 ERROR frame HTTP returns rather than a silent close. A
-        // throw after the first chunk still takes the §8.7 violation path.
-        if (error instanceof DecodeError || error instanceof SyncError) {
-          const sync =
-            error instanceof SyncError
-              ? error
-              : syncError(error.code, error.message);
-          finishRound();
-          await this.#sendRoundChunk(
-            errorResponseBytes(sync, this.wireVersion, this.logEpoch),
-          );
-          return;
-        }
-        throw error;
+        // The generator builds the whole response (and runs the read-verify
+        // refusal) on its first `next()`, before any chunk is yielded.
+        // Nothing has escaped, so a failure there is delivered as the same
+        // §8.4 ERROR frame HTTP returns rather than a silent close; a
+        // non-catalog exception is reported and answers
+        // `sync.internal_error`. A throw after the first chunk still takes
+        // the §8.7 violation path.
+        const sync =
+          error instanceof DecodeError
+            ? syncError(error.code, error.message)
+            : adapterSyncError(error, this.#hub.onError, 'realtime');
+        finishRound();
+        await this.#sendRoundChunk(
+          errorResponseBytes(sync, this.wireVersion, this.logEpoch),
+        );
+        return;
       }
       while (!step.done) {
         const chunk = step.value;
@@ -628,9 +632,10 @@ export class RealtimeSession {
         if (step.done) finishRound();
         await this.#sendRoundChunk(chunk);
       }
-    } catch {
+    } catch (error) {
       // A host failure mid-stream leaves the byte stream unfinishable —
       // fail loud, drop the connection (§8.7 / §1.4 abort rule).
+      reportError(error, this.#hub.onError, 'realtime');
       this.#violation('sync round failed mid-stream');
     } finally {
       finishRound();
@@ -1059,10 +1064,18 @@ export class RealtimeHub {
       ...(this.#config.events !== undefined
         ? { events: this.#config.events }
         : {}),
+      ...(this.#config.onError !== undefined
+        ? { onError: this.#config.onError }
+        : {}),
       realtime: this,
     };
   }
 
+  /** The host's error reporter (`SyncServerConfig.onError`), if any. */
+  get onError(): SyncularErrorHandler | undefined {
+    return this.#config.onError;
+  }
+
   requestContext(session: RealtimeSession): SyncRequestContext {
     return this.requestContextFor(session);
   }

package/src/relational-rows.ts

@@ -691,6 +691,49 @@ export const COMMIT_WINDOW_BATCH_SQL = `SELECT s.i AS sub, c.commit_seq AS commi
        ON ch.partition=$1 AND ch.commit_seq=c.commit_seq AND ch.tbl=$3
      ORDER BY s.i, c.commit_seq, ch.idx`;
 
+/**
+ * PostgreSQL push read-ahead: every requested row id with its stored row
+ * (NULL columns when absent, `selectRowSql` shape) and its §5 delete
+ * tombstone. Params: [partition, rowIds as JSON array text, tbl].
+ */
+export function prefetchRowsSql(table: CompiledTable): string {
+  return `SELECT ids.id AS lookup_id, r.${quoteIdent(SYNC_ROW_ID_COLUMN)} AS row_id, r.${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, r.${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, r.${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, r.${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions, t.commit_seq AS tombstone_seq
+     FROM jsonb_array_elements_text($2::text::jsonb) AS ids(id)
+     LEFT JOIN ${quoteIdent(table.name)} r
+       ON r.${quoteIdent(SYNC_PARTITION_COLUMN)}=$1 AND r.${quoteIdent(SYNC_ROW_ID_COLUMN)}=ids.id
+     LEFT JOIN sync_tombstones t
+       ON t.partition=$1 AND t.tbl=$3 AND t.row_id=ids.id`;
+}
+
+/**
+ * SQLite/D1: the newest `commit_seq` in the inverted change-scope index for
+ * one variable's values, as one index-only max per value (a `(partition,
+ * tbl, var, value)` prefix), so the cost is independent of the table's
+ * history. Bind order: [partition, tbl, var, value] per value.
+ */
+export function latestScopedChangeSeqSql(valueCount: number): string {
+  const perValue: string[] = [];
+  for (let i = 0; i < valueCount; i++) {
+    perValue.push(
+      'SELECT (SELECT max(commit_seq) FROM sync_change_scopes WHERE partition=? AND tbl=? AND var=? AND value=?) AS seq',
+    );
+  }
+  return `SELECT max(seq) AS seq FROM (${perValue.join(' UNION ALL ')})`;
+}
+
+/**
+ * PostgreSQL set-based form of `latestScopedChangeSeqSql` for the
+ * subscriptions of one table in a pull. `$2` is a JSON array of
+ * `{i, var, vals}` records. Params: [partition, specs (JSON text), tbl].
+ */
+export const LATEST_SCOPED_CHANGE_BATCH_SQL = `SELECT s.i AS sub,
+       (SELECT max(latest.seq) FROM (
+          SELECT (SELECT max(commit_seq) FROM sync_change_scopes
+                   WHERE partition=$1 AND tbl=$3 AND var=s.var AND value=v.value) AS seq
+            FROM jsonb_array_elements_text(s.vals) AS v(value)) AS latest) AS seq
+     FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, var text, vals jsonb)
+    ORDER BY s.i`;
+
 /** SELECT a row's version + scope map (admin/blob authz). Params: [partition, rowId]. */
 export function selectRowScopesSql(
   table: CompiledTable,

package/src/s3-segment-store.ts

@@ -49,6 +49,7 @@ import type {
   SegmentRecord,
   SegmentStore,
   SegmentStoreStats,
+  SegmentStream,
 } from './segment-store';
 import {
   mergeSegmentRecord,
@@ -619,6 +620,20 @@ export class S3SegmentStore implements SegmentStore {
     };
   }
 
+  /** The record plus the object GET's body, never buffered (see `get`). */
+  async open(segmentId: string): Promise<SegmentStream | undefined> {
+    if (!SEGMENT_ID_PATTERN.test(segmentId)) return undefined;
+    const record = await this.#readRecord(segmentId);
+    if (record === undefined) return undefined;
+    const response = await this.#request('GET', this.objectKeyFor(segmentId));
+    if (response === undefined || response.body === null) return undefined;
+    return {
+      record: record.record,
+      body: response.body,
+      byteLength: record.record.byteLength,
+    };
+  }
+
   async find(
     key: SegmentFindKey,
     nowMs: number,

package/src/segment-download.ts

@@ -19,7 +19,11 @@ import { SyncError, syncError } from './errors';
 import { emitEvent } from './events';
 import { compileSchema } from './schema';
 import { computeEffective, type ResolvedScopes, scopeDigest } from './scopes';
-import { publicationRecord, type SegmentRecord } from './segment-store';
+import {
+  bytesStream,
+  publicationRecord,
+  type SegmentRecord,
+} from './segment-store';
 
 export interface SegmentDownloadRequest {
   readonly segmentId: string;
@@ -34,6 +38,15 @@ export interface SegmentDownloadResult {
   readonly headers: Record<string, string>;
 }
 
+/** `openSegmentDownload`: the authorized segment as a byte stream. */
+export interface SegmentStreamResult {
+  readonly record: SegmentRecord;
+  readonly body: ReadableStream<Uint8Array<ArrayBuffer>>;
+  readonly byteLength: number;
+  /** Response headers per §5.5. */
+  readonly headers: Record<string, string>;
+}
+
 function parseScopesHeader(header: string): ScopeMap {
   let parsed: unknown;
   try {
@@ -60,16 +73,52 @@ function parseScopesHeader(header: string): ScopeMap {
   return scopes;
 }
 
-export async function handleSegmentDownload(
+export function handleSegmentDownload(
   ctx: SyncRequestContext,
   request: SegmentDownloadRequest,
 ): Promise<SegmentDownloadResult> {
+  return withDownloadEvent(
+    ctx,
+    request,
+    () => downloadSegment(ctx, request),
+    (result) => result.bytes.length,
+  );
+}
+
+/**
+ * The §5.5 download as a stream: the same authorization as
+ * `handleSegmentDownload`, with the bytes read through `SegmentStore.open`
+ * when the store provides it, so a host with bounded memory (a Worker)
+ * relays a large sqlite image without buffering it. A store without `open`
+ * serves its `get` bytes as a one-chunk stream.
+ */
+export function openSegmentDownload(
+  ctx: SyncRequestContext,
+  request: SegmentDownloadRequest,
+): Promise<SegmentStreamResult> {
+  return withDownloadEvent(
+    ctx,
+    request,
+    () => openSegment(ctx, request),
+    (result) => result.byteLength,
+  );
+}
+
+/** Emit `segment.downloaded` around a download when an events sink exists. */
+async function withDownloadEvent<
+  Result extends { readonly record: SegmentRecord },
+>(
+  ctx: SyncRequestContext,
+  request: SegmentDownloadRequest,
+  run: () => Promise<Result>,
+  bytesOf: (result: Result) => number,
+): Promise<Result> {
   const events = ctx.events;
-  if (events === undefined) return downloadSegment(ctx, request);
+  if (events === undefined) return run();
   const clock = clockOf(ctx);
   const startedAtMs = clock();
   try {
-    const result = await downloadSegment(ctx, request);
+    const result = await run();
     emitEvent(events, {
       type: 'segment.downloaded',
       atMs: clock(),
@@ -78,7 +127,7 @@ export async function handleSegmentDownload(
       segmentId: request.segmentId,
       outcome: 'ok',
       mediaType: result.record.mediaType,
-      bytes: result.bytes.length,
+      bytes: bytesOf(result),
       durationMs: clock() - startedAtMs,
     });
     return result;
@@ -106,13 +155,82 @@ async function downloadSegment(
   if (entry === undefined) {
     throw syncError('sync.not_found', 'unknown segment (§5.5)');
   }
+  const record = await authorizePublication(
+    ctx,
+    request,
+    registry.logEpoch,
+    entry.record,
+  );
+  return { record, bytes: entry.bytes, headers: segmentHeaders(record) };
+}
+
+async function openSegment(
+  ctx: SyncRequestContext,
+  request: SegmentDownloadRequest,
+): Promise<SegmentStreamResult> {
+  const registry = await touchAuthenticatedPartition(ctx);
+  const opened =
+    ctx.segments.open !== undefined
+      ? await ctx.segments.open(request.segmentId)
+      : await ctx.segments.get(request.segmentId).then((entry) =>
+          entry === undefined
+            ? undefined
+            : {
+                record: entry.record,
+                body: bytesStream(entry.bytes),
+                byteLength: entry.bytes.length,
+              },
+        );
+  if (opened === undefined) {
+    throw syncError('sync.not_found', 'unknown segment (§5.5)');
+  }
+  let record: SegmentRecord;
+  try {
+    record = await authorizePublication(
+      ctx,
+      request,
+      registry.logEpoch,
+      opened.record,
+    );
+  } catch (error) {
+    await opened.body.cancel();
+    throw error;
+  }
+  return {
+    record,
+    body: opened.body,
+    byteLength: opened.byteLength,
+    headers: segmentHeaders(record),
+  };
+}
+
+function segmentHeaders(record: SegmentRecord): Record<string, string> {
+  return {
+    'Content-Type': 'application/octet-stream',
+    ETag: `"${record.segmentId}"`,
+    'Cache-Control': 'private, max-age=0',
+    // Accept-Encoding: the body may be served compressed (§5.8).
+    Vary: 'Authorization, X-Syncular-Scopes, Accept-Encoding',
+  };
+}
+
+/**
+ * Select and authorize the caller's publication of a stored segment
+ * (§5.5), projected onto that publication.
+ */
+async function authorizePublication(
+  ctx: SyncRequestContext,
+  request: SegmentDownloadRequest,
+  logEpoch: string,
+  stored: SegmentRecord,
+): Promise<SegmentRecord> {
   // Select the caller's own publication context. A publication under a
   // different partition or a stale log epoch is not this caller's grant, and
   // the entry's existence must not leak across that boundary (§5.5).
-  const inContext = entry.record.publications.filter(
+  const inContext = stored.publications.filter(
     (publication) =>
       publication.partition === ctx.partition &&
-      publication.logEpoch === registry.logEpoch,
+      publication.logEpoch === logEpoch,
   );
   if (inContext.length === 0) {
     throw syncError('sync.not_found', 'unknown segment (§5.5)');
@@ -212,18 +330,8 @@ async function downloadSegment(
     );
   }
 
-  return {
-    // The returned record is projected onto the selected publication, so its
-    // `scopeDigests`/`publications` compatibility view cannot union a grant
-    // from another partition (§5.1).
-    record: publicationRecord(entry.record, publication),
-    bytes: entry.bytes,
-    headers: {
-      'Content-Type': 'application/octet-stream',
-      ETag: `"${entry.record.segmentId}"`,
-      'Cache-Control': 'private, max-age=0',
-      // Accept-Encoding: the body may be served compressed (§5.8).
-      Vary: 'Authorization, X-Syncular-Scopes, Accept-Encoding',
-    },
-  };
+  // The returned record is projected onto the selected publication, so its
+  // `scopeDigests`/`publications` compatibility view cannot union a grant
+  // from another partition (§5.1).
+  return publicationRecord(stored, publication);
 }

package/src/segment-store.ts

@@ -283,6 +283,25 @@ export interface SegmentStoreStats {
   readonly approximate?: boolean;
 }
 
+/** `SegmentStore.open`: a stored segment with its bytes as a stream. */
+export interface SegmentStream {
+  readonly record: SegmentRecord;
+  readonly body: ReadableStream<Uint8Array<ArrayBuffer>>;
+  readonly byteLength: number;
+}
+
+/** One buffered segment as a single-chunk `SegmentStream` body. */
+export function bytesStream(
+  bytes: Uint8Array,
+): ReadableStream<Uint8Array<ArrayBuffer>> {
+  return new ReadableStream({
+    start(controller) {
+      controller.enqueue(new Uint8Array(bytes));
+      controller.close();
+    },
+  });
+}
+
 export interface SegmentStore {
   put(
     metadata: SegmentMetadata,
@@ -307,6 +326,13 @@ export interface SegmentStore {
    * against the descriptor.
    */
   find(key: SegmentFindKey, nowMs: number): Promise<SegmentRecord | undefined>;
+  /**
+   * Optional streaming read: the record plus the bytes as a stream, so the
+   * download route relays a segment larger than the host's memory (a
+   * sqlite image through a Worker). Returns expired records like `get`.
+   * Stores without it are served through `get`.
+   */
+  open?(segmentId: string): Promise<SegmentStream | undefined>;
   /** Admin/console counters: ADDITIVE, optional. */
   stats?(): Promise<SegmentStoreStats>;
 }
@@ -353,6 +379,16 @@ export class MemorySegmentStore implements SegmentStore {
     return this.#entries.get(segmentId);
   }
 
+  async open(segmentId: string): Promise<SegmentStream | undefined> {
+    const entry = this.#entries.get(segmentId);
+    if (entry === undefined) return undefined;
+    return {
+      record: entry.record,
+      body: bytesStream(entry.bytes),
+      byteLength: entry.bytes.length,
+    };
+  }
+
   async find(
     key: SegmentFindKey,
     nowMs: number,

package/src/sqlite-storage.ts

@@ -15,11 +15,13 @@ import {
   bindAuthoritativePartition,
   prepareAuthoritativeQuery,
 } from './authoritative-query';
+import type { ScopeMap } from '@syncular/core';
 import { syncError } from './errors';
 import {
   assertPhysicalColumns,
   assertStoredLayouts,
   commitWindowPageSql,
+  latestScopedChangeSeqSql,
   deleteRowSql,
   deleteSqliteRowScopesSql,
   dropTableDdl,
@@ -1591,6 +1593,31 @@ END`);
     });
   }
 
+  async latestScopedChangeSeq(
+    partition: string,
+    query: { readonly table: string; readonly scopeFilter: ScopeMap },
+  ): Promise<number> {
+    const firstVariable = Object.keys(query.scopeFilter).sort()[0];
+    const values =
+      firstVariable === undefined
+        ? []
+        : (query.scopeFilter[firstVariable] ?? []);
+    if (firstVariable === undefined || values.length === 0) return 0;
+    const row = this.db
+      .query<{ seq: number | null }, string[]>(
+        latestScopedChangeSeqSql(values.length),
+      )
+      .get(
+        ...values.flatMap((value) => [
+          partition,
+          query.table,
+          firstVariable,
+          value,
+        ]),
+      );
+    return row?.seq ?? 0;
+  }
+
   async readCommitWindow(
     partition: string,
     query: CommitWindowQuery,

package/src/storage.ts

@@ -433,6 +433,14 @@ export interface AuthoritativeQueryResult {
  */
 export interface StorageTransaction {
   getRow(table: string, rowId: string): Promise<StoredRow | undefined>;
+  /**
+   * Optional read-ahead: the push layer names every row a commit's
+   * operations target, per table, before it applies them. A storage with
+   * per-statement network cost reads them in one statement and answers the
+   * following `getRow` calls from that snapshot until the row is written in
+   * this transaction.
+   */
+  prefetchRows?(table: string, rowIds: readonly string[]): Promise<void>;
   /**
    * §5 delete-precedence tombstone read: the `commit_seq` of the newest
    * applied delete of this row still within the pruning horizon, or
@@ -797,6 +805,19 @@ export interface ServerStorage {
     query: CommitWindowQuery,
   ): Promise<StoredCommit[]>;
 
+  /**
+   * Newest commitSeq whose change to `query.table` carries a value of the
+   * filter's first variable (keys sorted) in the inverted change-scope index:
+   * the `readCommitWindow` candidate rule, so no change a window read would
+   * deliver for this filter sits above it. 0 when the index holds none (never
+   * written, or pruned). The §4.7 sqlite-lane bootstrap pin reads it; it MUST
+   * be an index lookup per value, independent of the table's history.
+   */
+  latestScopedChangeSeq(
+    partition: string,
+    query: { readonly table: string; readonly scopeFilter: ScopeMap },
+  ): Promise<number>;
+
   /** Scope-filtered snapshot scan, ordered by rowId (bootstrap paging). */
   scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]>;