@syncular/server 0.26.5 → 0.27.0

package/src/pull.ts

@@ -9,13 +9,18 @@ import {
   type PullHeaderFrame,
   type ResponseFrame,
   type ScopeMap,
+  type SegmentRefFrame,
   type SegmentRow,
   type SubscriptionFrame,
 } from '@syncular/core';
-import type { SyncRequestContext } from './context';
+import type { SyncRequestContext, SyncServerConfig } from './context';
 import { clockOf, limitsOf } from './context';
 import type { PullSegmentSummary } from './events';
-import type { CompiledSchema, CompiledTable } from './schema';
+import {
+  type CompiledSchema,
+  type CompiledTable,
+  compileSchema,
+} from './schema';
 import { scopeDigest } from './scopes';
 import type { SegmentRecord, SegmentStore } from './segment-store';
 import { issueSegmentUrl } from './signed-url';
@@ -244,21 +249,11 @@ async function* sqliteImageSegment(
   digest: string,
   trace: PullSectionTrace | undefined,
   logEpoch: string,
+  existing: SegmentRecord | undefined,
+  probe: StoredRow[],
 ): AsyncGenerator<ResponseFrame, boolean> {
   const { storage, segments, partition } = ctx;
   const now = clockOf(ctx)();
-  const existing = await segments.find(
-    {
-      partition,
-      logEpoch,
-      table: plan.table.name,
-      schemaVersion: schema.version,
-      mediaType: 'sqlite',
-      scopeDigest: digest,
-      asOfCommitSeq: asOf,
-    },
-    now,
-  );
   if (existing !== undefined) {
     trace?.segments.push({
       mediaType: 'sqlite',
@@ -276,12 +271,7 @@ async function* sqliteImageSegment(
   }
   // Eligibility probe (§5.3): image only when the snapshot exceeds one
   // rows page — smaller tables stay on the (typically inline) rows lane.
-  const probe = await storage.scanRows(partition, {
-    table: plan.table.name,
-    scopeFilter: plan.effective,
-    afterRowId: null,
-    limit: limits.limitSnapshotRows + 1,
-  });
+  // The probe is the rows lane's first page, read by `prepareSections`.
   if (probe.length <= limits.limitSnapshotRows) return false;
   // §5.3: building an image needs a SQLite engine. The host injects the
   // builder through `sqliteImageBuilder`; when omitted we default to the
@@ -389,7 +379,8 @@ async function* bootstrapSegments(
   startRowCursor: string | null,
   trace: PullSectionTrace | undefined,
   logEpoch: string,
-  firstPage: StoredRow[] | undefined,
+  firstPage: StoredRow[],
+  image: SegmentRecord | undefined,
 ): AsyncGenerator<
   ResponseFrame,
   { complete: boolean; rowCursor: string | null }
@@ -408,13 +399,15 @@ async function* bootstrapSegments(
       digest,
       trace,
       logEpoch,
+      image,
+      firstPage,
     );
     if (imaged) return { complete: true, rowCursor: null };
   }
   let rowCursor = startRowCursor;
   for (let page = 0; page < limits.maxSnapshotPages; page++) {
     const scanned =
-      page === 0 && firstPage !== undefined
+      page === 0
         ? firstPage
         : await storage.scanRows(partition, {
             table: plan.table.name,
@@ -492,27 +485,45 @@ async function* bootstrapSegments(
 /**
  * The storage reads a section needs before its first frame. A pull starts
  * them for every subscription before awaiting any, so a storage that batches
- * concurrent page reads (PostgreSQL) answers all subscriptions of one table
- * with one statement, and the horizon re-check runs once per pull.
+ * concurrent page reads (PostgreSQL, D1) answers all subscriptions of one
+ * table with one statement, and the horizon re-check runs once per pull.
  */
 export interface PreparedSection {
   /** §4.5 window read for an incremental cursor; empty when none was read. */
   readonly commits: StoredCommit[];
   /** Horizon to judge the cursor against, re-read after any window read. */
   readonly horizonSeq: number;
-  /** First rows-lane snapshot page, read when the sqlite lane is excluded. */
+  /**
+   * A bootstrap's first snapshot page (`limitSnapshotRows + 1` rows from the
+   * resume cursor), which is also the §5.3 image eligibility probe.
+   */
   readonly firstPage: StoredRow[] | undefined;
+  /** §4.7 pin of a fresh bootstrap; absent ⇒ `maxCommitSeq`. */
+  readonly pin: number | undefined;
+  /** §5.3 stored image for a sqlite-lane bootstrap's reuse key, if any. */
+  readonly image: SegmentRecord | undefined;
+}
+
+interface PendingSection {
+  readonly commits?: Promise<StoredCommit[]>;
+  readonly firstPage?: Promise<StoredRow[]>;
+  readonly lane?: Promise<{
+    readonly pin: number | undefined;
+    readonly image: SegmentRecord | undefined;
+  }>;
 }
 
 /** Run the pre-frame storage reads of every subscription concurrently. */
 export async function prepareSections(
   ctx: SyncRequestContext,
+  schema: CompiledSchema,
   limits: PullLimits,
   plans: readonly SubscriptionPlan[],
   maxSeq: number,
   horizonSeq: number,
+  logEpoch: string,
 ): Promise<PreparedSection[]> {
-  const reads = plans.map((plan) => {
+  const reads = plans.map((plan): PendingSection => {
     const sub = plan.frame;
     if (plan.status === 'revoked') return {};
     const token = parseBootstrapToken(sub.bootstrapState, sub.table);
@@ -537,24 +548,55 @@ export async function prepareSections(
       return {};
     }
     // Bootstrapping: the same resume decision `preparedSection` makes (§4.7).
-    const startCursor =
+    const resume =
       token !== undefined && token.asOfCommitSeq >= horizonSeq
-        ? token.rowCursor
-        : null;
-    if (sqliteLaneEligible(limits, plan, startCursor)) return {};
-    return {
-      firstPage: ctx.storage.scanRows(ctx.partition, {
-        table: plan.table.name,
-        scopeFilter: plan.effective,
-        afterRowId: startCursor,
-        limit: limits.limitSnapshotRows + 1,
-      }),
-    };
+        ? token
+        : undefined;
+    const startCursor = resume?.rowCursor ?? null;
+    const firstPage = ctx.storage.scanRows(ctx.partition, {
+      table: plan.table.name,
+      scopeFilter: plan.effective,
+      afterRowId: startCursor,
+      limit: limits.limitSnapshotRows + 1,
+    });
+    if (!sqliteLaneEligible(limits, plan, startCursor)) return { firstPage };
+    // §4.7: a fresh sqlite-lane bootstrap pins at the newest change in its
+    // scope (never below the horizon, never above maxCommitSeq), so a stored
+    // image stays the reuse key's match until this scope changes (§5.3).
+    const pinned =
+      resume !== undefined
+        ? Promise.resolve({ asOf: resume.asOfCommitSeq, pin: undefined })
+        : ctx.storage
+            .latestScopedChangeSeq(ctx.partition, {
+              table: plan.table.name,
+              scopeFilter: plan.effective,
+            })
+            .then((latest) => {
+              const pin = Math.min(maxSeq, Math.max(latest, horizonSeq));
+              return { asOf: pin, pin };
+            });
+    const lane = pinned.then(async ({ asOf, pin }) => ({
+      pin,
+      image: await ctx.segments.find(
+        {
+          partition: ctx.partition,
+          logEpoch,
+          table: plan.table.name,
+          schemaVersion: schema.version,
+          mediaType: 'sqlite',
+          scopeDigest: await scopeDigest(plan.effective),
+          asOfCommitSeq: asOf,
+        },
+        clockOf(ctx)(),
+      ),
+    }));
+    return { firstPage, lane };
   });
   const settled = await Promise.all(
     reads.map(async (read) => ({
       commits: read.commits === undefined ? undefined : await read.commits,
       firstPage: await read.firstPage,
+      lane: await read.lane,
     })),
   );
   // Validate continuity before committing to an active section. A prune
@@ -566,6 +608,8 @@ export async function prepareSections(
     commits: read.commits ?? [],
     horizonSeq: read.commits === undefined ? horizonSeq : windowHorizon,
     firstPage: read.firstPage,
+    pin: read.lane?.pin,
+    image: read.lane?.image,
   }));
 }
 
@@ -583,12 +627,17 @@ export async function* subscriptionSection(
   trace?: PullSectionTrace,
   logEpoch?: string,
 ): AsyncGenerator<ResponseFrame, SubscriptionResult> {
+  if (logEpoch === undefined || logEpoch.length === 0) {
+    throw new Error('subscriptionSection requires a non-empty log epoch');
+  }
   const [prepared] = await prepareSections(
     ctx,
+    schema,
     limits,
     [plan],
     maxSeq,
     horizonSeq,
+    logEpoch,
   );
   if (prepared === undefined) throw new Error('prepareSections lost a plan');
   return yield* preparedSection(
@@ -660,8 +709,11 @@ export async function* preparedSection(
       token !== undefined && token.asOfCommitSeq >= horizonSeq
         ? token
         : undefined;
-    const asOf = resume?.asOfCommitSeq ?? maxSeq;
+    const asOf = resume?.asOfCommitSeq ?? prepared.pin ?? maxSeq;
     const startCursor = resume?.rowCursor ?? null;
+    if (prepared.firstPage === undefined) {
+      throw new Error('prepareSections did not read the bootstrap page');
+    }
     yield {
       type: 'SUB_START',
       id: sub.id,
@@ -680,6 +732,7 @@ export async function* preparedSection(
       trace,
       logEpoch,
       prepared.firstPage,
+      prepared.image,
     );
     if (outcome.complete) {
       yield { type: 'SUB_END', nextCursor: asOf };
@@ -740,3 +793,123 @@ export async function* preparedSection(
   yield { type: 'SUB_END', nextCursor };
   return { nextCursor, active: true };
 }
+
+export interface PublishSqliteImageInput {
+  /** The sync config the serving host runs, plus a `sqliteImageBuilder`. */
+  readonly config: SyncServerConfig;
+  readonly partition: string;
+  readonly table: string;
+  /**
+   * The effective scopes the subscribing clients hold for `table` (§3.2):
+   * the reuse key carries their digest, so publish once per distinct scope
+   * set that clients subscribe with.
+   */
+  readonly scopes: ScopeMap;
+}
+
+export interface PublishedSqliteImage {
+  readonly segmentId: string;
+  /** The §4.7 scope pin the image is stored under. */
+  readonly asOfCommitSeq: number;
+  readonly scopeDigest: string;
+  readonly rowCount: number;
+  readonly byteLength: number;
+  /** `reused` when an unexpired image for the same reuse key already existed. */
+  readonly origin: 'built' | 'reused';
+}
+
+/**
+ * Publish the §5.3 sqlite image that a pull for (`table`, `scopes`) looks
+ * up, from a host with a SQLite engine (a Bun process with the production
+ * storage). It runs the pull's own bootstrap path (the §4.7 scope pin, the
+ * reuse key, the build), so the stored identity is exactly the one a
+ * serving host without a builder finds. The image stays current until a
+ * change reaches the scope; its lifetime is the segment store's TTL at
+ * publication.
+ */
+export async function publishSqliteImage(
+  input: PublishSqliteImageInput,
+): Promise<PublishedSqliteImage> {
+  const { config, partition } = input;
+  if (config.sqliteImageBuilder === undefined) {
+    throw new Error('publishSqliteImage requires config.sqliteImageBuilder');
+  }
+  const schema = compileSchema(config.schema);
+  const table = schema.tables.get(input.table);
+  if (table === undefined) {
+    throw new Error('publishSqliteImage: the schema has no such table');
+  }
+  await config.storage.ensureSchema(schema, config.checkpoints);
+  const logEpoch = await config.storage.getPartitionLogEpoch(partition);
+  if (logEpoch === undefined) {
+    throw new Error('publishSqliteImage: the partition has no log epoch');
+  }
+  const ctx: SyncRequestContext = {
+    ...config,
+    partition,
+    actorId: 'syncular.image-publisher',
+  };
+  const plan: SubscriptionPlan = {
+    frame: {
+      type: 'SUBSCRIPTION',
+      id: 'image',
+      table: input.table,
+      scopes: input.scopes,
+      cursor: -1,
+    },
+    table,
+    status: 'active',
+    effective: input.scopes,
+  };
+  // One-row pages make every table above one row image-eligible (§5.3).
+  const limits: PullLimits = {
+    limitCommits: 1,
+    limitSnapshotRows: 1,
+    maxSnapshotPages: 1,
+    accept: ACCEPT_SQLITE,
+  };
+  const maxSeq = await config.storage.getMaxCommitSeq(partition);
+  const horizonSeq = await config.storage.getHorizonSeq(partition);
+  const trace: PullSectionTrace = { segments: [] };
+  const [prepared] = await prepareSections(
+    ctx,
+    schema,
+    limits,
+    [plan],
+    maxSeq,
+    horizonSeq,
+    logEpoch,
+  );
+  if (prepared === undefined) throw new Error('prepareSections lost a plan');
+  let ref: SegmentRefFrame | undefined;
+  for await (const frame of preparedSection(
+    ctx,
+    schema,
+    limits,
+    plan,
+    maxSeq,
+    prepared,
+    trace,
+    logEpoch,
+  )) {
+    if (frame.type === 'SEGMENT_REF' && frame.mediaType === 'sqlite') {
+      ref = frame;
+    }
+  }
+  const image = trace.segments.find(
+    (segment) => segment.mediaType === 'sqlite',
+  );
+  if (ref === undefined || image === undefined) {
+    throw new Error(
+      'publishSqliteImage: the scope holds at most one row, which pulls serve inline',
+    );
+  }
+  return {
+    segmentId: ref.segmentId,
+    asOfCommitSeq: ref.asOfCommitSeq,
+    scopeDigest: ref.scopeDigest,
+    rowCount: ref.rowCount,
+    byteLength: ref.byteLength,
+    origin: image.origin === 'reused' ? 'reused' : 'built',
+  };
+}

package/src/realtime.ts

@@ -39,7 +39,13 @@ import {
   RESOLVER_OUTAGE,
   touchAuthenticatedPartition,
 } from './context';
-import { SyncError, syncError } from './errors';
+import {
+  adapterSyncError,
+  reportError,
+  SyncError,
+  type SyncularErrorHandler,
+  syncError,
+} from './errors';
 import { emitEvent, type SyncularServerEvents } from './events';
 import { createSyncResponseStream } from './handler';
 import { type CompiledSchema, compileSchema } from './schema';
@@ -581,19 +587,18 @@ export class RealtimeSession {
         );
       } catch (error) {
         // §8.7 failures: what HTTP reports as status+JSON becomes a
-        // minimal RESP_HEADER/ERROR/END response stream on the socket.
-        if (error instanceof DecodeError || error instanceof SyncError) {
-          const sync =
-            error instanceof SyncError
-              ? error
-              : syncError(error.code, error.message);
-          finishRound(); // END is in this one chunk
-          await this.#sendRoundChunk(
-            errorResponseBytes(sync, this.wireVersion, this.logEpoch),
-          );
-          return;
-        }
-        throw error;
+        // minimal RESP_HEADER/ERROR/END response stream on the socket. An
+        // exception outside the catalog is reported and answers
+        // `sync.internal_error`, exactly like the HTTP adapter's 500.
+        const sync =
+          error instanceof DecodeError
+            ? syncError(error.code, error.message)
+            : adapterSyncError(error, this.#hub.onError, 'realtime');
+        finishRound(); // END is in this one chunk
+        await this.#sendRoundChunk(
+          errorResponseBytes(sync, this.wireVersion, this.logEpoch),
+        );
+        return;
       }
       // Fetch-ahead so the round stops being "in flight" the moment the
       // chunk carrying END is handed to send: the client may legally
@@ -604,23 +609,22 @@ export class RealtimeSession {
       try {
         step = await iterator.next();
       } catch (error) {
-        // The buffered read-verify refusal is thrown on the generator's first
-        // `next()`, before any chunk is yielded (the generator builds the whole
-        // response, then verifies). Nothing has escaped, so it is delivered as
-        // the same §8.4 ERROR frame HTTP returns rather than a silent close. A
-        // throw after the first chunk still takes the §8.7 violation path.
-        if (error instanceof DecodeError || error instanceof SyncError) {
-          const sync =
-            error instanceof SyncError
-              ? error
-              : syncError(error.code, error.message);
-          finishRound();
-          await this.#sendRoundChunk(
-            errorResponseBytes(sync, this.wireVersion, this.logEpoch),
-          );
-          return;
-        }
-        throw error;
+        // The generator builds the whole response (and runs the read-verify
+        // refusal) on its first `next()`, before any chunk is yielded.
+        // Nothing has escaped, so a failure there is delivered as the same
+        // §8.4 ERROR frame HTTP returns rather than a silent close; a
+        // non-catalog exception is reported and answers
+        // `sync.internal_error`. A throw after the first chunk still takes
+        // the §8.7 violation path.
+        const sync =
+          error instanceof DecodeError
+            ? syncError(error.code, error.message)
+            : adapterSyncError(error, this.#hub.onError, 'realtime');
+        finishRound();
+        await this.#sendRoundChunk(
+          errorResponseBytes(sync, this.wireVersion, this.logEpoch),
+        );
+        return;
       }
       while (!step.done) {
         const chunk = step.value;
@@ -628,9 +632,10 @@ export class RealtimeSession {
         if (step.done) finishRound();
         await this.#sendRoundChunk(chunk);
       }
-    } catch {
+    } catch (error) {
       // A host failure mid-stream leaves the byte stream unfinishable —
       // fail loud, drop the connection (§8.7 / §1.4 abort rule).
+      reportError(error, this.#hub.onError, 'realtime');
       this.#violation('sync round failed mid-stream');
     } finally {
       finishRound();
@@ -1059,10 +1064,18 @@ export class RealtimeHub {
       ...(this.#config.events !== undefined
         ? { events: this.#config.events }
         : {}),
+      ...(this.#config.onError !== undefined
+        ? { onError: this.#config.onError }
+        : {}),
       realtime: this,
     };
   }
 
+  /** The host's error reporter (`SyncServerConfig.onError`), if any. */
+  get onError(): SyncularErrorHandler | undefined {
+    return this.#config.onError;
+  }
+
   requestContext(session: RealtimeSession): SyncRequestContext {
     return this.requestContextFor(session);
   }

package/src/relational-rows.ts

@@ -705,6 +705,35 @@ export function prefetchRowsSql(table: CompiledTable): string {
        ON t.partition=$1 AND t.tbl=$3 AND t.row_id=ids.id`;
 }
 
+/**
+ * SQLite/D1: the newest `commit_seq` in the inverted change-scope index for
+ * one variable's values, as one index-only max per value (a `(partition,
+ * tbl, var, value)` prefix), so the cost is independent of the table's
+ * history. Bind order: [partition, tbl, var, value] per value.
+ */
+export function latestScopedChangeSeqSql(valueCount: number): string {
+  const perValue: string[] = [];
+  for (let i = 0; i < valueCount; i++) {
+    perValue.push(
+      'SELECT (SELECT max(commit_seq) FROM sync_change_scopes WHERE partition=? AND tbl=? AND var=? AND value=?) AS seq',
+    );
+  }
+  return `SELECT max(seq) AS seq FROM (${perValue.join(' UNION ALL ')})`;
+}
+
+/**
+ * PostgreSQL set-based form of `latestScopedChangeSeqSql` for the
+ * subscriptions of one table in a pull. `$2` is a JSON array of
+ * `{i, var, vals}` records. Params: [partition, specs (JSON text), tbl].
+ */
+export const LATEST_SCOPED_CHANGE_BATCH_SQL = `SELECT s.i AS sub,
+       (SELECT max(latest.seq) FROM (
+          SELECT (SELECT max(commit_seq) FROM sync_change_scopes
+                   WHERE partition=$1 AND tbl=$3 AND var=s.var AND value=v.value) AS seq
+            FROM jsonb_array_elements_text(s.vals) AS v(value)) AS latest) AS seq
+     FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, var text, vals jsonb)
+    ORDER BY s.i`;
+
 /** SELECT a row's version + scope map (admin/blob authz). Params: [partition, rowId]. */
 export function selectRowScopesSql(
   table: CompiledTable,

package/src/s3-segment-store.ts

@@ -49,6 +49,7 @@ import type {
   SegmentRecord,
   SegmentStore,
   SegmentStoreStats,
+  SegmentStream,
 } from './segment-store';
 import {
   mergeSegmentRecord,
@@ -619,6 +620,20 @@ export class S3SegmentStore implements SegmentStore {
     };
   }
 
+  /** The record plus the object GET's body, never buffered (see `get`). */
+  async open(segmentId: string): Promise<SegmentStream | undefined> {
+    if (!SEGMENT_ID_PATTERN.test(segmentId)) return undefined;
+    const record = await this.#readRecord(segmentId);
+    if (record === undefined) return undefined;
+    const response = await this.#request('GET', this.objectKeyFor(segmentId));
+    if (response === undefined || response.body === null) return undefined;
+    return {
+      record: record.record,
+      body: response.body,
+      byteLength: record.record.byteLength,
+    };
+  }
+
   async find(
     key: SegmentFindKey,
     nowMs: number,