@syncular/server 0.26.4 → 0.26.6

package/dist/segment-download.js

@@ -3,7 +3,7 @@ import { SyncError, syncError } from './errors.js';
 import { emitEvent } from './events.js';
 import { compileSchema } from './schema.js';
 import { computeEffective, scopeDigest } from './scopes.js';
-import { publicationRecord } from './segment-store.js';
+import { bytesStream, publicationRecord, } from './segment-store.js';
 function parseScopesHeader(header) {
     let parsed;
     try {
@@ -25,14 +25,28 @@ function parseScopesHeader(header) {
     }
     return scopes;
 }
-export async function handleSegmentDownload(ctx, request) {
+export function handleSegmentDownload(ctx, request) {
+    return withDownloadEvent(ctx, request, () => downloadSegment(ctx, request), (result) => result.bytes.length);
+}
+/**
+ * The §5.5 download as a stream: the same authorization as
+ * `handleSegmentDownload`, with the bytes read through `SegmentStore.open`
+ * when the store provides it, so a host with bounded memory (a Worker)
+ * relays a large sqlite image without buffering it. A store without `open`
+ * serves its `get` bytes as a one-chunk stream.
+ */
+export function openSegmentDownload(ctx, request) {
+    return withDownloadEvent(ctx, request, () => openSegment(ctx, request), (result) => result.byteLength);
+}
+/** Emit `segment.downloaded` around a download when an events sink exists. */
+async function withDownloadEvent(ctx, request, run, bytesOf) {
     const events = ctx.events;
     if (events === undefined)
-        return downloadSegment(ctx, request);
+        return run();
     const clock = clockOf(ctx);
     const startedAtMs = clock();
     try {
-        const result = await downloadSegment(ctx, request);
+        const result = await run();
         emitEvent(events, {
             type: 'segment.downloaded',
             atMs: clock(),
@@ -41,7 +55,7 @@ export async function handleSegmentDownload(ctx, request) {
             segmentId: request.segmentId,
             outcome: 'ok',
             mediaType: result.record.mediaType,
-            bytes: result.bytes.length,
+            bytes: bytesOf(result),
             durationMs: clock() - startedAtMs,
         });
         return result;
@@ -66,11 +80,57 @@ async function downloadSegment(ctx, request) {
     if (entry === undefined) {
         throw syncError('sync.not_found', 'unknown segment (§5.5)');
     }
+    const record = await authorizePublication(ctx, request, registry.logEpoch, entry.record);
+    return { record, bytes: entry.bytes, headers: segmentHeaders(record) };
+}
+async function openSegment(ctx, request) {
+    const registry = await touchAuthenticatedPartition(ctx);
+    const opened = ctx.segments.open !== undefined
+        ? await ctx.segments.open(request.segmentId)
+        : await ctx.segments.get(request.segmentId).then((entry) => entry === undefined
+            ? undefined
+            : {
+                record: entry.record,
+                body: bytesStream(entry.bytes),
+                byteLength: entry.bytes.length,
+            });
+    if (opened === undefined) {
+        throw syncError('sync.not_found', 'unknown segment (§5.5)');
+    }
+    let record;
+    try {
+        record = await authorizePublication(ctx, request, registry.logEpoch, opened.record);
+    }
+    catch (error) {
+        await opened.body.cancel();
+        throw error;
+    }
+    return {
+        record,
+        body: opened.body,
+        byteLength: opened.byteLength,
+        headers: segmentHeaders(record),
+    };
+}
+function segmentHeaders(record) {
+    return {
+        'Content-Type': 'application/octet-stream',
+        ETag: `"${record.segmentId}"`,
+        'Cache-Control': 'private, max-age=0',
+        // Accept-Encoding: the body may be served compressed (§5.8).
+        Vary: 'Authorization, X-Syncular-Scopes, Accept-Encoding',
+    };
+}
+/**
+ * Select and authorize the caller's publication of a stored segment
+ * (§5.5), projected onto that publication.
+ */
+async function authorizePublication(ctx, request, logEpoch, stored) {
     // Select the caller's own publication context. A publication under a
     // different partition or a stale log epoch is not this caller's grant, and
     // the entry's existence must not leak across that boundary (§5.5).
-    const inContext = entry.record.publications.filter((publication) => publication.partition === ctx.partition &&
-        publication.logEpoch === registry.logEpoch);
+    const inContext = stored.publications.filter((publication) => publication.partition === ctx.partition &&
+        publication.logEpoch === logEpoch);
     if (inContext.length === 0) {
         throw syncError('sync.not_found', 'unknown segment (§5.5)');
     }
@@ -144,18 +204,8 @@ async function downloadSegment(ctx, request) {
     if (checkpoints.some((checkpoint) => checkpoint.schemaVersion > publication.schemaVersion)) {
         throw syncError('sync.segment_expired', 'segment predates the partition backfill checkpoint — re-pull (§5.1)');
     }
-    return {
-        // The returned record is projected onto the selected publication, so its
-        // `scopeDigests`/`publications` compatibility view cannot union a grant
-        // from another partition (§5.1).
-        record: publicationRecord(entry.record, publication),
-        bytes: entry.bytes,
-        headers: {
-            'Content-Type': 'application/octet-stream',
-            ETag: `"${entry.record.segmentId}"`,
-            'Cache-Control': 'private, max-age=0',
-            // Accept-Encoding: the body may be served compressed (§5.8).
-            Vary: 'Authorization, X-Syncular-Scopes, Accept-Encoding',
-        },
-    };
+    // The returned record is projected onto the selected publication, so its
+    // `scopeDigests`/`publications` compatibility view cannot union a grant
+    // from another partition (§5.1).
+    return publicationRecord(stored, publication);
 }

package/dist/segment-store.d.ts

@@ -130,6 +130,14 @@ export interface SegmentStoreStats {
      */
     readonly approximate?: boolean;
 }
+/** `SegmentStore.open`: a stored segment with its bytes as a stream. */
+export interface SegmentStream {
+    readonly record: SegmentRecord;
+    readonly body: ReadableStream<Uint8Array<ArrayBuffer>>;
+    readonly byteLength: number;
+}
+/** One buffered segment as a single-chunk `SegmentStream` body. */
+export declare function bytesStream(bytes: Uint8Array): ReadableStream<Uint8Array<ArrayBuffer>>;
 export interface SegmentStore {
     put(metadata: SegmentMetadata, bytes: Uint8Array, nowMs: number): Promise<SegmentRecord>;
     /** Returns expired records too — expiry is the caller's check (§5.5). */
@@ -151,6 +159,13 @@ export interface SegmentStore {
      * against the descriptor.
      */
     find(key: SegmentFindKey, nowMs: number): Promise<SegmentRecord | undefined>;
+    /**
+     * Optional streaming read: the record plus the bytes as a stream, so the
+     * download route relays a segment larger than the host's memory (a
+     * sqlite image through a Worker). Returns expired records like `get`.
+     * Stores without it are served through `get`.
+     */
+    open?(segmentId: string): Promise<SegmentStream | undefined>;
     /** Admin/console counters: ADDITIVE, optional. */
     stats?(): Promise<SegmentStoreStats>;
 }
@@ -165,6 +180,7 @@ export declare class MemorySegmentStore implements SegmentStore {
         record: SegmentRecord;
         bytes: Uint8Array;
     } | undefined>;
+    open(segmentId: string): Promise<SegmentStream | undefined>;
     find(key: SegmentFindKey, nowMs: number): Promise<SegmentRecord | undefined>;
     stats(): Promise<SegmentStoreStats>;
 }

package/dist/segment-store.js

@@ -146,6 +146,15 @@ export function mergeSegmentRecord(existing, metadata, segmentId, byteLength, no
         expiresAtMs: Math.max(existing?.expiresAtMs ?? nowMs + ttlMs, nowMs + ttlMs),
     };
 }
+/** One buffered segment as a single-chunk `SegmentStream` body. */
+export function bytesStream(bytes) {
+    return new ReadableStream({
+        start(controller) {
+            controller.enqueue(new Uint8Array(bytes));
+            controller.close();
+        },
+    });
+}
 export async function segmentIdFor(bytes) {
     return `sha256:${await sha256Hex(bytes)}`;
 }
@@ -168,6 +177,16 @@ export class MemorySegmentStore {
     async get(segmentId) {
         return this.#entries.get(segmentId);
     }
+    async open(segmentId) {
+        const entry = this.#entries.get(segmentId);
+        if (entry === undefined)
+            return undefined;
+        return {
+            record: entry.record,
+            body: bytesStream(entry.bytes),
+            byteLength: entry.bytes.length,
+        };
+    }
     async find(key, nowMs) {
         for (const { record } of this.#entries.values()) {
             const publication = selectSegmentPublication(record, {

package/dist/sqlite-storage.d.ts

@@ -1,4 +1,5 @@
 import type { CommitPruneQuery, CommitPruneResult } from './storage.js';
+import type { ScopeMap } from '@syncular/core';
 import type { CompiledSchema, CompiledTable } from './schema.js';
 import { type SqliteDatabase } from './sqlite-driver.js';
 import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, CheckpointDeclaration, ClientCursorInfo, ClientRecord, CommitMetadata, CommitMetadataQuery, CommitWindowQuery, IndexRowScanQuery, PartitionRegistryEntry, PrunedReactionCounts, ReactionClaimQuery, ReactionFailureUpdate, ReactionListQuery, ReactionPruneQuery, RowScanQuery, ScopeActivityQuery, ScopeCommitActivity, ServerStorage, ServeGate, StorageTransaction, StoredCommit, StoredCheckpoint, StoredPushResult, StoredReaction, StoredRow } from './storage.js';
@@ -47,6 +48,10 @@ export declare class SqliteServerStorage implements ServerStorage {
     getReaction(partition: string, idempotencyKey: string): Promise<StoredReaction | undefined>;
     listReactions(partition: string, query: ReactionListQuery): Promise<StoredReaction[]>;
     pruneReactions(partition: string, query: ReactionPruneQuery): Promise<PrunedReactionCounts>;
+    latestScopedChangeSeq(partition: string, query: {
+        readonly table: string;
+        readonly scopeFilter: ScopeMap;
+    }): Promise<number>;
     readCommitWindow(partition: string, query: CommitWindowQuery): Promise<StoredCommit[]>;
     scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]>;
     scanRowsByIndex(partition: string, query: IndexRowScanQuery): Promise<StoredRow[]>;

package/dist/sqlite-storage.js

@@ -12,7 +12,7 @@ import { serveNotReadyError } from './readiness.js';
  */
 import { bindAuthoritativePartition, prepareAuthoritativeQuery, } from './authoritative-query.js';
 import { syncError } from './errors.js';
-import { assertPhysicalColumns, assertStoredLayouts, commitWindowPageSql, deleteRowSql, deleteSqliteRowScopesSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, scanRowPageSql, schemaDdl, selectRowScopesSql, selectRowSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
+import { assertPhysicalColumns, assertStoredLayouts, commitWindowPageSql, latestScopedChangeSeqSql, deleteRowSql, deleteSqliteRowScopesSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, scanRowPageSql, schemaDdl, selectRowScopesSql, selectRowSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
 import { matchesEffective } from './scopes.js';
 import { collectCommitWindowPage, deserializePushResult, placeholders, SQLITE_DDL, serializePushResult, toStoredRow, } from './sqlite-dialect.js';
 import { SqliteAdapterRequiredError, } from './sqlite-driver.js';
@@ -1009,6 +1009,23 @@ END`);
             };
         });
     }
+    async latestScopedChangeSeq(partition, query) {
+        const firstVariable = Object.keys(query.scopeFilter).sort()[0];
+        const values = firstVariable === undefined
+            ? []
+            : (query.scopeFilter[firstVariable] ?? []);
+        if (firstVariable === undefined || values.length === 0)
+            return 0;
+        const row = this.db
+            .query(latestScopedChangeSeqSql(values.length))
+            .get(...values.flatMap((value) => [
+            partition,
+            query.table,
+            firstVariable,
+            value,
+        ]));
+        return row?.seq ?? 0;
+    }
     async readCommitWindow(partition, query) {
         const variables = Object.keys(query.scopeFilter).sort();
         const firstVariable = variables[0];

package/dist/storage.d.ts

@@ -344,6 +344,14 @@ export interface AuthoritativeQueryResult {
  */
 export interface StorageTransaction {
     getRow(table: string, rowId: string): Promise<StoredRow | undefined>;
+    /**
+     * Optional read-ahead: the push layer names every row a commit's
+     * operations target, per table, before it applies them. A storage with
+     * per-statement network cost reads them in one statement and answers the
+     * following `getRow` calls from that snapshot until the row is written in
+     * this transaction.
+     */
+    prefetchRows?(table: string, rowIds: readonly string[]): Promise<void>;
     /**
      * §5 delete-precedence tombstone read: the `commit_seq` of the newest
      * applied delete of this row still within the pruning horizon, or
@@ -581,6 +589,18 @@ export interface ServerStorage {
      * reach `limitChanges` or the window is exhausted.
      */
     readCommitWindow(partition: string, query: CommitWindowQuery): Promise<StoredCommit[]>;
+    /**
+     * Newest commitSeq whose change to `query.table` carries a value of the
+     * filter's first variable (keys sorted) in the inverted change-scope index:
+     * the `readCommitWindow` candidate rule, so no change a window read would
+     * deliver for this filter sits above it. 0 when the index holds none (never
+     * written, or pruned). The §4.7 sqlite-lane bootstrap pin reads it; it MUST
+     * be an index lookup per value, independent of the table's history.
+     */
+    latestScopedChangeSeq(partition: string, query: {
+        readonly table: string;
+        readonly scopeFilter: ScopeMap;
+    }): Promise<number>;
     /** Scope-filtered snapshot scan, ordered by rowId (bootstrap paging). */
     scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]>;
     /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@syncular/server",
-  "version": "0.26.4",
+  "version": "0.26.6",
   "description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
   "license": "Apache-2.0",
   "author": "Benjamin Kniffler",
@@ -68,7 +68,7 @@
     "verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
   },
   "dependencies": {
-    "@syncular/core": "0.26.4"
+    "@syncular/core": "0.26.6"
   },
   "devDependencies": {
     "@electric-sql/pglite": "^0.5.4"

package/src/content-encoding.ts

@@ -92,3 +92,38 @@ export function encodeSegmentBody(
   }
   return { bytes };
 }
+
+/**
+ * Segments above this size are relayed as a stream instead of being
+ * buffered and compressed in memory: a sqlite image can exceed the memory of
+ * a Workers isolate (128 MB).
+ */
+export const SEGMENT_STREAM_THRESHOLD_BYTES = 16 * 1024 * 1024;
+
+export interface EncodedSegmentStream {
+  readonly body: ReadableStream<Uint8Array<ArrayBuffer>>;
+  /** Absent ⇒ identity (no `Content-Encoding` header). */
+  readonly contentEncoding?: 'gzip';
+}
+
+/**
+ * Compress a streamed segment body per §5.8 with the runtime's
+ * `CompressionStream`: gzip when the client accepts it, identity otherwise
+ * (zstd has no standard streaming codec).
+ */
+export function encodeSegmentStream(
+  body: ReadableStream<Uint8Array<ArrayBuffer>>,
+  acceptEncoding: string | undefined,
+): EncodedSegmentStream {
+  if (
+    acceptEncoding === undefined ||
+    !accepts(acceptEncoding, 'gzip') ||
+    typeof CompressionStream !== 'function'
+  ) {
+    return { body };
+  }
+  return {
+    body: body.pipeThrough(new CompressionStream('gzip')),
+    contentEncoding: 'gzip',
+  };
+}

package/src/context.ts

@@ -7,6 +7,7 @@
 import type { ScopeMap } from '@syncular/core';
 import type { BlobStore } from './blob-store';
 import type { CrdtMergerRegistry } from './crdt-merger';
+import type { SyncularErrorHandler } from './errors';
 import type { SyncularServerEvents } from './events';
 import type { LeaseStore } from './lease-store';
 import type { AnyReactionPlanner } from './reactions';
@@ -192,6 +193,14 @@ export interface SyncServerConfig {
    * event objects are built. A throwing sink never affects processing.
    */
   readonly events?: SyncularServerEvents;
+  /**
+   * Host error reporting (§10.2 `sync.internal_error`). Receives the original
+   * exception whenever a request, socket round, segment or blob download,
+   * or remote operation fails with something other than a `SyncError`; the
+   * client sees only the catalog code. Like the events sink, a throwing
+   * handler never changes the response.
+   */
+  readonly onError?: SyncularErrorHandler;
 }
 
 /** Per-request context: the config plus host-authenticated identity. */

package/src/d1-storage.ts

@@ -42,7 +42,7 @@ import type { CommitPruneQuery, CommitPruneResult } from './storage';
  * This mirrors PostgreSQL's per-partition row lock, achieved by placement
  * rather than a lock D1 does not expose.
  */
-import { decodeRow, type RowValue } from '@syncular/core';
+import { decodeRow, type RowValue, type ScopeMap } from '@syncular/core';
 import {
   bindAuthoritativePartition,
   type BoundAuthoritativeQuery,
@@ -54,6 +54,7 @@ import {
   assertPhysicalColumns,
   assertStoredLayouts,
   commitWindowPageSql,
+  latestScopedChangeSeqSql,
   deleteRowSql,
   deleteSqliteRowScopesSql,
   dropTableDdl,
@@ -2146,6 +2147,24 @@ export class D1ServerStorage implements ServerStorage {
     };
   }
 
+  async latestScopedChangeSeq(
+    partition: string,
+    query: { readonly table: string; readonly scopeFilter: ScopeMap },
+  ): Promise<number> {
+    const firstVariable = Object.keys(query.scopeFilter).sort()[0];
+    const values =
+      firstVariable === undefined
+        ? []
+        : (query.scopeFilter[firstVariable] ?? []);
+    if (firstVariable === undefined || values.length === 0) return 0;
+    // Concurrent reads of a pull share one `db.batch` round trip.
+    const [row] = await this.#batchedAll<{ seq: number | null }>(
+      latestScopedChangeSeqSql(values.length),
+      values.flatMap((value) => [partition, query.table, firstVariable, value]),
+    );
+    return row?.seq ?? 0;
+  }
+
   async readCommitWindow(
     partition: string,
     query: CommitWindowQuery,

package/src/errors.ts

@@ -171,6 +171,14 @@ export const ERROR_CATALOG: Readonly<Record<string, ErrorCatalogEntry>> = {
   // checkpoint is incomplete or the database is newer than the running build.
   // Formerly host-only; promoted to the §10.2 catalogue so an in-flight
   // request can be refused with a distinguishable identity.
+  // An exception outside this catalog (host code, storage, network) reached
+  // an adapter; the original goes to `onError`, never onto the wire.
+  'sync.internal_error': {
+    category: 'internal',
+    retryable: true,
+    recommendedAction: 'retryLater',
+    httpStatus: 500,
+  },
   'sync.schema_not_ready': {
     category: 'internal',
     retryable: true,
@@ -310,3 +318,51 @@ export function errorBody(error: SyncError): {
       : {}),
   };
 }
+
+/** The adapter surface that caught an exception reported through `onError`. */
+export type SyncularErrorRoute =
+  | 'sync'
+  | 'operations'
+  | 'segments'
+  | 'blobs'
+  | 'admin'
+  | 'realtime';
+
+/**
+ * Host error reporting: receives every exception that is not a `SyncError`
+ * before the adapter answers `sync.internal_error` (or the remote-operation
+ * `operation.*` failure code), with the surface that caught it.
+ */
+export type SyncularErrorHandler = (
+  error: unknown,
+  context: { readonly route: SyncularErrorRoute },
+) => void;
+
+/**
+ * The protocol error for an exception an adapter caught. A `SyncError`
+ * passes through unchanged. Anything else goes to `onError` and becomes
+ * `sync.internal_error` (§10.2), whose message never carries the exception
+ * text.
+ */
+export function adapterSyncError(
+  error: unknown,
+  onError: SyncularErrorHandler | undefined,
+  route: SyncularErrorRoute,
+): SyncError {
+  if (error instanceof SyncError) return error;
+  reportError(error, onError, route);
+  return new SyncError('sync.internal_error', 'internal server error');
+}
+
+/** Hand `error` to the host's `onError`; a throwing handler is contained. */
+export function reportError(
+  error: unknown,
+  onError: SyncularErrorHandler | undefined,
+  route: SyncularErrorRoute,
+): void {
+  try {
+    onError?.(error, { route });
+  } catch {
+    // The report is best effort; the response stays the catalog error.
+  }
+}

package/src/handler.ts

@@ -545,10 +545,12 @@ async function* streamResponse(
       const horizonSeq = await ctx.storage.getHorizonSeq(ctx.partition);
       const prepared = await prepareSections(
         ctx,
+        schema,
         limits,
         plan.subscriptions,
         maxSeq,
         horizonSeq,
+        plan.logEpoch,
       );
       for (const [index, subscription] of plan.subscriptions.entries()) {
         const trace: PullSectionTrace | undefined =

package/src/index.ts

@@ -81,6 +81,7 @@ export {
   IMAGE_VERSION_COLUMN,
   type SqliteImageBuilder,
   type SqliteImageInput,
+  writeSqliteImage,
 } from './sqlite-image';
 export * from './sqlite-blob-store';
 export * from './sqlite-lease-store';

package/src/operations-realtime.ts

@@ -5,7 +5,7 @@ import {
 } from '@syncular/core';
 import type { RealtimeNotifier, SyncRequestContext } from './context';
 import { REMOTE_COMMAND_CLIENT_ID_PREFIX } from './context';
-import { SyncError, syncError } from './errors';
+import { reportError, SyncError, syncError } from './errors';
 import type {
   RegisteredRemoteQuery,
   RemoteOperationRegistry,
@@ -181,6 +181,9 @@ class WatchSession implements RemoteOperationWatchSession {
           if (this.#isClosed || this.#watches.get(state.watchId) !== state) {
             return;
           }
+          if (!(error instanceof SyncError)) {
+            reportError(error, this.#ctx.onError, 'operations');
+          }
           this.#sendError(
             state.watchId,
             error instanceof SyncError

package/src/operations.ts

@@ -18,7 +18,7 @@ import {
   RESOLVER_OUTAGE,
   touchAuthenticatedPartition,
 } from './context';
-import { SyncError, syncError } from './errors';
+import { reportError, SyncError, syncError } from './errors';
 import { processPushOperationsWithTrace } from './push';
 import { compileSchema } from './schema';
 import { authorizeWrite, type ResolvedScopes } from './scopes';
@@ -450,6 +450,7 @@ export function registerRemoteQuery<Params>(
         );
       } catch (error) {
         if (error instanceof SyncError) throw error;
+        reportError(error, ctx.onError, 'operations');
         throw syncError(
           'operation.query_failed',
           'registered query execution failed',
@@ -463,6 +464,7 @@ export function registerRemoteQuery<Params>(
         rows = normalizeQueryRows(result.rows, descriptor.resultColumns);
       } catch (error) {
         if (error instanceof SyncError) throw error;
+        reportError(error, ctx.onError, 'operations');
         throw syncError(
           'operation.query_failed',
           'registered query result decoding failed',
@@ -766,6 +768,9 @@ export async function handleRemoteOperation(
       ),
     );
   } catch (error) {
+    if (!(error instanceof SyncError)) {
+      reportError(error, ctx.onError, 'operations');
+    }
     return encodeRemoteOperationError(error, 'operation.execution_failed');
   }
 }