@syncular/server 0.26.5 → 0.27.0

package/dist/segment-store.js

@@ -146,6 +146,15 @@ export function mergeSegmentRecord(existing, metadata, segmentId, byteLength, no
         expiresAtMs: Math.max(existing?.expiresAtMs ?? nowMs + ttlMs, nowMs + ttlMs),
     };
 }
+/** One buffered segment as a single-chunk `SegmentStream` body. */
+export function bytesStream(bytes) {
+    return new ReadableStream({
+        start(controller) {
+            controller.enqueue(new Uint8Array(bytes));
+            controller.close();
+        },
+    });
+}
 export async function segmentIdFor(bytes) {
     return `sha256:${await sha256Hex(bytes)}`;
 }
@@ -168,6 +177,16 @@ export class MemorySegmentStore {
     async get(segmentId) {
         return this.#entries.get(segmentId);
     }
+    async open(segmentId) {
+        const entry = this.#entries.get(segmentId);
+        if (entry === undefined)
+            return undefined;
+        return {
+            record: entry.record,
+            body: bytesStream(entry.bytes),
+            byteLength: entry.bytes.length,
+        };
+    }
     async find(key, nowMs) {
         for (const { record } of this.#entries.values()) {
             const publication = selectSegmentPublication(record, {

package/dist/sqlite-storage.d.ts

@@ -1,4 +1,5 @@
 import type { CommitPruneQuery, CommitPruneResult } from './storage.js';
+import type { ScopeMap } from '@syncular/core';
 import type { CompiledSchema, CompiledTable } from './schema.js';
 import { type SqliteDatabase } from './sqlite-driver.js';
 import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, CheckpointDeclaration, ClientCursorInfo, ClientRecord, CommitMetadata, CommitMetadataQuery, CommitWindowQuery, IndexRowScanQuery, PartitionRegistryEntry, PrunedReactionCounts, ReactionClaimQuery, ReactionFailureUpdate, ReactionListQuery, ReactionPruneQuery, RowScanQuery, ScopeActivityQuery, ScopeCommitActivity, ServerStorage, ServeGate, StorageTransaction, StoredCommit, StoredCheckpoint, StoredPushResult, StoredReaction, StoredRow } from './storage.js';
@@ -47,6 +48,10 @@ export declare class SqliteServerStorage implements ServerStorage {
     getReaction(partition: string, idempotencyKey: string): Promise<StoredReaction | undefined>;
     listReactions(partition: string, query: ReactionListQuery): Promise<StoredReaction[]>;
     pruneReactions(partition: string, query: ReactionPruneQuery): Promise<PrunedReactionCounts>;
+    latestScopedChangeSeq(partition: string, query: {
+        readonly table: string;
+        readonly scopeFilter: ScopeMap;
+    }): Promise<number>;
     readCommitWindow(partition: string, query: CommitWindowQuery): Promise<StoredCommit[]>;
     scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]>;
     scanRowsByIndex(partition: string, query: IndexRowScanQuery): Promise<StoredRow[]>;

package/dist/sqlite-storage.js

@@ -12,7 +12,7 @@ import { serveNotReadyError } from './readiness.js';
  */
 import { bindAuthoritativePartition, prepareAuthoritativeQuery, } from './authoritative-query.js';
 import { syncError } from './errors.js';
-import { assertPhysicalColumns, assertStoredLayouts, commitWindowPageSql, deleteRowSql, deleteSqliteRowScopesSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, scanRowPageSql, schemaDdl, selectRowScopesSql, selectRowSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
+import { assertPhysicalColumns, assertStoredLayouts, commitWindowPageSql, latestScopedChangeSeqSql, deleteRowSql, deleteSqliteRowScopesSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, scanRowPageSql, schemaDdl, selectRowScopesSql, selectRowSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
 import { matchesEffective } from './scopes.js';
 import { collectCommitWindowPage, deserializePushResult, placeholders, SQLITE_DDL, serializePushResult, toStoredRow, } from './sqlite-dialect.js';
 import { SqliteAdapterRequiredError, } from './sqlite-driver.js';
@@ -1009,6 +1009,23 @@ END`);
             };
         });
     }
+    async latestScopedChangeSeq(partition, query) {
+        const firstVariable = Object.keys(query.scopeFilter).sort()[0];
+        const values = firstVariable === undefined
+            ? []
+            : (query.scopeFilter[firstVariable] ?? []);
+        if (firstVariable === undefined || values.length === 0)
+            return 0;
+        const row = this.db
+            .query(latestScopedChangeSeqSql(values.length))
+            .get(...values.flatMap((value) => [
+            partition,
+            query.table,
+            firstVariable,
+            value,
+        ]));
+        return row?.seq ?? 0;
+    }
     async readCommitWindow(partition, query) {
         const variables = Object.keys(query.scopeFilter).sort();
         const firstVariable = variables[0];

package/dist/storage.d.ts

@@ -589,6 +589,18 @@ export interface ServerStorage {
      * reach `limitChanges` or the window is exhausted.
      */
     readCommitWindow(partition: string, query: CommitWindowQuery): Promise<StoredCommit[]>;
+    /**
+     * Newest commitSeq whose change to `query.table` carries a value of the
+     * filter's first variable (keys sorted) in the inverted change-scope index:
+     * the `readCommitWindow` candidate rule, so no change a window read would
+     * deliver for this filter sits above it. 0 when the index holds none (never
+     * written, or pruned). The §4.7 sqlite-lane bootstrap pin reads it; it MUST
+     * be an index lookup per value, independent of the table's history.
+     */
+    latestScopedChangeSeq(partition: string, query: {
+        readonly table: string;
+        readonly scopeFilter: ScopeMap;
+    }): Promise<number>;
     /** Scope-filtered snapshot scan, ordered by rowId (bootstrap paging). */
     scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]>;
     /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@syncular/server",
-  "version": "0.26.5",
+  "version": "0.27.0",
   "description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
   "license": "Apache-2.0",
   "author": "Benjamin Kniffler",
@@ -68,7 +68,7 @@
     "verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
   },
   "dependencies": {
-    "@syncular/core": "0.26.5"
+    "@syncular/core": "0.27.0"
   },
   "devDependencies": {
     "@electric-sql/pglite": "^0.5.4"

package/src/content-encoding.ts

@@ -92,3 +92,38 @@ export function encodeSegmentBody(
   }
   return { bytes };
 }
+
+/**
+ * Segments above this size are relayed as a stream instead of being
+ * buffered and compressed in memory: a sqlite image can exceed the memory of
+ * a Workers isolate (128 MB).
+ */
+export const SEGMENT_STREAM_THRESHOLD_BYTES = 16 * 1024 * 1024;
+
+export interface EncodedSegmentStream {
+  readonly body: ReadableStream<Uint8Array<ArrayBuffer>>;
+  /** Absent ⇒ identity (no `Content-Encoding` header). */
+  readonly contentEncoding?: 'gzip';
+}
+
+/**
+ * Compress a streamed segment body per §5.8 with the runtime's
+ * `CompressionStream`: gzip when the client accepts it, identity otherwise
+ * (zstd has no standard streaming codec).
+ */
+export function encodeSegmentStream(
+  body: ReadableStream<Uint8Array<ArrayBuffer>>,
+  acceptEncoding: string | undefined,
+): EncodedSegmentStream {
+  if (
+    acceptEncoding === undefined ||
+    !accepts(acceptEncoding, 'gzip') ||
+    typeof CompressionStream !== 'function'
+  ) {
+    return { body };
+  }
+  return {
+    body: body.pipeThrough(new CompressionStream('gzip')),
+    contentEncoding: 'gzip',
+  };
+}

package/src/context.ts

@@ -7,6 +7,7 @@
 import type { ScopeMap } from '@syncular/core';
 import type { BlobStore } from './blob-store';
 import type { CrdtMergerRegistry } from './crdt-merger';
+import type { SyncularErrorHandler } from './errors';
 import type { SyncularServerEvents } from './events';
 import type { LeaseStore } from './lease-store';
 import type { AnyReactionPlanner } from './reactions';
@@ -192,6 +193,14 @@ export interface SyncServerConfig {
    * event objects are built. A throwing sink never affects processing.
    */
   readonly events?: SyncularServerEvents;
+  /**
+   * Host error reporting (§10.2 `sync.internal_error`). Receives the original
+   * exception whenever a request, socket round, segment or blob download,
+   * or remote operation fails with something other than a `SyncError`; the
+   * client sees only the catalog code. Like the events sink, a throwing
+   * handler never changes the response.
+   */
+  readonly onError?: SyncularErrorHandler;
 }
 
 /** Per-request context: the config plus host-authenticated identity. */

package/src/d1-storage.ts

@@ -42,7 +42,7 @@ import type { CommitPruneQuery, CommitPruneResult } from './storage';
  * This mirrors PostgreSQL's per-partition row lock, achieved by placement
  * rather than a lock D1 does not expose.
  */
-import { decodeRow, type RowValue } from '@syncular/core';
+import { decodeRow, type RowValue, type ScopeMap } from '@syncular/core';
 import {
   bindAuthoritativePartition,
   type BoundAuthoritativeQuery,
@@ -54,6 +54,7 @@ import {
   assertPhysicalColumns,
   assertStoredLayouts,
   commitWindowPageSql,
+  latestScopedChangeSeqSql,
   deleteRowSql,
   deleteSqliteRowScopesSql,
   dropTableDdl,
@@ -2146,6 +2147,24 @@ export class D1ServerStorage implements ServerStorage {
     };
   }
 
+  async latestScopedChangeSeq(
+    partition: string,
+    query: { readonly table: string; readonly scopeFilter: ScopeMap },
+  ): Promise<number> {
+    const firstVariable = Object.keys(query.scopeFilter).sort()[0];
+    const values =
+      firstVariable === undefined
+        ? []
+        : (query.scopeFilter[firstVariable] ?? []);
+    if (firstVariable === undefined || values.length === 0) return 0;
+    // Concurrent reads of a pull share one `db.batch` round trip.
+    const [row] = await this.#batchedAll<{ seq: number | null }>(
+      latestScopedChangeSeqSql(values.length),
+      values.flatMap((value) => [partition, query.table, firstVariable, value]),
+    );
+    return row?.seq ?? 0;
+  }
+
   async readCommitWindow(
     partition: string,
     query: CommitWindowQuery,

package/src/errors.ts

@@ -171,6 +171,14 @@ export const ERROR_CATALOG: Readonly<Record<string, ErrorCatalogEntry>> = {
   // checkpoint is incomplete or the database is newer than the running build.
   // Formerly host-only; promoted to the §10.2 catalogue so an in-flight
   // request can be refused with a distinguishable identity.
+  // An exception outside this catalog (host code, storage, network) reached
+  // an adapter; the original goes to `onError`, never onto the wire.
+  'sync.internal_error': {
+    category: 'internal',
+    retryable: true,
+    recommendedAction: 'retryLater',
+    httpStatus: 500,
+  },
   'sync.schema_not_ready': {
     category: 'internal',
     retryable: true,
@@ -310,3 +318,51 @@ export function errorBody(error: SyncError): {
       : {}),
   };
 }
+
+/** The adapter surface that caught an exception reported through `onError`. */
+export type SyncularErrorRoute =
+  | 'sync'
+  | 'operations'
+  | 'segments'
+  | 'blobs'
+  | 'admin'
+  | 'realtime';
+
+/**
+ * Host error reporting: receives every exception that is not a `SyncError`
+ * before the adapter answers `sync.internal_error` (or the remote-operation
+ * `operation.*` failure code), with the surface that caught it.
+ */
+export type SyncularErrorHandler = (
+  error: unknown,
+  context: { readonly route: SyncularErrorRoute },
+) => void;
+
+/**
+ * The protocol error for an exception an adapter caught. A `SyncError`
+ * passes through unchanged. Anything else goes to `onError` and becomes
+ * `sync.internal_error` (§10.2), whose message never carries the exception
+ * text.
+ */
+export function adapterSyncError(
+  error: unknown,
+  onError: SyncularErrorHandler | undefined,
+  route: SyncularErrorRoute,
+): SyncError {
+  if (error instanceof SyncError) return error;
+  reportError(error, onError, route);
+  return new SyncError('sync.internal_error', 'internal server error');
+}
+
+/** Hand `error` to the host's `onError`; a throwing handler is contained. */
+export function reportError(
+  error: unknown,
+  onError: SyncularErrorHandler | undefined,
+  route: SyncularErrorRoute,
+): void {
+  try {
+    onError?.(error, { route });
+  } catch {
+    // The report is best effort; the response stays the catalog error.
+  }
+}

package/src/handler.ts

@@ -545,10 +545,12 @@ async function* streamResponse(
       const horizonSeq = await ctx.storage.getHorizonSeq(ctx.partition);
       const prepared = await prepareSections(
         ctx,
+        schema,
         limits,
         plan.subscriptions,
         maxSeq,
         horizonSeq,
+        plan.logEpoch,
       );
       for (const [index, subscription] of plan.subscriptions.entries()) {
         const trace: PullSectionTrace | undefined =

package/src/index.ts

@@ -81,6 +81,7 @@ export {
   IMAGE_VERSION_COLUMN,
   type SqliteImageBuilder,
   type SqliteImageInput,
+  writeSqliteImage,
 } from './sqlite-image';
 export * from './sqlite-blob-store';
 export * from './sqlite-lease-store';

package/src/operations-realtime.ts

@@ -5,7 +5,7 @@ import {
 } from '@syncular/core';
 import type { RealtimeNotifier, SyncRequestContext } from './context';
 import { REMOTE_COMMAND_CLIENT_ID_PREFIX } from './context';
-import { SyncError, syncError } from './errors';
+import { reportError, SyncError, syncError } from './errors';
 import type {
   RegisteredRemoteQuery,
   RemoteOperationRegistry,
@@ -181,6 +181,9 @@ class WatchSession implements RemoteOperationWatchSession {
           if (this.#isClosed || this.#watches.get(state.watchId) !== state) {
             return;
           }
+          if (!(error instanceof SyncError)) {
+            reportError(error, this.#ctx.onError, 'operations');
+          }
           this.#sendError(
             state.watchId,
             error instanceof SyncError

package/src/operations.ts

@@ -18,7 +18,7 @@ import {
   RESOLVER_OUTAGE,
   touchAuthenticatedPartition,
 } from './context';
-import { SyncError, syncError } from './errors';
+import { reportError, SyncError, syncError } from './errors';
 import { processPushOperationsWithTrace } from './push';
 import { compileSchema } from './schema';
 import { authorizeWrite, type ResolvedScopes } from './scopes';
@@ -450,6 +450,7 @@ export function registerRemoteQuery<Params>(
         );
       } catch (error) {
         if (error instanceof SyncError) throw error;
+        reportError(error, ctx.onError, 'operations');
         throw syncError(
           'operation.query_failed',
           'registered query execution failed',
@@ -463,6 +464,7 @@ export function registerRemoteQuery<Params>(
         rows = normalizeQueryRows(result.rows, descriptor.resultColumns);
       } catch (error) {
         if (error instanceof SyncError) throw error;
+        reportError(error, ctx.onError, 'operations');
         throw syncError(
           'operation.query_failed',
           'registered query result decoding failed',
@@ -766,6 +768,9 @@ export async function handleRemoteOperation(
       ),
     );
   } catch (error) {
+    if (!(error instanceof SyncError)) {
+      reportError(error, ctx.onError, 'operations');
+    }
     return encodeRemoteOperationError(error, 'operation.execution_failed');
   }
 }

package/src/postgres-storage.ts

@@ -40,7 +40,7 @@ import { serveNotReadyError } from './readiness';
  * would leave gaps on rollback, which the pull-window arithmetic in §4.5
  * does not tolerate). Cross-partition pushes never contend.
  */
-import type { PushOperationResult } from '@syncular/core';
+import type { PushOperationResult, ScopeMap } from '@syncular/core';
 import {
   bindAuthoritativePartition,
   type BoundAuthoritativeQuery,
@@ -59,6 +59,7 @@ import {
   assertPhysicalColumns,
   assertStoredLayouts,
   COMMIT_WINDOW_BATCH_SQL,
+  LATEST_SCOPED_CHANGE_BATCH_SQL,
   deleteRowSql,
   dropTableDdl,
   indexRowPageStatement,
@@ -2206,6 +2207,26 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`);
     };
   }
 
+  async latestScopedChangeSeq(
+    partition: string,
+    query: { readonly table: string; readonly scopeFilter: ScopeMap },
+  ): Promise<number> {
+    const firstVariable = Object.keys(query.scopeFilter).sort()[0];
+    const values =
+      firstVariable === undefined
+        ? []
+        : (query.scopeFilter[firstVariable] ?? []);
+    if (firstVariable === undefined || values.length === 0) return 0;
+    // Concurrent reads of one table in a pull share one statement.
+    const [row] = await this.#batchedPage<{ seq: unknown }>(
+      LATEST_SCOPED_CHANGE_BATCH_SQL,
+      partition,
+      query.table,
+      { var: firstVariable, vals: values },
+    );
+    return row === undefined ? 0 : asNumber(row.seq);
+  }
+
   async readCommitWindow(
     partition: string,
     query: CommitWindowQuery,