@syncular/server 0.30.5 → 0.30.7

package/dist/storage-errors.d.ts

@@ -31,3 +31,6 @@ export declare function isPostgresConstraintError(error: unknown): boolean;
  * the original message is copied to the public StorageConstraintError.
  */
 export declare function isD1ConstraintError(error: unknown): boolean;
+/** Preserve the operation failure when transaction cleanup also fails. */
+export declare function retainRollbackFailure(error: unknown, rollbackError: unknown): void;
+export declare function rollbackAfterError(run: () => void, error: unknown): never;

package/dist/storage-errors.js

@@ -84,3 +84,18 @@ export function isD1ConstraintError(error) {
     return (typeof message === 'string' &&
         /^(?:D1(?:_EXEC)?_ERROR:\s*)?(?:UNIQUE|NOT NULL|CHECK|FOREIGN KEY) constraint failed\b/i.test(message));
 }
+/** Preserve the operation failure when transaction cleanup also fails. */
+export function retainRollbackFailure(error, rollbackError) {
+    if (error instanceof Error && !('rollbackError' in error)) {
+        Object.defineProperty(error, 'rollbackError', { value: rollbackError });
+    }
+}
+export function rollbackAfterError(run, error) {
+    try {
+        run();
+    }
+    catch (rollbackError) {
+        retainRollbackFailure(error, rollbackError);
+    }
+    throw error;
+}

package/dist/storage.d.ts

@@ -219,6 +219,8 @@ export interface StoredPushResult {
     readonly results: readonly PushOperationResult[];
 }
 export interface ClientSubscription {
+    /** Codec accepted by the last pull; absent on records written before schema windows. */
+    readonly schemaVersion?: number;
     readonly id: string;
     readonly table: string;
     /** Requested scopes of the client's most recent pull (§8.1). */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@syncular/server",
-  "version": "0.30.5",
+  "version": "0.30.7",
   "description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
   "license": "Apache-2.0",
   "author": "Benjamin Kniffler",
@@ -68,7 +68,7 @@
     "verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
   },
   "dependencies": {
-    "@syncular/core": "0.30.5"
+    "@syncular/core": "0.30.7"
   },
   "devDependencies": {
     "@electric-sql/pglite": "^0.5.4"

package/src/context.ts

@@ -11,7 +11,7 @@ import type { SyncularErrorHandler } from './errors';
 import type { SyncularServerEvents } from './events';
 import type { LeaseStore } from './lease-store';
 import type { AnyReactionPlanner } from './reactions';
-import type { ServerSchema } from './schema';
+import type { ServerSchema, CompiledSchema } from './schema';
 import type { SegmentStore } from './segment-store';
 import type {
   BlobPresignConfig,
@@ -97,6 +97,8 @@ export interface RealtimeNotifier {
 
 export interface SyncServerConfig {
   readonly schema: ServerSchema;
+  /** Current compiled schema followed by the supported previous versions, newest first. */
+  readonly schemaWindow?: readonly CompiledSchema[];
   readonly storage: ServerStorage;
   /**
    * RFC 0007: the backfill checkpoints this host installs. Every readiness

package/src/handler.ts

@@ -55,6 +55,7 @@ import { type ProcessedPushCommit, processPushCommitWithTrace } from './push';
 import { serveNotReadyError } from './readiness';
 import type { CompiledSchema } from './schema';
 import { compileSchema } from './schema';
+import { schemaWindowOf, projectRowPayload } from './schema-window';
 import { computeEffective, type ResolvedScopes } from './scopes';
 import { StorageQueryError } from './storage-errors';
 import {
@@ -343,7 +344,7 @@ async function planRequest(
   const { resolved, leaseToEmit } = await resolveOnce(
     ctx,
     header.clientId,
-    schema,
+    compileSchema(ctx.schema),
   );
 
   const subscriptions: SubscriptionPlan[] = validated.map(({ frame }) => {
@@ -466,8 +467,10 @@ async function* streamResponse(
     yield encodeResponseFrame(
       {
         type: 'RESP_HEADER',
-        requiredSchemaVersion: schema.version,
-        latestSchemaVersion: schema.version,
+        requiredSchemaVersion: schemaWindowOf(ctx.schema, ctx.schemaWindow).at(
+          -1,
+        )!.version,
+        latestSchemaVersion: ctx.schema.version,
         ...(plan.wireVersion >= 2
           ? { logEpoch: plan.logEpoch, resetRequired: plan.epochReset }
           : {}),
@@ -480,7 +483,7 @@ async function* streamResponse(
   yield encodeResponseFrame(
     {
       type: 'RESP_HEADER',
-      latestSchemaVersion: schema.version,
+      latestSchemaVersion: ctx.schema.version,
       ...(plan.wireVersion >= 2
         ? { logEpoch: plan.logEpoch, resetRequired: plan.epochReset }
         : {}),
@@ -519,12 +522,35 @@ async function* streamResponse(
     for (const push of plan.pushes) {
       const processed = await processPushCommitWithTrace(
         ctx,
-        schema,
+        compileSchema(ctx.schema),
         plan.resolved,
         plan.header.clientId,
         push,
+        schema,
       );
-      const { frame } = processed;
+      const frame = {
+        ...processed.frame,
+        results: processed.frame.results.map((result) => {
+          if (result.status !== 'conflict') return result;
+          const operation = push.operations[result.opIndex]!;
+          const target = schema.tables.get(operation.table)!;
+          const current = compileSchema(ctx.schema).tables.get(
+            operation.table,
+          )!;
+          return {
+            ...result,
+            serverRow: projectRowPayload(current, target, result.serverRow),
+            conflictColumns: result.conflictColumns
+              .slice(0, Math.ceil(target.columns.length / 8))
+              .map((byte, i) =>
+                i === Math.ceil(target.columns.length / 8) - 1 &&
+                target.columns.length % 8 !== 0
+                  ? byte & ((1 << (target.columns.length % 8)) - 1)
+                  : byte,
+              ),
+          };
+        }),
+      };
       if (events !== undefined) {
         emitPushEvent(events, ctx, plan.header.clientId, push, processed);
       }
@@ -630,6 +656,7 @@ async function* streamResponse(
             id: s.frame.id,
             table: s.frame.table,
             scopes: s.frame.scopes,
+            schemaVersion: schema.version,
           }))
         : [...(previous?.subscriptions ?? [])];
     const cursor =
@@ -754,7 +781,15 @@ async function createStreamCore(
   const gate = await ctx.storage.readServeGate(ctx.partition, schema.version);
   const refusal = serveGateRefusal(gate, schema.version, ctx.checkpoints);
   if (refusal !== undefined) throw serveNotReadyError(refusal);
-  const plan = await planRequest(request, ctx, schema, registry);
+  const window = schemaWindowOf(ctx.schema, ctx.schemaWindow);
+  const header = request.frames[0];
+  const selected = window.find(
+    (candidate) =>
+      header?.type === 'REQ_HEADER' &&
+      candidate.version === header.schemaVersion,
+  );
+  const served = selected ?? schema;
+  const plan = await planRequest(request, ctx, served, registry);
   const report: RequestReport = { outcome: 'ok' };
   // RFC 0007 read-verify-refuse for the streamed read path: the generator
   // builds the whole response (its data reads) before yielding the first
@@ -766,7 +801,7 @@ async function createStreamCore(
   // key, so refusing the request is safe (proved in the serve-gate lane).
   const verified = async function* (): AsyncGenerator<Uint8Array> {
     const buffered: Uint8Array[] = [];
-    for await (const chunk of streamResponse(plan, ctx, schema, report)) {
+    for await (const chunk of streamResponse(plan, ctx, served, report)) {
       buffered.push(chunk);
     }
     const after = await ctx.storage.readServeGate(

package/src/index.ts

@@ -93,3 +93,5 @@ export {
   type StorageQueryErrorCode,
 } from './storage-errors';
 export * from './validate';
+
+export { schemaWindowOf } from './schema-window';

package/src/pg-executor-pglite.ts

@@ -1,3 +1,4 @@
+import { retainRollbackFailure } from './storage-errors';
 /**
  * A `PgExecutor` over @electric-sql/pglite (embedded WASM Postgres).
  *
@@ -64,7 +65,11 @@ export function pgliteExecutor(db: PGlite | PgliteLike): PgExecutor {
           await like.exec('COMMIT');
           return result;
         } catch (error) {
-          await like.exec('ROLLBACK');
+          try {
+            await like.exec('ROLLBACK');
+          } catch (rollbackError) {
+            retainRollbackFailure(error, rollbackError);
+          }
           throw error;
         }
       } finally {

package/src/pull.ts

@@ -4,7 +4,6 @@
  */
 import {
   type CommitChange,
-  decodeRow,
   encodeRowsSegment,
   type PullHeaderFrame,
   type ResponseFrame,
@@ -26,6 +25,7 @@ import type { SegmentRecord, SegmentStore } from './segment-store';
 import { issueSegmentUrl } from './signed-url';
 import type { SqliteImageBuilder } from './sqlite-image';
 import type { ServerStorage, StoredCommit, StoredRow } from './storage';
+import { projectRowPayload, projectRowValues } from './schema-window';
 
 // One artifact build per owning storage pair and complete immutable identity.
 const imageBuilds = new WeakMap<
@@ -136,7 +136,12 @@ function parseBootstrapToken(
   }
 }
 
-function commitFrame(table: string, commit: StoredCommit): ResponseFrame {
+function commitFrame(
+  table: string,
+  commit: StoredCommit,
+  current: CompiledTable,
+  target: CompiledTable,
+): ResponseFrame {
   const changes: CommitChange[] = commit.changes.map((change) => ({
     tableIndex: 0,
     rowId: change.rowId,
@@ -145,7 +150,9 @@ function commitFrame(table: string, commit: StoredCommit): ResponseFrame {
       ? { rowVersion: change.rowVersion }
       : {}),
     scopes: change.scopes,
-    ...(change.payload !== undefined ? { row: change.payload } : {}),
+    ...(change.payload !== undefined
+      ? { row: projectRowPayload(current, target, change.payload) }
+      : {}),
   }));
   return {
     type: 'COMMIT',
@@ -318,7 +325,14 @@ async function* sqliteImageSegment(
         scopeDigest: digest,
         rowBatches: (async function* () {
           rowCount += probe.length;
-          yield probe;
+          yield probe.map((row) => ({
+            ...row,
+            payload: projectRowPayload(
+              compileSchema(ctx.schema).tables.get(plan.table.name)!,
+              plan.table,
+              row.payload,
+            ),
+          }));
           let afterRowId = probe[probe.length - 1]!.rowId;
           for (;;) {
             const rows = await storage.scanRows(partition, {
@@ -328,7 +342,14 @@ async function* sqliteImageSegment(
               limit: 5_000,
             });
             rowCount += rows.length;
-            yield rows;
+            yield rows.map((row) => ({
+              ...row,
+              payload: projectRowPayload(
+                compileSchema(ctx.schema).tables.get(plan.table.name)!,
+                plan.table,
+                row.payload,
+              ),
+            }));
             const last = rows[rows.length - 1];
             if (rows.length < 5_000 || last === undefined) break;
             afterRowId = last.rowId;
@@ -420,7 +441,11 @@ async function* bootstrapSegments(
     // §5.2: every row record carries the row's current server_version.
     const decoded = pageRows.map((row: StoredRow) => ({
       serverVersion: row.serverVersion,
-      values: decodeRow(plan.table.columns, row.payload),
+      values: projectRowValues(
+        compileSchema(ctx.schema).tables.get(plan.table.name)!,
+        plan.table,
+        row.payload,
+      ),
     }));
     const bytes = encodeRowsSegment({
       table: plan.table.name,
@@ -772,7 +797,12 @@ export async function* preparedSection(
     ) {
       break;
     }
-    yield commitFrame(sub.table, commit);
+    yield commitFrame(
+      sub.table,
+      commit,
+      compileSchema(ctx.schema).tables.get(sub.table)!,
+      plan.table,
+    );
     delivered += commit.changes.length;
     deliveredCommits += 1;
     lastDeliveredSeq = commit.commitSeq;

package/src/push.ts

@@ -1,3 +1,4 @@
+import { retainRollbackFailure } from './storage-errors';
 /**
  * Push apply (SPEC.md §6) with §3.4 write-path authorization.
  *
@@ -293,6 +294,7 @@ async function mergeCrdtColumns(
 async function applyOperation(
   tx: StorageTransaction,
   schema: CompiledSchema,
+  wireSchema: CompiledSchema,
   resolved: ResolvedScopes,
   op: PushOperation,
   opIndex: number,
@@ -303,7 +305,8 @@ async function applyOperation(
   actorId: string,
 ): Promise<OperationOutcome> {
   const table = schema.tables.get(op.table);
-  if (table === undefined) {
+  const wireTable = wireSchema.tables.get(op.table);
+  if (table === undefined || wireTable === undefined) {
     return errorRecord(
       opIndex,
       'sync.unknown_table',
@@ -409,7 +412,13 @@ async function applyOperation(
   }
   let sparse: SparseRowValue[];
   try {
-    sparse = decodeSparseRow(table.columns, table.primaryKeyIndex, payload);
+    sparse = decodeSparseRow(
+      wireTable.columns,
+      wireTable.primaryKeyIndex,
+      payload,
+    );
+    while (sparse.length < table.columns.length)
+      sparse.push(stored === undefined ? null : undefined);
   } catch (error) {
     return errorRecord(
       opIndex,
@@ -1300,6 +1309,7 @@ export async function processPushCommitWithTrace(
   resolved: ResolvedScopes,
   clientId: string,
   frame: PushCommitFrame,
+  wireSchema: CompiledSchema = schema,
 ): Promise<ProcessedPushCommit> {
   return processPushOperationsWithTrace(
     ctx,
@@ -1308,6 +1318,7 @@ export async function processPushCommitWithTrace(
     clientId,
     frame.clientCommitId,
     async () => frame.operations,
+    wireSchema,
   );
 }
 
@@ -1325,6 +1336,7 @@ export async function processPushOperationsWithTrace(
   buildOperations: (
     tx: StorageTransaction,
   ) => Promise<readonly PushOperation[]>,
+  wireSchema: CompiledSchema = schema,
 ): Promise<ProcessedPushCommit> {
   const { storage, partition } = ctx;
   let persisted: StoredPushResult | undefined;
@@ -1437,6 +1449,7 @@ export async function processPushOperationsWithTrace(
       const outcome = await applyOperation(
         tx,
         schema,
+        wireSchema,
         resolved,
         op,
         opIndex,
@@ -1649,14 +1662,17 @@ export async function processPushOperationsWithTrace(
         // a no-op when the finalization already committed.
         try {
           await tx.rollback();
-        } catch {
-          // Surface the finalization failure; a rollback failure would
-          // otherwise mask it.
+        } catch (rollbackError) {
+          retainRollbackFailure(finalizationError, rollbackError);
         }
         throw finalizationError;
       }
     }
-    await tx.rollback();
+    try {
+      await tx.rollback();
+    } catch (rollbackError) {
+      retainRollbackFailure(error, rollbackError);
+    }
     throw error;
   }
 }

package/src/readiness.ts

@@ -9,6 +9,7 @@
 import type { SyncServerConfig } from './context';
 import { type SyncError, syncError } from './errors';
 import { type CompiledSchema, compileSchema } from './schema';
+import { schemaWindowOf } from './schema-window';
 import type { ServeGateRefusal } from './storage';
 
 export const SYNC_SERVER_READINESS_ERROR_CODE =
@@ -47,11 +48,15 @@ export class SyncServerReadinessError extends Error {
  * `cause` locally for the table/column or storage diagnostic.
  */
 export async function ensureSyncServerReady(
-  config: Pick<SyncServerConfig, 'schema' | 'storage' | 'checkpoints'>,
+  config: Pick<
+    SyncServerConfig,
+    'schema' | 'schemaWindow' | 'storage' | 'checkpoints'
+  >,
 ): Promise<void> {
   let compiled: CompiledSchema;
   try {
     compiled = compileSchema(config.schema);
+    schemaWindowOf(config.schema, config.schemaWindow);
   } catch (cause) {
     throw new SyncServerReadinessError({
       phase: 'schema_compile',

package/src/realtime.ts

@@ -48,7 +48,12 @@ import {
 } from './errors';
 import { emitEvent, type SyncularServerEvents } from './events';
 import { createSyncResponseStream } from './handler';
-import { type CompiledSchema, compileSchema } from './schema';
+import {
+  type CompiledSchema,
+  type CompiledTable,
+  compileSchema,
+} from './schema';
+import { schemaWindowOf, projectRowPayload } from './schema-window';
 import {
   computeEffective,
   matchesEffective,
@@ -99,6 +104,7 @@ export interface RealtimeConnectOptions {
 }
 
 interface Registration {
+  readonly codec: CompiledTable;
   readonly id: string;
   readonly table: string;
   readonly effective: ScopeMap;
@@ -794,7 +800,9 @@ export class RealtimeSession {
               ? { rowVersion: change.rowVersion }
               : {}),
             scopes: change.scopes,
-            ...(change.payload !== undefined ? { row: change.payload } : {}),
+            ...(change.payload !== undefined
+              ? { row: this.#hub.projectPayload(registration, change.payload) }
+              : {}),
           }),
         );
       if (changes.length > 0) sections.push({ registration, changes });
@@ -894,6 +902,15 @@ export class RealtimeHub {
   constructor(config: RealtimeHubConfig) {
     this.#config = config;
     this.#schema = compileSchema(config.schema);
+    schemaWindowOf(config.schema, config.schemaWindow);
+  }
+
+  projectPayload(registration: Registration, payload: Uint8Array): Uint8Array {
+    return projectRowPayload(
+      this.#schema.tables.get(registration.table)!,
+      registration.codec,
+      payload,
+    );
   }
 
   get sessionCount(): number {
@@ -983,7 +1000,18 @@ export class RealtimeHub {
     }
     const registrations: Registration[] = [];
     for (const subscription of record?.subscriptions ?? []) {
-      const table = this.#schema.tables.get(subscription.table);
+      const window = schemaWindowOf(
+        this.#config.schema,
+        this.#config.schemaWindow,
+      );
+      const selected =
+        subscription.schemaVersion === undefined &&
+        this.#config.schemaWindow === undefined
+          ? this.#schema
+          : window.find(
+              (schema) => schema.version === subscription.schemaVersion,
+            );
+      const table = selected?.tables.get(subscription.table);
       if (table === undefined) continue;
       const keysValid = Object.keys(subscription.scopes).every((key) =>
         table.declaredVariables.has(key),
@@ -993,6 +1021,7 @@ export class RealtimeHub {
       if (outcome.status !== 'active') continue;
       registrations.push({
         id: subscription.id,
+        codec: table,
         table: subscription.table,
         effective: outcome.effective,
       });
@@ -1022,6 +1051,9 @@ export class RealtimeHub {
       partition: identity.partition,
       actorId: identity.actorId,
       schema: this.#config.schema,
+      ...(this.#config.schemaWindow !== undefined
+        ? { schemaWindow: this.#config.schemaWindow }
+        : {}),
       storage: this.#config.storage,
       segments,
       resolveScopes: this.#config.resolveScopes,
@@ -1139,7 +1171,19 @@ export class RealtimeHub {
           clientId: options.clientId,
           cursor,
           latestCursor: latestSeq,
-          requiresSync: record === undefined || cursor < latestSeq,
+          requiresSync:
+            record === undefined ||
+            cursor < latestSeq ||
+            (this.#config.schemaWindow !== undefined &&
+              record.subscriptions.some(
+                (subscription) =>
+                  !schemaWindowOf(
+                    this.#config.schema,
+                    this.#config.schemaWindow,
+                  ).some(
+                    (schema) => schema.version === subscription.schemaVersion,
+                  ),
+              )),
           timestamp: clock(),
         },
       }),

package/src/schema-window.ts

@@ -0,0 +1,72 @@
+import { decodeRow, encodeRow, type RowValue } from '@syncular/core';
+import {
+  compileSchema,
+  type CompiledSchema,
+  type CompiledTable,
+  type ServerSchema,
+} from './schema';
+
+const validatedWindows = new WeakMap<
+  readonly CompiledSchema[],
+  CompiledSchema
+>();
+
+/** A host-reviewed newest-first window. Storage and validation always use current. */
+export function schemaWindowOf(
+  current: ServerSchema,
+  configured?: readonly CompiledSchema[],
+): readonly CompiledSchema[] {
+  const head = compileSchema(current);
+  if (configured === undefined) return [head];
+  if (validatedWindows.get(configured) === head) return configured;
+  if (configured.length === 0 || configured[0] !== head)
+    throw new Error('sync.schema_window.invalid_head');
+  let previous = head.version + 1;
+  for (const schema of configured) {
+    if (schema.version >= previous)
+      throw new Error('sync.schema_window.invalid_order');
+    previous = schema.version;
+    for (const table of schema.tables.values()) {
+      const next = head.tables.get(table.name);
+      if (
+        next === undefined ||
+        table.columns.length > next.columns.length ||
+        table.columns.some(
+          (column, i) =>
+            JSON.stringify(column) !== JSON.stringify(next.columns[i]),
+        ) ||
+        next.columns
+          .slice(table.columns.length)
+          .some((column) => !column.nullable) ||
+        table.primaryKeyIndex !== next.primaryKeyIndex ||
+        JSON.stringify(table.scopePatterns) !==
+          JSON.stringify(next.scopePatterns) ||
+        JSON.stringify(table.references) !== JSON.stringify(next.references)
+      ) {
+        throw new Error('sync.schema_window.incompatible_schema');
+      }
+    }
+  }
+  validatedWindows.set(configured, head);
+  return configured;
+}
+
+/** Canonical stored bytes never leave the server under another version's codec. */
+export function projectRowValues(
+  current: CompiledTable,
+  target: CompiledTable,
+  payload: Uint8Array,
+): RowValue[] {
+  const values = decodeRow(current.columns, payload);
+  return values.slice(0, target.columns.length);
+}
+
+export function projectRowPayload(
+  current: CompiledTable,
+  target: CompiledTable,
+  payload: Uint8Array,
+): Uint8Array {
+  return current === target
+    ? payload
+    : encodeRow(target.columns, projectRowValues(current, target, payload));
+}

package/src/segment-download.ts

@@ -18,6 +18,7 @@ import {
 import { SyncError, syncError } from './errors';
 import { emitEvent } from './events';
 import { compileSchema } from './schema';
+import { schemaWindowOf } from './schema-window';
 import { computeEffective, type ResolvedScopes, scopeDigest } from './scopes';
 import {
   bytesStream,
@@ -291,7 +292,10 @@ async function authorizePublication(
   // earlier valid publication of the same bytes exists.
   const schema = compileSchema(ctx.schema);
   const candidates = live.filter((publication) => {
-    const table = schema.tables.get(publication.table);
+    const served = schemaWindowOf(ctx.schema, ctx.schemaWindow).find(
+      (candidate) => candidate.version === publication.schemaVersion,
+    );
+    const table = served?.tables.get(publication.table);
     return (
       table !== undefined &&
       Object.entries(requested).every(

package/src/sqlite-image.ts

@@ -1,3 +1,4 @@
+import { rollbackAfterError } from './storage-errors';
 /**
  * SQLite-image segment generation (SPEC.md §5.3): a complete SQLite
  * database file carrying one table's whole effective-scope snapshot at
@@ -124,7 +125,8 @@ export async function writeSqliteImage(
 
     db.exec('COMMIT');
   } catch (error) {
-    db.exec('ROLLBACK');
-    throw error;
+    rollbackAfterError(() => {
+      db.exec('ROLLBACK');
+    }, error);
   }
 }

package/src/sqlite-segment-store.ts

@@ -1,3 +1,4 @@
+import { rollbackAfterError } from './storage-errors';
 /**
  * SQLite-backed segment store over the shared synchronous driver.
  */
@@ -238,8 +239,9 @@ export class SqliteSegmentStore implements SegmentStore {
       }
       this.db.exec('COMMIT');
     } catch (error) {
-      this.db.exec('ROLLBACK');
-      throw error;
+      rollbackAfterError(() => {
+        this.db.exec('ROLLBACK');
+      }, error);
     }
     return record;
   }