@syncular/server 0.15.47 → 0.16.1

package/src/realtime.ts

@@ -31,7 +31,11 @@ import {
   type WakeReason,
 } from '@syncular/core';
 import type { SyncRequestContext, SyncServerConfig } from './context';
-import { REMOTE_COMMAND_CLIENT_ID_PREFIX, RESOLVER_OUTAGE } from './context';
+import {
+  REMOTE_COMMAND_CLIENT_ID_PREFIX,
+  RESOLVER_OUTAGE,
+  touchAuthenticatedPartition,
+} from './context';
 import { SyncError, syncError } from './errors';
 import { emitEvent, type SyncularServerEvents } from './events';
 import { createSyncResponseStream } from './handler';
@@ -215,6 +219,9 @@ export class RealtimeSession {
   readonly partition: string;
   readonly actorId: string;
   readonly clientId: string;
+  readonly logEpoch: string;
+  /** Response/delta layout selected by the most recent socket round. */
+  wireVersion: number;
   /** Highest contiguously applied commitSeq acknowledged by the client. */
   cursor: number;
   /** Suppress deltas until the client catches up via pull + ack (§8.2). */
@@ -256,12 +263,16 @@ export class RealtimeSession {
     clock: () => number,
     maxDeltaBytes: number,
     storage: ServerStorage,
+    logEpoch: string,
+    wireVersion: number,
     events: SyncularServerEvents | undefined,
   ) {
     this.sessionId = crypto.randomUUID();
     this.partition = options.partition;
     this.actorId = options.actorId;
     this.clientId = options.clientId;
+    this.logEpoch = logEpoch;
+    this.wireVersion = wireVersion;
     this.cursor = cursor;
     this.lastKnownSeq = latestSeq;
     this.wakePending = cursor < latestSeq;
@@ -515,6 +526,11 @@ export class RealtimeSession {
       if (this.#activeRound === token) this.#activeRound = undefined;
     };
     try {
+      const requestedWireVersion =
+        (requestBytes[4] ?? 0) | ((requestBytes[5] ?? 0) << 8);
+      if (requestedWireVersion === 1 || requestedWireVersion === 2) {
+        this.wireVersion = requestedWireVersion;
+      }
       let stream: AsyncIterable<Uint8Array> | undefined;
       try {
         // §8.7: the round's clientId must match the connection's —
@@ -543,7 +559,9 @@ export class RealtimeSession {
               ? error
               : syncError(error.code, error.message);
           finishRound(); // END is in this one chunk
-          await this.#sendRoundChunk(errorResponseBytes(sync));
+          await this.#sendRoundChunk(
+            errorResponseBytes(sync, this.wireVersion, this.logEpoch),
+          );
           return;
         }
         throw error;
@@ -712,7 +730,14 @@ export class RealtimeSession {
       this.sendWake('catchup-required');
       return;
     }
-    const frames: ResponseFrame[] = [{ type: 'RESP_HEADER' }];
+    const frames: ResponseFrame[] = [
+      {
+        type: 'RESP_HEADER',
+        ...(this.wireVersion >= 2
+          ? { logEpoch: this.logEpoch, resetRequired: false }
+          : {}),
+      },
+    ];
     for (const section of sections) {
       frames.push({
         type: 'SUB_START',
@@ -733,7 +758,7 @@ export class RealtimeSession {
       frames.push({ type: 'SUB_END', nextCursor: commit.commitSeq });
     }
     const bytes = encodeMessage({
-      wireVersion: PROTOCOL_WIRE_VERSION,
+      wireVersion: this.wireVersion,
       msgKind: 'response',
       frames,
     });
@@ -963,6 +988,11 @@ export class RealtimeHub {
   async connect(options: RealtimeConnectOptions): Promise<RealtimeSession> {
     const { storage } = this.#config;
     const clock = this.#config.clock ?? Date.now;
+    const registry = await touchAuthenticatedPartition({
+      ...this.#config,
+      partition: options.partition,
+      actorId: options.actorId,
+    });
     if (options.clientId.startsWith(REMOTE_COMMAND_CLIENT_ID_PREFIX)) {
       throw syncError(
         'sync.invalid_client_id',
@@ -995,6 +1025,8 @@ export class RealtimeHub {
       clock,
       this.#config.maxDeltaBytes ?? DEFAULT_MAX_DELTA_BYTES,
       storage,
+      registry.logEpoch,
+      record?.wireVersion ?? PROTOCOL_WIRE_VERSION,
       this.#config.events,
     );
     this.#sessions.add(session);
@@ -1079,12 +1111,19 @@ export function createRealtimeHub(config: RealtimeHubConfig): RealtimeHub {
 /** §8.7 failures: the socket has no HTTP status surface, so a
  * request-level failure becomes a minimal RESP_HEADER/ERROR/END
  * response message delivered as the round's response stream. */
-function errorResponseBytes(error: SyncError): Uint8Array {
+function errorResponseBytes(
+  error: SyncError,
+  wireVersion: number,
+  logEpoch: string,
+): Uint8Array {
   return encodeMessage({
-    wireVersion: PROTOCOL_WIRE_VERSION,
+    wireVersion,
     msgKind: 'response',
     frames: [
-      { type: 'RESP_HEADER' },
+      {
+        type: 'RESP_HEADER',
+        ...(wireVersion >= 2 ? { logEpoch, resetRequired: false } : {}),
+      },
       {
         type: 'ERROR',
         code: error.code,

package/src/restore.ts

@@ -0,0 +1,28 @@
+import type { PartitionRegistryEntry, ServerStorage } from './storage';
+
+export interface RotatePartitionLogEpochOptions {
+  readonly storage: ServerStorage;
+  readonly partition: string;
+  readonly nowMs?: number;
+  /** Deterministic operator/test override. Defaults to a random UUID. */
+  readonly logEpoch?: string;
+}
+
+/**
+ * Fence clients from a prior authoritative database timeline after restore.
+ * Call while traffic and realtime delivery remain stopped (§2.1).
+ */
+export function rotatePartitionLogEpoch(
+  options: RotatePartitionLogEpochOptions,
+): Promise<PartitionRegistryEntry> {
+  if (options.partition.length === 0) {
+    throw new TypeError('partition must be non-empty');
+  }
+  const logEpoch = options.logEpoch ?? crypto.randomUUID();
+  if (logEpoch.length === 0) throw new TypeError('logEpoch must be non-empty');
+  return options.storage.rotatePartitionLogEpoch(
+    options.partition,
+    logEpoch,
+    options.nowMs ?? Date.now(),
+  );
+}

package/src/s3-segment-store.ts

@@ -152,7 +152,13 @@ function parseRecordJson(json: string, source: string): SegmentRecord {
     throw new Error(`S3SegmentStore: corrupt record in ${source}`);
   }
   const r = parsed as Record<string, unknown>;
-  const strings = ['segmentId', 'partition', 'table', 'scopeDigest'] as const;
+  const strings = [
+    'segmentId',
+    'partition',
+    'logEpoch',
+    'table',
+    'scopeDigest',
+  ] as const;
   const numbers = [
     'schemaVersion',
     'asOfCommitSeq',
@@ -182,6 +188,7 @@ function parseRecordJson(json: string, source: string): SegmentRecord {
   return {
     segmentId: r.segmentId as string,
     partition: r.partition as string,
+    logEpoch: r.logEpoch as string,
     table: r.table as string,
     schemaVersion: r.schemaVersion as number,
     mediaType: r.mediaType,
@@ -223,6 +230,7 @@ export class S3SegmentStore implements SegmentStore {
   async #findKeyFor(key: SegmentFindKey): Promise<string> {
     const canonical = JSON.stringify([
       key.partition,
+      key.logEpoch,
       key.table,
       key.schemaVersion,
       key.mediaType,
@@ -431,6 +439,7 @@ export class S3SegmentStore implements SegmentStore {
     const record = parseRecordJson(await response.text(), 'reuse pointer');
     if (
       record.partition !== key.partition ||
+      record.logEpoch !== key.logEpoch ||
       record.table !== key.table ||
       record.schemaVersion !== key.schemaVersion ||
       record.mediaType !== key.mediaType ||

package/src/seed.ts

@@ -186,8 +186,52 @@ export async function seedMutations(
     };
   });
 
+  const requestContext = {
+    ...config,
+    partition: target.partition,
+    actorId: target.actorId,
+  };
+  const acquisition = decodeMessage(
+    await handleSyncRequest(
+      encodeMessage({
+        wireVersion: PROTOCOL_WIRE_VERSION,
+        msgKind: 'request',
+        frames: [
+          {
+            type: 'REQ_HEADER',
+            clientId,
+            schemaVersion: config.schema.version,
+          },
+          {
+            type: 'PULL_HEADER',
+            limitCommits: 0,
+            limitSnapshotRows: 0,
+            maxSnapshotPages: 0,
+            accept: 0b0011,
+          },
+        ],
+      }),
+      requestContext,
+    ),
+  );
+  const acquisitionHeader = acquisition.frames[0];
+  if (
+    acquisition.msgKind !== 'response' ||
+    acquisitionHeader?.type !== 'RESP_HEADER' ||
+    acquisitionHeader.logEpoch === undefined
+  ) {
+    throw new SyncError(
+      'sync.invalid_request',
+      'seedMutations: epoch acquisition returned an invalid response',
+    );
+  }
   const frames: RequestFrame[] = [
-    { type: 'REQ_HEADER', clientId, schemaVersion: config.schema.version },
+    {
+      type: 'REQ_HEADER',
+      clientId,
+      schemaVersion: config.schema.version,
+      logEpoch: acquisitionHeader.logEpoch,
+    },
     { type: 'PUSH_COMMIT', clientCommitId, operations },
     // A pull that asks for nothing: the round exists for its push half.
     {
@@ -221,9 +265,7 @@ export async function seedMutations(
       frames,
     }),
     {
-      ...config,
-      partition: target.partition,
-      actorId: target.actorId,
+      ...requestContext,
       events:
         config.events === undefined
           ? capture

package/src/segment-download.ts

@@ -9,7 +9,11 @@
  */
 import type { ScopeMap } from '@syncular/core';
 import type { SyncRequestContext } from './context';
-import { clockOf, RESOLVER_OUTAGE } from './context';
+import {
+  clockOf,
+  RESOLVER_OUTAGE,
+  touchAuthenticatedPartition,
+} from './context';
 import { SyncError, syncError } from './errors';
 import { emitEvent } from './events';
 import { compileSchema } from './schema';
@@ -96,8 +100,13 @@ async function downloadSegment(
   ctx: SyncRequestContext,
   request: SegmentDownloadRequest,
 ): Promise<SegmentDownloadResult> {
+  const registry = await touchAuthenticatedPartition(ctx);
   const entry = await ctx.segments.get(request.segmentId);
-  if (entry === undefined || entry.record.partition !== ctx.partition) {
+  if (
+    entry === undefined ||
+    entry.record.partition !== ctx.partition ||
+    entry.record.logEpoch !== registry.logEpoch
+  ) {
     throw syncError('sync.not_found', 'unknown segment (§5.5)');
   }
   if (entry.record.expiresAtMs <= clockOf(ctx)()) {

package/src/segment-store.ts

@@ -12,6 +12,8 @@ export const DEFAULT_SEGMENT_TTL_MS = 24 * 60 * 60 * 1000;
 
 export interface SegmentMetadata {
   readonly partition: string;
+  /** Partition log continuity that produced these bytes (§2.1). */
+  readonly logEpoch: string;
   readonly table: string;
   readonly schemaVersion: number;
   readonly mediaType: 'rows' | 'sqlite';
@@ -36,6 +38,7 @@ export interface SegmentRecord extends SegmentMetadata {
  */
 export interface SegmentFindKey {
   readonly partition: string;
+  readonly logEpoch: string;
   readonly table: string;
   readonly schemaVersion: number;
   readonly mediaType: 'rows' | 'sqlite';
@@ -126,6 +129,7 @@ export class MemorySegmentStore implements SegmentStore {
     for (const { record } of this.#entries.values()) {
       if (
         record.partition === key.partition &&
+        record.logEpoch === key.logEpoch &&
         record.table === key.table &&
         record.schemaVersion === key.schemaVersion &&
         record.mediaType === key.mediaType &&

package/src/sqlite-bun-driver.ts

@@ -9,8 +9,12 @@ import type {
 export class BunSqliteDatabase implements SqliteDatabase {
   readonly native: Database;
 
-  constructor(path = ':memory:') {
-    this.native = new Database(path);
+  constructor(value: string | Database = ':memory:') {
+    this.native = typeof value === 'string' ? new Database(value) : value;
+  }
+
+  static deserialize(bytes: Uint8Array): BunSqliteDatabase {
+    return new BunSqliteDatabase(Database.deserialize(bytes));
   }
 
   exec(sql: string): void {

package/src/sqlite-bun.ts

@@ -40,10 +40,10 @@ export class SqliteLeaseStore extends SharedSqliteLeaseStore {
   }
 }
 
-export const buildSqliteImage: SqliteImageBuilder = (input) => {
+export const buildSqliteImage: SqliteImageBuilder = async (input) => {
   const db = new BunSqliteDatabase();
   try {
-    writeSqliteImage(db, input);
+    await writeSqliteImage(db, input);
     return db.serialize();
   } finally {
     db.close();

package/src/sqlite-dialect.ts

@@ -43,6 +43,12 @@ CREATE TABLE IF NOT EXISTS sync_partitions(
   max_commit_seq INTEGER NOT NULL DEFAULT 0,
   horizon_seq INTEGER NOT NULL DEFAULT 0
 );
+CREATE TABLE IF NOT EXISTS sync_partition_registry(
+  partition TEXT PRIMARY KEY,
+  log_epoch TEXT NOT NULL,
+  epoch_required INTEGER NOT NULL DEFAULT 0,
+  last_authenticated_at_ms INTEGER NOT NULL
+);
 CREATE TABLE IF NOT EXISTS sync_row_scopes(
   partition TEXT NOT NULL, tbl TEXT NOT NULL,
   var TEXT NOT NULL, value TEXT NOT NULL, row_id TEXT NOT NULL,
@@ -96,6 +102,7 @@ CREATE INDEX IF NOT EXISTS sync_reactions_dead_letter
   ON sync_reactions(partition, status, available_at_ms, idempotency_key);
 CREATE TABLE IF NOT EXISTS sync_clients(
   partition TEXT NOT NULL, client_id TEXT NOT NULL, actor_id TEXT NOT NULL,
+  wire_version INTEGER NOT NULL DEFAULT 1,
   cursor INTEGER NOT NULL, subscriptions TEXT NOT NULL,
   updated_at_ms INTEGER NOT NULL,
   PRIMARY KEY(partition, client_id)

package/src/sqlite-image.ts

@@ -57,7 +57,9 @@ export interface SqliteImageInput {
   readonly schemaVersion: number;
   readonly asOfCommitSeq: number;
   readonly scopeDigest: string;
-  readonly rows: readonly StoredRow[];
+  readonly rowBatches:
+    | AsyncIterable<readonly StoredRow[]>
+    | Iterable<readonly StoredRow[]>;
 }
 
 /**
@@ -68,14 +70,16 @@ export interface SqliteImageInput {
  * on the pull path. A Bun or Node host passes `buildSqliteImage`; a Workers
  * host omits it and serves the rows lane.
  */
-export type SqliteImageBuilder = (input: SqliteImageInput) => Uint8Array;
+export type SqliteImageBuilder = (
+  input: SqliteImageInput,
+) => Promise<Uint8Array>;
 
 /** Populate a §5.3 image database for a whole-table snapshot. */
-export function writeSqliteImage(
+export async function writeSqliteImage(
   db: SqliteDatabase,
   input: SqliteImageInput,
-): void {
-  const { table, rows } = input;
+): Promise<void> {
+  const { table, rowBatches } = input;
   const primaryKey = table.columns[table.primaryKeyIndex]?.name;
   const columnDefs = table.columns.map((column) => {
     const notNull = column.nullable ? '' : ' NOT NULL';
@@ -90,13 +94,6 @@ export function writeSqliteImage(
       "schemaVersion" INTEGER NOT NULL, "asOfCommitSeq" INTEGER NOT NULL,
       "scopeDigest" TEXT NOT NULL, "rowCount" INTEGER NOT NULL)`,
   );
-  db.query(`INSERT INTO ${IMAGE_METADATA_TABLE} VALUES (1, ?, ?, ?, ?, ?)`).run(
-    table.name,
-    input.schemaVersion,
-    input.asOfCommitSeq,
-    input.scopeDigest,
-    rows.length,
-  );
   const names = [
     ...table.columns.map((column) => quoteIdent(column.name)),
     quoteIdent(IMAGE_VERSION_COLUMN),
@@ -107,10 +104,24 @@ export function writeSqliteImage(
   );
   db.exec('BEGIN');
   try {
-    for (const row of rows) {
-      const values = decodeRow(table.columns, row.payload);
-      insert.run(...values.map(toSql), row.serverVersion);
+    let rowCount = 0;
+    for await (const rows of rowBatches) {
+      for (const row of rows) {
+        const values = decodeRow(table.columns, row.payload);
+        insert.run(...values.map(toSql), row.serverVersion);
+      }
+      rowCount += rows.length;
     }
+    db.query(
+      `INSERT INTO ${IMAGE_METADATA_TABLE} VALUES (1, ?, ?, ?, ?, ?)`,
+    ).run(
+      table.name,
+      input.schemaVersion,
+      input.asOfCommitSeq,
+      input.scopeDigest,
+      rowCount,
+    );
+
     db.exec('COMMIT');
   } catch (error) {
     db.exec('ROLLBACK');

package/src/sqlite-node.ts

@@ -43,13 +43,13 @@ export class SqliteLeaseStore extends SharedSqliteLeaseStore {
   }
 }
 
-export const buildSqliteImage: SqliteImageBuilder = (input) => {
+export const buildSqliteImage: SqliteImageBuilder = async (input) => {
   const directory = mkdtempSync(join(tmpdir(), 'syncular-server-image-'));
   const path = join(directory, 'segment.db');
   const db = new NodeSqliteDatabase(path);
   try {
     try {
-      writeSqliteImage(db, input);
+      await writeSqliteImage(db, input);
     } finally {
       db.close();
     }

package/src/sqlite-segment-store.ts

@@ -31,6 +31,7 @@ export class SqliteSegmentStore implements SegmentStore {
     this.db.exec(`
       CREATE TABLE IF NOT EXISTS sync_segments(
         segment_id TEXT PRIMARY KEY, partition TEXT NOT NULL,
+        log_epoch TEXT NOT NULL,
         tbl TEXT NOT NULL, schema_version INTEGER NOT NULL,
         media_type TEXT NOT NULL, scope_digest TEXT NOT NULL,
         as_of_commit_seq INTEGER NOT NULL, row_count INTEGER NOT NULL,
@@ -39,6 +40,14 @@ export class SqliteSegmentStore implements SegmentStore {
         expires_at_ms INTEGER NOT NULL, bytes BLOB NOT NULL
       );
     `);
+    const columns = this.db
+      .query<{ name: string }, []>('PRAGMA table_info(sync_segments)')
+      .all();
+    if (!columns.some((column) => column.name === 'log_epoch')) {
+      this.db.exec(
+        "ALTER TABLE sync_segments ADD COLUMN log_epoch TEXT NOT NULL DEFAULT ''",
+      );
+    }
   }
 
   async put(
@@ -57,14 +66,15 @@ export class SqliteSegmentStore implements SegmentStore {
     this.db
       .query(
         `INSERT OR REPLACE INTO sync_segments(
-          segment_id, partition, tbl, schema_version, media_type,
+          segment_id, partition, log_epoch, tbl, schema_version, media_type,
           scope_digest, as_of_commit_seq, row_count, row_cursor,
           next_row_cursor, byte_length, created_at_ms, expires_at_ms, bytes
-        ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
+        ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
       )
       .run(
         record.segmentId,
         record.partition,
+        record.logEpoch,
         record.table,
         record.schemaVersion,
         record.mediaType,
@@ -89,6 +99,7 @@ export class SqliteSegmentStore implements SegmentStore {
         {
           segment_id: string;
           partition: string;
+          log_epoch: string;
           tbl: string;
           schema_version: number;
           media_type: string;
@@ -110,6 +121,7 @@ export class SqliteSegmentStore implements SegmentStore {
       record: {
         segmentId: row.segment_id,
         partition: row.partition,
+        logEpoch: row.log_epoch,
         table: row.tbl,
         schemaVersion: row.schema_version,
         mediaType: row.media_type === 'sqlite' ? 'sqlite' : 'rows',
@@ -140,18 +152,19 @@ export class SqliteSegmentStore implements SegmentStore {
           created_at_ms: number;
           expires_at_ms: number;
         },
-        [string, string, number, string, string, number, number]
+        [string, string, string, number, string, string, number, number]
       >(
         `SELECT segment_id, row_count, next_row_cursor, byte_length,
                 created_at_ms, expires_at_ms
          FROM sync_segments
-         WHERE partition=? AND tbl=? AND schema_version=? AND media_type=?
+         WHERE partition=? AND log_epoch=? AND tbl=? AND schema_version=? AND media_type=?
            AND scope_digest=? AND as_of_commit_seq=? AND row_cursor IS NULL
            AND expires_at_ms > ?
          LIMIT 1`,
       )
       .get(
         key.partition,
+        key.logEpoch,
         key.table,
         key.schemaVersion,
         key.mediaType,
@@ -163,6 +176,7 @@ export class SqliteSegmentStore implements SegmentStore {
     return {
       segmentId: row.segment_id,
       partition: key.partition,
+      logEpoch: key.logEpoch,
       table: key.table,
       schemaVersion: key.schemaVersion,
       mediaType: key.mediaType,