@syncular/server 0.31.0 → 0.32.0

package/src/d1-storage.ts

@@ -68,6 +68,12 @@ import {
   rewriteRowSql,
   rewriteValues,
   SCHEMA_META_DDL_SQLITE,
+  SYNC_COLUMN_VERSIONS_COLUMN,
+  SYNC_PARTITION_COLUMN,
+  SYNC_PAYLOAD_COLUMN,
+  SYNC_ROW_ID_COLUMN,
+  SYNC_SCOPES_COLUMN,
+  SYNC_VERSION_COLUMN,
   type StoredColumnLayout,
   scanRowPageSql,
   schemaDdl,
@@ -86,10 +92,12 @@ import {
   collectCommitWindowPage,
   deserializePushResult,
   placeholders,
+  type SqliteChangeRecord,
   type SqliteCommitWindowRecord,
   type SqliteRowRecord,
   serializePushResult,
   sqliteDdlStatements,
+  toStoredChange,
   toStoredRow,
 } from './sqlite-dialect';
 import type {
@@ -97,8 +105,11 @@ import type {
   AuthoritativeQueryResult,
   CheckpointDeclaration,
   ClientCursorInfo,
+  ClientPushResult,
   ClientRecord,
   ClientSubscription,
+  CommitChangeSummary,
+  CommitDetail,
   CommitMetadata,
   CommitMetadataQuery,
   CommitWindowQuery,
@@ -113,6 +124,9 @@ import type {
   ReactionFailureUpdate,
   ReactionListQuery,
   ReactionPruneQuery,
+  RowChangeRecord,
+  RowChangesQuery,
+  RowListQuery,
   RowScanQuery,
   ScopeActivityQuery,
   ScopeCommitActivity,
@@ -2479,17 +2493,22 @@ export class D1ServerStorage implements ServerStorage {
     partition: string,
     query: CommitMetadataQuery,
   ): Promise<CommitMetadata[]> {
+    // Newest-first window, resumed by `afterSeq` (exclusive lower bound) and
+    // ended by `beforeSeq` (exclusive upper bound) when paging older. An
+    // omitted bound is the largest safe commitSeq, so one statement serves
+    // both the bounded and unbounded read.
+    const beforeSeq = query.beforeSeq ?? Number.MAX_SAFE_INTEGER;
     const { results: rows } = query.table
       ? await this.#db
           .prepare(
             `SELECT c.commit_seq, c.client_id, c.client_commit_id, c.actor_id, c.created_at_ms
              FROM sync_commits c
-             WHERE c.partition=? AND c.commit_seq>?
+             WHERE c.partition=? AND c.commit_seq>? AND c.commit_seq<?
                AND EXISTS (SELECT 1 FROM sync_changes ch
                  WHERE ch.partition=c.partition AND ch.commit_seq=c.commit_seq AND ch.tbl=?)
              ORDER BY c.commit_seq DESC LIMIT ?`,
           )
-          .bind(partition, query.afterSeq, query.table, query.limit)
+          .bind(partition, query.afterSeq, beforeSeq, query.table, query.limit)
           .all<{
             commit_seq: number;
             client_id: string;
@@ -2501,10 +2520,10 @@ export class D1ServerStorage implements ServerStorage {
           .prepare(
             `SELECT commit_seq, client_id, client_commit_id, actor_id, created_at_ms
              FROM sync_commits
-             WHERE partition=? AND commit_seq>?
+             WHERE partition=? AND commit_seq>? AND commit_seq<?
              ORDER BY commit_seq DESC LIMIT ?`,
           )
-          .bind(partition, query.afterSeq, query.limit)
+          .bind(partition, query.afterSeq, beforeSeq, query.limit)
           .all<{
             commit_seq: number;
             client_id: string;
@@ -2513,6 +2532,46 @@ export class D1ServerStorage implements ServerStorage {
             created_at_ms: number;
           }>();
     const out: CommitMetadata[] = [];
+    // `changeSample`: extra statement(s) for the whole page — the first
+    // `sample` changes (change-index order) of every listed commit.
+    const sampleCount =
+      query.changeSample !== undefined &&
+      Number.isInteger(query.changeSample) &&
+      query.changeSample > 0
+        ? query.changeSample
+        : 0;
+    const samples = new Map<number, CommitChangeSummary[]>();
+    if (sampleCount > 0 && rows.length > 0) {
+      const seqs = rows.map((row) => row.commit_seq);
+      // D1 caps the bound parameters of one statement, so a page larger than
+      // that cap splits into a few chunk statements, never one per commit.
+      const chunkSize = D1_MAX_BIND_PARAMS - 2;
+      for (let offset = 0; offset < seqs.length; offset += chunkSize) {
+        const chunk = seqs.slice(offset, offset + chunkSize);
+        const { results: changes } = await this.#db
+          .prepare(
+            `SELECT commit_seq, tbl, row_id, op FROM sync_changes
+             WHERE partition=? AND commit_seq IN (${placeholders(chunk.length)}) AND idx<?
+             ORDER BY commit_seq, idx`,
+          )
+          .bind(partition, ...chunk, sampleCount)
+          .all<{
+            commit_seq: number;
+            tbl: string;
+            row_id: string;
+            op: number;
+          }>();
+        for (const change of changes) {
+          const sample = samples.get(change.commit_seq) ?? [];
+          sample.push({
+            table: change.tbl,
+            rowId: change.row_id,
+            op: change.op === 1 ? 'upsert' : 'delete',
+          });
+          samples.set(change.commit_seq, sample);
+        }
+      }
+    }
     for (const row of rows) {
       const { results: changes } = await this.#db
         .prepare(
@@ -2528,11 +2587,135 @@ export class D1ServerStorage implements ServerStorage {
         createdAtMs: row.created_at_ms,
         changeCount: changes.reduce((sum, c) => sum + c.n, 0),
         tables: changes.map((c) => c.tbl),
+        ...(sampleCount > 0
+          ? { sample: samples.get(row.commit_seq) ?? [] }
+          : {}),
       });
     }
     return out;
   }
 
+  async getCommit(
+    partition: string,
+    commitSeq: number,
+  ): Promise<CommitDetail | undefined> {
+    const meta = await this.#db
+      .prepare(
+        'SELECT client_id, client_commit_id, actor_id, created_at_ms FROM sync_commits WHERE partition=? AND commit_seq=?',
+      )
+      .bind(partition, commitSeq)
+      .first<{
+        client_id: string;
+        client_commit_id: string;
+        actor_id: string;
+        created_at_ms: number;
+      }>();
+    if (meta === null) return undefined;
+    const { results: changeRows } = await this.#db
+      .prepare(
+        'SELECT tbl, row_id, op, row_version, scopes, payload FROM sync_changes WHERE partition=? AND commit_seq=? ORDER BY idx',
+      )
+      .bind(partition, commitSeq)
+      .all<SqliteChangeRecord>();
+    const changes = changeRows.map(toStoredChange);
+    return {
+      commitSeq,
+      clientId: meta.client_id,
+      clientCommitId: meta.client_commit_id,
+      actorId: meta.actor_id,
+      createdAtMs: meta.created_at_ms,
+      changeCount: changes.length,
+      tables: [...new Set(changes.map((change) => change.table))].sort(),
+      changes,
+    };
+  }
+
+  async listRowChanges(
+    partition: string,
+    query: RowChangesQuery,
+  ): Promise<RowChangeRecord[]> {
+    // Retained-log read for one row: no dedicated index (the write path pays
+    // nothing for it), bounded by retention (§4.6).
+    const { results: rows } = await this.#db
+      .prepare(
+        `SELECT ch.commit_seq, ch.op, ch.row_version, c.actor_id, c.client_id, c.created_at_ms
+         FROM sync_changes ch
+         JOIN sync_commits c
+           ON c.partition=ch.partition AND c.commit_seq=ch.commit_seq
+         WHERE ch.partition=? AND ch.tbl=? AND ch.row_id=?
+         ORDER BY ch.commit_seq DESC, ch.idx DESC LIMIT ?`,
+      )
+      .bind(partition, query.table, query.rowId, query.limit)
+      .all<{
+        commit_seq: number;
+        op: number;
+        row_version: number | null;
+        actor_id: string;
+        client_id: string;
+        created_at_ms: number;
+      }>();
+    return rows.map((row) => ({
+      commitSeq: row.commit_seq,
+      op: row.op === 1 ? 'upsert' : 'delete',
+      ...(row.row_version !== null ? { rowVersion: row.row_version } : {}),
+      actorId: row.actor_id,
+      clientId: row.client_id,
+      createdAtMs: row.created_at_ms,
+    }));
+  }
+
+  async listRows(partition: string, query: RowListQuery): Promise<StoredRow[]> {
+    // Keyset page over the row table's (partition, rowId) primary key: an
+    // index range read, no scope filter, no per-row lookup.
+    const { results } = await this.#schemaDatabase()
+      .prepare(
+        `SELECT ${quoteIdent(SYNC_ROW_ID_COLUMN)} AS row_id, ${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, ${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, ${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, ${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions
+         FROM ${quoteIdent(this.table(query.table).name)}
+         WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=? AND ${quoteIdent(SYNC_ROW_ID_COLUMN)}>?
+         ORDER BY ${quoteIdent(SYNC_ROW_ID_COLUMN)} LIMIT ?`,
+      )
+      .bind(partition, query.afterRowId ?? '', query.limit)
+      .all<SqliteRowRecord>();
+    return results.map(toStoredRow);
+  }
+
+  async countRows(partition: string, table: string): Promise<number> {
+    const record = await this.#schemaDatabase()
+      .prepare(
+        `SELECT count(*) AS n FROM ${quoteIdent(this.table(table).name)} WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=?`,
+      )
+      .bind(partition)
+      .first<{ n: number }>();
+    return record?.n ?? 0;
+  }
+
+  async listPushResults(
+    partition: string,
+    clientCommitId: string,
+  ): Promise<ClientPushResult[]> {
+    // Retained outcomes for one clientCommitId: no dedicated index, ordered
+    // by client so the read is deterministic.
+    const { results } = await this.#schemaDatabase()
+      .prepare(
+        'SELECT client_id, result FROM sync_push_results WHERE partition=? AND client_commit_id=? ORDER BY client_id',
+      )
+      .bind(partition, clientCommitId)
+      .all<{ client_id: string; result: string }>();
+    return results.map((record) => {
+      try {
+        return {
+          clientId: record.client_id,
+          result: deserializePushResult(record.result),
+        };
+      } catch {
+        throw syncError(
+          'sync.idempotency_cache_miss',
+          'persisted push result unreadable (§6.3)',
+        );
+      }
+    });
+  }
+
   async scopeActivity(
     partition: string,
     query: ScopeActivityQuery,
@@ -2571,25 +2754,6 @@ export class D1ServerStorage implements ServerStorage {
     return out;
   }
 
-  async getRowScopes(
-    partition: string,
-    table: string,
-    rowId: string,
-  ): Promise<
-    { serverVersion: number; scopes: Record<string, string> } | undefined
-  > {
-    const db = this.#schemaDatabase();
-    const record = await db
-      .prepare(selectRowScopesSql(this.table(table), 'sqlite'))
-      .bind(partition, rowId)
-      .first<{ server_version: number; scopes: string }>();
-    if (record === null) return undefined;
-    return {
-      serverVersion: record.server_version,
-      scopes: JSON.parse(record.scopes) as Record<string, string>,
-    };
-  }
-
   async listPartitions(): Promise<string[]> {
     return (await this.listPartitionRegistry()).map((entry) => entry.partition);
   }

package/src/events-ring.ts

@@ -3,8 +3,8 @@
  * without any infrastructure dependency.
  *
  * `RingBufferEvents` is a `SyncularServerEvents` sink that retains the last
- * N events in memory and exposes a `query({type?, sinceMs?, clientId?,
- * actorId?, limit})` plus a `subscribe(listener)` hook for live tails. It
+ * N events in memory and exposes a `query({type?, partition?, sinceMs?,
+ * clientId?, actorId?, limit})` plus a `subscribe(listener)` hook for live tails. It
  * composes with any other sink through `composeEvents(...sinks)`, so a host
  * can keep `consoleJsonEvents()` (or a Sentry adapter) AND feed the console
  * event tail from the same emissions. Fire-and-forget discipline is
@@ -20,6 +20,10 @@ import {
 export interface RingEventQuery {
   /** Restrict to one event `type` (e.g. `push.applied`). */
   readonly type?: SyncularServerEvent['type'];
+  /** Restrict to one partition (every event carries one). */
+  readonly partition?: string;
+  /** Restrict to the events of one sync request. */
+  readonly requestId?: string;
   /** Only events with `atMs >= sinceMs`. */
   readonly sinceMs?: number;
   /** Restrict to events carrying this `clientId` (not every type does). */
@@ -41,6 +45,12 @@ export function matchesRingQuery(
   query: Omit<RingEventQuery, 'limit'>,
 ): boolean {
   if (query.type !== undefined && event.type !== query.type) return false;
+  if (query.partition !== undefined && event.partition !== query.partition) {
+    return false;
+  }
+  if (query.requestId !== undefined && event.requestId !== query.requestId) {
+    return false;
+  }
   if (query.sinceMs !== undefined && event.atMs < query.sinceMs) return false;
   const carrier = event as { clientId?: string; actorId?: string };
   if (query.clientId !== undefined && carrier.clientId !== query.clientId) {
@@ -67,6 +77,7 @@ export class RingBufferEvents implements SyncularServerEvents {
   /** Index of the oldest element in the circular buffer. */
   #head = 0;
   #size = 0;
+  #evictedThroughMs: number | undefined;
   /** Live subscribers (the SSE tail). Each notify is guarded. */
   readonly #listeners = new Set<(event: SyncularServerEvent) => void>();
 
@@ -84,6 +95,8 @@ export class RingBufferEvents implements SyncularServerEvents {
       this.#size += 1;
     } else {
       // Full: overwrite the oldest and advance the head.
+      const evicted = this.#buffer[this.#head];
+      if (evicted !== undefined) this.#markEvicted(evicted.atMs);
       this.#buffer[this.#head] = event;
       this.#head = (this.#head + 1) % this.#capacity;
     }
@@ -119,6 +132,21 @@ export class RingBufferEvents implements SyncularServerEvents {
     return this.#size;
   }
 
+  /**
+   * The newest `atMs` among events the ring dropped (evicted when full, or
+   * cleared); `undefined` while nothing was dropped. A window that starts
+   * at or before it is incomplete.
+   */
+  get evictedThroughMs(): number | undefined {
+    return this.#evictedThroughMs;
+  }
+
+  #markEvicted(atMs: number): void {
+    if (this.#evictedThroughMs === undefined || atMs > this.#evictedThroughMs) {
+      this.#evictedThroughMs = atMs;
+    }
+  }
+
   /**
    * Newest-first slice of the retained events. Filters by `type` and
    * `sinceMs`, then caps to `limit`. Returns a fresh array — the caller may
@@ -139,6 +167,7 @@ export class RingBufferEvents implements SyncularServerEvents {
 
   /** Drop all retained events. */
   clear(): void {
+    for (const event of this.#buffer) this.#markEvicted(event.atMs);
     this.#buffer = [];
     this.#head = 0;
     this.#size = 0;

package/src/events.ts

@@ -29,11 +29,21 @@ export interface RequestHandledEvent {
   /**
    * `ok` — response streamed to END;
    * `schema_floor` — §2.4 required-schema answer;
-   * `reset` — §2.1 log-epoch reset answer;
+   * `epoch_acquired` — §2.1 answer to a request without `logEpoch` (a new
+   *   or wiped replica's first round);
+   * `reset` — §2.1 answer to a `logEpoch` that no longer matches the log;
    * `rejected` — request validation failed before any bytes (§1.7);
    * `error` — in-band ERROR frame (§1.6) or a thrown host failure.
    */
-  readonly outcome: 'ok' | 'schema_floor' | 'reset' | 'rejected' | 'error';
+  readonly outcome:
+    | 'ok'
+    | 'schema_floor'
+    | 'epoch_acquired'
+    | 'reset'
+    | 'rejected'
+    | 'error';
+  /** The REQ_HEADER clientId; absent when the request failed to decode. */
+  readonly clientId?: string;
   /** §10.2 code for `rejected`/`error`; `"internal"` for non-SyncErrors. */
   readonly errorCode?: string;
   readonly pushCommits: number;
@@ -64,6 +74,9 @@ export interface PushRejectedEvent extends PushEventBase {
   readonly type: 'push.rejected';
   readonly code: string;
   readonly opIndex: number;
+  /** The terminating operation's table and rowId. */
+  readonly table?: string;
+  readonly rowId?: string;
   /** True when the rejection was replayed from the idempotency cache. */
   readonly replay: boolean;
   /** Original host time for outcomes recorded by a metadata-aware server. */
@@ -76,6 +89,9 @@ export interface PushRejectedEvent extends PushEventBase {
 export interface PushConflictedEvent extends PushEventBase {
   readonly type: 'push.conflicted';
   readonly opIndex: number;
+  /** The conflicting operation's table and rowId. */
+  readonly table?: string;
+  readonly rowId?: string;
   /** True when the conflict was replayed from the idempotency cache. */
   readonly replay: boolean;
   readonly recordedAtMs?: number;
@@ -306,7 +322,13 @@ export interface LeaseRevokedEvent {
   readonly leaseId: string;
 }
 
-export type SyncularServerEvent =
+/**
+ * Every event. `requestId` is present on the events emitted while serving
+ * one sync request (`request.handled`, `push.*`, `pull.served`,
+ * `lease.issued`, `reaction.queued`, `scopes.resolve_failed`) and shared by
+ * all of them, so a sink can group one request's events.
+ */
+export type SyncularServerEvent = (
   | RequestHandledEvent
   | PushAppliedEvent
   | PushRejectedEvent
@@ -329,7 +351,8 @@ export type SyncularServerEvent =
   | PruneCompletedEvent
   | ScopesResolveFailedEvent
   | LeaseIssuedEvent
-  | LeaseRevokedEvent;
+  | LeaseRevokedEvent
+) & { readonly requestId?: string };
 
 /**
  * The seam. Optional on the server config — when absent, no event object

package/src/handler.ts

@@ -373,7 +373,7 @@ async function planRequest(
 
 /** Mutable outcome box shared with the instrumented stream wrapper. */
 interface RequestReport {
-  outcome: 'ok' | 'schema_floor' | 'reset' | 'error';
+  outcome: 'ok' | 'schema_floor' | 'epoch_acquired' | 'reset' | 'error';
   errorCode?: string;
 }
 
@@ -421,11 +421,14 @@ function emitPushEvent(
   }
   // Rejected (§6.3): exactly one terminating operation record.
   const record = frame.results[0];
+  const op = record !== undefined ? push.operations[record.opIndex] : undefined;
+  const target = op !== undefined ? { table: op.table, rowId: op.rowId } : {};
   if (record?.status === 'conflict') {
     emitEvent(events, {
       type: 'push.conflicted',
       ...base,
       opIndex: record.opIndex,
+      ...target,
       replay: processed.replayed,
       ...(processed.recordedAtMs !== undefined
         ? { recordedAtMs: processed.recordedAtMs }
@@ -444,6 +447,7 @@ function emitPushEvent(
         ? record.code
         : 'sync.invalid_request',
     opIndex: record?.opIndex ?? 0,
+    ...target,
     replay: processed.replayed,
     ...(processed.recordedAtMs !== undefined
       ? { recordedAtMs: processed.recordedAtMs }
@@ -491,7 +495,10 @@ async function* streamResponse(
     plan.wireVersion,
   );
   if (plan.epochReset) {
-    if (report !== undefined) report.outcome = 'reset';
+    if (report !== undefined) {
+      report.outcome =
+        plan.header.logEpoch === undefined ? 'epoch_acquired' : 'reset';
+    }
     yield END_FRAME_BYTES;
     return;
   }
@@ -729,14 +736,20 @@ export async function createSyncResponseStream(
   bytes: Uint8Array,
   ctx: SyncRequestContext,
 ): Promise<AsyncIterable<Uint8Array>> {
-  const events = ctx.events;
-  if (events === undefined) return createStreamCore(bytes, ctx, undefined);
+  const sink = ctx.events;
+  if (sink === undefined) return createStreamCore(bytes, ctx, undefined);
   // Instrumented path: measure with the ctx clock, count bytes at the
-  // seam, emit exactly one `request.handled` per request.
+  // seam, emit exactly one `request.handled` per request. Every event of
+  // this request carries one `requestId`.
+  const requestId = crypto.randomUUID();
+  const events: SyncularServerEvents = {
+    emit: (event) => sink.emit({ ...event, requestId }),
+  };
+  const requestCtx = { ...ctx, events };
   const clock = clockOf(ctx);
   const startedAtMs = clock();
   try {
-    return await createStreamCore(bytes, ctx, events, startedAtMs);
+    return await createStreamCore(bytes, requestCtx, events, startedAtMs);
   } catch (error) {
     emitEvent(events, {
       type: 'request.handled',
@@ -878,6 +891,7 @@ async function* instrumentedStream(
       bytesIn,
       bytesOut,
       outcome: report.outcome,
+      clientId: plan.header.clientId,
       ...(report.errorCode !== undefined
         ? { errorCode: report.errorCode }
         : {}),