@syncular/server 0.1.2 → 0.2.0

package/src/sqlite-dialect.ts

@@ -0,0 +1,337 @@
+/**
+ * Shared SQLite dialect for the two SQLite-family storages: `bun:sqlite`
+ * (synchronous, `SqliteServerStorage`) and Cloudflare D1 (async,
+ * `D1ServerStorage`). D1 *is* SQLite — same DDL, same statement grammar,
+ * same `?` positional placeholders, same `INSERT OR REPLACE` / `INSERT OR
+ * IGNORE` upsert idioms — so the schema and the value (de)serialization are
+ * genuinely common ground and live here.
+ *
+ * What is NOT shared: statement *execution*. `bun:sqlite` is sync
+ * (`db.query(sql).get(...)`) and D1 is async (`await
+ * db.prepare(sql).bind(...).all()`); a shared execution layer would have to
+ * pick one calling convention and adapt the other, which is uglier than two
+ * thin storage classes that each speak their driver's native shape while
+ * importing the same SQL text and codecs from here (TODO §4.2 judgment
+ * call — "clean parallel implementation against the storage contract").
+ * Both classes run the identical `test/storage-contract.ts`, so the
+ * behavior is held key-for-key regardless.
+ */
+import type { PushOperationResult, ScopeMap } from '@syncular/core';
+import { matchesEffective } from './scopes';
+import type {
+  StoredChange,
+  StoredCommit,
+  StoredPushResult,
+  StoredRow,
+} from './storage';
+
+/**
+ * Schema DDL — one statement per `;`-delimited chunk. `bun:sqlite` applies
+ * the whole string via `db.exec(SQLITE_DDL)`; D1 applies each statement
+ * separately (its `prepare`/`batch` API is one statement per call). Types
+ * are SQLite's: `INTEGER`/`TEXT`/`BLOB`. Scopes are stored as JSON `TEXT`
+ * (no JSONB on SQLite), payloads as `BLOB`.
+ *
+ * The inverted scope index carries the ordered column (`commit_seq` /
+ * `row_id`) last in the PRIMARY KEY, so the candidate scan is an index
+ * range that returns already-ordered rows — the same covering-index shape
+ * the Postgres storage documents (§3.1, REVISE B2 performance-by-
+ * construction).
+ */
+export const SQLITE_DDL = `
+CREATE TABLE IF NOT EXISTS sync_partitions(
+  partition TEXT PRIMARY KEY,
+  max_commit_seq INTEGER NOT NULL DEFAULT 0,
+  horizon_seq INTEGER NOT NULL DEFAULT 0
+);
+CREATE TABLE IF NOT EXISTS sync_row_scopes(
+  partition TEXT NOT NULL, tbl TEXT NOT NULL,
+  var TEXT NOT NULL, value TEXT NOT NULL, row_id TEXT NOT NULL,
+  PRIMARY KEY(partition, tbl, var, value, row_id)
+);
+CREATE TABLE IF NOT EXISTS sync_commits(
+  partition TEXT NOT NULL, commit_seq INTEGER NOT NULL,
+  client_id TEXT NOT NULL, client_commit_id TEXT NOT NULL,
+  actor_id TEXT NOT NULL, created_at_ms INTEGER NOT NULL,
+  PRIMARY KEY(partition, commit_seq)
+);
+CREATE INDEX IF NOT EXISTS sync_commits_by_time
+  ON sync_commits(partition, created_at_ms);
+CREATE TABLE IF NOT EXISTS sync_changes(
+  partition TEXT NOT NULL, commit_seq INTEGER NOT NULL, idx INTEGER NOT NULL,
+  tbl TEXT NOT NULL, row_id TEXT NOT NULL, op INTEGER NOT NULL,
+  row_version INTEGER, scopes TEXT NOT NULL, payload BLOB,
+  PRIMARY KEY(partition, commit_seq, idx)
+);
+CREATE INDEX IF NOT EXISTS sync_changes_by_table
+  ON sync_changes(partition, commit_seq, tbl, idx);
+CREATE TABLE IF NOT EXISTS sync_change_scopes(
+  partition TEXT NOT NULL, tbl TEXT NOT NULL,
+  var TEXT NOT NULL, value TEXT NOT NULL, commit_seq INTEGER NOT NULL,
+  PRIMARY KEY(partition, tbl, var, value, commit_seq)
+);
+CREATE TABLE IF NOT EXISTS sync_push_results(
+  partition TEXT NOT NULL, client_id TEXT NOT NULL,
+  client_commit_id TEXT NOT NULL, result TEXT NOT NULL,
+  PRIMARY KEY(partition, client_id, client_commit_id)
+);
+CREATE TABLE IF NOT EXISTS sync_clients(
+  partition TEXT NOT NULL, client_id TEXT NOT NULL, actor_id TEXT NOT NULL,
+  cursor INTEGER NOT NULL, subscriptions TEXT NOT NULL,
+  updated_at_ms INTEGER NOT NULL,
+  PRIMARY KEY(partition, client_id)
+);
+CREATE TABLE IF NOT EXISTS sync_blob_refs(
+  partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,
+  blob_id TEXT NOT NULL,
+  PRIMARY KEY(partition, tbl, row_id, blob_id)
+);
+CREATE INDEX IF NOT EXISTS sync_blob_refs_by_blob
+  ON sync_blob_refs(partition, blob_id);
+`;
+
+/** Split the DDL into individual statements (D1 applies them one by one). */
+export function sqliteDdlStatements(): string[] {
+  return SQLITE_DDL.split(';')
+    .map((statement) => statement.trim())
+    .filter((statement) => statement.length > 0);
+}
+
+/** `?,?,…` for an `IN (…)` clause of `count` positional parameters. */
+export function placeholders(count: number): string {
+  return new Array(count).fill('?').join(',');
+}
+
+// -- value (de)serialization, shared by both SQLite storages ----------------
+
+/** Runtime-neutral base64 (no `Buffer` — Workers-safe) for push results. */
+function bytesToBase64(bytes: Uint8Array): string {
+  let binary = '';
+  for (const byte of bytes) binary += String.fromCharCode(byte);
+  return btoa(binary);
+}
+
+function base64ToBytes(text: string): Uint8Array {
+  const binary = atob(text);
+  const bytes = new Uint8Array(binary.length);
+  for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
+  return bytes;
+}
+
+interface SerializedResult {
+  opIndex: number;
+  status: string;
+  code?: string;
+  message?: string;
+  serverVersion?: number;
+  serverRow?: string;
+  retryable?: boolean;
+}
+
+/** Serialize a push result to the JSON `TEXT` stored in `sync_push_results`. */
+export function serializePushResult(result: StoredPushResult): string {
+  return JSON.stringify({
+    status: result.status,
+    ...(result.commitSeq !== undefined ? { commitSeq: result.commitSeq } : {}),
+    results: result.results.map((record) => {
+      if (record.status === 'conflict') {
+        return {
+          opIndex: record.opIndex,
+          status: record.status,
+          code: record.code,
+          message: record.message,
+          serverVersion: record.serverVersion,
+          serverRow: bytesToBase64(record.serverRow),
+        };
+      }
+      if (record.status === 'error') {
+        return {
+          opIndex: record.opIndex,
+          status: record.status,
+          code: record.code,
+          message: record.message,
+          retryable: record.retryable,
+        };
+      }
+      return { opIndex: record.opIndex, status: record.status };
+    }),
+  });
+}
+
+export function deserializePushResult(text: string): StoredPushResult {
+  const parsed = JSON.parse(text) as {
+    status: 'applied' | 'rejected';
+    commitSeq?: number;
+    results: SerializedResult[];
+  };
+  const results: PushOperationResult[] = parsed.results.map((record) => {
+    if (record.status === 'conflict') {
+      return {
+        opIndex: record.opIndex,
+        status: 'conflict',
+        code: record.code ?? '',
+        message: record.message ?? '',
+        serverVersion: record.serverVersion ?? 0,
+        serverRow: base64ToBytes(record.serverRow ?? ''),
+      };
+    }
+    if (record.status === 'error') {
+      return {
+        opIndex: record.opIndex,
+        status: 'error',
+        code: record.code ?? '',
+        message: record.message ?? '',
+        retryable: record.retryable ?? false,
+      };
+    }
+    return { opIndex: record.opIndex, status: 'applied' };
+  });
+  return {
+    status: parsed.status,
+    ...(parsed.commitSeq !== undefined ? { commitSeq: parsed.commitSeq } : {}),
+    results,
+  };
+}
+
+/** Row-record shape the SQLite `SELECT`s in both storages return. */
+export interface SqliteRowRecord {
+  row_id: string;
+  server_version: number;
+  scopes: string;
+  payload: Uint8Array;
+}
+
+export interface SqliteChangeRecord {
+  tbl: string;
+  row_id: string;
+  op: number;
+  row_version: number | null;
+  scopes: string;
+  payload: Uint8Array | null;
+}
+
+/** `bun:sqlite` returns `Uint8Array`; D1 returns `ArrayBuffer` for BLOBs. */
+export function asUint8Array(value: unknown): Uint8Array {
+  if (value instanceof Uint8Array) return value;
+  if (value instanceof ArrayBuffer) return new Uint8Array(value);
+  if (Array.isArray(value)) return new Uint8Array(value);
+  throw new Error(`expected a BLOB column, got ${typeof value}`);
+}
+
+export function toStoredRow(record: SqliteRowRecord): StoredRow {
+  return {
+    rowId: record.row_id,
+    serverVersion: record.server_version,
+    scopes: JSON.parse(record.scopes) as Record<string, string>,
+    payload: asUint8Array(record.payload),
+  };
+}
+
+export function toStoredChange(record: SqliteChangeRecord): StoredChange {
+  return {
+    table: record.tbl,
+    rowId: record.row_id,
+    op: record.op === 1 ? 'upsert' : 'delete',
+    ...(record.row_version !== null ? { rowVersion: record.row_version } : {}),
+    scopes: JSON.parse(record.scopes) as Record<string, string>,
+    ...(record.payload !== null
+      ? { payload: asUint8Array(record.payload) }
+      : {}),
+  };
+}
+
+/**
+ * One result row of `commitWindowPageSql` (candidate LEFT JOIN commit meta
+ * LEFT JOIN changes): meta/change columns are NULL when the joined row
+ * vanished (see the builder's LEFT JOIN contract). `payload` is a BLOB —
+ * `bun:sqlite` hands back `Uint8Array`, D1 `ArrayBuffer`; `toStoredChange`
+ * normalizes via `asUint8Array`.
+ */
+export interface SqliteCommitWindowRecord {
+  commit_seq: number;
+  actor_id: string | null;
+  created_at_ms: number | null;
+  tbl: string | null;
+  row_id: string | null;
+  op: number | null;
+  row_version: number | null;
+  scopes: string | null;
+  payload: Uint8Array | null;
+}
+
+/**
+ * Fold one `commitWindowPageSql` page into commits, preserving the exact
+ * semantics of the old per-candidate loop:
+ *
+ *   - rows arrive ordered (commit_seq, idx); consecutive rows with the same
+ *     `commit_seq` are one candidate commit;
+ *   - every candidate advances the cursor (`lastSeq`) — including vanished
+ *     commits (NULL meta) and commits whose changes all fail the exact
+ *     multi-variable scope verification (`matchesEffective`);
+ *   - a commit is emitted only with its matching changes, in `idx` order;
+ *   - stops after the commit that accumulates at least `remaining` matching
+ *     changes (never splitting a commit).
+ *
+ * `candidateCount` counts the processed candidates so the caller can detect
+ * a short page (window exhausted) exactly as it did with the old candidate
+ * query.
+ */
+export function collectCommitWindowPage(
+  records: readonly SqliteCommitWindowRecord[],
+  scopeFilter: ScopeMap,
+  remaining: number,
+): {
+  commits: StoredCommit[];
+  delivered: number;
+  lastSeq: number;
+  candidateCount: number;
+} {
+  const commits: StoredCommit[] = [];
+  let delivered = 0;
+  let lastSeq = 0;
+  let candidateCount = 0;
+  let i = 0;
+  while (i < records.length) {
+    const head = records[i];
+    if (head === undefined) break;
+    const seq = head.commit_seq;
+    candidateCount += 1;
+    lastSeq = seq;
+    const changes: StoredChange[] = [];
+    for (; i < records.length; i++) {
+      const record = records[i];
+      if (record === undefined || record.commit_seq !== seq) break;
+      // NULL tbl: a candidate with no change rows for the table (LEFT JOIN
+      // contract) — it still advanced the cursor above.
+      if (
+        record.tbl === null ||
+        record.row_id === null ||
+        record.op === null ||
+        record.scopes === null
+      ) {
+        continue;
+      }
+      const change = toStoredChange({
+        tbl: record.tbl,
+        row_id: record.row_id,
+        op: record.op,
+        row_version: record.row_version,
+        scopes: record.scopes,
+        payload: record.payload,
+      });
+      if (matchesEffective(change.scopes, scopeFilter)) changes.push(change);
+    }
+    // NULL meta: the commit vanished — cursor advanced, nothing emitted.
+    if (head.actor_id === null || head.created_at_ms === null) continue;
+    if (changes.length === 0) continue;
+    commits.push({
+      commitSeq: seq,
+      createdAtMs: head.created_at_ms,
+      actorId: head.actor_id,
+      changes,
+    });
+    delivered += changes.length;
+    if (delivered >= remaining) break;
+  }
+  return { commits, delivered, lastSeq, candidateCount };
+}

package/src/sqlite-image.ts

@@ -0,0 +1,124 @@
+/**
+ * SQLite-image segment generation (SPEC.md §5.3): a complete SQLite
+ * database file carrying one table's whole effective-scope snapshot at
+ * the bootstrap pin, plus the single-row `_syncular_segment` metadata
+ * table. Built in memory on bun:sqlite — dependency-free.
+ *
+ * Images are NOT byte-deterministic (§5.3): the content address pins the
+ * served bytes, and cross-client dedup comes from the segment store's
+ * metadata lookup (`SegmentStore.find`), not from hash convergence.
+ */
+import { Database } from 'bun:sqlite';
+import { decodeRow, type RowColumn, type RowValue } from '@syncular/core';
+import type { CompiledTable } from './schema';
+import type { StoredRow } from './storage';
+
+/** The §5.6 version column as it appears inside a sqlite image (§5.3). */
+export const IMAGE_VERSION_COLUMN = '_syncular_version';
+
+/** The §5.3 metadata table name. */
+export const IMAGE_METADATA_TABLE = '_syncular_segment';
+
+function quoteIdent(name: string): string {
+  return `"${name.replaceAll('"', '""')}"`;
+}
+
+/** §5.3 column type affinities (string/json/blob_ref TEXT, boolean INTEGER, …). */
+function sqlType(column: RowColumn): string {
+  switch (column.type) {
+    case 'string':
+    case 'json':
+    case 'blob_ref':
+      return 'TEXT';
+    case 'integer':
+      return 'INTEGER';
+    case 'float':
+      return 'REAL';
+    case 'boolean':
+      return 'INTEGER';
+    case 'bytes':
+    case 'crdt':
+      // §5.10: a crdt column's opaque bytes ride the BLOB affinity, like
+      // bytes.
+      return 'BLOB';
+    default:
+      throw new Error(`unsupported column type: ${String(column.type)}`);
+  }
+}
+
+/** §5.3 cell encodings: boolean as 0/1, everything else its natural type. */
+function toSql(value: RowValue): string | number | bigint | Uint8Array | null {
+  if (typeof value === 'boolean') return value ? 1 : 0;
+  return value;
+}
+
+export interface SqliteImageInput {
+  readonly table: CompiledTable;
+  readonly schemaVersion: number;
+  readonly asOfCommitSeq: number;
+  readonly scopeDigest: string;
+  readonly rows: readonly StoredRow[];
+}
+
+/**
+ * The §5.3 image-builder capability, injected through
+ * `SyncServerConfig.sqliteImageBuilder` (TODO §4.2). Building an image needs
+ * a real SQLite engine (`bun:sqlite` here), which is not available on every
+ * runtime — Cloudflare Workers has none. So the core takes the builder as an
+ * optional capability rather than importing `bun:sqlite` on the pull path:
+ * a Bun/Node host passes `buildSqliteImage`; a Workers host omits it and the
+ * pull serves the rows lane (§5.3 clients advertise sqlite as an *accept*,
+ * never a requirement — the host chooses the served format from what it can
+ * produce; this is a support floor, not a fallback).
+ */
+export type SqliteImageBuilder = (input: SqliteImageInput) => Uint8Array;
+
+/** Build the §5.3 image bytes for a whole-table snapshot. */
+export const buildSqliteImage: SqliteImageBuilder = (input) => {
+  const { table, rows } = input;
+  const primaryKey = table.columns[table.primaryKeyIndex]?.name;
+  const db = new Database(':memory:');
+  try {
+    const columnDefs = table.columns.map((column) => {
+      const notNull = column.nullable ? '' : ' NOT NULL';
+      const pk = column.name === primaryKey ? ' PRIMARY KEY' : '';
+      return `${quoteIdent(column.name)} ${sqlType(column)}${notNull}${pk}`;
+    });
+    columnDefs.push(`${quoteIdent(IMAGE_VERSION_COLUMN)} INTEGER NOT NULL`);
+    db.exec(
+      `CREATE TABLE ${quoteIdent(table.name)} (${columnDefs.join(', ')})`,
+    );
+    db.exec(
+      `CREATE TABLE ${IMAGE_METADATA_TABLE} (
+        format INTEGER NOT NULL, "table" TEXT NOT NULL,
+        "schemaVersion" INTEGER NOT NULL, "asOfCommitSeq" INTEGER NOT NULL,
+        "scopeDigest" TEXT NOT NULL, "rowCount" INTEGER NOT NULL)`,
+    );
+    db.query(
+      `INSERT INTO ${IMAGE_METADATA_TABLE} VALUES (1, ?, ?, ?, ?, ?)`,
+    ).run(
+      table.name,
+      input.schemaVersion,
+      input.asOfCommitSeq,
+      input.scopeDigest,
+      rows.length,
+    );
+    const names = [
+      ...table.columns.map((column) => quoteIdent(column.name)),
+      quoteIdent(IMAGE_VERSION_COLUMN),
+    ];
+    const insert = db.query(
+      `INSERT INTO ${quoteIdent(table.name)} (${names.join(', ')})
+       VALUES (${names.map(() => '?').join(', ')})`,
+    );
+    db.exec('BEGIN');
+    for (const row of rows) {
+      const values = decodeRow(table.columns, row.payload);
+      insert.run(...values.map(toSql), row.serverVersion);
+    }
+    db.exec('COMMIT');
+    return new Uint8Array(db.serialize());
+  } finally {
+    db.close();
+  }
+};

package/src/sqlite-lease-store.ts

@@ -0,0 +1,144 @@
+/**
+ * SQLite-backed auth-lease store via `bun:sqlite` (§7.3.1 reference store,
+ * dependency-free). Bun-specific by design (top-level `bun:sqlite` import),
+ * so it lives in its own module — the runtime-neutral `LeaseStore` interface,
+ * `LeaseRecord`, and `MemoryLeaseStore` stay in `lease-store.ts` for the
+ * Workers/edge core (TODO §4.2 neutrality discipline; enforced by
+ * `test/runtime-neutrality.test.ts`).
+ */
+import { Database } from 'bun:sqlite';
+import type { ScopeMap } from '@syncular/core';
+import type { LeaseIdFactory, LeaseRecord, LeaseStore } from './lease-store';
+
+function defaultLeaseId(): string {
+  return `lease_${crypto.randomUUID()}`;
+}
+
+export class SqliteLeaseStore implements LeaseStore {
+  readonly db: Database;
+  readonly #newId: LeaseIdFactory;
+
+  constructor(
+    db: Database | string = ':memory:',
+    options?: { readonly leaseId?: LeaseIdFactory },
+  ) {
+    this.db = typeof db === 'string' ? new Database(db) : db;
+    this.#newId = options?.leaseId ?? defaultLeaseId;
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS sync_leases(
+        partition TEXT NOT NULL, client_id TEXT NOT NULL,
+        lease_id TEXT NOT NULL, actor_id TEXT NOT NULL,
+        allowed_scopes TEXT NOT NULL,
+        issued_at_ms INTEGER NOT NULL, expires_at_ms INTEGER NOT NULL,
+        revoked INTEGER NOT NULL DEFAULT 0,
+        PRIMARY KEY (partition, client_id)
+      );
+      CREATE TABLE IF NOT EXISTS sync_lease_revocations(
+        partition TEXT NOT NULL, lease_id TEXT NOT NULL,
+        PRIMARY KEY (partition, lease_id)
+      );
+    `);
+  }
+
+  async get(
+    partition: string,
+    clientId: string,
+  ): Promise<LeaseRecord | undefined> {
+    const row = this.db
+      .query<
+        {
+          lease_id: string;
+          actor_id: string;
+          allowed_scopes: string;
+          issued_at_ms: number;
+          expires_at_ms: number;
+          revoked: number;
+        },
+        [string, string]
+      >(
+        `SELECT lease_id, actor_id, allowed_scopes, issued_at_ms,
+                expires_at_ms, revoked
+         FROM sync_leases WHERE partition=? AND client_id=?`,
+      )
+      .get(partition, clientId);
+    if (row === null) return undefined;
+    return {
+      leaseId: row.lease_id,
+      actorId: row.actor_id,
+      allowedScopes: JSON.parse(row.allowed_scopes) as ScopeMap,
+      issuedAtMs: row.issued_at_ms,
+      expiresAtMs: row.expires_at_ms,
+      revoked: row.revoked !== 0,
+    };
+  }
+
+  #isRevoked(partition: string, leaseId: string): boolean {
+    const row = this.db
+      .query<{ n: number }, [string, string]>(
+        'SELECT 1 AS n FROM sync_lease_revocations WHERE partition=? AND lease_id=?',
+      )
+      .get(partition, leaseId);
+    return row !== null;
+  }
+
+  async issue(
+    partition: string,
+    clientId: string,
+    actorId: string,
+    allowedScopes: ScopeMap,
+    nowMs: number,
+    ttlMs: number,
+  ): Promise<LeaseRecord> {
+    const existing = await this.get(partition, clientId);
+    const reuse =
+      existing !== undefined &&
+      existing.actorId === actorId &&
+      !existing.revoked &&
+      !this.#isRevoked(partition, existing.leaseId);
+    const leaseId = reuse ? existing.leaseId : this.#newId();
+    const record: LeaseRecord = {
+      leaseId,
+      actorId,
+      allowedScopes,
+      issuedAtMs: nowMs,
+      expiresAtMs: nowMs + ttlMs,
+      revoked: false,
+    };
+    this.db
+      .query(
+        `INSERT INTO sync_leases(
+          partition, client_id, lease_id, actor_id, allowed_scopes,
+          issued_at_ms, expires_at_ms, revoked
+        ) VALUES (?,?,?,?,?,?,?,0)
+        ON CONFLICT(partition, client_id) DO UPDATE SET
+          lease_id=excluded.lease_id, actor_id=excluded.actor_id,
+          allowed_scopes=excluded.allowed_scopes,
+          issued_at_ms=excluded.issued_at_ms,
+          expires_at_ms=excluded.expires_at_ms, revoked=0`,
+      )
+      .run(
+        partition,
+        clientId,
+        leaseId,
+        actorId,
+        JSON.stringify(allowedScopes),
+        nowMs,
+        record.expiresAtMs,
+      );
+    return record;
+  }
+
+  async revoke(partition: string, leaseId: string): Promise<void> {
+    this.db
+      .query(
+        `INSERT OR IGNORE INTO sync_lease_revocations(partition, lease_id)
+         VALUES (?,?)`,
+      )
+      .run(partition, leaseId);
+    this.db
+      .query(
+        'UPDATE sync_leases SET revoked=1 WHERE partition=? AND lease_id=?',
+      )
+      .run(partition, leaseId);
+  }
+}