@syncular/server 0.1.2 → 0.2.0

package/src/index.ts

@@ -1,36 +1,53 @@
 /**
- * @syncular/server - Server-side sync infrastructure
+ * @syncular/server — framework-free embeddable SSP2 protocol library
+ * (SPEC.md is normative; REVISE.md B2 is the architectural mandate).
  *
- * Commit-log based sync with:
- * - commit log + change log
- * - scopes + subscriptions (partial sync + auth)
- * - commit-level idempotency
- * - blob/media storage
+ * Core surface: `handleSyncRequest(bytes, ctx) → bytes` over host-provided
+ * storage / scope-resolution / segment-store interfaces, plus a
+ * transport-agnostic realtime session (§8), the direct segment download
+ * handler (§5.5), and signed-URL token issuance/verification (§5.4).
  */
-export * from '@syncular/core';
-
-export * from './auth-leases';
-export * from './blobs';
-export * from './clients';
-export * from './commit-integrity';
-export * from './compaction';
-export * from './dialect';
-export * from './encrypted-crdt';
+export * from './admin';
+export * from './blob-handlers';
+export * from './blob-store';
+export * from './content-encoding';
+export * from './context';
+export * from './crdt-merger';
+export * from './d1-storage';
 export * from './errors';
-export * from './handlers';
-export * from './helpers';
-export * from './migrate';
-export * from './notify';
-export * from './plugins';
-export * from './proxy';
+export * from './events';
+export * from './events-ring';
+export * from './frame-bytes';
+export * from './handler';
+export * from './lease-store';
+// The `PgExecutor` seam + Postgres storage/fanout are driver-agnostic (zero
+// runtime deps). Concrete driver adapters (pglite for tests; Bun.sql /
+// node-postgres for production, documented in the README) live in separate
+// entry points so the barrel never imports a driver.
+export * from './pg-executor';
+export * from './postgres-fanout';
+export * from './postgres-storage';
 export * from './prune';
 export * from './pull';
 export * from './push';
 export * from './realtime';
+export * from './relational-rows';
+export * from './s3-blob-store';
+export * from './s3-segment-store';
 export * from './schema';
-export * from './snapshot-artifacts';
-export * from './snapshot-chunks';
-export type { SnapshotChunkStorage } from './snapshot-chunks/types';
-export * from './stats';
-export * from './subscriptions';
-export * from './sync';
+export * from './scopes';
+export * from './segment-download';
+export * from './segment-store';
+export * from './signed-url';
+// Bun-specific storages (top-level `bun:sqlite`): re-exported for Bun/Node
+// hosts. Workers/edge builds tree-shake them (and their `bun:sqlite` import)
+// away — the runtime-neutral core closure is enforced by
+// `test/runtime-neutrality.test.ts`.
+export * from './sqlite-blob-store';
+export * from './sqlite-dialect';
+export * from './sqlite-image';
+export * from './sqlite-lease-store';
+export * from './sqlite-segment-store';
+export * from './sqlite-storage';
+export * from './storage';
+export * from './validate';

package/src/pg-executor-pglite.ts

@@ -0,0 +1,63 @@
+/**
+ * A `PgExecutor` over @electric-sql/pglite (embedded WASM Postgres).
+ *
+ * This is the **test/dev** driver — hermetic, no docker, runs under bun.
+ * `@electric-sql/pglite` is a devDependency of `@syncular/server`; this
+ * module is exported for tests and local experiments, not the production
+ * path. Production wires Bun.sql or node-postgres against the same
+ * `PgExecutor` interface (see the server README).
+ *
+ * pglite is single-connection, so `transaction` runs `BEGIN`/`COMMIT`/
+ * `ROLLBACK` on the one connection and relies on the caller not interleaving
+ * unrelated queries during a transaction scope (the storage layer holds one
+ * transaction at a time per push).
+ */
+import type { PGlite } from '@electric-sql/pglite';
+import type { PgExecutor, PgQueryable, PgRow } from './pg-executor';
+
+/** Minimal structural type for a pglite instance (avoids a hard type dep). */
+interface PgliteLike {
+  query<Row>(
+    query: string,
+    params?: unknown[],
+  ): Promise<{ rows: Row[]; affectedRows?: number }>;
+  exec(query: string): Promise<unknown>;
+  close(): Promise<void>;
+}
+
+function queryable(db: PgliteLike): PgQueryable {
+  return {
+    async query<Row = PgRow>(text: string, params?: readonly unknown[]) {
+      const result = await db.query<Row>(
+        text,
+        params ? [...params] : undefined,
+      );
+      return {
+        rows: result.rows,
+        rowCount: result.affectedRows ?? result.rows.length,
+      };
+    },
+  };
+}
+
+export function pgliteExecutor(db: PGlite | PgliteLike): PgExecutor {
+  const like = db as PgliteLike;
+  const q = queryable(like);
+  return {
+    query: q.query,
+    async transaction<T>(fn: (client: PgQueryable) => Promise<T>): Promise<T> {
+      await like.exec('BEGIN');
+      try {
+        const result = await fn(q);
+        await like.exec('COMMIT');
+        return result;
+      } catch (error) {
+        await like.exec('ROLLBACK');
+        throw error;
+      }
+    },
+    async close() {
+      await like.close();
+    },
+  };
+}

package/src/pg-executor.ts

@@ -0,0 +1,87 @@
+/**
+ * The minimal Postgres executor seam (REVISE B2: zero runtime deps).
+ *
+ * `PostgresServerStorage` is written entirely against this interface so the
+ * server library never imports a specific driver. Production wires one of:
+ *
+ *   - **Bun.sql** (built into bun) — see `bunSqlExecutor` in the README;
+ *   - **node-postgres** (`pg`) — a `Pool`/`PoolClient` adapter;
+ *
+ * and the test lane wires **@electric-sql/pglite** (embedded WASM Postgres,
+ * a devDependency) via `pgliteExecutor`. All three speak the same three
+ * primitives below.
+ *
+ * ## Type-parser contract (load-bearing)
+ *
+ * `commitSeq`/`serverVersion` are `int8`/`bigint` columns. Drivers disagree
+ * on how they decode `int8`:
+ *
+ *   - pglite → JS `number`;
+ *   - node-postgres → `string` (its default `int8` parser is off to avoid
+ *     silent precision loss past 2^53);
+ *   - Bun.sql → `bigint`.
+ *
+ * The storage layer therefore coerces every sequence value it reads through
+ * `Number(...)` (see `asNumber`). Sequences here are per-partition commit
+ * counters and server_version counters — both comfortably inside 2^53 for
+ * any realistic deployment, so the coercion is safe; a host that expects to
+ * exceed 2^53 commits in a single partition has bigger problems than this
+ * seam. `bytea` columns MUST decode to `Uint8Array`/`Buffer` (all three
+ * drivers do); `NULL` decodes to JS `null`.
+ */
+
+/** One database row as a plain object keyed by (lower-cased) column name. */
+export type PgRow = Record<string, unknown>;
+
+/**
+ * A handle that runs parameterized statements. Both the pool-level executor
+ * and a per-transaction client implement it, so storage code is written once
+ * against `PgQueryable` regardless of transaction scope.
+ *
+ * `params` use Postgres positional placeholders (`$1`, `$2`, ...). `text`
+ * is a single statement (no multi-statement `;`-joined batches — those are
+ * driver-specific and defeat parameterization).
+ */
+export interface PgQueryable {
+  query<Row = PgRow>(
+    text: string,
+    params?: readonly unknown[],
+  ): Promise<{ rows: Row[]; rowCount: number }>;
+}
+
+/**
+ * The pool-level executor: `PgQueryable` plus transaction control. A
+ * `transaction` callback receives a client pinned to one connection for the
+ * duration of a real Postgres `BEGIN … COMMIT/ROLLBACK`; throwing rolls back.
+ */
+export interface PgExecutor extends PgQueryable {
+  /**
+   * Run `fn` inside a single Postgres transaction on one pinned connection.
+   * The executor issues `BEGIN` before `fn`, `COMMIT` on resolve, and
+   * `ROLLBACK` on throw (rethrowing the original error).
+   */
+  transaction<T>(fn: (client: PgQueryable) => Promise<T>): Promise<T>;
+  /** Release any pooled resources. Optional (pglite/Bun.sql). */
+  close?(): Promise<void>;
+}
+
+/** Coerce a driver-decoded sequence value (`number | string | bigint`). */
+export function asNumber(value: unknown): number {
+  if (typeof value === 'number') return value;
+  if (typeof value === 'bigint') return Number(value);
+  if (typeof value === 'string') return Number.parseInt(value, 10);
+  if (value === null || value === undefined) return 0;
+  throw new Error(`expected a numeric column, got ${typeof value}`);
+}
+
+/** Coerce a driver-decoded `bytea` value to a `Uint8Array`. */
+export function asBytes(value: unknown): Uint8Array {
+  if (value instanceof Uint8Array) return value;
+  if (value === null || value === undefined) {
+    throw new Error('expected bytea, got null');
+  }
+  // node-postgres/Bun.sql return Buffer (a Uint8Array subclass); this branch
+  // is a defensive copy for any array-like the driver hands back.
+  if (Array.isArray(value)) return new Uint8Array(value);
+  throw new Error(`expected bytea, got ${typeof value}`);
+}

package/src/postgres-fanout.ts

@@ -0,0 +1,143 @@
+/**
+ * LISTEN/NOTIFY multi-instance fanout (TODO §4.1) — the primitive that lets
+ * N server instances behind a load balancer wake each other's realtime
+ * sessions when a commit lands on any one of them.
+ *
+ * ## The multi-instance problem
+ *
+ * A commit applied on instance A fans out to A's *local* RealtimeHub
+ * sessions immediately (in-memory `notifyCommit`, full delta, no re-read).
+ * But a client whose socket lives on instance B never saw it. LISTEN/NOTIFY
+ * closes that gap:
+ *
+ *   1. after a commit lands, the originating instance calls `notify(payload)`
+ *      → `NOTIFY syncular_commit, '<partition>:<commitSeq>'`;
+ *   2. every instance runs a `listen()` loop on a dedicated connection;
+ *      on a notification it parses the payload and calls
+ *      `hub.wake(partition, 'sync')`.
+ *
+ * ## Why wake, not re-broadcast the delta
+ *
+ * Postgres NOTIFY payloads are capped (~8000 bytes) and are not an ordered,
+ * lossless delta channel — so we do NOT try to ship commit bytes through
+ * them. Remote sessions receive a `sync` wake and pull the delta from the
+ * shared Postgres storage they already read from (§8.3 wake semantics). The
+ * originating instance keeps the in-memory fast path (its own sessions get
+ * the full delta with no extra read); only *cross-instance* delivery pays
+ * the re-pull. Single-instance deployments never install a fanout at all.
+ *
+ * The payload carries `commitSeq` purely for observability/ordering in logs;
+ * the wake itself is seq-agnostic (a session pulls whatever it is behind).
+ *
+ * pglite cannot exercise cross-connection NOTIFY, so the integration
+ * (`postgres-fanout.integration.test.ts`) is env-gated on `SYNCULAR_PG_URL`
+ * and skips cleanly; the payload encode/parse is unit-tested hermetically.
+ */
+import type { WakeReason } from '@syncular/core';
+
+/** The channel every syncular instance LISTENs on. */
+export const FANOUT_CHANNEL = 'syncular_commit';
+
+/**
+ * The wake reason remote instances raise on a fanout notification (§8.3):
+ * `catchup-required` — a commit landed on another instance, so the local
+ * session pulls the delta it is now behind on from shared storage.
+ */
+const FANOUT_WAKE_REASON: WakeReason = 'catchup-required';
+
+export interface FanoutPayload {
+  readonly partition: string;
+  readonly commitSeq: number;
+}
+
+/**
+ * Encode a fanout payload as `<partition>:<commitSeq>`. The partition is
+ * base64url-encoded so a partition string containing `:` cannot corrupt the
+ * frame; commitSeq is a plain decimal integer.
+ */
+export function encodeFanoutPayload(payload: FanoutPayload): string {
+  const partition = Buffer.from(payload.partition, 'utf8').toString(
+    'base64url',
+  );
+  return `${partition}:${payload.commitSeq}`;
+}
+
+/** Parse a fanout payload; returns `undefined` for a malformed frame. */
+export function parseFanoutPayload(text: string): FanoutPayload | undefined {
+  const sep = text.indexOf(':');
+  if (sep <= 0) return undefined;
+  const partitionB64 = text.slice(0, sep);
+  const seqText = text.slice(sep + 1);
+  const commitSeq = Number.parseInt(seqText, 10);
+  if (!Number.isFinite(commitSeq) || commitSeq < 0) return undefined;
+  if (!/^\d+$/.test(seqText)) return undefined;
+  let partition: string;
+  try {
+    partition = Buffer.from(partitionB64, 'base64url').toString('utf8');
+  } catch {
+    return undefined;
+  }
+  if (partition.length === 0) return undefined;
+  return { partition, commitSeq };
+}
+
+/** The hub surface the fanout wakes (a `RealtimeHub` satisfies this). */
+export interface FanoutWakeTarget {
+  wake(partition: string, reason: WakeReason): void;
+}
+
+/**
+ * A raw notification connection. Production wires a driver's LISTEN support:
+ *   - node-postgres: a dedicated `Client` with `client.on('notification', …)`
+ *     after `LISTEN syncular_commit`;
+ *   - Bun.sql: `sql.listen(channel, handler)`.
+ * The `notify` side can reuse the main pool (`SELECT pg_notify($1,$2)`), but
+ * LISTEN needs its own long-lived connection.
+ */
+export interface PgNotificationConnection {
+  /** Register a payload handler for the channel and begin listening. */
+  listen(
+    channel: string,
+    handler: (payload: string) => void,
+  ): Promise<void> | void;
+  /** Send a notification on the channel. */
+  notify(channel: string, payload: string): Promise<void>;
+  close?(): Promise<void>;
+}
+
+/**
+ * The fanout primitive. `install(hub)` starts the LISTEN loop that wakes the
+ * hub on every remote commit; `notifyCommit` is called after a local commit
+ * lands to fan it to the other instances.
+ */
+export class PostgresFanout {
+  readonly #conn: PgNotificationConnection;
+  #installed = false;
+
+  constructor(conn: PgNotificationConnection) {
+    this.#conn = conn;
+  }
+
+  /** Start the LISTEN loop, waking `hub` on every notification received. */
+  async install(hub: FanoutWakeTarget): Promise<void> {
+    if (this.#installed) throw new Error('fanout already installed');
+    this.#installed = true;
+    await this.#conn.listen(FANOUT_CHANNEL, (payload) => {
+      const parsed = parseFanoutPayload(payload);
+      if (parsed === undefined) return; // ignore malformed frames
+      hub.wake(parsed.partition, FANOUT_WAKE_REASON);
+    });
+  }
+
+  /** Fan an applied commit out to the other instances. */
+  async notifyCommit(partition: string, commitSeq: number): Promise<void> {
+    await this.#conn.notify(
+      FANOUT_CHANNEL,
+      encodeFanoutPayload({ partition, commitSeq }),
+    );
+  }
+
+  async close(): Promise<void> {
+    await this.#conn.close?.();
+  }
+}