@pylonsync/sync 0.3.188 → 0.3.192

package/src/local-store.ts

@@ -0,0 +1,309 @@
+// ---------------------------------------------------------------------------
+// In-memory replica of server state.
+//
+// Tracks tables (entity → id → row), tombstones (delete seqs that
+// fence out late-arriving stale inserts), and optimistic tombstones
+// (pending client deletes that block incoming inserts until the
+// server's authoritative delete arrives). Persistence backends
+// (IndexedDB) wire through `_persistFn`.
+// ---------------------------------------------------------------------------
+
+import type { ChangeEvent, Row } from "./types";
+
+export class LocalStore {
+  private tables: Map<string, Map<string, Row>> = new Map();
+  /**
+   * `(entity, row_id) → deletedAt seq`. A row in this map has been
+   * deleted; any insert/update event older than its tombstone seq is
+   * ignored so an out-of-order replay can't resurrect it. Without
+   * this, a delete followed by a reconnect-driven replay of the
+   * original insert would re-materialize the row.
+   *
+   * Real server-issued tombstones use the event's own seq. Optimistic
+   * client tombstones live in `optimisticTombstones` instead.
+   */
+  private tombstones: Map<string, Map<string, number>> = new Map();
+  /**
+   * Pending optimistic deletes — `(entity, row_id)` pairs the client
+   * has dropped but the server hasn't yet confirmed. Stored
+   * separately from `tombstones` so the real server-issued delete
+   * (with its real, smaller seq) can supersede the optimistic entry
+   * without being max-merged out.
+   *
+   * Invariant: a future legitimate re-create of the same id must
+   * succeed once the server's authoritative delete arrives. Test:
+   * `optimistic_delete_releases_id_when_server_confirms`.
+   */
+  private optimisticTombstones: Map<string, Set<string>> = new Map();
+  private listeners: Set<() => void> = new Set();
+
+  /** Get all rows for an entity. */
+  list(entity: string): Row[] {
+    const table = this.tables.get(entity);
+    if (!table) return [];
+    return Array.from(table.values());
+  }
+
+  /** Get a row by ID. */
+  get(entity: string, id: string): Row | null {
+    return this.tables.get(entity)?.get(id) ?? null;
+  }
+
+  /** Snapshot of every entity name with at least one local row. Used
+   *  by `SyncEngine.reconcile` to know which tables to diff against
+   *  server truth. */
+  entityNames(): string[] {
+    const names: string[] = [];
+    for (const [name, table] of this.tables) {
+      if (table.size > 0) names.push(name);
+    }
+    return names;
+  }
+
+  /**
+   * Remove a row whose absence was confirmed by the server-truth
+   * reconciler. Records a tombstone at `tombstoneSeq` so a stale
+   * insert/update replayed afterwards (e.g. a slow WS frame) can't
+   * resurrect it. Callers pass the current sync cursor — any future
+   * change events have higher seqs and pass the tombstone check,
+   * while older replays are filtered.
+   *
+   * Differs from `optimisticDelete`: this is "the server says this
+   * row is gone right now"; that one is "the client wants this row
+   * gone before the server has confirmed."
+   */
+  reconcileRemove(entity: string, id: string, tombstoneSeq: number): boolean {
+    const table = this.tables.get(entity);
+    if (!table || !table.has(id)) return false;
+    table.delete(id);
+    this.recordTombstone(entity, id, tombstoneSeq);
+    return true;
+  }
+
+  private isTombstoned(entity: string, id: string, at_seq?: number): boolean {
+    if (this.optimisticTombstones.get(entity)?.has(id)) return true;
+    const tombSeq = this.tombstones.get(entity)?.get(id);
+    if (tombSeq === undefined) return false;
+    // No-seq caller means "this change has no provenance" — treat as
+    // older than any tombstone (safer default than admitting it).
+    if (at_seq === undefined) return true;
+    return at_seq < tombSeq;
+  }
+
+  private recordTombstone(entity: string, id: string, seq: number): void {
+    // A real (server-issued) tombstone supersedes any pending
+    // optimistic entry for this id. Drop the optimistic guard so a
+    // future legitimate re-create of the same id isn't blocked.
+    this.optimisticTombstones.get(entity)?.delete(id);
+    if (!this.tombstones.has(entity)) {
+      this.tombstones.set(entity, new Map());
+    }
+    const existing = this.tombstones.get(entity)!.get(id);
+    if (existing === undefined || seq > existing) {
+      this.tombstones.get(entity)!.set(id, seq);
+    }
+  }
+
+  /** Apply a change event to the local store. */
+  applyChange(change: ChangeEvent): void {
+    if (!this.tables.has(change.entity)) {
+      this.tables.set(change.entity, new Map());
+    }
+    const table = this.tables.get(change.entity)!;
+
+    // Drop insert/update events that arrive AFTER a delete for the
+    // same row. The tombstone map records the seq of the delete;
+    // anything strictly older is a stale resurrect.
+    if (
+      (change.kind === "insert" || change.kind === "update") &&
+      this.isTombstoned(change.entity, change.row_id, change.seq)
+    ) {
+      return;
+    }
+
+    switch (change.kind) {
+      case "insert":
+        if (change.data) {
+          // Spread data FIRST, then force `id = change.row_id` —
+          // otherwise an `id` field in `data` could shadow the
+          // authoritative row_id and corrupt the replica's primary key
+          // on reload.
+          table.set(change.row_id, {
+            ...change.data,
+            id: change.row_id,
+          });
+        }
+        break;
+      case "update":
+        if (change.data) {
+          const existing = table.get(change.row_id) ?? { id: change.row_id };
+          table.set(change.row_id, {
+            ...existing,
+            ...change.data,
+            id: change.row_id,
+          });
+        }
+        break;
+      case "delete":
+        table.delete(change.row_id);
+        this.recordTombstone(change.entity, change.row_id, change.seq);
+        break;
+    }
+  }
+
+  /** Apply multiple changes synchronously. Persistence runs fire-
+   *  and-forget. Prefer `applyChangesAsync` when you plan to advance
+   *  a cursor after — otherwise a crash can save the cursor before
+   *  rows hit disk, causing permanent missed changes on restart. */
+  applyChanges(changes: ChangeEvent[]): void {
+    for (const change of changes) {
+      this.applyChange(change);
+    }
+    this.notify();
+
+    if (this._persistFn) {
+      for (const change of changes) {
+        const merged = this.hydrateFromMemory(change);
+        void this._persistFn(merged);
+      }
+    }
+  }
+
+  /**
+   * Apply + persist, awaiting disk writes before returning. Callers
+   * that are about to advance a cursor based on `changes` MUST use
+   * this path — otherwise cursor durability is broken: a crash
+   * between the memory apply and the eventual disk write can persist
+   * a cursor that's ahead of the replica, skipping those rows
+   * forever on restart.
+   */
+  async applyChangesAsync(changes: ChangeEvent[]): Promise<void> {
+    for (const change of changes) {
+      this.applyChange(change);
+    }
+    this.notify();
+    if (this._persistFn) {
+      // Sequential await — concurrent IDB writes can resolve out of
+      // order, racing an update behind its own delete on disk. The
+      // engine's `applyQueue` sequences events into here; we
+      // preserve that order down to the disk layer.
+      for (const change of changes) {
+        const result = this._persistFn(this.hydrateFromMemory(change));
+        if (result instanceof Promise) {
+          await result;
+        }
+      }
+    }
+  }
+
+  /**
+   * Reshape a change event so its `data` field matches the row as it
+   * now exists in memory after `applyChange` merged the patch.
+   * Persistence callers (IndexedDB) save the full row, which only
+   * works if they receive the full row. Deletes pass through
+   * untouched.
+   */
+  private hydrateFromMemory(change: ChangeEvent): ChangeEvent {
+    if (change.kind === "delete") return change;
+    const merged = this.tables.get(change.entity)?.get(change.row_id);
+    if (!merged) return change;
+    return { ...change, data: merged };
+  }
+
+  /** Persistence callback for auto-saving changes. Returns
+   *  `Promise<void>` so callers can await. Void-returning callbacks
+   *  are accepted for backwards compatibility (just not awaitable). */
+  _persistFn: ((change: ChangeEvent) => void | Promise<void>) | null = null;
+
+  /** Subscribe to store changes. Returns unsubscribe function. */
+  subscribe(listener: () => void): () => void {
+    this.listeners.add(listener);
+    return () => this.listeners.delete(listener);
+  }
+
+  notify(): void {
+    for (const listener of this.listeners) {
+      listener();
+    }
+  }
+
+  /** Apply an optimistic insert. Returns a temporary id. */
+  optimisticInsert(entity: string, data: Row): string {
+    const tempId = `_pending_${Date.now()}_${Math.random().toString(36).slice(2)}`;
+    if (!this.tables.has(entity)) {
+      this.tables.set(entity, new Map());
+    }
+    this.tables.get(entity)!.set(tempId, { id: tempId, ...data });
+    this.notify();
+    return tempId;
+  }
+
+  /**
+   * Apply an optimistic insert with a caller-provided id.
+   *
+   * Used by `useMutation({ optimistic })`: the React hook generates a
+   * Pylon-shaped id (40-char hex via `generateId()`), threads it
+   * through the mutation args as `_optimisticId`, and the server
+   * function honors it on `ctx.db.insert("Entity", { id, ... })`.
+   * Because the optimistic ghost and the canonical row share the
+   * same row_id, the WS broadcast that follows lands as a field-
+   * level merge on top of the optimistic — no delete-then-replace
+   * flash, no temp-row swap.
+   */
+  optimisticInsertWithId(entity: string, id: string, data: Row): void {
+    if (!this.tables.has(entity)) {
+      this.tables.set(entity, new Map());
+    }
+    this.tables.get(entity)!.set(id, { ...data, id });
+    this.notify();
+  }
+
+  /**
+   * Roll back an optimistic insert without leaving a tombstone.
+   *
+   * Counterpart to `optimisticInsertWithId`. On rejection the ghost
+   * row must go away, but we MUST NOT leave a tombstone — a future
+   * legitimate insert with the same id (user retries, workflow
+   * eventually creates the row) must not be blocked.
+   * `optimisticDelete` records a MAX_SAFE_INTEGER tombstone, which is
+   * the wrong semantic here; this is a plain remove.
+   */
+  rollbackOptimisticInsert(entity: string, id: string): void {
+    const removed = this.tables.get(entity)?.delete(id);
+    if (removed) this.notify();
+  }
+
+  /** Apply an optimistic update. */
+  optimisticUpdate(entity: string, id: string, data: Partial<Row>): void {
+    const table = this.tables.get(entity);
+    if (!table) return;
+    const existing = table.get(id);
+    if (existing) {
+      table.set(id, { ...existing, ...data });
+      this.notify();
+    }
+  }
+
+  /** Apply an optimistic delete. Block any incoming insert/update
+   *  for this id until the server's authoritative delete arrives. */
+  optimisticDelete(entity: string, id: string): void {
+    this.tables.get(entity)?.delete(id);
+    if (!this.optimisticTombstones.has(entity)) {
+      this.optimisticTombstones.set(entity, new Set());
+    }
+    this.optimisticTombstones.get(entity)!.add(id);
+    this.notify();
+  }
+
+  /**
+   * Drop every table + tombstone in-place, then notify. Used by the
+   * sync engine's `resetReplica()` on identity flip (token or tenant
+   * changed — the old replica reflects a different visible set).
+   */
+  clearAll(): void {
+    this.tables.clear();
+    this.tombstones.clear();
+    this.optimisticTombstones.clear();
+    this.notify();
+  }
+}

package/src/mutation-queue.ts

@@ -0,0 +1,153 @@
+// ---------------------------------------------------------------------------
+// Offline-safe write queue.
+//
+// Memory-only by default. When a persistence backend is attached
+// (`attachPersistence`), every state transition writes through to
+// disk and `hydrate()` restores pending/failed mutations on startup.
+//
+// The mutation id is reused as the server-side `op_id` for idempotent
+// replay — a retry carrying the same id short-circuits on the server
+// instead of re-applying.
+// ---------------------------------------------------------------------------
+
+import type { ClientChange } from "./types";
+
+export interface PendingMutation {
+  id: string;
+  change: ClientChange;
+  status: "pending" | "applied" | "failed";
+  error?: string;
+}
+
+/**
+ * Optional persistence backend. The default IndexedDB persistence
+ * layer provides `savePending`/`loadPending`. Callers can supply a
+ * custom backend for tests or alternative storage.
+ */
+export interface MutationQueuePersistence {
+  saveAll(mutations: PendingMutation[]): Promise<void>;
+  loadAll(): Promise<PendingMutation[]>;
+}
+
+export class MutationQueue {
+  private queue: PendingMutation[] = [];
+  private persistence?: MutationQueuePersistence;
+
+  constructor(persistence?: MutationQueuePersistence) {
+    this.persistence = persistence;
+  }
+
+  /**
+   * Attach a persistence backend after construction. The SyncEngine
+   * swaps in IndexedDB-backed persistence once the DB has opened.
+   * Public so it doesn't need a `// @ts-expect-error` to reach in
+   * from the same package.
+   */
+  attachPersistence(persistence: MutationQueuePersistence): void {
+    this.persistence = persistence;
+  }
+
+  /** Load persisted queue state. Call once at startup. */
+  async hydrate(): Promise<void> {
+    if (!this.persistence) return;
+    try {
+      const loaded = await this.persistence.loadAll();
+      // Merge in-memory with on-disk. An `add()` that ran while
+      // hydrate was awaiting `loadAll()` will already have flushed a
+      // snapshot that didn't include the loaded rows — re-flush
+      // after merge so disk matches memory again.
+      const existingIds = new Set(this.queue.map((m) => m.id));
+      let mergedAny = false;
+      for (const m of loaded) {
+        if (!existingIds.has(m.id)) {
+          this.queue.push(m);
+          mergedAny = true;
+        }
+      }
+      if (mergedAny) this.flush();
+    } catch (err) {
+      // Broken storage shouldn't prevent the app from running — warn
+      // and degrade to memory-only mode.
+      console.warn("[sync] mutation-queue hydrate failed:", err);
+    }
+  }
+
+  /** Add a pending mutation. Returns the op_id used for server
+   *  idempotency. */
+  add(change: ClientChange): string {
+    const id = `mut_${Date.now()}_${Math.random().toString(36).slice(2)}`;
+    const changeWithOp: ClientChange = { ...change, op_id: id };
+    this.queue.push({ id, change: changeWithOp, status: "pending" });
+    this.flush();
+    return id;
+  }
+
+  pending(): PendingMutation[] {
+    return this.queue.filter((m) => m.status === "pending");
+  }
+
+  /**
+   * Set of `${entity}/${row_id}` keys for every mutation currently
+   * in Pending or Failed state. Used by reconcile() to skip rows
+   * whose canonical state on the server hasn't caught up with the
+   * local optimistic ghost yet — otherwise reconcile would tombstone
+   * the row (it's not yet on the server) and the still-pending push
+   * would later re-apply against the tombstone, fighting the local
+   * replica.
+   *
+   * Failed mutations are included too: a user-visible failure is
+   * recoverable, and sweeping the row would discard the local edit
+   * the user is still trying to push.
+   */
+  pendingRowKeys(): Set<string> {
+    const out = new Set<string>();
+    for (const m of this.queue) {
+      if (m.status === "pending" || m.status === "failed") {
+        out.add(`${m.change.entity}/${m.change.row_id}`);
+      }
+    }
+    return out;
+  }
+
+  markApplied(id: string): void {
+    const m = this.queue.find((m) => m.id === id);
+    if (m) m.status = "applied";
+    this.flush();
+  }
+
+  markFailed(id: string, error: string): void {
+    const m = this.queue.find((m) => m.id === id);
+    if (m) {
+      m.status = "failed";
+      m.error = error;
+    }
+    this.flush();
+  }
+
+  /**
+   * Prune applied mutations. Failed mutations are KEPT so the UI can
+   * surface them to the user and so retries are possible.
+   */
+  clear(): void {
+    this.queue = this.queue.filter(
+      (m) => m.status === "pending" || m.status === "failed",
+    );
+    this.flush();
+  }
+
+  /** Remove a specific mutation by id. Used by the UI after user
+   *  ack of failures. */
+  remove(id: string): void {
+    this.queue = this.queue.filter((m) => m.id !== id);
+    this.flush();
+  }
+
+  /** Fire-and-forget persistence write. */
+  private flush(): void {
+    if (!this.persistence) return;
+    const snapshot = this.queue.slice();
+    this.persistence.saveAll(snapshot).catch((err) => {
+      console.warn("[sync] mutation-queue persist failed:", err);
+    });
+  }
+}

package/src/transport.ts

@@ -0,0 +1,205 @@
+// ---------------------------------------------------------------------------
+// Shared client HTTP transport.
+//
+// One helper that every browser-side Pylon call funnels through:
+// `pylonFetch(config, path, init)`. Consistently handles:
+//
+//  - base URL resolution (config.baseUrl, falling back to
+//    window.location.origin on the browser when omitted)
+//  - bearer token attachment (config.token or token-getter callback)
+//  - `credentials: "include"` for cookie-auth apps
+//  - request-body JSON serialization (when `init.json` is set)
+//  - response JSON parsing with structured error envelope:
+//      { error: { code, message } }
+//  - non-JSON error bodies (HTML proxy pages, empty 204s) → synthetic
+//    Error with status + code
+//  - X-Pylon-Change-Seq response header surfaced via the optional
+//    `onChangeSeq` callback (so callers can trigger catch-up pulls
+//    when a mutation's seq exceeds their local cursor)
+//
+// Streaming + raw-binary callers (file uploads, SSE) bypass the
+// JSON-parse step but reuse the URL/auth/credentials logic via
+// `buildRequest`.
+// ---------------------------------------------------------------------------
+
+/**
+ * What every caller of `pylonFetch` must supply. The SyncEngine
+ * passes its own config; the React free helpers pass a lightweight
+ * shim built from `getBaseUrl()` + `currentAuthToken()`.
+ */
+export interface TransportConfig {
+  baseUrl?: string;
+  /** Static bearer token. Use `getToken` if the token can change. */
+  token?: string;
+  /** Lazy bearer-token resolver — called per request. Use this when
+   *  the token can be rotated mid-session (refresh, session-changed).
+   *  Returning null/undefined means "no auth header"; cookie-auth
+   *  apps rely on `credentials: "include"` instead. */
+  getToken?: () => string | null | undefined;
+  /** Invoked with the value of the X-Pylon-Change-Seq response
+   *  header when the server sets one. Returning a Promise pauses no
+   *  one — the transport doesn't await it. */
+  onChangeSeq?: (seq: number) => void;
+}
+
+/**
+ * Per-call init shape. Mirrors the standard `RequestInit` but
+ * normalizes the two body shapes (JSON-encode-this vs raw-pass-
+ * through) into separate fields so callers don't have to remember
+ * to set Content-Type.
+ */
+export interface PylonRequestInit {
+  method?: "GET" | "POST" | "PUT" | "PATCH" | "DELETE";
+  /** When set, JSON-stringified into the body and Content-Type
+   *  defaults to application/json. */
+  json?: unknown;
+  /** Raw body — passed through to fetch unchanged. Use this for file
+   *  uploads (Blob, ArrayBuffer, FormData). */
+  body?: BodyInit;
+  /** Extra headers. Caller-supplied keys win against the transport's
+   *  defaults. */
+  headers?: Record<string, string>;
+  /** Override the request's Accept header (useful for SSE streams). */
+  accept?: string;
+  /** AbortController signal for cancellation. */
+  signal?: AbortSignal;
+}
+
+/**
+ * Resolve the base URL. Browser-side fallback to
+ * `window.location.origin` matches the SDK contract: when
+ * `configureClient` was called with no baseUrl, calls go to the
+ * current document's origin so dev + prod work without env
+ * gymnastics.
+ */
+export function resolveBaseUrl(config: TransportConfig): string {
+  if (config.baseUrl) return config.baseUrl.replace(/\/+$/, "");
+  if (typeof window !== "undefined" && window.location?.origin) {
+    return window.location.origin;
+  }
+  return "";
+}
+
+/**
+ * Assemble fetch URL + RequestInit. Exposed for streaming / raw-
+ * response callers (SSE, file upload) that need to handle the
+ * Response themselves but want the transport's auth + credentials +
+ * URL logic.
+ */
+export function buildRequest(
+  config: TransportConfig,
+  path: string,
+  init: PylonRequestInit = {},
+): { url: string; init: RequestInit } {
+  const base = resolveBaseUrl(config);
+  const url = `${base}${path}`;
+  const headers: Record<string, string> = { ...(init.headers ?? {}) };
+  // Auth: explicit getToken wins over static token. Either may be
+  // null/undefined — cookie-auth apps rely solely on credentials.
+  const token = config.getToken?.() ?? config.token;
+  if (token) headers["Authorization"] = `Bearer ${token}`;
+  if (init.accept) headers["Accept"] = init.accept;
+  let body: BodyInit | undefined = init.body;
+  if (init.json !== undefined) {
+    if (!headers["Content-Type"]) headers["Content-Type"] = "application/json";
+    body = JSON.stringify(init.json);
+  }
+  return {
+    url,
+    init: {
+      method: init.method ?? (body !== undefined ? "POST" : "GET"),
+      headers,
+      // `credentials: "include"` is mandatory for cookie-auth apps.
+      // Bearer-auth apps don't care (no cookie to send) so the cost
+      // is the same. Browser CORS rules require the server to set
+      // `Access-Control-Allow-Credentials: true` and a specific
+      // origin (not `*`) for the cookie to be sent — Pylon's CORS
+      // middleware does this by default.
+      credentials: "include",
+      body,
+      signal: init.signal,
+    },
+  };
+}
+
+/** Error thrown by `pylonFetch` for non-2xx responses. Carries the
+ *  status + structured error code + the parsed JSON body (if any). */
+export class PylonHttpError extends Error {
+  status: number;
+  code?: string;
+  body?: unknown;
+  constructor(message: string, status: number, code?: string, body?: unknown) {
+    super(message);
+    this.name = "PylonHttpError";
+    this.status = status;
+    this.code = code;
+    this.body = body;
+  }
+}
+
+/**
+ * The canonical client HTTP call. Returns the parsed JSON body on
+ * 2xx. Throws `PylonHttpError` on non-2xx. Empty 204 bodies parse
+ * as `null`.
+ *
+ * Surfaces `X-Pylon-Change-Seq` via `config.onChangeSeq` so the
+ * SyncEngine can fire a catch-up pull when a mutation's seq is
+ * ahead of its local cursor — matches the
+ * `requestWithChangeSync` shape but lifts the logic out of the
+ * engine so React free helpers (`callFn`, `apiRequest`) get it too.
+ */
+export async function pylonFetch<T = unknown>(
+  config: TransportConfig,
+  path: string,
+  init: PylonRequestInit = {},
+): Promise<T> {
+  const { url, init: req } = buildRequest(config, path, init);
+  const res = await fetch(url, req);
+  const text = await res.text();
+  let parsed: unknown = null;
+  if (text) {
+    try {
+      parsed = JSON.parse(text);
+    } catch {
+      // Non-JSON body — proxy HTML error page, etc. Fall through;
+      // the !res.ok branch synthesizes a useful error.
+    }
+  }
+  // Surface change-seq for catch-up logic. Best-effort: a malformed
+  // header doesn't fail the request.
+  if (config.onChangeSeq) {
+    const header = res.headers.get("x-pylon-change-seq");
+    if (header) {
+      const seq = Number.parseInt(header, 10);
+      if (Number.isFinite(seq) && seq > 0) {
+        config.onChangeSeq(seq);
+      }
+    }
+  }
+  if (!res.ok) {
+    const err = parsed as { error?: { code?: string; message?: string } } | null;
+    const message =
+      err?.error?.message ??
+      `${req.method ?? "GET"} ${path} failed: ${res.status}`;
+    throw new PylonHttpError(message, res.status, err?.error?.code, parsed);
+  }
+  return parsed as T;
+}
+
+/**
+ * Streaming variant — returns the raw `Response` so callers can read
+ * `.body` (SSE), `.blob()` (file download), etc. Bypasses JSON
+ * parsing but keeps the URL + auth + credentials logic.
+ *
+ * Caller is responsible for status checking and the
+ * X-Pylon-Change-Seq callback (we don't read the response without
+ * the caller's involvement).
+ */
+export async function pylonFetchRaw(
+  config: TransportConfig,
+  path: string,
+  init: PylonRequestInit = {},
+): Promise<Response> {
+  const { url, init: req } = buildRequest(config, path, init);
+  return fetch(url, req);
+}