@rindle/normalized 0.1.0-rc.5

package/src/backend.ts

@@ -0,0 +1,285 @@
+// NormalizedBackend — the composition that turns a NORMALIZED server stream into the flat
+// `Backend` seam the existing `Store` already drives (NORMALIZED-CHANGES-DESIGN.md §5/§7).
+//
+// A normalized client runs its OWN local engine and queries over its base tables. This
+// backend wires that up by composing three pieces behind one `Backend`:
+//   - a local `@rindle/wasm` engine (the base tables + local IVM that materializes results),
+//   - `NormalizedSync` (cross-query refcount → net base-table mutations), and
+//   - a `NormalizedSource` (the server's per-query normalized stream).
+//
+// The flow per query: `registerQuery` registers it BOTH on the local engine (→ an empty
+// `FlatArrayView`) and on the server (→ its normalized footprint stream). Each normalized
+// batch folds through `NormalizedSync` into net base mutations, which are applied to the
+// local engine — whose own flat change stream then updates every affected view. Because the
+// local engine fans a base write to ALL local queries, a row synced for one query updates
+// any other query that reads it (the "local query resolution" the design is built for).
+//
+// Since this implements `Backend`, `new Store(schema, new NormalizedBackend(...))` reuses the
+// whole Store/ArrayView machinery unchanged — the normalized path adds composition, not a
+// second materialization layer.
+
+import { normalizedTableSchemas, Store, tableSpec } from "@rindle/client";
+import type {
+  Ast,
+  Backend,
+  ChangeEvent,
+  ColsMap,
+  Mutation,
+  NormalizedEvent,
+  NormalizedSource,
+  NormalizedTableSchema,
+  QueryId,
+  RemoteQuery,
+  Schema,
+} from "@rindle/client";
+import { WasmBackend } from "@rindle/wasm";
+
+import { aggTableSchemas, rewriteAggregates } from "./agg-table.ts";
+import { NormalizedSync, type ColCounts, type PkCols } from "./sync.ts";
+
+// The normalized seam + event/table-schema types live in `@rindle/client` (sibling of `Backend`),
+// so a ws (`@rindle/remote`) source and the in-process native source emit the same shape and
+// `@rindle/normalized` needs no dependency on `@rindle/remote`. Re-exported for callers.
+export type { NormalizedEvent, NormalizedSource, NormalizedTableSchema } from "@rindle/client";
+
+/** Each table's primary-key column indices, from the typed schema. */
+function pkColsFromSchema<S extends ColsMap>(schema: Schema<S>): PkCols {
+  const out: PkCols = {};
+  for (const name of Object.keys(schema.tables)) out[name] = tableSpec(schema.tables[name]).primaryKey;
+  return out;
+}
+
+/** Each table's FULL column count (the union-row width), from the typed schema. */
+function colCountsFromSchema<S extends ColsMap>(schema: Schema<S>): ColCounts {
+  const out: ColCounts = {};
+  for (const name of Object.keys(schema.tables)) out[name] = tableSpec(schema.tables[name]).columns.length;
+  return out;
+}
+
+/** Each table's column name → base ColId, from the typed schema (for mapping a projected
+ *  hello's columns back to base positions, PROJECTION-SUPPORT-DESIGN.md §5.2). */
+function colIndexFromSchema<S extends ColsMap>(schema: Schema<S>): Record<string, Map<string, number>> {
+  const out: Record<string, Map<string, number>> = {};
+  for (const name of Object.keys(schema.tables)) {
+    const cols = tableSpec(schema.tables[name]).columns;
+    out[name] = new Map(cols.map((c, i) => [c, i]));
+  }
+  return out;
+}
+
+export class NormalizedBackend<S extends ColsMap> implements Backend {
+  private readonly local: WasmBackend<S>;
+  private readonly sync: NormalizedSync;
+  private readonly source: NormalizedSource;
+  private handler: (qid: QueryId, ev: ChangeEvent) => void = () => {};
+  private readonly remoteSubs = new Map<string, RemoteSub>();
+  private readonly localToRemote = new Map<QueryId, string>();
+  /** The client's OWN typed per-table schemas (for CRIT#4 validation), fixed at construction. */
+  private readonly clientTables: NormalizedTableSchema[];
+  /** Synthetic aggregate tables (`__agg_*`) registered so far, by name (§3.3). Per aggregate
+   *  DEFINITION (not per query), so two queries over the same count share one table. */
+  private readonly synthetic = new Map<string, NormalizedTableSchema>();
+  /** table → (column name → base ColId), for mapping a projected hello to base positions. */
+  private readonly colIndex: Record<string, Map<string, number>>;
+  /** table → full column count, to detect whether a hello's table is projected. */
+  private readonly colCounts: ColCounts;
+  /** Synthetic table name → how many registered queries reference it. Materialized on `0→1`,
+   *  reclaimed (engine source + refcount layer) on `1→0` — so aggregate state is not permanent (§4). */
+  private readonly syntheticRefs = new Map<string, number>();
+  /** queryId → the synthetic tables it referenced at registration, to decrement on teardown. */
+  private readonly queryAggTables = new Map<QueryId, string[]>();
+
+  constructor(schema: Schema<S>, source: NormalizedSource) {
+    this.local = new WasmBackend(schema);
+    // The local engine's flat stream IS this backend's stream upward (→ the Store's views).
+    this.local.onEvent((qid, ev) => this.handler(qid, ev));
+    this.colIndex = colIndexFromSchema(schema);
+    this.colCounts = colCountsFromSchema(schema);
+    this.sync = new NormalizedSync(pkColsFromSchema(schema), this.colCounts);
+    this.source = source;
+    // Hand the source our OWN typed schema so it validates each server hello against it and
+    // rejects a column-order / PK skew instead of silently transposing positional cells (CRIT#4).
+    this.clientTables = normalizedTableSchemas(schema);
+    this.source.expectClientSchema?.(this.clientTables);
+    this.source.onNormalized((qid, ev) => this.onNormalized(qid, ev));
+  }
+
+  registerQuery(qid: QueryId, ast: Ast, remote?: RemoteQuery): void {
+    // A relationship aggregate (`count(child)`) is DISPLAYED from a server-authoritative
+    // synthetic base table, not recomputed locally (AGGREGATE-SYNC-DESIGN.md §3.3): register
+    // that table (engine + refcount layer + hello validation), then drive the local engine
+    // off a rewritten AST whose `count` relationships read it with a plain projected join.
+    this.ensureSyntheticTables(qid, ast);
+    // Local first: builds the (empty) view synchronously. Then the server stream hydrates it.
+    this.local.registerQuery(qid, rewriteAggregates(ast));
+    if (remote) this.retainRemote(qid, remote);
+  }
+
+  /** Register every synthetic aggregate table `ast` needs that we haven't seen yet: on the
+   *  local engine (so it can join to it), on `NormalizedSync` (so its rows refcount/GC by
+   *  group key), and into the source's expected-schema set (so the server's `hello` — which
+   *  advertises the same table — passes CRIT#4 validation, the client deriving the schema
+   *  identically). Idempotent across queries that share an aggregate definition. */
+  private ensureSyntheticTables(qid: QueryId, ast: Ast): void {
+    if (this.queryAggTables.has(qid)) return; // idempotent per qid
+    let added = false;
+    const names: string[] = [];
+    for (const t of aggTableSchemas(ast)) {
+      names.push(t.name);
+      const prev = this.syntheticRefs.get(t.name) ?? 0;
+      this.syntheticRefs.set(t.name, prev + 1);
+      if (prev > 0) continue; // another query already materialized it — just refcount
+      this.synthetic.set(t.name, t);
+      this.local.registerTable(t.name, { columns: t.columns, primaryKey: t.primaryKey });
+      this.sync.registerTable(t.name, t.primaryKey);
+      added = true;
+    }
+    if (names.length) this.queryAggTables.set(qid, names);
+    if (added) this.source.expectClientSchema?.([...this.clientTables, ...this.synthetic.values()]);
+  }
+
+  /** Decrement each synthetic table query `qid` referenced; remove the ones that reach 0 (no
+   *  reader left) from the engine + refcount layer — aggregate state reclaimed, not permanent
+   *  (§4). Runs AFTER `local.unregisterQuery(qid)` so the source has no live connection when
+   *  `unregisterTable` frees it. */
+  private releaseSyntheticTables(qid: QueryId): void {
+    const names = this.queryAggTables.get(qid);
+    if (!names) return;
+    this.queryAggTables.delete(qid);
+    let removed = false;
+    for (const name of names) {
+      const next = (this.syntheticRefs.get(name) ?? 1) - 1;
+      if (next > 0) {
+        this.syntheticRefs.set(name, next);
+        continue;
+      }
+      this.syntheticRefs.delete(name);
+      this.local.unregisterTable(name);
+      this.sync.unregisterTable(name);
+      this.synthetic.delete(name);
+      removed = true;
+    }
+    if (removed) this.source.expectClientSchema?.([...this.clientTables, ...this.synthetic.values()]);
+  }
+
+  unregisterQuery(qid: QueryId): void {
+    const remoteQid = this.releaseRemote(qid);
+    if (remoteQid !== undefined) {
+      // Drop this remote footprint's refcounts; rows referenced by no other named query are
+      // GC'd from the local base tables (→ other views shrink if they shared them).
+      const muts = this.sync.dropQuery(remoteQid);
+      if (muts.length) void this.local.mutate(muts);
+    }
+    this.local.unregisterQuery(qid);
+    // Pipeline gone (no live conn) + `__agg` rows GC'd above → free any now-unread synthetic table.
+    this.releaseSyntheticTables(qid);
+  }
+
+  retainRemoteQuery(qid: QueryId, remote: RemoteQuery): void {
+    this.retainRemote(qid, remote);
+  }
+
+  releaseRemoteQuery(qid: QueryId): void {
+    const remoteQid = this.releaseRemote(qid);
+    if (remoteQid === undefined) return;
+    const muts = this.sync.dropQuery(remoteQid);
+    if (muts.length) void this.local.mutate(muts);
+  }
+
+  /** Writes are authoritative-only here: send to the server, the stream reconciles locally.
+   *  (Optimistic local apply + rebase is Slice 6.) */
+  mutate(mutations: Mutation[]): Promise<void> {
+    return this.source.mutate(mutations);
+  }
+
+  onEvent(handler: (qid: QueryId, ev: ChangeEvent) => void): void {
+    this.handler = handler;
+  }
+
+  private onNormalized(qid: QueryId, ev: NormalizedEvent): void {
+    // The slim hello carries table schemas + fingerprint; envelope validation (epoch/seq/gap)
+    // is the source's job (the ws Subscriber, §5.3) — it precedes every (re)hydrate snapshot.
+    if (ev.type === "hello") {
+      // Learn this query's per-table column map (PROJECTION-SUPPORT-DESIGN.md §5.2): map each
+      // advertised column to its base ColId BY NAME. The hello may carry FEWER columns than the
+      // client's schema (a projection) or MORE (an EXPANDED server table mid an
+      // `expand-then-contract` migration) — a column the client lacks maps to `-1`, a DROP
+      // sentinel the sync layer discards. Register the map so the sync layer scatters the rows
+      // into the shared full-width union; a table whose map is an in-order, drop-free full width
+      // IS '*' (a verbatim full row) and is left unregistered. Idempotent across re-hydrate epochs.
+      for (const t of ev.tables) {
+        const full = this.colCounts[t.name];
+        if (full === undefined) continue; // unknown/synthetic table → '*' (full presence)
+        const index = this.colIndex[t.name];
+        const cols = t.columns.map((name) => index?.get(name) ?? -1);
+        // Register a non-trivial map; otherwise revert to '*' — and CLEAR any stale map a prior
+        // epoch left (a server that expanded then contracted back), so the now-exact rows don't
+        // scatter through a `-1`-bearing layout (silent cell corruption).
+        if (cols.length !== full || cols.some((c, i) => c !== i)) this.sync.registerProjection(qid, t.name, cols);
+        else this.sync.unregisterProjection(qid, t.name);
+      }
+      return;
+    }
+    // A `snapshot` is the seq-0 baseline — initial OR a re-hydrate under a new epoch (§5.3). Both
+    // go through `rehydrate` (a footprint DIFF): for the first one the footprint is empty so it
+    // degenerates to all-adds; for a re-hydrate it removes rows that left during the gap. A
+    // `batch` is an incremental delta → `applyBatch`.
+    const muts = ev.type === "snapshot" ? this.sync.rehydrate(qid, ev.ops) : this.sync.applyBatch(qid, ev.ops);
+    if (muts.length) void this.local.mutate(muts); // → local engine → flat stream → views
+  }
+
+  private retainRemote(localQid: QueryId, remote: RemoteQuery): void {
+    const key = remoteKey(remote);
+    let sub = this.remoteSubs.get(key);
+    if (!sub) {
+      sub = { sourceQid: localQid, remote, refCount: 0 };
+      this.remoteSubs.set(key, sub);
+      this.source.registerQuery(sub.sourceQid, remote);
+    }
+    sub.refCount++;
+    this.localToRemote.set(localQid, key);
+  }
+
+  private releaseRemote(localQid: QueryId): QueryId | undefined {
+    const key = this.localToRemote.get(localQid);
+    if (!key) return undefined;
+    this.localToRemote.delete(localQid);
+    const sub = this.remoteSubs.get(key);
+    if (!sub) return undefined;
+    sub.refCount--;
+    if (sub.refCount > 0) return undefined;
+    this.source.unregisterQuery(sub.sourceQid);
+    this.remoteSubs.delete(key);
+    return sub.sourceQid;
+  }
+}
+
+interface RemoteSub {
+  sourceQid: QueryId;
+  remote: RemoteQuery;
+  refCount: number;
+}
+
+function remoteKey(remote: RemoteQuery): string {
+  return stableJson([remote.name, remote.args]);
+}
+
+function stableJson(value: unknown): string {
+  if (value === null || typeof value !== "object") return JSON.stringify(value);
+  if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`;
+  const obj = value as Record<string, unknown>;
+  return `{${Object.keys(obj)
+    .sort()
+    .map((k) => `${JSON.stringify(k)}:${stableJson(obj[k])}`)
+    .join(",")}}`;
+}
+
+/** A local-first {@link Store} whose base tables are fed by a server's normalized stream.
+ *  The returned Store is the ordinary `@rindle/client` Store — `store.query…materialize()` and
+ *  `store.write(…)` work as always; only the backend composition differs. */
+export function createNormalizedStore<S extends ColsMap>(
+  schema: Schema<S>,
+  source: NormalizedSource,
+): Store<S> {
+  return new Store(schema, new NormalizedBackend(schema, source));
+}

package/src/index.ts

@@ -0,0 +1,16 @@
+// @rindle/normalized — the normalized local-first client glue (NORMALIZED-CHANGES-DESIGN.md §5/§7).
+//
+// - `NormalizedSync` (Slice 4): the cross-query refcount + GC layer + the `NormalizedOp` wire type.
+// - `NormalizedBackend` / `createNormalizedStore` (Slice 5): compose a NORMALIZED server stream
+//   (a `NormalizedSource`) with a local `@rindle/wasm` engine, so the ordinary `Store`/`ArrayView`
+//   materializes results from base tables fed by the server.
+
+export { NormalizedSync } from "./sync.ts";
+export type { ColCounts, NormalizedOp, PkCols } from "./sync.ts";
+
+// Synthetic aggregate tables (§3.3): the byte-exact twin of Rust `agg_table_name` /
+// `agg_table_schemas`, plus the local-engine AST rewrite. Exported for tests + advanced glue.
+export { aggTableName, aggTableSchemas, rewriteAggregates } from "./agg-table.ts";
+
+export { NormalizedBackend, createNormalizedStore } from "./backend.ts";
+export type { NormalizedEvent, NormalizedSource, NormalizedTableSchema } from "./backend.ts";