@loomwork/ui-lineage 0.0.0-stage → 0.1.1

package/src/seenCrypto.ts

@@ -0,0 +1,271 @@
+import { createCipheriv, createDecipheriv, createHmac, hkdfSync, randomBytes } from "node:crypto";
+import type { Finding } from "@loomwork/ui-core";
+
+// ── Per-viewer data keys for the record of what a viewer saw (U12) ─────────────────────────────
+// WHY THIS IS HERE AND NOT `@loomwork/plugin-crypto`'s `shredSubject`: loom's shredder refuses the
+// reserved `__loom_*` streams (a logged loom follow-up), and a seen log has to live in one. What IS
+// reused is the SHAPE of loom's `KeyStore` (getOrCreate / get / destroy, destroy = the erasure
+// primitive, never resurrected): a host's existing KMS-backed `KeyStore` satisfies `SeenKeyStore`
+// structurally and can be passed as is. What is NOT reused is `encryptValue`: it has no associated
+// data, and this design needs the ciphertext bound to (experience, viewer, record, entry) so a ref
+// cannot be transplanted. So the primitive (AES-256-GCM, node:crypto, as loom's) is local.
+
+/**
+ * Key material for one (experience, viewer) subject. Destroying a key is the erasure primitive, so a
+ * production store must really delete it (KMS/HSM), and `durable: false` stores lose every viewer's
+ * record on restart (indistinguishable from erasure).
+ */
+export interface SeenKeyStore {
+  /**
+   * The subject's 32-byte key, created on first use. MUST reject for a destroyed subject (never
+   * resurrect). Create-if-absent MUST be ATOMIC: two concurrent calls for one new subject return the
+   * SAME key (a durable implementation needs a conditional insert / compare-and-set, not
+   * read-then-write), or one writer's records are sealed under a key that is then lost.
+   */
+  getOrCreate(subjectId: string): Promise<Uint8Array>;
+  /**
+   * The key, or `null` if it never existed OR was destroyed. AFTER `destroy` an implementation MUST
+   * NOT serve the key from ANY layer, including an in-process cache warmed by an earlier `get`: a
+   * cache that outlives `destroy` makes erasure a lie on that process. (`SeenLog.record` re-reads
+   * the key just before it appends, so a destroyed key is observed there.)
+   */
+  get(subjectId: string): Promise<Uint8Array | null>;
+  /** Irreversibly destroy the key. Idempotent; a no-op for an absent subject (which stays destroyed). */
+  destroy(subjectId: string): Promise<void>;
+  readonly durable?: boolean;
+}
+
+/** Reference in-memory key store (tests / ephemeral apps): NOT durable. */
+export class InMemorySeenKeyStore implements SeenKeyStore {
+  readonly durable = false;
+  private readonly keys = new Map<string, Uint8Array>();
+  private readonly destroyed = new Set<string>();
+
+  async getOrCreate(subjectId: string): Promise<Uint8Array> {
+    if (this.destroyed.has(subjectId))
+      throw new Error(`the key for "${subjectId}" was destroyed; it cannot be recreated`);
+    let key = this.keys.get(subjectId);
+    if (key === undefined) {
+      key = new Uint8Array(randomBytes(32));
+      this.keys.set(subjectId, key);
+    }
+    return new Uint8Array(key); // a copy: a caller cannot mutate (or retain a handle on) the stored key
+  }
+  async get(subjectId: string): Promise<Uint8Array | null> {
+    const key = this.keys.get(subjectId);
+    return key === undefined ? null : new Uint8Array(key);
+  }
+  async destroy(subjectId: string): Promise<void> {
+    this.keys.get(subjectId)?.fill(0); // zeroize, then forget
+    this.keys.delete(subjectId);
+    this.destroyed.add(subjectId);
+  }
+}
+
+/** The subject (unit of erasure) of a stream key. */
+export const seenKeySubject = (streamKey: string): string => `loom-ui/seen/${streamKey}`;
+
+export interface DerivedSeenKeys {
+  readonly enc: Buffer;
+  readonly mac: Buffer;
+}
+
+/** Separate encryption and MAC keys from the stored data key (HKDF-SHA256). Throws on a malformed key. */
+export function deriveSeenKeys(master: Uint8Array): DerivedSeenKeys {
+  if (!(master instanceof Uint8Array) || master.length !== 32)
+    throw new Error("a seen data key must be 32 bytes");
+  const d = (info: string) =>
+    Buffer.from(hkdfSync("sha256", master, Buffer.alloc(0), `loom-ui/seen/${info}/v1`, 32));
+  return { enc: d("enc"), mac: d("mac") };
+}
+
+/** The associated data a sealed ref is bound to. */
+export const refAad = (
+  experienceId: string,
+  viewerId: string,
+  sequence: number,
+  entryIndex: number
+): Buffer =>
+  Buffer.from(
+    JSON.stringify(["loom-ui/seen/ref/v1", experienceId, viewerId, sequence, entryIndex])
+  );
+
+/**
+ * Plaintext length (bytes) every sealed ref is padded to, so the ciphertext length does not reveal
+ * the value's size (the `bytes` of an entry lives only INSIDE the seal). Addresses longer than this
+ * round up to the next multiple, which a real content address never needs.
+ */
+export const SEALED_REF_BLOCK = 256;
+/** Longest content address `sealRef` accepts. Real addresses are sha256-sized; anything longer fails closed so padding never rounds up (a length leak). */
+export const MAX_SEALED_ADDRESS_CHARS = 128;
+
+/** The plaintext inside a sealed ref: the content ref and the value's measured size, padded to a constant length. */
+export function sealRef(
+  enc: Buffer,
+  aad: Buffer,
+  ref: { readonly __loomContent: string },
+  bytes: number
+): string {
+  if (ref.__loomContent.length > MAX_SEALED_ADDRESS_CHARS)
+    throw new Error(
+      `sealRef: a content address over ${MAX_SEALED_ADDRESS_CHARS} characters is refused`
+    );
+  const body = JSON.stringify({ r: ref, b: bytes });
+  const len = Buffer.byteLength(body, "utf-8");
+  const padded = Math.ceil(len / SEALED_REF_BLOCK) * SEALED_REF_BLOCK;
+  return sealText(enc, aad, body + " ".repeat(padded - len));
+}
+
+/** AES-256-GCM: `base64url(0x01 | iv | ciphertext | tag)`. A fresh random IV per call. */
+export function sealText(enc: Buffer, aad: Buffer, plaintext: string): string {
+  const iv = randomBytes(12);
+  const c = createCipheriv("aes-256-gcm", enc, iv);
+  c.setAAD(aad);
+  const ct = Buffer.concat([c.update(Buffer.from(plaintext, "utf-8")), c.final()]);
+  return Buffer.concat([Buffer.from([1]), iv, ct, c.getAuthTag()]).toString("base64url");
+}
+
+/** The plaintext, or `undefined` for ANY failure (wrong key, wrong AAD, tampering, malformed): fail closed. */
+export function openText(enc: Buffer, aad: Buffer, sealed: unknown): string | undefined {
+  try {
+    if (typeof sealed !== "string") return undefined;
+    const raw = Buffer.from(sealed, "base64url");
+    if (raw.length < 1 + 12 + 16 || raw[0] !== 1) return undefined;
+    const d = createDecipheriv("aes-256-gcm", enc, raw.subarray(1, 13));
+    d.setAAD(aad);
+    d.setAuthTag(raw.subarray(raw.length - 16));
+    return Buffer.concat([d.update(raw.subarray(13, raw.length - 16)), d.final()]).toString(
+      "utf-8"
+    );
+  } catch {
+    return undefined;
+  }
+}
+
+/** `undefined` for any failure; otherwise the ref and size sealed by {@link sealRef}. */
+export function openRef(
+  enc: Buffer,
+  aad: Buffer,
+  sealed: unknown
+): { readonly ref: { readonly __loomContent: string }; readonly bytes: number } | undefined {
+  const text = openText(enc, aad, sealed);
+  if (text === undefined) return undefined;
+  try {
+    const v = JSON.parse(text) as { r?: { __loomContent?: unknown }; b?: unknown };
+    const addr = v?.r?.__loomContent;
+    if (
+      typeof addr !== "string" ||
+      addr.length === 0 ||
+      addr.length > 256 ||
+      !Number.isSafeInteger(v.b) ||
+      (v.b as number) < 0
+    )
+      return undefined;
+    return { ref: { __loomContent: addr }, bytes: v.b as number };
+  } catch {
+    return undefined;
+  }
+}
+
+/** Keyed digest: an unkeyed hash of a manifest would let anyone confirm a guessed value. */
+export const keyedDigest = (mac: Buffer, text: string): string =>
+  `hmac:${createHmac("sha256", mac).update(text).digest("hex")}`;
+
+/** The `SeenKeyStore` conformance kit: findings (a test asserts none), never throws. */
+export async function checkSeenKeyStoreConformance(
+  make: () => SeenKeyStore | Promise<SeenKeyStore>
+): Promise<Finding[]> {
+  const findings: Finding[] = [];
+  const fail = (code: string, message: string) =>
+    findings.push({ code, severity: "error", message });
+  const store = await make();
+  const same = (a: Uint8Array | null, b: Uint8Array | null) =>
+    a !== null && b !== null && Buffer.compare(Buffer.from(a), Buffer.from(b)) === 0;
+  const cases: Array<[string, () => Promise<void>]> = [
+    [
+      "an unknown subject has no key",
+      async () => {
+        if ((await store.get("k-none")) !== null)
+          fail("get-unknown", "get of an unknown subject was not null");
+      },
+    ],
+    [
+      "getOrCreate yields a stable 32-byte key",
+      async () => {
+        const a = await store.getOrCreate("k-a");
+        if (a.length !== 32) fail("key-length", `a key is ${a.length} bytes, not 32`);
+        if (!same(a, await store.getOrCreate("k-a")) || !same(a, await store.get("k-a")))
+          fail("key-stable", "getOrCreate/get did not return one key for one subject");
+      },
+    ],
+    [
+      "subjects get independent keys",
+      async () => {
+        if (same(await store.getOrCreate("k-1"), await store.getOrCreate("k-2")))
+          fail("key-distinct", "two subjects share a key");
+      },
+    ],
+    [
+      "destroy erases and never resurrects",
+      async () => {
+        await store.getOrCreate("k-d");
+        await store.destroy("k-d");
+        if ((await store.get("k-d")) !== null)
+          fail("destroy-get", "get after destroy was not null");
+        let recreated = false;
+        try {
+          await store.getOrCreate("k-d");
+          recreated = true;
+        } catch {
+          /* required */
+        }
+        if (recreated) fail("destroy-resurrect", "getOrCreate after destroy created a new key");
+      },
+    ],
+    [
+      "get after destroy is null even after a prior get (no cache outlives destroy)",
+      async () => {
+        await store.getOrCreate("k-w");
+        await store.get("k-w"); // warm any cache
+        await store.get("k-w");
+        await store.destroy("k-w");
+        if ((await store.get("k-w")) !== null)
+          fail("destroy-cache", "get after destroy served a key from a warm cache");
+      },
+    ],
+    [
+      "concurrent getOrCreate of one new subject returns one key",
+      async () => {
+        const [a, b, c] = await Promise.all([
+          store.getOrCreate("k-race"),
+          store.getOrCreate("k-race"),
+          store.getOrCreate("k-race"),
+        ]);
+        if (!same(a, b) || !same(b, c))
+          fail(
+            "create-race",
+            "concurrent getOrCreate returned different keys (create is not atomic)"
+          );
+        if (!same(a, await store.get("k-race")))
+          fail("create-race", "the stored key is not the one concurrent callers received");
+      },
+    ],
+    [
+      "destroy is idempotent and works on an absent subject",
+      async () => {
+        await store.destroy("k-d");
+        await store.destroy("k-never");
+        if ((await store.get("k-never")) !== null)
+          fail("destroy-absent", "an absent subject has a key");
+      },
+    ],
+  ];
+  for (const [name, run] of cases) {
+    try {
+      await run();
+    } catch (err) {
+      fail("store-threw", `${name}: ${err instanceof Error ? err.message : String(err)}`);
+    }
+  }
+  return findings;
+}

package/src/seenGc.ts

@@ -0,0 +1,233 @@
+import type { PinsExtra } from "./lineage";
+import type { RetentionPolicy } from "./retention";
+import { type SeenLog, type SeenRefusal, type SeenUnreadable, UNREADABLE_EXAMPLES } from "./seen";
+
+// ── Reclaiming the value blobs of the record of what a viewer saw (U12) ──────────────────────
+// MARK-AND-SWEEP, not a refcount. There is no transaction across the event log and the ContentStore,
+// so a counter drifts, and a crash between "append the manifest" and "increment" would delete a live
+// value. The roots are re-derived from the logs on every pass, so nothing here can drift:
+//
+//   MARK   every content address cited by (a) every line of the lineage (artifacts, op payloads,
+//          anything stored as a ref in an event) and (b) every DECRYPTABLE ref of every viewer's
+//          retention window (the age window is kept `maxSkewMs` wider than a read's); plus whatever the host's `extraRoots` names. Erased viewers and
+//          rolled-off views are unreadable or out of window, so they mark nothing.
+//   SWEEP  every blob the store lists that is not marked, deleted with the store's own conditional
+//          `deleteIfUntouchedSince` so a blob that is touched (a put, including a dedup'd re-put)
+//          after the cutoff is never deleted, in one atomic step.
+//
+// SAFETY WINDOW. `cutoff = store.touchClock() at mark start - graceMs`. A blob touched at or after the
+// cutoff is kept: that covers a blob written during the pass AND one younger than `graceMs`, i.e. a
+// value whose manifest is not appended yet. `graceMs` MUST exceed the longest a `record()` (or a
+// lineage commit) can take between putting a blob and appending the event that cites it; a crash in
+// that gap leaks the blob only until the first pass past its grace, and never loses a cited value.
+// The pass is idempotent and safe to rerun (or to crash in the middle of).
+//
+// FAIL CLOSED. Anything the collector cannot establish aborts BEFORE any delete: a store that cannot
+// enumerate its streams, a key-store failure, ANYTHING UNREADABLE in the live window (a stream key
+// that does not round-trip, a record that fails to parse, e.g. a newer writer's shape during a rolling
+// upgrade, a sealed ref that fails to open under a present key: `gc-unreadable-record`, with counts), a ContentStore without the SPI 1.3.0 set
+// (`listPage` + `deleteIfUntouchedSince` + `touchClock`), a root set over `maxMarked`.
+//
+// MEMORY. The marked set holds one address string per distinct live blob (O(live blobs); `maxMarked`
+// refuses rather than grow past it). Streams are read one at a time; the sweep pages by `pageSize`.
+
+export interface SeenGcOptions {
+  /** Blobs touched within this many ms before the pass began are never swept. MUST exceed one `record()`'s duration. */
+  readonly graceMs: number;
+  /**
+   * You assert EVERY blob in the log's ContentStore is either rooted by the lineage / the seen logs
+   * or named by `extraRoots`. The collector deletes what it cannot find a root for: pointing it at a
+   * store that also holds anything else would destroy that data. A literal `true`, so it is a decision.
+   */
+  readonly ownsContentStore: true;
+  /** Addresses this collector cannot see but must keep (other users of the same ContentStore). */
+  readonly extraRoots?: () => AsyncIterable<string> | Iterable<string>;
+  /**
+   * Reclaim old LINEAGE versions in the same pass. Absent (the default), every artifact any version
+   * ever cited is rooted (`Lineage.forEachContentRef`), so this collector reclaims only the seen
+   * log's value blobs. With a policy, the lineage half of the root set is `Lineage.pins(policy)` plus
+   * the versions the live views still name, so versions the policy dropped (and not a viewer's) stop
+   * being rooted and are swept. Use this when ONE collector owns the whole store; use
+   * `Lineage.pins` + loom's `sweepUnpinnedContent` when loom's sweep owns it and pass it
+   * `collectSeenRoots(log)` as `extra`. Never run both against the same store with different roots.
+   */
+  readonly lineageRetention?: RetentionPolicy;
+  /** Refuse (before deleting anything) if more than this many distinct addresses are live. Default 5,000,000. */
+  readonly maxMarked?: number;
+  /** Blobs per `listPage`. Default 500. */
+  readonly pageSize?: number;
+  /**
+   * Proceed although some live-window stream / record / sealed ref could not be read. OPT-IN MEANS
+   * "I ACCEPT SWEEPING WHAT THOSE CITE": their blobs are unrooted and WILL be deleted once past the
+   * grace period (for a rolling upgrade, finish the upgrade and rerun instead). Default: refuse.
+   * A viewer whose key is ABSENT (erased) is not unreadable: that is the intended shred path.
+   */
+  readonly allowUnreadable?: boolean;
+}
+
+export interface SeenGcReport {
+  /** Distinct live addresses found. */
+  readonly marked: number;
+  /** Blobs older than the cutoff that were examined. */
+  readonly scanned: number;
+  /** Blobs deleted. */
+  readonly deleted: number;
+  /** Unrooted blobs spared because they were touched at or after the cutoff (in grace). */
+  readonly spared: number;
+  readonly cutoffMs: number;
+  /** What could not be read (non-zero only with `allowUnreadable: true`). */
+  readonly unreadable: SeenUnreadable;
+}
+
+export type SeenGcResult =
+  | { readonly ok: true; readonly report: SeenGcReport }
+  | { readonly ok: false; readonly refusal: SeenRefusal };
+
+const no = (code: string, message: string): SeenGcResult => ({
+  ok: false,
+  refusal: {
+    kind: "refused",
+    finding: { code, severity: "error", message },
+    code,
+    message,
+  },
+});
+
+/** Run one collection pass over `log`'s content store. See the header for the contract. */
+export async function collectSeenGarbage(log: SeenLog, opts: SeenGcOptions): Promise<SeenGcResult> {
+  if (opts?.ownsContentStore !== true)
+    return no(
+      "gc-ownership-unasserted",
+      "ownsContentStore: true is required: the sweep deletes unrooted blobs"
+    );
+  if (!Number.isFinite(opts.graceMs) || opts.graceMs < 0)
+    return no("gc-grace-invalid", "graceMs must be a non-negative number of milliseconds");
+  const maxMarked = opts.maxMarked ?? 5_000_000;
+  const pageSize = opts.pageSize ?? 500;
+  if (
+    !Number.isSafeInteger(maxMarked) ||
+    maxMarked < 1 ||
+    !Number.isSafeInteger(pageSize) ||
+    pageSize < 1
+  )
+    return no("gc-options-invalid", "maxMarked and pageSize must be positive integers");
+  const content = log.contentStore;
+  if (
+    content.listPage === undefined ||
+    content.deleteIfUntouchedSince === undefined ||
+    content.touchClock === undefined
+  )
+    return no(
+      "gc-content-store-unsupported",
+      "the ContentStore must implement listPage, deleteIfUntouchedSince and touchClock (SPI 1.3.0): without a touch time a young blob cannot be told from an orphan"
+    );
+
+  const marked = new Set<string>();
+  const visit = (address: string) => {
+    marked.add(address);
+    if (marked.size > maxMarked) throw new Error("maxMarked exceeded");
+  };
+  let cutoffMs: number;
+  let unreadable: SeenUnreadable = {
+    streams: 0,
+    records: 0,
+    refs: 0,
+    examples: { streams: [], records: [], refs: [] },
+  };
+  try {
+    // The clock is read BEFORE marking: anything touched after this instant is spared, so a blob
+    // written while we mark (whose manifest we may have read too early to see) survives.
+    cutoffMs = (await content.touchClock()) - opts.graceMs;
+    const versions = new Map<string, { experienceId: string; versionHash: string }>();
+    unreadable = await log.forEachLiveRef(visit, (experienceId, versionHash) =>
+      versions.set(`${experienceId}\u0000${versionHash}`, { experienceId, versionHash })
+    );
+    if (opts.lineageRetention === undefined) await log.lineageOf.forEachContentRef(visit);
+    else
+      for await (const a of log.lineageOf.pins(opts.lineageRetention, {
+        keepVersions: versions.values(),
+      }))
+        visit(a);
+    if (opts.extraRoots !== undefined) for await (const a of opts.extraRoots()) visit(a);
+  } catch (err) {
+    return no(
+      "gc-roots-unavailable",
+      `the root set could not be established (nothing was deleted): ${err instanceof Error ? err.message : String(err)}`
+    );
+  }
+
+  if (
+    unreadable.streams + unreadable.records + unreadable.refs > 0 &&
+    opts.allowUnreadable !== true
+  )
+    return no(
+      "gc-unreadable-record",
+      `${unreadable.streams} stream(s), ${unreadable.records} record(s) and ${unreadable.refs} sealed ref(s) in the live window could not be read, so what they cite is unrooted (nothing was deleted). Fix or finish the upgrade and rerun, or pass allowUnreadable: true to sweep it. Offenders (first ${UNREADABLE_EXAMPLES} per kind): streams [${unreadable.examples.streams.join(", ")}], records [${unreadable.examples.records.join(", ")}], refs [${unreadable.examples.refs.join(", ")}]`
+    );
+
+  let scanned = 0;
+  let deleted = 0;
+  let spared = 0;
+  try {
+    let cursor: string | undefined;
+    for (;;) {
+      const page = await content.listPage({
+        ...(cursor !== undefined && { cursor }),
+        limit: pageSize,
+      });
+      for (const entry of page.entries) {
+        if (marked.has(entry.address)) continue;
+        // 0 = unknown touch time (never orphan-eligible); at/after the cutoff = young or just touched
+        if (entry.touchedAt === 0 || !(entry.touchedAt < cutoffMs)) {
+          spared += 1;
+          continue;
+        }
+        scanned += 1;
+        if (await content.deleteIfUntouchedSince({ __loomContent: entry.address }, cutoffMs))
+          deleted += 1;
+        else spared += 1; // touched between the listing and the delete
+      }
+      if (page.nextCursor === undefined || page.nextCursor === cursor) break;
+      cursor = page.nextCursor;
+    }
+  } catch (err) {
+    return no(
+      "gc-sweep-interrupted",
+      `the sweep stopped after deleting ${deleted} blobs (safe to rerun): ${err instanceof Error ? err.message : String(err)}`
+    );
+  }
+  return {
+    ok: true,
+    report: { marked: marked.size, scanned, deleted, spared, cutoffMs, unreadable },
+  };
+}
+
+/**
+ * The seen log's share of a shared content store's root set, for `Lineage.pins(policy, extra)`:
+ * the sealed value blobs of every live view and the (experience, version) each live view reads.
+ *
+ *   roots: () => lineage.pins(policy, await collectSeenRoots(seenLog))   // sweepUnpinnedContent
+ *
+ * With `reclaimOrphans: { completeScan: true }` loom erases EVERYTHING in the store no root names, so
+ * the roots must come from every user of it (this seen log, other lineages, loom's executions). THROWS
+ * if anything in the live window could not be read (a root set that silently omits what it could not
+ * open would delete it): fix or finish the upgrade, or pass `allowUnreadable: true` to accept that.
+ */
+export async function collectSeenRoots(
+  log: SeenLog,
+  opts: { readonly allowUnreadable?: boolean } = {}
+): Promise<Required<Pick<PinsExtra, "keepVersions" | "keepAddresses">>> {
+  const addresses = new Set<string>();
+  const versions = new Map<string, { experienceId: string; versionHash: string }>();
+  const unreadable = await log.forEachLiveRef(
+    (a) => addresses.add(a),
+    (experienceId, versionHash) =>
+      versions.set(`${experienceId}\u0000${versionHash}`, { experienceId, versionHash })
+  );
+  const n = unreadable.streams + unreadable.records + unreadable.refs;
+  if (n > 0 && opts.allowUnreadable !== true)
+    throw new Error(
+      `collectSeenRoots: ${n} stream(s)/record(s)/sealed ref(s) in the live window could not be read, so what they cite would be unrooted (examples: ${[...unreadable.examples.streams, ...unreadable.examples.records, ...unreadable.examples.refs].slice(0, UNREADABLE_EXAMPLES).join(", ")}). Nothing was collected; pass allowUnreadable: true to sweep it anyway`
+    );
+  return { keepVersions: [...versions.values()], keepAddresses: [...addresses] };
+}