@loomwork/ui-lineage 0.0.0-stage → 0.1.1

package/src/state.ts

@@ -0,0 +1,268 @@
+import {
+  type Finding,
+  MAX_TREE_DEPTH,
+  type Provenance,
+  type RecordedOperation,
+  boundSummary,
+  treeDepth,
+} from "@loomwork/ui-core";
+import {
+  type BranchedPayload,
+  type ComposedPayload,
+  EXPERIENCE_BRANCHED,
+  EXPERIENCE_COMPOSED,
+  EXPERIENCE_PROMOTED,
+  EXPERIENCE_REVERTED,
+  PAYLOAD_SCHEMAS,
+  type RevertedPayload,
+  type SummaryClass,
+  isLineageEventType,
+} from "./events";
+import type { StoredLineageRecord } from "./store";
+
+// ── Folding a lineage's events into its state ─────────────────────────────────
+// The fold is pure and total (a malformed log is a finding, never a throw) and it is where the
+// log's STRUCTURE is checked: dense sequences, every version's `parents` naming the head it was
+// appended on, one version per content address. What a version's artifact IS, and whether its ops
+// reproduce it, needs the content store and is `Lineage.verify`'s job.
+
+export type VersionKind = "composed" | "reverted" | "branched";
+
+export interface VersionRecord {
+  readonly experienceId: string;
+  readonly versionHash: string;
+  readonly ref: { readonly __loomContent: string };
+  readonly parents: readonly string[];
+  readonly kind: VersionKind;
+  /** Position in the line's event log. */
+  readonly sequence: number;
+  readonly at: string;
+  readonly by: Provenance;
+  /** The ops folded onto the parent (`composed` only; `[]` otherwise and for a genesis). */
+  readonly ops: readonly RecordedOperation[];
+  /** `composed`: what the author said about the change; untrusted text, already bounded. */
+  readonly summary?: string;
+  /** `composed`: the class of what the summary was written from (host-stamped); see `ComposedPayload`. */
+  readonly summaryClass?: SummaryClass;
+  /** `reverted`: the version whose content this restores. */
+  readonly revertedTo?: string;
+  /** `branched`: the version of another line this forked, and the source head classes came from. */
+  readonly from?: { readonly experienceId: string; readonly versionHash: string };
+  readonly carriedFrom?: string;
+}
+
+export interface LineageState {
+  readonly experienceId: string;
+  /** The event log's tip (0 = empty): the compare-and-set token of the STORE. */
+  readonly sequence: number;
+  readonly versions: ReadonlyMap<string, VersionRecord>;
+  /** Version hashes in append order. */
+  readonly order: readonly string[];
+  /** The latest version (the CAS token of `advance`); null = the line has no version yet. */
+  readonly head: string | null;
+  /** The version the app chose to show; null = none promoted. */
+  readonly promoted: string | null;
+}
+
+/** Most ops one version may record. A longer array is a finding before the schema ever walks it. */
+export const MAX_OPS_PER_VERSION = 10_000;
+/** How deep `prompt` ops may nest their `materialized` ops while being checked. */
+export const MAX_OP_NESTING = 16;
+
+/**
+ * Why a stored ops array must not reach the (recursive) zod `Operation` schema, or undefined when
+ * it may. The schema recurses through `insertNode.node` and `prompt.materialized`, so a payload
+ * nested past the caps would blow the stack inside `safeParse` instead of returning an issue;
+ * a log is untrusted input (anyone with append access to the stream wrote it).
+ */
+function opsRefusal(ops: unknown): string | undefined {
+  if (!Array.isArray(ops)) return undefined; // the schema reports the wrong shape
+  if (ops.length > MAX_OPS_PER_VERSION)
+    return `records ${ops.length} ops, over the ${MAX_OPS_PER_VERSION} limit`;
+  const stack: Array<{ op: unknown; nest: number }> = [];
+  for (const r of ops) stack.push({ op: (r as { op?: unknown } | null)?.op, nest: 1 });
+  let seen = 0;
+  for (let top = stack.pop(); top !== undefined; top = stack.pop()) {
+    const { op, nest } = top;
+    if (typeof op !== "object" || op === null) continue;
+    if (++seen > MAX_OPS_PER_VERSION * 2) return "nests too many ops";
+    const node = (op as { node?: unknown }).node;
+    if (node !== undefined) {
+      const depth = treeDepth(node as never);
+      if (depth > MAX_TREE_DEPTH)
+        return `an op's node nests ${depth} levels, over the ${MAX_TREE_DEPTH} limit`;
+    }
+    const inner = (op as { materialized?: unknown }).materialized;
+    if (Array.isArray(inner)) {
+      if (nest >= MAX_OP_NESTING) return `prompt ops nest over ${MAX_OP_NESTING} levels`;
+      if (inner.length > MAX_OPS_PER_VERSION) return "a prompt op materializes too many ops";
+      for (const i of inner) stack.push({ op: i, nest: nest + 1 });
+    }
+  }
+  return undefined;
+}
+
+/**
+ * The WRITE-SIDE twin of the fold's payload check: why `payload` would make the line unreadable
+ * (`lineage-event-invalid` on every later read), or undefined when it folds. A write that fails this
+ * must be refused BEFORE the append — the fold stays strict, the log has no delete, and one bad event
+ * would brick the line for everyone. `by` marks a provenance problem (the caller gives it its own code).
+ */
+export function payloadRefusal(
+  type: string,
+  payload: unknown
+):
+  | { readonly code: "provenance-invalid" | "event-payload-invalid"; readonly message: string }
+  | undefined {
+  if (!isLineageEventType(type))
+    return { code: "event-payload-invalid", message: `unknown event type ${String(type)}` };
+  try {
+    const tooBig = opsRefusal((payload as { ops?: unknown } | null)?.ops);
+    if (tooBig !== undefined) return { code: "event-payload-invalid", message: tooBig };
+    const parsed = PAYLOAD_SCHEMAS[type].safeParse(payload);
+    if (parsed.success) return undefined;
+    const issue = parsed.error.issues[0];
+    const inBy = parsed.error.issues.some((i) => i.path.includes("by"));
+    return {
+      code: inBy ? "provenance-invalid" : "event-payload-invalid",
+      message: `${inBy ? "a provenance (`by`) is invalid" : "the event payload is malformed"}: ${issue?.message ?? "invalid"}${issue ? ` at ${issue.path.join(".")}` : ""}`,
+    };
+  } catch (err) {
+    return { code: "event-payload-invalid", message: errMessage(err) };
+  }
+}
+
+export type Fold = { ok: true; state: LineageState } | { ok: false; finding: Finding };
+
+const bad = (code: string, message: string): Fold => ({
+  ok: false,
+  finding: { code, severity: "error", message },
+});
+
+export function foldLineage(experienceId: string, records: readonly StoredLineageRecord[]): Fold {
+  try {
+    return foldChecked(experienceId, records);
+  } catch (err) {
+    // Total by construction: whatever a hostile record does (a throwing getter, a shape the
+    // checks above did not anticipate) is a finding, never an exception out of every read.
+    return bad("lineage-event-invalid", `lineage "${experienceId}": ${errMessage(err)}`);
+  }
+}
+
+const errMessage = (err: unknown): string => {
+  try {
+    return err instanceof Error ? err.message : String(err);
+  } catch {
+    return "unreadable record";
+  }
+};
+
+function foldChecked(experienceId: string, records: readonly StoredLineageRecord[]): Fold {
+  const versions = new Map<string, VersionRecord>();
+  const order: string[] = [];
+  let head: string | null = null;
+  let promoted: string | null = null;
+  let expectedSequence = 0;
+  for (const rec of records) {
+    expectedSequence += 1;
+    if (typeof rec !== "object" || rec === null)
+      return bad(
+        "lineage-event-invalid",
+        `lineage "${experienceId}" event #${expectedSequence}: the record is not an object`
+      );
+    const where = `lineage "${experienceId}" event #${rec.sequence} (${rec.type})`;
+    if (rec.sequence !== expectedSequence)
+      return bad(
+        "lineage-sequence-gap",
+        `${where}: expected sequence ${expectedSequence} — the log has a hole or a reorder`
+      );
+    if (!isLineageEventType(rec.type))
+      return bad("lineage-event-unknown", `${where}: unknown type`);
+    const tooBig = opsRefusal((rec.payload as { ops?: unknown } | null)?.ops);
+    if (tooBig !== undefined) return bad("lineage-event-invalid", `${where}: ${tooBig}`);
+    const parsed = PAYLOAD_SCHEMAS[rec.type].safeParse(rec.payload);
+    if (!parsed.success)
+      return bad(
+        "lineage-event-invalid",
+        `${where}: ${parsed.error.issues[0]?.message ?? "payload is malformed"}`
+      );
+    const p = parsed.data;
+    if (p.experienceId !== experienceId)
+      return bad("lineage-event-foreign", `${where}: names experience "${p.experienceId}"`);
+
+    if (rec.type === EXPERIENCE_PROMOTED) {
+      if (!versions.has(p.versionHash))
+        return bad(
+          "lineage-promote-unknown",
+          `${where}: promotes unknown version ${p.versionHash}`
+        );
+      promoted = p.versionHash;
+      continue;
+    }
+
+    const v = p as ComposedPayload | RevertedPayload | BranchedPayload;
+    if (versions.has(v.versionHash))
+      return bad("lineage-version-duplicate", `${where}: version ${v.versionHash} already exists`);
+    if (rec.type === EXPERIENCE_BRANCHED) {
+      if (head !== null)
+        return bad("lineage-branch-midline", `${where}: only a line's first event`);
+    } else if (head === null) {
+      if (rec.type !== EXPERIENCE_COMPOSED || v.parents.length !== 0)
+        return bad(
+          "lineage-genesis-invalid",
+          `${where}: a line starts with a parentless composition`
+        );
+    } else if (v.parents.length !== 1 || v.parents[0] !== head) {
+      return bad(
+        "lineage-parent-mismatch",
+        `${where}: parents ${JSON.stringify(v.parents)} are not the head ${head} it was appended on`
+      );
+    }
+    const base = {
+      experienceId,
+      versionHash: v.versionHash,
+      ref: v.ref,
+      parents: v.parents,
+      sequence: rec.sequence,
+      at: rec.at,
+      by: v.by,
+    };
+    const record: VersionRecord =
+      rec.type === EXPERIENCE_COMPOSED
+        ? {
+            ...base,
+            kind: "composed",
+            ops: (v as ComposedPayload).ops as RecordedOperation[],
+            ...summaryOf((v as ComposedPayload).summary),
+            ...((v as ComposedPayload).summaryClass !== undefined && {
+              summaryClass: (v as ComposedPayload).summaryClass as SummaryClass,
+            }),
+          }
+        : rec.type === EXPERIENCE_REVERTED
+          ? {
+              ...base,
+              kind: "reverted",
+              ops: [],
+              revertedTo: (v as RevertedPayload).revertedTo,
+            }
+          : {
+              ...base,
+              kind: "branched",
+              ops: [],
+              from: (v as BranchedPayload).from,
+              carriedFrom: (v as BranchedPayload).carriedFrom,
+            };
+    versions.set(v.versionHash, record);
+    order.push(v.versionHash);
+    head = v.versionHash;
+  }
+  return {
+    ok: true,
+    state: { experienceId, sequence: expectedSequence, versions, order, head, promoted },
+  };
+}
+
+const summaryOf = (raw: string | undefined): { summary?: string } => {
+  const summary = boundSummary(raw);
+  return summary === undefined ? {} : { summary };
+};

package/src/store.ts

@@ -0,0 +1,274 @@
+import {
+  DuplicateEnvelopeIdError,
+  DuplicateSequenceError,
+  type EventStore,
+  headOf,
+} from "@loomwork/event-runtime";
+
+// ── The store seam ────────────────────────────────────────────────────────────
+// What a lineage needs from storage is exactly one thing loom's event log already is: an
+// append-only, per-stream, SEQUENCED log whose append refuses a taken slot. The compare-and-set
+// is the sequence: "append as event n+1 iff the tip is n". `LineageStore` is that, narrowed to
+// what a lineage uses, so a backend that is not an `EventStore` (a table, a KV with a conditional
+// put) can serve it, and `eventStoreLineageStore` adapts the loom SPI that is.
+
+export interface LineageRecord {
+  readonly type: string;
+  readonly payload: unknown;
+  /** ISO-8601 time the event was recorded. */
+  readonly at: string;
+}
+
+export interface StoredLineageRecord extends LineageRecord {
+  /** 1-based position in this line. Dense: 1..n with no gaps. */
+  readonly sequence: number;
+}
+
+export type AppendOutcome =
+  | { readonly ok: true; readonly sequence: number }
+  /** The tip was not `expectedSequence`; `tip` is what it is now. Nothing was written. */
+  | { readonly ok: false; readonly tip: number };
+
+export interface LineageStore {
+  /**
+   * Append `record` as sequence `expectedSequence + 1` iff the line's tip is EXACTLY
+   * `expectedSequence` (0 = the line is empty). Atomic: of N concurrent appends at the same
+   * `expectedSequence` exactly one wins and the rest get `{ ok: false }`. NEVER overwrites,
+   * never skips a sequence, and a refusal is a returned outcome, not a throw (a throw is an
+   * infrastructure failure and leaves the line untouched).
+   */
+  append(
+    experienceId: string,
+    expectedSequence: number,
+    record: LineageRecord
+  ): Promise<AppendOutcome>;
+  /** Every record of the line in sequence order; `[]` for an unknown line. The caller owns the result. */
+  read(experienceId: string): Promise<readonly StoredLineageRecord[]>;
+  /**
+   * OPTIONAL — the newest `limit` records of the line, in sequence order (`[]` for an unknown line).
+   * Lets a long-lived per-viewer log answer "what is the tip?" without reading its whole history.
+   * Implement it together with {@link readAfter}; absent, callers fall back to `read` (O(log length)).
+   */
+  readTail?(experienceId: string, limit: number): Promise<readonly StoredLineageRecord[]>;
+  /**
+   * OPTIONAL — the records with `sequence > afterSequence`, in sequence order, at most `limit` of
+   * them (default: all). The retention window of a per-viewer log is read this way, so a read costs
+   * the size of the window, not the age of the log.
+   */
+  readAfter?(
+    experienceId: string,
+    afterSequence: number,
+    limit?: number
+  ): Promise<readonly StoredLineageRecord[]>;
+  /**
+   * OPTIONAL — enumerate the ids of every line this store holds, in bounded pages. A collector
+   * (`SeenLog.collectGarbage`) needs it to find every root; a store that omits it cannot be
+   * garbage-collected (the collector refuses rather than guess a root set). Ids are exactly what
+   * `read` takes. Concurrent creation during a scan may be missed (a maintenance pass, not a snapshot).
+   */
+  listStreams?(): AsyncIterable<string>;
+}
+
+/**
+ * A lost slot, by class OR name (a store decorator or a second copy of the package throws an error
+ * that is not `instanceof` ours); mirrors the kernel's `isWriteConflict`.
+ */
+function isWriteConflict(err: unknown): boolean {
+  if (err instanceof DuplicateSequenceError || err instanceof DuplicateEnvelopeIdError) return true;
+  const name = (err as { name?: unknown } | null)?.name;
+  return name === "DuplicateSequenceError" || name === "DuplicateEnvelopeIdError";
+}
+
+type EventEnvelope = Awaited<ReturnType<EventStore["read"]>>[number];
+
+const clone = <T>(v: T): T => structuredClone(v);
+
+/** Reference in-memory store. Copies on the way in and out, so no caller can mutate the log. */
+export class InMemoryLineageStore implements LineageStore {
+  private readonly lines = new Map<string, StoredLineageRecord[]>();
+
+  async append(
+    experienceId: string,
+    expectedSequence: number,
+    record: LineageRecord
+  ): Promise<AppendOutcome> {
+    const line = this.lines.get(experienceId) ?? [];
+    if (line.length !== expectedSequence) return { ok: false, tip: line.length };
+    const sequence = line.length + 1;
+    line.push(clone({ ...record, sequence }));
+    this.lines.set(experienceId, line);
+    return { ok: true, sequence };
+  }
+
+  async read(experienceId: string): Promise<readonly StoredLineageRecord[]> {
+    return clone(this.lines.get(experienceId) ?? []);
+  }
+
+  async readTail(experienceId: string, limit: number): Promise<readonly StoredLineageRecord[]> {
+    const line = this.lines.get(experienceId) ?? [];
+    return clone(limit >= line.length ? line : line.slice(line.length - Math.max(0, limit)));
+  }
+
+  async readAfter(
+    experienceId: string,
+    afterSequence: number,
+    limit = Number.POSITIVE_INFINITY
+  ): Promise<readonly StoredLineageRecord[]> {
+    // dense 1..n: the records after `a` start at index `a`
+    const line = this.lines.get(experienceId) ?? [];
+    const from = Math.max(0, afterSequence);
+    return clone(line.slice(from, Number.isFinite(limit) ? from + Math.max(0, limit) : undefined));
+  }
+
+  async *listStreams(): AsyncGenerator<string, void, void> {
+    for (const id of [...this.lines.keys()]) yield id;
+  }
+}
+
+/** The reserved-stream namespace a lineage's events live under in an `EventStore`. */
+export const LINEAGE_STREAM_PREFIX = "__loom_ui_lineage__:";
+/** The `workflowId` a lineage event is written under unless `eventStoreLineageStore` is given another. */
+export const LINEAGE_WORKFLOW_ID = "loom-ui.lineage";
+const WORKFLOW_ID = LINEAGE_WORKFLOW_ID;
+
+/** Every stream prefix this package may write under; the kernel's own reserved prefixes are not ours. */
+export const LOOM_UI_STREAM_NAMESPACE = "__loom_ui_";
+const WORKFLOW_ID_SHAPE = /^[A-Za-z0-9._:-]{1,128}$/;
+
+/** A `LineageStore` that knows which stream prefix it writes under (see {@link eventStoreLineageStore}). */
+export interface PrefixedLineageStore extends LineageStore {
+  readonly streamPrefix: string;
+  /**
+   * The `workflowId` its events are written under. A `Lineage` passes it, with the stream id, as the
+   * `ContentAccess` of every artifact blob, so one host `subjectOf` covers events and blobs alike.
+   */
+  readonly workflowId?: string;
+}
+
+/**
+ * A `LineageStore` over loom's `EventStore`. One stream per experience
+ * (`__loom_ui_lineage__:<experienceId>`, a RESERVED stream so enumerators of real executions skip
+ * it; see the README for what a reserved stream costs); the lineage sequence IS the envelope sequence, so the CAS is the log's own
+ * `(executionId, sequence)` uniqueness. The adapter checks the tip first (a store enforces a
+ * taken slot, not a skipped one) and treats a `DuplicateSequenceError` from a racing writer as a
+ * lost CAS. Fencing, tenancy and sharding decorators wrap `store` as usual.
+ *
+ * `streamPrefix` must start with `__loom_ui_` (and say more): it is the namespace this package owns
+ * inside loom's reserved `__loom_` space, so a configuration can never write into the kernel's own
+ * control streams (`__loom_lifecycle__`, `__loom_erasure__`) or an execution's id space. `workflowId`
+ * is a short identifier. A bad value THROWS at construction (a misconfiguration, found at startup).
+ */
+export function eventStoreLineageStore(
+  store: EventStore,
+  opts: { readonly streamPrefix?: string; readonly workflowId?: string } = {}
+): PrefixedLineageStore {
+  const prefix = opts.streamPrefix ?? LINEAGE_STREAM_PREFIX;
+  const workflowId = opts.workflowId ?? WORKFLOW_ID;
+  if (
+    typeof prefix !== "string" ||
+    !prefix.startsWith(LOOM_UI_STREAM_NAMESPACE) ||
+    prefix.length <= LOOM_UI_STREAM_NAMESPACE.length ||
+    prefix.length > 128
+  )
+    throw new Error(
+      `streamPrefix must start with "${LOOM_UI_STREAM_NAMESPACE}" and name its stream (got ${JSON.stringify(String(prefix).slice(0, 40))})`
+    );
+  if (typeof workflowId !== "string" || !WORKFLOW_ID_SHAPE.test(workflowId))
+    throw new Error("workflowId must be 1-128 characters of [A-Za-z0-9._:-]");
+  const streamOf = (experienceId: string) => `${prefix}${experienceId}`;
+  return {
+    streamPrefix: prefix,
+    workflowId,
+    async append(experienceId, expectedSequence, record) {
+      const executionId = streamOf(experienceId);
+      const tip = (await headOf(store, executionId)) ?? 0;
+      if (tip !== expectedSequence) return { ok: false, tip };
+      const sequence = expectedSequence + 1;
+      try {
+        await store.append({
+          id: `${executionId}#${sequence}`,
+          workflowId,
+          executionId,
+          eventType: record.type,
+          sequence,
+          timestamp: Date.parse(record.at) || 0,
+          payload: clone(record.payload),
+          metadata: { at: record.at },
+        });
+      } catch (err) {
+        if (isWriteConflict(err))
+          return { ok: false, tip: (await headOf(store, executionId)) ?? sequence };
+        throw err;
+      }
+      return { ok: true, sequence };
+    },
+    async read(experienceId) {
+      return toRecords(await store.read(streamOf(experienceId)));
+    },
+    async readTail(experienceId, limit) {
+      const id = streamOf(experienceId);
+      const tip = (await headOf(store, id)) ?? 0;
+      if (tip === 0 || limit <= 0) return [];
+      return toRecords(await readWindow(store, id, Math.max(1, tip - limit + 1), tip));
+    },
+    async readAfter(experienceId, afterSequence, limit = Number.POSITIVE_INFINITY) {
+      const id = streamOf(experienceId);
+      const tip = (await headOf(store, id)) ?? 0;
+      const from = Math.max(1, afterSequence + 1);
+      const to = Number.isFinite(limit) ? Math.min(tip, from + Math.max(0, limit) - 1) : tip;
+      if (to < from) return [];
+      return toRecords(await readWindow(store, id, from, to));
+    },
+    ...(store.listExecutionsPage !== undefined || store.listExecutions !== undefined
+      ? {
+          async *listStreams(): AsyncGenerator<string, void, void> {
+            // Reserved streams ARE surfaced by the SPI's enumeration (`__loom_*`), so ours are found
+            // by prefix. Pages are keyset-ordered by code point, so with pages the scan starts just
+            // before our prefix and stops once past it; the unpaged fallback filters the full list.
+            if (store.listExecutionsPage === undefined) {
+              for (const id of (await store.listExecutions?.()) ?? [])
+                if (id.startsWith(prefix)) yield id.slice(prefix.length);
+              return;
+            }
+            let cursor: string | undefined = prefix.slice(0, -1);
+            for (;;) {
+              const page: { executionIds: string[]; nextCursor?: string } =
+                await store.listExecutionsPage({
+                  ...(cursor !== undefined && { cursor }),
+                  limit: 500,
+                });
+              for (const id of page.executionIds) {
+                if (id.startsWith(prefix)) yield id.slice(prefix.length);
+                else if (id > prefix) return;
+              }
+              if (page.nextCursor === undefined || page.nextCursor === cursor) return;
+              cursor = page.nextCursor;
+            }
+          },
+        }
+      : {}),
+  };
+}
+
+function toRecords(events: readonly EventEnvelope[]): StoredLineageRecord[] {
+  return events
+    .slice()
+    .sort((a, b) => a.sequence - b.sequence)
+    .map((e) => ({
+      sequence: e.sequence,
+      type: e.eventType,
+      payload: clone(e.payload),
+      at: typeof e.metadata?.at === "string" ? e.metadata.at : new Date(e.timestamp).toISOString(),
+    }));
+}
+
+/** Sequences `from..to` (inclusive) of a stream: the store's `readRange`, else `readFrom`/`read` filtered. */
+async function readWindow(
+  store: EventStore,
+  id: string,
+  from: number,
+  to: number
+): Promise<EventEnvelope[]> {
+  if (store.readRange !== undefined) return store.readRange(id, from, to);
+  return (await store.read(id)).filter((e) => e.sequence >= from && e.sequence <= to);
+}