@smartmemory/compose 0.3.6-beta → 0.3.8

package/lib/fluid/record-shape.js

@@ -0,0 +1,214 @@
+/**
+ * lib/fluid/record-shape.js — record rules that belong to the seam, not to a provider.
+ *
+ * Two rules lived as private members of `LocalFluidProvider` while it was the
+ * only provider, and the moment a second one exists they are drift generators:
+ *
+ *   1. `normalizeRecord()` — the contract-default filling and defensive cloning
+ *      that `_normalize` did (COMP-FOH C12).
+ *   2. `assertPatchable()` — the `UNPATCHABLE` refusal that `updateRecord` did
+ *      (COMP-FOH C16).
+ *
+ * WHY THESE ARE SHARED AND NOT COPIED
+ * -----------------------------------
+ * Both encode a promise the *seam* makes, not a choice a provider gets to make.
+ * `getRecord` returning a record whose optional fields are filled to contract
+ * defaults is what lets a caller read a record written before a field existed;
+ * `updateRecord` refusing to patch `discussion` is what makes the deliberation
+ * trail evidence rather than a mutable blob. A provider that implements either
+ * one differently is not a slower or simpler provider — it is a provider with a
+ * different contract, which defeats the point of the swap.
+ *
+ * The second copy is the dangerous one, because it is the one nobody re-reads.
+ * `local-provider.js` enforces `UNPATCHABLE` and says why in a comment that a
+ * new provider's author would never see.
+ *
+ * Pure functions only — no I/O, no store, no provider reference — so both
+ * providers and the tests can call them directly.
+ */
+
+import { KIND } from './provider.js';
+
+/** Handle prefix per record kind. Handles are quoted in docs and must be short
+ *  and stable; the prefix set is fixed here so two providers allocate the same
+ *  shape of handle for the same kind. */
+export const HANDLE_PREFIX = Object.freeze({
+  [KIND.IDEA]: 'IDEA',
+  [KIND.POSITION]: 'POS',
+  [KIND.JOINT]: 'JOINT',
+  [KIND.DECISION]: 'DEC',
+  [KIND.THREAD]: 'THREAD',
+  [KIND.QUESTION]: 'Q',
+  [KIND.CLUSTER]: 'CLUS',
+});
+
+/**
+ * Handle grammar, with the suffix bounded to 9 digits.
+ *
+ * The bound is load-bearing, not cosmetic. Allocation reads the suffix as a
+ * Number; past the exact-integer range `max + 1 === max`, so importing a
+ * gigantic handle would make the allocator hand the same one out forever. The
+ * contract carries the same bound.
+ */
+export const HANDLE_RE = /^([A-Z][A-Z0-9]*)-([1-9][0-9]{0,8})$/;
+
+/** Canonical fluid lifecycle states, per the contract's `status` enum. */
+export const FLUID_STATUSES = Object.freeze(new Set(['new', 'discussing', 'promoted', 'killed']));
+
+/** Link types the contract defines. Validated so a typo becomes an error at the
+ *  write rather than a silently unqueryable edge. */
+export const LINK_TYPES = Object.freeze(new Set([
+  'promoted_to', 'maps_to', 'informs', 'blocks',
+  'supports', 'contradicts', 'supersedes', 'duplicate_of',
+]));
+
+export function assertStatus(status) {
+  if (!FLUID_STATUSES.has(status)) {
+    throw new Error(
+      `fluid: invalid status "${status}" (expected one of: ${[...FLUID_STATUSES].join(', ')})`
+    );
+  }
+  return status;
+}
+
+/** A caller-supplied handle must be well-formed AND belong to its kind — an
+ *  `idea` carrying `DEC-4` would be invisible to idea handle allocation and
+ *  would collide with a real decision later. */
+export function assertHandle(handle, kind) {
+  const m = HANDLE_RE.exec(handle ?? '');
+  if (!m) {
+    throw new Error(`fluid: malformed handle "${handle}" (expected PREFIX-N, N >= 1)`);
+  }
+  const expected = HANDLE_PREFIX[kind];
+  if (m[1] !== expected) {
+    throw new Error(
+      `fluid: handle "${handle}" does not belong to kind "${kind}" (expected prefix ${expected})`
+    );
+  }
+  return handle;
+}
+
+export function assertLink(link) {
+  if (!link || !LINK_TYPES.has(link.type)) {
+    throw new Error(
+      `fluid: invalid link type "${link?.type}" (expected one of: ${[...LINK_TYPES].join(', ')})`
+    );
+  }
+  if (typeof link.target !== 'string' || !link.target) {
+    throw new Error('fluid: link requires a target');
+  }
+  return link;
+}
+
+/**
+ * Which lifecycle fact an update represents.
+ *
+ * Status transitions win over field edits because they are the events other
+ * rungs read (promotion and kill are the two outcomes a conviction or
+ * calibration layer scores against). Checked in that order so a promote that
+ * also retitles is still recorded as a promotion.
+ *
+ * Shared because the event stream is the thing a later semantic layer learns
+ * from: two providers that classified the same edit differently would make that
+ * history unreadable across a swap.
+ */
+export function eventTypeForUpdate(before, after, patch) {
+  if (after.status !== before.status) {
+    if (after.status === 'killed') return 'killed';
+    if (after.status === 'promoted') return 'promoted';
+  }
+  if (patch.priority !== undefined && after.priority !== before.priority) return 'triaged';
+  if (after.discussion.length > before.discussion.length) return 'discussed';
+  return 'updated';
+}
+
+/**
+ * Fields a caller may never patch through `updateRecord`.
+ *
+ * `handle`/`kind`/`id` are identity: `handle` is quoted in docs and commits, and
+ * `kind` additionally decides a record's storage type on providers that key
+ * behaviour off it — a provider whose backing store freezes the type at creation
+ * would diverge permanently from a record claiming a new one.
+ *
+ * `provenance` is write-time-stamped and never retrofitted. `discussion` is
+ * append-only evidence — a deliberation trail that can be replaced wholesale is
+ * not evidence, so it moves only through `appendDiscussion()`. Timestamps are
+ * provider-assigned.
+ */
+export const UNPATCHABLE = Object.freeze([
+  'handle', 'kind', 'provenance', 'discussion', 'id', 'created_at', 'updated_at',
+]);
+
+/**
+ * Refuse a patch that touches an unpatchable field.
+ *
+ * Refuses rather than silently dropping. Silently ignoring `discussion: []`
+ * would look to the caller exactly like a successful erase of the deliberation
+ * trail, and looking successful is the dangerous half.
+ *
+ * Keys are tested with `!== undefined`, so an explicit `null` is a violation
+ * too: `{ provenance: null }` is an attempt to clear provenance, not an absent
+ * field.
+ *
+ * @param {object} patch caller-supplied patch
+ * @param {string} providerName for the error message
+ * @throws {Error} naming every offending field at once, so a caller fixes one
+ *   call rather than discovering the list one rejection at a time.
+ */
+export function assertPatchable(patch, providerName) {
+  const forbidden = UNPATCHABLE.filter((f) => patch?.[f] !== undefined);
+  if (forbidden.length) {
+    throw new Error(
+      `fluid: field(s) ${forbidden.join(', ')} cannot be changed through updateRecord ` +
+      `(identity and append-only evidence are not patchable; use appendDiscussion for discussion)` +
+      (providerName ? ` [${providerName}]` : '')
+    );
+  }
+}
+
+/**
+ * Normalize a record on the way out.
+ *
+ * Every structured field is cloned. Returning stored objects by reference lets
+ * a caller mutate what the provider considers canonical with no write, no
+ * timestamp, no event and no save — a change that appeared to take effect and
+ * then silently vanished. Nothing shares a reference with the store.
+ *
+ * Absent optional fields are filled to their contract defaults rather than left
+ * `undefined`, so a record written before a field existed reads back the same
+ * shape as one written after it. This is also what makes a record recovered
+ * from a store that erases nulls and empty containers (COMP-FOH C14) read back
+ * intact, without a migration.
+ *
+ * @param {object} record raw stored record
+ * @returns {object} a fresh object sharing no references with `record`
+ */
+export function normalizeRecord(record) {
+  return {
+    id: record.id,
+    handle: record.handle,
+    kind: record.kind,
+    title: record.title,
+    body: record.body ?? '',
+    status: record.status,
+    status_label: record.status_label ?? null,
+    priority: record.priority ?? null,
+    // The 2x2 matrix's two axes. Filled to null here rather than left absent for
+    // the reason stated above: every record written before COMP-PLAN-IDEA-UNIFY
+    // S3b-2 predates both fields, and a caller that has to distinguish "absent"
+    // from "unassigned" would be reading the migration's timeline out of the
+    // record shape.
+    effort: record.effort ?? null,
+    impact: record.impact ?? null,
+    cluster: record.cluster ?? null,
+    cluster_order: record.cluster_order ?? null,
+    tags: Array.isArray(record.tags) ? [...record.tags] : [],
+    source: record.source ?? null,
+    links: Array.isArray(record.links) ? record.links.map((l) => ({ ...l })) : [],
+    killed: record.killed ? { ...record.killed } : null,
+    discussion: Array.isArray(record.discussion) ? record.discussion.map((d) => ({ ...d })) : [],
+    provenance: { ...record.provenance },
+    created_at: record.created_at,
+    updated_at: record.updated_at,
+  };
+}

package/lib/fluid/record-store.js

@@ -0,0 +1,328 @@
+/**
+ * lib/fluid/record-store.js — durable, git-tracked persistence for the floor
+ * provider.
+ *
+ * WHY THIS EXISTS (S3 entry-gate ruling, 2026-08-04)
+ * --------------------------------------------------
+ * S1 hosted records on vision-store items, in an additive `fluid_ext`
+ * namespace. That was the right call for a seam pilot and the wrong one for
+ * durability: `.compose/data/vision-state.json` is gitignored (`.gitignore:3`
+ * matches `data/`), so wiring the CLI, the API and the UI onto the provider
+ * would have moved idea canon from a tracked file to an untracked one on a
+ * single machine — and committed a GENERATED `ideabox.md` with no source of
+ * truth behind it on any other clone or in CI.
+ *
+ * The owner ruled: track the records, split them out of vision-state. That is
+ * not merely a path change. It inverts which side is canon:
+ *
+ *     BEFORE  vision item = canon,   nothing tracked
+ *     AFTER   record file = canon,   vision item = optional projection
+ *
+ * Vision-state could not simply be un-ignored: it is ~540KB of runtime state
+ * (gates, connections, 465 items) rewritten on every save, so tracking it would
+ * mean an unreadable diff and a merge conflict per parallel session. Records
+ * need the opposite properties, so they get their own files.
+ *
+ * LAYOUT — mirrors `docs/judgment/`, deliberately
+ * -----------------------------------------------
+ * The judgment layer already solved "tool-owned canon plus a generated markdown
+ * projection" and is the house convention; a third pattern here would be drift
+ * for its own sake.
+ *
+ *     docs/judgment/records/joints/*.json   ↔   <root>/records/<HANDLE>.json
+ *     docs/judgment/records/ledger.jsonl    ↔   <root>/events.jsonl
+ *     docs/judgment/REGISTER.md (generated) ↔   docs/product/ideabox.md
+ *
+ * ONE FILE PER RECORD, not one array file. A whole-array rewrite makes every
+ * `ideabox add` a diff against all records and a merge conflict between two
+ * clones that each added an idea. Per-record files make an add a pure file
+ * creation. The handle is the filename because it is unique, never reused
+ * (retired handles stay retired — see the events log), and already constrained
+ * to a filesystem-safe grammar by the contract.
+ *
+ * NO IN-MEMORY SNAPSHOT. Every read hits the directory. This is what retires
+ * the S1 `_sync()` hazard rather than merely narrowing it: there is no cached
+ * state for a second writer to invalidate, so the CLI and a running server
+ * cannot erase each other's records by saving a stale view. Interleaving is
+ * still unserialized — see the known gap at the foot of this file.
+ */
+
+import {
+  appendFileSync,
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  readdirSync,
+  renameSync,
+  rmSync,
+  writeFileSync,
+} from 'node:fs';
+import { join, resolve } from 'node:path';
+import { randomUUID } from 'node:crypto';
+
+/** Tracked root, relative to the project. Under `docs/` on purpose: this is
+ *  product canon that belongs in review and in git history, not runtime state.
+ *  It sits beside its own projection (`docs/product/ideabox.md`). */
+export const DEFAULT_RECORDS_ROOT = join('docs', 'product', 'fluid');
+
+const RECORDS_DIR = 'records';
+const EVENTS_FILE = 'events.jsonl';
+
+/**
+ * Filenames are handles, so the grammar is enforced on the way to disk as well
+ * as at the seam. A handle is interpolated into a path; anything carrying a
+ * separator or a traversal segment must never reach `join()`. The seam already
+ * validates handles, but a store that writes wherever it is pointed is one
+ * caller away from writing outside its root.
+ */
+const HANDLE_RE = /^[A-Z][A-Z0-9]*-[1-9][0-9]{0,8}$/;
+
+function assertHandleSafe(handle) {
+  if (typeof handle !== 'string' || !HANDLE_RE.test(handle)) {
+    throw new Error(`fluid: unsafe record handle "${handle}" — refusing to build a path from it`);
+  }
+  return handle;
+}
+
+/**
+ * Scans RAW log text for handle tokens, independent of JSON parseability.
+ *
+ * Carried over from S1 unchanged, and for the same reason: the watermark must
+ * survive a corrupt or partially-written line. If the only tombstone for a
+ * retired handle were lost to a parse failure, that handle would be reissued,
+ * and handles are external citations — reuse silently repoints a citation in a
+ * doc or a commit at a different idea. Tolerant history reading and durable
+ * handle retirement are separate jobs, so they read the log differently.
+ */
+const HANDLE_TOKEN_RE = /"handle"\s*:\s*"([A-Z][A-Z0-9]*-[1-9][0-9]{0,8})"/g;
+
+/**
+ * Durable per-record storage plus the append-only lifecycle log.
+ *
+ * Deliberately dumb: no validation of record CONTENT (the provider validates
+ * against `contracts/fluid-record.schema.json` before handing anything over),
+ * no lifecycle rules, no handle allocation. It knows about paths, atomicity and
+ * durability, and nothing else.
+ */
+export class FluidRecordStore {
+  /**
+   * @param {string} cwd project root
+   * @param {object} [config]
+   * @param {string} [config.recordsRoot] absolute or project-relative override
+   *   (tests point this at a tmp dir)
+   */
+  constructor(cwd, config = {}) {
+    this.cwd = cwd;
+    const root = config.recordsRoot ?? DEFAULT_RECORDS_ROOT;
+    // resolve(), not join(): an absolute override is documented and a plain
+    // join turns `/tmp/fluid` into `<cwd>/tmp/fluid`, silently writing records
+    // somewhere the caller never named.
+    this.root = resolve(cwd, root);
+    this.recordsDir = join(this.root, RECORDS_DIR);
+    this.eventsPath = join(this.root, EVENTS_FILE);
+  }
+
+  _pathFor(handle) {
+    return join(this.recordsDir, `${assertHandleSafe(handle)}.json`);
+  }
+
+  // -------------------------------------------------------------------------
+  // Records
+  // -------------------------------------------------------------------------
+
+  /**
+   * Write a record atomically.
+   *
+   * tmp + rename, matching `VisionStore._save()`. A half-written record file is
+   * worse here than in a single state file: it is not a corrupt blob you notice
+   * on load, it is one idea that silently fails to parse while its 19 siblings
+   * load fine. The rename makes the file appear whole or not at all.
+   *
+   * The tmp name carries a random suffix rather than a timestamp so two writers
+   * in the same millisecond cannot land on the same tmp path and interleave
+   * their bytes.
+   */
+  write(record) {
+    const path = this._pathFor(record.handle);
+    mkdirSync(this.recordsDir, { recursive: true });
+    // Pretty-printed with a trailing newline: these files are reviewed as
+    // diffs. Key order is deterministic because the provider builds every
+    // record through one object literal and a re-read preserves that order —
+    // NOT via a sorted replacer array, which would also filter nested objects
+    // and silently drop `provenance` and every link field.
+    const data = JSON.stringify(record, null, 2) + '\n';
+    const tmp = `${path}.tmp.${randomUUID()}`;
+    try {
+      writeFileSync(tmp, data, 'utf8');
+      renameSync(tmp, path);
+    } catch (err) {
+      // A leftover tmp file is invisible to read() (it does not end in .json
+      // alone — see list()), but leaving litter in tracked canon would show up
+      // in `git status` and in review.
+      try { rmSync(tmp, { force: true }); } catch { /* best effort */ }
+      throw new Error(`fluid: failed to persist record ${record.handle} — ${err.message}`);
+    }
+    return record;
+  }
+
+  /** @returns {object|null} the record, or null when absent. */
+  read(handle) {
+    const path = this._pathFor(handle);
+    if (!existsSync(path)) return null;
+    let record;
+    try {
+      record = JSON.parse(readFileSync(path, 'utf8'));
+    } catch (err) {
+      // Loud, not null. Returning null for an unparseable file would make a
+      // corrupt record indistinguishable from a deleted one, and the caller's
+      // next move on "absent" is to allocate the handle again or report it
+      // missing — both of which quietly destroy the damaged record.
+      throw new Error(`fluid: record file for ${handle} is unreadable — ${err.message}`);
+    }
+    return this._assertIdentity(record, handle, path);
+  }
+
+  /**
+   * The filename and the record's own `handle` must agree.
+   *
+   * These are git-tracked files a human can open, and every write path routes
+   * through `record.handle` to choose its destination. So a file whose name and
+   * contents disagree is not a cosmetic inconsistency: reading `IDEA-1.json`
+   * that contains `"handle": "IDEA-2"` and then saving it writes IDEA-2.json,
+   * destroying a DIFFERENT record while leaving the malformed one untouched.
+   *
+   * Cheap to check, and the check has to live here because this is the only
+   * layer that knows both the name asked for and the bytes on disk.
+   */
+  _assertIdentity(record, handle, path) {
+    if (record?.handle !== handle) {
+      throw new Error(
+        `fluid: record file ${path} declares handle "${record?.handle}" but is named ` +
+        `"${handle}" — refusing to read it, because writing it back would overwrite ` +
+        `a different record`
+      );
+    }
+    return record;
+  }
+
+  /**
+   * Every record on disk.
+   *
+   * Sorted by filename so the enumeration is deterministic regardless of
+   * `readdirSync` order, which is filesystem-dependent. The provider re-sorts
+   * for presentation; this only guarantees the input to that sort is stable, so
+   * a regenerated projection does not churn on an unrelated machine.
+   *
+   * ONE CORRUPT FILE FAILS THE WHOLE LIST, deliberately. Skipping it would be
+   * the friendlier-looking choice and the destructive one: this list is what
+   * regenerates `ideabox.md`, so a silently omitted record becomes an idea
+   * deleted from tracked markdown by a routine write. Refusing to list at all
+   * is recoverable; a projection that quietly drops an idea is not.
+   */
+  list() {
+    if (!existsSync(this.recordsDir)) return [];
+    const names = readdirSync(this.recordsDir)
+      .filter((n) => n.endsWith('.json') && HANDLE_RE.test(n.slice(0, -'.json'.length)))
+      .sort();
+    return names.map((n) => {
+      const handle = n.slice(0, -'.json'.length);
+      // read() enforces filename↔handle identity, so a mismatched file fails
+      // the whole list rather than being projected under the wrong identity.
+      const record = this.read(handle);
+      if (!record) {
+        // Raced against a delete between readdir and read. Not an error; the
+        // record is genuinely gone.
+        return null;
+      }
+      return record;
+    }).filter(Boolean);
+  }
+
+  /** Handles of live records only. Retired handles live in the log. */
+  liveHandles() {
+    if (!existsSync(this.recordsDir)) return new Set();
+    const out = new Set();
+    for (const n of readdirSync(this.recordsDir)) {
+      if (!n.endsWith('.json')) continue;
+      const handle = n.slice(0, -'.json'.length);
+      if (HANDLE_RE.test(handle)) out.add(handle);
+    }
+    return out;
+  }
+
+  /** @returns {boolean} whether a file was removed. */
+  remove(handle) {
+    const path = this._pathFor(handle);
+    if (!existsSync(path)) return false;
+    rmSync(path);
+    return true;
+  }
+
+  // -------------------------------------------------------------------------
+  // Events — append-only
+  // -------------------------------------------------------------------------
+
+  _rawLog() {
+    return existsSync(this.eventsPath) ? readFileSync(this.eventsPath, 'utf8') : '';
+  }
+
+  /**
+   * Append one event.
+   *
+   * The log is the tombstone ledger: a `created` entry burns its handle
+   * permanently, which is what stops a deleted record's handle being handed to
+   * a different idea later. It is therefore tracked alongside the records
+   * rather than left in the ignored runtime directory — losing it does not lose
+   * history, it loses the guarantee.
+   */
+  appendEvent(event) {
+    mkdirSync(this.root, { recursive: true });
+    appendFileSync(this.eventsPath, JSON.stringify(event) + '\n', 'utf8');
+    return { ok: true };
+  }
+
+  /**
+   * Handles named anywhere in the raw log, including on lines that fail to
+   * parse. See HANDLE_TOKEN_RE.
+   */
+  issuedHandlesFromLog() {
+    const handles = new Set();
+    const text = this._rawLog();
+    HANDLE_TOKEN_RE.lastIndex = 0;
+    let m;
+    while ((m = HANDLE_TOKEN_RE.exec(text)) !== null) handles.add(m[1]);
+    return handles;
+  }
+
+  /** Parsed history, skipping unparseable lines. Handle retirement never
+   *  depends on this path — see issuedHandlesFromLog. */
+  readEvents() {
+    const out = [];
+    for (const line of this._rawLog().split('\n')) {
+      const trimmed = line.trim();
+      if (!trimmed) continue;
+      try { out.push(JSON.parse(trimmed)); } catch { /* a corrupt line must not
+        take the readable history with it */ }
+    }
+    return out;
+  }
+}
+
+/*
+ * KNOWN GAP, accepted for the floor (unchanged from S1, narrowed by this slice)
+ * ----------------------------------------------------------------------------
+ * There is still no lock. Two processes writing the SAME handle concurrently
+ * can interleave read-modify-write, and last writer wins.
+ *
+ * What this slice did fix is the worse failure it used to sit behind: with a
+ * shared state file and an in-memory snapshot, a stale writer saving unrelated
+ * work erased records it had never read. Per-record files plus read-per-
+ * operation mean a concurrent write can now only lose an update to the ONE
+ * record being contended, never to its siblings.
+ *
+ * Handle allocation is the remaining sharp edge: two creates racing can compute
+ * the same next handle. The pre-write tombstone narrows the window but does not
+ * close it. Serializing that needs a lock, and `.compose/locks/` already exists
+ * for exactly this — it is S3b's job, when the CLI, the API and the UI are all
+ * actually writing.
+ */