@smartmemory/compose 0.4.1 → 0.5.1

package/lib/fluid/ideabox-migrate.js

@@ -34,9 +34,16 @@
 import { existsSync, readFileSync } from 'node:fs';
 
 import { parseIdeabox } from '../ideabox.js';
+import { hashMarkdown, readManifest } from './ideabox-manifest.js';
+// FU-2: the readability check lives in its own leaf module so the PARSER's own
+// module can import it without closing a cycle through this one. Re-exported
+// here because this was its address for its whole life so far.
+import { IdeaboxUnreadable, assertIdeaboxReadable } from './ideabox-readable.js';
 import { importIdeabox } from './import-ideabox.js';
 import { KIND } from './provider.js';
 
+export { IdeaboxUnreadable, assertIdeaboxReadable };
+
 export class IdeaboxMigrationConflict extends Error {
   constructor(missing, ideaboxPath) {
     super(
@@ -54,43 +61,129 @@ export class IdeaboxMigrationConflict extends Error {
 }
 
 /**
- * Ensure the record store reflects the markdown before any mutation touches it.
+ * An interrupted migration whose source document has changed underneath it.
  *
- * Runs before every mutating ideabox command. Three states, one of which stops
- * the command:
+ * Distinct from a conflict: the entries are ones this store PLANNED to write,
+ * so they are not strays — but the plan was made against a different version of
+ * the file, and importing the current one would import a document nobody
+ * checked. FU-1.
+ */
+export class IdeaboxManifestStale extends Error {
+  constructor(ideaboxPath) {
+    super(
+      `compose: the ideabox at ${ideaboxPath} has an unfinished migration recorded against a ` +
+      `DIFFERENT version of this file — the document was most likely edited while the migration ` +
+      `was running, or after it was interrupted. Resuming would import a version nobody checked, ` +
+      `and the entries added since would be projected away. Nothing has been changed. Decide which ` +
+      `document is right and say so, with one of:\n` +
+      `  compose ideabox adopt-file      the file on disk is right — finish the migration against ` +
+      `it, updating what was already imported to match and importing the rest\n` +
+      `  compose ideabox discard-edits   the migration is right — put the file back as it was read ` +
+      `and finish it (a copy of the current file is saved first, and its path printed)\n` +
+      `Deleting the record of the unfinished migration is NOT a shortcut: it makes the partially ` +
+      `migrated store canon, and the next render replaces this file with a projection of it.`
+    );
+    this.name = 'IdeaboxManifestStale';
+    this.code = 'IDEABOX_MANIFEST_STALE';
+  }
+}
+
+/**
+ * The source document changed between the gate's check and the projection's
+ * write. FU-3.
  *
- *   - **no records, markdown has entries** → import it (the upgrade path)
- *   - **records exist, markdown adds nothing** → already migrated, proceed
- *   - **records exist, markdown has entries with no record** → refuse
+ * The remedy is genuinely different from a conflict's — nothing is wrong with
+ * the file or the store, the two writers simply interleaved — so it does not
+ * reuse `IdeaboxMigrationConflict`, whose text tells the user to re-add ideas
+ * by hand.
+ */
+export class IdeaboxChangedUnderLock extends Error {
+  constructor(ideaboxPath, reason) {
+    super(
+      `compose: the ideabox at ${ideaboxPath} changed between the check and the write ` +
+      `(${reason}), so the projection was abandoned rather than replacing content that was ` +
+      `never checked. Nothing has been changed. Run the command again.`
+    );
+    this.name = 'IdeaboxChangedUnderLock';
+    this.code = 'IDEABOX_CHANGED_UNDER_LOCK';
+    this.reason = reason;
+  }
+}
+
+/**
+ * PURE. What the gate WOULD do, computed without doing any of it.
  *
- * A fresh project with no markdown and no records is the trivial first case and
- * simply proceeds.
+ * Split out of `ensureIdeaboxMigrated` for FU-3: the projection has to re-run
+ * this decision INSIDE its write lock, and it can only do that if making the
+ * decision writes nothing and takes no lock. Reads exactly three things —
+ * `listRecords`, `readEvents`, the markdown — none of which lock, so running it
+ * under the projection lock cannot deadlock the way moving the whole gate under
+ * it does (measured: a full 30s lock timeout, see `render-ideabox.js`).
  *
- * @param {import('./provider.js').FluidProvider} provider
- * @param {string} ideaboxPath absolute path to the markdown ideabox
- * @returns {Promise<{migrated: boolean, imported: string[]}>}
+ * Refusals are RETURNED, not thrown, because a decision procedure that throws
+ * cannot be used to ask a question.
+ *
+ * @returns {Promise<{action: 'none'|'import', markdown?: string} |
+ *                   {action: 'refuse', reason: string, error: Error}>}
  */
-export async function ensureIdeaboxMigrated(provider, ideaboxPath) {
+export async function assessIdeabox(provider, ideaboxPath) {
   const records = await provider.listRecords({ kind: KIND.IDEA });
   const markdown = existsSync(ideaboxPath) ? readFileSync(ideaboxPath, 'utf8') : null;
 
-  if (markdown === null) return { migrated: false, imported: [] };
+  if (markdown === null) return { action: 'none' };
 
   const parsed = parseIdeabox(markdown);
   const inMarkdown = [...(parsed.ideas ?? []), ...(parsed.killed ?? [])].map((i) => i.id);
 
+  try {
+    assertIdeaboxReadable(parsed, ideaboxPath);
+  } catch (error) {
+    return { action: 'refuse', reason: 'unreadable', error };
+  }
+
   const known = new Set(records.map((r) => r.handle));
   const missing = inMarkdown.filter((id) => !known.has(id));
 
   if (records.length === 0) {
-    if (inMarkdown.length === 0) return { migrated: false, imported: [] };
+    if (inMarkdown.length === 0) return { action: 'none' };
     // The upgrade path.
-    const result = await importIdeabox(provider, { markdown, path: ideaboxPath });
-    return { migrated: true, imported: result.imported };
+    return { action: 'import', markdown };
+  }
+
+  // AN OPEN MANIFEST MEANS THE MIGRATION NEVER FINISHED, and that is a fact
+  // about the DOCUMENT, not only about the handles that happen to be absent.
+  //
+  // Scoping this to the missing-handles branch was wrong in a way the ID
+  // comparison hides: an edit to an idea that ALREADY has a record changes no
+  // ID, so nothing is missing, and the gate waved a file through that the
+  // projection then rewrote from the version read at the start of the import.
+  // Until the migration completes this file is still the SOURCE document, so
+  // the question here is whether it is the same document, not whether its ids
+  // line up. (After the migration there is no manifest, so `render` still
+  // discards hand edits to what is by then generated output — that contract is
+  // untouched.)
+  const manifest = readManifest(provider, ideaboxPath);
+  const planned = new Set();
+  if (manifest) {
+    if (manifest.hash !== hashMarkdown(markdown)) {
+      // Planned against a different document. Every handle it names is suspect,
+      // and so is every handle in the file now.
+      return {
+        action: 'refuse',
+        reason: 'manifest-stale',
+        error: new IdeaboxManifestStale(ideaboxPath),
+      };
+    }
+    for (const handle of manifest.planned) planned.add(handle);
+    // Same document, unfinished import. Finishing it is a no-op for handles
+    // that already landed, and it is what closes the manifest — so a crash
+    // between the last write and the close heals on the next command instead of
+    // leaving the store in a state every later check has to reason about.
+    if (missing.length === 0) return { action: 'import', markdown };
   }
 
   if (missing.length) {
-    // RESUME versus REFUSE, decided PER HANDLE from the events log.
+    // RESUME versus REFUSE, decided PER HANDLE.
     //
     // A crash partway through the first-use import leaves some records written
     // and the rest missing, which lands here rather than in the empty-store
@@ -99,29 +192,84 @@ export async function ensureIdeaboxMigrated(provider, ideaboxPath) {
     // fails and there is no way out — and the reclaim path built for exactly
     // this case is never reached.
     //
-    // The log distinguishes the two populations. A handle the import already
-    // burned carries an event; `importIdeabox` skips live records and reclaims
-    // its own aborted allocations, so resuming is safe and lossless.
-    //
     // The evidence has to be per-handle, not "did an import ever run". Once the
     // first import succeeds the log carries `imported` events forever, so a
     // global check would quietly import anything later hand-added to what is
     // now generated output — losing the very protection this gate exists for.
-    // A handle with no event was never issued here: it was typed into the file
-    // by hand, and importing it would treat the markdown as authoritative when
-    // it no longer is.
-    // Three populations, and only one of them is resumable:
+    // A handle with no event and no manifest entry was never issued here: it
+    // was typed into the file by hand, and importing it would treat the
+    // markdown as authoritative when it no longer is.
+    //
+    // Two sources of evidence, because neither alone covers the whole failure:
+    //
+    //   - THE EVENT LOG covers handles the import REACHED. `importIdeabox`
+    //     skips live records and reclaims its own aborted allocations, so a
+    //     handle that was issued and not deleted is a create that crashed, and
+    //     resuming it is safe and lossless.
+    //   - THE MANIFEST covers the handles it never reached. An import that dies
+    //     on idea 2 leaves ideas 3..N with no event anywhere, and no amount of
+    //     reading the log afterwards can distinguish them from strays — the
+    //     intention has to have been written down first (FU-1).
+    //
+    // Three populations, and only two of them are resumable:
     //   - issued, no `deleted` event  → a create that crashed. RESUME.
+    //   - planned by an OPEN manifest whose hash still matches this file
+    //                                 → never attempted. RESUME.
     //   - issued, `deleted` event     → deliberately retired; this file is just
     //                                   stale output. REFUSE (a render fixes it,
     //                                   and importing would resurrect it).
-    //   - never issued               → hand-typed into generated output. REFUSE.
+    //   - neither                     → hand-typed into generated output. REFUSE.
     const { issued, deleted } = await handleHistory(provider);
-    const resumable = missing.filter((id) => issued.has(id) && !deleted.has(id));
+
+    const resumable = missing.filter(
+      (id) => (issued.has(id) && !deleted.has(id)) || (planned.has(id) && !deleted.has(id)),
+    );
     const strays = missing.filter((id) => !resumable.includes(id));
-    if (strays.length) throw new IdeaboxMigrationConflict(strays, ideaboxPath);
+    if (strays.length) {
+      return {
+        action: 'refuse',
+        reason: 'stray-entries',
+        error: new IdeaboxMigrationConflict(strays, ideaboxPath),
+      };
+    }
+
+    return { action: 'import', markdown };
+  }
+
+  return { action: 'none' };
+}
+
+/**
+ * Ensure the record store reflects the markdown before any mutation touches it.
+ *
+ * Runs before every mutating ideabox command. Three states, one of which stops
+ * the command:
+ *
+ *   - **no records, markdown has entries** → import it (the upgrade path)
+ *   - **records exist, markdown adds nothing** → already migrated, proceed
+ *   - **records exist, markdown has entries with no record** → refuse
+ *
+ * A fresh project with no markdown and no records is the trivial first case and
+ * simply proceeds.
+ *
+ * Assess, then act. The decision is `assessIdeabox` and this is the only thing
+ * that executes it; the split exists so the projection can ask the same
+ * question under its lock (FU-3) without re-implementing the answer.
+ *
+ * @param {import('./provider.js').FluidProvider} provider
+ * @param {string} ideaboxPath absolute path to the markdown ideabox
+ * @returns {Promise<{migrated: boolean, imported: string[]}>}
+ */
+export async function ensureIdeaboxMigrated(provider, ideaboxPath) {
+  const assessment = await assessIdeabox(provider, ideaboxPath);
+
+  if (assessment.action === 'refuse') throw assessment.error;
 
-    const result = await importIdeabox(provider, { markdown, path: ideaboxPath });
+  if (assessment.action === 'import') {
+    const result = await importIdeabox(provider, {
+      markdown: assessment.markdown,
+      path: ideaboxPath,
+    });
     return { migrated: true, imported: result.imported };
   }
 

package/lib/fluid/ideabox-preamble.js

@@ -0,0 +1,155 @@
+/**
+ * lib/fluid/ideabox-preamble.js — a project's own heading and introduction,
+ * kept beside the projection rather than inside it.
+ *
+ * COMP-IDEABOX-MIGRATE-DIALECT FU-4.
+ *
+ * THE DEFECT
+ * ----------
+ * `renderIdeabox` emits a hardcoded template preamble, so the first projection
+ * after a migration replaced a project's own title and introductory prose with
+ * the standard one. Ideas, clusters, custom fields and bodies all survived; the
+ * document AROUND them did not. Measured on forge-top: `# Forge Ideabox` and
+ * its two-line introduction were destroyed.
+ *
+ * WHY A SIDECAR AND NOT A RECORD
+ * ------------------------------
+ * The obvious fix — carry the preamble forward from the file being replaced —
+ * was implemented and rejected, because it makes the DESTINATION authoritative
+ * and `render` is documented as the way back from any hand edit
+ * (`lib/ideabox-cli.js`). A projection that reads its own destination cannot
+ * repair it.
+ *
+ * The alternative was to make the preamble a record. The owner rejected that:
+ * it means a new ontology type in every SmartMemory tenant and a decision about
+ * what a remote store does with per-document text. The preamble is not a
+ * property of the idea corpus at all — it is a property of THIS FILE, and the
+ * file is local no matter which provider holds the records.
+ *
+ * The projection therefore stays independent of its destination: it is rendered
+ * from the records plus this sidecar, and a hand edit to the file's heading is
+ * still discarded by the next render, exactly as a hand edit to an idea is.
+ *
+ * WHY IT IS TRACKED, AND NOT BESIDE THE MANIFEST
+ * ----------------------------------------------
+ * The first version of this put the sidecar in gitignored `.compose/data/`,
+ * next to the migration manifest, keyed off `provider.lockPath`. That was
+ * wrong twice.
+ *
+ * It made a TRACKED projection depend on UNTRACKED input. The heading then
+ * survived only on the machine that ran the migration: any other clone — a
+ * teammate, CI — found no sidecar, rendered the template over the custom
+ * heading and committed that, and the migrating machine restored it on its next
+ * render. FU-4's own defect, recurring on every clone, plus git churn on a
+ * tracked file. This is the exact failure the owner already ruled on: records
+ * were moved OUT of gitignored `vision-state.json` in the S3 entry-gate ruling
+ * of 2026-08-04 (see the header of `local-provider.js`) because canon that a
+ * tracked file is generated from cannot itself be ignored. The manifest is a
+ * fair neighbour for none of this: it is TRANSIENT, alive only between the
+ * start and the end of one import, while the preamble is durable content with
+ * the same lifecycle as the records.
+ *
+ * And keying off `provider.lockPath` excluded SmartMemory, which has no lock —
+ * the very provider whose existence was the argument for a local sidecar rather
+ * than a record. Keyed off the ideabox path instead, every provider gets one.
+ *
+ * Beside the document it describes, it is also discoverable: the hand-written
+ * recovery below is a file someone can find, not a hashed name under a hidden
+ * directory. A deliberately named, deliberately committed file is not the stray
+ * temp file that tracked directories have to be protected from.
+ */
+
+import { randomUUID } from 'node:crypto';
+import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
+import { dirname } from 'node:path';
+
+/**
+ * The preamble file for this ideabox: a tracked sibling of it.
+ *
+ * `docs/product/ideabox.md` → `docs/product/ideabox.preamble.md`. Derived from
+ * the ideabox path alone, so it does not depend on which provider holds the
+ * records and every provider gets one. Plain markdown rather than JSON so a
+ * project that has to write one by hand can just write it.
+ */
+export function preamblePath(ideaboxPath) {
+  if (!ideaboxPath) return null;
+  return String(ideaboxPath).replace(/(\.md)?$/i, '.preamble.md');
+}
+
+/**
+ * The generated banner is not part of anyone's introduction.
+ *
+ * A document being imported is normally a legacy one and carries no banner, but
+ * a store whose records were removed re-imports its own PROJECTION — and the
+ * parser hands back everything before `## Ideas`, banner included. Capturing
+ * that would make the next render emit the banner twice, and the one after that
+ * three times.
+ */
+function withoutBanner(text) {
+  const lines = String(text ?? '').split('\n');
+  if (!/^\s*<!--/.test(lines[0] ?? '') || !/GENERATED FILE/.test(lines[0] ?? '')) return text;
+  const end = lines.findIndex((l) => /-->/.test(l));
+  if (end === -1) return text;
+  return lines.slice(end + 1).join('\n').replace(/^\n+/, '');
+}
+
+/**
+ * Capture the source document's own preamble.
+ *
+ * @param {string} ideaboxPath
+ * @param {string} text `parseIdeabox(...).preamble` — the parser's own output,
+ *   never a second scan of the same document. Two readers of one document is
+ *   the exact shape that produced this whole bug class.
+ * @returns {string|null} the path written, or null when there is nothing to
+ *   write or nowhere to write it
+ */
+export function writePreamble(ideaboxPath, text) {
+  const path = preamblePath(ideaboxPath);
+  if (!path) return null;
+  const body = withoutBanner(text);
+  if (!body || !body.trim()) return null;
+  // Nothing to do when it already says this. Same reasoning as the manifest: a
+  // rewrite that changes nothing is a window in which a crash can lose
+  // something, for no gain.
+  if (readPreamble(ideaboxPath) === body) return path;
+
+  mkdirSync(dirname(path), { recursive: true });
+  const tmp = `${path}.tmp.${randomUUID()}`;
+  try {
+    writeFileSync(tmp, body, 'utf8');
+    renameSync(tmp, path);
+  } catch (err) {
+    rmSync(tmp, { force: true });
+    throw err;
+  }
+  return path;
+}
+
+/**
+ * The captured preamble, or null.
+ *
+ * Null is the ordinary answer for a project that migrated before this existed,
+ * and it means the renderer falls back to the standard template — which is what
+ * that project already has in its file, so nothing changes for it.
+ */
+export function readPreamble(ideaboxPath) {
+  const path = preamblePath(ideaboxPath);
+  if (!path || !existsSync(path)) return null;
+  try {
+    const text = readFileSync(path, 'utf8');
+    if (!text.trim()) return null;
+    // Trailing blank lines are structural, not content — the renderer puts its
+    // own separator before `## Ideas`, and the parser pops them on the way back
+    // (`lib/ideabox.js`), so leaving one here costs the fixed point that the
+    // whole cutover rests on. `writePreamble` never stores one because it
+    // stores what the parser already popped; a sidecar written BY HAND does,
+    // because every editor ends a file with a newline — and writing one by hand
+    // is the documented recovery for a project that migrated before this
+    // existed.
+    const lines = text.split('\n');
+    while (lines.length && lines.at(-1).trim() === '') lines.pop();
+    return lines.length ? lines.join('\n') : null;
+  } catch {
+    return null;
+  }
+}

package/lib/fluid/ideabox-readable.js

@@ -0,0 +1,83 @@
+/**
+ * lib/fluid/ideabox-readable.js — ONE readability check, for every caller.
+ *
+ * COMP-IDEABOX-MIGRATE-DIALECT FU-2.
+ *
+ * The original bug was a parse that failed being read as "this file has no
+ * ideas", after which the next projection write replaced the file with a
+ * projection that did not contain them — 18 ideas destroyed by one command.
+ * The fix for that bug put the readability check in the MIGRATION GATE. That
+ * guarded the destructive path but not the parse: every other caller of
+ * `parseIdeabox` still read failure as absence.
+ *
+ * This module is the check itself, extracted so there is one implementation
+ * instead of one per caller. It is a LEAF on purpose: it takes an already
+ * parsed document and knows nothing about providers, stores or files.
+ * `lib/ideabox.js` (the parser) has to be able to import it, and
+ * `ideabox-migrate.js` already imports `parseIdeabox` FROM `lib/ideabox.js` —
+ * so keeping the assertion in the gate and wiring it into `readIdeabox` would
+ * close an import cycle. Hence its own module, re-exported from
+ * `ideabox-migrate.js` so existing importers are unaffected.
+ */
+
+export class IdeaboxUnreadable extends Error {
+  constructor(unread, ideaboxPath) {
+    super(
+      `compose: the ideabox at ${ideaboxPath} declares ${unread.length} idea(s) this version cannot ` +
+      `read: ${unread.join(', ')}. Refusing rather than proceeding, because continuing would treat ` +
+      `them as absent and the next write would overwrite this file with a projection that does not ` +
+      `contain them. This usually means the file is in a dialect newer or older than this install, ` +
+      `or is partly converted. Nothing has been changed. Back the file up, then either upgrade ` +
+      `compose or convert the entries by hand.`
+    );
+    this.name = 'IdeaboxUnreadable';
+    this.code = 'IDEABOX_UNREADABLE';
+    this.unread = unread;
+  }
+}
+
+/**
+ * Throw unless the parse actually understood the document.
+ *
+ * @param {object} parsed a `parseIdeabox` result
+ * @param {string} ideaboxPath named in the error, for the human who has to fix it
+ * @returns {object} the same `parsed`, so callers can `return assertIdeaboxReadable(...)`
+ */
+export function assertIdeaboxReadable(parsed, ideaboxPath) {
+  const inMarkdown = [...(parsed.ideas ?? []), ...(parsed.killed ?? [])].map((i) => i.id);
+
+  // READABILITY BEFORE SEMANTICS.
+  //
+  // Every caller reads `inMarkdown` as a statement about what the user has.
+  // That is only true if the parse actually understood the file. When it did
+  // not, an id vanishes from `inMarkdown` and every branch silently reads its
+  // absence as consent — "no ideas here" — and the next write projects over it.
+  // That is not a hypothetical: it destroyed 18 ideas in one command
+  // (COMP-IDEABOX-MIGRATE-DIALECT).
+  //
+  // So compare what the file DECLARES against what the parser PRODUCED, and
+  // stop on any gap. This deliberately catches more than the empty parse that
+  // motivated it: a half-converted file yields some ideas and hides the rest,
+  // which every count-based check (`inMarkdown.length === 0`) waves straight
+  // through while it is just as destructive.
+  // What the parse could see but not read. Taken from the parser itself, not
+  // from a second scan of the same text: two readers of one document is the
+  // exact shape that produced this bug and three of its follow-ons.
+  const unread = [...new Set(parsed.unconsumed ?? [])].filter((id) => !inMarkdown.includes(id));
+  if (unread.length) throw new IdeaboxUnreadable(unread, ideaboxPath);
+
+  // The same id declared twice is the half-converted document: the parser reads
+  // one copy, so the check above is satisfied while the other — routinely the
+  // older, richer one — is invisible and would be deleted by the next render.
+  const seen = new Set();
+  const duplicated = [...new Set([
+    ...[...inMarkdown, ...(parsed.unconsumed ?? [])].filter((id) => seen.size === seen.add(id).size),
+    // An umbrella named after an idea that also exists here as a real idea:
+    // the half-converted document, where the richer original survives only as
+    // the heading the parser turned into a cluster.
+    ...(parsed.collisions ?? []),
+  ])];
+  if (duplicated.length) throw new IdeaboxUnreadable(duplicated, ideaboxPath);
+
+  return parsed;
+}