@smartmemory/compose 0.4.1 → 0.5.0

package/lib/fluid/render-ideabox.js

@@ -16,11 +16,12 @@
  */
 
 import { randomUUID } from 'node:crypto';
-import { mkdirSync, renameSync, rmSync, writeFileSync } from 'node:fs';
+import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
 import { dirname, join } from 'node:path';
 
 import { withDirLock } from '../dir-lock.js';
 import { toMarkdownDate } from './ideabox-dates.js';
+import { readPreamble } from './ideabox-preamble.js';
 import { KIND } from './provider.js';
 
 // Says only what is true. The cockpit's write path is NOT on the store yet
@@ -31,7 +32,20 @@ const BANNER = [
   '     Projection of the fluid-store idea records (COMP-PLAN-IDEA-UNIFY).',
   '     Edits here are overwritten on the next render. Change ideas with',
   '     `compose ideabox add|pri|kill|discuss|promote`, then `compose ideabox',
-  '     render` if this file ever looks stale. -->',
+  '     render` if this file ever looks stale.',
+  // The heading and introduction come from a DIFFERENT file, and nothing in the
+  // document says so. Someone correcting the title here loses it on the next
+  // render with no clue where it went — the one edit this file invites that has
+  // no visible home (COMP-IDEABOX-MIGRATE-DIALECT FU-4 follow-up).
+  '     The title and introduction above are not generated: they live in',
+  '     `<this file>.preamble.md`. Edit them THERE, not here.',
+  // The inverse rule, and nothing said it. Everything else in this file is
+  // generated output whose hand edits are discarded on the next render; the
+  // sidecar is the one input, so ITS hand edits are canon and are kept
+  // untouched forever. Two files side by side following opposite rules, with
+  // only one of them stated, is a trap for the next person to edit either.
+  '     That file is the opposite of this one: it is yours, nothing',
+  '     regenerates it, and what you write there is kept. -->',
 ];
 
 const PREAMBLE = [
@@ -85,6 +99,19 @@ function renderIdea(record) {
   if (record.source) out.push(`**Source:** ${record.source}`);
   if (record.body) out.push(`**Idea:** ${record.body}`);
 
+  // Unrecognised hand-authored fields. The two serializers put these in
+  // DIFFERENT slots and the projection has to match whichever one will read it
+  // back: `serializeIdea` emits them after `**Idea:**` and before the trailing
+  // known fields (`lib/ideabox.js`), while `serializeKilledIdea` emits them
+  // after `**Killed:**`. Using the live slot for both meant a killed idea
+  // carrying a custom field reordered the file on every write, so the
+  // projection stopped being a fixed point of the serializer — the property the
+  // whole cutover rests on. Killed ideas are emitted further down, next to
+  // `**Killed:**`.
+  if (!record.killed) {
+    for (const extra of record.extra_fields ?? []) out.push(extra);
+  }
+
   // `Promoted to:` is emitted BEFORE `Maps to:`, which reads backwards and is
   // deliberate. The legacy parser knows `Maps to` and does not know `Promoted
   // to`, so the latter lands in `_extraLines` — and the legacy serializer emits
@@ -112,6 +139,9 @@ function renderIdea(record) {
   // discussed — which is most of the ones anyone would want to kill.
   if (record.status === 'killed' && record.killed) {
     out.push(`**Killed:** ${toMarkdownDate(record.killed.at)} — ${record.killed.reason}`);
+    // The killed serializer's slot for unrecognised fields: immediately after
+    // `**Killed:**`, before the discussion block.
+    for (const extra of record.extra_fields ?? []) out.push(extra);
   }
 
   if (record.discussion?.length) {
@@ -132,8 +162,14 @@ function renderIdea(record) {
  * @param {Array<object>} data.clusters
  * @returns {string}
  */
-export function renderIdeabox({ ideas, clusters }) {
-  const lines = [...BANNER, '', ...PREAMBLE, '', '## Ideas', ''];
+export function renderIdeabox({ ideas, clusters, preamble = null }) {
+  // A project's own title and introduction, when it has one. The default
+  // PREAMBLE is a template, and using it unconditionally deleted the heading and
+  // prose an upgrading project had written for itself — the migration kept every
+  // idea and silently rewrote the document around them
+  // (COMP-IDEABOX-MIGRATE-DIALECT, Codex review finding 4).
+  const preambleLines = preamble && preamble.length ? preamble : PREAMBLE;
+  const lines = [...BANNER, '', ...preambleLines, '', '## Ideas', ''];
   // The legacy serializer emits a placeholder comment when the file contains no
   // `###` grouping heading at all, and the projection has to match it byte for
   // byte or `serialize(parse(projection))` stops being the identity. That state
@@ -203,14 +239,44 @@ export function renderIdeabox({ ideas, clusters }) {
  * Read the records from a provider and render.
  * @param {import('./provider.js').FluidProvider} provider
  */
-export async function renderIdeaboxFrom(provider) {
+export async function renderIdeaboxFrom(provider, { preamble = null, outPath = null } = {}) {
   const [ideas, clusters] = await Promise.all([
     provider.listRecords({ kind: KIND.IDEA }),
     provider.listRecords({ kind: KIND.CLUSTER }),
   ]);
-  return renderIdeabox({ ideas, clusters });
+  // The project's own heading and introduction, captured at migration into a
+  // tracked sibling of the ideabox (FU-4). NOT read from the destination —
+  // `<ideabox>.preamble.md` is a different file; see the note below. `outPath`
+  // names which ideabox's sidecar to read, not a file to read from. An explicit
+  // `preamble` still wins, and a project with no sidecar gets the standard
+  // template exactly as before.
+  const captured = preamble ?? (outPath ? readPreamble(outPath) : null);
+  return renderIdeabox({
+    ideas,
+    clusters,
+    preamble: captured ? captured.split('\n') : null,
+  });
 }
 
+/**
+ * STILL NOT READ FROM THE DESTINATION (FU-4, resolved 2026-09-07).
+ *
+ * Carrying the existing file's preamble forward would have preserved a
+ * project's own title and introduction through migration, but it makes the
+ * DESTINATION authoritative, and `render` is documented as the way back from
+ * any hand edit (`lib/ideabox-cli.js`). Reading the file we are about to repair
+ * means a hand-corrupted heading survives the repair. Both behaviours are
+ * legitimate and they were mutually exclusive while the preamble lived only in
+ * the markdown.
+ *
+ * So it no longer lives only there. `importIdeabox` captures it at migration
+ * into a tracked SIBLING of the ideabox, `<ideabox>.preamble.md`
+ * (`ideabox-preamble.js`), and the renderer reads THAT. A sibling is a
+ * different file, so the projection stays a function of the records plus the
+ * sidecar and never of its own output: a hand edit to the heading is discarded
+ * by the next render exactly as a hand edit to an idea is.
+ */
+
 /**
  * Render and write atomically (temp + rename), following the roadmap-gen
  * pattern — a half-written projection of canonical data is worse than none.
@@ -234,16 +300,78 @@ export async function renderIdeaboxFrom(provider) {
  * which is the documented state of that provider rather than a new gap.
  */
 export async function writeIdeaboxProjection(provider, outPath) {
+  // THE SINGLE DOOR.
+  //
+  // Every projection write replaces the user's ideabox wholesale, so every
+  // projection write is a potential erasure — and the migration gate that
+  // prevents it used to be applied caller-by-caller. It was on the CLI render
+  // and absent from the HTTP render, which meant a button in the cockpit could
+  // do what the equivalent command refused to
+  // (COMP-IDEABOX-MIGRATE-DIALECT, Codex review finding 1). Guarding each
+  // caller is a list that has to stay complete forever; guarding the boundary
+  // they all pass through is a fact. It goes here.
+  //
+  // Idempotent for the ordinary case: after a normal mutation the store is
+  // ahead of the file, which the gate reads as already-migrated and returns
+  // without work.
+  //
+  // DELIBERATELY OUTSIDE `withDirLock`, and it must stay there. The gate can
+  // import, importing calls `provider.createRecord`, and that takes this same
+  // lock — which is a non-reentrant `mkdirSync` lock (`lib/dir-lock.js`). Moving
+  // this call under the lock to close the read-then-write window deadlocks every
+  // migration: measured, it fails after the full 30s lock timeout rather than
+  // completing.
+  //
+  // The window that leaves — the file changing between this check and the
+  // replacement below — is closed by `publishProjection` re-running the gate's
+  // PURE assessment inside the lock (FU-3). That is the shape the deadlock
+  // forces: the decision moves under the lock, the execution stays outside it.
+  const { ensureIdeaboxMigrated } = await import('./ideabox-migrate.js');
+  await ensureIdeaboxMigrated(provider, outPath);
+
   return provider.lockPath
     ? withDirLock(provider.lockPath, () => publishProjection(provider, outPath))
     : publishProjection(provider, outPath);
 }
 
 async function publishProjection(provider, outPath) {
+  // THE SECOND LOOK, INSIDE THE LOCK.
+  //
+  // The gate above ran before this lock was acquired, so the file it approved
+  // is not necessarily the file about to be replaced: an editor saving a new
+  // idea while a migration runs had it destroyed, because the projection was
+  // built from records made out of the OLDER document
+  // (COMP-IDEABOX-MIGRATE-DIALECT FU-3).
+  //
+  // Re-ASSESSING rather than comparing a hash captured at gate time. A hash
+  // comparison cannot tell an edit from an ordinary concurrent render: two
+  // legitimate writers (the CLI and the REST API) each re-read the records, and
+  // whichever lands second would refuse for no reason. The assessment asks the
+  // question that actually matters — is this file still consistent with the
+  // store? — so a concurrent render assesses as `none` and proceeds, while a
+  // hand-added idea assesses as a stray and stops the write.
+  //
+  // Safe under the lock because the assessment is pure and takes no lock: it
+  // reads `listRecords`, `readEvents` and the markdown, and `renderIdeaboxFrom`
+  // below already performs the first of those under this same lock.
+  const { assessIdeabox, IdeaboxChangedUnderLock } = await import('./ideabox-migrate.js');
+  // The exact bytes the assessment below is about to approve. Re-read
+  // immediately before the rename — see the note there.
+  const approved = existsSync(outPath) ? readFileSync(outPath, 'utf8') : null;
+  const assessment = await assessIdeabox(provider, outPath);
+  if (assessment.action === 'refuse') throw assessment.error;
+  if (assessment.action !== 'none') {
+    // `import` here means the store and the file diverged after the gate
+    // approved them — records deleted concurrently, or an interrupted migration
+    // resumable from its manifest. Either way the write is abandoned rather
+    // than replacing a document this pass never checked.
+    throw new IdeaboxChangedUnderLock(outPath, `the store and the file now need ${assessment.action}`);
+  }
+
   // Rendered BEFORE the destination is touched, so a render that refuses (an
   // idea naming a cluster that does not exist) leaves the previous good file
   // exactly where it was rather than replacing it with a partial view.
-  const markdown = await renderIdeaboxFrom(provider);
+  const markdown = await renderIdeaboxFrom(provider, { outPath });
   mkdirSync(dirname(outPath), { recursive: true });
   // randomUUID, not pid: two renders from one process must not collide on the
   // temp name, and a recycled pid must not adopt a stranded file. Cleaned up on
@@ -252,6 +380,24 @@ async function publishProjection(provider, outPath) {
   const tmp = join(dirname(outPath), `.ideabox.md.tmp.${randomUUID()}`);
   try {
     writeFileSync(tmp, markdown, 'utf8');
+    // THE LOCK DOES NOT COORDINATE THE PERSON.
+    //
+    // The assessment above closed the window between the gate and the lock, but
+    // not the window inside the lock: reading the records and rendering them
+    // takes time, and a human saving a new idea in that moment never acquires
+    // this lock — nothing invites them to. Their whole idea was still lost, one
+    // step later than before.
+    //
+    // So the destination is compared against ITSELF, over milliseconds, with
+    // the lock held. That is not the gate-time fingerprint this deliberately
+    // does not use: the lock serializes every render Compose performs, so no
+    // legitimate concurrent render can change this file between the two reads,
+    // and none can be refused by this check. Only a writer outside the lock —
+    // which means a person with the file open — trips it.
+    const current = existsSync(outPath) ? readFileSync(outPath, 'utf8') : null;
+    if (current !== approved) {
+      throw new IdeaboxChangedUnderLock(outPath, 'it was saved from outside compose mid-write');
+    }
     renameSync(tmp, outPath);
   } catch (err) {
     rmSync(tmp, { force: true });

package/lib/fluid/smartmemory-provider.js

@@ -959,6 +959,12 @@ export class SmartMemoryFluidProvider extends FluidProvider {
       cluster_order: input.cluster_order ?? null,
       tags: input.tags ?? [],
       source: input.source ?? null,
+      // Unrecognised hand-authored markdown fields, carried verbatim. This
+      // literal is one of FOUR places the record's fields are enumerated (the
+      // contract, `record-shape.js`, and both providers); a field missing from
+      // any one of them is silently dropped on write
+      // (COMP-IDEABOX-MIGRATE-DIALECT).
+      extra_fields: input.extra_fields ?? [],
       links: input.links ?? [],
       killed: input.killed ?? null,
       discussion: input.discussion ?? [],

package/lib/gate-prompt.js

@@ -136,28 +136,32 @@ export class GateInputUnavailableError extends Error {
  *
  * @param {import('node:readline').Interface} rl
  * @param {string} question
- * @param {{ armGuard?: boolean, deadlineMs?: number }} [opts]
+ * @param {{ armGuard?: boolean, deadlineMs?: number, signal?: AbortSignal }} [opts]
  * @returns {Promise<string>}
  */
-export function ask(rl, question, { armGuard = false, deadlineMs = GATE_NONINTERACTIVE_TIMEOUT_MS } = {}) {
-  if (!armGuard) {
+export function ask(rl, question, { armGuard = false, deadlineMs = GATE_NONINTERACTIVE_TIMEOUT_MS, signal } = {}) {
+  if (!armGuard && !signal) {
     return new Promise(resolve => rl.question(question, resolve));
   }
   return new Promise((resolve, reject) => {
     let settled = false;
     const onEnd = () => finish(() => reject(new GateInputUnavailableError('stdin was closed before a decision')));
-    const timer = setTimeout(
+    const onAbort = () => finish(() => reject(signal.reason));
+    const timer = armGuard ? setTimeout(
       () => finish(() => reject(new GateInputUnavailableError(`no input arrived within ${Math.round(deadlineMs / 1000)}s`))),
       deadlineMs,
-    );
+    ) : null;
     function finish(act) {
       if (settled) return;
       settled = true;
       clearTimeout(timer);
       rl.input.removeListener('end', onEnd);
+      signal?.removeEventListener('abort', onAbort);
       act();
     }
-    rl.input.once('end', onEnd);
+    if (armGuard) rl.input.once('end', onEnd);
+    if (signal?.aborted) { onAbort(); return; }
+    signal?.addEventListener('abort', onAbort, { once: true });
     rl.question(question, answer => finish(() => resolve(answer)));
   });
 }
@@ -230,7 +234,8 @@ function drawGatePanel(out, gateDispatch, { artifact, gateExtras } = {}) {
  * @param {object}   [options.gateExtras]  - { fromPhase, toPhase } for panel display
  * @returns {Promise<{ outcome: string, rationale: string }>}
  */
-export async function promptGate(gateDispatch, { input, output, artifact, askAgent, gateExtras, nonInteractive, timeoutMs } = {}) {
+export async function promptGate(gateDispatch, { input, output, artifact, askAgent, gateExtras, nonInteractive, timeoutMs, signal } = {}) {
+  signal?.throwIfAborted();
   if (nonInteractive) {
     return { outcome: 'approve', rationale: 'auto-approved (--all mode)' };
   }
@@ -247,6 +252,7 @@ export async function promptGate(gateDispatch, { input, output, artifact, askAge
   const askGuard = {
     armGuard: input === undefined && !process.stdin.isTTY,
     deadlineMs: timeoutMs ?? GATE_NONINTERACTIVE_TIMEOUT_MS,
+    signal,
   };
 
   try {
@@ -312,6 +318,7 @@ export async function promptGate(gateDispatch, { input, output, artifact, askAge
           const answer = await askAgent(trimmed, artifact);
           rl.output.write(`\n  ${answer}\n`);
         } catch (err) {
+          if (signal?.aborted) throw err;
           rl.output.write(`  (agent error: ${err.message})\n`);
         }
         notes.push(trimmed);

package/lib/ideabox-cli.js

@@ -48,6 +48,7 @@ import {
 import { toMarkdownDate } from './fluid/ideabox-dates.js';
 import { KIND } from './fluid/provider.js';
 import { ensureIdeaboxMigrated } from './fluid/ideabox-migrate.js';
+import { adoptFile, discardEdits } from './fluid/ideabox-recover.js';
 import { writeIdeaboxProjection } from './fluid/render-ideabox.js';
 
 const USAGE = [
@@ -63,6 +64,10 @@ const USAGE = [
   '  discuss <ID> "<comment>"     Add a discussion comment',
   '  triage [--lens <name>]       Walk untriaged ideas and assign priorities',
   '  render                       Rewrite the ideabox file from the records',
+  '',
+  'Recovering an interrupted migration whose file then changed:',
+  '  adopt-file                   The file on disk is right — finish the migration against it',
+  '  discard-edits                The migration is right — restore the file (a copy is saved first)',
 ];
 
 const PRIORITIES = ['P0', 'P1', 'P2'];
@@ -91,6 +96,20 @@ function reportOpFailure(err) {
     console.error(err.message);
     return 1;
   }
+  // Every refusal in the migration family — unreadable, conflict, stale
+  // manifest, changed under the lock, nothing to recover — is a message written
+  // FOR the user, naming what was found and what to do about it. Rethrowing
+  // meant the CLI died with a Node stack trace and the guidance buried in the
+  // middle of it. That is not a hypothetical cost: the stale-manifest message
+  // is the one place the two recovery commands are named, and a supported exit
+  // nobody can read is not an exit.
+  //
+  // Matched by code prefix rather than by class so a refusal added later is
+  // reported the same way without anyone having to remember this list.
+  if (typeof err?.code === 'string' && err.code.startsWith('IDEABOX_')) {
+    console.error(err.message);
+    return 1;
+  }
   throw err;
 }
 
@@ -304,6 +323,55 @@ export async function runIdeaboxCommand(cwd, args, opts = {}) {
         return 0;
       }
 
+      // The two recovery commands. Deliberately NOT preceded by
+      // `ensureIdeaboxMigrated`: that gate throws on exactly the state these
+      // exist to leave. They run their own, narrower check instead — both
+      // refuse unless an interrupted migration is recorded AND its document has
+      // changed, so neither is a general "make the file canon" door.
+      case 'adopt-file': {
+        const { updated, discussed, imported, kept, reclustered } = await adoptFile(provider, ideaboxPath);
+        console.log(`Adopted ${ideaboxPath} as the source of the interrupted migration.`);
+        // A recovery that does not say what it did is not a recovery.
+        console.log(`  Updated to match the file: ${updated.length ? updated.join(', ') : 'none'}`);
+        if (discussed.length) console.log(`  Discussion entries added: ${discussed.join(', ')}`);
+        if (reclustered.length) console.log(`  Umbrellas updated: ${reclustered.join(', ')}`);
+        console.log(`  Imported: ${imported.length ? imported.join(', ') : 'none'}`);
+        if (kept.length) {
+          console.log(`  Kept, though absent from the file: ${kept.join(', ')}`);
+          console.log('    Nothing is ever deleted by this command. They are back in the file now;');
+          console.log('    remove them deliberately with `compose ideabox kill <ID>` if they are stale.');
+        }
+        return 0;
+      }
+
+      case 'discard-edits': {
+        // Printed from the callback rather than the return value: if the resume
+        // throws after the overwrite, the user has just lost the text and this
+        // line is the only thing telling them where the copy is.
+        const { result, reverted, leftover } = await discardEdits(provider, ideaboxPath, {
+          onBackup: (path) => console.log(`Saved a copy of the discarded file: ${path}`),
+        });
+        console.log(`Restored ${ideaboxPath} to the document the migration read, and finished it.`);
+        // Records are restored as well as the file — an adoption that got as far
+        // as patching them is what makes this more than a file copy.
+        if (reverted.updated.length) {
+          console.log(`  Put back to the migrated version: ${reverted.updated.join(', ')}`);
+        }
+        console.log(`  Imported: ${result.imported.length ? result.imported.join(', ') : 'none'}`);
+        // Nothing is deleted here either, so anything an interrupted adoption
+        // added and this cannot take back is named rather than left as a
+        // surprise in the file.
+        if (leftover.clusters.length) {
+          console.log(`  Umbrellas left in place, unnamed by the restored file: ${leftover.clusters.join(', ')}`);
+          console.log('    They render with no ideas under them. Nothing was deleted to make room.');
+        }
+        if (leftover.discussed.length) {
+          console.log(`  Comments that stay on their idea: ${leftover.discussed.join(', ')}`);
+          console.log('    The discussion trail is append-only, so a comment typed during the outage is kept.');
+        }
+        return 0;
+      }
+
       default:
         console.error(`Unknown ideabox subcommand: ${sub}`);
         console.error('Run: compose ideabox --help');