@smartmemory/compose 0.4.1 → 0.5.1

package/lib/gsd.js

@@ -26,7 +26,7 @@ import { validateBoundaryMap } from './boundary-map.js';
 import { enrichTaskGraph } from './gsd-decompose-enrich.js';
 import { buildTaskDescription } from './gsd-prompt.js';
 import { writeAll, validate as validateTaskResult, read as readBlackboard } from './gsd-blackboard.js';
-import { executeShipStep, toPhaseResultOutput, runConsumerIssuance, ConsumerStuckError, filesOwnedConflict, toEngineUsage, reportUsageReceipts } from './build.js';
+import { executeShipStep, toPhaseResultOutput, runConsumerIssuance, ConsumerStuckError, filesOwnedConflict, toEngineUsage, reportUsageReceipts, loadPipelineProfiles, preflightPipelineProfiles, admitConsumerWave, publishConsumerCheckpoint, evaluateConfiguredGate, reportWaveEvidence, waveProfilesEnabled } from './build.js';
 import {
   ConsumerFanoutArtifacts,
   ConsumerMergeDecisionError,
@@ -144,7 +144,8 @@ export async function runGsd(featureCode, opts = {}) {
   const preMergeGate = resolvePreMergeGate(cwd, opts.preMergeGate);
 
   // 4. Load pipeline spec
-  const specPath = join(PACKAGE_ROOT, 'pipelines', 'gsd.stratum.yaml');
+  const localPath = join(cwd, 'pipelines', 'gsd.stratum.yaml');
+  const specPath = existsSync(localPath) ? localPath : join(PACKAGE_ROOT, 'pipelines', 'gsd.stratum.yaml');
   // 4a. COMP-GSD-4: inject the stratum flow budget block from `gsd.budget.*`.
   // injectBudget is IDENTITY when nothing is configured, so an un-budgeted gsd
   // run (and plain `compose build`) is byte-identical.
@@ -152,6 +153,9 @@ export async function runGsd(featureCode, opts = {}) {
   const specYaml = injectBudget(readFileSync(specPath, 'utf-8'), budgetCfg);
   const localSpec = YAML.parse(specYaml);
   const localSpecDigest = sha256(JSON.stringify(localSpec));
+  const profileCheck = preflightPipelineProfiles(loadPipelineProfiles(specPath), localSpec, specPath);
+  const pipelineProfiles = profileCheck.normalized;
+  const profileDigest = waveProfilesEnabled(pipelineProfiles) ? profileCheck.profilesDigest : undefined;
 
   // 4a. COMP-GSD-4: cumulative cross-session ceiling pre-check (tokens/cost).
   // Refuse to start/resume a run that has already spent its lifetime budget —
@@ -235,7 +239,7 @@ export async function runGsd(featureCode, opts = {}) {
     stepCtx = {
       stratum, cwd, featureCode, blueprintText, gateCommands, preMergeGate,
       receiptsMode,
-      localSpec, localSpecDigest,
+      localSpec, localSpecDigest, pipelineProfiles, profileDigest,
       filesChanged: [],
       stuckDetector,
       // D2(a): per-ITEM wall-clock ceiling from gsd.budget.per_task_ms, enforced
@@ -294,6 +298,11 @@ export async function runGsd(featureCode, opts = {}) {
     }, { workspaceRoot: cwd });
     const flowId = response.runId;
     stepCtx.flowId = flowId;
+    if (waveProfilesEnabled(pipelineProfiles)) {
+      stepCtx.consumerArtifacts = new ConsumerFanoutArtifacts({ runId: flowId, targetCwd: cwd,
+        revisionDigest: response.revisionDigest, specDigest: localSpecDigest, profilesDigest: profileDigest });
+      stepCtx.artifacts = stepCtx.consumerArtifacts;
+    }
     flushState(stepCtx, { flowId, phase: 'decompose' });
     emitPhaseOnce(stepCtx, 'decompose'); // COMP-GSD-7-EVENTLOG
 
@@ -303,11 +312,17 @@ export async function runGsd(featureCode, opts = {}) {
       response.status !== 'completed' &&
       response.status !== 'failed' &&
       response.status !== 'stuck' &&
+      response.status !== 'waiting_gate' &&
       response.status !== 'budget_exhausted'
     ) {
       response = await runOneStep(response, stepCtx);
     }
 
+    if (response.status === 'waiting_gate') {
+      flushState(stepCtx, { status: 'waiting_gate', gateToken: response.gateToken, reason: response.reason });
+      return response;
+    }
+
     if (response.status === 'stuck') {
       // Artifacts (stuck.md/json + pause.json) were written by runOneStep.
       // COMP-GSD-7-EVENTLOG: flush any completions that finished before the stuck
@@ -494,25 +509,35 @@ async function runOneStep(response, ctx) {
           targetCwd: cwd,
           revisionDigest: descriptor.revisionDigest,
           specDigest: localSpecDigest,
+          profilesDigest: ctx.profileDigest,
         });
       }
       ctx.consumerArtifacts.bindRunRevision({
         revisionDigest: descriptor.revisionDigest,
         specDigest: localSpecDigest,
+        profilesDigest: ctx.profileDigest,
       });
+      ctx.artifacts = ctx.consumerArtifacts;
+      // This audit already serves issuance recovery on legacy runs; admission
+      // uses it only when this step opts into wave policies.
+      const audit = await stratum.audit(flowId);
+      const configured = ctx.pipelineProfiles?.[descriptor.step]?.tier_from || ctx.pipelineProfiles?._consumer?.[descriptor.step];
+      const admission = configured ? await admitConsumerWave({ descriptor, descriptors: ready.filter(isConsumerDescriptor), audit,
+        localSpec, profiles: ctx.pipelineProfiles, artifacts: ctx.consumerArtifacts, stratum, flowId }) : null;
       // D7(a): render the exact TaskResult path from the item's task id (the
       // fanout `over` is decompose_gsd.output.tasks, indexed by itemIndex).
-      const consumerItem = ctx.lastTaskGraph?.tasks?.[descriptor.itemIndex];
+      const consumerItem = descriptor.item;
       const taskResultPath = consumerItem?.id
         ? gsdTaskResultPath(featureCode, consumerItem.id)
         : undefined;
       try {
         return await runConsumerIssuance({
           descriptor,
+          admission,
           flowId,
           stratum,
           artifacts: ctx.consumerArtifacts,
-          audit: await stratum.audit(flowId),
+          audit,
           localSpec,
           // D2(b): onUsage debits each item's agent usage into the cumulative
           // ledger. D2(a): per-item wall-clock ceiling from gsd.budget.per_task_ms.
@@ -523,12 +548,15 @@ async function runOneStep(response, ctx) {
           // by definition a GSD task, so timing.json + diffs/<id>.diff are written
           // for the milestone report. The build consumer path omits it (no marker),
           // so build-mode fanout stays byte-identical.
-          context: { cwd, projectCwd: cwd, featureCode, flowId, receiptsMode: ctx.receiptsMode, gsd: true, gsdTaskId: consumerItem?.id, filesChanged: ctx.filesChanged, onUsage: (usage, meta) => recordTsAgentUsage(ctx, usage, meta), taskResultPath },
+          context: { pipelineProfiles: ctx.pipelineProfiles, artifacts: ctx.consumerArtifacts, cwd, projectCwd: cwd, featureCode, flowId, receiptsMode: ctx.receiptsMode, gsd: true, gsdTaskId: consumerItem?.id, filesChanged: ctx.filesChanged, onUsage: (usage, meta) => recordTsAgentUsage(ctx, usage, meta), taskResultPath },
           // runAndNormalize narrates via progress.debug/warn/info/toolUse — gsd
           // has no cockpit, so a COMPLETE no-op (not just stepStart/stepDone) is
           // required or the item throws "progress.debug is not a function".
           progress: NOOP_PROGRESS,
-          streamWriter: { write() {} },
+          streamWriter: { write(event) {
+            if (event.type === 'step_model') appendGsdEvent(cwd, featureCode, 'step_model', event);
+          } },
+          profile: ctx.pipelineProfiles?.[descriptor.step]?.default ?? ctx.pipelineProfiles?.[descriptor.step],
           perItemTimeoutMs: ctx.perItemTimeoutMs,
           stuckDetector: ctx.stuckDetector,
         });
@@ -555,7 +583,7 @@ async function runOneStep(response, ctx) {
         featureCode,
         cwd,
         cwd,
-        { cwd, featureCode, mode: 'feature', filesChanged: ctx.filesChanged ?? [] },
+        { ...ctx, cwd, featureCode, mode: 'feature', artifacts: ctx.consumerArtifacts, filesChanged: ctx.filesChanged ?? [] },
         '',
         null,
       );
@@ -663,46 +691,65 @@ async function runOneStep(response, ctx) {
     const gateStep = steps.find((step) => step.id === gateStepId);
     const predecessorIds = new Set(gateStep?.after ?? []);
     const fanoutStep = steps.find((step) => predecessorIds.has(step.id) && step.fanout?.dispatch === 'consumer');
-    if (!fanoutStep) {
-      throw new Error(`runGsd: unexpected non-consumer gate ${gateStepId} on the TS path`);
-    }
-    if (!ctx.consumerArtifacts) {
-      ctx.consumerArtifacts = new ConsumerFanoutArtifacts({
-        runId: flowId,
-        targetCwd: cwd,
-        revisionDigest: response.revisionDigest,
-        specDigest: localSpecDigest,
-      });
-    }
-    const artifacts = ctx.consumerArtifacts;
-    artifacts.recordGateBinding({ gateStepId, fanoutStepId: fanoutStep.id });
-    let transaction;
-    let outcome = 'approve';
-    let rationale = 'consumer fanout artifacts merged';
-    try {
-      transaction = artifacts.prepareMerge({
-        gateStepId,
-        gateToken: gateState.gateToken,
-        fanoutStepId: fanoutStep.id,
-        audit,
-      });
-      await artifacts.applyMerge(transaction);
-    } catch (error) {
-      if (!(error instanceof ConsumerMergeDecisionError)) throw error;
-      outcome = gateStep?.gate?.on_revise ? 'revise' : 'kill';
-      rationale = `${error.code}: ${error.message}`;
-      transaction ??= artifacts.journal.mergeTransactions.find(
-        (entry) => entry.gateToken === gateState.gateToken,
-      );
-      if (transaction) artifacts.restoreMergeBaseline(transaction, audit);
-    }
-    const next = await stratum.gateResolve(
-      flowId, gateStepId, outcome, rationale, 'system', gateState.gateToken,
-    );
-    artifacts.markGateResolved(transaction, outcome);
-    if (outcome !== 'revise') artifacts.cleanupWorktrees(`GSD merge gate ${outcome}`);
-    if (outcome === 'approve') ctx.filesChanged = collectChangedFiles(cwd);
-    return next;
+    const outputDecision = ctx.artifacts ? await evaluateConfiguredGate(ctx, { localSpec, gateStepId,
+      gateToken: gateState.gateToken, audit }) : null;
+    if (outputDecision && !outputDecision.outcome) return { status: 'waiting_gate', flowId,
+      gateToken: gateState.gateToken, reason: outputDecision.reason };
+    const resolveGateWithConsumerMerge = async (requestedOutcome, requestedRationale) => {
+      if (!fanoutStep && !outputDecision) throw new Error(`runGsd: unexpected non-consumer gate ${gateStepId} on the TS path`);
+      if (fanoutStep && !ctx.consumerArtifacts) {
+        ctx.consumerArtifacts = new ConsumerFanoutArtifacts({ runId: flowId, targetCwd: cwd,
+          revisionDigest: response.revisionDigest, specDigest: localSpecDigest, profilesDigest: ctx.profileDigest });
+      }
+      const artifacts = fanoutStep ? ctx.consumerArtifacts : null;
+      if (artifacts) ctx.artifacts = artifacts;
+      if (ctx.pipelineProfiles?._consumer?.[fanoutStep?.id]?.checkpoint_gate === gateStepId) {
+        artifacts.initializeWave({ ref: `refs/heads/compose/wave/${flowId}`, profilesDigest: ctx.profileDigest });
+      }
+      let transaction;
+      let outcome = requestedOutcome;
+      let rationale = requestedRationale;
+      if (artifacts) {
+        artifacts.recordGateBinding({ gateStepId, fanoutStepId: fanoutStep.id });
+        try {
+          transaction = artifacts.prepareMerge({ gateStepId, gateToken: gateState.gateToken, fanoutStepId: fanoutStep.id, audit });
+          if (outcome === 'approve') await artifacts.applyMerge(transaction);
+          else artifacts.restoreMergeBaseline(transaction, audit);
+        } catch (error) {
+          if (!(error instanceof ConsumerMergeDecisionError)) throw error;
+          outcome = gateStep?.gate?.on_revise ? 'revise' : 'kill';
+          rationale = `${error.code}: ${error.message}`;
+          transaction ??= artifacts.journal.mergeTransactions.find(e => e.gateToken === gateState.gateToken);
+          if (transaction) artifacts.restoreMergeBaseline(transaction, audit);
+        }
+      }
+      let next;
+      try {
+        next = await stratum.gateResolve(flowId, gateStepId, outcome, rationale, 'system', gateState.gateToken);
+      } catch (error) {
+        const current = await stratum.audit(flowId);
+        const ordinal = (audit.events ?? []).filter(e => e.type === 'gate_resolved' && e.stepId === gateStepId).length;
+        const confirmed = (current.events ?? []).filter(e => e.type === 'gate_resolved' && e.stepId === gateStepId)[ordinal]?.detail?.decision;
+        if (!confirmed) throw error;
+        outcome = confirmed;
+        if (artifacts && outcome === 'approve') {
+          artifacts.markGateResolved(transaction, outcome);
+          await publishConsumerCheckpoint(ctx, transaction);
+        }
+        next = await stratum.resume(flowId);
+      }
+      if (artifacts) {
+        artifacts.markGateResolved(transaction, outcome);
+        if (outcome === 'approve') await publishConsumerCheckpoint(ctx, transaction);
+        if (outcome !== 'revise') artifacts.cleanupWorktrees(`GSD merge gate ${outcome}`,
+          artifacts.journal.wave ? { dispatchTokens: transaction.acceptedDispatchTokens } : {});
+        if (outcome === 'approve' && !artifacts.journal.wave) ctx.filesChanged = collectChangedFiles(cwd);
+      }
+      if (outputDecision) await reportWaveEvidence(ctx, 'gate_decision', gateState.gateToken,
+        { ...outputDecision, outcome, rationale }, 'accepted');
+      return next;
+    };
+    return resolveGateWithConsumerMerge(outputDecision?.outcome ?? 'approve', outputDecision?.rationale ?? 'consumer fanout artifacts merged');
   }
 
   return response;

package/lib/ideabox-cli.js

@@ -48,6 +48,7 @@ import {
 import { toMarkdownDate } from './fluid/ideabox-dates.js';
 import { KIND } from './fluid/provider.js';
 import { ensureIdeaboxMigrated } from './fluid/ideabox-migrate.js';
+import { adoptFile, discardEdits } from './fluid/ideabox-recover.js';
 import { writeIdeaboxProjection } from './fluid/render-ideabox.js';
 
 const USAGE = [
@@ -63,6 +64,10 @@ const USAGE = [
   '  discuss <ID> "<comment>"     Add a discussion comment',
   '  triage [--lens <name>]       Walk untriaged ideas and assign priorities',
   '  render                       Rewrite the ideabox file from the records',
+  '',
+  'Recovering an interrupted migration whose file then changed:',
+  '  adopt-file                   The file on disk is right — finish the migration against it',
+  '  discard-edits                The migration is right — restore the file (a copy is saved first)',
 ];
 
 const PRIORITIES = ['P0', 'P1', 'P2'];
@@ -91,6 +96,20 @@ function reportOpFailure(err) {
     console.error(err.message);
     return 1;
   }
+  // Every refusal in the migration family — unreadable, conflict, stale
+  // manifest, changed under the lock, nothing to recover — is a message written
+  // FOR the user, naming what was found and what to do about it. Rethrowing
+  // meant the CLI died with a Node stack trace and the guidance buried in the
+  // middle of it. That is not a hypothetical cost: the stale-manifest message
+  // is the one place the two recovery commands are named, and a supported exit
+  // nobody can read is not an exit.
+  //
+  // Matched by code prefix rather than by class so a refusal added later is
+  // reported the same way without anyone having to remember this list.
+  if (typeof err?.code === 'string' && err.code.startsWith('IDEABOX_')) {
+    console.error(err.message);
+    return 1;
+  }
   throw err;
 }
 
@@ -304,6 +323,55 @@ export async function runIdeaboxCommand(cwd, args, opts = {}) {
         return 0;
       }
 
+      // The two recovery commands. Deliberately NOT preceded by
+      // `ensureIdeaboxMigrated`: that gate throws on exactly the state these
+      // exist to leave. They run their own, narrower check instead — both
+      // refuse unless an interrupted migration is recorded AND its document has
+      // changed, so neither is a general "make the file canon" door.
+      case 'adopt-file': {
+        const { updated, discussed, imported, kept, reclustered } = await adoptFile(provider, ideaboxPath);
+        console.log(`Adopted ${ideaboxPath} as the source of the interrupted migration.`);
+        // A recovery that does not say what it did is not a recovery.
+        console.log(`  Updated to match the file: ${updated.length ? updated.join(', ') : 'none'}`);
+        if (discussed.length) console.log(`  Discussion entries added: ${discussed.join(', ')}`);
+        if (reclustered.length) console.log(`  Umbrellas updated: ${reclustered.join(', ')}`);
+        console.log(`  Imported: ${imported.length ? imported.join(', ') : 'none'}`);
+        if (kept.length) {
+          console.log(`  Kept, though absent from the file: ${kept.join(', ')}`);
+          console.log('    Nothing is ever deleted by this command. They are back in the file now;');
+          console.log('    remove them deliberately with `compose ideabox kill <ID>` if they are stale.');
+        }
+        return 0;
+      }
+
+      case 'discard-edits': {
+        // Printed from the callback rather than the return value: if the resume
+        // throws after the overwrite, the user has just lost the text and this
+        // line is the only thing telling them where the copy is.
+        const { result, reverted, leftover } = await discardEdits(provider, ideaboxPath, {
+          onBackup: (path) => console.log(`Saved a copy of the discarded file: ${path}`),
+        });
+        console.log(`Restored ${ideaboxPath} to the document the migration read, and finished it.`);
+        // Records are restored as well as the file — an adoption that got as far
+        // as patching them is what makes this more than a file copy.
+        if (reverted.updated.length) {
+          console.log(`  Put back to the migrated version: ${reverted.updated.join(', ')}`);
+        }
+        console.log(`  Imported: ${result.imported.length ? result.imported.join(', ') : 'none'}`);
+        // Nothing is deleted here either, so anything an interrupted adoption
+        // added and this cannot take back is named rather than left as a
+        // surprise in the file.
+        if (leftover.clusters.length) {
+          console.log(`  Umbrellas left in place, unnamed by the restored file: ${leftover.clusters.join(', ')}`);
+          console.log('    They render with no ideas under them. Nothing was deleted to make room.');
+        }
+        if (leftover.discussed.length) {
+          console.log(`  Comments that stay on their idea: ${leftover.discussed.join(', ')}`);
+          console.log('    The discussion trail is append-only, so a comment typed during the outage is kept.');
+        }
+        return 0;
+      }
+
       default:
         console.error(`Unknown ideabox subcommand: ${sub}`);
         console.error('Run: compose ideabox --help');

package/lib/ideabox.js

@@ -16,6 +16,7 @@
 
 import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs'
 import { join, dirname } from 'node:path'
+import { assertIdeaboxReadable } from './fluid/ideabox-readable.js'
 import { resolvePathValue } from './paths-core.js'
 
 // ---------------------------------------------------------------------------
@@ -47,6 +48,36 @@ export const IDEABOX_TEMPLATE = `# Ideabox
 // Matches: #### IDEA-42 — Some Title   (or "- " variant)
 const IDEA_HEADING_RE = /^####\s+(IDEA-(\d+))\s+[—–-]+\s+(.+)$/
 
+// The LEGACY (pre-umbrella) dialect: ideas are H3 under arbitrary `## Topic`
+// headings, with no `## Ideas` wrapper and no cluster level. Published installs
+// upgrading from before COMP-PLAN-IDEA-UNIFY have this shape, so it is the
+// dialect the migration path actually meets — see COMP-IDEABOX-MIGRATE-DIALECT,
+// where failing to read it destroyed 18 ideas in one command.
+const LEGACY_IDEA_HEADING_RE = /^###\s+(IDEA-(\d+))\s+[—–-]+\s+(.+)$/
+
+// An idea heading at EITHER level. A third vintage in the wild is a hybrid: it
+// has the modern `## Ideas` wrapper (so it is not the flat dialect) but writes
+// both its ideas AND its umbrellas at H3. Those two are still unambiguous —
+// an H3 that starts with an `IDEA-N` id is an idea, and any other H3 is an
+// umbrella — so the level alone was never what distinguished them.
+// Measured 2026-09-06: three projects on this machine are in this dialect
+// (books 9 ideas, ScaleMate 2, trustflow 1). Before the guard landed they were
+// silently destroyed; with the guard but without this they are refused, which
+// is safe but leaves them unable to use the ideabox at all.
+const IDEA_HEADING_ANY_RE = /^#{3,4}\s+(IDEA-(\d+))\s+[—–-]+\s+(.+)$/
+
+/**
+ * A line SHAPED like an idea declaration, at any heading level.
+ *
+ * Used only to notice a heading the dialect's own idea pattern could not
+ * consume — the "I can see something here and cannot read it" case.
+ */
+const IDEA_DECL_SHAPE_RE = /^\s*#{1,6}\s+(IDEA-\d+)\b/
+
+/** The bullet form, for a document that uses no idea headings at all. */
+const IDEA_BULLET_SHAPE_RE = /^\s*[-*]\s+(IDEA-\d+)\s*[—–:-]/
+
+
 // Matches a field line: **FieldName:** value  (colon inside bold markers)
 const FIELD_RE = /^\*\*([^*:]+):\*\*\s*(.*)$/
 
@@ -80,6 +111,41 @@ const DISCUSSION_ENTRY_RE = /^-\s+\[(\d{4}-\d{2}-\d{2})\]\s+([^:\n]+?):\s+(.+)$/
 export function parseIdeabox(markdown) {
   const lines = markdown.split('\n')
 
+  // Legacy dialect detection, fenced as narrowly as possible: it activates ONLY
+  // when the document has no `## Ideas` section AND carries H3 idea headings.
+  // A new-dialect file always opens its ideas with `## Ideas`, so legacy mode
+  // can never engage on one. A file with BOTH shapes is deliberately left to
+  // the modern path — a half-converted document is not something to guess at,
+  // and the migration gate refuses it by declaration count instead.
+  // A COMPLETE heading, not a prefix: `/^##\s+Ideas/` also matched a legitimate
+  // legacy topic called `## Ideas for Later`, which switched legacy mode off and
+  // made that document's ideas unreadable. Fenced blocks are skipped for the
+  // same reason — an example containing `## Ideas` must not decide the dialect
+  // of the file quoting it.
+  const structural = []
+  let fenced = false
+  for (const l of lines) {
+    if (/^\s*```/.test(l)) { fenced = !fenced; continue }
+    if (!fenced) structural.push(l)
+  }
+  const hasIdeasSection = structural.some((l) => /^##\s+Ideas\s*$/.test(l))
+  const legacy = !hasIdeasSection && structural.some((l) => LEGACY_IDEA_HEADING_RE.test(l))
+
+  // The hybrid dialect only exists in a document the tools have NEVER written:
+  // every write emits ideas at H4, so a document containing any `#### IDEA-` is
+  // modern, and an H3 there is an umbrella no matter what it is called.
+  //
+  // That distinction is load-bearing rather than cosmetic. `compose ideabox`
+  // will happily create a cluster named `IDEA-9 — Cache research`, and reading
+  // that back as an idea made the next command fail: the guard saw a declared
+  // id with no record behind it and refused. Recognising H3 ideas everywhere
+  // broke round-tripping the tool's own output.
+  const hasModernIdeaHeadings = structural.some((l) => IDEA_HEADING_RE.test(l))
+  const hybrid = hasIdeasSection && !hasModernIdeaHeadings
+  const ideaHeadingRe = legacy
+    ? LEGACY_IDEA_HEADING_RE
+    : (hybrid ? IDEA_HEADING_ANY_RE : IDEA_HEADING_RE)
+
   const ideas = []
   const killed = []
   // Cluster-scoped data. An umbrella heading carries a hand-authored multi-
@@ -89,11 +155,25 @@ export function parseIdeabox(markdown) {
   // parse→serialize cycle (i.e. every `compose ideabox` mutation) deleted it.
   const clusters = []
   const clusterIndex = new Map()
+  // Idea-shaped headings that NO branch consumed. The migration gate compares
+  // these against the ideas actually produced, so that "the parser could not
+  // read this" can never be mistaken for "there was nothing here". Collected
+  // HERE, by the parser itself, rather than by a second scanner: the whole class
+  // of bug this module keeps producing is two readers disagreeing about what the
+  // document says, and a separate scanner is a second reader by construction.
+  const shapes = []
+  // Idea-shaped headings taken as UMBRELLA names. Legitimate on its own — the
+  // tools will create a cluster called `IDEA-9 — Cache research` — but an
+  // umbrella named after an id that is ALSO a real idea in the same document is
+  // the half-converted file, not a naming choice.
+  const clusterNamedIds = []
   // Everything before `## Ideas`. Regenerating this from IDEABOX_TEMPLATE
   // instead of preserving it drops hand-authored convention bullets.
   const preambleLines = []
 
-  let inIdeasSection = false
+  // In the legacy dialect there is no `## Ideas` wrapper — the ideas simply
+  // live under topic headings — so the whole document is the ideas section.
+  let inIdeasSection = legacy
   let inKilledSection = false
   let currentCluster = null
   let currentIdea = null
@@ -111,11 +191,43 @@ export function parseIdeabox(markdown) {
     currentIdea = null
   }
 
+  // Fenced blocks are documentation, not content. Detection and the declaration
+  // scan already skipped them; the parse loop did not, so an example in a
+  // ```markdown``` block was imported as a real idea — and, worse, satisfied the
+  // readability guard on behalf of a genuine entry with the same id that the
+  // parser could NOT read, letting the destructive write through. All three
+  // readers have to agree on what is content.
+  let inFence = false
+
   for (let i = 0; i < lines.length; i++) {
     const line = lines[i]
 
+    if (/^\s*```/.test(line)) {
+      inFence = !inFence
+      if (!inIdeasSection && !inKilledSection && !seenAnySection) preambleLines.push(line)
+      // The DELIMITERS are content too. Keeping the enclosed lines while
+      // dropping the fence turned a fenced `#### IDEA-2 — example` inside an
+      // idea's body into a real heading on the next projection, and the render
+      // after that refused the file it had just written.
+      else if (currentIdea) currentIdea._extraLines.push(line)
+      continue
+    }
+    if (inFence) {
+      if (!inIdeasSection && !inKilledSection && !seenAnySection) preambleLines.push(line)
+      else if (currentIdea) currentIdea._extraLines.push(line)
+      continue
+    }
+
+    // Every idea-shaped heading in the document, recorded BEFORE any branch can
+    // swallow it. What is consumed as a real idea or as an umbrella is
+    // subtracted at the end; whatever is left is content the parser could see
+    // and could not read, which the migration gate must refuse rather than
+    // treat as absent.
+    const shapeHere = line.match(IDEA_DECL_SHAPE_RE)
+    if (shapeHere) shapes.push(shapeHere[1])
+
     // Detect section boundaries
-    if (/^##\s+Ideas/.test(line)) {
+    if (/^##\s+Ideas\s*$/.test(line)) {
       flushCurrentIdea()
       inIdeasSection = true
       inKilledSection = false
@@ -123,7 +235,7 @@ export function parseIdeabox(markdown) {
       currentCluster = null
       continue
     }
-    if (/^##\s+Killed\s+Ideas/.test(line)) {
+    if (/^##\s+Killed\s+Ideas\s*$/.test(line)) {
       flushCurrentIdea()
       inIdeasSection = false
       inKilledSection = true
@@ -132,8 +244,34 @@ export function parseIdeabox(markdown) {
       continue
     }
     // Other H2 sections end both
-    if (/^##\s/.test(line) && !(/^##\s+Ideas/.test(line)) && !(/^##\s+Killed\s+Ideas/.test(line))) {
+    if (/^##\s/.test(line) && !(/^##\s+Ideas\s*$/.test(line)) && !(/^##\s+Killed\s+Ideas\s*$/.test(line))) {
       flushCurrentIdea()
+      if (legacy) {
+        // A `## Topic` heading in the legacy dialect is not the end of the
+        // ideas — it is how that dialect GROUPED them, which is precisely what
+        // an umbrella is. Mapping it to a cluster instead of discarding it
+        // means the upgrade preserves the author's grouping rather than
+        // flattening ten topics into one undifferentiated list. Treating it as
+        // preamble (the previous behaviour) collected every topic heading at
+        // the top of the file, detached from its ideas.
+        const name = line.replace(/^##\s+/, '').trim()
+        seenAnySection = true
+        // Leaving the killed section matters: without this, every idea under a
+        // topic heading that happened to follow `## Killed Ideas` was imported
+        // as KILLED. A topic heading opens a group, it does not inherit the
+        // previous section's disposition — and it re-enters the ideas section,
+        // which `## Killed Ideas` had closed, or the ideas below it are read as
+        // neither live nor killed and vanish entirely.
+        inKilledSection = false
+        inIdeasSection = true
+        currentCluster = name
+        if (!clusterIndex.has(name)) {
+          const entry = { name, theme: '', order: clusters.length }
+          clusters.push(entry)
+          clusterIndex.set(name, entry)
+        }
+        continue
+      }
       inIdeasSection = false
       inKilledSection = false
       // Still part of the preamble when it precedes the first real section —
@@ -142,6 +280,15 @@ export function parseIdeabox(markdown) {
       continue
     }
 
+    // Legacy preamble: everything before the first topic heading or idea. The
+    // modern collector below is unreachable here because legacy mode is inside
+    // the ideas section from line one, so without this the document's title and
+    // introduction were dropped on the first render after migration.
+    if (legacy && !seenAnySection && !currentIdea) {
+      preambleLines.push(line)
+      continue
+    }
+
     if (!inIdeasSection && !inKilledSection) {
       // Preamble = everything before the first section heading.
       if (!seenAnySection) preambleLines.push(line)
@@ -156,9 +303,14 @@ export function parseIdeabox(markdown) {
       continue
     }
 
-    // H3 = cluster heading
-    if (/^###\s/.test(line)) {
+    // H3 = cluster heading (modern dialect only — in the legacy dialect H3 IS
+    // the idea heading, handled below, and there is no cluster level at all).
+    // An H3 carrying an idea id is an IDEA at this level too, not an umbrella
+    // named after one; the hybrid dialect writes both at H3.
+    if (!legacy && /^###\s/.test(line) && !(hybrid && IDEA_HEADING_ANY_RE.test(line))) {
       flushCurrentIdea()
+      const eaten = line.match(IDEA_DECL_SHAPE_RE)
+      if (eaten) clusterNamedIds.push(eaten[1])
       currentCluster = line.replace(/^###\s+/, '').trim()
       if (!clusterIndex.has(currentCluster)) {
         const entry = { name: currentCluster, theme: '', order: clusters.length }
@@ -179,7 +331,7 @@ export function parseIdeabox(markdown) {
     }
 
     // H4 = idea heading
-    const headingMatch = line.match(IDEA_HEADING_RE)
+    const headingMatch = line.match(ideaHeadingRe)
     if (headingMatch) {
       flushCurrentIdea()
       currentIdea = {
@@ -273,7 +425,47 @@ export function parseIdeabox(markdown) {
   // serializer re-adds the separator itself.
   while (preambleLines.length && preambleLines.at(-1).trim() === '') preambleLines.pop()
 
-  return { ideas, killed, nextId, clusters, preamble: preambleLines.join('\n') }
+  // The bullet form is a fallback, not a rule: it counts ONLY for a document
+  // with no idea headings whatsoever. Counting bullets unconditionally made
+  // `- IDEA-1 needs research`, written inside IDEA-1's own body, a second
+  // declaration of IDEA-1 — and the gate then refused a perfectly readable file.
+  // A reference is not a declaration, and the only document where a bullet
+  // plausibly IS one is a document that declares nothing any other way.
+  const parsedIds = [...ideas, ...killed].map((i) => i.id)
+
+  // By COUNT, not by membership. A half-converted document declares the same id
+  // twice — once in each dialect — and the parser consumes only one of them; a
+  // membership test sees the id in `parsedIds` and calls it accounted for, while
+  // the other copy (routinely the older, richer one) is invisible and would be
+  // deleted by the next render. Counting says two were written and one was read.
+  const count = (arr, id) => arr.filter((x) => x === id).length
+  const unconsumed = [...new Set(shapes)].filter(
+    (id) => count(shapes, id) > count(parsedIds, id) + count(clusterNamedIds, id),
+  )
+  // An umbrella named after an id that is also a real idea here.
+  const collisions = clusterNamedIds.filter((id) => parsedIds.includes(id))
+
+  if (!ideas.length && !killed.length && !unconsumed.length) {
+    let bulletFence = false
+    for (const line of lines) {
+      if (/^\s*```/.test(line)) { bulletFence = !bulletFence; continue }
+      if (bulletFence) continue
+      const m = line.match(IDEA_BULLET_SHAPE_RE)
+      if (m && !unconsumed.includes(m[1])) unconsumed.push(m[1])
+    }
+  }
+
+  return {
+    ideas,
+    killed,
+    nextId,
+    clusters,
+    preamble: preambleLines.join('\n'),
+    // Idea-shaped content this parse could NOT turn into an idea.
+    unconsumed,
+    // Umbrellas named after ids that are also real ideas in this document.
+    collisions,
+  }
 }
 
 function extractStatus(raw) {
@@ -647,7 +839,15 @@ export function readIdeabox(cwd, ideaboxPath) {
     return { ideas: [], killed: [], nextId: 1 }
   }
   const markdown = readFileSync(fullPath, 'utf-8')
-  return parseIdeabox(markdown)
+  // FU-2. Returning a partial parse from here made "I could not read this file"
+  // indistinguishable from "this file has nothing in it" for every caller —
+  // most visibly `compose new --from-idea`, which reported "idea not found" for
+  // an idea plainly present in the file and then built a feature without the
+  // content it had been asked for. The assertion lives in its own leaf module
+  // (`fluid/ideabox-readable.js`) precisely so the parser's module can call it:
+  // the migration gate imports `parseIdeabox` from here, so keeping it there
+  // would close an import cycle.
+  return assertIdeaboxReadable(parseIdeabox(markdown), fullPath)
 }
 
 /**