@smartmemory/compose 0.3.6-beta → 0.3.8

package/lib/ideabox-cli.js

@@ -0,0 +1,315 @@
+/**
+ * lib/ideabox-cli.js — `compose ideabox …`, writing to the record store.
+ *
+ * COMP-PLAN-IDEA-UNIFY S3b-1 (D18), rebased onto the shared operations in S3b-2
+ * (D20).
+ *
+ * Lifted out of a ~300-line inline `else if` block in `bin/compose.js` with no
+ * named symbol, in a 3,400-line file. The move is not cosmetic: it is what lets
+ * the cutover be driven in-process by a test instead of only through a
+ * subprocess, and the cutover is the part that needed testing most.
+ *
+ * WHAT THIS MODULE IS NOW
+ * -----------------------
+ * Argument parsing, presentation, and exit codes. Nothing else. Every mutation
+ * lives in `fluid/ideabox-ops.js`, which the REST API calls too — including the
+ * two invariants that used to be spelled out here and would have had to be
+ * re-remembered by every new surface:
+ *
+ *   1. the migration gate runs before any write, and
+ *   2. the projection is rewritten only after the record is durable.
+ *
+ * A surface that does not implement an invariant cannot forget it. That is the
+ * whole reason the ops module exists; see its header for the failure it prevents.
+ *
+ * Behaviour is preserved deliberately, including output wording, exit codes and
+ * flag parsing, because a cutover that also redesigns the commands cannot be
+ * reviewed: any difference becomes ambiguous between "intended" and
+ * "regression". The exceptions are documented — `normalizeTags` in the ops
+ * module, the killed-idea promote refusal (D22), and the new `resurrect`
+ * subcommand (D23).
+ */
+
+import { existsSync } from 'node:fs';
+
+import {
+  ideaboxContext,
+  addIdea,
+  addDiscussion,
+  killIdea,
+  promoteIdea,
+  resurrectIdea,
+  setPriority,
+  updateIdea,
+  IdeaboxConflict,
+  IdeaboxInvalid,
+  IdeaboxNotFound,
+} from './fluid/ideabox-ops.js';
+import { toMarkdownDate } from './fluid/ideabox-dates.js';
+import { KIND } from './fluid/provider.js';
+import { ensureIdeaboxMigrated } from './fluid/ideabox-migrate.js';
+import { writeIdeaboxProjection } from './fluid/render-ideabox.js';
+
+const USAGE = [
+  'Usage: compose ideabox <subcommand>',
+  '',
+  'Subcommands:',
+  '  add "<title>"                Add a new idea',
+  '  list                         List all ideas',
+  '  promote <ID>                 Mark idea as PROMOTED (creates feature folder)',
+  '  kill <ID> "<reason>"         Move idea to Killed Ideas',
+  '  resurrect <ID>               Return a killed idea to the live set',
+  '  pri <ID> <P0|P1|P2>          Set priority',
+  '  discuss <ID> "<comment>"     Add a discussion comment',
+  '  triage [--lens <name>]       Walk untriaged ideas and assign priorities',
+  '  render                       Rewrite the ideabox file from the records',
+];
+
+const PRIORITIES = ['P0', 'P1', 'P2'];
+const flagValue = (args, name) => {
+  const i = args.indexOf(name);
+  return i !== -1 ? args[i + 1] : undefined;
+};
+
+/** Status enum → the legacy display label the list output has always used. */
+const displayStatus = (record) => {
+  if (record.status === 'promoted') return 'PROMOTED';
+  if (record.status === 'discussing') return 'DISCUSSING';
+  return 'NEW';
+};
+
+const displayPriority = (record) => record.priority ?? '—';
+
+/**
+ * Turn an op failure into the terminal's contract: a message on stderr and a
+ * non-zero exit. Only the ops' own typed failures are handled — anything else
+ * is a bug or a broken store, and swallowing those into a tidy exit code is how
+ * a corrupt project looks healthy.
+ */
+function reportOpFailure(err) {
+  if (err instanceof IdeaboxNotFound || err instanceof IdeaboxInvalid || err instanceof IdeaboxConflict) {
+    console.error(err.message);
+    return 1;
+  }
+  throw err;
+}
+
+/**
+ * @param {string} cwd project root
+ * @param {string[]} args argv after `ideabox`
+ * @param {object} [opts]
+ * @param {object} [opts.config] already-loaded `.compose/compose.json`
+ * @returns {Promise<number>} process exit code
+ */
+export async function runIdeaboxCommand(cwd, args, opts = {}) {
+  const sub = args[0];
+  if (!sub || sub === '--help' || sub === '-h') {
+    for (const line of USAGE) console.log(line);
+    return 0;
+  }
+
+  const ctx = await ideaboxContext(cwd, { config: opts.config, origin: 'cli:ideabox' });
+  const { provider, ideaboxPath } = ctx;
+
+  try {
+    switch (sub) {
+      case 'add': {
+        const title = args.slice(1).find((a) => !a.startsWith('-')) || args[1];
+        if (!title) {
+          console.error('Usage: compose ideabox add "<title>" [--source "..."] [--desc "..."] [--cluster "..."]');
+          return 1;
+        }
+        const { record, createdCluster } = await addIdea(ctx, {
+          title,
+          body: flagValue(args, '--desc') ?? '',
+          source: flagValue(args, '--source') ?? '',
+          tags: flagValue(args, '--tags') ?? [],
+          cluster: flagValue(args, '--cluster') ?? null,
+        });
+        if (createdCluster) console.log(`Created cluster ${createdCluster.handle}: ${createdCluster.title}`);
+        console.log(`Added ${record.handle}: ${record.title}`);
+        return 0;
+      }
+
+      case 'list': {
+        // Read-only paths still migrate: `list` on an upgraded project must show
+        // the user's real ideas, not an empty store. It is the same import
+        // either way, and running it lazily on first read rather than first
+        // write means the upgrade happens at the least destructive moment
+        // available.
+        if (!existsSync(ideaboxPath) && !(await provider.listRecords({ kind: KIND.IDEA })).length) {
+          console.log('No ideabox found. Run: compose ideabox add "<title>"');
+          return 0;
+        }
+        await ensureIdeaboxMigrated(provider, ideaboxPath);
+        const ideas = await provider.listRecords({ kind: KIND.IDEA });
+        if (ideas.length === 0) {
+          console.log('No ideas yet.');
+          return 0;
+        }
+
+        const live = ideas.filter((i) => i.status !== 'killed');
+        const killed = ideas.filter((i) => i.status === 'killed');
+        const order = { P0: 0, P1: 1, P2: 2, '—': 3 };
+
+        for (const status of ['NEW', 'DISCUSSING', 'PROMOTED']) {
+          const group = live.filter((i) => displayStatus(i) === status);
+          if (!group.length) continue;
+          group.sort((a, b) => (order[displayPriority(a)] ?? 3) - (order[displayPriority(b)] ?? 3));
+          console.log(`\n[${status}]`);
+          for (const idea of group) {
+            const pri = displayPriority(idea) !== '—' ? ` [${displayPriority(idea)}]` : '';
+            const tags = idea.tags.length ? ` ${idea.tags.join(' ')}` : '';
+            console.log(`  ${idea.handle}${pri}  ${idea.title}${tags}`);
+          }
+        }
+
+        if (killed.length) {
+          console.log(`\n[KILLED] (${killed.length})`);
+          for (const idea of killed) {
+            console.log(`  ${idea.handle}  ${idea.title}  — ${idea.killed?.reason ?? ''}`);
+          }
+        }
+        return 0;
+      }
+
+      case 'promote': {
+        const id = args[1];
+        if (!id) {
+          console.error('Usage: compose ideabox promote <ID> [<FEATURE-CODE>]');
+          return 1;
+        }
+        const { record, featureCode, featurePath, createdFeature } =
+          await promoteIdea(ctx, id, args[2] || '');
+        if (createdFeature) console.log(`Created feature folder: ${featurePath}/`);
+        console.log(`Promoted ${record.handle} → ${featureCode}`);
+        return 0;
+      }
+
+      case 'kill': {
+        const id = args[1];
+        if (!id) {
+          console.error('Usage: compose ideabox kill <ID> "<reason>"');
+          return 1;
+        }
+        const reason = args[2] || '';
+        const { record, alreadyKilled } = await killIdea(ctx, id, reason);
+        if (alreadyKilled) {
+          console.log(`${record.handle} was already killed on ${toMarkdownDate(record.killed?.at)}: ${record.killed?.reason ?? ''}`);
+        } else {
+          console.log(`Killed ${record.handle}: ${reason}`);
+        }
+        return 0;
+      }
+
+      case 'resurrect': {
+        const id = args[1];
+        if (!id) {
+          console.error('Usage: compose ideabox resurrect <ID>');
+          return 1;
+        }
+        const { record } = await resurrectIdea(ctx, id);
+        console.log(`Resurrected ${record.handle}: ${record.title}`);
+        return 0;
+      }
+
+      case 'pri': {
+        const id = args[1];
+        const priority = args[2];
+        if (!id || !priority) {
+          console.error('Usage: compose ideabox pri <ID> <P0|P1|P2>');
+          return 1;
+        }
+        if (!PRIORITIES.includes(priority.toUpperCase()) && priority !== '—') {
+          console.error(`Invalid priority: ${priority}. Use P0, P1, P2 or —`);
+          return 1;
+        }
+        const { record } = await setPriority(ctx, id, priority);
+        console.log(`Set ${record.handle} priority → ${priority}`);
+        return 0;
+      }
+
+      case 'discuss': {
+        const id = args[1];
+        const comment = args[2];
+        if (!id || !comment) {
+          console.error('Usage: compose ideabox discuss <ID> "<comment>"');
+          return 1;
+        }
+        await addDiscussion(ctx, id, { text: comment, author: 'human' });
+        console.log(`[${new Date().toISOString().slice(0, 10)}] human: ${comment}`);
+        return 0;
+      }
+
+      case 'triage': {
+        const lensName = flagValue(args, '--lens');
+        await ensureIdeaboxMigrated(provider, ideaboxPath);
+        const ideas = await provider.listRecords({ kind: KIND.IDEA });
+        const untriaged = ideas.filter((i) => !i.priority && i.status === 'new');
+        if (!untriaged.length) {
+          console.log('No untriaged ideas.');
+          return 0;
+        }
+
+        if (lensName) {
+          const { loadLens } = await import('./ideabox.js');
+          if (!loadLens(cwd, lensName)) {
+            console.warn(`Lens not found: docs/product/ideabox-priority-${lensName}.md`);
+          } else {
+            console.log(`Using lens: ${lensName}`);
+          }
+        }
+
+        const { createInterface } = await import('node:readline');
+        const rl = createInterface({ input: process.stdin, output: process.stdout });
+        const ask = (q) => new Promise((resolve) => rl.question(q, resolve));
+
+        let changed = false;
+        try {
+          for (const idea of untriaged) {
+            console.log(`\n${idea.handle}: ${idea.title}`);
+            if (idea.body) console.log(`  ${idea.body.slice(0, 120)}`);
+            if (lensName) console.log(`  [lens: ${lensName}]`);
+            const answer = (await ask('  Priority [P0/P1/P2/skip]: ')).trim().toUpperCase();
+            if (PRIORITIES.includes(answer)) {
+              // The record write only. Triage is a bulk pass, so it renders once
+              // at the end rather than rewriting the whole file per answer.
+              await provider.updateRecord(idea.handle, { priority: answer });
+              changed = true;
+              console.log(`  Set ${idea.handle} → ${answer}`);
+            } else {
+              console.log('  Skipped');
+            }
+          }
+        } finally {
+          rl.close();
+        }
+
+        if (changed) {
+          await writeIdeaboxProjection(provider, ideaboxPath);
+          console.log('\nSaved.');
+        }
+        return 0;
+      }
+
+      case 'render': {
+        // The repair path. Every mutation writes the record before the
+        // projection, so a failed render leaves durable canon and a stale file;
+        // this is how the file catches up without touching a record. Also the
+        // fix for a projection that was interrupted, and the way back from any
+        // hand edit.
+        await ensureIdeaboxMigrated(provider, ideaboxPath);
+        await writeIdeaboxProjection(provider, ideaboxPath);
+        console.log(`Rendered ${ideaboxPath}`);
+        return 0;
+      }
+
+      default:
+        console.error(`Unknown ideabox subcommand: ${sub}`);
+        console.error('Run: compose ideabox --help');
+        return 1;
+    }
+  } catch (err) {
+    return reportOpFailure(err);
+  }
+}

package/lib/ideabox.js

@@ -51,7 +51,22 @@ const IDEA_HEADING_RE = /^####\s+(IDEA-(\d+))\s+[—–-]+\s+(.+)$/
 const FIELD_RE = /^\*\*([^*:]+):\*\*\s*(.*)$/
 
 // Matches a discussion entry: - [2026-04-10] author: text
-const DISCUSSION_ENTRY_RE = /^-\s+\[(\d{4}-\d{2}-\d{2})\]\s+(\w+):\s+(.+)$/
+//
+// The author is "everything up to the first colon", not `\w+`.
+//
+// COMP-FLUID-SEAM-GUARANTEES F7-1. `\w+` matched no real person's name: an entry
+// written by `Jane Doe` rendered correctly into the file and then parsed to
+// ZERO discussion entries, silently, taking the comment with it and breaking the
+// `serialize(parse(projection))` fixed point the cutover rests on. It was
+// unreachable while the CLI was the only writer (it always writes `human`) and
+// became reachable the moment the REST API accepted an author from a request
+// body (COMP-PLAN-IDEA-UNIFY S3b-2).
+//
+// Lazy, so the FIRST colon delimits: an author cannot contain one (the contract
+// forbids it, since the file could not represent it), while the comment text
+// routinely does — `- [2026-08-05] human: see this: it matters` keeps the whole
+// sentence.
+const DISCUSSION_ENTRY_RE = /^-\s+\[(\d{4}-\d{2}-\d{2})\]\s+([^:\n]+?):\s+(.+)$/
 
 // ---------------------------------------------------------------------------
 // parseIdeabox(markdown) → { ideas, killed, nextId }
@@ -67,11 +82,22 @@ export function parseIdeabox(markdown) {
 
   const ideas = []
   const killed = []
+  // Cluster-scoped data. An umbrella heading carries a hand-authored multi-
+  // sentence `**Theme:**` paragraph, which is information, not decoration —
+  // before this was captured it sat between the H3 and the first H4 where the
+  // loop had no `currentIdea`, so it was silently dropped and every
+  // parse→serialize cycle (i.e. every `compose ideabox` mutation) deleted it.
+  const clusters = []
+  const clusterIndex = new Map()
+  // Everything before `## Ideas`. Regenerating this from IDEABOX_TEMPLATE
+  // instead of preserving it drops hand-authored convention bullets.
+  const preambleLines = []
 
   let inIdeasSection = false
   let inKilledSection = false
   let currentCluster = null
   let currentIdea = null
+  let seenAnySection = false
 
   function flushCurrentIdea() {
     if (!currentIdea) return
@@ -93,6 +119,7 @@ export function parseIdeabox(markdown) {
       flushCurrentIdea()
       inIdeasSection = true
       inKilledSection = false
+      seenAnySection = true
       currentCluster = null
       continue
     }
@@ -100,6 +127,7 @@ export function parseIdeabox(markdown) {
       flushCurrentIdea()
       inIdeasSection = false
       inKilledSection = true
+      seenAnySection = true
       currentCluster = null
       continue
     }
@@ -108,18 +136,48 @@ export function parseIdeabox(markdown) {
       flushCurrentIdea()
       inIdeasSection = false
       inKilledSection = false
+      // Still part of the preamble when it precedes the first real section —
+      // `## Conventions` lives here and was being dropped on the floor.
+      if (!seenAnySection) preambleLines.push(line)
       continue
     }
 
-    if (!inIdeasSection && !inKilledSection) continue
+    if (!inIdeasSection && !inKilledSection) {
+      // Preamble = everything before the first section heading.
+      if (!seenAnySection) preambleLines.push(line)
+      continue
+    }
+
+    // Horizontal rules inside the ideas section are structural separators
+    // between umbrellas. They are regenerated by the serializer from the
+    // cluster list, so capturing them here would attach each one to the
+    // PRECEDING idea's extra lines and duplicate every rule on write.
+    if (/^---\s*$/.test(line)) {
+      continue
+    }
 
     // H3 = cluster heading
     if (/^###\s/.test(line)) {
       flushCurrentIdea()
       currentCluster = line.replace(/^###\s+/, '').trim()
+      if (!clusterIndex.has(currentCluster)) {
+        const entry = { name: currentCluster, theme: '', order: clusters.length }
+        clusters.push(entry)
+        clusterIndex.set(currentCluster, entry)
+      }
       continue
     }
 
+    // Cluster-scoped `**Theme:**` paragraph — appears after the H3 and before
+    // the first H4, i.e. exactly where there is no current idea to attach it to.
+    if (currentCluster && !currentIdea) {
+      const themeMatch = line.match(/^\*\*Theme:\*\*\s*(.*)$/)
+      if (themeMatch) {
+        clusterIndex.get(currentCluster).theme = themeMatch[1].trim()
+        continue
+      }
+    }
+
     // H4 = idea heading
     const headingMatch = line.match(IDEA_HEADING_RE)
     if (headingMatch) {
@@ -211,7 +269,11 @@ export function parseIdeabox(markdown) {
   const maxNum = allNums.length ? Math.max(...allNums) : 0
   const nextId = maxNum + 1
 
-  return { ideas, killed, nextId }
+  // Trailing blank lines before `## Ideas` are structural, not content — the
+  // serializer re-adds the separator itself.
+  while (preambleLines.length && preambleLines.at(-1).trim() === '') preambleLines.pop()
+
+  return { ideas, killed, nextId, clusters, preamble: preambleLines.join('\n') }
 }
 
 function extractStatus(raw) {
@@ -232,7 +294,17 @@ function applyField(idea, key, val) {
       idea.priority = val.replace(/`/g, '').trim() || '—'
       break
     case 'Tags':
-      idea.tags = (val.match(/#\w+/g) || [])
+      // Accepts BOTH spellings. The documented convention is `` `#ux` ``, but
+      // the real ideabox writes bare words (`stratum integrity research-
+      // influence`), and the old `/#\w+/g` matched none of them — so all 20
+      // ideas parsed with zero tags and every CLI mutation stripped the lot.
+      // Tokens are kept verbatim (a leading `#` is preserved, never added) so
+      // whichever spelling a file uses survives a round-trip unchanged.
+      idea.tags = val
+        .replace(/`/g, ' ')
+        .trim()
+        .split(/\s+/)
+        .filter(Boolean)
       break
     case 'Source':
       idea.source = val
@@ -280,28 +352,42 @@ function applyField(idea, key, val) {
  * @param {{ ideas: IdeaEntry[], killed: IdeaEntry[], nextId: number }} parsedData
  * @returns {string}
  */
-export function serializeIdeabox({ ideas, killed }) {
+export function serializeIdeabox({ ideas, killed, clusters: clusterMeta = [], preamble = '' }) {
   const lines = []
 
-  lines.push('# Ideabox')
-  lines.push('')
-  lines.push('**Purpose:** Capture raw ideas before they\'re ready for the roadmap.')
-  lines.push('')
-  lines.push('## Conventions')
-  lines.push('- **ID:** `IDEA-N` (sequential, never reuse)')
-  lines.push('- **Status:** `NEW` | `DISCUSSING` | `PROMOTED` | `KILLED`')
-  lines.push('- **Priority:** `P0` (promote now) | `P1` (next up) | `P2` (backlog) | `—` (untriaged)')
-  lines.push('- **Source:** Where the idea came from')
-  lines.push('- **Tags:** `#ux` `#core` `#distribution` `#integration` `#research` `#infra`')
-  lines.push('')
+  if (preamble) {
+    // Round-trip the author's own preamble. Regenerating it from the template
+    // silently deletes any convention the project added (the real ideabox
+    // documents an `**Umbrella:**` rule the template has never known about).
+    lines.push(...preamble.split('\n'))
+    lines.push('')
+  } else {
+    lines.push('# Ideabox')
+    lines.push('')
+    lines.push('**Purpose:** Capture raw ideas before they\'re ready for the roadmap.')
+    lines.push('')
+    lines.push('## Conventions')
+    lines.push('- **ID:** `IDEA-N` (sequential, never reuse)')
+    lines.push('- **Status:** `NEW` | `DISCUSSING` | `PROMOTED` | `KILLED`')
+    lines.push('- **Priority:** `P0` (promote now) | `P1` (next up) | `P2` (backlog) | `—` (untriaged)')
+    lines.push('- **Source:** Where the idea came from')
+    lines.push('- **Tags:** `#ux` `#core` `#distribution` `#integration` `#research` `#infra`')
+    lines.push('')
+  }
   lines.push('## Ideas')
   lines.push('')
-  lines.push('<!-- Ideas grouped by potential feature cluster -->')
-  lines.push('')
+  if (!clusterMeta.length) {
+    lines.push('<!-- Ideas grouped by potential feature cluster -->')
+    lines.push('')
+  }
 
   // Group active ideas by cluster
   const clusters = new Map()
   const unclustered = []
+  // Seed in declared cluster order so an empty cluster keeps its place and its
+  // theme rather than disappearing.
+  for (const c of clusterMeta) clusters.set(c.name, [])
+  const themeOf = new Map(clusterMeta.map((c) => [c.name, c.theme]))
   for (const idea of ideas) {
     if (idea.cluster) {
       if (!clusters.has(idea.cluster)) clusters.set(idea.cluster, [])
@@ -312,8 +398,17 @@ export function serializeIdeabox({ ideas, killed }) {
   }
 
   for (const [cluster, clusterIdeas] of clusters) {
+    if (clusterMeta.length) {
+      lines.push('---')
+      lines.push('')
+    }
     lines.push(`### ${cluster}`)
     lines.push('')
+    const theme = themeOf.get(cluster)
+    if (theme) {
+      lines.push(`**Theme:** ${theme}`)
+      lines.push('')
+    }
     for (const idea of clusterIdeas) {
       lines.push(...serializeIdea(idea))
     }
@@ -348,14 +443,19 @@ function serializeIdea(idea) {
 
   if (idea.source) out.push(`**Source:** ${idea.source}`)
   if (idea.description) out.push(`**Idea:** ${idea.description}`)
-  if (idea.mapsTo) out.push(`**Maps to:** ${idea.mapsTo}`)
-  if (idea.effort) out.push(`**Effort:** ${idea.effort}`)
-  if (idea.impact) out.push(`**Impact:** ${idea.impact}`)
 
+  // Unrecognized field lines are emitted here, BEFORE the trailing known
+  // fields, because that is where they occur in practice: an idea that carries
+  // custom `**Idea (original):**` / `**Re-aim:**` blocks ends with `**Maps
+  // to:**`, and emitting extras last reordered the file on every write.
   for (const extra of (idea._extraLines || [])) {
     out.push(extra)
   }
 
+  if (idea.mapsTo) out.push(`**Maps to:** ${idea.mapsTo}`)
+  if (idea.effort) out.push(`**Effort:** ${idea.effort}`)
+  if (idea.impact) out.push(`**Impact:** ${idea.impact}`)
+
   // Discussion thread
   if (idea.discussion && idea.discussion.length > 0) {
     out.push('**Discussion:**')