@ktpartners/dgs-platform 2.8.0 → 3.0.4

package/deliver-great-systems/bin/lib/ideas.cjs

@@ -72,7 +72,8 @@ function allocateId(cwd) {
 
 // ─── Directory & File Helpers ─────────────────────────────────────────────────
 
-const IDEA_STATES = ['pending', 'done', 'rejected', 'consolidated'];
+const IDEA_STATUSES = ['pending', 'done', 'rejected', 'consolidated'];
+const IDEA_STATES = IDEA_STATUSES;
 
 /**
  * Ensure ideas/{pending,done,rejected}/ directories exist.
@@ -208,6 +209,9 @@ function buildIdeaContent(frontmatter, body, notes, discussionLog, researchLog)
   let yaml = '---\n';
   yaml += `id: ${frontmatter.id}\n`;
   yaml += `title: "${frontmatter.title}"\n`;
+  if (frontmatter.status) {
+    yaml += `status: ${frontmatter.status}\n`;
+  }
   if (frontmatter.tags && frontmatter.tags.length > 0) {
     yaml += `tags: [${frontmatter.tags.map(t => `"${t}"`).join(', ')}]\n`;
   } else {
@@ -227,6 +231,22 @@ function buildIdeaContent(frontmatter, body, notes, discussionLog, researchLog)
   if (frontmatter.consolidated_into) {
     yaml += `consolidated_into: "${frontmatter.consolidated_into}"\n`;
   }
+  // Passthrough: preserve any unknown frontmatter fields (round-trip integrity)
+  const knownFields = new Set(['id', 'title', 'status', 'tags', 'created', 'updated', 'author', 'updated_by', 'consolidated_from', 'consolidated_into']);
+  for (const [key, value] of Object.entries(frontmatter)) {
+    if (knownFields.has(key)) continue;
+    if (value === null || value === undefined) continue;
+    if (Array.isArray(value)) {
+      yaml += `${key}: [${value.map(v => `"${v}"`).join(', ')}]\n`;
+    } else {
+      const sv = String(value);
+      if (sv.includes(':') || sv.includes('#') || sv.startsWith('[') || sv.startsWith('{')) {
+        yaml += `${key}: "${sv}"\n`;
+      } else {
+        yaml += `${key}: ${sv}\n`;
+      }
+    }
+  }
   yaml += '---\n';
 
   // Body
@@ -270,11 +290,12 @@ function buildIdeaContent(frontmatter, body, notes, discussionLog, researchLog)
 }
 
 /**
- * Find an idea file by numeric ID or exact filename across all state directories.
+ * Find an idea file by numeric ID or exact filename.
+ * Scans flat ideas/ directory first (frontmatter status), then falls back to legacy subdirectories.
  *
  * @param {string} cwd - Working directory
  * @param {string} idOrFilename - Numeric ID (e.g., "42", "042") or filename
- * @returns {{ path: string, state: string, filename: string, frontmatter: object, body: string, notes: string, discussionLog: string, researchLog: string } | null}
+ * @returns {{ path: string, state: string, status: string, filename: string, frontmatter: object, body: string, notes: string, discussionLog: string, researchLog: string } | null}
  */
 function findIdeaFile(cwd, idOrFilename) {
   if (!idOrFilename) return null;
@@ -282,6 +303,24 @@ function findIdeaFile(cwd, idOrFilename) {
   const idStr = idOrFilename.replace(/^0+/, '') || '0';
   const paddedId = String(parseInt(idStr, 10)).padStart(3, '0');
 
+  // Flat-first scan: check ideas/ root directory
+  const flatDir = path.join(getPlanningRoot(cwd), 'ideas');
+  try {
+    const flatFiles = fs.readdirSync(flatDir).filter(f => f.endsWith('.md') && !fs.statSync(path.join(flatDir, f)).isDirectory());
+    for (const file of flatFiles) {
+      if (file.startsWith(paddedId + '-') || file === idOrFilename) {
+        const content = safeReadFile(path.join(flatDir, file));
+        if (!content) continue;
+        const { frontmatter, body, notes, discussionLog, researchLog } = parseIdeaFrontmatter(content);
+        const status = frontmatter.status || 'pending';
+        return { path: path.join(flatDir, file), state: status, status, filename: file, frontmatter, body, notes, discussionLog, researchLog };
+      }
+    }
+  } catch {
+    // Flat directory may not exist yet — continue to legacy scan
+  }
+
+  // Legacy subdirectory fallback
   for (const state of IDEA_STATES) {
     const dir = path.join(getPlanningRoot(cwd), 'ideas', state);
     let files;
@@ -293,18 +332,22 @@ function findIdeaFile(cwd, idOrFilename) {
 
     for (const file of files) {
       // Match by ID prefix
-      if (file.startsWith(paddedId + '-')) {
+      if (file.startsWith(paddedId + '-') || file === idOrFilename) {
         const content = safeReadFile(path.join(dir, file));
         if (!content) continue;
         const { frontmatter, body, notes, discussionLog, researchLog } = parseIdeaFrontmatter(content);
-        return { path: path.join(dir, file), state, filename: file, frontmatter, body, notes, discussionLog, researchLog };
-      }
-      // Match by exact filename
-      if (file === idOrFilename) {
-        const content = safeReadFile(path.join(dir, file));
-        if (!content) continue;
-        const { frontmatter, body, notes, discussionLog, researchLog } = parseIdeaFrontmatter(content);
-        return { path: path.join(dir, file), state, filename: file, frontmatter, body, notes, discussionLog, researchLog };
+
+        // Legacy fallback warning
+        process.stderr.write(`[DGS] Warning: idea '${paddedId}' found in legacy ${state}/ directory. Run migration to flatten.\n`);
+
+        // Check frontmatter status vs directory disagreement
+        const fmStatus = frontmatter.status;
+        if (fmStatus && fmStatus !== state) {
+          process.stderr.write(`[DGS] Warning: idea '${paddedId}' frontmatter status '${fmStatus}' disagrees with directory '${state}'. Frontmatter wins.\n`);
+        }
+
+        const effectiveStatus = fmStatus || state;
+        return { path: path.join(dir, file), state: effectiveStatus, status: effectiveStatus, filename: file, frontmatter, body, notes, discussionLog, researchLog };
       }
     }
   }
@@ -329,14 +372,14 @@ function cmdIdeasCreate(cwd, title, body, tags, raw, author) {
     error('title required for ideas create');
   }
 
-  ensureIdeasDirs(cwd);
+  const planRoot = getPlanningRoot(cwd);
+  fs.mkdirSync(path.join(planRoot, 'ideas'), { recursive: true });
 
   const id = allocateId(cwd);
   const slug = generateSlugInternal(title);
   const filename = `${id}-${slug}.md`;
-  const planRoot = getPlanningRoot(cwd);
   const planRootRel = path.relative(cwd, planRoot) || '.';
-  const filePath = path.join(planRoot, 'ideas', 'pending', filename);
+  const filePath = path.join(planRoot, 'ideas', filename);
   const now = new Date().toISOString();
 
   const parsedTags = tags
@@ -346,6 +389,7 @@ function cmdIdeasCreate(cwd, title, body, tags, raw, author) {
   const frontmatter = {
     id: parseInt(id, 10),
     title,
+    status: 'pending',
     tags: parsedTags,
     created: now,
     updated: now,
@@ -362,7 +406,7 @@ function cmdIdeasCreate(cwd, title, body, tags, raw, author) {
   const result = {
     id: parseInt(id, 10),
     filename,
-    path: path.join(planRootRel, 'ideas', 'pending', filename),
+    path: path.join(planRootRel, 'ideas', filename),
     title,
   };
   output(result, raw);
@@ -491,7 +535,8 @@ function cmdIdeasAppendNote(cwd, idOrFilename, noteText, raw, author) {
 }
 
 /**
- * List ideas grouped by state with optional filtering.
+ * List ideas grouped by status with optional filtering.
+ * Scans flat ideas/ directory first (frontmatter status), then falls back to legacy subdirectories.
  *
  * @param {string} cwd - Working directory
  * @param {{ state: string|null, tag: string|null, include_orphan_check: boolean, include_consolidated: boolean, show_all: boolean }} options
@@ -502,7 +547,10 @@ function cmdIdeasList(cwd, options, raw) {
   const counts = { pending: 0, done: 0, rejected: 0, consolidated: 0, total: 0 };
   const orphanedIds = [];
 
-  // Determine which states to iterate
+  // Track IDs seen in flat directory to avoid duplicates from legacy fallback
+  const seenIds = new Set();
+
+  // Determine which states to iterate (for legacy fallback and status filter)
   let statesToList;
   if (options.state) {
     statesToList = [options.state];
@@ -514,6 +562,64 @@ function cmdIdeasList(cwd, options, raw) {
     statesToList = ['pending', 'done', 'rejected']; // default: hide consolidated
   }
 
+  // Flat-first scan: check ideas/ root directory
+  const flatDir = path.join(getPlanningRoot(cwd), 'ideas');
+  try {
+    const flatFiles = fs.readdirSync(flatDir).filter(f => f.endsWith('.md') && !fs.statSync(path.join(flatDir, f)).isDirectory());
+    for (const file of flatFiles) {
+      const content = safeReadFile(path.join(flatDir, file));
+      if (!content) continue;
+      const { frontmatter, discussionLog, researchLog } = parseIdeaFrontmatter(content);
+      const status = frontmatter.status || 'pending';
+
+      // Apply status filter (using existing --state flag for backward compat)
+      if (options.state && status !== options.state) continue;
+      if (!options.state && options.include_consolidated && status !== 'consolidated') continue;
+      if (!options.state && !options.include_consolidated && !options.show_all && status === 'consolidated') continue;
+      if (!options.state && !options.show_all && !options.include_consolidated && !['pending', 'done', 'rejected'].includes(status)) continue;
+
+      // Tag filter (case-insensitive)
+      if (options.tag) {
+        const tagLower = options.tag.toLowerCase();
+        const tags = (frontmatter.tags || []).map(t => t.toLowerCase());
+        if (!tags.includes(tagLower)) continue;
+      }
+
+      // Author filter (case-insensitive substring match on name-only)
+      const authorName = extractNameFromAuthor(frontmatter.author || '');
+      if (options.author) {
+        const filterAuthor = options.author.toLowerCase();
+        if (!authorName.toLowerCase().includes(filterAuthor)) continue;
+      }
+
+      const ideaObj = {
+        id: frontmatter.id,
+        title: frontmatter.title || '',
+        tags: frontmatter.tags || [],
+        created: frontmatter.created || '',
+        updated: frontmatter.updated || '',
+        author: authorName,
+        state: status,   // backward compat
+        status,          // new preferred field
+        filename: file,
+        discussed: !!(discussionLog && discussionLog.trim().length > 0),
+        researched: !!(researchLog && researchLog.trim().length > 0),
+      };
+
+      if (status === 'consolidated' && frontmatter.consolidated_into) {
+        ideaObj.consolidated_into = frontmatter.consolidated_into;
+      }
+
+      ideas.push(ideaObj);
+      if (counts[status] !== undefined) counts[status]++;
+      counts.total++;
+      seenIds.add(frontmatter.id);
+    }
+  } catch {
+    // Flat directory may not exist — continue to legacy scan
+  }
+
+  // Legacy subdirectory fallback
   for (const state of statesToList) {
 
     const dir = path.join(getPlanningRoot(cwd), 'ideas', state);
@@ -530,6 +636,14 @@ function cmdIdeasList(cwd, options, raw) {
 
       const { frontmatter, discussionLog, researchLog } = parseIdeaFrontmatter(content);
 
+      // Skip if already found in flat directory
+      if (seenIds.has(frontmatter.id)) continue;
+
+      // Legacy fallback warning
+      process.stderr.write(`[DGS] Warning: idea '${file}' found in legacy ${state}/ directory. Run migration to flatten.\n`);
+
+      const effectiveStatus = frontmatter.status || state;
+
       // Tag filter (case-insensitive)
       if (options.tag) {
         const tagLower = options.tag.toLowerCase();
@@ -551,26 +665,28 @@ function cmdIdeasList(cwd, options, raw) {
         created: frontmatter.created || '',
         updated: frontmatter.updated || '',
         author: authorName,
-        state,
+        state: effectiveStatus,   // backward compat
+        status: effectiveStatus,  // new preferred field
         filename: file,
         discussed: !!(discussionLog && discussionLog.trim().length > 0),
         researched: !!(researchLog && researchLog.trim().length > 0),
       };
 
-      if (state === 'consolidated' && frontmatter.consolidated_into) {
+      if (effectiveStatus === 'consolidated' && frontmatter.consolidated_into) {
         ideaObj.consolidated_into = frontmatter.consolidated_into;
       }
 
       ideas.push(ideaObj);
 
-      counts[state]++;
+      if (counts[effectiveStatus] !== undefined) counts[effectiveStatus]++;
       counts.total++;
+      seenIds.add(frontmatter.id);
     }
   }
 
   // Orphan check: scan specs for references to done idea IDs
   if (options.include_orphan_check) {
-    const doneIdeas = ideas.filter(i => i.state === 'done');
+    const doneIdeas = ideas.filter(i => i.status === 'done');
     const specsDir = path.join(getPlanningRoot(cwd), 'specs');
 
     let specContent = '';
@@ -649,13 +765,8 @@ function cmdIdeasReject(cwd, idOrFilename, reason, raw, author) {
   const rebuilt = buildIdeaContent(fm, bd, updatedNotes, dl, rl);
   fs.writeFileSync(idea.path, rebuilt, 'utf-8');
 
-  ensureIdeasDirs(cwd);
-
-  // Git mv from pending to rejected
-  const planRootRel = path.relative(cwd, getPlanningRoot(cwd)) || '.';
-  const fromRel = path.join(planRootRel, 'ideas', 'pending', idea.filename);
-  const toRel = path.join(planRootRel, 'ideas', 'rejected', idea.filename);
-  execGit(cwd, ['mv', fromRel, toRel]);
+  // Set status to rejected via frontmatter edit
+  setIdeaStatus(cwd, idOrFilename, 'rejected');
 
   // Auto-commit
   execGit(cwd, ['add', '-A']);
@@ -714,13 +825,8 @@ function cmdIdeasRestore(cwd, idOrFilename, raw, author) {
     fs.writeFileSync(idea.path, buildIdeaContent(fm, bd, nt, dl, rl), 'utf-8');
   }
 
-  ensureIdeasDirs(cwd);
-
-  // Git mv to pending (keep rejection reason in file as history)
-  const planRootRel = path.relative(cwd, getPlanningRoot(cwd)) || '.';
-  const fromRel = path.join(planRootRel, 'ideas', idea.state, idea.filename);
-  const toRel = path.join(planRootRel, 'ideas', 'pending', idea.filename);
-  execGit(cwd, ['mv', fromRel, toRel]);
+  // Set status to pending via frontmatter edit
+  setIdeaStatus(cwd, idOrFilename, 'pending');
 
   // Auto-commit
   execGit(cwd, ['add', '-A']);
@@ -737,6 +843,48 @@ function cmdIdeasRestore(cwd, idOrFilename, raw, author) {
   output(result, raw);
 }
 
+/**
+ * Set an idea's frontmatter status field (internal helper — throws on error).
+ *
+ * This is the foundation function that Phase 133's cmdSpecsFinalize will call
+ * instead of using git mv. It edits frontmatter in-place without moving the file.
+ *
+ * @param {string} cwd - Working directory
+ * @param {string} idOrFilename - Idea ID or filename
+ * @param {string} status - Target status (must be in IDEA_STATUSES)
+ * @param {string} [author] - Author string for updated_by
+ * @returns {{ id: number, filename: string, previous_status: string, status: string }}
+ * @throws {Error} If status is invalid, idea not found, or already in target status
+ */
+function setIdeaStatus(cwd, idOrFilename, status, author) {
+  if (!IDEA_STATUSES.includes(status)) {
+    throw new Error(`Invalid status: ${status}. Allowed: ${IDEA_STATUSES.join(', ')}`);
+  }
+
+  const idea = findIdeaFile(cwd, idOrFilename);
+  if (!idea) {
+    throw new Error(`Idea not found: ${idOrFilename}`);
+  }
+
+  const previousStatus = idea.frontmatter.status || idea.state;
+
+  // Update frontmatter
+  idea.frontmatter.status = status;
+  idea.frontmatter.updated = new Date().toISOString();
+  if (author) {
+    idea.frontmatter.updated_by = author;
+  }
+
+  // Write back using buildIdeaContent (preserves all fields)
+  const content = buildIdeaContent(
+    idea.frontmatter, idea.body, idea.notes,
+    idea.discussionLog, idea.researchLog
+  );
+  fs.writeFileSync(idea.path, content, 'utf-8');
+
+  return { id: idea.frontmatter.id, filename: idea.filename, previous_status: previousStatus, status };
+}
+
 /**
  * Move an idea to a target state (general-purpose state transition).
  *
@@ -753,49 +901,18 @@ function cmdIdeasMoveState(cwd, idOrFilename, targetState, raw, author) {
   if (!targetState) {
     error('target state required for ideas move-state');
   }
-  if (!IDEA_STATES.includes(targetState)) {
-    error(`invalid state: ${targetState}. Use ${IDEA_STATES.join(', ')}`);
-  }
 
-  const idea = findIdeaFile(cwd, idOrFilename);
-  if (!idea) {
-    error(`idea not found: ${idOrFilename}`);
-  }
+  try {
+    const result = setIdeaStatus(cwd, idOrFilename, targetState, author);
 
-  if (idea.state === targetState) {
-    error(`idea ${idOrFilename} is already in ${targetState}`);
-  }
+    // Git add + commit (replaces the old git mv + commit)
+    execGit(cwd, ['add', '-A']);
+    execGit(cwd, ['commit', '-m', `ideas: set #${result.id} status to ${targetState}`]);
 
-  // Set updated_by if author provided
-  if (author) {
-    const currentContent = safeReadFile(idea.path) || '';
-    const { frontmatter: fm, body: bd, notes: nt, discussionLog: dl, researchLog: rl } = parseIdeaFrontmatter(currentContent);
-    fm.updated_by = author;
-    fm.updated = new Date().toISOString();
-    fs.writeFileSync(idea.path, buildIdeaContent(fm, bd, nt, dl, rl), 'utf-8');
+    output(result, raw);
+  } catch (err) {
+    error(err.message);
   }
-
-  ensureIdeasDirs(cwd);
-
-  // Git mv
-  const planRootRel = path.relative(cwd, getPlanningRoot(cwd)) || '.';
-  const fromRel = path.join(planRootRel, 'ideas', idea.state, idea.filename);
-  const toRel = path.join(planRootRel, 'ideas', targetState, idea.filename);
-  execGit(cwd, ['mv', fromRel, toRel]);
-
-  // Auto-commit
-  execGit(cwd, ['add', '-A']);
-  const title = idea.frontmatter.title || idea.filename;
-  const idNum = idea.frontmatter.id || idOrFilename;
-  execGit(cwd, ['commit', '-m', `ideas: move #${idNum} - ${title} to ${targetState}`]);
-
-  const result = {
-    id: idea.frontmatter.id,
-    filename: idea.filename,
-    from: idea.state,
-    to: targetState,
-  };
-  output(result, raw);
 }
 
 // ─── Section Entry Helpers ────────────────────────────────────────────────────
@@ -1061,8 +1178,6 @@ function cmdIdeasConsolidate(cwd, options, raw, author) {
     sourceIdeas.push(idea);
   }
 
-  ensureIdeasDirs(cwd);
-
   // Collect tags: union of all source tags + provided tags (deduplicated)
   const allTags = new Set();
   for (const idea of sourceIdeas) {
@@ -1080,7 +1195,8 @@ function cmdIdeasConsolidate(cwd, options, raw, author) {
   const filename = `${newId}-${slug}.md`;
   const planRoot = getPlanningRoot(cwd);
   const planRootRel = path.relative(cwd, planRoot) || '.';
-  const filePath = path.join(planRoot, 'ideas', 'pending', filename);
+  const filePath = path.join(planRoot, 'ideas', filename);
+  fs.mkdirSync(path.join(planRoot, 'ideas'), { recursive: true });
   const now = new Date().toISOString();
   const dateStr = now.split('T')[0];
 
@@ -1088,6 +1204,7 @@ function cmdIdeasConsolidate(cwd, options, raw, author) {
   const frontmatter = {
     id: parseInt(newId, 10),
     title,
+    status: 'pending',
     tags: [...allTags],
     created: now,
     updated: now,
@@ -1098,7 +1215,7 @@ function cmdIdeasConsolidate(cwd, options, raw, author) {
   }
   frontmatter.consolidated_from = sourceIdeas.map(i => String(i.frontmatter.id).padStart(3, '0'));
 
-  // Write new consolidated idea to pending/
+  // Write new consolidated idea to flat ideas/
   const content = buildIdeaContent(frontmatter, body || '', '', discussion || '', research || '');
   fs.writeFileSync(filePath, content, 'utf-8');
 
@@ -1116,15 +1233,13 @@ function cmdIdeasConsolidate(cwd, options, raw, author) {
     // Append consolidation note
     appendConsolidationNote(idea.path, newId, title, dateStr);
 
-    // Git mv source from pending/ to consolidated/
-    const fromRel = path.join(planRootRel, 'ideas', 'pending', idea.filename);
-    const toRel = path.join(planRootRel, 'ideas', 'consolidated', idea.filename);
-    execGit(cwd, ['mv', fromRel, toRel]);
+    // Set status to consolidated via frontmatter edit
+    setIdeaStatus(cwd, String(idea.frontmatter.id), 'consolidated');
 
     movedFiles.push({
       id: idea.frontmatter.id,
       filename: idea.filename,
-      from: 'pending',
+      from: idea.state,
       to: 'consolidated',
     });
   }
@@ -1138,7 +1253,7 @@ function cmdIdeasConsolidate(cwd, options, raw, author) {
   const result = {
     id: parseInt(newId, 10),
     filename,
-    path: path.join(planRootRel, 'ideas', 'pending', filename),
+    path: path.join(planRootRel, 'ideas', filename),
     title,
     consolidated_from: frontmatter.consolidated_from,
     moved_files: movedFiles,
@@ -1350,16 +1465,14 @@ function cmdIdeasUndoConsolidation(cwd, idOrFilename, raw, author) {
     if (author) parsed.frontmatter.updated_by = author;
     fs.writeFileSync(sourceIdea.path, buildIdeaContent(parsed.frontmatter, parsed.body, parsed.notes, parsed.discussionLog, parsed.researchLog), 'utf-8');
 
-    // Git mv from consolidated/ to pending/
-    const fromRel = path.join(planRootRel, 'ideas', 'consolidated', sourceIdea.filename);
-    const toRel = path.join(planRootRel, 'ideas', 'pending', sourceIdea.filename);
-    execGit(cwd, ['mv', fromRel, toRel]);
+    // Set status to pending via frontmatter edit
+    setIdeaStatus(cwd, sourceId, 'pending');
 
     restoredIds.push(parseInt(sourceId, 10));
   }
 
-  // Delete the consolidated result idea
-  const ideaRel = path.join(planRootRel, 'ideas', idea.state, idea.filename);
+  // Delete the consolidated result idea (flat directory — no state subdir)
+  const ideaRel = path.join(planRootRel, 'ideas', idea.filename);
   execGit(cwd, ['rm', ideaRel]);
 
   // Atomic git commit
@@ -1380,7 +1493,9 @@ function cmdIdeasUndoConsolidation(cwd, idOrFilename, raw, author) {
 // ─── Exports ──────────────────────────────────────────────────────────────────
 
 module.exports = {
+  IDEA_STATUSES,
   IDEA_STATES,
+  setIdeaStatus,
   loadManifest,
   saveManifest,
   allocateId,