@smartmemory/compose 0.2.57-beta → 0.3.0

package/lib/new.js

@@ -1,391 +1,246 @@
 /**
- * new.js — Product kickoff runner for `compose new`.
- *
- * Orchestrates project creation through a Stratum workflow:
- * research → brainstorm → gate → roadmap → gate → scaffold.
- *
- * Reuses the same dispatch loop pattern as build.js.
+ * Product kickoff runner for `compose new` using the Stratum TS ready loop.
  */
 
 import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs';
 import { join, basename } from 'node:path';
+import YAML from 'yaml';
 
 import { StratumMcpClient } from './stratum-mcp-client.js';
 import { runAndNormalize } from './result-normalizer.js';
-import { installFactoryShim } from './connector-factory-shim.js';
-import { buildStepPrompt, buildRetryPrompt, buildGateContext } from './step-prompt.js';
+import { buildStepPrompt, buildGateContext } from './step-prompt.js';
 import { promptGate } from './gate-prompt.js';
 import { VisionWriter } from './vision-writer.js';
 import { readFlowRound } from './flow-state.js';
-
 import { validateStep } from './step-validator.js';
 
-// STRAT-DEDUP-AGENTRUN-V3: connectors live in stratum-mcp; runAndNormalize
-// dispatches via stratum.agentRun(...) directly.
+const KICKOFF_VALIDATION = new Map([
+  ['research', {
+    artifact: 'docs/discovery/research.md',
+    criteria: [
+      'Contains at least 2 existing tools or prior art entries',
+      'Mentions architectural patterns or common approaches',
+      'Lists risks or pitfalls',
+    ],
+  }],
+  ['brainstorm', {
+    artifact: 'docs/discovery/brainstorm.md',
+    criteria: [
+      'Contains at least 3 features with short codes',
+      'Contains user stories in As a, I want, so that format',
+      'Contains at least 2 architecture options with trade-offs',
+    ],
+  }],
+  ['roadmap', {
+    artifact: 'ROADMAP.md',
+    criteria: [
+      'Contains a markdown table with feature codes and status columns',
+      'Features are organized into phases',
+      'All features have PLANNED status',
+    ],
+  }],
+]);
+
+function outputContract(spec, stepId) {
+  const step = spec.flows?.new?.steps?.find((candidate) => candidate.id === stepId);
+  const fields = typeof step?.out === 'string' ? spec.contracts?.[step.out] : null;
+  return fields && typeof fields === 'object' ? fields : {};
+}
 
-/**
- * Run the product kickoff pipeline.
- *
- * @param {string} intent - Product description / intent
- * @param {object} opts
- * @param {string}   [opts.cwd]              - Working directory (default: process.cwd())
- * @param {string}   [opts.projectName]      - Project name override
- * @param {boolean}  [opts.skipResearch]     - Skip the research step
- * @param {object}   [opts.gateOpts]         - Options for gate prompt (input/output streams)
- */
+function waitingGate(audit) {
+  return Object.entries(audit?.steps ?? {})
+    .find(([, state]) => state?.status === 'waiting_gate');
+}
+
+function gateDispatch(spec, stepId) {
+  const step = spec.flows?.new?.steps?.find((candidate) => candidate.id === stepId);
+  if (!step?.gate) throw new Error(`Kickoff gate ${stepId} is missing from the local TS spec`);
+  return { step_id: stepId, ...step.gate };
+}
+
+/** Run the product kickoff pipeline. */
 export async function runNew(intent, opts = {}) {
   const cwd = opts.cwd ?? process.cwd();
   const projectName = opts.projectName ?? basename(cwd);
-  const skipResearch = opts.skipResearch ?? false;
 
-  // Check for existing design doc — use as enriched intent if present
   const designDocPath = join(cwd, 'docs', 'design.md');
   if (existsSync(designDocPath)) {
-    const designDoc = readFileSync(designDocPath, 'utf-8');
     console.log('Found design doc at docs/design.md — using as enriched intent');
-    intent = `${intent}\n\n## Design Document\n${designDoc}`;
+    intent = `${intent}\n\n## Design Document\n${readFileSync(designDocPath, 'utf-8')}`;
   }
 
-  // Resolve project paths
   const composeDir = join(cwd, '.compose');
   const dataDir = join(composeDir, 'data');
-
-  // Ensure compose is initialized
-  const configPath = join(composeDir, 'compose.json');
-  if (!existsSync(configPath)) {
+  if (!existsSync(join(composeDir, 'compose.json'))) {
     throw new Error(`No .compose/compose.json found at ${cwd}. Run 'compose init' first.`);
   }
-
-  // Ensure discovery dir exists for brainstorm output
   mkdirSync(join(cwd, 'docs', 'discovery'), { recursive: true });
 
-  // Load kickoff spec
   const specPath = join(cwd, 'pipelines', 'new.stratum.yaml');
   if (!existsSync(specPath)) {
     throw new Error(`Kickoff spec not found: ${specPath}. Run 'compose init' to get default pipelines.`);
   }
 
-  // If skipResearch, temporarily add skip_if to research step
-  if (skipResearch) {
-    const { parse, stringify } = await import('yaml');
-    const spec = parse(readFileSync(specPath, 'utf-8'));
-    const researchStep = spec.flows?.new?.steps?.find(s => s.id === 'research');
-    if (researchStep && !researchStep.skip_if) {
-      researchStep.skip_if = 'true';
-      researchStep.skip_reason = 'Skipped by user (questionnaire)';
-      writeFileSync(specPath, stringify(spec, { lineWidth: 120 }));
-      console.log('Skipping research step (per questionnaire).\n');
-    }
+  const spec = YAML.parse(readFileSync(specPath, 'utf-8'));
+  if (opts.skipResearch) {
+    const research = spec.flows?.new?.steps?.find((step) => step.id === 'research');
+    if (research) research.when = 'false';
+    console.log('Skipping research step (per questionnaire).\n');
   }
 
-  const specYaml = readFileSync(specPath, 'utf-8');
+  const validateConfigs = KICKOFF_VALIDATION;
+  const cleanSpecYaml = YAML.stringify(spec, { lineWidth: 120 });
 
-  // Parse spec to extract validate configs per step
-  const { parse: parseYaml, stringify: stringifyYaml } = await import('yaml');
-  const specObj = parseYaml(specYaml);
-  const validateConfigs = new Map();
-  for (const step of specObj.flows?.new?.steps ?? []) {
-    if (step.validate) {
-      validateConfigs.set(step.id, step.validate);
-      delete step.validate;  // strip before sending to stratum
-    }
-  }
-  // Re-serialize with validate fields stripped
-  const cleanSpecYaml = stringifyYaml(specObj, { lineWidth: 120 });
-
-  // Vision writer
   const visionWriter = new VisionWriter(dataDir);
   const itemId = await visionWriter.ensureFeatureItem(projectName, projectName);
-
-  // Stratum MCP client (test override permitted via opts.stratum)
   const stratum = opts.stratum ?? new StratumMcpClient();
   if (!opts.stratum) await stratum.connect({ cwd });
 
-  if (opts.connectorFactory && !opts.stratum) {
-    installFactoryShim(stratum, opts.connectorFactory, cwd);
-  }
-
   try {
     console.log(`Starting product kickoff for "${projectName}"...`);
     console.log(`Intent: ${intent}\n`);
-
-    let response = await stratum.plan(cleanSpecYaml, 'new', { projectName, intent });
-
+    let response = await stratum.plan(
+      cleanSpecYaml,
+      'new',
+      { projectName, intent },
+      { workspaceRoot: cwd },
+    );
+    const flowId = response.runId;
     await visionWriter.updateItemStatus(itemId, 'in_progress');
-
-    const context = { cwd, featureCode: projectName, projectName, intent };
-
-    // Dispatch loop — same pattern as build.js
-    while (response.status !== 'complete' && response.status !== 'killed') {
-      const stepId = response.step_id;
-      const flowId = response.flow_id;
-      const stepNum = response.step_number ?? '?';
-      const totalSteps = response.total_steps ?? '?';
-
-      if (response.status === 'execute_step') {
-        console.log(`[${stepNum}/${totalSteps}] ${stepId}...`);
-
-        await visionWriter.updateItemPhase(itemId, stepId);
-
-        const agentType = response.agent ?? 'claude';
-        const prompt = buildStepPrompt(response, context);
-        const { result } = await runAndNormalize(null, prompt, response, { stratum, cwd });
-
-        // Agent-as-validator: if step has validate config, check the artifact
-        const valConfig = validateConfigs.get(stepId);
-        if (valConfig) {
-          console.log(`  ✓ Validating ${stepId}...`);
-          const { valid, issues } = await validateStep({
-            artifact: valConfig.artifact,
-            criteria: valConfig.criteria,
-            stepId,
+    const context = { cwd, featureCode: projectName, projectName, intent, stepHistory: [] };
+
+    while (!['completed', 'failed', 'budget_exhausted'].includes(response.status)) {
+      if (response.status === 'ready') {
+        const step = response.ready?.[0];
+        if (!step) throw new Error('Stratum returned ready without a kickoff step');
+        const dispatch = {
+          ...step,
+          step_id: step.id,
+          flow_id: response.runId,
+          intent: step.do,
+          output_fields: outputContract(spec, step.id),
+          has_out_contract: true,
+        };
+        console.log(`${step.id}...`);
+        await visionWriter.updateItemPhase(itemId, step.id);
+        const prompt = buildStepPrompt(dispatch, context);
+        const { result } = await runAndNormalize(null, prompt, dispatch, { stratum, cwd });
+
+        const validation = validateConfigs.get(step.id);
+        if (validation) {
+          console.log(`  ✓ Validating ${step.id}...`);
+          const checked = await validateStep({
+            artifact: validation.artifact,
+            criteria: validation.criteria,
+            stepId: step.id,
             stratum,
             cwd,
           });
-          if (!valid) {
-            console.log(`  ✗ Validation failed:`);
-            for (const issue of issues) console.log(`    - ${issue}`);
-            console.log(`  ↻ Fix (claude) for ${stepId}`);
+          if (!checked.valid) {
+            console.log('  ✗ Validation failed:');
+            for (const issue of checked.issues) console.log(`    - ${issue}`);
             const fixPrompt =
-              `Read "${valConfig.artifact}" and fix these issues:\n` +
-              issues.map(i => `- ${i}`).join('\n') + '\n\n' +
-              `Update the file in place. Do not skip any issue.\n\n` +
+              `Read "${validation.artifact}" and fix these issues:\n` +
+              checked.issues.map((issue) => `- ${issue}`).join('\n') +
+              `\n\nUpdate the file in place. Do not skip any issue.\n\n` +
               `## Context\nWorking directory: ${cwd}\nProject: ${projectName}`;
-            const fixDispatch = { ...response, agent: 'claude' };
-            await runAndNormalize(null, fixPrompt, fixDispatch, { stratum, cwd });
+            await runAndNormalize(null, fixPrompt, { ...dispatch, agent: 'claude' }, { stratum, cwd });
           }
         }
 
-        // Print step summary
-        const valConfig2 = validateConfigs.get(stepId);
-        const artifactPath = valConfig2?.artifact;
-        if (result?.summary) {
-          console.log(`  ✓ ${result.summary}`);
-        } else if (artifactPath && existsSync(join(cwd, artifactPath))) {
-          // Summarize from artifact — first few lines
-          const content = readFileSync(join(cwd, artifactPath), 'utf-8');
-          const heading = content.split('\n').find(l => l.startsWith('# '));
-          const lineCount = content.split('\n').length;
-          console.log(`  ✓ Wrote ${artifactPath} (${lineCount} lines)${heading ? ' — ' + heading.replace(/^#+\s*/, '') : ''}`);
-        } else {
-          console.log(`  ✓ ${stepId} complete`);
-        }
-
-        response = await stratum.stepDone(flowId, stepId, result ?? { summary: 'Step complete' });
+        if (result?.summary) console.log(`  ✓ ${result.summary}`);
+        else console.log(`  ✓ ${step.id} complete`);
+        context.stepHistory.push({ stepId: step.id, summary: result?.summary ?? 'Step complete' });
+        response = await stratum.stepDone(
+          response.runId,
+          step.id,
+          { output: result ?? { summary: 'Step complete' } },
+          step.dispatchToken,
+        );
+        continue;
+      }
 
-      } else if (response.status === 'await_gate') {
+      if (response.status === 'running') {
+        const audit = await stratum.audit(response.runId);
+        const waiting = waitingGate(audit);
+        if (!waiting) throw new Error('Kickoff flow is running without ready work or a waiting gate');
+        const [stepId, state] = waiting;
+        const dispatch = gateDispatch(spec, stepId);
         console.log(`\nGate: ${stepId}`);
 
-        // Show the artifact that's being gated so user can make an informed decision
-        // Try multiple sources for the prior step: gate's on_revise, depends_on, or spec lookup
-        const priorStepId = response.on_revise ?? response.depends_on?.[0]
-          ?? specObj.flows?.new?.steps?.find(s => s.id === stepId)?.on_revise;
-        const priorValConfig = priorStepId ? validateConfigs.get(priorStepId) : null;
-        if (priorValConfig?.artifact) {
-          const artPath = join(cwd, priorValConfig.artifact);
-          if (existsSync(artPath)) {
-            const content = readFileSync(artPath, 'utf-8');
-            console.log(`\n--- ${priorValConfig.artifact} ---`);
-            // Show full content (it's a discovery doc, should be readable)
-            const lines = content.split('\n');
-            if (lines.length <= 80) {
-              console.log(content);
-            } else {
-              console.log(lines.slice(0, 60).join('\n'));
-              console.log(`\n... (${lines.length - 60} more lines — see ${priorValConfig.artifact})`);
-            }
-            console.log(`--- end ---\n`);
-          }
+        const priorStepId = dispatch.on_revise;
+        const priorValidation = validateConfigs.get(priorStepId);
+        const artifact = priorValidation?.artifact ? join(cwd, priorValidation.artifact) : null;
+        if (artifact && existsSync(artifact)) {
+          const content = readFileSync(artifact, 'utf-8');
+          console.log(`\n--- ${priorValidation.artifact} ---`);
+          const lines = content.split('\n');
+          console.log(lines.length <= 80 ? content : `${lines.slice(0, 60).join('\n')}\n\n... (${lines.length - 60} more lines)`);
+          console.log('--- end ---\n');
         }
 
-        // COMP-PLAN-GATE-LOOP: round-aware gate id so a revise re-entry mints a
-        // fresh gate instead of reusing the prior resolved one.
-        const gateId = await visionWriter.createGate(flowId, stepId, itemId, { round: readFlowRound(flowId) });
-
-        // Resolve artifact path for this gate
-        const gateArtifact = priorValConfig?.artifact
-          ? join(cwd, priorValConfig.artifact)
-          : null;
-
-        // Agent Q&A callback for interactive gate — with full workflow context
-        const gatePreamble = buildGateContext(response, context, null);
+        const gateId = await visionWriter.createGate(
+          response.runId,
+          stepId,
+          itemId,
+          { round: readFlowRound(response.runId) },
+        );
+        const preamble = buildGateContext(dispatch, context, null);
         const askAgent = async (question, artifactPath) => {
           const fileRef = artifactPath
             ? `Read the file "${artifactPath}" and answer`
-            : `Look at the project files in the working directory and answer`;
-          const qaPrompt =
-            `${gatePreamble}\n\n---\n\n` +
-            `${fileRef} this question concisely:\n\n` +
-            `${question}\n\n` +
-            `Keep your answer brief — 2-3 sentences max.`;
-          const text = await stratum.runAgentText('claude', qaPrompt, { cwd });
-          return text || '(no answer)';
+            : 'Look at the project files in the working directory and answer';
+          return stratum.runAgentText(
+            'claude',
+            `${preamble}\n\n---\n\n${fileRef} this question concisely:\n\n${question}`,
+            { cwd },
+          );
         };
-
-        const { outcome, rationale } = await promptGate(response, {
+        const decision = await promptGate(dispatch, {
           ...(opts.gateOpts ?? {}),
-          artifact: gateArtifact ?? cwd,
+          artifact: artifact ?? cwd,
           askAgent,
         });
-        await visionWriter.resolveGate(gateId, outcome);
-
-        // Inject gate decision into context so the re-run step sees reviewer feedback
-        if (!context.stepHistory) context.stepHistory = [];
+        await visionWriter.resolveGate(gateId, decision.outcome);
         context.stepHistory.push({
           stepId,
-          artifact: null,
-          summary: `Gate ${outcome}${rationale ? ': ' + rationale : ''}`,
-          outcome,
+          summary: `Gate ${decision.outcome}${decision.rationale ? `: ${decision.rationale}` : ''}`,
+          outcome: decision.outcome,
         });
-
-        response = await stratum.gateResolve(flowId, stepId, outcome, rationale, 'human');
-
-      } else if (response.status === 'execute_flow') {
-        const parentFlowId = response.parent_flow_id;
-        const parentStepId = response.parent_step_id;
-        const childFlowName = response.child_flow_name ?? 'sub-flow';
-        console.log(`[sub-flow] ${childFlowName}...`);
-
-        const childResult = await executeChildFlow(
-          response, stratum, context,
-          visionWriter, itemId, opts.gateOpts ?? {}
-        );
-
-        response = await stratum.stepDone(parentFlowId, parentStepId, childResult);
-
-      } else if (response.status === 'ensure_failed' || response.status === 'schema_failed') {
-        console.log(`  ↻ Retrying ${stepId} (postconditions failed)`);
-        const violations = response.violations ?? [];
-        const agentType = response.agent ?? 'claude';
-
-        // Fix pass before retry
-        const fixAgent = agentType === 'codex' ? 'claude' : agentType;
-        console.log(`  ↻ Fix (${fixAgent}) for ${stepId}`);
-        const fixPrompt =
-          `Fix step "${stepId}" — postconditions failed:\n` +
-          violations.map(v => `- ${v}`).join('\n') + '\n\n' +
-          `Fix every issue. Do not skip any.\n\n` +
-          `## Context\nWorking directory: ${cwd}\nProject: ${projectName}`;
-        const fixDispatch = { ...response, agent: fixAgent };
-        await runAndNormalize(null, fixPrompt, fixDispatch, { stratum, cwd });
-
-        console.log(`  ↻ Retrying ${stepId} (${agentType})`);
-        const prompt = buildRetryPrompt(response, violations, context);
-        const { result } = await runAndNormalize(null, prompt, response, { stratum, cwd });
-
-        response = await stratum.stepDone(
-          response.flow_id, response.step_id,
-          result ?? { summary: 'Retry complete' }
+        response = await stratum.gateResolve(
+          response.runId,
+          stepId,
+          decision.outcome,
+          decision.rationale,
+          'human',
+          state.gateToken,
         );
-
-      } else {
-        console.warn(`Unknown dispatch status: ${response.status}`);
-        break;
+        continue;
       }
+
+      throw new Error(`Unknown TS kickoff status: ${response.status}`);
     }
 
-    // Flow complete
-    if (response.status === 'complete') {
+    if (response.status === 'completed') {
       console.log('\nProduct kickoff complete.');
       await visionWriter.updateItemStatus(itemId, 'complete');
-    } else if (response.status === 'killed') {
-      console.log('\nProduct kickoff killed.');
-      await visionWriter.updateItemStatus(itemId, 'killed');
-    }
-
-    // Write audit trace
-    if (response.trace) {
-      try {
-        const auditPath = join(cwd, 'docs', 'discovery', 'kickoff-audit.json');
-        writeFileSync(auditPath, JSON.stringify(response, null, 2));
-        console.log(`Audit trace written to docs/discovery/kickoff-audit.json`);
-      } catch (err) {
-        console.warn(`Warning: could not write audit trace: ${err.message}`);
-      }
-    }
-
-    // Summary
-    if (response.status === 'complete') {
       console.log('\nNext steps:');
       console.log('  compose pipeline show          # review the build pipeline');
       console.log('  compose build <FEATURE-CODE>   # build the first feature');
+    } else {
+      await visionWriter.updateItemStatus(itemId, 'failed');
     }
 
+    try {
+      const audit = await stratum.audit(flowId);
+      const auditPath = join(cwd, 'docs', 'discovery', 'kickoff-audit.json');
+      writeFileSync(auditPath, JSON.stringify(audit, null, 2));
+      console.log('Audit trace written to docs/discovery/kickoff-audit.json');
+    } catch (err) {
+      console.warn(`Warning: could not write audit trace: ${err.message}`);
+    }
+    return response;
   } finally {
     await stratum.close();
   }
 }
-
-// ---------------------------------------------------------------------------
-// Child flow execution (simplified from build.js — no active-build tracking)
-// ---------------------------------------------------------------------------
-
-async function executeChildFlow(
-  flowDispatch, stratum, context,
-  visionWriter, itemId, gateOpts
-) {
-  let resp = flowDispatch.child_step;
-  const childFlowId = flowDispatch.child_flow_id;
-  const childFlowName = flowDispatch.child_flow_name ?? 'sub-flow';
-
-  while (resp.status !== 'complete' && resp.status !== 'killed') {
-    if (resp.status === 'execute_step') {
-      console.log(`  [${childFlowName}] ${resp.step_id}...`);
-      await visionWriter.updateItemPhase(itemId, `${childFlowName}:${resp.step_id}`);
-
-      const prompt = buildStepPrompt(resp, context);
-      const { result } = await runAndNormalize(null, prompt, resp, { stratum, cwd: context.cwd });
-
-      resp = await stratum.stepDone(
-        childFlowId, resp.step_id,
-        result ?? { summary: 'Step complete' }
-      );
-
-    } else if (resp.status === 'await_gate') {
-      console.log(`  [${childFlowName}] Gate: ${resp.step_id}`);
-      // COMP-PLAN-GATE-LOOP: round-aware gate id (child flow has its own round).
-      const gateId = await visionWriter.createGate(childFlowId, resp.step_id, itemId, { round: readFlowRound(childFlowId) });
-      const { outcome, rationale } = await promptGate(resp, gateOpts);
-      await visionWriter.resolveGate(gateId, outcome);
-      resp = await stratum.gateResolve(childFlowId, resp.step_id, outcome, rationale, 'human');
-
-    } else if (resp.status === 'ensure_failed' || resp.status === 'schema_failed') {
-      const violations = resp.violations ?? [];
-      const stepAgent = resp.agent ?? 'claude';
-      const fixAgent = stepAgent === 'codex' ? 'claude' : stepAgent;
-
-      console.log(`  [${childFlowName}] ↻ Fix (${fixAgent}) for ${resp.step_id}`);
-      const fixPrompt =
-        `Fix step "${resp.step_id}" — postconditions failed:\n` +
-        violations.map(v => `- ${v}`).join('\n') + '\n\n' +
-        `Fix every issue.\n\n` +
-        `## Context\nWorking directory: ${context.cwd}\nProject: ${context.projectName}`;
-      const fixDispatch = { ...resp, agent: fixAgent };
-      await runAndNormalize(null, fixPrompt, fixDispatch, { stratum, cwd: context.cwd });
-
-      console.log(`  [${childFlowName}] ↻ Retrying ${resp.step_id} (${stepAgent})`);
-      const prompt = buildRetryPrompt(resp, violations, context);
-      const { result } = await runAndNormalize(null, prompt, resp, { stratum, cwd: context.cwd });
-
-      resp = await stratum.stepDone(
-        resp.flow_id ?? childFlowId, resp.step_id,
-        result ?? { summary: 'Retry complete' }
-      );
-
-    } else if (resp.status === 'execute_flow') {
-      const nestedParentFlowId = resp.parent_flow_id;
-      const nestedParentStepId = resp.parent_step_id;
-      const nestedResult = await executeChildFlow(
-        resp, stratum, context,
-        visionWriter, itemId, gateOpts
-      );
-      resp = await stratum.stepDone(nestedParentFlowId, nestedParentStepId, nestedResult);
-
-    } else {
-      console.warn(`  [${childFlowName}] Unknown status: ${resp.status}`);
-      break;
-    }
-  }
-
-  return resp;
-}