opencastle 0.32.8 → 0.32.10

package/dist/cli/pipeline.js

@@ -1,4 +1,4 @@
-import { readFile, writeFile, mkdir } from 'node:fs/promises';
+import { readFile, writeFile, mkdir, unlink } from 'node:fs/promises';
 import { existsSync } from 'node:fs';
 import { resolve } from 'node:path';
 import { stringify } from 'yaml';
@@ -6,7 +6,7 @@ import { c, confirm, closePrompts } from './prompt.js';
 import { runPromptStep, readProjectMcpServers } from './plan.js';
 import { cleanupAdapters } from './run/adapters/index.js';
 import { parseYaml, validateSpec } from './run/schema.js';
-import { buildConvoyYaml, parseTaskPlan, parsePatches, applyPatches, deriveSpecEnrichment } from './convoy/spec-builder.js';
+import { buildConvoyYaml, parseTaskPlanWithReason, parsePatches, applyPatches, deriveSpecEnrichment } from './convoy/spec-builder.js';
 function appendTaskComplexity(base, taskComplexity) {
     if (!taskComplexity?.length)
         return base;
@@ -18,6 +18,89 @@ function appendTaskComplexity(base, taskComplexity) {
     }
     return result;
 }
+/**
+ * For chain mode, extract only the PRD sections relevant to the given phases.
+ * Keeps Overview, Technical Requirements, and the matching phase sections from
+ * Task Breakdown, while trimming the full User Stories, Implementation Scope,
+ * and non-matching phases to reduce context size and avoid output truncation.
+ */
+function extractRelevantPrdSections(prdContent, phases) {
+    const phaseSet = new Set(phases);
+    const lines = prdContent.split('\n');
+    const result = [];
+    // Sections to always include (key context)
+    const alwaysInclude = ['overview', 'goals', 'non-goals', 'technical requirements'];
+    // Sections to include condensed
+    const condenseSection = ['user stories & acceptance criteria', 'implementation scope', 'success criteria', 'risks & open questions'];
+    let currentSection = '';
+    let inTaskBreakdown = false;
+    let inRelevantPhase = false;
+    let skipSection = false;
+    for (const line of lines) {
+        // Detect heading (## level)
+        const h2Match = line.match(/^## (.+)/);
+        if (h2Match) {
+            const heading = h2Match[1].trim().toLowerCase();
+            currentSection = heading;
+            inTaskBreakdown = heading === 'task breakdown';
+            inRelevantPhase = false;
+            skipSection = false;
+            if (alwaysInclude.some(s => heading.startsWith(s))) {
+                result.push(line);
+                continue;
+            }
+            if (inTaskBreakdown) {
+                result.push(line);
+                result.push('');
+                result.push(`*(Only phases ${phases.join(', ')} shown — other phases omitted for brevity)*`);
+                result.push('');
+                continue;
+            }
+            if (condenseSection.some(s => heading.startsWith(s))) {
+                // Include the heading but mark as condensed
+                result.push(line);
+                result.push('');
+                result.push('*(Condensed — see full PRD for details)*');
+                result.push('');
+                skipSection = true;
+                continue;
+            }
+            // # title heading — always include
+            result.push(line);
+            continue;
+        }
+        // H1 heading — always include
+        if (line.match(/^# /)) {
+            result.push(line);
+            continue;
+        }
+        if (skipSection)
+            continue;
+        if (inTaskBreakdown) {
+            // Detect phase headers like "Phase 1 —" or "Phase 2 —"
+            const phaseMatch = line.match(/Phase\s+(\d+)/i);
+            if (phaseMatch) {
+                const phaseNum = parseInt(phaseMatch[1], 10);
+                inRelevantPhase = phaseSet.has(phaseNum);
+            }
+            if (inRelevantPhase) {
+                result.push(line);
+            }
+            continue;
+        }
+        result.push(line);
+    }
+    return result.join('\n');
+}
+/**
+ * Filter task complexity entries to only those matching the given phases.
+ */
+function filterTaskComplexityByPhases(taskComplexity, phases) {
+    if (!taskComplexity?.length)
+        return taskComplexity;
+    const phaseSet = new Set(phases);
+    return taskComplexity.filter(tc => phaseSet.has(tc.phase));
+}
 export function parseComplexityAssessment(jsonText) {
     try {
         const parsed = JSON.parse(jsonText.trim());
@@ -266,6 +349,62 @@ function relPath(abs) {
 function stepLabel(n, total, name) {
     return c.bold(c.cyan(`  [${n}/${total}] ${name}`));
 }
+async function fixPrd(sharedOpts, prdPath, prdContent, initialErrors, totalSteps, adapterFlag) {
+    const MAX_PRD_FIX_RETRIES = 2;
+    let fixedPrdContent = prdContent;
+    let prdValidationErrors = initialErrors;
+    let prdFixed = false;
+    for (let attempt = 1; attempt <= MAX_PRD_FIX_RETRIES; attempt++) {
+        const label = `Fix PRD attempt ${attempt}/${MAX_PRD_FIX_RETRIES}…`;
+        console.log(stepLabel(3, totalSteps, label));
+        try {
+            await runPromptStep({
+                ...sharedOpts,
+                template: 'fix-prd',
+                goalText: fixedPrdContent,
+                contextText: prdValidationErrors,
+                outputPath: prdPath,
+            });
+        }
+        catch (err) {
+            console.error(`\n  ✗ Step 3 (attempt ${attempt}) failed: ${err instanceof Error ? err.message : String(err)}`);
+            process.exit(1);
+        }
+        console.log(c.dim(`  Re-validating after fix…`));
+        fixedPrdContent = await readFile(prdPath, 'utf8');
+        let revalidation;
+        try {
+            revalidation = await runPromptStep({
+                ...sharedOpts,
+                template: 'validate-prd',
+                goalText: `<!-- validation-pass: ${attempt + 1} -->\n${fixedPrdContent}`,
+            });
+        }
+        catch (err) {
+            console.error(`\n  ✗ Re-validation failed: ${err instanceof Error ? err.message : String(err)}`);
+            process.exit(1);
+        }
+        if (revalidation.isValid) {
+            console.log(c.green(`  ✓ PRD fixed and validated\n`));
+            prdFixed = true;
+            break;
+        }
+        prdValidationErrors = revalidation.errors ?? revalidation.rawOutput;
+        if (attempt < MAX_PRD_FIX_RETRIES) {
+            console.log(c.yellow(`  ⚠ Still has issues after fix attempt ${attempt} — retrying…\n`));
+            console.log(c.dim(prdValidationErrors));
+            console.log();
+        }
+    }
+    if (!prdFixed) {
+        console.log(c.yellow(`\n  ⚠ Could not fully auto-fix the PRD after ${MAX_PRD_FIX_RETRIES} attempts — continuing with best-effort PRD.\n`));
+        console.log(c.dim(`  Remaining issues:\n`));
+        console.log(c.dim(prdValidationErrors));
+        console.log(c.dim(`\n  PRD saved to ${relPath(prdPath)} with best available fixes.`) +
+            c.dim(`\n  You can re-validate later with:\n`) +
+            `    opencastle start --prd ${relPath(prdPath)}${adapterFlag ? ` --adapter ${adapterFlag}` : ''}\n`);
+    }
+}
 export default async function pipeline({ args, pkgRoot }) {
     const opts = parseArgs(args);
     if (opts.help) {
@@ -331,92 +470,16 @@ export default async function pipeline({ args, pkgRoot }) {
         console.log(c.dim('\n  [dry-run] Nothing to preview — PRD already provided via --prd. Remove --dry-run to continue.'));
         return;
     }
-    // ── Step 2: Validate PRD ──────────────────────────────────────────────────
-    if (!opts.skipValidation) {
-        console.log(stepLabel(2, totalSteps, 'Validating PRD…'));
-        const prdContent = await readFile(prdPath, 'utf8');
-        let result;
-        try {
-            result = await runPromptStep({
-                ...sharedOpts,
-                template: 'validate-prd',
-                goalText: `<!-- validation-pass: 1 -->\n${prdContent}`,
-            });
-        }
-        catch (err) {
-            console.error(`\n  ✗ Step 2 failed: ${err instanceof Error ? err.message : String(err)}`);
-            process.exit(1);
-        }
-        if (!result.isValid) {
-            let prdValidationErrors = result.errors ?? result.rawOutput;
-            console.log(c.yellow(`  ⚠ PRD has validation issues — attempting auto-fix…\n`));
-            console.log(c.dim(prdValidationErrors));
-            console.log();
-            // ── Step 3: Fix PRD (up to 2 retries) ──────────────────────────────────
-            const MAX_PRD_FIX_RETRIES = 2;
-            let fixedPrdContent = prdContent;
-            let prdFixed = false;
-            for (let attempt = 1; attempt <= MAX_PRD_FIX_RETRIES; attempt++) {
-                const label = `Fix PRD attempt ${attempt}/${MAX_PRD_FIX_RETRIES}…`;
-                console.log(stepLabel(3, totalSteps, label));
-                try {
-                    await runPromptStep({
-                        ...sharedOpts,
-                        template: 'fix-prd',
-                        goalText: fixedPrdContent,
-                        contextText: prdValidationErrors,
-                        outputPath: prdPath, // overwrite in place
-                    });
-                }
-                catch (err) {
-                    console.error(`\n  ✗ Step 3 (attempt ${attempt}) failed: ${err instanceof Error ? err.message : String(err)}`);
-                    process.exit(1);
-                }
-                console.log(c.dim(`  Re-validating after fix…`));
-                fixedPrdContent = await readFile(prdPath, 'utf8');
-                let revalidation;
-                try {
-                    revalidation = await runPromptStep({
-                        ...sharedOpts,
-                        template: 'validate-prd',
-                        goalText: `<!-- validation-pass: ${attempt + 1} -->\n${fixedPrdContent}`,
-                    });
-                }
-                catch (err) {
-                    console.error(`\n  ✗ Re-validation failed: ${err instanceof Error ? err.message : String(err)}`);
-                    process.exit(1);
-                }
-                if (revalidation.isValid) {
-                    console.log(c.green(`  ✓ PRD fixed and validated\n`));
-                    prdFixed = true;
-                    break;
-                }
-                prdValidationErrors = revalidation.errors ?? revalidation.rawOutput;
-                if (attempt < MAX_PRD_FIX_RETRIES) {
-                    console.log(c.yellow(`  ⚠ Still has issues after fix attempt ${attempt} — retrying…\n`));
-                    console.log(c.dim(prdValidationErrors));
-                    console.log();
-                }
-            }
-            if (!prdFixed) {
-                console.log(c.yellow(`\n  ⚠ Could not fully auto-fix the PRD after ${MAX_PRD_FIX_RETRIES} attempts — continuing with best-effort PRD.\n`));
-                console.log(c.dim(`  Remaining issues:\n`));
-                console.log(c.dim(prdValidationErrors));
-                console.log(c.dim(`\n  PRD saved to ${relPath(prdPath)} with best available fixes.`) +
-                    c.dim(`\n  You can re-validate later with:\n`) +
-                    `    opencastle start --prd ${relPath(prdPath)}${opts.adapter ? ` --adapter ${opts.adapter}` : ''}\n`);
-            }
-        }
-        else {
-            console.log(c.green(`  ✓ PRD is valid\n`));
-        }
-    }
-    // ── Complexity-aware strategy decision ────────────────────────────────────
+    // ── Steps 2 + 4: Validate PRD & Assess complexity (in parallel) ────────
+    // Both only read the PRD — run them concurrently to save one LLM round-trip.
+    // If validation fails we still use the complexity result (fix-prd patches
+    // issues without changing the overall structure/phases).
     const complexityStep = opts.skipValidation ? 2 : 4;
     let complexity = null;
     const complexityFilePath = opts.complexity
         ? resolve(process.cwd(), opts.complexity)
         : deriveComplexityPath(prdPath);
+    // Check for cached / provided complexity before launching LLM calls
     if (opts.complexity) {
         if (!existsSync(complexityFilePath)) {
             console.error(`  ✗ Complexity file not found: ${opts.complexity}`);
@@ -451,24 +514,103 @@ export default async function pipeline({ args, pkgRoot }) {
             // ignore — fall through to LLM assessment
         }
     }
-    if (!complexity) {
-        console.log(stepLabel(complexityStep, totalSteps, 'Assessing complexity…'));
-        try {
-            const complexityResult = await runPromptStep({
-                ...sharedOpts,
-                template: 'assess-complexity',
-                filePath: prdPath,
-                contextText: opts.text ?? undefined,
-            });
-            complexity = parseComplexityAssessment(complexityResult.rawOutput);
-            if (complexity) {
-                await writeFile(complexityFilePath, JSON.stringify(complexity, null, 2), 'utf8');
-                console.log(c.green(`  ✓ Complexity assessment saved to ${relPath(complexityFilePath)}\n`));
+    const needsValidation = !opts.skipValidation;
+    const needsComplexity = !complexity;
+    if (needsValidation || needsComplexity) {
+        const prdContent = await readFile(prdPath, 'utf8');
+        // Launch validation and complexity in parallel when both are needed
+        if (needsValidation && needsComplexity) {
+            console.log(stepLabel(2, totalSteps, 'Validating PRD…'));
+            console.log(stepLabel(complexityStep, totalSteps, 'Assessing complexity…'));
+            console.log(c.dim(`  (running in parallel)\n`));
+            const [validationResult, complexityResult] = await Promise.all([
+                runPromptStep({
+                    ...sharedOpts,
+                    template: 'validate-prd',
+                    goalText: `<!-- validation-pass: 1 -->\n${prdContent}`,
+                }).catch((err) => {
+                    console.error(`\n  ✗ Validation failed: ${err instanceof Error ? err.message : String(err)}`);
+                    return null;
+                }),
+                runPromptStep({
+                    ...sharedOpts,
+                    template: 'assess-complexity',
+                    filePath: prdPath,
+                    contextText: opts.text ?? undefined,
+                }).catch((err) => {
+                    console.warn(c.yellow(`  ⚠ Complexity assessment failed: ${err instanceof Error ? err.message : String(err)}`));
+                    return null;
+                }),
+            ]);
+            // Process complexity result
+            if (complexityResult) {
+                complexity = parseComplexityAssessment(complexityResult.rawOutput);
+                if (complexity) {
+                    await writeFile(complexityFilePath, JSON.stringify(complexity, null, 2), 'utf8');
+                    console.log(c.green(`  ✓ Complexity assessment saved to ${relPath(complexityFilePath)}`));
+                }
+            }
+            // Process validation result
+            if (!validationResult) {
+                process.exit(1);
+            }
+            if (!validationResult.isValid) {
+                let prdValidationErrors = validationResult.errors ?? validationResult.rawOutput;
+                console.log(c.yellow(`  ⚠ PRD has validation issues — attempting auto-fix…\n`));
+                console.log(c.dim(prdValidationErrors));
+                console.log();
+                await fixPrd(sharedOpts, prdPath, prdContent, prdValidationErrors, totalSteps, opts.adapter);
+            }
+            else {
+                console.log(c.green(`  ✓ PRD is valid\n`));
             }
         }
-        catch (err) {
-            console.warn(c.yellow(`  ⚠ Complexity assessment failed: ${err instanceof Error ? err.message : String(err)}`));
-            console.warn(c.dim(`    Falling back to single convoy strategy.\n`));
+        else if (needsValidation) {
+            // Only validation needed (complexity was cached)
+            console.log(stepLabel(2, totalSteps, 'Validating PRD…'));
+            let result;
+            try {
+                result = await runPromptStep({
+                    ...sharedOpts,
+                    template: 'validate-prd',
+                    goalText: `<!-- validation-pass: 1 -->\n${prdContent}`,
+                });
+            }
+            catch (err) {
+                console.error(`\n  ✗ Step 2 failed: ${err instanceof Error ? err.message : String(err)}`);
+                process.exit(1);
+            }
+            if (!result.isValid) {
+                let prdValidationErrors = result.errors ?? result.rawOutput;
+                console.log(c.yellow(`  ⚠ PRD has validation issues — attempting auto-fix…\n`));
+                console.log(c.dim(prdValidationErrors));
+                console.log();
+                await fixPrd(sharedOpts, prdPath, prdContent, prdValidationErrors, totalSteps, opts.adapter);
+            }
+            else {
+                console.log(c.green(`  ✓ PRD is valid\n`));
+            }
+        }
+        else {
+            // Only complexity needed (validation skipped)
+            console.log(stepLabel(complexityStep, totalSteps, 'Assessing complexity…'));
+            try {
+                const complexityResult = await runPromptStep({
+                    ...sharedOpts,
+                    template: 'assess-complexity',
+                    filePath: prdPath,
+                    contextText: opts.text ?? undefined,
+                });
+                complexity = parseComplexityAssessment(complexityResult.rawOutput);
+                if (complexity) {
+                    await writeFile(complexityFilePath, JSON.stringify(complexity, null, 2), 'utf8');
+                    console.log(c.green(`  ✓ Complexity assessment saved to ${relPath(complexityFilePath)}\n`));
+                }
+            }
+            catch (err) {
+                console.warn(c.yellow(`  ⚠ Complexity assessment failed: ${err instanceof Error ? err.message : String(err)}`));
+                console.warn(c.dim(`    Falling back to single convoy strategy.\n`));
+            }
         }
     }
     if (!complexity) {
@@ -497,9 +639,16 @@ export default async function pipeline({ args, pkgRoot }) {
                 console.log();
                 const convoyDir = resolve(process.cwd(), '.opencastle', 'convoys');
                 await mkdir(convoyDir, { recursive: true });
+                // Extract feature name early for convoy naming
+                const chainPrdContent = await readFile(prdPath, 'utf8');
+                const featureNameMatch = chainPrdContent.match(/^# (.+?)\s*(?:—|-)?\s*PRD/m);
+                const featureName = featureNameMatch
+                    ? featureNameMatch[1].trim().toLowerCase().replace(/[^a-z0-9]+/g, '-')
+                    : 'feature';
                 const groupSpecPaths = [];
                 for (let i = 0; i < complexity.convoy_groups.length; i++) {
                     const group = complexity.convoy_groups[i];
+                    console.log(c.cyan(`  [${i + 1}/${complexity.convoy_groups.length}] Generating convoy: ${group.name}`) + c.dim(` (phases: ${group.phases.join(', ')})`));
                     const chainGoal = [
                         complexity.original_prompt,
                         '',
@@ -514,8 +663,10 @@ export default async function pipeline({ args, pkgRoot }) {
                         group.depends_on.length ? `- **Depends on groups:** ${group.depends_on.join(', ')}` : '',
                     ].filter(Boolean).join('\n');
                     const prdContent = await readFile(prdPath, 'utf8');
-                    const contextForSpec = appendTaskComplexity(prdContent, complexity?.task_complexity);
-                    const groupSpecPath = resolve(convoyDir, `${group.name}.convoy.yml`);
+                    const relevantPrd = extractRelevantPrdSections(prdContent, group.phases);
+                    const relevantComplexity = filterTaskComplexityByPhases(complexity?.task_complexity, group.phases);
+                    const contextForSpec = appendTaskComplexity(relevantPrd, relevantComplexity);
+                    const groupSpecPath = resolve(convoyDir, `${featureName}-${group.name}.convoy.yml`);
                     const { specPath: resolvedGroupSpecPath } = await generateAndValidateSpec({
                         sharedOpts,
                         goalText: chainGoal,
@@ -523,16 +674,12 @@ export default async function pipeline({ args, pkgRoot }) {
                         specPath: groupSpecPath,
                         skipValidation: opts.skipValidation,
                         groupName: group.name,
+                        featureName,
                         enrichment: complexity ? deriveSpecEnrichment(complexity) : undefined,
                     });
                     groupSpecPaths.push(resolvedGroupSpecPath);
                 }
                 // Build master pipeline spec (version 2)
-                const chainPrdContent = await readFile(prdPath, 'utf8');
-                const featureNameMatch = chainPrdContent.match(/^# (.+?)\s*(?:—|-)?\s*PRD/m);
-                const featureName = featureNameMatch
-                    ? featureNameMatch[1].trim().toLowerCase().replace(/[^a-z0-9]+/g, '-')
-                    : 'feature';
                 const branchMatch = chainPrdContent.match(/`feat\/([^`]+)`/);
                 const branch = branchMatch ? `feat/${branchMatch[1]}` : `feat/${featureName}`;
                 const masterSpec = {
@@ -674,6 +821,13 @@ async function generateAndValidateSpec(params) {
         ? `Generating task plan: ${params.groupName}…`
         : 'Generating task plan…';
     console.log(c.cyan(`  ${label}`));
+    // Temp file for debugging — kept on failure, deleted on success
+    const convoyDir = resolve(process.cwd(), '.opencastle', 'convoys');
+    await mkdir(convoyDir, { recursive: true });
+    const tempJsonName = params.groupName
+        ? `${params.featureName ? `${params.featureName}-` : ''}${params.groupName}.task-plan.json`
+        : `${params.featureName ?? 'task-plan'}.task-plan.json`;
+    const tempJsonPath = resolve(convoyDir, tempJsonName);
     let taskPlanResult;
     try {
         taskPlanResult = await runPromptStep({
@@ -687,12 +841,14 @@ async function generateAndValidateSpec(params) {
         console.error(`\n  ✗ Task plan generation failed: ${err instanceof Error ? err.message : String(err)}`);
         process.exit(1);
     }
-    let taskPlan = parseTaskPlan(taskPlanResult.rawOutput);
-    if (!taskPlan) {
-        console.log(c.yellow(`  ⚠ Failed to parse task plan JSON — retrying generation…\n`));
-        if (params.sharedOpts.verbose) {
-            console.log(c.dim(taskPlanResult.rawOutput.slice(0, 500)));
-        }
+    // Write raw JSON to temp file for debugging
+    await writeFile(tempJsonPath, taskPlanResult.rawOutput + '\n', 'utf8');
+    let result = parseTaskPlanWithReason(taskPlanResult.rawOutput);
+    if (!result.plan) {
+        console.log(c.yellow(`  ⚠ Failed to parse task plan JSON: ${result.reason}`));
+        console.log(c.dim(`    Output length: ${taskPlanResult.rawOutput.length} chars`));
+        console.log(c.dim(`    Temp file: ${relPath(tempJsonPath)}`));
+        console.log(c.yellow(`    Retrying generation…\n`));
         let retryResult;
         try {
             retryResult = await runPromptStep({
@@ -706,13 +862,20 @@ async function generateAndValidateSpec(params) {
             console.error(`\n  ✗ Retry failed: ${err instanceof Error ? err.message : String(err)}`);
             process.exit(1);
         }
-        taskPlan = parseTaskPlan(retryResult.rawOutput);
-        if (!taskPlan) {
-            console.error('  ✗ Failed to parse task plan JSON after retry');
-            console.error(c.dim(retryResult.rawOutput.slice(0, 500)));
+        // Overwrite temp file with retry output
+        await writeFile(tempJsonPath, retryResult.rawOutput + '\n', 'utf8');
+        result = parseTaskPlanWithReason(retryResult.rawOutput);
+        if (!result.plan) {
+            console.error(`  ✗ Failed to parse task plan JSON after retry: ${result.reason}`);
+            console.error(c.dim(`    Output length: ${retryResult.rawOutput.length} chars`));
+            console.error(c.dim(`    Raw JSON saved to ${relPath(tempJsonPath)} for inspection.`));
+            console.error(c.dim(`\n${retryResult.rawOutput}`));
             process.exit(1);
         }
     }
+    let taskPlan = result.plan;
+    // Success — clean up temp file
+    await unlink(tempJsonPath).catch(() => { });
     console.log(c.green(`  ✓ Task plan generated (${taskPlan.tasks.length} tasks)`));
     // Derive spec path from plan name if not provided
     let resolvedSpecPath = params.specPath;