@auto-engineer/component-implementor-react 1.98.0 → 1.100.0

package/ketchup-plan.md

@@ -1,13 +1,48 @@
-# Ketchup Plan: component-implementor-react
+# Ketchup Plan: TDD Pipeline Overhaul
 
 ## TODO
 
+### Phase 6: Wire into ui-runner
+- [ ] Burst 21: Update ui-runner/src/run.ts to pass per-step model config [depends: 20]
+
 ## DONE
 
-- [x] Burst 1: Infrastructure — package.json, tsconfig.json, vitest.config.ts (8d0ef1f1)
-- [x] Burst 2: extract-code-block.ts with test (c272077c)
-- [x] Burst 3: generate-test.ts with test (ca284764)
-- [x] Burst 4: generate-component.ts with test (2263623a)
-- [x] Burst 5: generate-story.ts with test (ceaa70df)
-- [x] Burst 6: implement-component command handler with test (150fcb02)
-- [x] Burst 7: index.ts exports + type-check (a54bd956)
+### Phase 5: Integration
+- [x] Burst 17: Rewrite commands/implement-component.ts [depends: 16]
+- [x] Burst 18: Rewrite commands/implement-component.test.ts [depends: 17]
+- [x] Burst 19: Update index.ts exports + delete old files [depends: 18]
+- [x] Burst 20: Coverage verification — 123 tests, 100% coverage [depends: 19]
+
+### Phase 4: Pipeline Runner
+- [x] Burst 16: pipeline/run-pipeline.ts + test [depends: 7-15]
+
+### Phase 3: Pipeline Steps
+- [x] Burst 7: pipeline/steps/generate-test.ts + test [depends: 1]
+- [x] Burst 8: pipeline/steps/generate-component.ts + test [depends: 1]
+- [x] Burst 9: pipeline/steps/type-fix-loop.ts + test [depends: 1, 2, 6]
+- [x] Burst 10: pipeline/steps/test-fix-loop.ts + test [depends: 1, 3, 6]
+- [x] Burst 11: pipeline/steps/lint-fix-loop.ts + test [depends: 1, 4, 6]
+- [x] Burst 12: pipeline/steps/generate-story.ts + test [depends: 1]
+- [x] Burst 13: pipeline/steps/story-fix-loop.ts + test [depends: 1, 2, 6]
+- [x] Burst 14: pipeline/steps/storybook-test.ts + test [depends: 1, 2, 5]
+- [x] Burst 15: pipeline/steps/visual-test.ts + test [depends: 1]
+
+### Phase 2: Fixer Prompts
+- [x] Burst 6: Add 4 fixer prompt builders to prompt.ts + tests [depends: none]
+
+### Phase 1: Foundation
+- [x] Burst 1: pipeline/types.ts — PipelineContext, StepResult, ModelConfig, PipelineConfig [depends: none]
+- [x] Burst 2: tools/type-checker.ts + test — runTypeCheck wrapping tsc [depends: none]
+- [x] Burst 3: tools/test-runner.ts + test — runTests wrapping vitest [depends: none]
+- [x] Burst 4: tools/lint-runner.ts + test — runLint/runLintFix wrapping biome [depends: none]
+- [x] Burst 5: tools/storybook-runner.ts + test — runStorybookTest [depends: none]
+
+## DONE (previous)
+
+- [x] Burst: Infrastructure — package.json, tsconfig.json, vitest.config.ts (8d0ef1f1)
+- [x] Burst: extract-code-block.ts with test (c272077c)
+- [x] Burst: generate-test.ts with test (ca284764)
+- [x] Burst: generate-component.ts with test (2263623a)
+- [x] Burst: generate-story.ts with test (ceaa70df)
+- [x] Burst: implement-component command handler with test (150fcb02)
+- [x] Burst: index.ts exports + type-check (a54bd956)

package/package.json

@@ -6,13 +6,13 @@
   "dependencies": {
     "ai": "^6.0.0",
     "debug": "^4.4.1",
-    "@auto-engineer/message-bus": "1.98.0",
-    "@auto-engineer/model-factory": "1.98.0"
+    "@auto-engineer/message-bus": "1.100.0",
+    "@auto-engineer/model-factory": "1.100.0"
   },
   "devDependencies": {
     "vitest": "^3.2.1"
   },
-  "version": "1.98.0",
+  "version": "1.100.0",
   "publishConfig": {
     "access": "public"
   },

package/scoring-heuristic.md

@@ -0,0 +1,138 @@
+# Scoring Heuristic: component-implementor-react
+
+Total score: **100 points** across 5 categories.
+
+Scored **per component**, then aggregated across the batch.
+
+## 1. First-Pass Success (30 pts)
+
+Measures how often the LLM gets it right without fix loops. This is the strongest quality signal — it reflects upstream spec quality AND model capability.
+
+| Check | Points | How to measure |
+|---|---|---|
+| Type check passes on first try (0 type-fix iterations) | 10 | `typeFixIterations === 0` |
+| Tests pass on first try (0 test-fix iterations) | 10 | `testFixIterations === 0` |
+| Lint passes on first try (0 lint-fix iterations) | 5 | `lintFixIterations === 0` |
+| Story compiles on first try (0 story-fix iterations) | 5 | `storyFixIterations === 0` |
+
+**Partial credit:** Each fix loop iteration deducts points proportionally.
+- 1 iteration: 60% of points
+- 2 iterations: 30% of points
+- 3 iterations (max): 0 points
+
+## 2. Test Quality (25 pts)
+
+| Check | Points | How to measure |
+|---|---|---|
+| Test-to-spec ratio ≥ 0.8 | 8 | `testCount / totalSpecBullets`. Count `it()` blocks in test file vs total spec strings (structure + rendering + interaction + styling) |
+| Tests exercise all spec categories | 6 | For each non-empty spec category (structure, rendering, interaction, styling), at least one `it()` block's assertion touches that behavior. Measure by matching test assertion targets (CSS queries, text content, event triggers, class checks) against spec category keywords — not by describe block naming |
+| Low weak-assertion ratio | 5 | Grep test file for `toBeDefined`, `toBeTruthy`, `toBeFalsy`, `not.toBeNull`. Score = `max(0, 1 - (weakCount / totalAssertions))` * 5. One weak assertion in 50 total = ~5 pts; 10 weak in 12 total = ~0.8 pts |
+| All tests pass | 6 | Final test run: `passedTests === totalTests` |
+
+**Test-to-spec ratio scoring:**
+- ≥ 1.0: 8 pts (1:1 or better)
+- 0.8-0.99: 6 pts
+- 0.5-0.79: 3 pts
+- < 0.5: 0 pts
+
+## 3. Implementation Quality (20 pts)
+
+| Check | Points | How to measure |
+|---|---|---|
+| Named function export (not default) | 3 | Grep for `export function ComponentName` |
+| Props type exported | 3 | Grep for `export type ComponentNameProps` or `export interface` |
+| No TypeScript errors in final output | 4 | Final type check passes |
+| No lint violations in final output | 3 | Final biome check passes |
+| No `// @ts-ignore` or `// @ts-expect-error` | 3 | Grep for these patterns; 0 = pass |
+| No `any` type in component code | 4 | Grep for `: any` or `as any`; 0 = pass |
+
+## 4. Fix Loop Efficiency (15 pts)
+
+Measures total cost/effort to produce one component.
+
+| Metric | Points | Formula |
+|---|---|---|
+| Total LLM calls ≤ 3 (generate test + component + story) | 8 | Ideal = 3 calls. Score = `max(0, 8 - (totalCalls - 3))` |
+| Total fix iterations = 0 | 7 | Score = `max(0, 7 - totalFixIterations * 2)` |
+
+**Breakdown of ideal vs degraded:**
+
+| Total LLM Calls | Fix Iterations | Score (of 15) | Interpretation |
+|---|---|---|---|
+| 3 | 0 | 15 | Perfect — no fixes needed |
+| 4 | 1 | 11 | Good — one minor fix |
+| 5 | 2 | 7 | Acceptable — some churn |
+| 6 | 3 | 3 | Concerning — spec may be poor |
+| 7+ | 4+ | 0 | Bad — upstream spec issue |
+
+## 5. Story Quality (10 pts)
+
+| Check | Points | How to measure |
+|---|---|---|
+| Story file compiles (no type errors) | 4 | Type check on story file passes |
+| Has Default story export | 2 | Grep for `export const Default` |
+| ≥ 2 named story exports | 2 | Count `export const` in story file |
+| No `export default` in story (CSF3) | 2 | Only default export should be meta |
+
+## Aggregation: Batch Score
+
+When running multiple components in a batch:
+
+```
+batchScore = sum(componentScores) / componentCount
+
+additionalMetrics:
+  - totalLLMCalls across batch
+  - totalFixIterations across batch
+  - totalTokensUsed
+  - estimatedCost
+  - avgTimePerComponent
+```
+
+## Grading
+
+### Per Component
+
+| Score | Grade | Interpretation |
+|---|---|---|
+| 90-100 | A | Clean implementation, excellent specs |
+| 75-89 | B | Minor fixes needed, good result |
+| 60-74 | C | Multiple fix loops, but final output works |
+| < 60 | F | Component may need manual intervention |
+
+### Batch
+
+| Avg Score | Grade | Interpretation |
+|---|---|---|
+| 85+ | A | Pipeline is healthy, specs are good |
+| 70-84 | B | Some specs need improvement |
+| 55-69 | C | Systematic spec quality issue — review architect output |
+| < 55 | F | Pipeline problem — check architect or model selection |
+
+## Pipeline Gate
+
+**Per component: minimum 60.** Below this, flag the component for manual review. A component at 50 can pass with failing tests if other categories compensate — 60 ensures core quality checks (types + tests passing) aren't bypassable.
+
+**Batch average: minimum 65.** Below this, consider re-running the frontend-architect-agent with different instructions before re-implementing.
+
+## Feedback Loop Signals
+
+| Pattern | Root Cause | Fix |
+|---|---|---|
+| High type-fix iterations across many components | Props specs are wrong or incomplete | Improve architect props generation |
+| High test-fix iterations | Tests assert things not in the component | Improve test agent prompt OR spec clarity |
+| High lint-fix iterations | Code style issues | Consider adding lint rules to component agent prompt |
+| Story failures across batch | Missing prop types or wrong exports | Check architect story variant quality |
+| First-pass success < 30% batch-wide | Model capability issue | Try a stronger model for generation |
+
+## Cross-Stage Correlation
+
+Track upstream scores alongside implementor outcomes to pinpoint root causes:
+
+| Upstream Score | Implementor Outcome | Root Cause | Action |
+|---|---|---|---|
+| ID coverage < 90% | Missing components in batch | Interaction model dropped narratives | Re-run interaction-designer |
+| Architect styling prescriptiveness < 10 | High test-fix on styling assertions | Vague specs → ambiguous implementations | Re-run architect visual-design pass |
+| Architect props quality < 12 | High type-fix iterations | Bad prop types cascade to component + story | Re-run architect architecture pass |
+| Architect story completeness < 8 | High story-fix iterations | Missing required props in story args | Fix architect story variant generation |
+| Architect composition score < 8 | Import errors across batch | Missing or wrong `composes` references | Fix architect composition logic |

package/scripts/improve.ts

@@ -195,7 +195,7 @@ console.log('    -> Wrote component.tsx, test.tsx, story.tsx');
 }
 
 // ─── Evaluate one scenario with Claude CLI ───────────────────────────────────
-function evaluateScenario(scenario: string): { totalScore: number; raw: any } | null {
+function evaluateScenario(scenario: string): { totalScore: number; raw: Record<string, unknown> } | null {
   log(`  Evaluating ${scenario}...`);
 
   const improvementPrompt = readFileSync(IMPROVEMENT_PROMPT_FILE, 'utf-8');
@@ -251,8 +251,11 @@ function evaluateScenario(scenario: string): { totalScore: number; raw: any } |
       encoding: 'utf-8',
       stdio: ['pipe', 'pipe', 'pipe'],
     });
-  } catch (err: any) {
-    console.error(`    Claude CLI evaluation failed for ${scenario}:`, err.message);
+  } catch (err: unknown) {
+    console.error(
+      `    Claude CLI evaluation failed for ${scenario}:`,
+      err instanceof Error ? err.message : String(err),
+    );
     return null;
   }
 
@@ -265,15 +268,17 @@ function evaluateScenario(scenario: string): { totalScore: number; raw: any } |
   try {
     const parsed = JSON.parse(jsonMatch[0]);
     return { totalScore: parsed.totalScore ?? 0, raw: parsed };
-  } catch (err: any) {
-    console.error(`    JSON parse error for ${scenario}:`, err.message);
+  } catch (err: unknown) {
+    console.error(`    JSON parse error for ${scenario}:`, err instanceof Error ? err.message : String(err));
     return null;
   }
 }
 
 // ─── Aggregate improvements across scenarios ─────────────────────────────────
-function aggregateImprovements(evaluations: { scenario: string; eval: any }[]): any[] {
-  const allImprovements: { improvement: any; scenario: string }[] = [];
+function aggregateImprovements(
+  evaluations: { scenario: string; eval: Record<string, unknown> }[],
+): Record<string, unknown>[] {
+  const allImprovements: { improvement: Record<string, unknown>; scenario: string }[] = [];
   for (const e of evaluations) {
     for (const imp of e.eval.promptImprovements ?? []) {
       allImprovements.push({ improvement: imp, scenario: e.scenario });
@@ -281,7 +286,7 @@ function aggregateImprovements(evaluations: { scenario: string; eval: any }[]):
   }
 
   // Deduplicate by category + priority — prefer improvements that appear in multiple scenarios
-  const seen = new Map<string, { improvement: any; count: number; scenarios: string[] }>();
+  const seen = new Map<string, { improvement: Record<string, unknown>; count: number; scenarios: string[] }>();
 
   for (const { improvement, scenario } of allImprovements) {
     const key = `${improvement.category}::${improvement.priority}::${improvement.desiredBehavior?.slice(0, 80)}`;
@@ -323,7 +328,7 @@ function priorityOrder(p: string): number {
 }
 
 // ─── Apply improvements ─────────────────────────────────────────────────────
-function applyImprovements(improvements: any[]): boolean {
+function applyImprovements(improvements: Record<string, unknown>[]): boolean {
   if (improvements.length === 0) {
     log('No improvements to apply.');
     return false;
@@ -340,7 +345,7 @@ function applyImprovements(improvements: any[]): boolean {
   const currentPrompt = readFileSync(PROMPT_FILE, 'utf-8');
 
   const improvementList = improvements
-    .map((imp: any, i: number) => {
+    .map((imp: Record<string, unknown>, i: number) => {
       const freq =
         imp._frequency > 1
           ? ` (appeared in ${imp._frequency} scenarios: ${imp._appearsInScenarios?.join(', ')})`
@@ -402,8 +407,8 @@ function applyImprovements(improvements: any[]): boolean {
       encoding: 'utf-8',
       stdio: ['pipe', 'pipe', 'pipe'],
     });
-  } catch (err: any) {
-    console.error('  Claude CLI prompt-improvement failed:', err.message);
+  } catch (err: unknown) {
+    console.error('  Claude CLI prompt-improvement failed:', err instanceof Error ? err.message : String(err));
     return false;
   }
 
@@ -424,7 +429,7 @@ function applyImprovements(improvements: any[]): boolean {
 
 // ─── Print scorecard ─────────────────────────────────────────────────────────
 function printAggregateScorecard(
-  results: { scenario: string; totalScore: number; eval: any }[],
+  results: { scenario: string; totalScore: number; eval: Record<string, unknown> }[],
   avgScore: number,
 ): void {
   console.log('');
@@ -443,7 +448,7 @@ function printAggregateScorecard(
   const categories = new Map<string, { total: number; max: number; count: number }>();
   for (const r of results) {
     for (const [key, val] of Object.entries(r.eval.categories ?? {})) {
-      const v = val as any;
+      const v = val as Record<string, number>;
       const existing = categories.get(key) ?? { total: 0, max: v.maxScore, count: 0 };
       existing.total += v.score;
       existing.count++;
@@ -463,7 +468,7 @@ function printAggregateScorecard(
 // ─── Save iteration artifacts ────────────────────────────────────────────────
 function saveIteration(
   iteration: number,
-  results: { scenario: string; totalScore: number; eval: any }[],
+  results: { scenario: string; totalScore: number; eval: Record<string, unknown> }[],
   avgScore: number,
 ): void {
   const iterDir = resolve(OUTPUTS_DIR, `iteration-${iteration}`);
@@ -525,14 +530,14 @@ async function main() {
     for (const scenario of scenarios) {
       try {
         runAgentForScenario(scenario);
-      } catch (err: any) {
-        console.error(`  Agent failed on ${scenario}: ${err.message}`);
+      } catch (err: unknown) {
+        console.error(`  Agent failed on ${scenario}: ${err instanceof Error ? err.message : String(err)}`);
       }
     }
 
     // 3. Evaluate all scenarios
     log('\nEvaluating all scenarios...');
-    const results: { scenario: string; totalScore: number; eval: any }[] = [];
+    const results: { scenario: string; totalScore: number; eval: Record<string, unknown> }[] = [];
 
     for (const scenario of scenarios) {
       const outputPath = resolve(OUTPUTS_DIR, scenario, 'component.tsx');