@agile-vibe-coding/avc 0.1.1 → 0.3.1

package/cli/micro-check-validator.js

@@ -0,0 +1,538 @@
+import { readFileSync, existsSync } from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import { runTier1Check, runTier2Check, runTier1Batch, runTier2Batch } from './micro-check-runner.js';
+import { scoreChecks } from './micro-check-scorer.js';
+import { fixFailedChecks } from './micro-check-fixer.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+// ---------------------------------------------------------------------------
+// Debug helper — matches existing patterns in epic-story-validator.js
+// ---------------------------------------------------------------------------
+
+// ---------------------------------------------------------------------------
+// Utility
+// ---------------------------------------------------------------------------
+
+function chunkArray(array, size) {
+  const chunks = [];
+  for (let i = 0; i < array.length; i += size) {
+    chunks.push(array.slice(i, i + size));
+  }
+  return chunks;
+}
+
+function debug(message, data = null) {
+  const timestamp = new Date().toISOString();
+  if (data) {
+    console.log(`[DEBUG][${timestamp}] ${message}`, JSON.stringify(data, null, 2));
+  } else {
+    console.log(`[DEBUG][${timestamp}] ${message}`);
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Check-definition loader
+// ---------------------------------------------------------------------------
+
+/**
+ * Load Tier 1 and Tier 2 check definitions for the requested perspectives.
+ *
+ * Resolution order per perspective:
+ *   1. Project override  .avc/customized-agents/checks/{workItemType}/{perspective}.json
+ *   2. Built-in          src/cli/checks/{workItemType}/{perspective}.json
+ *
+ * @param {string}   workItemType  - "epic" or "story"
+ * @param {string[]} perspectives  - List of perspective names
+ * @param {string}   [projectRoot] - Project root for override loading
+ * @returns {{ tier1: Object[], tier2: Object[] }}
+ */
+function loadCheckDefinitions(workItemType, perspectives, projectRoot) {
+  const checks = { tier1: [], tier2: [] };
+  const checksDir = path.join(__dirname, 'checks');
+
+  debug('Loading check definitions', { workItemType, perspectives, checksDir });
+
+  for (const perspective of perspectives) {
+    // Resolve Tier 1 file — project override takes precedence
+    const builtinPath = path.join(checksDir, workItemType, `${perspective}.json`);
+    const overridePath = projectRoot
+      ? path.join(projectRoot, '.avc', 'customized-agents', 'checks', workItemType, `${perspective}.json`)
+      : null;
+    const filePath = (overridePath && existsSync(overridePath)) ? overridePath : builtinPath;
+
+    if (existsSync(filePath)) {
+      try {
+        const raw = JSON.parse(readFileSync(filePath, 'utf8'));
+        // Support both formats: plain array or { checks: [...] } wrapper
+        const perspectiveChecks = Array.isArray(raw) ? raw : (Array.isArray(raw.checks) ? raw.checks : []);
+        debug(`Loaded ${perspectiveChecks.length} Tier 1 checks from ${filePath}`);
+        checks.tier1.push(...perspectiveChecks);
+      } catch (err) {
+        console.error(`[micro-check-validator] Failed to parse ${filePath}: ${err.message}`);
+      }
+    } else {
+      debug(`No Tier 1 check file found for perspective "${perspective}" (looked at ${builtinPath})`);
+    }
+  }
+
+  // Load Tier 2 cross-reference checks
+  const crossRefBuiltin = path.join(checksDir, 'cross-refs', `${workItemType}.json`);
+  const crossRefOverride = projectRoot
+    ? path.join(projectRoot, '.avc', 'customized-agents', 'checks', 'cross-refs', `${workItemType}.json`)
+    : null;
+  const crossRefPath = (crossRefOverride && existsSync(crossRefOverride)) ? crossRefOverride : crossRefBuiltin;
+
+  if (existsSync(crossRefPath)) {
+    try {
+      const raw = JSON.parse(readFileSync(crossRefPath, 'utf8'));
+      const crossRefChecks = Array.isArray(raw) ? raw : (Array.isArray(raw.checks) ? raw.checks : []);
+      // Only include cross-refs whose perspectives overlap with selected perspectives
+      const applicable = crossRefChecks.filter(c =>
+        c.perspectives && c.perspectives.some(p => perspectives.includes(p))
+      );
+      debug(`Loaded ${applicable.length}/${crossRefChecks.length} Tier 2 cross-ref checks from ${crossRefPath}`);
+      checks.tier2.push(...applicable);
+    } catch (err) {
+      console.error(`[micro-check-validator] Failed to parse ${crossRefPath}: ${err.message}`);
+    }
+  } else {
+    debug(`No Tier 2 cross-ref file found (looked at ${crossRefBuiltin})`);
+  }
+
+  debug('Check definitions loaded', {
+    tier1Count: checks.tier1.length,
+    tier2Count: checks.tier2.length
+  });
+
+  return checks;
+}
+
+// ---------------------------------------------------------------------------
+// Concurrency-limited parallel runner
+// ---------------------------------------------------------------------------
+
+/**
+ * Run an array of async task factories with a concurrency cap.
+ *
+ * @param {Array<() => Promise>} tasks       - Factory functions returning promises
+ * @param {number}               concurrency - Max parallel tasks
+ * @returns {Promise<any[]>}
+ */
+async function runWithConcurrency(tasks, concurrency) {
+  const results = [];
+  const executing = new Set();
+
+  for (const task of tasks) {
+    const p = task().then(result => {
+      executing.delete(p);
+      return result;
+    });
+    executing.add(p);
+    results.push(p);
+
+    if (executing.size >= concurrency) {
+      await Promise.race(executing);
+    }
+  }
+
+  return Promise.all(results);
+}
+
+// ---------------------------------------------------------------------------
+// Format helpers
+// ---------------------------------------------------------------------------
+
+/**
+ * Convert a micro-check result into the issue format used by the existing
+ * aggregateValidationResults() shape.
+ *
+ * @param {Object} check - A single check result object
+ * @returns {Object}
+ */
+function toIssueFormat(check) {
+  return {
+    severity: check.severity,
+    category: check.category,
+    description: check.failDescription || `Failed: ${check.id}`,
+    suggestion: check.failSuggestion || '',
+    example: ''
+  };
+}
+
+/**
+ * Compute a per-perspective score by filtering allResults down to checks
+ * belonging to a single perspective and scoring them independently.
+ *
+ * @param {string}   perspective - Perspective name
+ * @param {Object[]} allResults  - Combined Tier 1 + Tier 2 results
+ * @returns {Object} scoreChecks() output for this perspective slice
+ */
+function perspectiveSlice(perspective, allResults) {
+  const slice = allResults.filter(r =>
+    r.perspective === perspective ||
+    (Array.isArray(r.perspectives) && r.perspectives.includes(perspective))
+  );
+  if (slice.length === 0) {
+    return { score: 100, status: 'excellent', applicableCount: 0, skippedCount: 0 };
+  }
+  return scoreChecks(slice);
+}
+
+/**
+ * Derive issues for a single perspective from allResults.
+ *
+ * @param {string}   perspective - Perspective name
+ * @param {Object[]} allResults  - Combined Tier 1 + Tier 2 results
+ * @returns {Object[]}
+ */
+function perspectiveIssues(perspective, allResults) {
+  return allResults
+    .filter(r =>
+      (r.perspective === perspective ||
+        (Array.isArray(r.perspectives) && r.perspectives.includes(perspective))) &&
+      r.passed === false  // excludes null (errored) and true (passed)
+    )
+    .map(toIssueFormat);
+}
+
+// ---------------------------------------------------------------------------
+// Main entry point
+// ---------------------------------------------------------------------------
+
+/**
+ * Validate a work item using the 3-tier micro-check system.
+ * Single flow for ALL models — decomposed checks with programmatic scoring.
+ *
+ * @param {Object}   workItem              - Epic or story work.json object
+ * @param {string}   workItemText          - Work item context markdown
+ * @param {string}   workItemType          - "epic" or "story"
+ * @param {string[]} perspectives          - Perspective names to validate
+ * @param {Object}   llmProvider           - LLM provider instance
+ * @param {Object}   [options]             - Configuration options
+ * @param {number}   [options.concurrency=5]       - Max parallel LLM calls
+ * @param {number}   [options.maxFixAttempts=3]    - Max fix iterations
+ * @param {Function} [options.generateContextFn]   - Regenerate context markdown from work item
+ * @param {Function} [options.progressCallback]    - Progress reporting callback
+ * @param {string}   [options.projectRoot]         - Project root for check override loading
+ * @returns {Promise<Object>} Result compatible with existing validateEpic()/validateStory() return shape
+ */
+export async function validateWithMicroChecks(
+  workItem,
+  workItemText,
+  workItemType,
+  perspectives,
+  llmProvider,
+  options = {}
+) {
+  const {
+    concurrency = 5,
+    batchSize = 8,
+    maxFixAttempts = 3,
+    generateContextFn = null,
+    progressCallback = null,
+    projectRoot = null
+  } = options;
+
+  const startTime = Date.now();
+  debug('validateWithMicroChecks START', {
+    workItemType,
+    perspectives,
+    concurrency,
+    maxFixAttempts,
+    hasGenerateContextFn: !!generateContextFn
+  });
+
+  const progress = (msg) => {
+    debug(msg);
+    if (progressCallback) {
+      progressCallback(msg);
+    }
+  };
+
+  // -----------------------------------------------------------------
+  // Step 1 — Load check definitions
+  // -----------------------------------------------------------------
+  const checks = loadCheckDefinitions(workItemType, perspectives, projectRoot);
+  progress(`Loaded ${checks.tier1.length} Tier 1 + ${checks.tier2.length} Tier 2 checks for ${perspectives.length} perspective(s)`);
+
+  // -----------------------------------------------------------------
+  // Step 2 — Phase 1: Run Tier 1 checks in parallel
+  // -----------------------------------------------------------------
+  progress('Phase 1: Running Tier 1 checks...');
+  const tier1Start = Date.now();
+
+  const tier1Batches = chunkArray(checks.tier1, batchSize);
+  debug(`Tier 1: ${checks.tier1.length} checks in ${tier1Batches.length} batch(es) of up to ${batchSize}`);
+
+  const tier1Tasks = tier1Batches.map(batch =>
+    () => runTier1Batch(batch, workItemText, llmProvider, workItemType).catch(err => {
+      debug(`Tier 1 batch error (${batch.length} checks): ${err.message}`);
+      // Fallback: run each check individually
+      return Promise.all(batch.map(check =>
+        runTier1Check(check, workItemText, llmProvider, workItemType).catch(indivErr => {
+          debug(`Tier 1 check ${check.id} individual fallback error: ${indivErr.message}`);
+          return {
+            id: check.id, tier: 1, severity: check.severity, category: check.category,
+            perspective: check.perspective, universal: check.universal,
+            applicable: true, passed: null, evidence: `Error: ${indivErr.message}`,
+            failDescription: check.failDescription, failSuggestion: check.failSuggestion,
+          };
+        })
+      ));
+    })
+  );
+
+  const tier1BatchResults = await runWithConcurrency(tier1Tasks, concurrency);
+  const tier1ResultsArray = tier1BatchResults.flat();
+
+  // Build Map: checkId → result
+  const tier1Results = new Map();
+  let tier1Passed = 0;
+  let tier1Failed = 0;
+  let tier1Skipped = 0;
+  let tier1Errored = 0;
+
+  for (const result of tier1ResultsArray) {
+    tier1Results.set(result.id, result);
+    if (result.applicable === false) {
+      tier1Skipped++;
+    } else if (result.passed === null) {
+      tier1Errored++;
+    } else if (result.passed) {
+      tier1Passed++;
+    } else {
+      tier1Failed++;
+    }
+  }
+
+  const tier1Elapsed = Date.now() - tier1Start;
+  const errorSuffix = tier1Errored > 0 ? `, ${tier1Errored} errored` : '';
+  progress(
+    `Tier 1: ${tier1ResultsArray.length}/${checks.tier1.length} checks in ${tier1Batches.length} batch(es) ` +
+    `(${tier1Passed} passed, ${tier1Failed} failed, ${tier1Skipped} skipped${errorSuffix}) — ${tier1Elapsed}ms`
+  );
+
+  // -----------------------------------------------------------------
+  // Step 3 — Phase 2: Run Tier 2 cross-reference checks in parallel
+  // -----------------------------------------------------------------
+  progress('Phase 2: Running Tier 2 cross-reference checks...');
+  const tier2Start = Date.now();
+
+  // Only run Tier 2 checks whose dependsOn checks are ALL present in tier1Results
+  const applicableTier2 = checks.tier2.filter(check => {
+    if (!Array.isArray(check.dependsOn) || check.dependsOn.length === 0) {
+      return true;
+    }
+    return check.dependsOn.every(depId => tier1Results.has(depId));
+  });
+
+  debug(`Tier 2: ${applicableTier2.length}/${checks.tier2.length} checks have satisfied dependencies`);
+
+  const tier2Batches = chunkArray(applicableTier2, batchSize);
+  debug(`Tier 2: ${applicableTier2.length} checks in ${tier2Batches.length} batch(es) of up to ${batchSize}`);
+
+  const tier2Tasks = tier2Batches.map(batch =>
+    () => runTier2Batch(batch, workItemText, tier1Results, llmProvider, workItemType).catch(err => {
+      debug(`Tier 2 batch error (${batch.length} checks): ${err.message}`);
+      // Fallback: run each check individually
+      return Promise.all(batch.map(check =>
+        runTier2Check(check, workItemText, tier1Results, llmProvider).catch(indivErr => {
+          debug(`Tier 2 check ${check.id} individual fallback error: ${indivErr.message}`);
+          return {
+            id: check.id, tier: 2, severity: check.severity, category: check.category,
+            perspectives: check.perspectives,
+            applicable: true, passed: null, evidence: `Error: ${indivErr.message}`,
+            failDescription: check.failDescription, failSuggestion: check.failSuggestion,
+          };
+        })
+      ));
+    })
+  );
+
+  const tier2BatchResults = await runWithConcurrency(tier2Tasks, concurrency);
+  const tier2ResultsArray = tier2BatchResults.flat();
+
+  let tier2Passed = 0;
+  let tier2Failed = 0;
+  let tier2Skipped = 0;
+  let tier2Errored = 0;
+
+  for (const result of tier2ResultsArray) {
+    if (result.applicable === false) {
+      tier2Skipped++;
+    } else if (result.passed === null) {
+      tier2Errored++;
+    } else if (result.passed) {
+      tier2Passed++;
+    } else {
+      tier2Failed++;
+    }
+  }
+
+  const tier2Elapsed = Date.now() - tier2Start;
+  const tier2ErrorSuffix = tier2Errored > 0 ? `, ${tier2Errored} errored` : '';
+  progress(
+    `Tier 2: ${tier2ResultsArray.length}/${applicableTier2.length} checks in ${tier2Batches.length} batch(es) ` +
+    `(${tier2Passed} passed, ${tier2Failed} failed, ${tier2Skipped} skipped${tier2ErrorSuffix}) — ${tier2Elapsed}ms`
+  );
+
+  // -----------------------------------------------------------------
+  // Step 4 — Phase 3: Programmatic scoring (Tier 3 patterns included)
+  // -----------------------------------------------------------------
+  progress('Phase 3: Scoring...');
+
+  let allResults = [...tier1ResultsArray, ...tier2ResultsArray];
+  let scored = scoreChecks(allResults);
+
+  const criticalFails = allResults.filter(r => r.passed === false && r.severity === 'critical').length;
+  const majorFails = allResults.filter(r => r.passed === false && r.severity === 'major').length;
+  const minorFails = allResults.filter(r => r.passed === false && r.severity === 'minor').length;
+  const totalErrored = allResults.filter(r => r.passed === null && r.applicable !== false).length;
+
+  progress(
+    `Score: ${scored.score}/100 (${scored.status}) — ` +
+    `${criticalFails} critical, ${majorFails} major, ${minorFails} minor` +
+    (totalErrored > 0 ? ` (${totalErrored} checks excluded due to LLM errors)` : '')
+  );
+
+  // Log individual failed checks for diagnostics
+  const failedDetails = allResults.filter(r => r.passed === false);
+  if (failedDetails.length > 0) {
+    debug('Failed checks detail', failedDetails.map(r => ({
+      id: r.id,
+      severity: r.severity,
+      perspective: r.perspective || r.perspectives,
+      evidence: (r.evidence || '').slice(0, 200),
+    })));
+  }
+
+  // -----------------------------------------------------------------
+  // Step 5 — Phase 4: Fix failed checks (if critical or major)
+  // -----------------------------------------------------------------
+  let fixResults = null;
+
+  if (criticalFails > 0 || majorFails > 0) {
+    progress('Phase 4: Attempting fixes for critical/major failures...');
+    const fixStart = Date.now();
+
+    // Collect failed checks sorted by severity (critical first, then major)
+    const severityOrder = { critical: 0, major: 1, minor: 2 };
+    const failedChecks = allResults
+      .filter(r => r.passed === false && (r.severity === 'critical' || r.severity === 'major'))
+      .sort((a, b) => (severityOrder[a.severity] || 99) - (severityOrder[b.severity] || 99));
+
+    debug(`Attempting fixes for ${failedChecks.length} failed checks`, {
+      ids: failedChecks.map(c => c.id)
+    });
+
+    try {
+      fixResults = await fixFailedChecks({
+        failedChecks,
+        allCheckDefinitions: [...checks.tier1, ...checks.tier2],
+        allCheckResults: allResults,
+        workItem,
+        workItemText,
+        workItemType,
+        llmProvider,
+        maxFixAttempts,
+        generateContextFn,
+        progressCallback: progress,
+        concurrency,
+        tier1Results,
+      });
+
+      const fixElapsed = Date.now() - fixStart;
+      debug(`Fix phase completed in ${fixElapsed}ms`, {
+        fixesAttempted: fixResults?.fixResults?.length || 0,
+        regressionDetected: fixResults?.regressionDetected || false
+      });
+
+      // If fixes were applied, rescore using the rerun results
+      if (fixResults && fixResults.rerunResults && fixResults.rerunResults.length > 0) {
+        progress('Rescoring after fixes...');
+        allResults = fixResults.rerunResults;
+        scored = scoreChecks(allResults);
+
+        const postFixCritical = allResults.filter(r => r.passed === false && r.severity === 'critical').length;
+        const postFixMajor = allResults.filter(r => r.passed === false && r.severity === 'major').length;
+        const postFixMinor = allResults.filter(r => r.passed === false && r.severity === 'minor').length;
+
+        progress(
+          `Post-fix score: ${scored.score}/100 (${scored.status}) — ` +
+          `${postFixCritical} critical, ${postFixMajor} major, ${postFixMinor} minor`
+        );
+      }
+    } catch (fixErr) {
+      console.error(`[micro-check-validator] Fix phase error: ${fixErr.message}`);
+      debug('Fix phase failed', { error: fixErr.message, stack: fixErr.stack });
+    }
+  } else {
+    debug('Phase 4 skipped — no critical or major failures');
+  }
+
+  // -----------------------------------------------------------------
+  // Step 6 — Phase 5: Build result in existing format
+  // -----------------------------------------------------------------
+  progress('Phase 5: Building result...');
+
+  const finalScore = scored;
+
+  // Collect final failed checks across all severities (exclude errored/skipped)
+  const finalFailedChecks = allResults.filter(r => r.passed === false);
+
+  // Build per-perspective validator results
+  const validatorResults = perspectives.map(p => {
+    const pScore = perspectiveSlice(p, allResults);
+    const pIssues = perspectiveIssues(p, allResults);
+    return {
+      validator: `micro-check-${workItemType}-${p}`,
+      score: pScore.score,
+      status: pScore.status,
+      issues: pIssues
+    };
+  });
+
+  const result = {
+    averageScore: finalScore.score,
+    overallStatus: finalScore.status,
+    readyToPublish: finalScore.status !== 'needs-improvement',
+    readyForImplementation: finalScore.status !== 'needs-improvement',
+    criticalIssues: finalFailedChecks
+      .filter(c => c.severity === 'critical')
+      .map(toIssueFormat),
+    majorIssues: finalFailedChecks
+      .filter(c => c.severity === 'major')
+      .map(toIssueFormat),
+    minorIssues: finalFailedChecks
+      .filter(c => c.severity === 'minor')
+      .map(toIssueFormat),
+    validatorResults,
+    validatedAt: new Date().toISOString(),
+    microCheckDetails: {
+      tier1Count: tier1Results.size,
+      tier2Count: tier2ResultsArray.length,
+      applicableCount: finalScore.applicableCount,
+      skippedCount: finalScore.skippedCount,
+      splitRecommendation: finalScore.splitRecommendation,
+      splitReason: finalScore.splitReason,
+      clusterWarnings: finalScore.clusterWarnings,
+      fixesAttempted: fixResults?.fixResults?.length || 0,
+      regressionDetected: fixResults?.regressionDetected || false
+    }
+  };
+
+  const totalElapsed = Date.now() - startTime;
+  debug('validateWithMicroChecks COMPLETE', {
+    score: result.averageScore,
+    status: result.overallStatus,
+    criticalIssues: result.criticalIssues.length,
+    majorIssues: result.majorIssues.length,
+    minorIssues: result.minorIssues.length,
+    totalElapsedMs: totalElapsed
+  });
+
+  return result;
+}