agentic-qe 3.13.0 → 3.13.1

package/dist/coordination/handlers/coverage-handlers.js

@@ -5,106 +5,155 @@
  * Covers: analyze-coverage
  */
 import * as fs from 'fs/promises';
-import * as fsSync from 'fs';
 import * as path from 'path';
 import { ok, err } from '../../shared/types';
 import { toError } from '../../shared/error-utils.js';
 import { safeJsonParse } from '../../shared/safe-json.js';
 import { loadCoverageData } from './handler-utils';
+import { buildEstimatedCoverage, collectRustCoverage, isCoverageExecDisabled, isRustProject, isTestPath, } from './coverage-collection';
 /**
- * Build heuristic coverage data from source files when no instrumented
- * coverage is available. Uses the same logic as the CLI's buildCoverageData.
+ * Try to produce a JS/TS coverage report by running the project's test runner.
+ * Returns true if the runner completed (which does not guarantee a report was
+ * written — the caller re-checks disk).
  */
-function buildHeuristicCoverage(targetPath) {
-    const SOURCE_EXTENSIONS = new Set(['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs', '.py', '.go', '.rs', '.java', '.rb']);
-    const SKIP_DIRS = new Set(['node_modules', 'dist', 'build', 'coverage', '.git', '.claude', '.agentic-qe', '__pycache__', '.venv']);
-    const TEST_PATTERNS = ['.test.', '.spec.', '_test.', '_spec.'];
-    if (!fsSync.existsSync(targetPath) || !fsSync.statSync(targetPath).isDirectory())
-        return null;
-    const sourceFiles = [];
-    function walk(dir, depth) {
-        if (depth > 6)
-            return;
-        let entries;
+async function tryRunJsCoverage(targetPath) {
+    // Running the project's test suite executes code from the analyzed repository
+    // (the tests themselves, plus npm lifecycle scripts via npx). Honor the
+    // opt-out before doing that.
+    if (isCoverageExecDisabled())
+        return false;
+    try {
+        const { execSync } = await import('child_process');
+        let coverageCmd = 'npx vitest run --coverage --reporter=json 2>/dev/null';
         try {
-            entries = fsSync.readdirSync(dir, { withFileTypes: true });
-        }
-        catch {
-            return;
-        }
-        for (const entry of entries) {
-            if (entry.isDirectory()) {
-                if (SKIP_DIRS.has(entry.name))
-                    continue;
-                walk(path.join(dir, entry.name), depth + 1);
+            const pkgContent = await fs.readFile(path.join(targetPath, 'package.json'), 'utf-8');
+            const pkg = safeJsonParse(pkgContent);
+            const deps = {
+                ...(pkg.devDependencies || {}),
+                ...(pkg.dependencies || {}),
+            };
+            if (deps['jest'] || deps['@jest/core']) {
+                coverageCmd = 'npx jest --coverage --json 2>/dev/null';
             }
-            else if (entry.isFile()) {
-                const ext = path.extname(entry.name);
-                if (!SOURCE_EXTENSIONS.has(ext))
-                    continue;
-                if (TEST_PATTERNS.some(p => entry.name.includes(p)))
-                    continue;
-                sourceFiles.push(path.join(dir, entry.name));
+            else if (deps['mocha'] || deps['nyc']) {
+                coverageCmd = 'npx nyc mocha 2>/dev/null';
             }
+            // vitest is the default — covers vitest, @vitest/coverage-v8, etc.
         }
+        catch {
+            // No package.json — use default vitest
+        }
+        execSync(coverageCmd, {
+            cwd: targetPath,
+            timeout: 120000,
+            encoding: 'utf-8',
+            stdio: ['pipe', 'pipe', 'pipe'],
+        });
+        return true;
     }
-    walk(targetPath, 0);
-    if (sourceFiles.length === 0)
+    catch {
+        // Test runner failed or not available.
+        return false;
+    }
+}
+/**
+ * Narrow an instrumented report to production files only, recomputing the
+ * aggregate from what survives.
+ *
+ * Returns `null` when the report contains no production files at all — the
+ * caller must NOT fall back to the unfiltered set, because that reports the test
+ * files' own coverage as the project's (#569's third contradiction).
+ */
+function productionOnly(report, targetPath) {
+    const files = report.files.filter(f => !isTestPath(f.path, targetPath));
+    if (files.length === 0)
         return null;
-    const files = sourceFiles.map(filePath => {
-        let content = '';
-        try {
-            content = fsSync.readFileSync(filePath, 'utf-8');
-        }
-        catch { /* skip */ }
-        const lines = content.split('\n');
-        const totalLines = lines.length;
-        const functionCount = Math.max((content.match(/\b(function|=>)\b/g) || []).length, 1);
-        const branchCount = Math.max((content.match(/\b(if|switch|case|\?\?|\|\|)\b/g) || []).length, 1);
-        // Check for co-located test file
-        const testVariants = [
-            filePath.replace('.ts', '.test.ts').replace('/src/', '/tests/'),
-            filePath.replace('.ts', '.spec.ts').replace('/src/', '/tests/'),
-            filePath.replace('.ts', '.test.ts'),
-            filePath.replace('.js', '.test.js'),
-            filePath.replace('.js', '.test.js').replace('/src/', '/tests/'),
-        ];
-        const hasTest = testVariants.some(t => fsSync.existsSync(t));
-        const complexityPenalty = Math.min(branchCount * 0.005, 0.15);
-        const sizePenalty = Math.min(totalLines * 0.0001, 0.1);
-        const coverageRate = Math.max(0.05, Math.min(0.95, hasTest ? 0.85 - complexityPenalty - sizePenalty : 0.20));
-        const coveredLines = Math.floor(totalLines * coverageRate);
-        const branchCoverage = Math.min(Math.floor(coveredLines * 0.7), branchCount);
-        const functionCoverage = Math.min(Math.floor(functionCount * coverageRate * 0.9), functionCount);
-        const uncoveredLines = Array.from({ length: totalLines - coveredLines }, (_, i) => i + coveredLines + 1);
-        return {
-            path: filePath,
-            lines: { covered: coveredLines, total: totalLines },
-            branches: { covered: branchCoverage, total: branchCount },
-            functions: { covered: functionCoverage, total: functionCount },
-            statements: { covered: coveredLines, total: totalLines },
-            uncoveredLines,
-            uncoveredBranches: uncoveredLines.slice(0, Math.floor(uncoveredLines.length / 2)),
-        };
-    });
-    const totalLines = files.reduce((s, f) => s + f.lines.total, 0);
-    const coveredLines = files.reduce((s, f) => s + f.lines.covered, 0);
-    const totalBranches = files.reduce((s, f) => s + f.branches.total, 0);
-    const coveredBranches = files.reduce((s, f) => s + f.branches.covered, 0);
-    const totalFunctions = files.reduce((s, f) => s + f.functions.total, 0);
-    const coveredFunctions = files.reduce((s, f) => s + f.functions.covered, 0);
-    const safeDiv = (a, b) => b > 0 ? Math.round((a / b) * 100) : 0;
+    // Recompute rather than carrying the original summary: it was computed over
+    // the unfiltered set, so keeping it would blend test-file coverage into the
+    // production headline figure.
+    const sum = (pick) => files.reduce((acc, f) => acc + pick(f), 0);
+    const pct = (covered, total) => total > 0 ? Math.round((covered / total) * 100) : 0;
+    const totalLines = sum(f => f.lines.total);
+    const coveredLines = sum(f => f.lines.covered);
+    const branchDataCollected = files.some(f => f.branches.total > 0);
+    const functionDataCollected = files.some(f => f.functions.total > 0);
     return {
-        files,
-        summary: {
-            line: safeDiv(coveredLines, totalLines),
-            branch: safeDiv(coveredBranches, totalBranches),
-            function: safeDiv(coveredFunctions, totalFunctions),
-            statement: safeDiv(coveredLines, totalLines),
-            files: files.length,
+        data: {
+            files,
+            summary: {
+                line: pct(coveredLines, totalLines),
+                branch: pct(sum(f => f.branches.covered), sum(f => f.branches.total)),
+                function: pct(sum(f => f.functions.covered), sum(f => f.functions.total)),
+                statement: pct(coveredLines, totalLines),
+                files: files.length,
+            },
+        },
+        provenance: {
+            method: 'instrumented-report',
+            estimated: false,
+            branchDataCollected,
+            functionDataCollected,
+            notes: [],
         },
     };
 }
+/**
+ * Collect coverage for a target, preferring real measurement over estimation
+ * and always reporting which one happened (#569).
+ *
+ * Cascade:
+ *   1. an instrumented report already on disk (istanbul/lcov/cobertura)
+ *   2. language-specific instrumentation we can run — `cargo llvm-cov` for Rust,
+ *      the project's test runner for JS/TS
+ *   3. static estimation, explicitly labelled `estimated: true`
+ */
+async function collectCoverage(targetPath) {
+    const existing = await loadCoverageData(targetPath);
+    if (existing) {
+        const fromReport = productionOnly(existing, targetPath);
+        // A report whose only entries are test files yields NO production coverage.
+        // Falling back to the unfiltered set here would report the test files'
+        // own coverage as the project's — recreating #569's third contradiction
+        // (`tests/verifier_matrix.rs` analysed at 79.2% as if it were a target).
+        if (!fromReport)
+            return { collected: null, ranTests: false, testOnlyReport: true };
+        return { collected: fromReport, ranTests: false };
+    }
+    // #569 work item 1: delegate to real instrumentation where available.
+    if (isRustProject(targetPath)) {
+        const rust = await collectRustCoverage(targetPath);
+        if (rust)
+            return { collected: rust, ranTests: true };
+    }
+    const ranTests = await tryRunJsCoverage(targetPath);
+    const afterRun = await loadCoverageData(targetPath);
+    if (afterRun) {
+        const fromRun = productionOnly(afterRun, targetPath);
+        if (!fromRun)
+            return { collected: null, ranTests, testOnlyReport: true };
+        return { collected: fromRun, ranTests };
+    }
+    return { collected: buildEstimatedCoverage(targetPath), ranTests };
+}
+/**
+ * Build the guidance string shown when a result is an estimate rather than a
+ * measurement, tailored to what the target actually is.
+ */
+function estimationGuidance(targetPath) {
+    // When the caller turned execution off, naming the tool to install is the
+    // wrong advice — nothing will be run either way until they opt back in.
+    if (isCoverageExecDisabled()) {
+        return 'Coverage tooling execution is disabled (AQE_COVERAGE_NO_EXEC). No tests ' +
+            'were run and no build tooling was invoked. Unset it, or generate a coverage ' +
+            'report yourself and re-run, to get real measurements.';
+    }
+    if (isRustProject(targetPath)) {
+        return 'To get real measurements for this Rust crate, install cargo-llvm-cov ' +
+            '(`cargo install cargo-llvm-cov`) and re-run; agentic-qe will invoke it automatically.';
+    }
+    return 'To get real measurements, run your tests with coverage enabled ' +
+        '(e.g. `npm test -- --coverage`, `pytest --cov`) and re-run this analysis.';
+}
 export function registerCoverageHandlers(ctx) {
     // Register coverage analysis handler - REAL IMPLEMENTATION
     ctx.registerHandler('analyze-coverage', async (task) => {
@@ -113,63 +162,30 @@ export function registerCoverageHandlers(ctx) {
             const analyzer = ctx.getCoverageAnalyzer();
             const targetPath = payload.target || process.cwd();
             const threshold = payload.threshold || 80;
-            // Try to find and read actual coverage files
-            let coverageData = await loadCoverageData(targetPath);
-            if (!coverageData) {
-                // No coverage data found — attempt to collect it by running tests with coverage
-                let collected = false;
-                try {
-                    const { execSync } = await import('child_process');
-                    // Detect test runner from package.json
-                    let coverageCmd = 'npx vitest run --coverage --reporter=json 2>/dev/null';
-                    try {
-                        const pkgContent = await fs.readFile(path.join(targetPath, 'package.json'), 'utf-8');
-                        const pkg = safeJsonParse(pkgContent);
-                        const deps = { ...(pkg.devDependencies || {}), ...(pkg.dependencies || {}) };
-                        if (deps['jest'] || deps['@jest/core']) {
-                            coverageCmd = 'npx jest --coverage --json 2>/dev/null';
-                        }
-                        else if (deps['mocha'] || deps['nyc']) {
-                            coverageCmd = 'npx nyc mocha 2>/dev/null';
-                        }
-                        // vitest is the default — covers vitest, @vitest/coverage-v8, etc.
-                    }
-                    catch {
-                        // No package.json — use default vitest
-                    }
-                    execSync(coverageCmd, {
-                        cwd: targetPath,
-                        timeout: 120000,
-                        encoding: 'utf-8',
-                        stdio: ['pipe', 'pipe', 'pipe'],
-                    });
-                    collected = true;
-                }
-                catch {
-                    // Test runner failed or not available — that's OK, we'll check for output anyway
-                }
-                // Re-check for coverage data after collection attempt
-                coverageData = await loadCoverageData(targetPath);
-                // Still no instrumented coverage — fall back to heuristic estimation
-                if (!coverageData) {
-                    coverageData = buildHeuristicCoverage(targetPath);
-                }
-                if (!coverageData) {
-                    return ok({
-                        lineCoverage: 0,
-                        branchCoverage: 0,
-                        functionCoverage: 0,
-                        statementCoverage: 0,
-                        totalFiles: 0,
-                        coverageByFile: [],
-                        gaps: [],
-                        algorithm: 'sublinear-O(log n)',
-                        warning: collected
+            const { collected, ranTests, testOnlyReport } = await collectCoverage(targetPath);
+            if (!collected) {
+                return ok({
+                    lineCoverage: null,
+                    branchCoverage: null,
+                    functionCoverage: null,
+                    statementCoverage: null,
+                    totalFiles: 0,
+                    coverageByFile: [],
+                    gaps: [],
+                    estimated: false,
+                    measured: false,
+                    coverageMethod: 'none',
+                    algorithm: 'sublinear-O(log n)',
+                    warning: testOnlyReport
+                        ? 'The coverage report contains only test files, so there is no production ' +
+                            'coverage to report. Coverage OF a test file is not a meaningful metric. ' +
+                            'Check that your coverage tool is instrumenting your source directory.'
+                        : ranTests
                             ? 'Tests ran but no coverage output was generated. Ensure a coverage provider is configured (e.g., @vitest/coverage-v8, istanbul).'
-                            : 'No coverage data found and could not run tests automatically. Run: npm test -- --coverage',
-                    });
-                }
+                            : 'No coverage data found and could not run tests automatically. ' + estimationGuidance(targetPath),
+                });
             }
+            const { data: coverageData, provenance } = collected;
             // Analyze coverage using the real CoverageAnalyzerService
             const analysisResult = await analyzer.analyze({
                 coverageData,
@@ -189,26 +205,63 @@ export function registerCoverageHandlers(ctx) {
                         file: gap.file,
                         lines: gap.lines,
                         risk: gap.severity,
+                        // #569: a gap derived from an estimate is a hypothesis, not a
+                        // finding. Carry that all the way to the caller rather than letting
+                        // it inherit a default riskScore/confidence downstream.
+                        estimated: provenance.estimated,
+                        ...(provenance.estimated
+                            ? {
+                                confidence: 0.2,
+                                reason: 'Possible coverage gap (STATIC ESTIMATE — no instrumentation ran)',
+                            }
+                            : {}),
                     }));
                 }
             }
+            const round = (n) => Math.round(n * 10) / 10;
+            const pct = (covered, total) => total > 0 ? Math.round((covered / total) * 1000) / 10 : null;
+            const warnings = [...provenance.notes];
+            if (provenance.estimated)
+                warnings.push(estimationGuidance(targetPath));
+            // #569: the analyzer generates recommendations from the summary numbers.
+            // When branch or function data was never collected, those numbers are 0
+            // by construction, and the resulting advice ("Branch coverage is
+            // significantly lower than line coverage", "Function coverage is below
+            // 70%") is a statement about a metric nobody measured. Drop it rather
+            // than send a developer to fix a number that doesn't exist.
+            const recommendations = report.recommendations.filter(rec => {
+                if (!provenance.branchDataCollected && /branch coverage/i.test(rec))
+                    return false;
+                if (!provenance.functionDataCollected && /function coverage/i.test(rec))
+                    return false;
+                return true;
+            });
             return ok({
-                lineCoverage: Math.round(report.summary.line * 10) / 10,
-                branchCoverage: Math.round(report.summary.branch * 10) / 10,
-                functionCoverage: Math.round(report.summary.function * 10) / 10,
-                statementCoverage: Math.round(report.summary.statement * 10) / 10,
+                lineCoverage: round(report.summary.line),
+                // #569 work item 4: never report a branch percentage — least of all
+                // 100% — when no branch data was collected.
+                branchCoverage: provenance.branchDataCollected ? round(report.summary.branch) : null,
+                functionCoverage: provenance.functionDataCollected ? round(report.summary.function) : null,
+                statementCoverage: round(report.summary.statement),
                 totalFiles: report.summary.files,
                 coverageByFile: coverageData.files.map(f => ({
                     file: f.path,
-                    lineCoverage: f.lines.total > 0 ? Math.round((f.lines.covered / f.lines.total) * 1000) / 10 : 0,
-                    branchCoverage: f.branches.total > 0 ? Math.round((f.branches.covered / f.branches.total) * 1000) / 10 : 0,
-                    functionCoverage: f.functions.total > 0 ? Math.round((f.functions.covered / f.functions.total) * 1000) / 10 : 0,
+                    lineCoverage: pct(f.lines.covered, f.lines.total) ?? 0,
+                    branchCoverage: provenance.branchDataCollected ? pct(f.branches.covered, f.branches.total) : null,
+                    functionCoverage: provenance.functionDataCollected ? pct(f.functions.covered, f.functions.total) : null,
                 })),
                 gaps,
                 meetsThreshold: report.meetsThreshold,
                 delta: report.delta,
-                recommendations: report.recommendations,
+                recommendations,
                 algorithm: 'sublinear-O(log n)',
+                // #569 work item 5: the caller must be able to tell measurement from guess.
+                estimated: provenance.estimated,
+                measured: !provenance.estimated,
+                coverageMethod: provenance.method,
+                branchDataCollected: provenance.branchDataCollected,
+                functionDataCollected: provenance.functionDataCollected,
+                ...(warnings.length > 0 ? { warning: warnings.join(' ') } : {}),
             });
         }
         catch (error) {

package/dist/coordination/handlers/handler-types.d.ts

@@ -23,7 +23,12 @@ export interface TaskHandlerContext {
     /** Lazy service getters */
     getCoverageAnalyzer(): CoverageAnalyzerService;
     getSecurityScanner(): SecurityScannerService;
-    getTestGenerator(): TestGeneratorService;
+    /**
+     * Async (#567): resolves the ADR-043 LLM router before constructing the
+     * service. Pass `useLLM: false` to get a router-free generator (deterministic
+     * templates only) for callers that set `aiEnhancement: false`.
+     */
+    getTestGenerator(useLLM?: boolean): Promise<TestGeneratorService>;
     getKnowledgeGraph(): KnowledgeGraphService;
     getQualityAnalyzer(): QualityAnalyzerService;
     /** Executor configuration */

package/dist/coordination/handlers/handler-utils.js

@@ -295,10 +295,17 @@ export function parseCoberturaXml(content) {
                     functionsCovered++;
             }
         }
-        // If no lines parsed, estimate from rates
+        // No per-line <line> entries — fall back to the file's reported line-rate.
+        //
+        // #569/ADR-126: the previous form (`linesTotal = 1; linesCovered = rate >= 0.5
+        // ? 1 : 0`) turned a measured 47% into 0% and a measured 51% into 100%, and
+        // the result was still labelled `estimated: false`. The line-rate itself IS
+        // instrumented data; only the per-line detail is missing. Scale it to a
+        // 100-line denominator so the measured rate survives at 1% granularity
+        // instead of being rounded to an extreme.
         if (linesTotal === 0 && !isNaN(lineRate)) {
-            linesTotal = 1;
-            linesCovered = lineRate >= 0.5 ? 1 : 0;
+            linesTotal = 100;
+            linesCovered = Math.round(Math.max(0, Math.min(1, lineRate)) * 100);
         }
         files.push({
             path: filename,

package/dist/coordination/handlers/security-handlers.js

@@ -242,8 +242,23 @@ export function registerSecurityHandlers(ctx) {
                 jstsFilesScanned: jstsFiles.length,
                 otherFilesScanned: otherFiles.length,
                 coverage: sastResult?.coverage,
-                ...(otherFiles.length > 0 && jstsFiles.length === 0 ? {
-                    note: 'Non-JS/TS files were scanned with cross-language pattern matching. For deeper analysis, use language-specific security tools.',
+                // #569 (companion note): "zero findings" from a scan that ran no
+                // language-specific analysis is not evidence of absence, and combined
+                // with a coverage number it reads as "this code is clean and covered".
+                // Say plainly which depth of analysis actually ran.
+                deepAnalysisPerformed: jstsFiles.length > 0,
+                analysisDepth: jstsFiles.length > 0
+                    ? (otherFiles.length > 0 ? 'full-sast-on-js-ts; pattern-matching-on-other-languages' : 'full-sast')
+                    : 'pattern-matching-only',
+                ...(jstsFiles.length === 0 ? {
+                    note: otherFiles.length > 0
+                        ? 'NO LANGUAGE-SPECIFIC ANALYSIS RAN. Only cross-language pattern matching ' +
+                            '(secrets, CORS, eval/exec) was applied to these files — full SAST is ' +
+                            'implemented for JS/TS only. A zero-finding result here means "nothing ' +
+                            'matched a generic pattern", NOT "no vulnerabilities". Run a ' +
+                            'language-specific tool (cargo audit / clippy, bandit, gosec, semgrep) ' +
+                            'for real coverage of this codebase.'
+                        : 'No source files were analyzed.',
                 } : {}),
             });
         }

package/dist/coordination/handlers/test-execution-handlers.js

@@ -56,7 +56,8 @@ export function registerTestExecutionHandlers(ctx) {
     ctx.registerHandler('generate-tests', async (task) => {
         const payload = task.payload;
         try {
-            const generator = ctx.getTestGenerator();
+            // #567: honor the tool's documented `aiEnhancement` knob (default true).
+            const generator = await ctx.getTestGenerator(payload.aiEnhancement !== false);
             // Determine source files to analyze
             let sourceFiles = [];
             let tempPath;
@@ -134,11 +135,25 @@ export function registerTestExecutionHandlers(ctx) {
                     testCode: rewritten.testCode,
                 };
             });
+            // #567: surface whether the LLM branch actually ran. Previously the only
+            // clue that generation had silently fallen back to template scaffolding
+            // was a suspiciously constant coverage estimate and a sub-20ms duration.
+            const llmEnhanced = generatedTests.tests.some(t => t.llmEnhanced === true);
             return ok({
                 testsGenerated: tests.length,
                 coverageEstimate: generatedTests.coverageEstimate,
                 tests,
                 patternsUsed: generatedTests.patternsUsed,
+                llmEnhanced,
+                ...(llmEnhanced ? {} : {
+                    generationMode: 'deterministic-template',
+                    note: payload.aiEnhancement === false
+                        ? 'LLM enhancement was disabled by the caller (aiEnhancement: false).'
+                        : 'No LLM provider was available, so tests are deterministic template ' +
+                            'scaffolding rather than source-aware. Configure a provider in ' +
+                            '.agentic-qe/llm-config.json (or set a provider API key) and ensure ' +
+                            'AQE_LLM_ROUTER_DISABLED is not set.',
+                }),
             });
         }
         catch (error) {