agentic-qe 3.14.7 → 3.15.0

package/dist/domains/quality-assessment/quality-evidence.js

@@ -91,9 +91,14 @@ export async function loadQualityEvidence(memory, options = {}) {
     }
     return values;
 }
-export function evaluateQualityEvidence(values) {
+export function evaluateQualityEvidence(values, thresholds = {}) {
+    if (thresholds.coverage !== undefined && (!Number.isFinite(thresholds.coverage) || thresholds.coverage < 0 || thresholds.coverage > 100)) {
+        throw new Error('Coverage quality threshold must be a finite number between 0 and 100.');
+    }
     const checks = QUALITY_METRICS.map((name) => {
-        const threshold = DEFAULT_QUALITY_THRESHOLDS[name];
+        const threshold = name === 'coverage' && thresholds.coverage !== undefined
+            ? { ...DEFAULT_QUALITY_THRESHOLDS[name], value: thresholds.coverage }
+            : DEFAULT_QUALITY_THRESHOLDS[name];
         const value = values[name];
         return {
             name,

package/dist/domains/test-execution/services/retry-handler.js

@@ -11,6 +11,7 @@ import { TEST_EXECUTION_CONSTANTS, RETRY_CONSTANTS } from '../../constants.js';
 import { toError } from '../../../shared/error-utils.js';
 import { safeJsonParse } from '../../../shared/safe-json.js';
 import { secureRandom } from '../../../shared/utils/crypto-random.js';
+import { terminateTestRunner, trackTestRunnerExit } from '../../../shared/test-runner-process.js';
 const DEFAULT_RETRY_CONFIG = {
     simulateForTesting: false,
     simulatedRetrySuccessRate: RETRY_CONSTANTS.DEFAULT_FLAKY_PASS_RATE,
@@ -335,6 +336,9 @@ export class RetryHandlerService {
      */
     buildTestCommand(runner, testFile, testName) {
         const npx = this.config.npxPath;
+        // These runners interpret name filters as regular expressions, but a
+        // FailedTest contains the literal assertion name from the previous run.
+        const namePattern = testName?.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') ?? '';
         switch (runner) {
             case 'vitest':
                 // vitest run --reporter=json testFile -t "testName"
@@ -343,7 +347,7 @@ export class RetryHandlerService {
                     const report = createVitestJsonReport();
                     const vitestArgs = ['vitest', 'run', '--reporter=json', testFile];
                     if (testName && testName !== testFile) {
-                        vitestArgs.push('-t', testName);
+                        vitestArgs.push('-t', namePattern);
                     }
                     vitestArgs.push(...report.args);
                     return { command: npx, args: vitestArgs, report };
@@ -353,7 +357,7 @@ export class RetryHandlerService {
                 {
                     const jestArgs = ['jest', '--json', '--testPathPattern', testFile];
                     if (testName && testName !== testFile) {
-                        jestArgs.push('-t', testName);
+                        jestArgs.push('-t', namePattern);
                     }
                     return { command: npx, args: jestArgs };
                 }
@@ -362,7 +366,7 @@ export class RetryHandlerService {
                 {
                     const mochaArgs = ['mocha', '--reporter', 'json', testFile];
                     if (testName && testName !== testFile) {
-                        mochaArgs.push('--grep', testName);
+                        mochaArgs.push('--grep', namePattern);
                     }
                     return { command: npx, args: mochaArgs };
                 }
@@ -379,6 +383,8 @@ export class RetryHandlerService {
             let stdout = '';
             let stderr = '';
             let timedOut = false;
+            let termination;
+            const timeoutError = new Error(`Test execution timed out after ${timeout}ms for command: ${command} ${args.join(' ')}`);
             const settle = (fn, value) => {
                 report?.cleanup();
                 fn(value);
@@ -387,13 +393,20 @@ export class RetryHandlerService {
             // Arguments are passed as array to avoid shell interpretation
             const proc = spawn(command, args, {
                 cwd,
+                detached: process.platform !== 'win32',
                 env: { ...process.env, FORCE_COLOR: '0', CI: 'true' },
             });
+            const untrackExit = trackTestRunnerExit(proc);
             const timeoutHandle = setTimeout(() => {
                 timedOut = true;
-                proc.kill('SIGTERM');
-                // Give it a moment to terminate gracefully, then force kill
-                setTimeout(() => proc.kill('SIGKILL'), 1000);
+                termination = terminateTestRunner(proc);
+                termination.catch(error => {
+                    logger.warn(`Retry runner cleanup failed: ${toError(error).message}`);
+                    // A denied cleanup can leave the root alive: preserve the timeout
+                    // outcome immediately, keeping artifacts because settlement is unknown.
+                    reject(timeoutError);
+                });
+                termination.then(untrackExit, () => undefined);
             }, timeout);
             proc.stdout?.on('data', (data) => {
                 stdout += data.toString();
@@ -404,9 +417,10 @@ export class RetryHandlerService {
             proc.on('close', (code) => {
                 clearTimeout(timeoutHandle);
                 if (timedOut) {
-                    settle(reject, new Error(`Test execution timed out after ${timeout}ms for command: ${command} ${args.join(' ')}`));
+                    termination?.then(() => settle(reject, timeoutError), () => reject(timeoutError));
                     return;
                 }
+                untrackExit();
                 // Parse result based on exit code and output (Vitest writes the JSON
                 // report to the --outputFile; read it back rather than trusting stdout).
                 try {
@@ -422,6 +436,7 @@ export class RetryHandlerService {
             });
             proc.on('error', (err) => {
                 clearTimeout(timeoutHandle);
+                untrackExit();
                 report?.cleanup();
                 reject(new Error(`Failed to spawn test process: ${err.message}. ` +
                     `Command: ${command} ${args.join(' ')}. ` +
@@ -433,20 +448,23 @@ export class RetryHandlerService {
      * Parse test runner output to determine pass/fail status
      */
     parseTestResult(exitCode, stdout, stderr) {
-        // A passing process and a passing report must agree when JSON is available.
+        // A retry can clear a prior failure only when the report proves a test
+        // actually passed. Empty/fully skipped selections can also exit zero.
         try {
             // Vitest JSON output
             const vitestMatch = stdout.match(/\{[\s\S]*"testResults"[\s\S]*\}/);
             if (vitestMatch) {
                 const result = safeJsonParse(vitestMatch[0]);
                 if (exitCode === 0 && result.success !== false
-                    && result.numFailedTests === 0 && result.numFailedTestSuites === 0) {
+                    && result.numFailedTests === 0 && result.numFailedTestSuites === 0
+                    && Number.isFinite(result.numPassedTests) && result.numPassedTests > 0) {
                     return { passed: true };
                 }
                 const failedTest = result.testResults?.[0]?.assertionResults?.find((r) => r.status === 'failed');
                 return {
                     passed: false,
-                    error: failedTest?.failureMessages?.join('\n') || stderr || `Test failed with exit code ${exitCode}`,
+                    error: failedTest?.failureMessages?.join('\n') || stderr
+                        || (exitCode === 0 ? 'Retry runner did not report any passing tests.' : `Test failed with exit code ${exitCode}`),
                 };
             }
             // Jest JSON output
@@ -454,34 +472,38 @@ export class RetryHandlerService {
             if (jestMatch) {
                 const result = safeJsonParse(jestMatch[0]);
                 if (exitCode === 0 && result.success !== false
-                    && result.numFailedTests === 0 && result.numFailedTestSuites === 0) {
+                    && result.numFailedTests === 0 && result.numFailedTestSuites === 0
+                    && Number.isFinite(result.numPassedTests) && result.numPassedTests > 0) {
                     return { passed: true };
                 }
                 const failedTest = result.testResults?.[0]?.assertionResults?.find((r) => r.status === 'failed');
                 return {
                     passed: false,
-                    error: failedTest?.failureMessages?.join('\n') || stderr || `Test failed with exit code ${exitCode}`,
+                    error: failedTest?.failureMessages?.join('\n') || stderr
+                        || (exitCode === 0 ? 'Retry runner did not report any passing tests.' : `Test failed with exit code ${exitCode}`),
                 };
             }
             // Mocha JSON output
             const mochaMatch = stdout.match(/\{[\s\S]*"stats"[\s\S]*"failures"[\s\S]*\}/);
             if (mochaMatch) {
                 const result = safeJsonParse(mochaMatch[0]);
-                if (exitCode === 0 && result.stats?.failures === 0) {
+                if (exitCode === 0 && result.stats?.failures === 0
+                    && Number.isFinite(result.stats?.passes) && result.stats.passes > 0) {
                     return { passed: true };
                 }
                 const failure = result.failures?.[0];
                 return {
                     passed: false,
-                    error: failure?.err?.message || stderr || `Test failed with exit code ${exitCode}`,
+                    error: failure?.err?.message || stderr
+                        || (exitCode === 0 ? 'Retry runner did not report any passing tests.' : `Test failed with exit code ${exitCode}`),
                 };
             }
         }
         catch {
-            // JSON parsing failed, fall back to simple exit code check
+            // Unreadable output cannot prove that the retry executed a passing test.
         }
         if (exitCode === 0)
-            return { passed: true };
+            return { passed: false, error: 'Retry runner did not report any passing tests.' };
         // Non-zero exit code means failure
         const errorOutput = stderr || stdout || `Test failed with exit code ${exitCode}`;
         return {

package/dist/domains/test-execution/services/test-executor.d.ts

@@ -170,9 +170,9 @@ export declare class TestExecutorService implements ITestExecutionService {
      */
     private extractCoverageFromJson;
     /**
-     * Read coverage from disk files written by vitest/jest.
-     * Both runners write coverage-summary.json to a coverage/ directory when
-     * --coverage is passed, rather than embedding it in stdout JSON.
+     * Read coverage from the private output directory of a completed runner.
+     * Setting the directory does not enable coverage; project configuration
+     * still decides whether coverage is collected.
      */
     private readCoverageFromDisk;
     private storeResults;

package/dist/domains/test-execution/services/test-executor.js

@@ -4,7 +4,8 @@
  */
 import { LoggerFactory } from '../../../logging/index.js';
 import { spawn } from 'node:child_process';
-import { existsSync, readFileSync } from 'node:fs';
+import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { v4 as uuidv4 } from 'uuid';
 import { ok, err } from '../../../shared/types/index.js';
@@ -16,6 +17,7 @@ import { writeQualityEvidence } from '../../quality-assessment/quality-evidence.
 import { getTestRunnerExecutionError, TestRunnerExecutionError } from '../../../shared/test-runner-verdict.js';
 import { createVitestJsonReport } from '../../../shared/vitest-json-report.js';
 import { getClaudeModelForTier } from '../../../shared/llm/model-registry.js';
+import { terminateTestRunner, trackTestRunnerExit } from '../../../shared/test-runner-process.js';
 const DEFAULT_CONFIG = {
     simulateForTesting: false,
     simulatedTestsPerFile: 5,
@@ -368,19 +370,29 @@ Provide:
             return err(new Error(`Test file not found: ${missingFile}`));
         }
         // Build command based on framework
-        const { command, args, report } = this.buildTestCommand(files, framework);
+        const { command, args, report, coverageDirectory } = this.buildTestCommand(files, framework);
         const fileLabel = files.join(', ');
         return new Promise((resolve) => {
             let stdout = '';
             let stderr = '';
             let killed = false;
+            let termination;
             let settled = false;
+            const cleanupArtifacts = () => {
+                report?.cleanup();
+                if (coverageDirectory) {
+                    try {
+                        rmSync(coverageDirectory, { recursive: true, force: true });
+                    }
+                    catch { /* Best effort. */ }
+                }
+            };
             const finish = (result, cleanup = true) => {
                 if (settled)
                     return;
                 settled = true;
                 if (cleanup)
-                    report?.cleanup();
+                    cleanupArtifacts();
                 resolve(result);
             };
             // Spawn the test runner process
@@ -390,6 +402,7 @@ Provide:
             try {
                 proc = spawn(command, args, {
                     cwd: process.cwd(),
+                    detached: process.platform !== 'win32',
                     env: {
                         ...process.env,
                         FORCE_COLOR: '0', // Disable color codes for easier parsing
@@ -401,10 +414,15 @@ Provide:
                 finish(err(new Error(`Failed to spawn test runner: ${toErrorMessage(error)}. Is '${command}' installed?`)));
                 return;
             }
+            const untrackExit = trackTestRunnerExit(proc);
             // Set timeout
             const timeoutId = setTimeout(() => {
                 killed = true;
-                proc.kill('SIGTERM');
+                termination = terminateTestRunner(proc);
+                // Keep the timeout outcome immediate; cleanup waits for the owned
+                // group, rather than treating the root's close as descendant exit.
+                termination.catch(error => logger.warn(`Test runner cleanup failed: ${toErrorMessage(error)}`));
+                termination.then(untrackExit, () => undefined);
                 finish(err(new Error(`Test execution timed out after ${timeout}ms for files: ${fileLabel}`)), false);
             }, timeout);
             proc.stdout?.on('data', (data) => {
@@ -416,9 +434,12 @@ Provide:
             proc.on('close', (code) => {
                 clearTimeout(timeoutId);
                 if (killed) {
-                    report?.cleanup();
-                    return; // Timeout result was already returned; child has now closed.
+                    // Coverage and JSON reports belong to this run; clean both only
+                    // after the entire owned process group has stopped.
+                    termination?.then(() => cleanupArtifacts(), () => undefined);
+                    return; // The timeout result has already been returned.
                 }
+                untrackExit();
                 // Parse results based on framework. Vitest 5 writes the JSON report to
                 // a file instead of stdout, so read it back through the report handle.
                 let reportText;
@@ -434,10 +455,10 @@ Provide:
                     return;
                 }
                 const parseResult = this.parseTestOutput(reportText, stderr, fileLabel, framework, code);
-                // If no coverage in stdout JSON, try reading from disk
-                // (vitest/jest write coverage to coverage/coverage-summary.json)
-                if (parseResult.success && !parseResult.value.coverage && ['vitest', 'jest'].includes(framework.toLowerCase())) {
-                    const diskCoverage = this.readCoverageFromDisk();
+                // Read only this invocation's coverage artifacts. A previous or
+                // concurrent run's project coverage directory is not evidence.
+                if (parseResult.success && !parseResult.value.coverage && coverageDirectory) {
+                    const diskCoverage = this.readCoverageFromDisk(coverageDirectory);
                     if (diskCoverage) {
                         parseResult.value.coverage = diskCoverage.summary;
                         if (diskCoverage.perFile.length > 0) {
@@ -451,6 +472,7 @@ Provide:
                 clearTimeout(timeoutId);
                 if (killed)
                     return;
+                untrackExit();
                 finish(err(new Error(`Failed to spawn test runner: ${error.message}. Is '${command}' installed?`)));
             });
         });
@@ -460,22 +482,35 @@ Provide:
      */
     buildTestCommand(fileOrFiles, framework) {
         const files = Array.isArray(fileOrFiles) ? fileOrFiles : [fileOrFiles];
+        const createCoverageDirectory = () => mkdtempSync(join(tmpdir(), 'aqe-test-coverage-'));
         switch (framework.toLowerCase()) {
             case 'vitest': {
                 // --outputFile makes Vitest 4 and 5 both write the JSON report to a
                 // file we own; Vitest 5 no longer prints it to stdout.
-                const report = createVitestJsonReport();
+                const coverageDirectory = createCoverageDirectory();
+                let report;
+                try {
+                    report = createVitestJsonReport();
+                }
+                catch (error) {
+                    rmSync(coverageDirectory, { recursive: true, force: true });
+                    throw error;
+                }
                 return {
                     command: 'npx',
-                    args: ['vitest', 'run', ...files, '--reporter=json', '--no-color', ...report.args],
+                    args: ['vitest', 'run', ...files, '--reporter=json', '--no-color', ...report.args, `--coverage.reportsDirectory=${coverageDirectory}`],
                     report,
+                    coverageDirectory,
                 };
             }
-            case 'jest':
+            case 'jest': {
+                const coverageDirectory = createCoverageDirectory();
                 return {
                     command: 'npx',
-                    args: ['jest', ...files, '--json', '--no-colors', '--testLocationInResults'],
+                    args: ['jest', ...files, '--json', '--no-colors', '--testLocationInResults', `--coverageDirectory=${coverageDirectory}`],
+                    coverageDirectory,
                 };
+            }
             case 'mocha':
                 // Note: mocha has no built-in coverage — requires external nyc/c8 wrapper.
                 // Coverage data will be unavailable for mocha-based test runs.
@@ -491,11 +526,20 @@ Provide:
                 };
             default: {
                 // Default to vitest
-                const report = createVitestJsonReport();
+                const coverageDirectory = createCoverageDirectory();
+                let report;
+                try {
+                    report = createVitestJsonReport();
+                }
+                catch (error) {
+                    rmSync(coverageDirectory, { recursive: true, force: true });
+                    throw error;
+                }
                 return {
                     command: 'npx',
-                    args: ['vitest', 'run', ...files, '--reporter=json', '--no-color', ...report.args],
+                    args: ['vitest', 'run', ...files, '--reporter=json', '--no-color', ...report.args, `--coverage.reportsDirectory=${coverageDirectory}`],
                     report,
+                    coverageDirectory,
                 };
             }
         }
@@ -1003,16 +1047,16 @@ Provide:
         return { perFile };
     }
     /**
-     * Read coverage from disk files written by vitest/jest.
-     * Both runners write coverage-summary.json to a coverage/ directory when
-     * --coverage is passed, rather than embedding it in stdout JSON.
+     * Read coverage from the private output directory of a completed runner.
+     * Setting the directory does not enable coverage; project configuration
+     * still decides whether coverage is collected.
      */
-    readCoverageFromDisk() {
+    readCoverageFromDisk(coverageDirectory) {
         try {
             // Check common coverage output paths
             const candidates = [
-                join(process.cwd(), 'coverage', 'coverage-summary.json'),
-                join(process.cwd(), 'coverage', 'coverage-final.json'),
+                join(coverageDirectory, 'coverage-summary.json'),
+                join(coverageDirectory, 'coverage-final.json'),
             ];
             for (const filePath of candidates) {
                 if (!existsSync(filePath))

package/dist/governance/ab-benchmarking.d.ts

@@ -168,7 +168,8 @@ export interface WinnerResult {
     benchmarkId: string;
     winnerId: string;
     winnerName: string;
-    confidence: number;
+    /** Ranking heuristic, not a probability or statistical confidence level. */
+    heuristicScore: number;
     combinedScore: number;
     significantMetrics: string[];
     recommendation: string;
@@ -290,10 +291,13 @@ export declare class ABBenchmarkingFramework {
      * Calculate statistical significance for a benchmark
      */
     calculateStatisticalSignificance(benchmarkId: string): SignificanceResult;
+    private calculateSignificance;
     /**
      * Get the winner of a benchmark
      */
     getWinner(benchmarkId: string): WinnerResult | null;
+    /** Require evidence for this candidate over its ranked runner-up. */
+    private isSignificantlyBetterThanRunnerUp;
     /**
      * Compare two specific variants
      */

package/dist/governance/ab-benchmarking.js

@@ -523,11 +523,17 @@ export class ABBenchmarkingFramework {
      * Calculate statistical significance for a benchmark
      */
     calculateStatisticalSignificance(benchmarkId) {
+        return this.calculateSignificance(benchmarkId);
+    }
+    calculateSignificance(benchmarkId, variantIds) {
         const benchmark = this.benchmarks.get(benchmarkId);
         if (!benchmark) {
             throw new Error(`Benchmark ${benchmarkId} not found`);
         }
-        const { config, variantMetrics } = benchmark;
+        const { config } = benchmark;
+        const variantMetrics = variantIds
+            ? new Map(variantIds.map(id => [id, benchmark.variantMetrics.get(id)]))
+            : benchmark.variantMetrics;
         const alpha = 1 - config.confidenceLevel;
         // Bonferroni correction for multiple comparisons
         const numComparisons = config.metrics.length;
@@ -540,10 +546,10 @@ export class ABBenchmarkingFramework {
         const chiSquareResult = chiSquareTest(variants);
         // Per-metric statistics (pairwise comparisons between first two variants)
         const metricStatistics = [];
-        const variantIds = Array.from(variantMetrics.keys());
-        if (variantIds.length >= 2) {
-            const v1 = variantMetrics.get(variantIds[0]);
-            const v2 = variantMetrics.get(variantIds[1]);
+        const comparedIds = Array.from(variantMetrics.keys());
+        if (comparedIds.length >= 2) {
+            const v1 = variantMetrics.get(comparedIds[0]);
+            const v2 = variantMetrics.get(comparedIds[1]);
             for (const metricConfig of config.metrics) {
                 const values1 = (v1.metrics.get(metricConfig.name) || []).map(d => d.value);
                 const values2 = (v2.metrics.get(metricConfig.name) || []).map(d => d.value);
@@ -618,10 +624,10 @@ export class ABBenchmarkingFramework {
         // Sort by combined score (descending)
         scores.sort((a, b) => b.score - a.score);
         const winner = scores[0];
-        // Calculate confidence based on significance and score gap
+        // Calculate a ranking heuristic based on significance and score gap
         const significance = this.calculateStatisticalSignificance(benchmarkId);
         const scoreGap = scores.length > 1 ? winner.score - scores[1].score : winner.score;
-        const confidence = significance.isSignificant
+        const heuristicScore = significance.isSignificant
             ? Math.min(0.99, 0.5 + scoreGap * 0.5 + (significance.powerAnalysis.currentPower * 0.25))
             : Math.min(0.5, scoreGap * 0.5);
         const significantMetrics = significance.metricStatistics
@@ -632,12 +638,38 @@ export class ABBenchmarkingFramework {
             benchmarkId,
             winnerId: winner.variantId,
             winnerName: variantConfig?.name || winner.variantId,
-            confidence,
+            heuristicScore,
             combinedScore: winner.score,
             significantMetrics,
-            recommendation: this.generateWinnerRecommendation(winner, confidence, significantMetrics),
+            recommendation: this.generateWinnerRecommendation(winner, heuristicScore, significantMetrics),
         };
     }
+    /** Require evidence for this candidate over its ranked runner-up. */
+    isSignificantlyBetterThanRunnerUp(benchmarkId, winnerId) {
+        const benchmark = this.benchmarks.get(benchmarkId);
+        const scores = this.calculateCombinedScores(benchmark).sort((a, b) => b.score - a.score);
+        const [winner, runnerUp] = scores;
+        if (!runnerUp || winner.variantId !== winnerId || winner.score <= runnerUp.score) {
+            return false;
+        }
+        const significance = this.calculateSignificance(benchmarkId, [winnerId, runnerUp.variantId]);
+        const winnerMetrics = benchmark.variantMetrics.get(winnerId);
+        const runnerUpMetrics = benchmark.variantMetrics.get(runnerUp.variantId);
+        const successMetrics = benchmark.config.metrics.filter(metric => metric.type === 'success_rate');
+        const successContributesToScore = successMetrics.length === 0 ||
+            successMetrics.some(metric => metric.weight > 0);
+        if (successContributesToScore && significance.chiSquareTest?.isSignificant &&
+            this.getSuccessRate(winnerMetrics) > this.getSuccessRate(runnerUpMetrics)) {
+            return true;
+        }
+        return significance.metricStatistics.some(statistic => {
+            const metric = benchmark.config.metrics.find(config => config.name === statistic.metric);
+            return statistic.isSignificant && metric.weight > 0 &&
+                (metric.higherIsBetter
+                    ? statistic.variantA.mean > statistic.variantB.mean
+                    : statistic.variantA.mean < statistic.variantB.mean);
+        });
+    }
     /**
      * Compare two specific variants
      */
@@ -693,9 +725,6 @@ export class ABBenchmarkingFramework {
             scoreA += normalizedA * metricConfig.weight;
             scoreB += normalizedB * metricConfig.weight;
         }
-        // Add success rate comparison
-        const successRateA = this.getSuccessRate(metricsA);
-        const successRateB = this.getSuccessRate(metricsB);
         const overallWinner = scoreA > scoreB ? variantA : scoreB > scoreA ? variantB : null;
         const confidence = Math.abs(scoreA - scoreB) / Math.max(scoreA, scoreB, 0.001);
         return {
@@ -760,16 +789,19 @@ export class ABBenchmarkingFramework {
             if (winner.significantMetrics.length > 0) {
                 reasoning.push(`Significant improvements in: ${winner.significantMetrics.join(', ')}`);
             }
-            if (winner.confidence >= 0.95) {
+            if (!this.isSignificantlyBetterThanRunnerUp(benchmarkId, winner.winnerId)) {
+                caveats.push('Leading variant is not significantly better than the runner-up');
+            }
+            else if (winner.heuristicScore >= 0.95) {
                 readyToApply = true;
-                reasoning.push('High confidence - ready to apply');
+                reasoning.push('High heuristic score with runner-up significance - ready to apply');
             }
-            else if (winner.confidence >= 0.8 && significance?.isSignificant) {
+            else if (winner.heuristicScore >= 0.8) {
                 readyToApply = true;
-                reasoning.push('Good confidence with statistical significance - ready to apply');
+                reasoning.push('Good heuristic score with runner-up significance - ready to apply');
             }
             else {
-                caveats.push(`Confidence (${(winner.confidence * 100).toFixed(1)}%) below recommended threshold`);
+                caveats.push(`Heuristic score (${winner.heuristicScore.toFixed(3)}) below recommended threshold`);
             }
         }
         else {
@@ -778,7 +810,7 @@ export class ABBenchmarkingFramework {
         return {
             benchmarkId,
             suggestedWinnerId,
-            confidence: winner?.confidence ?? 0,
+            confidence: winner?.heuristicScore ?? 0,
             reasoning,
             caveats,
             readyToApply,
@@ -798,12 +830,15 @@ export class ABBenchmarkingFramework {
         if (!winner) {
             throw new Error(`No winner determined for benchmark ${benchmarkId}`);
         }
+        if (!this.isSignificantlyBetterThanRunnerUp(benchmarkId, winner.winnerId)) {
+            throw new Error(`Winner is not significantly better than the runner-up for benchmark ${benchmarkId}`);
+        }
         // Mark benchmark as completed
         benchmark.status = 'completed';
         benchmark.endTime = Date.now();
         benchmark.winnerId = winner.winnerId;
         // Promote winner in Evolution Pipeline
-        evolutionPipelineIntegration.promoteRule(winner.winnerId, `Won A/B benchmark ${benchmarkId} with ${(winner.confidence * 100).toFixed(1)}% confidence`);
+        evolutionPipelineIntegration.promoteRule(winner.winnerId, `Won A/B benchmark ${benchmarkId} with heuristic score ${winner.heuristicScore.toFixed(3)} and runner-up significance`);
         // Complete the variant test
         evolutionPipelineIntegration.completeVariantTest(benchmarkId);
         this.logEvent('winner_applied', `Applied winner ${winner.winnerId} for benchmark ${benchmarkId}`);
@@ -979,25 +1014,25 @@ export class ABBenchmarkingFramework {
     /**
      * Generate winner recommendation text
      */
-    generateWinnerRecommendation(winner, confidence, significantMetrics) {
-        if (confidence >= 0.95) {
+    generateWinnerRecommendation(winner, heuristicScore, significantMetrics) {
+        if (heuristicScore >= 0.95) {
             return `Strongly recommend applying ${winner.variantId}. ` +
-                `High confidence (${(confidence * 100).toFixed(1)}%) with significant improvements ` +
+                `High heuristic score (${heuristicScore.toFixed(3)}) with significant improvements ` +
                 `in: ${significantMetrics.length > 0 ? significantMetrics.join(', ') : 'overall performance'}.`;
         }
-        else if (confidence >= 0.8) {
+        else if (heuristicScore >= 0.8) {
             return `Recommend applying ${winner.variantId}. ` +
-                `Good confidence (${(confidence * 100).toFixed(1)}%). ` +
-                `Consider collecting more samples for higher confidence.`;
+                `Good heuristic score (${heuristicScore.toFixed(3)}). ` +
+                `Consider collecting more samples for a higher heuristic score.`;
         }
-        else if (confidence >= 0.6) {
-            return `${winner.variantId} shows promise with moderate confidence ` +
-                `(${(confidence * 100).toFixed(1)}%). ` +
+        else if (heuristicScore >= 0.6) {
+            return `${winner.variantId} shows promise with moderate heuristic score ` +
+                `(${heuristicScore.toFixed(3)}). ` +
                 `Recommend collecting more samples before applying.`;
         }
         else {
             return `Results are inconclusive. ${winner.variantId} is currently leading but ` +
-                `confidence (${(confidence * 100).toFixed(1)}%) is low. Continue collecting data.`;
+                `heuristic score (${heuristicScore.toFixed(3)}) is low. Continue collecting data.`;
         }
     }
     /**

package/dist/init/fleet-integration.js

@@ -122,12 +122,13 @@ export class FleetInitEnhancer {
                 enableVectorEmbeddings: true,
             });
             // Find all source files
-            const glob = await import('fast-glob');
-            const files = await glob.default([
+            const { glob } = await import('tinyglobby');
+            const files = await glob([
                 '**/*.ts', '**/*.tsx', '**/*.js', '**/*.jsx', '**/*.py'
             ], {
                 cwd: this.projectRoot,
                 ignore: ['node_modules/**', 'dist/**', 'coverage/**', '.agentic-qe/**'],
+                expandDirectories: false,
             });
             console.log(chalk.gray(`  Indexing ${files.length} files...`));
             // Index files

package/dist/init/init-wizard-steps.js

@@ -200,12 +200,13 @@ export async function runCodeIntelligenceScan(projectPath) {
             namespace: 'code-intelligence:kg',
             enableVectorEmbeddings: true,
         });
-        const glob = await import('fast-glob');
-        const files = await glob.default([
+        const { glob } = await import('tinyglobby');
+        const files = await glob([
             '**/*.ts', '**/*.tsx', '**/*.js', '**/*.jsx', '**/*.py'
         ], {
             cwd: projectPath,
             ignore: ['node_modules/**', 'dist/**', 'coverage/**', '.agentic-qe/**'],
+            expandDirectories: false,
         });
         const result = await kgService.index({
             paths: files.map(f => join(projectPath, f)),