agentic-qe 3.14.8 → 3.15.1

package/dist/cli/utils/workflow-parser.js

@@ -694,33 +694,50 @@ function detectCircularDependencies(stages) {
  * Validate cron expression (basic validation)
  */
 export function isValidCronExpression(expression) {
-    // Check for aliases
-    if (Object.keys(CRON_PATTERNS).includes(expression)) {
-        return true;
-    }
-    // Basic cron validation: 5 fields separated by spaces
-    const parts = expression.trim().split(/\s+/);
-    if (parts.length !== 5) {
-        return false;
-    }
-    // Validate each field
-    const fieldPatterns = [
-        /^(\*|[0-5]?\d)(-[0-5]?\d)?(\/\d+)?$/, // minute (0-59)
-        /^(\*|1?\d|2[0-3])(-\d+)?(\/\d+)?$/, // hour (0-23)
-        /^(\*|[1-9]|[12]\d|3[01])(-\d+)?(\/\d+)?$/, // day of month (1-31)
-        /^(\*|[1-9]|1[0-2])(-\d+)?(\/\d+)?$/, // month (1-12)
-        /^(\*|[0-7])(-[0-7])?(\/\d+)?$/, // day of week (0-7)
+    return parseCronFields(expression) !== undefined;
+}
+/** Parse the existing numeric five-field grammar and its four named aliases. */
+function parseCronFields(expression) {
+    const normalized = Object.hasOwn(CRON_PATTERNS, expression)
+        ? CRON_PATTERNS[expression]
+        : expression;
+    const parts = normalized.trim().split(/\s+/);
+    if (parts.length !== 5)
+        return undefined;
+    const bounds = [[0, 59], [0, 23], [1, 31], [1, 12], [0, 7]];
+    const syntax = [
+        /^(\*|[0-5]?\d)(-[0-5]?\d)?(\/\d+)?$/,
+        /^(\*|1?\d|2[0-3])(-\d+)?(\/\d+)?$/,
+        /^(\*|[1-9]|[12]\d|3[01])(-\d+)?(\/\d+)?$/,
+        /^(\*|[1-9]|1[0-2])(-\d+)?(\/\d+)?$/,
+        /^(\*|[0-7])(-[0-7])?(\/\d+)?$/,
     ];
-    for (let i = 0; i < 5; i++) {
-        // Allow comma-separated values
-        const values = parts[i].split(',');
-        for (const value of values) {
-            if (!fieldPatterns[i].test(value) && value !== '*') {
-                return false;
+    const fields = [];
+    for (let index = 0; index < parts.length; index++) {
+        const [minimum, maximum] = bounds[index];
+        const values = new Set();
+        for (const item of parts[index].split(',')) {
+            if (!syntax[index].test(item))
+                return undefined;
+            const match = /^(\*|\d+)(?:-(\d+))?(?:\/(\d+))?$/.exec(item);
+            if (!match || (match[1] === '*' && match[2] !== undefined))
+                return undefined;
+            const start = match[1] === '*' ? minimum : Number(match[1]);
+            const end = match[2] !== undefined ? Number(match[2])
+                : match[1] === '*' || match[3] !== undefined ? maximum : start;
+            const step = match[3] === undefined ? 1 : Number(match[3]);
+            if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) ||
+                !Number.isSafeInteger(step) || step <= 0 ||
+                start < minimum || end > maximum || start > end)
+                return undefined;
+            for (let value = start; value <= end; value += step) {
+                values.add(index === 4 && value === 7 ? 0 : value);
             }
         }
+        fields.push(values);
     }
-    return true;
+    // Cronie records DOM_STAR/DOW_STAR from the first character, before parsing lists.
+    return { fields: fields, wildcardDay: parts[2].startsWith('*') || parts[4].startsWith('*') };
 }
 /**
  * Parse cron expression to human-readable string
@@ -773,40 +790,39 @@ export function describeCronSchedule(expression) {
  * Calculate next run time from cron expression
  */
 export function calculateNextRun(cronExpression, fromDate = new Date()) {
-    // Simplified next run calculation
-    const parts = cronExpression.trim().split(/\s+/);
-    if (parts.length !== 5) {
-        // Default to 1 day from now
-        return new Date(fromDate.getTime() + 24 * 60 * 60 * 1000);
-    }
-    const [minute, hour] = parts;
-    const nextRun = new Date(fromDate);
-    // Handle specific hour and minute
-    if (hour !== '*' && minute !== '*') {
-        nextRun.setHours(parseInt(hour, 10), parseInt(minute, 10), 0, 0);
-        if (nextRun <= fromDate) {
-            nextRun.setDate(nextRun.getDate() + 1);
-        }
-    }
-    else if (hour !== '*') {
-        // Every day at specific hour
-        nextRun.setHours(parseInt(hour, 10), 0, 0, 0);
-        if (nextRun <= fromDate) {
-            nextRun.setDate(nextRun.getDate() + 1);
-        }
-    }
-    else if (minute !== '*') {
-        // Every hour at specific minute
-        nextRun.setMinutes(parseInt(minute, 10), 0, 0);
-        if (nextRun <= fromDate) {
-            nextRun.setHours(nextRun.getHours() + 1);
+    const parsed = parseCronFields(cronExpression);
+    if (!parsed)
+        throw new Error(`Invalid cron expression: ${cronExpression}`);
+    if (!Number.isFinite(fromDate.getTime()))
+        throw new Error('Invalid schedule start date');
+    const [minutes, hours, days, months, weekdays] = parsed.fields;
+    // Preserve the existing host-local timezone. Advance elapsed minutes to
+    // skip absent DST times and inspect both occurrences of repeated times.
+    let candidate = new Date(Math.floor(fromDate.getTime() / 60000) * 60000 + 60000);
+    const deadline = new Date(fromDate);
+    deadline.setFullYear(fromDate.getFullYear() + 400);
+    // A complete Gregorian cycle contains 146097 days and repeats weekdays.
+    // Near the Date maximum a full cycle is not representable: clamp the
+    // search to that finite boundary and report the limitation on exhaustion.
+    const deadlineMs = Number.isFinite(deadline.getTime())
+        ? deadline.getTime() : 8_640_000_000_000_000;
+    while (Number.isFinite(candidate.getTime()) && candidate.getTime() <= deadlineMs) {
+        const dayMatches = days.has(candidate.getDate());
+        const weekdayMatches = weekdays.has(candidate.getDay());
+        const calendarMatches = months.has(candidate.getMonth() + 1) &&
+            (parsed.wildcardDay ? dayMatches && weekdayMatches : dayMatches || weekdayMatches);
+        if (!calendarMatches) {
+            // Skip a nonmatching local calendar day instead of scanning its minutes.
+            candidate.setHours(24, 0, 0, 0);
+            continue;
         }
-    }
-    else {
-        // Every minute
-        nextRun.setSeconds(0, 0);
-        nextRun.setMinutes(nextRun.getMinutes() + 1);
-    }
-    return nextRun;
+        if (hours.has(candidate.getHours()) && minutes.has(candidate.getMinutes()))
+            return candidate;
+        candidate = new Date(candidate.getTime() + 60000);
+    }
+    // An impossible date such as February 31 must not spin indefinitely or
+    // silently become a daily schedule. Leap-day/weekday intersections may
+    // exceed a leap-day-only horizon, so search one complete Gregorian cycle.
+    throw new Error(`No matching cron date within a 400-year Gregorian cycle and the supported Date range: ${cronExpression}`);
 }
 //# sourceMappingURL=workflow-parser.js.map

package/dist/domains/code-intelligence/services/metric-collector/test-counter.d.ts

@@ -6,7 +6,7 @@
  *
  * Detection priority by project type:
  * 1. Node.js: vitest > jest > fallback pattern matching
- * 2. Rust: cargo test --list
+ * 2. Rust: cargo test -- --list
  * 3. Python: pytest --collect-only
  * 4. Go: go test -list
  *

package/dist/domains/code-intelligence/services/metric-collector/test-counter.js

@@ -6,7 +6,7 @@
  *
  * Detection priority by project type:
  * 1. Node.js: vitest > jest > fallback pattern matching
- * 2. Rust: cargo test --list
+ * 2. Rust: cargo test -- --list
  * 3. Python: pytest --collect-only
  * 4. Go: go test -list
  *
@@ -262,11 +262,11 @@ function countJestTests(projectPath, config) {
 }
 /**
  * Count tests using Cargo (Rust)
- * Uses `cargo test --list` for accurate test enumeration
+ * Uses `cargo test -- --list` for accurate test enumeration
  */
 function countCargoTests(projectPath, config) {
     try {
-        const output = execSync('cargo test --list', {
+        const output = execSync('cargo test -- --list', {
             cwd: projectPath,
             encoding: 'utf-8',
             timeout: config.timeout,
@@ -310,13 +310,14 @@ function countPytestTests(projectPath, config) {
         // Count test function lines (format: "tests/test_foo.py::test_bar")
         const testLines = output.split('\n').filter(line => line.includes('::test_') || line.includes('::Test'));
         const total = testLines.length;
-        // Classify by path patterns
-        const unit = testLines.filter(t => t.includes('unit') || (!t.includes('integration') && !t.includes('e2e'))).length;
-        const integration = testLines.filter(t => t.includes('integration') || t.includes('_integration')).length;
+        // Match fallback classification precedence so every collected test belongs
+        // to exactly one category, even when its path names multiple categories.
         const e2e = testLines.filter(t => t.includes('e2e') || t.includes('end_to_end')).length;
+        const integration = testLines.filter(t => !t.includes('e2e') && !t.includes('end_to_end') && t.includes('integration')).length;
+        const unit = total - integration - e2e;
         return {
             total,
-            unit: unit - integration - e2e, // Exclude overlap
+            unit,
             integration,
             e2e,
             source: 'pytest',
@@ -343,13 +344,14 @@ function countGoTests(projectPath, config) {
             encoding: 'utf-8',
             timeout: config.timeout,
         });
-        // Count lines starting with "Test" (Go test naming convention)
-        const testLines = output.split('\n').filter(line => line.startsWith('Test') || line.startsWith('Example') || line.startsWith('Benchmark'));
-        const total = testLines.filter(l => l.startsWith('Test')).length;
-        // Classify by test name patterns
-        const unit = testLines.filter(t => t.startsWith('Test') && !t.includes('Integration') && !t.includes('E2E')).length;
-        const integration = testLines.filter(t => t.includes('Integration')).length;
+        // Benchmarks and examples are listed too, but are not included in the
+        // test total. Apply that same selection to every category.
+        const testLines = output.split('\n').filter(line => line.startsWith('Test'));
+        const total = testLines.length;
+        // Use disjoint categories, with e2e taking precedence over integration.
         const e2e = testLines.filter(t => t.includes('E2E')).length;
+        const integration = testLines.filter(t => !t.includes('E2E') && t.includes('Integration')).length;
+        const unit = total - integration - e2e;
         return {
             total,
             unit,
@@ -474,8 +476,8 @@ function countTestsInPythonFile(filePath) {
         const content = readFileSync(filePath, 'utf-8');
         // Count def test_ functions and async def test_ functions
         const funcMatches = content.match(/\bdef\s+test_\w+\s*\(/g) || [];
-        const asyncMatches = content.match(/\basync\s+def\s+test_\w+\s*\(/g) || [];
-        return funcMatches.length + asyncMatches.length;
+        // The def pattern also matches async def; every declaration counts once.
+        return funcMatches.length;
     }
     catch {
         return 0;

package/dist/domains/test-execution/services/retry-handler.js

@@ -336,6 +336,9 @@ export class RetryHandlerService {
      */
     buildTestCommand(runner, testFile, testName) {
         const npx = this.config.npxPath;
+        // These runners interpret name filters as regular expressions, but a
+        // FailedTest contains the literal assertion name from the previous run.
+        const namePattern = testName?.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') ?? '';
         switch (runner) {
             case 'vitest':
                 // vitest run --reporter=json testFile -t "testName"
@@ -344,7 +347,7 @@ export class RetryHandlerService {
                     const report = createVitestJsonReport();
                     const vitestArgs = ['vitest', 'run', '--reporter=json', testFile];
                     if (testName && testName !== testFile) {
-                        vitestArgs.push('-t', testName);
+                        vitestArgs.push('-t', namePattern);
                     }
                     vitestArgs.push(...report.args);
                     return { command: npx, args: vitestArgs, report };
@@ -354,7 +357,7 @@ export class RetryHandlerService {
                 {
                     const jestArgs = ['jest', '--json', '--testPathPattern', testFile];
                     if (testName && testName !== testFile) {
-                        jestArgs.push('-t', testName);
+                        jestArgs.push('-t', namePattern);
                     }
                     return { command: npx, args: jestArgs };
                 }
@@ -363,7 +366,7 @@ export class RetryHandlerService {
                 {
                     const mochaArgs = ['mocha', '--reporter', 'json', testFile];
                     if (testName && testName !== testFile) {
-                        mochaArgs.push('--grep', testName);
+                        mochaArgs.push('--grep', namePattern);
                     }
                     return { command: npx, args: mochaArgs };
                 }

package/dist/governance/ab-benchmarking.d.ts

@@ -168,7 +168,8 @@ export interface WinnerResult {
     benchmarkId: string;
     winnerId: string;
     winnerName: string;
-    confidence: number;
+    /** Ranking heuristic, not a probability or statistical confidence level. */
+    heuristicScore: number;
     combinedScore: number;
     significantMetrics: string[];
     recommendation: string;
@@ -290,10 +291,13 @@ export declare class ABBenchmarkingFramework {
      * Calculate statistical significance for a benchmark
      */
     calculateStatisticalSignificance(benchmarkId: string): SignificanceResult;
+    private calculateSignificance;
     /**
      * Get the winner of a benchmark
      */
     getWinner(benchmarkId: string): WinnerResult | null;
+    /** Require evidence for this candidate over its ranked runner-up. */
+    private isSignificantlyBetterThanRunnerUp;
     /**
      * Compare two specific variants
      */

package/dist/governance/ab-benchmarking.js

@@ -523,11 +523,17 @@ export class ABBenchmarkingFramework {
      * Calculate statistical significance for a benchmark
      */
     calculateStatisticalSignificance(benchmarkId) {
+        return this.calculateSignificance(benchmarkId);
+    }
+    calculateSignificance(benchmarkId, variantIds) {
         const benchmark = this.benchmarks.get(benchmarkId);
         if (!benchmark) {
             throw new Error(`Benchmark ${benchmarkId} not found`);
         }
-        const { config, variantMetrics } = benchmark;
+        const { config } = benchmark;
+        const variantMetrics = variantIds
+            ? new Map(variantIds.map(id => [id, benchmark.variantMetrics.get(id)]))
+            : benchmark.variantMetrics;
         const alpha = 1 - config.confidenceLevel;
         // Bonferroni correction for multiple comparisons
         const numComparisons = config.metrics.length;
@@ -540,10 +546,10 @@ export class ABBenchmarkingFramework {
         const chiSquareResult = chiSquareTest(variants);
         // Per-metric statistics (pairwise comparisons between first two variants)
         const metricStatistics = [];
-        const variantIds = Array.from(variantMetrics.keys());
-        if (variantIds.length >= 2) {
-            const v1 = variantMetrics.get(variantIds[0]);
-            const v2 = variantMetrics.get(variantIds[1]);
+        const comparedIds = Array.from(variantMetrics.keys());
+        if (comparedIds.length >= 2) {
+            const v1 = variantMetrics.get(comparedIds[0]);
+            const v2 = variantMetrics.get(comparedIds[1]);
             for (const metricConfig of config.metrics) {
                 const values1 = (v1.metrics.get(metricConfig.name) || []).map(d => d.value);
                 const values2 = (v2.metrics.get(metricConfig.name) || []).map(d => d.value);
@@ -618,10 +624,10 @@ export class ABBenchmarkingFramework {
         // Sort by combined score (descending)
         scores.sort((a, b) => b.score - a.score);
         const winner = scores[0];
-        // Calculate confidence based on significance and score gap
+        // Calculate a ranking heuristic based on significance and score gap
         const significance = this.calculateStatisticalSignificance(benchmarkId);
         const scoreGap = scores.length > 1 ? winner.score - scores[1].score : winner.score;
-        const confidence = significance.isSignificant
+        const heuristicScore = significance.isSignificant
             ? Math.min(0.99, 0.5 + scoreGap * 0.5 + (significance.powerAnalysis.currentPower * 0.25))
             : Math.min(0.5, scoreGap * 0.5);
         const significantMetrics = significance.metricStatistics
@@ -632,12 +638,38 @@ export class ABBenchmarkingFramework {
             benchmarkId,
             winnerId: winner.variantId,
             winnerName: variantConfig?.name || winner.variantId,
-            confidence,
+            heuristicScore,
             combinedScore: winner.score,
             significantMetrics,
-            recommendation: this.generateWinnerRecommendation(winner, confidence, significantMetrics),
+            recommendation: this.generateWinnerRecommendation(winner, heuristicScore, significantMetrics),
         };
     }
+    /** Require evidence for this candidate over its ranked runner-up. */
+    isSignificantlyBetterThanRunnerUp(benchmarkId, winnerId) {
+        const benchmark = this.benchmarks.get(benchmarkId);
+        const scores = this.calculateCombinedScores(benchmark).sort((a, b) => b.score - a.score);
+        const [winner, runnerUp] = scores;
+        if (!runnerUp || winner.variantId !== winnerId || winner.score <= runnerUp.score) {
+            return false;
+        }
+        const significance = this.calculateSignificance(benchmarkId, [winnerId, runnerUp.variantId]);
+        const winnerMetrics = benchmark.variantMetrics.get(winnerId);
+        const runnerUpMetrics = benchmark.variantMetrics.get(runnerUp.variantId);
+        const successMetrics = benchmark.config.metrics.filter(metric => metric.type === 'success_rate');
+        const successContributesToScore = successMetrics.length === 0 ||
+            successMetrics.some(metric => metric.weight > 0);
+        if (successContributesToScore && significance.chiSquareTest?.isSignificant &&
+            this.getSuccessRate(winnerMetrics) > this.getSuccessRate(runnerUpMetrics)) {
+            return true;
+        }
+        return significance.metricStatistics.some(statistic => {
+            const metric = benchmark.config.metrics.find(config => config.name === statistic.metric);
+            return statistic.isSignificant && metric.weight > 0 &&
+                (metric.higherIsBetter
+                    ? statistic.variantA.mean > statistic.variantB.mean
+                    : statistic.variantA.mean < statistic.variantB.mean);
+        });
+    }
     /**
      * Compare two specific variants
      */
@@ -693,9 +725,6 @@ export class ABBenchmarkingFramework {
             scoreA += normalizedA * metricConfig.weight;
             scoreB += normalizedB * metricConfig.weight;
         }
-        // Add success rate comparison
-        const successRateA = this.getSuccessRate(metricsA);
-        const successRateB = this.getSuccessRate(metricsB);
         const overallWinner = scoreA > scoreB ? variantA : scoreB > scoreA ? variantB : null;
         const confidence = Math.abs(scoreA - scoreB) / Math.max(scoreA, scoreB, 0.001);
         return {
@@ -760,16 +789,19 @@ export class ABBenchmarkingFramework {
             if (winner.significantMetrics.length > 0) {
                 reasoning.push(`Significant improvements in: ${winner.significantMetrics.join(', ')}`);
             }
-            if (winner.confidence >= 0.95) {
+            if (!this.isSignificantlyBetterThanRunnerUp(benchmarkId, winner.winnerId)) {
+                caveats.push('Leading variant is not significantly better than the runner-up');
+            }
+            else if (winner.heuristicScore >= 0.95) {
                 readyToApply = true;
-                reasoning.push('High confidence - ready to apply');
+                reasoning.push('High heuristic score with runner-up significance - ready to apply');
             }
-            else if (winner.confidence >= 0.8 && significance?.isSignificant) {
+            else if (winner.heuristicScore >= 0.8) {
                 readyToApply = true;
-                reasoning.push('Good confidence with statistical significance - ready to apply');
+                reasoning.push('Good heuristic score with runner-up significance - ready to apply');
             }
             else {
-                caveats.push(`Confidence (${(winner.confidence * 100).toFixed(1)}%) below recommended threshold`);
+                caveats.push(`Heuristic score (${winner.heuristicScore.toFixed(3)}) below recommended threshold`);
             }
         }
         else {
@@ -778,7 +810,7 @@ export class ABBenchmarkingFramework {
         return {
             benchmarkId,
             suggestedWinnerId,
-            confidence: winner?.confidence ?? 0,
+            confidence: winner?.heuristicScore ?? 0,
             reasoning,
             caveats,
             readyToApply,
@@ -798,12 +830,15 @@ export class ABBenchmarkingFramework {
         if (!winner) {
             throw new Error(`No winner determined for benchmark ${benchmarkId}`);
         }
+        if (!this.isSignificantlyBetterThanRunnerUp(benchmarkId, winner.winnerId)) {
+            throw new Error(`Winner is not significantly better than the runner-up for benchmark ${benchmarkId}`);
+        }
         // Mark benchmark as completed
         benchmark.status = 'completed';
         benchmark.endTime = Date.now();
         benchmark.winnerId = winner.winnerId;
         // Promote winner in Evolution Pipeline
-        evolutionPipelineIntegration.promoteRule(winner.winnerId, `Won A/B benchmark ${benchmarkId} with ${(winner.confidence * 100).toFixed(1)}% confidence`);
+        evolutionPipelineIntegration.promoteRule(winner.winnerId, `Won A/B benchmark ${benchmarkId} with heuristic score ${winner.heuristicScore.toFixed(3)} and runner-up significance`);
         // Complete the variant test
         evolutionPipelineIntegration.completeVariantTest(benchmarkId);
         this.logEvent('winner_applied', `Applied winner ${winner.winnerId} for benchmark ${benchmarkId}`);
@@ -979,25 +1014,25 @@ export class ABBenchmarkingFramework {
     /**
      * Generate winner recommendation text
      */
-    generateWinnerRecommendation(winner, confidence, significantMetrics) {
-        if (confidence >= 0.95) {
+    generateWinnerRecommendation(winner, heuristicScore, significantMetrics) {
+        if (heuristicScore >= 0.95) {
             return `Strongly recommend applying ${winner.variantId}. ` +
-                `High confidence (${(confidence * 100).toFixed(1)}%) with significant improvements ` +
+                `High heuristic score (${heuristicScore.toFixed(3)}) with significant improvements ` +
                 `in: ${significantMetrics.length > 0 ? significantMetrics.join(', ') : 'overall performance'}.`;
         }
-        else if (confidence >= 0.8) {
+        else if (heuristicScore >= 0.8) {
             return `Recommend applying ${winner.variantId}. ` +
-                `Good confidence (${(confidence * 100).toFixed(1)}%). ` +
-                `Consider collecting more samples for higher confidence.`;
+                `Good heuristic score (${heuristicScore.toFixed(3)}). ` +
+                `Consider collecting more samples for a higher heuristic score.`;
         }
-        else if (confidence >= 0.6) {
-            return `${winner.variantId} shows promise with moderate confidence ` +
-                `(${(confidence * 100).toFixed(1)}%). ` +
+        else if (heuristicScore >= 0.6) {
+            return `${winner.variantId} shows promise with moderate heuristic score ` +
+                `(${heuristicScore.toFixed(3)}). ` +
                 `Recommend collecting more samples before applying.`;
         }
         else {
             return `Results are inconclusive. ${winner.variantId} is currently leading but ` +
-                `confidence (${(confidence * 100).toFixed(1)}%) is low. Continue collecting data.`;
+                `heuristic score (${heuristicScore.toFixed(3)}) is low. Continue collecting data.`;
         }
     }
     /**