agentic-qe 3.15.5 → 3.15.7

package/dist/workers/workers/regression-monitor.js

@@ -6,9 +6,14 @@
  * - Test result regressions
  * - Performance regressions
  * - Coverage regressions
- * - Quality metric regressions
+ *
+ * #1028: every figure comes from measured evidence — the stamped
+ * `coverage:latest` record and the latest testsPassing quality evidence (with
+ * its run's recorded result). Nothing measured → nothing stored or compared.
  */
 import { BaseWorker } from '../base-worker.js';
+import { measuredCoverageSummary, } from '../../domains/quality-assessment/coverage-report-event.js';
+import { readQualityEvidenceRecord } from '../../domains/quality-assessment/quality-evidence.js';
 const CONFIG = {
     id: 'regression-monitor',
     name: 'Regression Monitor',
@@ -21,6 +26,74 @@ const CONFIG = {
     retryCount: 2,
     retryDelayMs: 10000,
 };
+const BASELINE_KEY = 'regression:baseline';
+const HISTORY_KEY = 'regression:history';
+const BASELINE_SCHEMA_VERSION = 2;
+const isPercent = (v) => typeof v === 'number' && Number.isFinite(v) && v >= 0 && v <= 100;
+/** Two decimals, for reporting measured figures. */
+const round = (v) => Math.round(v * 100) / 100;
+/** The stored snapshot when it is a measured baseline, else `null` (#1028). */
+function measuredBaseline(raw) {
+    if (!raw || typeof raw !== 'object')
+        return null;
+    const r = raw;
+    if (r.measured !== true || r.schemaVersion !== BASELINE_SCHEMA_VERSION)
+        return null;
+    const coverage = r.coverage === null ? null : measuredCoverageSummary(r.coverage);
+    if (r.coverage !== null && !coverage)
+        return null;
+    const tests = r.tests === null ? null : measuredTests(r.tests);
+    if (r.tests !== null && !tests)
+        return null;
+    if (!coverage && !tests)
+        return null;
+    return {
+        measured: true,
+        schemaVersion: BASELINE_SCHEMA_VERSION,
+        timestamp: typeof r.timestamp === 'string' ? r.timestamp : '',
+        coverage,
+        tests,
+    };
+}
+function measuredTests(raw) {
+    if (!raw || typeof raw !== 'object')
+        return null;
+    const t = raw;
+    if (!isPercent(t.passRate) || typeof t.measuredAt !== 'string' || typeof t.source !== 'string')
+        return null;
+    const failedTests = Array.isArray(t.failedTests) && t.failedTests.every((n) => typeof n === 'string')
+        ? t.failedTests : null;
+    const avg = t.avgTestDurationMs;
+    return {
+        testSet: parseTestSet(t.testSet),
+        passRate: t.passRate,
+        runId: typeof t.runId === 'string' ? t.runId : null,
+        runStatus: t.runStatus === 'passed' || t.runStatus === 'failed' ? t.runStatus : null,
+        measuredAt: t.measuredAt,
+        source: t.source,
+        failedTests,
+        avgTestDurationMs: typeof avg === 'number' && Number.isFinite(avg) && avg >= 0 ? avg : null,
+    };
+}
+/** A recorded test set `{ total, files }`, or `null`. */
+function parseTestSet(raw) {
+    if (!raw || typeof raw !== 'object')
+        return null;
+    const r = raw;
+    const { total } = r;
+    if (typeof total !== 'number' || !Number.isInteger(total) || total <= 0)
+        return null;
+    const files = testFileList(r.files);
+    return files ? { total, files } : null;
+}
+/** A non-empty list of test file names, sorted and de-duplicated, or `null`. */
+function testFileList(raw) {
+    if (!Array.isArray(raw) || raw.length === 0 || !raw.every((f) => typeof f === 'string'))
+        return null;
+    return [...new Set(raw)].sort();
+}
+const sameTestSet = (a, b) => a !== null && b !== null && a.total === b.total
+    && a.files.length === b.files.length && a.files.every((f, i) => f === b.files[i]);
 export class RegressionMonitorWorker extends BaseWorker {
     constructor() {
         super(CONFIG);
@@ -30,139 +103,235 @@ export class RegressionMonitorWorker extends BaseWorker {
         context.logger.info('Starting regression monitoring');
         const findings = [];
         const recommendations = [];
-        // Get current snapshot
-        const currentSnapshot = await this.collectCurrentSnapshot(context);
-        // Get baseline (previous snapshot)
-        const baseline = await context.memory.get('regression:baseline');
-        // Detect regressions
+        // #1028: only measured figures — never a hardcoded or estimated snapshot.
+        const current = await this.collectCurrentSnapshot(context);
+        // A baseline counts only with the measured stamp: the invented snapshot
+        // older versions stored is ignored (left in place, never compared).
+        const storedBaseline = await context.memory.get(BASELINE_KEY);
+        const baseline = measuredBaseline(storedBaseline);
+        const baselineStatus = this.baselineStatus(storedBaseline, baseline);
+        if (baselineStatus === 'legacy-ignored') {
+            context.logger.info('Ignoring a regression:baseline without the measured stamp (written by <= 3.15.5)');
+        }
+        else if (baselineStatus === 'invalid-ignored') {
+            context.logger.warn('Ignoring a malformed measured regression:baseline');
+        }
         let regressions = [];
-        if (baseline) {
-            regressions = this.detectRegressions(baseline, currentSnapshot);
+        let metricsCompared = 0;
+        if (baseline && current) {
+            ({ regressions, metricsCompared } = this.detectRegressions(baseline, current));
             this.analyzeRegressions(regressions, findings, recommendations);
         }
-        // Store current as new baseline
-        await context.memory.set('regression:baseline', currentSnapshot);
-        // Store regression history
-        const history = await context.memory.get('regression:history') || [];
+        if (current) {
+            // A side not measured now keeps its last measured value: an estimate or
+            // a missing run neither erases the baseline nor gets compared.
+            const next = {
+                ...current,
+                coverage: current.coverage ?? baseline?.coverage ?? null,
+                tests: current.tests ?? baseline?.tests ?? null,
+            };
+            await context.memory.set(BASELINE_KEY, next);
+        }
+        else {
+            findings.push({
+                type: 'regression-unmeasured',
+                severity: 'info',
+                domain: 'quality-assessment',
+                title: 'Regression Baseline Unmeasured',
+                description: 'No measured coverage (stamped coverage:latest) and no current testsPassing evidence exist, ' +
+                    'so nothing was stored or compared. Run the tests (with coverage) to start regression tracking.',
+            });
+        }
         if (regressions.length > 0) {
-            history.push(...regressions);
+            const detectedAt = new Date().toISOString();
+            const stored = await context.memory.get(HISTORY_KEY);
+            const history = Array.isArray(stored) ? stored : [];
+            history.push(...regressions.map((r) => ({ ...r, measured: true, detectedAt })));
             // Keep last 100 regressions
             while (history.length > 100)
                 history.shift();
-            await context.memory.set('regression:history', history);
+            await context.memory.set(HISTORY_KEY, history);
         }
         const healthScore = this.calculateHealthScore(regressions);
         context.logger.info('Regression monitoring complete', {
             healthScore,
+            measured: current !== null,
+            baselineStatus,
             regressionsDetected: regressions.length,
         });
+        const pct = (v) => (v === null ? 'n/a' : `${round(v)}%`);
+        const tests = current?.tests ?? null;
+        const coverage = current?.coverage ?? null;
         return this.createResult(Date.now() - startTime, {
-            itemsAnalyzed: 5, // Number of metrics checked
+            itemsAnalyzed: metricsCompared,
             issuesFound: regressions.length,
             healthScore,
             trend: regressions.length > 0 ? 'degrading' : 'stable',
             domainMetrics: {
-                testPassRate: `${currentSnapshot.testPassRate}%`,
-                lineCoverage: `${currentSnapshot.coverage.line}%`,
-                avgTestDuration: `${currentSnapshot.avgTestDuration}ms`,
-                qualityScore: currentSnapshot.qualityScore,
+                baselineStatus,
+                testPassRate: tests ? pct(tests.passRate) : 'unmeasured',
+                lineCoverage: coverage ? pct(coverage.line) : 'unmeasured',
+                branchCoverage: coverage ? pct(coverage.branch) : 'unmeasured',
+                avgTestDuration: tests && tests.avgTestDurationMs !== null ? `${round(tests.avgTestDurationMs)}ms` : 'unmeasured',
                 regressionsDetected: regressions.length,
             },
         }, findings, recommendations);
     }
-    async collectCurrentSnapshot(_context) {
-        // In a real implementation, this would query actual metrics
+    /** Why a stored baseline is (not) used: absent, legacy (unstamped), stamped but malformed, or measured. */
+    baselineStatus(stored, baseline) {
+        if (baseline)
+            return 'measured';
+        if (stored === undefined || stored === null)
+            return 'none';
+        const stamped = typeof stored === 'object' && stored.measured === true;
+        return stamped ? 'invalid-ignored' : 'legacy-ignored';
+    }
+    /**
+     * #1028: the current snapshot from measured sources only — the stamped
+     * `coverage:latest` (same validator as the coverage tracker and the quality
+     * gate) and the latest usable testsPassing evidence (same validation as the
+     * gate) with that run's recorded `test-run:<runId>` result. `null` when
+     * nothing is measured.
+     */
+    async collectCurrentSnapshot(context) {
+        const coverage = measuredCoverageSummary(await context.memory.get('coverage:latest'));
+        const tests = await this.collectMeasuredTests(context);
+        if (!coverage && !tests)
+            return null;
         return {
-            timestamp: new Date(),
-            testPassRate: 98.2,
-            coverage: {
-                line: 78.5,
-                branch: 65.2,
-            },
-            avgTestDuration: 245,
-            failedTests: [
-                'test-execution/retry-handler.test.ts::should retry failed tests',
-                'coordination/workflow.test.ts::should handle concurrent workflows',
-            ],
-            qualityScore: 82,
+            measured: true,
+            schemaVersion: BASELINE_SCHEMA_VERSION,
+            timestamp: new Date().toISOString(),
+            coverage,
+            tests,
         };
     }
-    detectRegressions(baseline, current) {
-        const regressions = [];
-        // Test pass rate regression
-        if (current.testPassRate < baseline.testPassRate - 1) {
-            const change = current.testPassRate - baseline.testPassRate;
-            regressions.push({
-                type: 'test',
-                metric: 'testPassRate',
-                previousValue: baseline.testPassRate,
-                currentValue: current.testPassRate,
-                changePercent: change,
-                severity: change < -5 ? 'critical' : change < -2 ? 'high' : 'medium',
-                affectedTests: current.failedTests,
-            });
-        }
-        // Coverage regression
-        if (current.coverage.line < baseline.coverage.line - 2) {
-            const change = current.coverage.line - baseline.coverage.line;
-            regressions.push({
-                type: 'coverage',
-                metric: 'lineCoverage',
-                previousValue: baseline.coverage.line,
-                currentValue: current.coverage.line,
-                changePercent: change,
-                severity: change < -5 ? 'high' : 'medium',
-            });
-        }
-        if (current.coverage.branch < baseline.coverage.branch - 3) {
-            const change = current.coverage.branch - baseline.coverage.branch;
-            regressions.push({
-                type: 'coverage',
-                metric: 'branchCoverage',
-                previousValue: baseline.coverage.branch,
-                currentValue: current.coverage.branch,
-                changePercent: change,
-                severity: change < -5 ? 'high' : 'medium',
-            });
-        }
-        // Performance regression
-        const durationChange = ((current.avgTestDuration - baseline.avgTestDuration) / baseline.avgTestDuration) * 100;
-        if (durationChange > 20) {
-            regressions.push({
-                type: 'performance',
-                metric: 'avgTestDuration',
-                previousValue: baseline.avgTestDuration,
-                currentValue: current.avgTestDuration,
-                changePercent: durationChange,
-                severity: durationChange > 50 ? 'high' : 'medium',
+    async collectMeasuredTests(context) {
+        const evidence = await readQualityEvidenceRecord(context.memory, 'testsPassing');
+        if (!evidence || !isPercent(evidence.value))
+            return null;
+        let failedTests = null;
+        let avgTestDurationMs = null;
+        let testSet = null;
+        if (evidence.runId) {
+            const run = await context.memory.get(`test-run:${evidence.runId}`, {
+                namespace: 'test-execution',
             });
+            if (run && typeof run === 'object') {
+                if (Array.isArray(run.failedTests)) {
+                    failedTests = run.failedTests
+                        .map((t) => (t && typeof t === 'object' ? t.testName : undefined))
+                        .filter((n) => typeof n === 'string');
+                }
+                const { total, duration } = run;
+                if (typeof total === 'number' && total > 0 && typeof duration === 'number' && Number.isFinite(duration) && duration >= 0) {
+                    avgTestDurationMs = duration / total;
+                }
+                testSet = parseTestSet({ total, files: run.testFiles });
+            }
         }
-        // Quality score regression
-        if (current.qualityScore < baseline.qualityScore - 5) {
-            const change = current.qualityScore - baseline.qualityScore;
-            regressions.push({
-                type: 'quality',
-                metric: 'qualityScore',
-                previousValue: baseline.qualityScore,
-                currentValue: current.qualityScore,
-                changePercent: change,
-                severity: change < -10 ? 'critical' : change < -5 ? 'high' : 'medium',
-            });
+        return {
+            passRate: evidence.value,
+            runId: evidence.runId ?? null,
+            runStatus: evidence.runStatus ?? null,
+            measuredAt: evidence.measuredAt,
+            source: evidence.source,
+            failedTests,
+            avgTestDurationMs,
+            testSet,
+        };
+    }
+    /**
+     * Compare two measured snapshots. A metric is compared only when both sides
+     * measured it; an uncollected (`null`) metric is never compared (#1028).
+     */
+    detectRegressions(baseline, current) {
+        const regressions = [];
+        let metricsCompared = 0;
+        const prevTests = baseline.tests;
+        const curTests = current.tests;
+        if (prevTests && curTests) {
+            metricsCompared++;
+            if (curTests.passRate < prevTests.passRate - 1) {
+                const change = curTests.passRate - prevTests.passRate;
+                regressions.push({
+                    type: 'test',
+                    metric: 'testPassRate',
+                    previousValue: round(prevTests.passRate),
+                    currentValue: round(curTests.passRate),
+                    changePercent: change,
+                    severity: change < -5 ? 'critical' : change < -2 ? 'high' : 'medium',
+                    ...(curTests.failedTests ? { affectedTests: curTests.failedTests } : {}),
+                });
+            }
+            if (prevTests.failedTests && curTests.failedTests) {
+                metricsCompared++;
+                const known = new Set(prevTests.failedTests);
+                const newFailures = curTests.failedTests.filter((t) => !known.has(t));
+                if (newFailures.length > 0) {
+                    regressions.push({
+                        type: 'test',
+                        metric: 'newFailures',
+                        previousValue: 0,
+                        currentValue: newFailures.length,
+                        changePercent: 100,
+                        severity: newFailures.length > 5 ? 'critical' : newFailures.length > 2 ? 'high' : 'medium',
+                        affectedTests: newFailures,
+                    });
+                }
+            }
+            // #1028 N1: the recorded duration is the run's wall time, runner start-up
+            // included, and the runner records no per-test durations for passing
+            // tests. Per-test time is comparable only between runs of the same test
+            // set; across different sets it mostly measures overhead.
+            const prevAvg = prevTests.avgTestDurationMs;
+            const curAvg = curTests.avgTestDurationMs;
+            if (prevAvg !== null && curAvg !== null && prevAvg > 0 && sameTestSet(prevTests.testSet, curTests.testSet)) {
+                metricsCompared++;
+                const durationChange = ((curAvg - prevAvg) / prevAvg) * 100;
+                if (durationChange > 20) {
+                    regressions.push({
+                        type: 'performance',
+                        metric: 'avgTestDuration',
+                        previousValue: round(prevAvg),
+                        currentValue: round(curAvg),
+                        changePercent: durationChange,
+                        severity: durationChange > 50 ? 'high' : 'medium',
+                    });
+                }
+            }
         }
-        // New test failures
-        const newFailures = current.failedTests.filter(t => !baseline.failedTests.includes(t));
-        if (newFailures.length > 0) {
-            regressions.push({
-                type: 'test',
-                metric: 'newFailures',
-                previousValue: 0,
-                currentValue: newFailures.length,
-                changePercent: 100,
-                severity: newFailures.length > 5 ? 'critical' : newFailures.length > 2 ? 'high' : 'medium',
-                affectedTests: newFailures,
-            });
+        const prevCov = baseline.coverage;
+        const curCov = current.coverage;
+        if (prevCov && curCov) {
+            metricsCompared++;
+            if (curCov.line < prevCov.line - 2) {
+                const change = curCov.line - prevCov.line;
+                regressions.push({
+                    type: 'coverage',
+                    metric: 'lineCoverage',
+                    previousValue: prevCov.line,
+                    currentValue: curCov.line,
+                    changePercent: change,
+                    severity: change < -5 ? 'high' : 'medium',
+                });
+            }
+            if (prevCov.branch !== null && curCov.branch !== null) {
+                metricsCompared++;
+                if (curCov.branch < prevCov.branch - 3) {
+                    const change = curCov.branch - prevCov.branch;
+                    regressions.push({
+                        type: 'coverage',
+                        metric: 'branchCoverage',
+                        previousValue: prevCov.branch,
+                        currentValue: curCov.branch,
+                        changePercent: change,
+                        severity: change < -5 ? 'high' : 'medium',
+                    });
+                }
+            }
         }
-        return regressions;
+        return { regressions, metricsCompared };
     }
     analyzeRegressions(regressions, findings, recommendations) {
         for (const regression of regressions) {
@@ -174,6 +343,7 @@ export class RegressionMonitorWorker extends BaseWorker {
                 title: `${this.formatMetricName(regression.metric)} Regression Detected`,
                 description: this.formatRegressionDescription(regression),
                 context: {
+                    metric: regression.metric,
                     previousValue: regression.previousValue,
                     currentValue: regression.currentValue,
                     changePercent: regression.changePercent.toFixed(2),
@@ -223,17 +393,7 @@ export class RegressionMonitorWorker extends BaseWorker {
         }
     }
     getDomainForType(type) {
-        switch (type) {
-            case 'test':
-            case 'performance':
-                return 'test-execution';
-            case 'coverage':
-                return 'coverage-analysis';
-            case 'quality':
-                return 'quality-assessment';
-            default:
-                return 'quality-assessment';
-        }
+        return type === 'coverage' ? 'coverage-analysis' : 'test-execution';
     }
     formatMetricName(metric) {
         const names = {
@@ -241,7 +401,6 @@ export class RegressionMonitorWorker extends BaseWorker {
             lineCoverage: 'Line Coverage',
             branchCoverage: 'Branch Coverage',
             avgTestDuration: 'Test Duration',
-            qualityScore: 'Quality Score',
             newFailures: 'New Test Failures',
         };
         return names[metric] || metric;

package/dist/workers/workers/security-scan-records.d.ts

@@ -0,0 +1,111 @@
+/**
+ * Agentic QE v3 - security-scan worker: recorded scan parsing (#1043, #1049)
+ *
+ * Parses the `security:scan:<id>` records the security-compliance domain writes
+ * (OSV dependency scans and SAST scans) into what the security-scan worker
+ * reports. Nothing is filled in: a field a record does not carry stays absent.
+ */
+export declare const SCHEMA_VERSION = 1;
+/** The dependency scanner keeps its records for 7 days; nothing older is current. */
+export declare const MAX_SCAN_AGE_MS: number;
+export type Severity = 'critical' | 'high' | 'medium' | 'low' | 'informational';
+export declare const SEVERITIES: readonly Severity[];
+export declare const CVE_ID: RegExp;
+export interface ScanCounts {
+    critical: number;
+    high: number;
+    medium: number;
+    low: number;
+    informational: number;
+}
+/** A vulnerability as a recorded scan reported it; nothing is filled in. */
+export interface ReportedVulnerability {
+    severity: Severity;
+    title: string;
+    description: string;
+    category: string;
+    /** Dependency name (dependency scan) — only when the record names it. */
+    package?: string;
+    /** `file:line` (SAST) — only when the record locates it. */
+    location?: string;
+    /** CVE id exactly as the advisory carried it; absent when it carried none. */
+    cve?: string;
+    fixedIn?: string;
+    references: string[];
+}
+export interface RecordedScan {
+    scanId: string;
+    scannedAt: string;
+    summary: ScanCounts;
+    /** Vulnerabilities the record lists (SAST keeps only critical/high). */
+    vulnerabilities: ReportedVulnerability[];
+    /** Total the scan found, including ones the record does not list. */
+    total: number;
+}
+/** A package the OSV lookup could not check (#1049 record contract). */
+export interface UncheckedPackage {
+    name: string;
+    version: string;
+    reason: string;
+}
+/**
+ * A dependency scan whose OSV lookup was not complete (#1049): `incomplete`,
+ * `failed`, or a pre-#1049 record without `status` — which cannot be told
+ * apart from a failed lookup recorded as clean. Never "0 vulnerabilities".
+ */
+export interface IncompleteDependencyScan {
+    scan: RecordedScan;
+    status: 'incomplete' | 'failed' | 'unknown';
+    error?: string;
+    uncheckedPackages: UncheckedPackage[];
+    /** Packages the lookup did check, when the record states it. */
+    packagesChecked?: number;
+}
+/** The `security:lastScan` record: only scans that actually ran (#1043). */
+export interface MeasuredSecurityScan {
+    measured: true;
+    schemaVersion: typeof SCHEMA_VERSION;
+    evaluatedAt: string;
+    dependency: (RecordedScan & {
+        source: 'osv';
+    }) | null;
+    code: (RecordedScan & {
+        source: 'sast';
+    }) | null;
+}
+export declare const isCount: (v: unknown) => v is number;
+export declare const isRecord: (v: unknown) => v is Record<string, unknown>;
+export declare function parseCounts(raw: unknown): ScanCounts | null;
+/** `scannedAt` when the record's timestamp is valid, not in the future and within the scanner's retention. */
+export declare function parseScannedAt(raw: unknown, now: number): string | null;
+export declare function parseVulnerability(raw: unknown, kind: 'dependency' | 'sast'): ReportedVulnerability | null;
+export declare function parseUnchecked(raw: unknown): UncheckedPackage[];
+/**
+ * #1049: a dependency scan is a measurement only when its OSV lookup checked
+ * every requested package (`measured: true` and `status: 'complete'`).
+ */
+export declare function dependencyCompleteness(raw: Record<string, unknown>): {
+    complete: true;
+} | {
+    complete: false;
+    status: IncompleteDependencyScan['status'];
+    error?: string;
+    uncheckedPackages: UncheckedPackage[];
+    packagesChecked?: number;
+};
+export declare const SEVERITY_LABEL: Record<Severity, string>;
+/** Health penalty for the real critical/high advisories of a partial dependency scan (#1049). */
+export declare function partialDependencyPenalty(incomplete: IncompleteDependencyScan | null): number;
+/** What a partial dependency scan did check; the dependency totals stay 'unmeasured' (#1049). */
+export declare function partialDependencyMetrics(incomplete: IncompleteDependencyScan | null): Record<string, number>;
+export type ParsedScan = {
+    kind: 'sast';
+    scan: RecordedScan;
+} | {
+    kind: 'dependency';
+    scan: RecordedScan;
+    completeness: ReturnType<typeof dependencyCompleteness>;
+};
+/** A recorded dependency (OSV) or SAST scan, or `null` when the record is not one. */
+export declare function parseRecordedScan(raw: unknown, now: number): ParsedScan | null;
+//# sourceMappingURL=security-scan-records.d.ts.map