@yasserkhanorg/impact-gate 2.0.0

package/dist/agent/feedback.d.ts

@@ -0,0 +1,91 @@
+export interface RecommendationFeedbackEntry {
+    timestamp: string;
+    runSet: 'smoke' | 'targeted' | 'full';
+    recommendedTests: string[];
+    executedTests: string[];
+    failedTests: string[];
+    escapedFailures?: string[];
+}
+export interface CalibrationSummary {
+    schemaVersion: '1.1.0';
+    generatedAt: string;
+    samples: number;
+    overall: {
+        precision: number;
+        recall: number;
+        falseNegativeRate: number;
+    };
+    recent7d: {
+        precision: number;
+        recall: number;
+        falseNegativeRate: number;
+        samples: number;
+    };
+    recent30d: {
+        precision: number;
+        recall: number;
+        falseNegativeRate: number;
+        samples: number;
+    };
+    bySubsystem: Record<string, {
+        precision: number;
+        recall: number;
+        falseNegativeRate: number;
+        samples: number;
+        recent7d: {
+            precision: number;
+            recall: number;
+            falseNegativeRate: number;
+            samples: number;
+        };
+        recent30d: {
+            precision: number;
+            recall: number;
+            falseNegativeRate: number;
+            samples: number;
+        };
+    }>;
+}
+export interface FlakySummary {
+    schemaVersion: '1.1.0';
+    generatedAt: string;
+    tests: Array<{
+        test: string;
+        subsystem: string;
+        owners: string[];
+        flakeRate: number;
+        flakeRate7d: number;
+        flakeRate30d: number;
+        trend: 'up' | 'down' | 'stable';
+        quarantine: boolean;
+        quarantineState: 'none' | 'active' | 'retire-candidate';
+        lastFailureAt?: string;
+        samples: number;
+        samples7d: number;
+        samples30d: number;
+    }>;
+}
+export declare function appendFeedbackAndRecompute(appRoot: string, input: RecommendationFeedbackEntry): {
+    feedbackPath: string;
+    calibrationPath: string;
+    calibration: CalibrationSummary;
+};
+export declare function readCalibration(appRoot: string): CalibrationSummary | null;
+export interface AdaptiveThresholds {
+    minConfidenceForTargeted: number;
+    safeMergeMinConfidence: number;
+    /** Subsystems that should always be included regardless of confidence */
+    alwaysIncludeSubsystems: string[];
+    /** Human-readable adjustment reasons for logging */
+    adjustmentReasons: string[];
+}
+/**
+ * Compute adaptive thresholds based on calibration data.
+ * - If recent recall < 0.8: lower minConfidence (catch more escapes)
+ * - If recent precision > 0.9: raise minConfidence (fewer unnecessary tests)
+ * - Per-subsystem: if falseNegativeRate > 0.3 in 30d, always include tests
+ * Returns defaults if no calibration data exists.
+ */
+export declare function getAdaptiveThresholds(appRoot: string): AdaptiveThresholds;
+export declare function readFlakyTests(appRoot: string): FlakySummary | null;
+//# sourceMappingURL=feedback.d.ts.map

package/dist/agent/feedback.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"feedback.d.ts","sourceRoot":"","sources":["../../src/agent/feedback.ts"],"names":[],"mappings":"AAOA,MAAM,WAAW,2BAA2B;IACxC,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,OAAO,GAAG,UAAU,GAAG,MAAM,CAAC;IACtC,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;CAC9B;AAED,MAAM,WAAW,kBAAkB;IAC/B,aAAa,EAAE,OAAO,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE;QACL,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,iBAAiB,EAAE,MAAM,CAAC;KAC7B,CAAC;IACF,QAAQ,EAAE;QACN,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,iBAAiB,EAAE,MAAM,CAAC;QAC1B,OAAO,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,SAAS,EAAE;QACP,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,iBAAiB,EAAE,MAAM,CAAC;QAC1B,OAAO,EAAE,MAAM,CAAC;KACnB,CAAC;IACF,WAAW,EAAE,MAAM,CACnB,MAAM,EACN;QACI,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,iBAAiB,EAAE,MAAM,CAAC;QAC1B,OAAO,EAAE,MAAM,CAAC;QAChB,QAAQ,EAAE;YACN,SAAS,EAAE,MAAM,CAAC;YAClB,MAAM,EAAE,MAAM,CAAC;YACf,iBAAiB,EAAE,MAAM,CAAC;YAC1B,OAAO,EAAE,MAAM,CAAC;SACnB,CAAC;QACF,SAAS,EAAE;YACP,SAAS,EAAE,MAAM,CAAC;YAClB,MAAM,EAAE,MAAM,CAAC;YACf,iBAAiB,EAAE,MAAM,CAAC;YAC1B,OAAO,EAAE,MAAM,CAAC;SACnB,CAAC;KACL,CACA,CAAC;CACL;AAOD,MAAM,WAAW,YAAY;IACzB,aAAa,EAAE,OAAO,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,KAAK,CAAC;QACT,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,MAAM,EAAE,CAAC;QACjB,SAAS,EAAE,MAAM,CAAC;QAClB,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;QACrB,KAAK,EAAE,IAAI,GAAG,MAAM,GAAG,QAAQ,CAAC;QAChC,UAAU,EAAE,OAAO,CAAC;QACpB,eAAe,EAAE,MAAM,GAAG,QAAQ,GAAG,kBAAkB,CAAC;QACxD,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,OAAO,EAAE,MAAM,CAAC;QAChB,SAAS,EAAE,MAAM,CAAC;QAClB,UAAU,EAAE,MAAM,CAAC;KACtB,CAAC,CAAC;CACN;AAyQD,wBAAgB,0BAA0B,CACtC,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,2BAA2B,GACnC;IAAC,YAAY,EAAE,MAAM,CAAC;IAAC,eAAe,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,kBAAkB,CAAA;CAAC,CAwBlF;AAED,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,kBAAkB,GAAG,IAAI,CAE1E;AAED,MAAM,WAAW,kBAAkB;IAC/B,wBAAwB,EAAE,MAAM,CAAC;IACjC,sBAAsB,EAAE,MAAM,CAAC;IAC/B,yEAAyE;IACzE,uBAAuB,EAAE,MAAM,EAAE,CAAC;IAClC,oDAAoD;IACpD,iBAAiB,EAAE,MAAM,EAAE,CAAC;CAC/B;AAOD;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,MAAM,GAAG,kBAAkB,CA6DzE;AAED,wBAAgB,cAAc,CAAC,OAAO,EAAE,MAAM,GAAG,YAAY,GAAG,IAAI,CAEnE"}

package/dist/agent/feedback.js

@@ -0,0 +1,323 @@
+"use strict";
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.appendFeedbackAndRecompute = appendFeedbackAndRecompute;
+exports.readCalibration = readCalibration;
+exports.getAdaptiveThresholds = getAdaptiveThresholds;
+exports.readFlakyTests = readFlakyTests;
+const fs_1 = require("fs");
+const path_1 = require("path");
+const test_path_js_1 = require("./test_path.js");
+function readJson(path) {
+    if (!(0, fs_1.existsSync)(path)) {
+        return null;
+    }
+    try {
+        return JSON.parse((0, fs_1.readFileSync)(path, 'utf-8'));
+    }
+    catch {
+        return null;
+    }
+}
+function normalizeTestName(test) {
+    return test.replace(/ \(flags:.*\)$/, '').trim();
+}
+function asSet(values) {
+    return new Set(values.map(normalizeTestName).filter(Boolean));
+}
+function ratio(numerator, denominator) {
+    if (denominator <= 0) {
+        return 0;
+    }
+    return Number((numerator / denominator).toFixed(4));
+}
+function subsystemForTest(test) {
+    return (0, test_path_js_1.inferSubsystemFromTestPath)(test);
+}
+function parseTimestamp(value) {
+    const time = Date.parse(value);
+    if (Number.isNaN(time)) {
+        return null;
+    }
+    return time;
+}
+function filterRecent(entries, days) {
+    const cutoff = Date.now() - days * 24 * 60 * 60 * 1000;
+    return entries.filter((entry) => {
+        const time = parseTimestamp(entry.timestamp);
+        return time !== null && time >= cutoff;
+    });
+}
+function aggregateMetrics(entries) {
+    let truePositives = 0;
+    let recommendedTotal = 0;
+    let failuresTotal = 0;
+    let escapedTotal = 0;
+    for (const entry of entries) {
+        const recommended = asSet(entry.recommendedTests || []);
+        const failed = asSet(entry.failedTests || []);
+        const escaped = asSet(entry.escapedFailures || []);
+        const tp = Array.from(recommended).filter((test) => failed.has(test)).length;
+        truePositives += tp;
+        recommendedTotal += recommended.size;
+        failuresTotal += failed.size;
+        escapedTotal += escaped.size;
+    }
+    return {
+        precision: ratio(truePositives, recommendedTotal),
+        recall: ratio(truePositives, failuresTotal),
+        falseNegativeRate: ratio(escapedTotal, failuresTotal + escapedTotal),
+        samples: entries.length,
+    };
+}
+function aggregate(entries) {
+    const subsystemAcc = new Map();
+    for (const entry of entries) {
+        const recommended = asSet(entry.recommendedTests || []);
+        const failed = asSet(entry.failedTests || []);
+        const escaped = asSet(entry.escapedFailures || []);
+        const allSubsystems = new Set([
+            ...Array.from(recommended).map(subsystemForTest),
+            ...Array.from(failed).map(subsystemForTest),
+            ...Array.from(escaped).map(subsystemForTest),
+        ]);
+        for (const subsystem of allSubsystems) {
+            const bucket = subsystemAcc.get(subsystem) || { entries: [] };
+            bucket.entries.push({
+                ...entry,
+                recommendedTests: Array.from(recommended).filter((test) => subsystemForTest(test) === subsystem),
+                executedTests: (entry.executedTests || []).filter((test) => subsystemForTest(test) === subsystem),
+                failedTests: Array.from(failed).filter((test) => subsystemForTest(test) === subsystem),
+                escapedFailures: Array.from(escaped).filter((test) => subsystemForTest(test) === subsystem),
+            });
+            subsystemAcc.set(subsystem, bucket);
+        }
+    }
+    const bySubsystem = {};
+    for (const [subsystem, bucket] of subsystemAcc.entries()) {
+        const all = aggregateMetrics(bucket.entries);
+        const recent7d = aggregateMetrics(filterRecent(bucket.entries, 7));
+        const recent30d = aggregateMetrics(filterRecent(bucket.entries, 30));
+        bySubsystem[subsystem] = {
+            precision: all.precision,
+            recall: all.recall,
+            falseNegativeRate: all.falseNegativeRate,
+            samples: all.samples,
+            recent7d,
+            recent30d,
+        };
+    }
+    const overall = aggregateMetrics(entries);
+    const recent7d = aggregateMetrics(filterRecent(entries, 7));
+    const recent30d = aggregateMetrics(filterRecent(entries, 30));
+    return {
+        schemaVersion: '1.1.0',
+        generatedAt: new Date().toISOString(),
+        samples: entries.length,
+        overall,
+        recent7d,
+        recent30d,
+        bySubsystem,
+    };
+}
+function rateFor(runs, failed) {
+    return runs > 0 ? Number((failed / runs).toFixed(4)) : 0;
+}
+function trendFor(rate7d, rate30d) {
+    if (rate7d - rate30d >= 0.08) {
+        return 'up';
+    }
+    if (rate30d - rate7d >= 0.08) {
+        return 'down';
+    }
+    return 'stable';
+}
+function loadOwners(appRoot) {
+    const path = (0, path_1.join)(appRoot, '.e2e-ai-agents', 'subsystem-owners.json');
+    const manifest = readJson(path);
+    if (!manifest) {
+        return {};
+    }
+    if (manifest.ownersBySubsystem) {
+        return manifest.ownersBySubsystem;
+    }
+    if (manifest.subsystems) {
+        return manifest.subsystems;
+    }
+    return {};
+}
+function daysSince(value) {
+    if (!value) {
+        return null;
+    }
+    const time = parseTimestamp(value);
+    if (time === null) {
+        return null;
+    }
+    return (Date.now() - time) / (24 * 60 * 60 * 1000);
+}
+function aggregateFlaky(entries, appRoot) {
+    const acc = new Map();
+    const cutoff7 = Date.now() - 7 * 24 * 60 * 60 * 1000;
+    const cutoff30 = Date.now() - 30 * 24 * 60 * 60 * 1000;
+    const ownersBySubsystem = loadOwners(appRoot);
+    for (const entry of entries) {
+        const executed = asSet(entry.executedTests || []);
+        const failed = asSet(entry.failedTests || []);
+        const time = parseTimestamp(entry.timestamp);
+        for (const test of executed) {
+            const bucket = acc.get(test) || {
+                runs: 0,
+                failed: 0,
+                runs7d: 0,
+                failed7d: 0,
+                runs30d: 0,
+                failed30d: 0,
+                lastFailureAt: undefined,
+            };
+            bucket.runs += 1;
+            if (time !== null && time >= cutoff7) {
+                bucket.runs7d += 1;
+            }
+            if (time !== null && time >= cutoff30) {
+                bucket.runs30d += 1;
+            }
+            if (failed.has(test)) {
+                bucket.failed += 1;
+                bucket.lastFailureAt = entry.timestamp;
+                if (time !== null && time >= cutoff7) {
+                    bucket.failed7d += 1;
+                }
+                if (time !== null && time >= cutoff30) {
+                    bucket.failed30d += 1;
+                }
+            }
+            acc.set(test, bucket);
+        }
+    }
+    const tests = Array.from(acc.entries())
+        .map(([test, bucket]) => {
+        const flakeRate = rateFor(bucket.runs, bucket.failed);
+        const flakeRate7d = rateFor(bucket.runs7d, bucket.failed7d);
+        const flakeRate30d = rateFor(bucket.runs30d, bucket.failed30d);
+        const trend = trendFor(flakeRate7d, flakeRate30d);
+        const quarantine = (bucket.runs30d >= 5 && flakeRate30d >= 0.35) || (bucket.runs >= 8 && flakeRate >= 0.4);
+        const daysFromLastFailure = daysSince(bucket.lastFailureAt);
+        const quarantineState = quarantine
+            ? (daysFromLastFailure !== null && daysFromLastFailure >= 14 && flakeRate7d <= 0.05 ? 'retire-candidate' : 'active')
+            : 'none';
+        const subsystem = subsystemForTest(test);
+        const owners = ownersBySubsystem[subsystem] || [];
+        return {
+            test,
+            subsystem,
+            owners,
+            flakeRate,
+            flakeRate7d,
+            flakeRate30d,
+            trend,
+            quarantine,
+            quarantineState,
+            lastFailureAt: bucket.lastFailureAt,
+            samples: bucket.runs,
+            samples7d: bucket.runs7d,
+            samples30d: bucket.runs30d,
+        };
+    })
+        .filter((entry) => entry.flakeRate > 0)
+        .sort((a, b) => (b.flakeRate30d || b.flakeRate) - (a.flakeRate30d || a.flakeRate));
+    return {
+        schemaVersion: '1.1.0',
+        generatedAt: new Date().toISOString(),
+        tests,
+    };
+}
+function appendFeedbackAndRecompute(appRoot, input) {
+    const baseDir = (0, path_1.join)(appRoot, '.e2e-ai-agents');
+    (0, fs_1.mkdirSync)(baseDir, { recursive: true });
+    const feedbackPath = (0, path_1.join)(baseDir, 'feedback.json');
+    const existing = readJson(feedbackPath) || { schemaVersion: '1.0.0', entries: [] };
+    existing.entries.push({
+        ...input,
+        recommendedTests: input.recommendedTests || [],
+        executedTests: input.executedTests || [],
+        failedTests: input.failedTests || [],
+        escapedFailures: input.escapedFailures || [],
+    });
+    (0, fs_1.writeFileSync)(feedbackPath, JSON.stringify(existing, null, 2), 'utf-8');
+    const calibration = aggregate(existing.entries);
+    const calibrationPath = (0, path_1.join)(baseDir, 'calibration.json');
+    (0, fs_1.writeFileSync)(calibrationPath, JSON.stringify(calibration, null, 2), 'utf-8');
+    const flaky = aggregateFlaky(existing.entries, appRoot);
+    const flakyPath = (0, path_1.join)(baseDir, 'flaky-tests.json');
+    (0, fs_1.writeFileSync)(flakyPath, JSON.stringify(flaky, null, 2), 'utf-8');
+    return { feedbackPath, calibrationPath, calibration };
+}
+function readCalibration(appRoot) {
+    return readJson((0, path_1.join)(appRoot, '.e2e-ai-agents', 'calibration.json'));
+}
+const DEFAULT_MIN_CONFIDENCE = 60;
+const DEFAULT_SAFE_MERGE = 85;
+const MIN_CONFIDENCE_FLOOR = 40;
+const MIN_CONFIDENCE_CEILING = 80;
+/**
+ * Compute adaptive thresholds based on calibration data.
+ * - If recent recall < 0.8: lower minConfidence (catch more escapes)
+ * - If recent precision > 0.9: raise minConfidence (fewer unnecessary tests)
+ * - Per-subsystem: if falseNegativeRate > 0.3 in 30d, always include tests
+ * Returns defaults if no calibration data exists.
+ */
+function getAdaptiveThresholds(appRoot) {
+    const calibration = readCalibration(appRoot);
+    const reasons = [];
+    const alwaysInclude = [];
+    if (!calibration || calibration.samples === 0) {
+        return {
+            minConfidenceForTargeted: DEFAULT_MIN_CONFIDENCE,
+            safeMergeMinConfidence: DEFAULT_SAFE_MERGE,
+            alwaysIncludeSubsystems: [],
+            adjustmentReasons: ['No calibration data — using defaults'],
+        };
+    }
+    let minConfidence = DEFAULT_MIN_CONFIDENCE;
+    let safeMerge = DEFAULT_SAFE_MERGE;
+    // Adjust based on 7-day recall
+    if (calibration.recent7d.samples >= 3) {
+        if (calibration.recent7d.recall < 0.8) {
+            const adjustment = 10;
+            minConfidence -= adjustment;
+            safeMerge -= adjustment;
+            reasons.push(`Lowering confidence threshold by ${adjustment} (7d recall: ${calibration.recent7d.recall.toFixed(2)})`);
+        }
+        else if (calibration.recent7d.precision > 0.9) {
+            const adjustment = 5;
+            minConfidence += adjustment;
+            safeMerge += adjustment;
+            reasons.push(`Raising confidence threshold by ${adjustment} (7d precision: ${calibration.recent7d.precision.toFixed(2)})`);
+        }
+    }
+    // Clamp to safe ranges
+    minConfidence = Math.max(MIN_CONFIDENCE_FLOOR, Math.min(MIN_CONFIDENCE_CEILING, minConfidence));
+    safeMerge = Math.max(70, Math.min(95, safeMerge));
+    // Per-subsystem blind spot detection (30-day window)
+    for (const [subsystem, metrics] of Object.entries(calibration.bySubsystem)) {
+        const recent = metrics.recent30d;
+        if (recent.samples >= 3 && recent.falseNegativeRate > 0.3) {
+            alwaysInclude.push(subsystem);
+            reasons.push(`Always including ${subsystem} tests (30d false-negative rate: ${recent.falseNegativeRate.toFixed(2)})`);
+        }
+    }
+    if (reasons.length === 0) {
+        reasons.push('Calibration data within normal range — using defaults');
+    }
+    return {
+        minConfidenceForTargeted: minConfidence,
+        safeMergeMinConfidence: safeMerge,
+        alwaysIncludeSubsystems: alwaysInclude,
+        adjustmentReasons: reasons,
+    };
+}
+function readFlakyTests(appRoot) {
+    return readJson((0, path_1.join)(appRoot, '.e2e-ai-agents', 'flaky-tests.json'));
+}

package/dist/agent/git.d.ts

@@ -0,0 +1,19 @@
+export interface GitChangeResult {
+    files: string[];
+    /** Test/spec files from the diff that were filtered by isRelevantFile(). Only includes files matching TEST_FILE_PATTERNS — not config, docs, or other non-test filtered files. */
+    filteredTestFiles: string[];
+    error?: string;
+    baseRef?: string;
+    baseStrategy?: 'merge-base' | 'direct';
+}
+export interface GitChangeOptions {
+    includeUncommitted?: boolean;
+}
+export declare function runGitRaw(args: string[], cwd: string): string | null;
+/**
+ * Check if a file path is a test file (spec, test, or in test directories).
+ * Shared across pipeline and crew orchestrators.
+ */
+export declare function isTestFile(file: string): boolean;
+export declare function getChangedFiles(appRoot: string, since: string, options?: GitChangeOptions): GitChangeResult;
+//# sourceMappingURL=git.d.ts.map

package/dist/agent/git.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"git.d.ts","sourceRoot":"","sources":["../../src/agent/git.ts"],"names":[],"mappings":"AAkHA,MAAM,WAAW,eAAe;IAC5B,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,kLAAkL;IAClL,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,YAAY,GAAG,QAAQ,CAAC;CAC1C;AAED,MAAM,WAAW,gBAAgB;IAC7B,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAChC;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAUpE;AAqED;;;GAGG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAShD;AAED,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,eAAe,CA8D3G"}