@esbenwiberg/corpus-default 1.1.0 → 1.2.0

package/dist/probes/tests-deterministic.js

@@ -0,0 +1,155 @@
+import { defineProbe } from "@esbenwiberg/repofit/sdk";
+const TEST_FILE = /(?:\.test\.|\.spec\.|__tests__|^tests?\/|^e2e\/)/i;
+const SOURCE_FILE = /\.(?:ts|tsx|js|jsx|mjs|cjs|py|cs|go|rs|java|kt)$/i;
+const SKIP_DIRS = /(?:^|\/)(?:node_modules|dist|build|coverage|\.next|\.nuxt|out|target|bin|obj)\//;
+const FLAKE_PATTERNS = [
+    {
+        pattern: /\b(?:test|it|describe)\.only\s*\(/,
+        severity: "error",
+        message: "commits a focused-only test",
+    },
+    {
+        pattern: /\b(?:test|it|describe)\.skip\s*\(/,
+        severity: "warn",
+        message: "skips a test without an executable signal",
+    },
+    {
+        pattern: /\bMath\.random\s*\(|\brandomUUID\s*\(|\brandint\s*\(|\brandom\.\w+\s*\(/,
+        severity: "warn",
+        message: "uses randomness without an obvious seed",
+    },
+    {
+        pattern: /\bDate\.now\s*\(|\bnew Date\s*\(|\btime\.time\s*\(|\bInstant\.now\s*\(/,
+        severity: "warn",
+        message: "depends on wall-clock time",
+    },
+    {
+        pattern: /\bsetTimeout\s*\([^,]+,\s*(?:[1-9]\d{2,}|[1-9]\d{3,})|\bwaitForTimeout\s*\(/,
+        severity: "warn",
+        message: "uses fixed sleeps instead of waiting on a condition",
+    },
+    {
+        pattern: /\bfetch\s*\(\s*["'`]https?:\/\/|\baxios\.\w+\s*\(\s*["'`]https?:\/\//,
+        severity: "warn",
+        message: "calls an external network endpoint from a test",
+    },
+];
+export default defineProbe({
+    id: "tests.deterministic",
+    version: "1.0.0",
+    dimensions: [
+        { id: "feedback", weight: 1 },
+        { id: "latency", weight: 0.3 },
+    ],
+    tier: "static",
+    evidence: ["files", "size_stats"],
+    rationale: `
+    Flaky tests poison the agent feedback loop: a red result no longer means
+    the last change broke behavior. This static probe avoids running large
+    suites and instead flags common flake smells in test files: focused tests,
+    skips, unseeded randomness, wall-clock time, fixed sleeps, and live
+    external network calls.
+  `,
+    remediation: "Remove focused tests before committing, turn skips into tracked TODOs, seed randomness, inject clocks, replace fixed sleeps with condition waits, and mock or record external network calls. Deterministic failures are the currency agents can spend.",
+    async detect(ev) {
+        const testPaths = ev.size_stats.files
+            .map((f) => f.path)
+            .filter((p) => SOURCE_FILE.test(p) && TEST_FILE.test(p) && !SKIP_DIRS.test(`/${p}`))
+            .sort();
+        if (testPaths.length === 0) {
+            return { kind: "na", reason: "no test files detected" };
+        }
+        const items = [];
+        for (const path of testPaths) {
+            const text = await ev.files.readText(path);
+            if (!text)
+                continue;
+            const lines = text.split(/\n/);
+            for (let i = 0; i < lines.length; i += 1) {
+                const line = lines[i] ?? "";
+                for (const { pattern, severity, message } of FLAKE_PATTERNS) {
+                    if (pattern.test(line)) {
+                        items.push({
+                            location: { path, range: { startLine: i + 1 } },
+                            severity,
+                            message,
+                        });
+                    }
+                }
+            }
+        }
+        return { kind: "inventory", items };
+    },
+    score: {
+        kind: "inventory",
+        severityWeights: { info: 1, warn: 2, error: 10 },
+        bands: [{ upTo: 0, score: 100 }, { upTo: 2, score: 80 }, { upTo: 6, score: 50 }, { score: 0 }],
+    },
+    fixtures: [
+        {
+            name: "no-tests",
+            evidence: {
+                size_stats: {
+                    source: "git-ls-files",
+                    totalBytes: 100,
+                    totalFiles: 1,
+                    files: [{ path: "src/index.ts", bytes: 100, lines: 8, depth: 1 }],
+                },
+            },
+            expect: { reading: { kind: "na", reason: "no test files detected" }, score: null },
+        },
+        {
+            name: "deterministic-tests",
+            evidence: {
+                files: {
+                    "src/math.test.ts": 'test("adds numbers", () => {\n  expect(add(1, 2)).toBe(3);\n});\n',
+                },
+                size_stats: {
+                    source: "git-ls-files",
+                    totalBytes: 100,
+                    totalFiles: 1,
+                    files: [{ path: "src/math.test.ts", bytes: 100, lines: 3, depth: 1 }],
+                },
+            },
+            expect: { reading: { kind: "inventory", items: [] }, score: 100 },
+        },
+        {
+            name: "flake-smells",
+            evidence: {
+                files: {
+                    "e2e/login.spec.ts": 'test.only("login", async ({ page }) => {\n  await page.waitForTimeout(1000);\n  expect(Date.now()).toBeGreaterThan(0);\n});\n',
+                },
+                size_stats: {
+                    source: "git-ls-files",
+                    totalBytes: 180,
+                    totalFiles: 1,
+                    files: [{ path: "e2e/login.spec.ts", bytes: 180, lines: 4, depth: 1 }],
+                },
+            },
+            expect: {
+                reading: {
+                    kind: "inventory",
+                    items: [
+                        {
+                            location: { path: "e2e/login.spec.ts", range: { startLine: 1 } },
+                            severity: "error",
+                            message: "commits a focused-only test",
+                        },
+                        {
+                            location: { path: "e2e/login.spec.ts", range: { startLine: 2 } },
+                            severity: "warn",
+                            message: "uses fixed sleeps instead of waiting on a condition",
+                        },
+                        {
+                            location: { path: "e2e/login.spec.ts", range: { startLine: 3 } },
+                            severity: "warn",
+                            message: "depends on wall-clock time",
+                        },
+                    ],
+                },
+                score: 0,
+            },
+        },
+    ],
+});
+//# sourceMappingURL=tests-deterministic.js.map

package/dist/probes/tests-deterministic.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tests-deterministic.js","sourceRoot":"","sources":["../../src/probes/tests-deterministic.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAEvD,MAAM,SAAS,GAAG,mDAAmD,CAAC;AACtE,MAAM,WAAW,GAAG,mDAAmD,CAAC;AACxE,MAAM,SAAS,GAAG,iFAAiF,CAAC;AAEpG,MAAM,cAAc,GAA+D;IACjF;QACE,OAAO,EAAE,mCAAmC;QAC5C,QAAQ,EAAE,OAAO;QACjB,OAAO,EAAE,6BAA6B;KACvC;IACD;QACE,OAAO,EAAE,mCAAmC;QAC5C,QAAQ,EAAE,MAAM;QAChB,OAAO,EAAE,2CAA2C;KACrD;IACD;QACE,OAAO,EAAE,yEAAyE;QAClF,QAAQ,EAAE,MAAM;QAChB,OAAO,EAAE,yCAAyC;KACnD;IACD;QACE,OAAO,EAAE,wEAAwE;QACjF,QAAQ,EAAE,MAAM;QAChB,OAAO,EAAE,4BAA4B;KACtC;IACD;QACE,OAAO,EAAE,6EAA6E;QACtF,QAAQ,EAAE,MAAM;QAChB,OAAO,EAAE,qDAAqD;KAC/D;IACD;QACE,OAAO,EAAE,sEAAsE;QAC/E,QAAQ,EAAE,MAAM;QAChB,OAAO,EAAE,gDAAgD;KAC1D;CACF,CAAC;AAEF,eAAe,WAAW,CAAC;IACzB,EAAE,EAAE,qBAAqB;IACzB,OAAO,EAAE,OAAO;IAChB,UAAU,EAAE;QACV,EAAE,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,EAAE;QAC7B,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE;KAC/B;IACD,IAAI,EAAE,QAAQ;IACd,QAAQ,EAAE,CAAC,OAAO,EAAE,YAAY,CAAC;IAEjC,SAAS,EAAE;;;;;;GAMV;IAED,WAAW,EACT,wPAAwP;IAE1P,KAAK,CAAC,MAAM,CAAC,EAAE;QACb,MAAM,SAAS,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK;aAClC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;aAClB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;aACnF,IAAI,EAAE,CAAC;QAEV,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,wBAAwB,EAAE,CAAC;QAC1D,CAAC;QAED,MAAM,KAAK,GAAoB,EAAE,CAAC;QAClC,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC7B,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YAC3C,IAAI,CAAC,IAAI;gBAAE,SAAS;YACpB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;gBACzC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC5B,KAAK,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,cAAc,EAAE,CAAC;oBAC5D,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;wBACvB,KAAK,CAAC,IAAI,CAAC;4BACT,QAAQ,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,SAAS,EAAE,CAAC,GAAG,CAAC,EAAE,EAAE;4BAC/C,QAAQ;4BACR,OAAO;yBACR,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;IACtC,CAAC;IAED,KAAK,EAAE;QACL,IAAI,EAAE,WAAW;QACjB,eAAe,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE;QAChD,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;KAC/F;IAED,QAAQ,EAAE;QACR;YACE,IAAI,EAAE,UAAU;YAChB,QAAQ,EAAE;gBACR,UAAU,EAAE;oBACV,MAAM,EAAE,cAAc;oBACtB,UAAU,EAAE,GAAG;oBACf,UAAU,EAAE,CAAC;oBACb,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;iBAClE;aACF;YACD,MAAM,EAAE,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,wBAAwB,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE;SACnF;QACD;YACE,IAAI,EAAE,qBAAqB;YAC3B,QAAQ,EAAE;gBACR,KAAK,EAAE;oBACL,kBAAkB,EAAE,mEAAmE;iBACxF;gBACD,UAAU,EAAE;oBACV,MAAM,EAAE,cAAc;oBACtB,UAAU,EAAE,GAAG;oBACf,UAAU,EAAE,CAAC;oBACb,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;iBACtE;aACF;YACD,MAAM,EAAE,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE;SAClE;QACD;YACE,IAAI,EAAE,cAAc;YACpB,QAAQ,EAAE;gBACR,KAAK,EAAE;oBACL,mBAAmB,EACjB,+HAA+H;iBAClI;gBACD,UAAU,EAAE;oBACV,MAAM,EAAE,cAAc;oBACtB,UAAU,EAAE,GAAG;oBACf,UAAU,EAAE,CAAC;oBACb,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,mBAAmB,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;iBACvE;aACF;YACD,MAAM,EAAE;gBACN,OAAO,EAAE;oBACP,IAAI,EAAE,WAAW;oBACjB,KAAK,EAAE;wBACL;4BACE,QAAQ,EAAE,EAAE,IAAI,EAAE,mBAAmB,EAAE,KAAK,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,EAAE;4BAChE,QAAQ,EAAE,OAAO;4BACjB,OAAO,EAAE,6BAA6B;yBACvC;wBACD;4BACE,QAAQ,EAAE,EAAE,IAAI,EAAE,mBAAmB,EAAE,KAAK,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,EAAE;4BAChE,QAAQ,EAAE,MAAM;4BAChB,OAAO,EAAE,qDAAqD;yBAC/D;wBACD;4BACE,QAAQ,EAAE,EAAE,IAAI,EAAE,mBAAmB,EAAE,KAAK,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,EAAE;4BAChE,QAAQ,EAAE,MAAM;4BAChB,OAAO,EAAE,4BAA4B;yBACtC;qBACF;iBACF;gBACD,KAAK,EAAE,CAAC;aACT;SACF;KACF;CACF,CAAC,CAAC"}

package/dist/probes/tests-failure-actionability.d.ts

@@ -0,0 +1,3 @@
+declare const _default: import("@esbenwiberg/repofit/sdk").Probe;
+export default _default;
+//# sourceMappingURL=tests-failure-actionability.d.ts.map

package/dist/probes/tests-failure-actionability.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tests-failure-actionability.d.ts","sourceRoot":"","sources":["../../src/probes/tests-failure-actionability.ts"],"names":[],"mappings":";AAqCA,wBAqHG"}

package/dist/probes/tests-failure-actionability.js

@@ -0,0 +1,139 @@
+import { defineProbe } from "@esbenwiberg/repofit/sdk";
+const PROBE_VERSION = "1.0.0";
+const MAX_TEST_FILES = 6;
+const MAX_CHARS_PER_TEST = 2_500;
+const MAX_INPUT_CHARS = 18_000;
+const TEST_FILE = /(?:\.test\.|\.spec\.|__tests__|^tests?\/|^e2e\/)/i;
+const SOURCE_FILE = /\.(?:ts|tsx|js|jsx|mjs|cjs|py|cs|go|rs|java|kt)$/i;
+const SKIP_DIRS = /(?:^|\/)(?:node_modules|dist|build|coverage|\.next|\.nuxt|out|target|bin|obj)\//;
+const RUBRIC = {
+    task: "Judge whether test failures in this repository would be actionable for a coding agent trying to fix a regression.",
+    criteria: [
+        {
+            id: "scenario-names",
+            description: "Do test names describe the condition and expected behavior clearly enough to identify the broken contract? Generic names like 'works' or 'handles input' score low.",
+        },
+        {
+            id: "diagnostic-assertions",
+            description: "Do assertions expose useful expected/actual values, error messages, snapshots with narrow scope, or custom messages? Bare booleans and opaque helper failures score low.",
+        },
+        {
+            id: "localized-failures",
+            description: "Would a failing test point to a small behavior or subsystem, rather than a giant end-to-end path with many possible causes?",
+        },
+        {
+            id: "reproducible-setup",
+            description: "Is each test's setup explicit enough that an agent can reproduce and reason about the failure without hidden global state or undocumented fixtures?",
+        },
+    ],
+};
+export default defineProbe({
+    id: "tests.failure-actionability",
+    version: PROBE_VERSION,
+    dimensions: [{ id: "feedback", weight: 1 }],
+    tier: "reasoned",
+    evidence: ["files", "size_stats", "judge"],
+    rationale: `
+    Tests are only useful feedback when failures explain the broken fact.
+    Agents struggle with generic test names, opaque helpers, bare boolean
+    assertions, and giant flows where many things could be wrong. This probe
+    samples tests and judges whether failures would be diagnostic.
+  `,
+    remediation: "Name tests after the scenario and expected behavior, assert concrete expected/actual values, keep failures localized, and make setup explicit. Prefer `throws /message/`, table cases with named cases, and narrow snapshots over giant opaque failures.",
+    async detect(ev) {
+        const testPaths = ev.size_stats.files
+            .map((f) => f.path)
+            .filter((p) => SOURCE_FILE.test(p) && TEST_FILE.test(p) && !SKIP_DIRS.test(`/${p}`))
+            .sort();
+        if (testPaths.length === 0) {
+            return { kind: "na", reason: "no test files detected" };
+        }
+        const sampled = [];
+        let totalChars = 0;
+        for (const p of testPaths) {
+            if (sampled.length >= MAX_TEST_FILES)
+                break;
+            const text = await ev.files.readText(p);
+            if (!text)
+                continue;
+            const slice = text.slice(0, MAX_CHARS_PER_TEST);
+            sampled.push({ path: p, text: slice });
+            totalChars += slice.length;
+            if (totalChars >= MAX_INPUT_CHARS)
+                break;
+        }
+        if (sampled.length === 0) {
+            return { kind: "na", reason: "test files declared but unreadable" };
+        }
+        const input = sampled.map((s) => `# ${s.path}\n\n${s.text}`).join("\n\n---\n\n");
+        const result = await ev.judge.score({
+            probeId: "tests.failure-actionability",
+            probeVersion: PROBE_VERSION,
+            input,
+            rubric: RUBRIC,
+        });
+        return {
+            kind: "judge",
+            score: result.score,
+            perCriterion: result.perCriterion,
+            rationale: result.rationale,
+            model: result.model,
+        };
+    },
+    score: { kind: "judge" },
+    fixtures: [
+        {
+            name: "no-tests",
+            evidence: {
+                size_stats: {
+                    source: "git-ls-files",
+                    totalBytes: 100,
+                    totalFiles: 1,
+                    files: [{ path: "src/index.ts", bytes: 100, lines: 8, depth: 1 }],
+                },
+            },
+            expect: { reading: { kind: "na", reason: "no test files detected" }, score: null },
+        },
+        {
+            name: "actionable-failures",
+            evidence: {
+                files: {
+                    "src/cache.test.ts": 'test("expired cache entry returns a miss with stale reason", () => {\n  const cache = cacheWith({ key: "a", expiresAt: 0 });\n  expect(cache.get("a")).toEqual({ hit: false, reason: "expired" });\n});\n',
+                },
+                size_stats: {
+                    source: "git-ls-files",
+                    totalBytes: 220,
+                    totalFiles: 1,
+                    files: [{ path: "src/cache.test.ts", bytes: 220, lines: 5, depth: 1 }],
+                },
+                judge: {
+                    score: 80,
+                    perCriterion: {
+                        "scenario-names": 80,
+                        "diagnostic-assertions": 80,
+                        "localized-failures": 80,
+                        "reproducible-setup": 80,
+                    },
+                    rationale: "The test name, setup, and expected value identify the broken contract.",
+                    model: "fixture",
+                },
+            },
+            expect: {
+                reading: {
+                    kind: "judge",
+                    score: 80,
+                    perCriterion: {
+                        "scenario-names": 80,
+                        "diagnostic-assertions": 80,
+                        "localized-failures": 80,
+                        "reproducible-setup": 80,
+                    },
+                    rationale: "The test name, setup, and expected value identify the broken contract.",
+                    model: "fixture",
+                },
+                score: 80,
+            },
+        },
+    ],
+});
+//# sourceMappingURL=tests-failure-actionability.js.map

package/dist/probes/tests-failure-actionability.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tests-failure-actionability.js","sourceRoot":"","sources":["../../src/probes/tests-failure-actionability.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAEvD,MAAM,aAAa,GAAG,OAAO,CAAC;AAC9B,MAAM,cAAc,GAAG,CAAC,CAAC;AACzB,MAAM,kBAAkB,GAAG,KAAK,CAAC;AACjC,MAAM,eAAe,GAAG,MAAM,CAAC;AAE/B,MAAM,SAAS,GAAG,mDAAmD,CAAC;AACtE,MAAM,WAAW,GAAG,mDAAmD,CAAC;AACxE,MAAM,SAAS,GAAG,iFAAiF,CAAC;AAEpG,MAAM,MAAM,GAAG;IACb,IAAI,EAAE,mHAAmH;IACzH,QAAQ,EAAE;QACR;YACE,EAAE,EAAE,gBAAgB;YACpB,WAAW,EACT,qKAAqK;SACxK;QACD;YACE,EAAE,EAAE,uBAAuB;YAC3B,WAAW,EACT,0KAA0K;SAC7K;QACD;YACE,EAAE,EAAE,oBAAoB;YACxB,WAAW,EACT,6HAA6H;SAChI;QACD;YACE,EAAE,EAAE,oBAAoB;YACxB,WAAW,EACT,qJAAqJ;SACxJ;KACF;CACO,CAAC;AAEX,eAAe,WAAW,CAAC;IACzB,EAAE,EAAE,6BAA6B;IACjC,OAAO,EAAE,aAAa;IACtB,UAAU,EAAE,CAAC,EAAE,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;IAC3C,IAAI,EAAE,UAAU;IAChB,QAAQ,EAAE,CAAC,OAAO,EAAE,YAAY,EAAE,OAAO,CAAC;IAE1C,SAAS,EAAE;;;;;GAKV;IAED,WAAW,EACT,0PAA0P;IAE5P,KAAK,CAAC,MAAM,CAAC,EAAE;QACb,MAAM,SAAS,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK;aAClC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;aAClB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;aACnF,IAAI,EAAE,CAAC;QAEV,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,wBAAwB,EAAE,CAAC;QAC1D,CAAC;QAED,MAAM,OAAO,GAAqC,EAAE,CAAC;QACrD,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;YAC1B,IAAI,OAAO,CAAC,MAAM,IAAI,cAAc;gBAAE,MAAM;YAC5C,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YACxC,IAAI,CAAC,IAAI;gBAAE,SAAS;YACpB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC;YAChD,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;YACvC,UAAU,IAAI,KAAK,CAAC,MAAM,CAAC;YAC3B,IAAI,UAAU,IAAI,eAAe;gBAAE,MAAM;QAC3C,CAAC;QAED,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,oCAAoC,EAAE,CAAC;QACtE,CAAC;QAED,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QACjF,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC;YAClC,OAAO,EAAE,6BAA6B;YACtC,YAAY,EAAE,aAAa;YAC3B,KAAK;YACL,MAAM,EAAE,MAAM;SACf,CAAC,CAAC;QAEH,OAAO;YACL,IAAI,EAAE,OAAO;YACb,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,KAAK,EAAE,MAAM,CAAC,KAAK;SACpB,CAAC;IACJ,CAAC;IAED,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE;IAExB,QAAQ,EAAE;QACR;YACE,IAAI,EAAE,UAAU;YAChB,QAAQ,EAAE;gBACR,UAAU,EAAE;oBACV,MAAM,EAAE,cAAc;oBACtB,UAAU,EAAE,GAAG;oBACf,UAAU,EAAE,CAAC;oBACb,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;iBAClE;aACF;YACD,MAAM,EAAE,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,wBAAwB,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE;SACnF;QACD;YACE,IAAI,EAAE,qBAAqB;YAC3B,QAAQ,EAAE;gBACR,KAAK,EAAE;oBACL,mBAAmB,EACjB,2MAA2M;iBAC9M;gBACD,UAAU,EAAE;oBACV,MAAM,EAAE,cAAc;oBACtB,UAAU,EAAE,GAAG;oBACf,UAAU,EAAE,CAAC;oBACb,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,mBAAmB,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;iBACvE;gBACD,KAAK,EAAE;oBACL,KAAK,EAAE,EAAE;oBACT,YAAY,EAAE;wBACZ,gBAAgB,EAAE,EAAE;wBACpB,uBAAuB,EAAE,EAAE;wBAC3B,oBAAoB,EAAE,EAAE;wBACxB,oBAAoB,EAAE,EAAE;qBACzB;oBACD,SAAS,EAAE,wEAAwE;oBACnF,KAAK,EAAE,SAAS;iBACjB;aACF;YACD,MAAM,EAAE;gBACN,OAAO,EAAE;oBACP,IAAI,EAAE,OAAO;oBACb,KAAK,EAAE,EAAE;oBACT,YAAY,EAAE;wBACZ,gBAAgB,EAAE,EAAE;wBACpB,uBAAuB,EAAE,EAAE;wBAC3B,oBAAoB,EAAE,EAAE;wBACxB,oBAAoB,EAAE,EAAE;qBACzB;oBACD,SAAS,EAAE,wEAAwE;oBACnF,KAAK,EAAE,SAAS;iBACjB;gBACD,KAAK,EAAE,EAAE;aACV;SACF;KACF;CACF,CAAC,CAAC"}

package/dist/probes/tests-oracle-quality.d.ts

@@ -0,0 +1,3 @@
+declare const _default: import("@esbenwiberg/repofit/sdk").Probe;
+export default _default;
+//# sourceMappingURL=tests-oracle-quality.d.ts.map

package/dist/probes/tests-oracle-quality.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tests-oracle-quality.d.ts","sourceRoot":"","sources":["../../src/probes/tests-oracle-quality.ts"],"names":[],"mappings":";AAqCA,wBAsHG"}

package/dist/probes/tests-oracle-quality.js

@@ -0,0 +1,140 @@
+import { defineProbe } from "@esbenwiberg/repofit/sdk";
+const PROBE_VERSION = "1.0.0";
+const MAX_TEST_FILES = 6;
+const MAX_CHARS_PER_TEST = 2_500;
+const MAX_INPUT_CHARS = 18_000;
+const TEST_FILE = /(?:\.test\.|\.spec\.|__tests__|^tests?\/|^e2e\/)/i;
+const SOURCE_FILE = /\.(?:ts|tsx|js|jsx|mjs|cjs|py|cs|go|rs|java|kt)$/i;
+const SKIP_DIRS = /(?:^|\/)(?:node_modules|dist|build|coverage|\.next|\.nuxt|out|target|bin|obj)\//;
+const RUBRIC = {
+    task: "Judge whether this repository's tests act as useful oracles for a coding agent, rather than merely pleasing reviewers with superficial coverage.",
+    criteria: [
+        {
+            id: "behavioral-assertions",
+            description: "Do tests assert externally meaningful behavior, outputs, state changes, errors, or contracts? Tests with no assertions, trivial truthiness, or assertions that only mirror implementation details score low.",
+        },
+        {
+            id: "negative-and-edge-cases",
+            description: "Do tests include failure paths, edge cases, invalid inputs, permissions, empty states, or regression scenarios? Happy-path-only tests score lower.",
+        },
+        {
+            id: "mock-discipline",
+            description: "Are mocks/stubs used to isolate boundaries while preserving the behavior under test? Tests that mock the unit itself, assert only that mocks were called, or overfit to implementation score low.",
+        },
+        {
+            id: "review-resistance",
+            description: "Would these tests reject a plausible but wrong agent patch, or could an agent satisfy them with a shallow implementation? Tests that encode real facts and invariants score high.",
+        },
+    ],
+};
+export default defineProbe({
+    id: "tests.oracle-quality",
+    version: PROBE_VERSION,
+    dimensions: [{ id: "feedback", weight: 1 }],
+    tier: "reasoned",
+    evidence: ["files", "size_stats", "judge"],
+    rationale: `
+    A passing test suite is only useful when the tests can reject wrong
+    behavior. Agents often add "pleasing" tests: smoke checks, snapshots,
+    mock-call assertions, or tests that merely document the implementation
+    they just wrote. This probe samples test files and asks whether they are
+    real behavioral oracles.
+  `,
+    remediation: "Strengthen tests around behavior and contracts: assert outputs and state, cover edge/failure cases, keep mocks at boundaries, and add regression tests that would fail for the bug or requirement being addressed. Avoid assertion-free smoke tests, pure snapshot tests, and tests that only assert mock calls.",
+    async detect(ev) {
+        const testPaths = ev.size_stats.files
+            .map((f) => f.path)
+            .filter((p) => SOURCE_FILE.test(p) && TEST_FILE.test(p) && !SKIP_DIRS.test(`/${p}`))
+            .sort();
+        if (testPaths.length === 0) {
+            return { kind: "na", reason: "no test files detected" };
+        }
+        const sampled = [];
+        let totalChars = 0;
+        for (const p of testPaths) {
+            if (sampled.length >= MAX_TEST_FILES)
+                break;
+            const text = await ev.files.readText(p);
+            if (!text)
+                continue;
+            const slice = text.slice(0, MAX_CHARS_PER_TEST);
+            sampled.push({ path: p, text: slice });
+            totalChars += slice.length;
+            if (totalChars >= MAX_INPUT_CHARS)
+                break;
+        }
+        if (sampled.length === 0) {
+            return { kind: "na", reason: "test files declared but unreadable" };
+        }
+        const input = sampled.map((s) => `# ${s.path}\n\n${s.text}`).join("\n\n---\n\n");
+        const result = await ev.judge.score({
+            probeId: "tests.oracle-quality",
+            probeVersion: PROBE_VERSION,
+            input,
+            rubric: RUBRIC,
+        });
+        return {
+            kind: "judge",
+            score: result.score,
+            perCriterion: result.perCriterion,
+            rationale: result.rationale,
+            model: result.model,
+        };
+    },
+    score: { kind: "judge" },
+    fixtures: [
+        {
+            name: "no-tests",
+            evidence: {
+                size_stats: {
+                    source: "git-ls-files",
+                    totalBytes: 100,
+                    totalFiles: 1,
+                    files: [{ path: "src/index.ts", bytes: 100, lines: 8, depth: 1 }],
+                },
+            },
+            expect: { reading: { kind: "na", reason: "no test files detected" }, score: null },
+        },
+        {
+            name: "strong-oracles",
+            evidence: {
+                files: {
+                    "src/parse.test.ts": 'import { parsePort } from "./parse";\n\ntest("rejects out of range ports", () => {\n  expect(() => parsePort("70000")).toThrow(/port/i);\n});\n\ntest("accepts valid port", () => {\n  expect(parsePort("5432")).toBe(5432);\n});\n',
+                },
+                size_stats: {
+                    source: "git-ls-files",
+                    totalBytes: 240,
+                    totalFiles: 1,
+                    files: [{ path: "src/parse.test.ts", bytes: 240, lines: 9, depth: 1 }],
+                },
+                judge: {
+                    score: 80,
+                    perCriterion: {
+                        "behavioral-assertions": 80,
+                        "negative-and-edge-cases": 80,
+                        "mock-discipline": 80,
+                        "review-resistance": 80,
+                    },
+                    rationale: "Tests assert behavior and failure cases without over-mocking.",
+                    model: "fixture",
+                },
+            },
+            expect: {
+                reading: {
+                    kind: "judge",
+                    score: 80,
+                    perCriterion: {
+                        "behavioral-assertions": 80,
+                        "negative-and-edge-cases": 80,
+                        "mock-discipline": 80,
+                        "review-resistance": 80,
+                    },
+                    rationale: "Tests assert behavior and failure cases without over-mocking.",
+                    model: "fixture",
+                },
+                score: 80,
+            },
+        },
+    ],
+});
+//# sourceMappingURL=tests-oracle-quality.js.map

package/dist/probes/tests-oracle-quality.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tests-oracle-quality.js","sourceRoot":"","sources":["../../src/probes/tests-oracle-quality.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAEvD,MAAM,aAAa,GAAG,OAAO,CAAC;AAC9B,MAAM,cAAc,GAAG,CAAC,CAAC;AACzB,MAAM,kBAAkB,GAAG,KAAK,CAAC;AACjC,MAAM,eAAe,GAAG,MAAM,CAAC;AAE/B,MAAM,SAAS,GAAG,mDAAmD,CAAC;AACtE,MAAM,WAAW,GAAG,mDAAmD,CAAC;AACxE,MAAM,SAAS,GAAG,iFAAiF,CAAC;AAEpG,MAAM,MAAM,GAAG;IACb,IAAI,EAAE,kJAAkJ;IACxJ,QAAQ,EAAE;QACR;YACE,EAAE,EAAE,uBAAuB;YAC3B,WAAW,EACT,8MAA8M;SACjN;QACD;YACE,EAAE,EAAE,yBAAyB;YAC7B,WAAW,EACT,oJAAoJ;SACvJ;QACD;YACE,EAAE,EAAE,iBAAiB;YACrB,WAAW,EACT,mMAAmM;SACtM;QACD;YACE,EAAE,EAAE,mBAAmB;YACvB,WAAW,EACT,mLAAmL;SACtL;KACF;CACO,CAAC;AAEX,eAAe,WAAW,CAAC;IACzB,EAAE,EAAE,sBAAsB;IAC1B,OAAO,EAAE,aAAa;IACtB,UAAU,EAAE,CAAC,EAAE,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;IAC3C,IAAI,EAAE,UAAU;IAChB,QAAQ,EAAE,CAAC,OAAO,EAAE,YAAY,EAAE,OAAO,CAAC;IAE1C,SAAS,EAAE;;;;;;GAMV;IAED,WAAW,EACT,kTAAkT;IAEpT,KAAK,CAAC,MAAM,CAAC,EAAE;QACb,MAAM,SAAS,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK;aAClC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;aAClB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;aACnF,IAAI,EAAE,CAAC;QAEV,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,wBAAwB,EAAE,CAAC;QAC1D,CAAC;QAED,MAAM,OAAO,GAAqC,EAAE,CAAC;QACrD,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;YAC1B,IAAI,OAAO,CAAC,MAAM,IAAI,cAAc;gBAAE,MAAM;YAC5C,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YACxC,IAAI,CAAC,IAAI;gBAAE,SAAS;YACpB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC;YAChD,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;YACvC,UAAU,IAAI,KAAK,CAAC,MAAM,CAAC;YAC3B,IAAI,UAAU,IAAI,eAAe;gBAAE,MAAM;QAC3C,CAAC;QAED,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,oCAAoC,EAAE,CAAC;QACtE,CAAC;QAED,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QACjF,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC;YAClC,OAAO,EAAE,sBAAsB;YAC/B,YAAY,EAAE,aAAa;YAC3B,KAAK;YACL,MAAM,EAAE,MAAM;SACf,CAAC,CAAC;QAEH,OAAO;YACL,IAAI,EAAE,OAAO;YACb,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,KAAK,EAAE,MAAM,CAAC,KAAK;SACpB,CAAC;IACJ,CAAC;IAED,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE;IAExB,QAAQ,EAAE;QACR;YACE,IAAI,EAAE,UAAU;YAChB,QAAQ,EAAE;gBACR,UAAU,EAAE;oBACV,MAAM,EAAE,cAAc;oBACtB,UAAU,EAAE,GAAG;oBACf,UAAU,EAAE,CAAC;oBACb,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;iBAClE;aACF;YACD,MAAM,EAAE,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,wBAAwB,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE;SACnF;QACD;YACE,IAAI,EAAE,gBAAgB;YACtB,QAAQ,EAAE;gBACR,KAAK,EAAE;oBACL,mBAAmB,EACjB,qOAAqO;iBACxO;gBACD,UAAU,EAAE;oBACV,MAAM,EAAE,cAAc;oBACtB,UAAU,EAAE,GAAG;oBACf,UAAU,EAAE,CAAC;oBACb,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,mBAAmB,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;iBACvE;gBACD,KAAK,EAAE;oBACL,KAAK,EAAE,EAAE;oBACT,YAAY,EAAE;wBACZ,uBAAuB,EAAE,EAAE;wBAC3B,yBAAyB,EAAE,EAAE;wBAC7B,iBAAiB,EAAE,EAAE;wBACrB,mBAAmB,EAAE,EAAE;qBACxB;oBACD,SAAS,EAAE,+DAA+D;oBAC1E,KAAK,EAAE,SAAS;iBACjB;aACF;YACD,MAAM,EAAE;gBACN,OAAO,EAAE;oBACP,IAAI,EAAE,OAAO;oBACb,KAAK,EAAE,EAAE;oBACT,YAAY,EAAE;wBACZ,uBAAuB,EAAE,EAAE;wBAC3B,yBAAyB,EAAE,EAAE;wBAC7B,iBAAiB,EAAE,EAAE;wBACrB,mBAAmB,EAAE,EAAE;qBACxB;oBACD,SAAS,EAAE,+DAA+D;oBAC1E,KAAK,EAAE,SAAS;iBACjB;gBACD,KAAK,EAAE,EAAE;aACV;SACF;KACF;CACF,CAAC,CAAC"}

package/dist/probes/types-escape-hatches.d.ts

@@ -0,0 +1,3 @@
+declare const _default: import("@esbenwiberg/repofit/sdk").Probe;
+export default _default;
+//# sourceMappingURL=types-escape-hatches.d.ts.map

package/dist/probes/types-escape-hatches.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types-escape-hatches.d.ts","sourceRoot":"","sources":["../../src/probes/types-escape-hatches.ts"],"names":[],"mappings":";AA8DA,wBAiJG"}