d365fo-mcp 1.0.0

package/dist/eval/improver/flakeDetection.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"flakeDetection.js","sourceRoot":"","sources":["../../../src/eval/improver/flakeDetection.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,oEAAoE;AACpE,MAAM,YAAY,GAAG,CAAC,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,SAAS,CAAU,CAAC;AA4B/E;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CAAC,IAAsB;IAC1D,MAAM,MAAM,GAAG,IAAI,GAAG,EAA4B,CAAC;IACnD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,CAAC,GAAG,CAAC,cAAc;YAAE,SAAS;QAClC,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,cAAc,EAAE,CAAC;QACnD,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,IAAI,GAAG;YAAE,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;;YAAM,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACtD,CAAC;IAED,MAAM,UAAU,GAAqB,EAAE,CAAC;IACxC,KAAK,MAAM,CAAC,GAAG,EAAE,SAAS,CAAC,IAAI,MAAM,EAAE,CAAC;QACtC,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS;QACnC,MAAM,CAAC,OAAO,EAAE,cAAc,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAEjD,MAAM,aAAa,GAAwB,EAAE,CAAC;QAC9C,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;YACjC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;YACvE,IAAI,QAAQ,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;gBACtB,MAAM,MAAM,GAA8C,EAAE,CAAC;gBAC7D,KAAK,MAAM,CAAC,IAAI,SAAS;oBAAE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC;gBACvE,aAAa,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;YACxC,CAAC;QACH,CAAC;QAED,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7B,UAAU,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,aAAa,EAAE,CAAC,CAAC;QACpG,CAAC;IACH,CAAC;IAED,mEAAmE;IACnE,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,aAAa,CAAC,MAAM,IAAI,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACjH,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,gEAAgE;AAChE,MAAM,UAAU,qBAAqB,CAAC,UAA4B;IAChE,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,0FAA0F,CAAC;IAC/H,MAAM,KAAK,GAAa,CAAC,uBAAuB,UAAU,CAAC,MAAM,KAAK,CAAC,CAAC;IACxE,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAC1B,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,OAAO,MAAM,CAAC,CAAC,cAAc,MAAM,CAAC,CAAC,MAAM,CAAC,MAAM,UAAU,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC9G,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,aAAa,EAAE,CAAC;YAChC,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,KAAK,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACvF,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,KAAK,eAAe,KAAK,EAAE,CAAC,CAAC;QACpD,CAAC;IACH,CAAC,CAAC,CAAC;IACH,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}

package/dist/eval/improver/flakeDetectionCli.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Flake detection CLI — print same-sha score disagreements across corpus runs
+ * (docs/AGENT_EVAL_LOOP.md §9). VM-free.
+ *
+ *   tsx src/eval/improver/flakeDetectionCli.ts [--json]
+ */
+export {};
+//# sourceMappingURL=flakeDetectionCli.d.ts.map

package/dist/eval/improver/flakeDetectionCli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"flakeDetectionCli.d.ts","sourceRoot":"","sources":["../../../src/eval/improver/flakeDetectionCli.ts"],"names":[],"mappings":"AAAA;;;;;GAKG"}

package/dist/eval/improver/flakeDetectionCli.js

@@ -0,0 +1,27 @@
+/**
+ * Flake detection CLI — print same-sha score disagreements across corpus runs
+ * (docs/AGENT_EVAL_LOOP.md §9). VM-free.
+ *
+ *   tsx src/eval/improver/flakeDetectionCli.ts [--json]
+ */
+import * as path from 'path';
+import { fileURLToPath } from 'url';
+import { detectFlakeCandidates, renderFlakeCandidates } from './flakeDetection.js';
+import { loadJsonRecords } from './corpusIO.js';
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+const REPO_ROOT = path.resolve(HERE, '..', '..', '..');
+function loadRuns() {
+    const dir = path.join(REPO_ROOT, 'eval', 'corpus', 'runs');
+    return loadJsonRecords(dir, (r) => r != null && typeof r === 'object' && typeof r.run_id === 'string');
+}
+const asJson = process.argv.includes('--json');
+const runs = loadRuns();
+const candidates = detectFlakeCandidates(runs);
+if (asJson) {
+    console.log(JSON.stringify(candidates, null, 2));
+}
+else {
+    console.log(`Loaded ${runs.length} corpus run(s).\n`);
+    console.log(renderFlakeCandidates(candidates));
+}
+//# sourceMappingURL=flakeDetectionCli.js.map

package/dist/eval/improver/flakeDetectionCli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"flakeDetectionCli.js","sourceRoot":"","sources":["../../../src/eval/improver/flakeDetectionCli.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AACpC,OAAO,EAAE,qBAAqB,EAAE,qBAAqB,EAAuB,MAAM,qBAAqB,CAAC;AACxG,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAEhD,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAC1D,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AAEvD,SAAS,QAAQ;IACf,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC3D,OAAO,eAAe,CACpB,GAAG,EACH,CAAC,CAAC,EAAuB,EAAE,CAAC,CAAC,IAAI,IAAI,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAQ,CAAoB,CAAC,MAAM,KAAK,QAAQ,CACnH,CAAC;AACJ,CAAC;AAED,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;AAC/C,MAAM,IAAI,GAAG,QAAQ,EAAE,CAAC;AACxB,MAAM,UAAU,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC;AAE/C,IAAI,MAAM,EAAE,CAAC;IACX,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AACnD,CAAC;KAAM,CAAC;IACN,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,CAAC,MAAM,mBAAmB,CAAC,CAAC;IACtD,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,UAAU,CAAC,CAAC,CAAC;AACjD,CAAC"}

package/dist/eval/improver/heldout.d.ts

@@ -0,0 +1,44 @@
+/**
+ * Train/holdout split + anti-overfitting regression gate (docs/AGENT_EVAL_LOOP.md §10).
+ *
+ * A fix is accepted only if **holdout** scores do not regress — improvements
+ * tuned to the cases they were derived from (train) don't count. New cases enter
+ * the holdout set first, so a fix cannot be overfit to them.
+ *
+ * Pure + VM-free: callers supply already-scored cases (from the oracle / corpus).
+ */
+export type Split = 'train' | 'holdout';
+export interface ScoredCase {
+    caseId: string;
+    split: Split;
+    score: {
+        build: number;
+        bp_clean: number;
+        golden_match: number;
+    };
+}
+export interface SplitAggregate {
+    count: number;
+    /** Fractions in [0,1]. */
+    pass_at_build: number;
+    pass_at_bp_clean: number;
+    pass_at_golden: number;
+}
+export declare function aggregate(cases: ScoredCase[]): SplitAggregate;
+export declare function aggregateBySplit(cases: ScoredCase[]): Record<Split, SplitAggregate>;
+export interface RegressionResult {
+    ok: boolean;
+    regressions: Array<{
+        metric: keyof Omit<SplitAggregate, 'count'>;
+        baseline: number;
+        candidate: number;
+    }>;
+}
+/**
+ * Compare a candidate holdout aggregate against a baseline. Fails if any metric
+ * drops by more than `epsilon` (default 0 — no regression tolerated). A candidate
+ * that improves or ties on every metric passes.
+ */
+export declare function holdoutRegressed(baseline: SplitAggregate, candidate: SplitAggregate, epsilon?: number): RegressionResult;
+export declare function renderSplitReport(agg: Record<Split, SplitAggregate>): string;
+//# sourceMappingURL=heldout.d.ts.map

package/dist/eval/improver/heldout.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"heldout.d.ts","sourceRoot":"","sources":["../../../src/eval/improver/heldout.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,MAAM,MAAM,KAAK,GAAG,OAAO,GAAG,SAAS,CAAC;AAExC,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,KAAK,CAAC;IACb,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC;CAClE;AAED,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,0BAA0B;IAC1B,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;CACxB;AAMD,wBAAgB,SAAS,CAAC,KAAK,EAAE,UAAU,EAAE,GAAG,cAAc,CAQ7D;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,UAAU,EAAE,GAAG,MAAM,CAAC,KAAK,EAAE,cAAc,CAAC,CAKnF;AAED,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,OAAO,CAAC;IACZ,WAAW,EAAE,KAAK,CAAC;QAAE,MAAM,EAAE,MAAM,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC1G;AAMD;;;;GAIG;AACH,wBAAgB,gBAAgB,CAC9B,QAAQ,EAAE,cAAc,EACxB,SAAS,EAAE,cAAc,EACzB,OAAO,SAAI,GACV,gBAAgB,CAQlB;AAED,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,MAAM,CAAC,KAAK,EAAE,cAAc,CAAC,GAAG,MAAM,CAI5E"}

package/dist/eval/improver/heldout.js

@@ -0,0 +1,52 @@
+/**
+ * Train/holdout split + anti-overfitting regression gate (docs/AGENT_EVAL_LOOP.md §10).
+ *
+ * A fix is accepted only if **holdout** scores do not regress — improvements
+ * tuned to the cases they were derived from (train) don't count. New cases enter
+ * the holdout set first, so a fix cannot be overfit to them.
+ *
+ * Pure + VM-free: callers supply already-scored cases (from the oracle / corpus).
+ */
+function frac(n, d) {
+    return d === 0 ? 0 : n / d;
+}
+export function aggregate(cases) {
+    const count = cases.length;
+    return {
+        count,
+        pass_at_build: frac(cases.filter(c => c.score.build === 1).length, count),
+        pass_at_bp_clean: frac(cases.filter(c => c.score.bp_clean === 1).length, count),
+        pass_at_golden: frac(cases.filter(c => c.score.golden_match === 1).length, count),
+    };
+}
+export function aggregateBySplit(cases) {
+    return {
+        train: aggregate(cases.filter(c => c.split === 'train')),
+        holdout: aggregate(cases.filter(c => c.split === 'holdout')),
+    };
+}
+const METRICS = [
+    'pass_at_build', 'pass_at_bp_clean', 'pass_at_golden',
+];
+/**
+ * Compare a candidate holdout aggregate against a baseline. Fails if any metric
+ * drops by more than `epsilon` (default 0 — no regression tolerated). A candidate
+ * that improves or ties on every metric passes.
+ */
+export function holdoutRegressed(baseline, candidate, epsilon = 0) {
+    const regressions = [];
+    for (const m of METRICS) {
+        if (candidate[m] < baseline[m] - epsilon) {
+            regressions.push({ metric: m, baseline: baseline[m], candidate: candidate[m] });
+        }
+    }
+    return { ok: regressions.length === 0, regressions };
+}
+export function renderSplitReport(agg) {
+    const row = (name, a) => `  ${name.padEnd(8)} n=${a.count}  build=${pct(a.pass_at_build)}  bp=${pct(a.pass_at_bp_clean)}  golden=${pct(a.pass_at_golden)}`;
+    return ['# Scores by split', row('train', agg.train), row('holdout', agg.holdout)].join('\n');
+}
+function pct(f) {
+    return `${Math.round(f * 100)}%`;
+}
+//# sourceMappingURL=heldout.js.map

package/dist/eval/improver/heldout.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"heldout.js","sourceRoot":"","sources":["../../../src/eval/improver/heldout.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAkBH,SAAS,IAAI,CAAC,CAAS,EAAE,CAAS;IAChC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,KAAmB;IAC3C,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;IAC3B,OAAO;QACL,KAAK;QACL,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC;QACzE,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC;QAC/E,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC;KAClF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,KAAmB;IAClD,OAAO;QACL,KAAK,EAAE,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,OAAO,CAAC,CAAC;QACxD,OAAO,EAAE,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;KAC7D,CAAC;AACJ,CAAC;AAOD,MAAM,OAAO,GAA+C;IAC1D,eAAe,EAAE,kBAAkB,EAAE,gBAAgB;CACtD,CAAC;AAEF;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAC9B,QAAwB,EACxB,SAAyB,EACzB,OAAO,GAAG,CAAC;IAEX,MAAM,WAAW,GAAoC,EAAE,CAAC;IACxD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IAAI,SAAS,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC;YACzC,WAAW,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAClF,CAAC;IACH,CAAC;IACD,OAAO,EAAE,EAAE,EAAE,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,WAAW,EAAE,CAAC;AACvD,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,GAAkC;IAClE,MAAM,GAAG,GAAG,CAAC,IAAY,EAAE,CAAiB,EAAE,EAAE,CAC9C,KAAK,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,WAAW,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,gBAAgB,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,cAAc,CAAC,EAAE,CAAC;IACpI,OAAO,CAAC,mBAAmB,EAAE,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAChG,CAAC;AAED,SAAS,GAAG,CAAC,CAAS;IACpB,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC;AACnC,CAAC"}

package/dist/eval/improver/knowledgeFeedback.d.ts

@@ -0,0 +1,56 @@
+/**
+ * Knowledge-base feedback channel (docs/AGENT_EVAL_LOOP.md §9, "Self-improving
+ * upgrades"). VM-free.
+ *
+ * MODEL_ERROR clusters are NOT tool defects — `clusterRuns()` deliberately
+ * excludes them from the actionable (code-fix) set. But a recurring
+ * MODEL_ERROR is still signal: if the same mistake shows up across multiple
+ * runs, the agent is plausibly missing (or being misled by) a rule that
+ * belongs in the X++ knowledge base (`src/tools/xppKnowledge.ts`), not a tool
+ * bug. This module clusters MODEL_ERROR runs the same way the code-fix path
+ * clusters TOOL_DEFECT/KNOWLEDGE_GAP/VALIDATOR_GAP runs, then proposes which
+ * existing knowledge topic (by keyword overlap) is the best edit target — or
+ * flags that no existing topic matches, meaning a new one is needed.
+ *
+ * This module only PROPOSES; it never edits xppKnowledge.ts itself — unlike a
+ * code fix (which is mechanically verifiable via a regression test), an
+ * instruction/knowledge edit changes agent *behaviour* and needs a human (or
+ * the improver agent, with explicit review) to judge whether the proposed
+ * rule is actually correct before it lands.
+ */
+import { type CorpusRun, type Cluster } from './cluster.js';
+export interface KnowledgeEntryLike {
+    id: string;
+    title: string;
+    keywords: string[];
+}
+export interface KnowledgeProposal {
+    cluster: Cluster;
+    /** Best-matching existing topic, or null if nothing scored above zero. */
+    suggestedTopicId: string | null;
+    suggestedTopicTitle: string | null;
+    matchScore: number;
+    /** True when no existing topic matched — this cluster likely needs a new entry. */
+    isNewTopic: boolean;
+}
+/** MODEL_ERROR clusters, ranked the same way as the code-fix clusters (frequency × tier_weight). */
+export declare function modelErrorClusters(runs: CorpusRun[]): Cluster[];
+/**
+ * Score a symptom string against one knowledge entry's keyword list: each
+ * keyword that appears as a substring of a symptom token (or vice versa)
+ * scores 1. Mirrors the simple substring-overlap approach xppKnowledge.ts's
+ * own searchKnowledge() uses, so proposals are consistent with what a live
+ * get_knowledge(topic=...) call would actually surface.
+ */
+export declare function scoreTopicMatch(symptom: string, entry: KnowledgeEntryLike): number;
+/** Best-matching knowledge entry for a symptom, or null if no entry scores above zero. */
+export declare function suggestKnowledgeTopic(symptom: string, knowledgeBase: KnowledgeEntryLike[]): {
+    id: string;
+    title: string;
+    score: number;
+} | null;
+/** Build one proposal per MODEL_ERROR cluster, ranked by cluster priority. */
+export declare function buildKnowledgeProposals(runs: CorpusRun[], knowledgeBase: KnowledgeEntryLike[]): KnowledgeProposal[];
+/** Render proposals as a short, human-readable report. */
+export declare function renderKnowledgeProposals(proposals: KnowledgeProposal[]): string;
+//# sourceMappingURL=knowledgeFeedback.d.ts.map

package/dist/eval/improver/knowledgeFeedback.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"knowledgeFeedback.d.ts","sourceRoot":"","sources":["../../../src/eval/improver/knowledgeFeedback.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAe,KAAK,SAAS,EAAE,KAAK,OAAO,EAAE,MAAM,cAAc,CAAC;AAEzE,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,OAAO,CAAC;IACjB,0EAA0E;IAC1E,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,UAAU,EAAE,MAAM,CAAC;IACnB,mFAAmF;IACnF,UAAU,EAAE,OAAO,CAAC;CACrB;AAED,oGAAoG;AACpG,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,SAAS,EAAE,GAAG,OAAO,EAAE,CAE/D;AAOD;;;;;;GAMG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,kBAAkB,GAAG,MAAM,CAQlF;AAED,0FAA0F;AAC1F,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,MAAM,EACf,aAAa,EAAE,kBAAkB,EAAE,GAClC;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CASrD;AAED,8EAA8E;AAC9E,wBAAgB,uBAAuB,CACrC,IAAI,EAAE,SAAS,EAAE,EACjB,aAAa,EAAE,kBAAkB,EAAE,GAClC,iBAAiB,EAAE,CAWrB;AAED,0DAA0D;AAC1D,wBAAgB,wBAAwB,CAAC,SAAS,EAAE,iBAAiB,EAAE,GAAG,MAAM,CAa/E"}

package/dist/eval/improver/knowledgeFeedback.js

@@ -0,0 +1,88 @@
+/**
+ * Knowledge-base feedback channel (docs/AGENT_EVAL_LOOP.md §9, "Self-improving
+ * upgrades"). VM-free.
+ *
+ * MODEL_ERROR clusters are NOT tool defects — `clusterRuns()` deliberately
+ * excludes them from the actionable (code-fix) set. But a recurring
+ * MODEL_ERROR is still signal: if the same mistake shows up across multiple
+ * runs, the agent is plausibly missing (or being misled by) a rule that
+ * belongs in the X++ knowledge base (`src/tools/xppKnowledge.ts`), not a tool
+ * bug. This module clusters MODEL_ERROR runs the same way the code-fix path
+ * clusters TOOL_DEFECT/KNOWLEDGE_GAP/VALIDATOR_GAP runs, then proposes which
+ * existing knowledge topic (by keyword overlap) is the best edit target — or
+ * flags that no existing topic matches, meaning a new one is needed.
+ *
+ * This module only PROPOSES; it never edits xppKnowledge.ts itself — unlike a
+ * code fix (which is mechanically verifiable via a regression test), an
+ * instruction/knowledge edit changes agent *behaviour* and needs a human (or
+ * the improver agent, with explicit review) to judge whether the proposed
+ * rule is actually correct before it lands.
+ */
+import { clusterRuns } from './cluster.js';
+/** MODEL_ERROR clusters, ranked the same way as the code-fix clusters (frequency × tier_weight). */
+export function modelErrorClusters(runs) {
+    return clusterRuns(runs, true).filter(c => c.classification === 'MODEL_ERROR');
+}
+/** Tokenize free text into lowercase words of length > 2 (drops stopword-ish noise). */
+function tokenize(text) {
+    return text.toLowerCase().split(/[^a-z0-9]+/).filter(w => w.length > 2);
+}
+/**
+ * Score a symptom string against one knowledge entry's keyword list: each
+ * keyword that appears as a substring of a symptom token (or vice versa)
+ * scores 1. Mirrors the simple substring-overlap approach xppKnowledge.ts's
+ * own searchKnowledge() uses, so proposals are consistent with what a live
+ * get_knowledge(topic=...) call would actually surface.
+ */
+export function scoreTopicMatch(symptom, entry) {
+    const words = tokenize(symptom);
+    let score = 0;
+    for (const kw of entry.keywords) {
+        const kwLower = kw.toLowerCase();
+        if (words.some(w => kwLower.includes(w) || w.includes(kwLower)))
+            score += 1;
+    }
+    return score;
+}
+/** Best-matching knowledge entry for a symptom, or null if no entry scores above zero. */
+export function suggestKnowledgeTopic(symptom, knowledgeBase) {
+    let best = null;
+    for (const entry of knowledgeBase) {
+        const score = scoreTopicMatch(symptom, entry);
+        if (score > 0 && (!best || score > best.score)) {
+            best = { id: entry.id, title: entry.title, score };
+        }
+    }
+    return best;
+}
+/** Build one proposal per MODEL_ERROR cluster, ranked by cluster priority. */
+export function buildKnowledgeProposals(runs, knowledgeBase) {
+    return modelErrorClusters(runs).map(cluster => {
+        const match = suggestKnowledgeTopic(cluster.symptom, knowledgeBase);
+        return {
+            cluster,
+            suggestedTopicId: match?.id ?? null,
+            suggestedTopicTitle: match?.title ?? null,
+            matchScore: match?.score ?? 0,
+            isNewTopic: match === null,
+        };
+    });
+}
+/** Render proposals as a short, human-readable report. */
+export function renderKnowledgeProposals(proposals) {
+    if (proposals.length === 0)
+        return 'No MODEL_ERROR clusters — nothing to propose. 🎉';
+    const lines = [`# Knowledge-base feedback proposals (${proposals.length})\n`];
+    proposals.forEach((p, i) => {
+        lines.push(`${i + 1}. ${p.cluster.symptom}`);
+        lines.push(`   freq=${p.cluster.frequency}  cases=${p.cluster.caseIds.join(', ')}`);
+        if (p.isNewTopic) {
+            lines.push(`   → no existing topic matched; likely needs a NEW xppKnowledge.ts entry`);
+        }
+        else {
+            lines.push(`   → edit target: topic "${p.suggestedTopicId}" (${p.suggestedTopicTitle}), match score ${p.matchScore}`);
+        }
+    });
+    return lines.join('\n');
+}
+//# sourceMappingURL=knowledgeFeedback.js.map

package/dist/eval/improver/knowledgeFeedback.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"knowledgeFeedback.js","sourceRoot":"","sources":["../../../src/eval/improver/knowledgeFeedback.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,WAAW,EAAgC,MAAM,cAAc,CAAC;AAkBzE,oGAAoG;AACpG,MAAM,UAAU,kBAAkB,CAAC,IAAiB;IAClD,OAAO,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,cAAc,KAAK,aAAa,CAAC,CAAC;AACjF,CAAC;AAED,wFAAwF;AACxF,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AAC1E,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,eAAe,CAAC,OAAe,EAAE,KAAyB;IACxE,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC;IAChC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,EAAE,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;QAChC,MAAM,OAAO,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC;QACjC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YAAE,KAAK,IAAI,CAAC,CAAC;IAC9E,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,0FAA0F;AAC1F,MAAM,UAAU,qBAAqB,CACnC,OAAe,EACf,aAAmC;IAEnC,IAAI,IAAI,GAAwD,IAAI,CAAC;IACrE,KAAK,MAAM,KAAK,IAAI,aAAa,EAAE,CAAC;QAClC,MAAM,KAAK,GAAG,eAAe,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAC9C,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/C,IAAI,GAAG,EAAE,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC;QACrD,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,uBAAuB,CACrC,IAAiB,EACjB,aAAmC;IAEnC,OAAO,kBAAkB,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE;QAC5C,MAAM,KAAK,GAAG,qBAAqB,CAAC,OAAO,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QACpE,OAAO;YACL,OAAO;YACP,gBAAgB,EAAE,KAAK,EAAE,EAAE,IAAI,IAAI;YACnC,mBAAmB,EAAE,KAAK,EAAE,KAAK,IAAI,IAAI;YACzC,UAAU,EAAE,KAAK,EAAE,KAAK,IAAI,CAAC;YAC7B,UAAU,EAAE,KAAK,KAAK,IAAI;SAC3B,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,0DAA0D;AAC1D,MAAM,UAAU,wBAAwB,CAAC,SAA8B;IACrE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,kDAAkD,CAAC;IACtF,MAAM,KAAK,GAAa,CAAC,wCAAwC,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC;IACxF,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACzB,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;QAC7C,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,OAAO,CAAC,SAAS,WAAW,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACpF,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;YACjB,KAAK,CAAC,IAAI,CAAC,0EAA0E,CAAC,CAAC;QACzF,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC,gBAAgB,MAAM,CAAC,CAAC,mBAAmB,kBAAkB,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;QACxH,CAAC;IACH,CAAC,CAAC,CAAC;IACH,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}

package/dist/eval/improver/knowledgeFeedbackCli.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Knowledge-base feedback CLI — print MODEL_ERROR clusters + suggested
+ * xppKnowledge.ts edit targets (docs/AGENT_EVAL_LOOP.md §9). VM-free.
+ *
+ *   tsx src/eval/improver/knowledgeFeedbackCli.ts [--json]
+ */
+export {};
+//# sourceMappingURL=knowledgeFeedbackCli.d.ts.map

package/dist/eval/improver/knowledgeFeedbackCli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"knowledgeFeedbackCli.d.ts","sourceRoot":"","sources":["../../../src/eval/improver/knowledgeFeedbackCli.ts"],"names":[],"mappings":"AAAA;;;;;GAKG"}

package/dist/eval/improver/knowledgeFeedbackCli.js

@@ -0,0 +1,28 @@
+/**
+ * Knowledge-base feedback CLI — print MODEL_ERROR clusters + suggested
+ * xppKnowledge.ts edit targets (docs/AGENT_EVAL_LOOP.md §9). VM-free.
+ *
+ *   tsx src/eval/improver/knowledgeFeedbackCli.ts [--json]
+ */
+import * as path from 'path';
+import { fileURLToPath } from 'url';
+import { KNOWLEDGE_BASE } from '../../tools/xppKnowledge.js';
+import { buildKnowledgeProposals, renderKnowledgeProposals } from './knowledgeFeedback.js';
+import { loadJsonRecords } from './corpusIO.js';
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+const REPO_ROOT = path.resolve(HERE, '..', '..', '..');
+function loadRuns() {
+    const dir = path.join(REPO_ROOT, 'eval', 'corpus', 'runs');
+    return loadJsonRecords(dir, (r) => r != null && typeof r === 'object' && typeof r.classification === 'string');
+}
+const asJson = process.argv.includes('--json');
+const runs = loadRuns();
+const proposals = buildKnowledgeProposals(runs, KNOWLEDGE_BASE);
+if (asJson) {
+    console.log(JSON.stringify(proposals, null, 2));
+}
+else {
+    console.log(`Loaded ${runs.length} corpus run(s).\n`);
+    console.log(renderKnowledgeProposals(proposals));
+}
+//# sourceMappingURL=knowledgeFeedbackCli.js.map

package/dist/eval/improver/knowledgeFeedbackCli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"knowledgeFeedbackCli.js","sourceRoot":"","sources":["../../../src/eval/improver/knowledgeFeedbackCli.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AACpC,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,uBAAuB,EAAE,wBAAwB,EAAE,MAAM,wBAAwB,CAAC;AAE3F,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAEhD,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAC1D,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AAEvD,SAAS,QAAQ;IACf,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC3D,OAAO,eAAe,CACpB,GAAG,EACH,CAAC,CAAC,EAAkB,EAAE,CAAC,CAAC,IAAI,IAAI,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAQ,CAAe,CAAC,cAAc,KAAK,QAAQ,CACjH,CAAC;AACJ,CAAC;AAED,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;AAC/C,MAAM,IAAI,GAAG,QAAQ,EAAE,CAAC;AACxB,MAAM,SAAS,GAAG,uBAAuB,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;AAEhE,IAAI,MAAM,EAAE,CAAC;IACX,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AAClD,CAAC;KAAM,CAAC;IACN,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,CAAC,MAAM,mBAAmB,CAAC,CAAC;IACtD,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,SAAS,CAAC,CAAC,CAAC;AACnD,CAAC"}

package/dist/eval/improver/report.d.ts

@@ -0,0 +1,36 @@
+/**
+ * Corpus scoreboard (docs/AGENT_EVAL_LOOP.md §7): aggregate run records into
+ * per-tier pass-rates and the headline tool-defect rate, tracked over the catalog.
+ * Pure + VM-free.
+ */
+export interface RunForReport {
+    case_id: string;
+    tier: number;
+    classification: string;
+    score?: {
+        build?: number;
+        bp_clean?: number;
+        golden_match?: number;
+        tier_weight?: number;
+    };
+}
+export interface TierStats {
+    tier: number;
+    count: number;
+    pass_at_build: number;
+    pass_at_bp_clean: number;
+    pass_at_golden: number;
+}
+export interface Report {
+    total: number;
+    byTier: TierStats[];
+    /** Fraction of runs whose class is an actionable server gap (the headline metric). */
+    toolDefectRate: number;
+    pass_at_build: number;
+    pass_at_bp_clean: number;
+    pass_at_golden: number;
+    classificationCounts: Record<string, number>;
+}
+export declare function buildReport(runs: RunForReport[]): Report;
+export declare function renderReport(r: Report): string;
+//# sourceMappingURL=report.d.ts.map

package/dist/eval/improver/report.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../../src/eval/improver/report.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,CAAC;IACvB,KAAK,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAC5F;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,MAAM;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,SAAS,EAAE,CAAC;IACpB,sFAAsF;IACtF,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC9C;AAkBD,wBAAgB,WAAW,CAAC,IAAI,EAAE,YAAY,EAAE,GAAG,MAAM,CAoBxD;AAMD,wBAAgB,YAAY,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAkB9C"}

package/dist/eval/improver/report.js

@@ -0,0 +1,61 @@
+/**
+ * Corpus scoreboard (docs/AGENT_EVAL_LOOP.md §7): aggregate run records into
+ * per-tier pass-rates and the headline tool-defect rate, tracked over the catalog.
+ * Pure + VM-free.
+ */
+/** Classes that count as an actionable server gap for the tool-defect rate. */
+const ACTIONABLE = new Set(['TOOL_DEFECT', 'KNOWLEDGE_GAP', 'VALIDATOR_GAP']);
+function frac(n, d) {
+    return d === 0 ? 0 : n / d;
+}
+function passRates(runs) {
+    const n = runs.length;
+    return {
+        pass_at_build: frac(runs.filter(r => r.score?.build === 1).length, n),
+        pass_at_bp_clean: frac(runs.filter(r => r.score?.bp_clean === 1).length, n),
+        pass_at_golden: frac(runs.filter(r => r.score?.golden_match === 1).length, n),
+    };
+}
+export function buildReport(runs) {
+    const tiers = [...new Set(runs.map(r => r.tier))].sort((a, b) => a - b);
+    const byTier = tiers.map(tier => {
+        const subset = runs.filter(r => r.tier === tier);
+        return { tier, count: subset.length, ...passRates(subset) };
+    });
+    const classificationCounts = {};
+    for (const r of runs) {
+        classificationCounts[r.classification] = (classificationCounts[r.classification] ?? 0) + 1;
+    }
+    const actionable = runs.filter(r => ACTIONABLE.has(r.classification)).length;
+    return {
+        total: runs.length,
+        byTier,
+        toolDefectRate: frac(actionable, runs.length),
+        ...passRates(runs),
+        classificationCounts,
+    };
+}
+function pct(f) {
+    return `${Math.round(f * 100)}%`;
+}
+export function renderReport(r) {
+    if (r.total === 0)
+        return 'No corpus runs to report.';
+    const lines = [
+        `# Corpus scoreboard — ${r.total} run(s)`,
+        '',
+        `overall   build=${pct(r.pass_at_build)}  bp=${pct(r.pass_at_bp_clean)}  golden=${pct(r.pass_at_golden)}`,
+        `tool-defect rate: ${pct(r.toolDefectRate)} (TOOL_DEFECT/KNOWLEDGE_GAP/VALIDATOR_GAP)`,
+        '',
+        '## By tier',
+    ];
+    for (const t of r.byTier) {
+        lines.push(`  L${t.tier}  n=${t.count}  build=${pct(t.pass_at_build)}  bp=${pct(t.pass_at_bp_clean)}  golden=${pct(t.pass_at_golden)}`);
+    }
+    lines.push('', '## Classifications');
+    for (const [cls, n] of Object.entries(r.classificationCounts).sort((a, b) => b[1] - a[1])) {
+        lines.push(`  ${cls.padEnd(14)} ${n}`);
+    }
+    return lines.join('\n');
+}
+//# sourceMappingURL=report.js.map

package/dist/eval/improver/report.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"report.js","sourceRoot":"","sources":["../../../src/eval/improver/report.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AA4BH,+EAA+E;AAC/E,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,CAAC,aAAa,EAAE,eAAe,EAAE,eAAe,CAAC,CAAC,CAAC;AAE9E,SAAS,IAAI,CAAC,CAAS,EAAE,CAAS;IAChC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,SAAS,CAAC,IAAoB;IACrC,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;IACtB,OAAO;QACL,aAAa,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;QACrE,gBAAgB,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,QAAQ,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;QAC3E,cAAc,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,YAAY,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;KAC9E,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,IAAoB;IAC9C,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACxE,MAAM,MAAM,GAAgB,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;QAC3C,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;QACjD,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;IAC9D,CAAC,CAAC,CAAC;IAEH,MAAM,oBAAoB,GAA2B,EAAE,CAAC;IACxD,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,oBAAoB,CAAC,CAAC,CAAC,cAAc,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IAC7F,CAAC;IACD,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC;IAE7E,OAAO;QACL,KAAK,EAAE,IAAI,CAAC,MAAM;QAClB,MAAM;QACN,cAAc,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC;QAC7C,GAAG,SAAS,CAAC,IAAI,CAAC;QAClB,oBAAoB;KACrB,CAAC;AACJ,CAAC;AAED,SAAS,GAAG,CAAC,CAAS;IACpB,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC;AACnC,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,CAAS;IACpC,IAAI,CAAC,CAAC,KAAK,KAAK,CAAC;QAAE,OAAO,2BAA2B,CAAC;IACtD,MAAM,KAAK,GAAa;QACtB,yBAAyB,CAAC,CAAC,KAAK,SAAS;QACzC,EAAE;QACF,mBAAmB,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,gBAAgB,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,cAAc,CAAC,EAAE;QACzG,qBAAqB,GAAG,CAAC,CAAC,CAAC,cAAc,CAAC,4CAA4C;QACtF,EAAE;QACF,YAAY;KACb,CAAC;IACF,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,KAAK,WAAW,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,gBAAgB,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;IAC1I,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,oBAAoB,CAAC,CAAC;IACrC,KAAK,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1F,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzC,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}

package/dist/eval/improver/reportCli.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Corpus scoreboard CLI — `npm run eval:report [--json]`.
+ * Aggregates every eval/corpus/runs/*.json (latest run per case) into per-tier
+ * pass-rates and the headline tool-defect rate.
+ */
+export {};
+//# sourceMappingURL=reportCli.d.ts.map

package/dist/eval/improver/reportCli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reportCli.d.ts","sourceRoot":"","sources":["../../../src/eval/improver/reportCli.ts"],"names":[],"mappings":"AAAA;;;;GAIG"}

package/dist/eval/improver/reportCli.js

@@ -0,0 +1,27 @@
+/**
+ * Corpus scoreboard CLI — `npm run eval:report [--json]`.
+ * Aggregates every eval/corpus/runs/*.json (latest run per case) into per-tier
+ * pass-rates and the headline tool-defect rate.
+ */
+import * as path from 'path';
+import { fileURLToPath } from 'url';
+import { buildReport, renderReport } from './report.js';
+import { loadJsonRecords } from './corpusIO.js';
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+const REPO_ROOT = path.resolve(HERE, '..', '..', '..');
+function loadLatestPerCase() {
+    const dir = path.join(REPO_ROOT, 'eval', 'corpus', 'runs');
+    const records = loadJsonRecords(dir, (r) => r != null && typeof r === 'object' &&
+        !!r.case_id && typeof r.classification === 'string');
+    const latest = new Map();
+    for (const r of records) {
+        const prev = latest.get(r.case_id);
+        if (!prev || r.run_id > prev.run_id)
+            latest.set(r.case_id, r);
+    }
+    return [...latest.values()];
+}
+const asJson = process.argv.includes('--json');
+const report = buildReport(loadLatestPerCase());
+console.log(asJson ? JSON.stringify(report, null, 2) : renderReport(report));
+//# sourceMappingURL=reportCli.js.map

package/dist/eval/improver/reportCli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reportCli.js","sourceRoot":"","sources":["../../../src/eval/improver/reportCli.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AACpC,OAAO,EAAE,WAAW,EAAE,YAAY,EAAqB,MAAM,aAAa,CAAC;AAC3E,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAEhD,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAC1D,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AAEvD,SAAS,iBAAiB;IACxB,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC3D,MAAM,OAAO,GAAG,eAAe,CAC7B,GAAG,EACH,CAAC,CAAC,EAA0C,EAAE,CAC5C,CAAC,IAAI,IAAI,IAAI,OAAO,CAAC,KAAK,QAAQ;QAClC,CAAC,CAAE,CAAS,CAAC,OAAO,IAAI,OAAQ,CAAS,CAAC,cAAc,KAAK,QAAQ,CACxE,CAAC;IACF,MAAM,MAAM,GAAG,IAAI,GAAG,EAA6C,CAAC;IACpE,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QACnC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM;YAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IAChE,CAAC;IACD,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;AAC9B,CAAC;AAED,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;AAC/C,MAAM,MAAM,GAAG,WAAW,CAAC,iBAAiB,EAAE,CAAC,CAAC;AAChD,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC"}

package/dist/eval/oracle/actualArtifactResolution.d.ts

@@ -0,0 +1,49 @@
+/**
+ * Multi-artifact (`--actual-dir`) actual-file resolution for src/eval/oracle/cli.ts.
+ * Pure logic + fs reads — no CLI argv/process side effects — split out so it can be
+ * unit-tested without triggering the CLI script's `main()` (docs/AGENT_EVAL_LOOP.md §6).
+ */
+/**
+ * Resolve the actual-dir file matching a golden artifact filename. Tries an
+ * exact filename match first (fast path, and the only path when golden and
+ * actual happen to share the same EXTENSION_PREFIX session); if that misses,
+ * falls back to matching on the PREFIX-CANONICALISED filename (the golden's
+ * filename is itself typically a prefixed object name, e.g.
+ * "ContosoMyContract.metadata.xml", produced under a different session than the
+ * one that generated the actual artifacts) so a whole L3/L4 multi-artifact
+ * case doesn't spuriously score every artifact as missing/extra under prefix
+ * drift alone.
+ */
+export declare function resolveActualFile(actualDir: string, goldenName: string, goldenPrefix: string, actualPrefix: string): string | undefined;
+/**
+ * Build the `actualArtifacts` map for a multi-artifact (`--actual-dir`) run,
+ * one entry per golden artifact name.
+ *
+ * Regression: this used to key every entry by the GOLDEN's own filename
+ * (`actualArtifacts[name] = ...` inside a `for (const name of
+ * artifactNames)` loop) even when the resolved actual file had a DIFFERENT
+ * literal prefix (e.g. golden "ContosoMyContract.metadata.xml" resolved to actual
+ * file "DemoMyContract.metadata.xml" under prefix-agnostic matching —
+ * `resolveActualFile`'s whole point). `evaluateMulti`/`normalizeMultiArtifact`
+ * then canonicalises each artifact KEY with `actualPrefix` — but a key that's
+ * still the GOLDEN's literal name doesn't contain `actualPrefix` at all, so
+ * `canonicalizePrefix` is a no-op on it, and the golden side's key (correctly
+ * canonicalised from ITS OWN prefix) never matches. Every path in the
+ * artifact then shows up as wholesale `missing` (under the golden's canonical
+ * key) AND `extra` (under the actual's un-canonicalised key), even when the
+ * content is byte-identical. Keying by the RESOLVED actual file's own
+ * basename (which DOES contain `actualPrefix`) fixes the canonicalisation on
+ * both sides consistently — matching the documented multi-artifact contract
+ * (src/eval/oracle/normalize.ts's `normalizeMultiArtifact` doc comment).
+ *
+ * A golden artifact with NO resolvable actual file (genuinely missing, not a
+ * prefix-matching miss) keeps the golden's own name as the key with empty
+ * content — unchanged from before; there is no real actual basename to key it
+ * by, and the empty content correctly registers every one of that artifact's
+ * paths as `missing`.
+ */
+export declare function buildActualArtifactsMap(actualDir: string, artifactNames: string[], goldenPrefix: string, actualPrefix: string): {
+    actualArtifacts: Record<string, string>;
+    matchedActualFiles: Set<string>;
+};
+//# sourceMappingURL=actualArtifactResolution.d.ts.map

package/dist/eval/oracle/actualArtifactResolution.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"actualArtifactResolution.d.ts","sourceRoot":"","sources":["../../../src/eval/oracle/actualArtifactResolution.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAMH;;;;;;;;;;GAUG;AACH,wBAAgB,iBAAiB,CAC/B,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,EAClB,YAAY,EAAE,MAAM,EACpB,YAAY,EAAE,MAAM,GACnB,MAAM,GAAG,SAAS,CAQpB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,wBAAgB,uBAAuB,CACrC,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EAAE,EACvB,YAAY,EAAE,MAAM,EACpB,YAAY,EAAE,MAAM,GACnB;IAAE,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAAC,kBAAkB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;CAAE,CAc9E"}