@holmes-lab/holmes-kit 0.2.1 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/CHANGELOG.md +55 -0
  2. package/dist/.build-id +1 -1
  3. package/dist/holmes/assoc/assoc-arm.d.ts +94 -0
  4. package/dist/holmes/assoc/assoc-arm.js +187 -0
  5. package/dist/holmes/assoc/explore.d.ts +21 -0
  6. package/dist/holmes/assoc/explore.js +160 -0
  7. package/dist/holmes/assoc/impact-baseline.d.ts +13 -0
  8. package/dist/holmes/assoc/impact-baseline.js +48 -0
  9. package/dist/holmes/assoc/ppr.d.ts +64 -0
  10. package/dist/holmes/assoc/ppr.js +110 -0
  11. package/dist/holmes/cli/agents.d.ts +24 -11
  12. package/dist/holmes/cli/agents.js +93 -21
  13. package/dist/holmes/cli/doctor.d.ts +56 -0
  14. package/dist/holmes/cli/doctor.js +311 -5
  15. package/dist/holmes/cli/index.js +13 -1
  16. package/dist/holmes/cli/semantic-key.d.ts +19 -0
  17. package/dist/holmes/cli/semantic-key.js +93 -0
  18. package/dist/holmes/config/config.d.ts +9 -0
  19. package/dist/holmes/config/config.js +8 -1
  20. package/dist/holmes/cpg/language-capability.d.ts +65 -0
  21. package/dist/holmes/cpg/language-capability.js +145 -0
  22. package/dist/holmes/cpg/language-parser-walk.js +179 -34
  23. package/dist/holmes/cpg/language-parser.d.ts +1 -1
  24. package/dist/holmes/governance/ledger-rechain.d.ts +12 -0
  25. package/dist/holmes/governance/ledger-rechain.js +17 -2
  26. package/dist/holmes/governance/provenance-ledger.js +21 -0
  27. package/dist/holmes/guardrail/blind-spots.js +12 -1
  28. package/dist/holmes/guardrail/impact-gate.d.ts +77 -0
  29. package/dist/holmes/guardrail/impact-gate.js +263 -0
  30. package/dist/holmes/guardrail/write-target.d.ts +38 -1
  31. package/dist/holmes/guardrail/write-target.js +41 -4
  32. package/dist/holmes/hooks/adapters/antigravity.js +12 -1
  33. package/dist/holmes/hooks/corrupt-state-run.d.ts +33 -0
  34. package/dist/holmes/hooks/corrupt-state-run.js +16 -0
  35. package/dist/holmes/hooks/pre-tool-use.d.ts +26 -0
  36. package/dist/holmes/hooks/pre-tool-use.js +203 -11
  37. package/dist/holmes/hooks/stop.d.ts +65 -0
  38. package/dist/holmes/hooks/stop.js +200 -2
  39. package/dist/holmes/mcp/elicit-approval.d.ts +32 -2
  40. package/dist/holmes/mcp/elicit-approval.js +30 -2
  41. package/dist/holmes/mcp/handlers.d.ts +88 -0
  42. package/dist/holmes/mcp/handlers.js +636 -35
  43. package/dist/holmes/mcp/maintenance-analyze.d.ts +435 -0
  44. package/dist/holmes/mcp/maintenance-analyze.js +994 -0
  45. package/dist/holmes/mcp/maintenance-evidence.d.ts +140 -0
  46. package/dist/holmes/mcp/maintenance-evidence.js +253 -0
  47. package/dist/holmes/mcp/server.js +13 -10
  48. package/dist/holmes/mcp/tool-schemas.js +71 -0
  49. package/dist/holmes/project/root.js +3 -1
  50. package/dist/holmes/review/baseline-arm.d.ts +37 -0
  51. package/dist/holmes/review/baseline-arm.js +51 -0
  52. package/dist/holmes/review/captured-stdin-guard.d.ts +8 -0
  53. package/dist/holmes/review/captured-stdin-guard.js +48 -0
  54. package/dist/holmes/review/coherence-verify.d.ts +31 -0
  55. package/dist/holmes/review/coherence-verify.js +144 -0
  56. package/dist/holmes/review/commit-text.d.ts +50 -0
  57. package/dist/holmes/review/commit-text.js +76 -0
  58. package/dist/holmes/review/confidence-calibration.d.ts +39 -0
  59. package/dist/holmes/review/confidence-calibration.js +39 -0
  60. package/dist/holmes/review/content-baseline.d.ts +38 -0
  61. package/dist/holmes/review/content-baseline.js +103 -0
  62. package/dist/holmes/review/content-verify.d.ts +20 -0
  63. package/dist/holmes/review/content-verify.js +73 -0
  64. package/dist/holmes/review/dense-retrieval.d.ts +66 -0
  65. package/dist/holmes/review/dense-retrieval.js +97 -0
  66. package/dist/holmes/review/edge-quality.d.ts +44 -0
  67. package/dist/holmes/review/edge-quality.js +117 -0
  68. package/dist/holmes/review/evaluation-metrics.d.ts +138 -0
  69. package/dist/holmes/review/evaluation-metrics.js +175 -0
  70. package/dist/holmes/review/graph-verifier.d.ts +34 -0
  71. package/dist/holmes/review/graph-verifier.js +62 -0
  72. package/dist/holmes/review/hop-ablation.d.ts +100 -0
  73. package/dist/holmes/review/hop-ablation.js +89 -0
  74. package/dist/holmes/review/manual-baseline.d.ts +209 -0
  75. package/dist/holmes/review/manual-baseline.js +2846 -0
  76. package/dist/holmes/review/oracle-gap.d.ts +32 -0
  77. package/dist/holmes/review/oracle-gap.js +102 -0
  78. package/dist/holmes/review/point-in-time-replay.d.ts +41 -0
  79. package/dist/holmes/review/point-in-time-replay.js +161 -0
  80. package/dist/holmes/review/rank-diagnosis.d.ts +43 -0
  81. package/dist/holmes/review/rank-diagnosis.js +163 -0
  82. package/dist/holmes/review/replay-calibration.d.ts +62 -0
  83. package/dist/holmes/review/replay-calibration.js +83 -0
  84. package/dist/holmes/review/replay-corpus.d.ts +135 -0
  85. package/dist/holmes/review/replay-corpus.js +210 -0
  86. package/dist/holmes/review/run-replay.d.ts +260 -0
  87. package/dist/holmes/review/run-replay.js +729 -0
  88. package/dist/holmes/review/semantic-arm.d.ts +271 -0
  89. package/dist/holmes/review/semantic-arm.js +717 -0
  90. package/dist/holmes/review/semantic-retrieval.d.ts +55 -0
  91. package/dist/holmes/review/semantic-retrieval.js +156 -0
  92. package/dist/holmes/review/spec-layer-stats.d.ts +38 -0
  93. package/dist/holmes/review/spec-layer-stats.js +52 -0
  94. package/dist/holmes/review/temporal-prior.d.ts +33 -0
  95. package/dist/holmes/review/temporal-prior.js +53 -0
  96. package/dist/holmes/review/test-runner.d.ts +15 -0
  97. package/dist/holmes/review/test-runner.js +41 -4
  98. package/dist/holmes/review/test-selection-breadth.d.ts +75 -0
  99. package/dist/holmes/review/test-selection-breadth.js +57 -0
  100. package/dist/holmes/review/traceability-benchmark.d.ts +81 -0
  101. package/dist/holmes/review/traceability-benchmark.js +135 -0
  102. package/dist/holmes/review/union-verify.d.ts +12 -0
  103. package/dist/holmes/review/union-verify.js +70 -0
  104. package/dist/holmes/rtm/graph-store.d.ts +51 -0
  105. package/dist/holmes/rtm/graph-store.js +122 -0
  106. package/dist/holmes/rtm/incremental.d.ts +25 -1
  107. package/dist/holmes/rtm/incremental.js +18 -1
  108. package/dist/holmes/rtm/localize.d.ts +28 -0
  109. package/dist/holmes/rtm/localize.js +272 -14
  110. package/dist/holmes/rtm/rtm-builder.d.ts +118 -3
  111. package/dist/holmes/rtm/rtm-builder.js +265 -28
  112. package/dist/holmes/rtm/rtm-graph.d.ts +117 -2
  113. package/dist/holmes/rtm/rtm-graph.js +194 -31
  114. package/dist/holmes/rtm/taint-benchmark.d.ts +97 -0
  115. package/dist/holmes/rtm/taint-benchmark.js +141 -0
  116. package/dist/holmes/rtm/test-scope.js +24 -1
  117. package/dist/holmes/semantic/credentials.d.ts +15 -0
  118. package/dist/holmes/semantic/credentials.js +134 -0
  119. package/dist/holmes/semantic/embedder.d.ts +44 -0
  120. package/dist/holmes/semantic/embedder.js +185 -0
  121. package/dist/holmes/semantic/hit-rerank.d.ts +4 -0
  122. package/dist/holmes/semantic/hit-rerank.js +38 -0
  123. package/dist/holmes/semantic/tier.d.ts +37 -0
  124. package/dist/holmes/semantic/tier.js +54 -0
  125. package/dist/holmes/semantic/vector-cache.d.ts +11 -0
  126. package/dist/holmes/semantic/vector-cache.js +91 -0
  127. package/dist/holmes/spec/acceptance-quality.d.ts +81 -0
  128. package/dist/holmes/spec/acceptance-quality.js +169 -0
  129. package/dist/holmes/spec/validator.js +33 -1
  130. package/dist/holmes/spec/yaml-scalar.d.ts +1 -0
  131. package/dist/holmes/spec/yaml-scalar.js +43 -0
  132. package/package.json +1 -1
@@ -0,0 +1,32 @@
1
+ import type { Spec } from '../spec/spec-parser';
2
+ /**
3
+ * The distance between "a test file claims this requirement" and "a test checks it".
4
+ *
5
+ * Traceability here runs from an A-SPEC to a test FILE: an `@implements` anchor makes the link and
6
+ * the constitution gate enforces it. The anchor is a line at the top of a file, and it says nothing
7
+ * about whether any assertion inside that file examines the requirement. A suite can be green, the
8
+ * anchor present, the gate satisfied, and the criterion untested.
9
+ *
10
+ * What is countable without executing anything: whether the requirement's vocabulary appears in the
11
+ * NAMES of the tests that claim it. A test whose describe and it titles share no significant word
12
+ * with the spec does not even mention what it claims to verify.
13
+ *
14
+ * **This is a lower bound and not an oracle check.** Sharing words does not mean the assertion
15
+ * examines the requirement, and sharing none does not prove it fails to — a test named
16
+ * `returns 404 for unknown ids` may verify a requirement about error handling that uses none of
17
+ * those words. What the count measures is how much of the traceability claim rests on the anchor
18
+ * alone. Calling it anything stronger would repeat the mistake this repository has made before.
19
+ */
20
+ export interface OracleGap {
21
+ /** A-SPECs with at least one anchored test file. */
22
+ claimed: number;
23
+ /** Of those, how many have NO test title sharing a significant term with the spec. */
24
+ titlesShareNothing: number;
25
+ /** Mean share of the spec's terms appearing in the titles of tests that claim it. */
26
+ meanTitleOverlap: number | null;
27
+ /** Specs whose anchored test files could not be read — unmeasured, not clean. */
28
+ unreadable: number;
29
+ }
30
+ /** Test titles in a file: the strings passed to describe/it/test. */
31
+ export declare function testTitles(source: string): string[];
32
+ export declare function oracleGap(specs: readonly Spec[], anchors: Readonly<Record<string, readonly string[]>>, root: string): OracleGap;
@@ -0,0 +1,102 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || (function () {
19
+ var ownKeys = function(o) {
20
+ ownKeys = Object.getOwnPropertyNames || function (o) {
21
+ var ar = [];
22
+ for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
23
+ return ar;
24
+ };
25
+ return ownKeys(o);
26
+ };
27
+ return function (mod) {
28
+ if (mod && mod.__esModule) return mod;
29
+ var result = {};
30
+ if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
31
+ __setModuleDefault(result, mod);
32
+ return result;
33
+ };
34
+ })();
35
+ Object.defineProperty(exports, "__esModule", { value: true });
36
+ exports.testTitles = testTitles;
37
+ exports.oracleGap = oracleGap;
38
+ // @implements A-SPEC-391
39
+ const fs = __importStar(require("node:fs"));
40
+ const path = __importStar(require("node:path"));
41
+ const localize_1 = require("../rtm/localize");
42
+ /** Test titles in a file: the strings passed to describe/it/test. */
43
+ function testTitles(source) {
44
+ const out = [];
45
+ const re = /\b(?:describe|it|test)\s*(?:\.\w+)?\s*\(\s*(['"`])((?:\\.|(?!\1)[\s\S])*)\1/g;
46
+ let m;
47
+ while ((m = re.exec(source)) !== null)
48
+ out.push(m[2]);
49
+ return out;
50
+ }
51
+ function oracleGap(specs, anchors, root) {
52
+ const filesBySpec = new Map();
53
+ for (const [file, ids] of Object.entries(anchors)) {
54
+ for (const id of ids)
55
+ (filesBySpec.get(id) ?? filesBySpec.set(id, []).get(id)).push(file);
56
+ }
57
+ let claimed = 0, shareNothing = 0, unreadable = 0, overlapSum = 0, overlapCount = 0;
58
+ for (const s of specs) {
59
+ if (s.type !== 'A-SPEC')
60
+ continue;
61
+ const files = filesBySpec.get(s.id);
62
+ if (!files || files.length === 0)
63
+ continue;
64
+ claimed++;
65
+ const specTerms = new Set((0, localize_1.significantTerms)(`${s.title} ${Object.values(s.sections).join(' ')}`));
66
+ if (specTerms.size === 0)
67
+ continue;
68
+ const titleTerms = new Set();
69
+ let read = 0;
70
+ for (const file of files) {
71
+ let source;
72
+ try {
73
+ source = fs.readFileSync(path.join(root, file), 'utf8');
74
+ }
75
+ catch {
76
+ continue;
77
+ }
78
+ read++;
79
+ for (const title of testTitles(source))
80
+ for (const t of (0, localize_1.significantTerms)(title))
81
+ titleTerms.add(t);
82
+ }
83
+ if (read === 0) {
84
+ unreadable++;
85
+ continue;
86
+ }
87
+ let shared = 0;
88
+ for (const t of specTerms)
89
+ if (titleTerms.has(t))
90
+ shared++;
91
+ if (shared === 0)
92
+ shareNothing++;
93
+ overlapSum += shared / specTerms.size;
94
+ overlapCount++;
95
+ }
96
+ return {
97
+ claimed,
98
+ titlesShareNothing: shareNothing,
99
+ meanTitleOverlap: overlapCount === 0 ? null : overlapSum / overlapCount,
100
+ unreadable,
101
+ };
102
+ }
@@ -0,0 +1,41 @@
1
+ export type Materialized = {
2
+ ok: true;
3
+ files: string[];
4
+ } | {
5
+ ok: false;
6
+ reason: string;
7
+ };
8
+ export interface Fidelity {
9
+ /** Files whose materialized bytes were compared against the commit's blob. */
10
+ checked: number;
11
+ /** Repo-relative paths whose bytes differ from the commit. Empty means the tree is faithful. */
12
+ mismatches: string[];
13
+ }
14
+ /**
15
+ * @implements A-SPEC-315
16
+ * Every failure carries a CLASSIFICATION prefix — `unsafe-pathspec`, `destination-not-empty`,
17
+ * `git-ls-tree-failed`, `empty-pathspec`, `archive-or-extract-failed` — followed by the original
18
+ * detail. Two full-suite runs went red here in two different tests and both reported only `false`;
19
+ * isolation, two clean full runs and a 320-call concurrency probe all passed, so the cause is still
20
+ * unknown. Deliberately NO retry: a retry would hide whatever is actually happening.
21
+ *
22
+ * Lay out the tree a commit recorded, WITHOUT touching the working tree or index.
23
+ *
24
+ * `git archive` reads straight from the object database, so no checkout, stash or index write is
25
+ * involved — the caller's uncommitted work is untouched, which is the property this whole slice
26
+ * exists to obtain. The alternative (checkout into the same worktree) would destroy user state.
27
+ */
28
+ export declare function materializeTreeAt(repoRoot: string, commitish: string, pathspec: string, destDir: string, opts?: {
29
+ /**
30
+ * @implements A-SPEC-303
31
+ * Permit a second pathspec from the SAME commit into the same destination — a project is its
32
+ * code AND its specs, and one scan should see both as they were. Opt-in on purpose: making it
33
+ * the default would quietly restore the mixed-tree hazard the emptiness rule exists to prevent.
34
+ */
35
+ allowNonEmpty?: boolean;
36
+ }): Materialized;
37
+ /**
38
+ * Confirm the materialized tree really is that commit. The leak this slice removes is invisible by
39
+ * construction, so it must be asserted rather than assumed: a wrong tree still produces numbers.
40
+ */
41
+ export declare function verifyTreeFidelity(repoRoot: string, commitish: string, pathspec: string, destDir: string): Fidelity;
@@ -0,0 +1,161 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || (function () {
19
+ var ownKeys = function(o) {
20
+ ownKeys = Object.getOwnPropertyNames || function (o) {
21
+ var ar = [];
22
+ for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
23
+ return ar;
24
+ };
25
+ return ownKeys(o);
26
+ };
27
+ return function (mod) {
28
+ if (mod && mod.__esModule) return mod;
29
+ var result = {};
30
+ if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
31
+ __setModuleDefault(result, mod);
32
+ return result;
33
+ };
34
+ })();
35
+ Object.defineProperty(exports, "__esModule", { value: true });
36
+ exports.materializeTreeAt = materializeTreeAt;
37
+ exports.verifyTreeFidelity = verifyTreeFidelity;
38
+ // @implements A-SPEC-297
39
+ const fs = __importStar(require("node:fs"));
40
+ const os = __importStar(require("node:os"));
41
+ const path = __importStar(require("node:path"));
42
+ const crypto = __importStar(require("node:crypto"));
43
+ const node_child_process_1 = require("node:child_process");
44
+ /**
45
+ * A pathspec is ours to control, but a caller mistake here writes outside the destination, so it is
46
+ * checked rather than trusted: repo-relative, no escape, no absolute root.
47
+ */
48
+ function pathspecIsSafe(pathspec) {
49
+ if (pathspec === '' || path.isAbsolute(pathspec))
50
+ return false;
51
+ const normalized = path.normalize(pathspec);
52
+ return !normalized.startsWith('..') && !normalized.split(path.sep).includes('..');
53
+ }
54
+ function git(repoRoot, args) {
55
+ return (0, node_child_process_1.execFileSync)('git', ['-C', repoRoot, ...args], { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 });
56
+ }
57
+ /** `path mode sha` triples the commit records for the pathspec, keyed by repo-relative path. */
58
+ function blobsAt(repoRoot, commitish, pathspec) {
59
+ const out = git(repoRoot, ['ls-tree', '-r', `${commitish}^{tree}`, '--', pathspec]);
60
+ const blobs = new Map();
61
+ for (const line of out.split('\n')) {
62
+ if (line.trim() === '')
63
+ continue;
64
+ const [meta, file] = line.split('\t');
65
+ const parts = meta.split(/\s+/);
66
+ if (parts[1] !== 'blob')
67
+ continue; // submodules and trees are not files we can compare
68
+ blobs.set(file, parts[2]);
69
+ }
70
+ return blobs;
71
+ }
72
+ /**
73
+ * @implements A-SPEC-315
74
+ * Every failure carries a CLASSIFICATION prefix — `unsafe-pathspec`, `destination-not-empty`,
75
+ * `git-ls-tree-failed`, `empty-pathspec`, `archive-or-extract-failed` — followed by the original
76
+ * detail. Two full-suite runs went red here in two different tests and both reported only `false`;
77
+ * isolation, two clean full runs and a 320-call concurrency probe all passed, so the cause is still
78
+ * unknown. Deliberately NO retry: a retry would hide whatever is actually happening.
79
+ *
80
+ * Lay out the tree a commit recorded, WITHOUT touching the working tree or index.
81
+ *
82
+ * `git archive` reads straight from the object database, so no checkout, stash or index write is
83
+ * involved — the caller's uncommitted work is untouched, which is the property this whole slice
84
+ * exists to obtain. The alternative (checkout into the same worktree) would destroy user state.
85
+ */
86
+ function materializeTreeAt(repoRoot, commitish, pathspec, destDir, opts) {
87
+ if (!pathspecIsSafe(pathspec))
88
+ return { ok: false, reason: `unsafe-pathspec: ${pathspec}` };
89
+ if (!opts?.allowNonEmpty && fs.existsSync(destDir) && fs.readdirSync(destDir).length > 0) {
90
+ // Extracting over existing files yields a tree that is partly one commit and partly another —
91
+ // and it would still score, silently and wrongly.
92
+ return { ok: false, reason: `destination-not-empty: destination is not empty: ${destDir}` };
93
+ }
94
+ fs.mkdirSync(destDir, { recursive: true });
95
+ let blobs;
96
+ try {
97
+ blobs = blobsAt(repoRoot, commitish, pathspec);
98
+ }
99
+ catch (err) {
100
+ // The original message is kept, not replaced: a classification that DISCARDS the detail makes
101
+ // diagnosis worse, not better.
102
+ return { ok: false, reason: `git-ls-tree-failed: unknown or unreadable commit-ish ${commitish}: ${err.message}` };
103
+ }
104
+ if (blobs.size === 0) {
105
+ // Zero files is not a score of zero. Nothing was measured.
106
+ return { ok: false, reason: `empty-pathspec: no files at ${commitish} for pathspec ${pathspec}` };
107
+ }
108
+ // @implements A-SPEC-329
109
+ // The tar goes through a FILE, not through tar's stdin. Feeding a multi-megabyte buffer to
110
+ // `execFileSync('tar', …, { input })` failed intermittently under full-suite load with
111
+ // `spawnSync tar EPIPE` — observed three times, and only nameable because A-SPEC-315 had added
112
+ // failure classifications instead of the retry that would have hidden it. A retry would have
113
+ // masked a fixable bug; removing the pipe removes the failure mode rather than surviving it.
114
+ //
115
+ // The scratch file lives OUTSIDE the destination: inside, it would mix with the extracted tree and
116
+ // the fidelity check would see a file the commit never had.
117
+ const tarPath = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'pit-tar-')), 'tree.tar');
118
+ try {
119
+ const tar = (0, node_child_process_1.execFileSync)('git', ['-C', repoRoot, 'archive', '--format=tar', commitish, '--', pathspec], { maxBuffer: 512 * 1024 * 1024 });
120
+ fs.writeFileSync(tarPath, tar);
121
+ (0, node_child_process_1.execFileSync)('tar', ['-x', '-C', destDir, '-f', tarPath], { maxBuffer: 512 * 1024 * 1024 });
122
+ }
123
+ catch (err) {
124
+ return { ok: false, reason: `archive-or-extract-failed: could not materialize ${commitish}: ${err.message}` };
125
+ }
126
+ finally {
127
+ // Removed on every path — a scratch file that survives a failure is litter in the temp dir.
128
+ fs.rmSync(path.dirname(tarPath), { recursive: true, force: true });
129
+ }
130
+ return { ok: true, files: [...blobs.keys()].sort() };
131
+ }
132
+ /** Git's blob id: sha1 over `blob <byteLength>\0` followed by the content. */
133
+ function blobId(content) {
134
+ return crypto.createHash('sha1').update(`blob ${content.length}\0`).update(content).digest('hex');
135
+ }
136
+ /**
137
+ * Confirm the materialized tree really is that commit. The leak this slice removes is invisible by
138
+ * construction, so it must be asserted rather than assumed: a wrong tree still produces numbers.
139
+ */
140
+ function verifyTreeFidelity(repoRoot, commitish, pathspec, destDir) {
141
+ let blobs;
142
+ try {
143
+ blobs = blobsAt(repoRoot, commitish, pathspec);
144
+ }
145
+ catch {
146
+ return { checked: 0, mismatches: [`<unreadable commit-ish: ${commitish}>`] };
147
+ }
148
+ const mismatches = [];
149
+ let checked = 0;
150
+ for (const [file, expected] of blobs) {
151
+ const onDisk = path.join(destDir, file);
152
+ checked++;
153
+ if (!fs.existsSync(onDisk)) {
154
+ mismatches.push(file);
155
+ continue;
156
+ }
157
+ if (blobId(fs.readFileSync(onDisk)) !== expected)
158
+ mismatches.push(file);
159
+ }
160
+ return { checked, mismatches: mismatches.sort() };
161
+ }
@@ -0,0 +1,43 @@
1
+ /**
2
+ * Why the shipped harness lost on recall (S-404), decomposed.
3
+ *
4
+ * "Recall is low" has two completely different causes and they call for opposite fixes: either the
5
+ * answer never became a candidate (admission — widen the pool, add a retrieval channel), or it was
6
+ * a candidate and something else outranked it (ordering — change the scoring, and widening the pool
7
+ * does nothing). Thirteen rejected retrieval mechanisms were all attempts at the first one. This
8
+ * module measures which it actually is, so the next attempt is aimed at the measured cause.
9
+ *
10
+ * On the eight blind jarvis cases of A-SPEC-404 the answer is unambiguous, and it is the second:
11
+ * every truth file was already in the pool.
12
+ */
13
+ export interface PooledTruth {
14
+ file: string;
15
+ /** Position in the full candidate pool, best first. `-1` when the file never entered it. */
16
+ rank: number;
17
+ /** Lexical/spec score before the history re-rank. Zero means only commit history reached it. */
18
+ lexical: number;
19
+ }
20
+ export interface RankDiagnosis {
21
+ truthTotal: number;
22
+ /** Answers the pipeline never proposed at any depth — the admission failure. */
23
+ notInPool: number;
24
+ /** Answers that landed inside the reported cut. */
25
+ insideCut: number;
26
+ /** Answers present in the pool but ranked past the cut — the ordering failure. */
27
+ pooledBelowCut: number;
28
+ /** Pooled answers the lexical/spec layer scored at zero (reached only through commit history). */
29
+ pooledWithZeroLexical: number;
30
+ /** Recall the cut could reach if ordering were perfect. 1 means ranking is the only obstacle. */
31
+ cutRecallCeiling: number;
32
+ }
33
+ export declare function diagnoseRanking(cases: ReadonlyArray<{
34
+ truth: readonly PooledTruth[];
35
+ }>, cut: number): RankDiagnosis;
36
+ /**
37
+ * Measured 2026-08-29 on the eight A-SPEC-404 blind jarvis cases, through the shipped pipeline
38
+ * (localizeIssue at pool 500, then the recency x commit-text re-rank of A-SPEC-378/388).
39
+ */
40
+ export declare const S405_POOLED_TRUTH: ReadonlyArray<{
41
+ truth: readonly PooledTruth[];
42
+ }>;
43
+ export declare const S405_DIAGNOSIS: RankDiagnosis;
@@ -0,0 +1,163 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.S405_DIAGNOSIS = exports.S405_POOLED_TRUTH = void 0;
4
+ exports.diagnoseRanking = diagnoseRanking;
5
+ function diagnoseRanking(cases, cut) {
6
+ let truthTotal = 0;
7
+ let notInPool = 0;
8
+ let insideCut = 0;
9
+ let pooledBelowCut = 0;
10
+ let pooledWithZeroLexical = 0;
11
+ for (const c of cases) {
12
+ for (const t of c.truth) {
13
+ truthTotal += 1;
14
+ if (t.rank < 0) {
15
+ notInPool += 1;
16
+ continue;
17
+ }
18
+ if (t.rank < cut)
19
+ insideCut += 1;
20
+ else
21
+ pooledBelowCut += 1;
22
+ if (t.lexical === 0)
23
+ pooledWithZeroLexical += 1;
24
+ }
25
+ }
26
+ return {
27
+ truthTotal,
28
+ notInPool,
29
+ insideCut,
30
+ pooledBelowCut,
31
+ pooledWithZeroLexical,
32
+ cutRecallCeiling: truthTotal === 0 ? 0 : (truthTotal - notInPool) / truthTotal,
33
+ };
34
+ }
35
+ /**
36
+ * Measured 2026-08-29 on the eight A-SPEC-404 blind jarvis cases, through the shipped pipeline
37
+ * (localizeIssue at pool 500, then the recency x commit-text re-rank of A-SPEC-378/388).
38
+ */
39
+ exports.S405_POOLED_TRUTH = [
40
+ {
41
+ "truth": [
42
+ {
43
+ "file": "src/gateway/handlers.py",
44
+ "rank": 6,
45
+ "lexical": 0
46
+ },
47
+ {
48
+ "file": "src/gateway/persona.py",
49
+ "rank": 87,
50
+ "lexical": 0
51
+ }
52
+ ]
53
+ },
54
+ {
55
+ "truth": [
56
+ {
57
+ "file": "src/structured_logger.py",
58
+ "rank": 93,
59
+ "lexical": 0
60
+ }
61
+ ]
62
+ },
63
+ {
64
+ "truth": [
65
+ {
66
+ "file": "src/gateway/self_review.py",
67
+ "rank": 27,
68
+ "lexical": 0
69
+ },
70
+ {
71
+ "file": "src/llm_providers/base_cli.py",
72
+ "rank": 2,
73
+ "lexical": 32.271010116342076
74
+ },
75
+ {
76
+ "file": "src/llm_providers/claude_cli.py",
77
+ "rank": 0,
78
+ "lexical": 40.47872273076396
79
+ },
80
+ {
81
+ "file": "src/main.py",
82
+ "rank": 6,
83
+ "lexical": 8.415456200622677
84
+ },
85
+ {
86
+ "file": "src/web_server.py",
87
+ "rank": 44,
88
+ "lexical": 0
89
+ }
90
+ ]
91
+ },
92
+ {
93
+ "truth": [
94
+ {
95
+ "file": "src/llm_providers/base_cli.py",
96
+ "rank": 14,
97
+ "lexical": 0
98
+ }
99
+ ]
100
+ },
101
+ {
102
+ "truth": [
103
+ {
104
+ "file": "src/gateway/hardening_apply.py",
105
+ "rank": 24,
106
+ "lexical": 0
107
+ }
108
+ ]
109
+ },
110
+ {
111
+ "truth": [
112
+ {
113
+ "file": "src/core/memory_audit.py",
114
+ "rank": 12,
115
+ "lexical": 0
116
+ },
117
+ {
118
+ "file": "src/core/profile.py",
119
+ "rank": 2,
120
+ "lexical": 0
121
+ },
122
+ {
123
+ "file": "src/proactive.py",
124
+ "rank": 0,
125
+ "lexical": 139.77894930506255
126
+ }
127
+ ]
128
+ },
129
+ {
130
+ "truth": [
131
+ {
132
+ "file": "src/core/memory_gate.py",
133
+ "rank": 7,
134
+ "lexical": 0.7049978210990919
135
+ },
136
+ {
137
+ "file": "src/graph_memory.py",
138
+ "rank": 0,
139
+ "lexical": 57.809821330125516
140
+ },
141
+ {
142
+ "file": "src/knowledge_distiller.py",
143
+ "rank": 6,
144
+ "lexical": 0
145
+ }
146
+ ]
147
+ },
148
+ {
149
+ "truth": [
150
+ {
151
+ "file": "src/tools/capabilities.py",
152
+ "rank": 7,
153
+ "lexical": 0
154
+ },
155
+ {
156
+ "file": "src/tools/changelog.py",
157
+ "rank": 8,
158
+ "lexical": 0
159
+ }
160
+ ]
161
+ }
162
+ ];
163
+ exports.S405_DIAGNOSIS = diagnoseRanking(exports.S405_POOLED_TRUTH, 10);
@@ -0,0 +1,62 @@
1
+ /**
2
+ * Does the confidence number mean anything?
3
+ *
4
+ * Goal Phase D asks for the closed loop, and the mechanism has been verified for a while — what is
5
+ * missing is data (`.ax/evidence` is n=0). Filling that directory with synthetic records would be
6
+ * worse than leaving it empty: it is a user-visible governance ledger the user deliberately chose
7
+ * not to git-ignore, and calibration computed from replay would read as calibration observed in use.
8
+ * So this measures in a temporary ledger and says plainly what it measured.
9
+ *
10
+ * The question it CAN answer honestly is not the one `computeCalibration` scores. That function
11
+ * grades classification correctness, and point-in-time replay carries no ground truth for "new
12
+ * requirement vs change to an existing one" — inventing that label would be scoring our own guess
13
+ * against our own guess. What replay does carry is the files the commit actually touched, so the
14
+ * answerable question is whether a HIGHER confidence goes with a MORE accurate answer.
15
+ *
16
+ * The prior is not neutral. The confidence formula is
17
+ * `0.55 + 0.15·(direct symbols exist) + 0.15·(tests found) + 0.1·(history found)` — it counts
18
+ * whether evidence is PRESENT, never whether it is any good. A number built that way can be
19
+ * expected to correlate weakly with accuracy, and if it does, saying so is the finding: a confidence
20
+ * that does not track correctness sells unearned certainty to whoever reads it.
21
+ */
22
+ export interface ConfidenceObservation {
23
+ confidence: number;
24
+ /** Per-case recall of the ranked answer against what the commit really changed. */
25
+ accuracy: number;
26
+ }
27
+ export interface ConfidenceCalibration {
28
+ n: number;
29
+ /** Pearson correlation between confidence and accuracy; null when confidence never varies. */
30
+ correlation: number | null;
31
+ /** Distinct confidence values observed — a formula that emits one value cannot rank anything. */
32
+ distinctConfidences: number;
33
+ /** Mean accuracy among the cases at or above the median confidence, and below it. */
34
+ meanAccuracyHigh: number | null;
35
+ meanAccuracyLow: number | null;
36
+ /** Brier-style mean squared gap between the promise and the delivery. */
37
+ meanSquaredGap: number | null;
38
+ }
39
+ export declare function confidenceCalibration(obs: readonly ConfidenceObservation[]): ConfidenceCalibration;
40
+ /**
41
+ * @implements A-SPEC-359
42
+ * Does ANY quantity the analysis already computes predict how accurate it will be?
43
+ *
44
+ * The confidence field does not (A-SPEC-357: correlation -0.062, and the confident cases were
45
+ * marginally *less* accurate than the ambiguous ones). Before proposing a better formula, the honest
46
+ * step is to find out whether a better formula is possible at all with the signals to hand — a
47
+ * search that can come back empty, and an empty result is publishable.
48
+ *
49
+ * A signal counts only when both corpora agree on its SIGN. A correlation seen in one repository can
50
+ * be a property of that repository, which this work has now been caught by twice.
51
+ */
52
+ export interface SignalCorrelation {
53
+ signal: string;
54
+ correlation: number | null;
55
+ n: number;
56
+ }
57
+ export declare function correlate(pairs: readonly {
58
+ x: number;
59
+ y: number;
60
+ }[]): number | null;
61
+ /** Signals that agree in sign across corpora AND clear the threshold in both. */
62
+ export declare function agreeingSignals(a: readonly SignalCorrelation[], b: readonly SignalCorrelation[], threshold: number): string[];