@holmes-lab/holmes-kit 0.2.1 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +55 -0
- package/dist/.build-id +1 -1
- package/dist/holmes/assoc/assoc-arm.d.ts +94 -0
- package/dist/holmes/assoc/assoc-arm.js +187 -0
- package/dist/holmes/assoc/explore.d.ts +21 -0
- package/dist/holmes/assoc/explore.js +160 -0
- package/dist/holmes/assoc/impact-baseline.d.ts +13 -0
- package/dist/holmes/assoc/impact-baseline.js +48 -0
- package/dist/holmes/assoc/ppr.d.ts +64 -0
- package/dist/holmes/assoc/ppr.js +110 -0
- package/dist/holmes/cli/agents.d.ts +24 -11
- package/dist/holmes/cli/agents.js +93 -21
- package/dist/holmes/cli/doctor.d.ts +56 -0
- package/dist/holmes/cli/doctor.js +311 -5
- package/dist/holmes/cli/index.js +13 -1
- package/dist/holmes/cli/semantic-key.d.ts +19 -0
- package/dist/holmes/cli/semantic-key.js +93 -0
- package/dist/holmes/config/config.d.ts +9 -0
- package/dist/holmes/config/config.js +8 -1
- package/dist/holmes/cpg/language-capability.d.ts +65 -0
- package/dist/holmes/cpg/language-capability.js +145 -0
- package/dist/holmes/cpg/language-parser-walk.js +179 -34
- package/dist/holmes/cpg/language-parser.d.ts +1 -1
- package/dist/holmes/governance/ledger-rechain.d.ts +12 -0
- package/dist/holmes/governance/ledger-rechain.js +17 -2
- package/dist/holmes/governance/provenance-ledger.js +21 -0
- package/dist/holmes/guardrail/blind-spots.js +12 -1
- package/dist/holmes/guardrail/impact-gate.d.ts +77 -0
- package/dist/holmes/guardrail/impact-gate.js +263 -0
- package/dist/holmes/guardrail/write-target.d.ts +38 -1
- package/dist/holmes/guardrail/write-target.js +41 -4
- package/dist/holmes/hooks/adapters/antigravity.js +12 -1
- package/dist/holmes/hooks/corrupt-state-run.d.ts +33 -0
- package/dist/holmes/hooks/corrupt-state-run.js +16 -0
- package/dist/holmes/hooks/pre-tool-use.d.ts +26 -0
- package/dist/holmes/hooks/pre-tool-use.js +203 -11
- package/dist/holmes/hooks/stop.d.ts +65 -0
- package/dist/holmes/hooks/stop.js +200 -2
- package/dist/holmes/mcp/elicit-approval.d.ts +32 -2
- package/dist/holmes/mcp/elicit-approval.js +30 -2
- package/dist/holmes/mcp/handlers.d.ts +88 -0
- package/dist/holmes/mcp/handlers.js +636 -35
- package/dist/holmes/mcp/maintenance-analyze.d.ts +435 -0
- package/dist/holmes/mcp/maintenance-analyze.js +994 -0
- package/dist/holmes/mcp/maintenance-evidence.d.ts +140 -0
- package/dist/holmes/mcp/maintenance-evidence.js +253 -0
- package/dist/holmes/mcp/server.js +13 -10
- package/dist/holmes/mcp/tool-schemas.js +71 -0
- package/dist/holmes/project/root.js +3 -1
- package/dist/holmes/review/baseline-arm.d.ts +37 -0
- package/dist/holmes/review/baseline-arm.js +51 -0
- package/dist/holmes/review/captured-stdin-guard.d.ts +8 -0
- package/dist/holmes/review/captured-stdin-guard.js +48 -0
- package/dist/holmes/review/coherence-verify.d.ts +31 -0
- package/dist/holmes/review/coherence-verify.js +144 -0
- package/dist/holmes/review/commit-text.d.ts +50 -0
- package/dist/holmes/review/commit-text.js +76 -0
- package/dist/holmes/review/confidence-calibration.d.ts +39 -0
- package/dist/holmes/review/confidence-calibration.js +39 -0
- package/dist/holmes/review/content-baseline.d.ts +38 -0
- package/dist/holmes/review/content-baseline.js +103 -0
- package/dist/holmes/review/content-verify.d.ts +20 -0
- package/dist/holmes/review/content-verify.js +73 -0
- package/dist/holmes/review/dense-retrieval.d.ts +66 -0
- package/dist/holmes/review/dense-retrieval.js +97 -0
- package/dist/holmes/review/edge-quality.d.ts +44 -0
- package/dist/holmes/review/edge-quality.js +117 -0
- package/dist/holmes/review/evaluation-metrics.d.ts +138 -0
- package/dist/holmes/review/evaluation-metrics.js +175 -0
- package/dist/holmes/review/graph-verifier.d.ts +34 -0
- package/dist/holmes/review/graph-verifier.js +62 -0
- package/dist/holmes/review/hop-ablation.d.ts +100 -0
- package/dist/holmes/review/hop-ablation.js +89 -0
- package/dist/holmes/review/manual-baseline.d.ts +209 -0
- package/dist/holmes/review/manual-baseline.js +2846 -0
- package/dist/holmes/review/oracle-gap.d.ts +32 -0
- package/dist/holmes/review/oracle-gap.js +102 -0
- package/dist/holmes/review/point-in-time-replay.d.ts +41 -0
- package/dist/holmes/review/point-in-time-replay.js +161 -0
- package/dist/holmes/review/rank-diagnosis.d.ts +43 -0
- package/dist/holmes/review/rank-diagnosis.js +163 -0
- package/dist/holmes/review/replay-calibration.d.ts +62 -0
- package/dist/holmes/review/replay-calibration.js +83 -0
- package/dist/holmes/review/replay-corpus.d.ts +135 -0
- package/dist/holmes/review/replay-corpus.js +210 -0
- package/dist/holmes/review/run-replay.d.ts +260 -0
- package/dist/holmes/review/run-replay.js +729 -0
- package/dist/holmes/review/semantic-arm.d.ts +271 -0
- package/dist/holmes/review/semantic-arm.js +717 -0
- package/dist/holmes/review/semantic-retrieval.d.ts +55 -0
- package/dist/holmes/review/semantic-retrieval.js +156 -0
- package/dist/holmes/review/spec-layer-stats.d.ts +38 -0
- package/dist/holmes/review/spec-layer-stats.js +52 -0
- package/dist/holmes/review/temporal-prior.d.ts +33 -0
- package/dist/holmes/review/temporal-prior.js +53 -0
- package/dist/holmes/review/test-runner.d.ts +15 -0
- package/dist/holmes/review/test-runner.js +41 -4
- package/dist/holmes/review/test-selection-breadth.d.ts +75 -0
- package/dist/holmes/review/test-selection-breadth.js +57 -0
- package/dist/holmes/review/traceability-benchmark.d.ts +81 -0
- package/dist/holmes/review/traceability-benchmark.js +135 -0
- package/dist/holmes/review/union-verify.d.ts +12 -0
- package/dist/holmes/review/union-verify.js +70 -0
- package/dist/holmes/rtm/graph-store.d.ts +51 -0
- package/dist/holmes/rtm/graph-store.js +122 -0
- package/dist/holmes/rtm/incremental.d.ts +25 -1
- package/dist/holmes/rtm/incremental.js +18 -1
- package/dist/holmes/rtm/localize.d.ts +28 -0
- package/dist/holmes/rtm/localize.js +272 -14
- package/dist/holmes/rtm/rtm-builder.d.ts +118 -3
- package/dist/holmes/rtm/rtm-builder.js +265 -28
- package/dist/holmes/rtm/rtm-graph.d.ts +117 -2
- package/dist/holmes/rtm/rtm-graph.js +194 -31
- package/dist/holmes/rtm/taint-benchmark.d.ts +97 -0
- package/dist/holmes/rtm/taint-benchmark.js +141 -0
- package/dist/holmes/rtm/test-scope.js +24 -1
- package/dist/holmes/semantic/credentials.d.ts +15 -0
- package/dist/holmes/semantic/credentials.js +134 -0
- package/dist/holmes/semantic/embedder.d.ts +44 -0
- package/dist/holmes/semantic/embedder.js +185 -0
- package/dist/holmes/semantic/hit-rerank.d.ts +4 -0
- package/dist/holmes/semantic/hit-rerank.js +38 -0
- package/dist/holmes/semantic/tier.d.ts +37 -0
- package/dist/holmes/semantic/tier.js +54 -0
- package/dist/holmes/semantic/vector-cache.d.ts +11 -0
- package/dist/holmes/semantic/vector-cache.js +91 -0
- package/dist/holmes/spec/acceptance-quality.d.ts +81 -0
- package/dist/holmes/spec/acceptance-quality.js +169 -0
- package/dist/holmes/spec/validator.js +33 -1
- package/dist/holmes/spec/yaml-scalar.d.ts +1 -0
- package/dist/holmes/spec/yaml-scalar.js +43 -0
- package/package.json +1 -1
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import type { Spec } from '../spec/spec-parser';
|
|
2
|
+
/**
|
|
3
|
+
* The distance between "a test file claims this requirement" and "a test checks it".
|
|
4
|
+
*
|
|
5
|
+
* Traceability here runs from an A-SPEC to a test FILE: an `@implements` anchor makes the link and
|
|
6
|
+
* the constitution gate enforces it. The anchor is a line at the top of a file, and it says nothing
|
|
7
|
+
* about whether any assertion inside that file examines the requirement. A suite can be green, the
|
|
8
|
+
* anchor present, the gate satisfied, and the criterion untested.
|
|
9
|
+
*
|
|
10
|
+
* What is countable without executing anything: whether the requirement's vocabulary appears in the
|
|
11
|
+
* NAMES of the tests that claim it. A test whose describe and it titles share no significant word
|
|
12
|
+
* with the spec does not even mention what it claims to verify.
|
|
13
|
+
*
|
|
14
|
+
* **This is a lower bound and not an oracle check.** Sharing words does not mean the assertion
|
|
15
|
+
* examines the requirement, and sharing none does not prove it fails to — a test named
|
|
16
|
+
* `returns 404 for unknown ids` may verify a requirement about error handling that uses none of
|
|
17
|
+
* those words. What the count measures is how much of the traceability claim rests on the anchor
|
|
18
|
+
* alone. Calling it anything stronger would repeat the mistake this repository has made before.
|
|
19
|
+
*/
|
|
20
|
+
export interface OracleGap {
|
|
21
|
+
/** A-SPECs with at least one anchored test file. */
|
|
22
|
+
claimed: number;
|
|
23
|
+
/** Of those, how many have NO test title sharing a significant term with the spec. */
|
|
24
|
+
titlesShareNothing: number;
|
|
25
|
+
/** Mean share of the spec's terms appearing in the titles of tests that claim it. */
|
|
26
|
+
meanTitleOverlap: number | null;
|
|
27
|
+
/** Specs whose anchored test files could not be read — unmeasured, not clean. */
|
|
28
|
+
unreadable: number;
|
|
29
|
+
}
|
|
30
|
+
/** Test titles in a file: the strings passed to describe/it/test. */
|
|
31
|
+
export declare function testTitles(source: string): string[];
|
|
32
|
+
export declare function oracleGap(specs: readonly Spec[], anchors: Readonly<Record<string, readonly string[]>>, root: string): OracleGap;
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.testTitles = testTitles;
|
|
37
|
+
exports.oracleGap = oracleGap;
|
|
38
|
+
// @implements A-SPEC-391
|
|
39
|
+
const fs = __importStar(require("node:fs"));
|
|
40
|
+
const path = __importStar(require("node:path"));
|
|
41
|
+
const localize_1 = require("../rtm/localize");
|
|
42
|
+
/** Test titles in a file: the strings passed to describe/it/test. */
|
|
43
|
+
function testTitles(source) {
|
|
44
|
+
const out = [];
|
|
45
|
+
const re = /\b(?:describe|it|test)\s*(?:\.\w+)?\s*\(\s*(['"`])((?:\\.|(?!\1)[\s\S])*)\1/g;
|
|
46
|
+
let m;
|
|
47
|
+
while ((m = re.exec(source)) !== null)
|
|
48
|
+
out.push(m[2]);
|
|
49
|
+
return out;
|
|
50
|
+
}
|
|
51
|
+
function oracleGap(specs, anchors, root) {
|
|
52
|
+
const filesBySpec = new Map();
|
|
53
|
+
for (const [file, ids] of Object.entries(anchors)) {
|
|
54
|
+
for (const id of ids)
|
|
55
|
+
(filesBySpec.get(id) ?? filesBySpec.set(id, []).get(id)).push(file);
|
|
56
|
+
}
|
|
57
|
+
let claimed = 0, shareNothing = 0, unreadable = 0, overlapSum = 0, overlapCount = 0;
|
|
58
|
+
for (const s of specs) {
|
|
59
|
+
if (s.type !== 'A-SPEC')
|
|
60
|
+
continue;
|
|
61
|
+
const files = filesBySpec.get(s.id);
|
|
62
|
+
if (!files || files.length === 0)
|
|
63
|
+
continue;
|
|
64
|
+
claimed++;
|
|
65
|
+
const specTerms = new Set((0, localize_1.significantTerms)(`${s.title} ${Object.values(s.sections).join(' ')}`));
|
|
66
|
+
if (specTerms.size === 0)
|
|
67
|
+
continue;
|
|
68
|
+
const titleTerms = new Set();
|
|
69
|
+
let read = 0;
|
|
70
|
+
for (const file of files) {
|
|
71
|
+
let source;
|
|
72
|
+
try {
|
|
73
|
+
source = fs.readFileSync(path.join(root, file), 'utf8');
|
|
74
|
+
}
|
|
75
|
+
catch {
|
|
76
|
+
continue;
|
|
77
|
+
}
|
|
78
|
+
read++;
|
|
79
|
+
for (const title of testTitles(source))
|
|
80
|
+
for (const t of (0, localize_1.significantTerms)(title))
|
|
81
|
+
titleTerms.add(t);
|
|
82
|
+
}
|
|
83
|
+
if (read === 0) {
|
|
84
|
+
unreadable++;
|
|
85
|
+
continue;
|
|
86
|
+
}
|
|
87
|
+
let shared = 0;
|
|
88
|
+
for (const t of specTerms)
|
|
89
|
+
if (titleTerms.has(t))
|
|
90
|
+
shared++;
|
|
91
|
+
if (shared === 0)
|
|
92
|
+
shareNothing++;
|
|
93
|
+
overlapSum += shared / specTerms.size;
|
|
94
|
+
overlapCount++;
|
|
95
|
+
}
|
|
96
|
+
return {
|
|
97
|
+
claimed,
|
|
98
|
+
titlesShareNothing: shareNothing,
|
|
99
|
+
meanTitleOverlap: overlapCount === 0 ? null : overlapSum / overlapCount,
|
|
100
|
+
unreadable,
|
|
101
|
+
};
|
|
102
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
export type Materialized = {
|
|
2
|
+
ok: true;
|
|
3
|
+
files: string[];
|
|
4
|
+
} | {
|
|
5
|
+
ok: false;
|
|
6
|
+
reason: string;
|
|
7
|
+
};
|
|
8
|
+
export interface Fidelity {
|
|
9
|
+
/** Files whose materialized bytes were compared against the commit's blob. */
|
|
10
|
+
checked: number;
|
|
11
|
+
/** Repo-relative paths whose bytes differ from the commit. Empty means the tree is faithful. */
|
|
12
|
+
mismatches: string[];
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* @implements A-SPEC-315
|
|
16
|
+
* Every failure carries a CLASSIFICATION prefix — `unsafe-pathspec`, `destination-not-empty`,
|
|
17
|
+
* `git-ls-tree-failed`, `empty-pathspec`, `archive-or-extract-failed` — followed by the original
|
|
18
|
+
* detail. Two full-suite runs went red here in two different tests and both reported only `false`;
|
|
19
|
+
* isolation, two clean full runs and a 320-call concurrency probe all passed, so the cause is still
|
|
20
|
+
* unknown. Deliberately NO retry: a retry would hide whatever is actually happening.
|
|
21
|
+
*
|
|
22
|
+
* Lay out the tree a commit recorded, WITHOUT touching the working tree or index.
|
|
23
|
+
*
|
|
24
|
+
* `git archive` reads straight from the object database, so no checkout, stash or index write is
|
|
25
|
+
* involved — the caller's uncommitted work is untouched, which is the property this whole slice
|
|
26
|
+
* exists to obtain. The alternative (checkout into the same worktree) would destroy user state.
|
|
27
|
+
*/
|
|
28
|
+
export declare function materializeTreeAt(repoRoot: string, commitish: string, pathspec: string, destDir: string, opts?: {
|
|
29
|
+
/**
|
|
30
|
+
* @implements A-SPEC-303
|
|
31
|
+
* Permit a second pathspec from the SAME commit into the same destination — a project is its
|
|
32
|
+
* code AND its specs, and one scan should see both as they were. Opt-in on purpose: making it
|
|
33
|
+
* the default would quietly restore the mixed-tree hazard the emptiness rule exists to prevent.
|
|
34
|
+
*/
|
|
35
|
+
allowNonEmpty?: boolean;
|
|
36
|
+
}): Materialized;
|
|
37
|
+
/**
|
|
38
|
+
* Confirm the materialized tree really is that commit. The leak this slice removes is invisible by
|
|
39
|
+
* construction, so it must be asserted rather than assumed: a wrong tree still produces numbers.
|
|
40
|
+
*/
|
|
41
|
+
export declare function verifyTreeFidelity(repoRoot: string, commitish: string, pathspec: string, destDir: string): Fidelity;
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.materializeTreeAt = materializeTreeAt;
|
|
37
|
+
exports.verifyTreeFidelity = verifyTreeFidelity;
|
|
38
|
+
// @implements A-SPEC-297
|
|
39
|
+
const fs = __importStar(require("node:fs"));
|
|
40
|
+
const os = __importStar(require("node:os"));
|
|
41
|
+
const path = __importStar(require("node:path"));
|
|
42
|
+
const crypto = __importStar(require("node:crypto"));
|
|
43
|
+
const node_child_process_1 = require("node:child_process");
|
|
44
|
+
/**
|
|
45
|
+
* A pathspec is ours to control, but a caller mistake here writes outside the destination, so it is
|
|
46
|
+
* checked rather than trusted: repo-relative, no escape, no absolute root.
|
|
47
|
+
*/
|
|
48
|
+
function pathspecIsSafe(pathspec) {
|
|
49
|
+
if (pathspec === '' || path.isAbsolute(pathspec))
|
|
50
|
+
return false;
|
|
51
|
+
const normalized = path.normalize(pathspec);
|
|
52
|
+
return !normalized.startsWith('..') && !normalized.split(path.sep).includes('..');
|
|
53
|
+
}
|
|
54
|
+
function git(repoRoot, args) {
|
|
55
|
+
return (0, node_child_process_1.execFileSync)('git', ['-C', repoRoot, ...args], { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 });
|
|
56
|
+
}
|
|
57
|
+
/** `path mode sha` triples the commit records for the pathspec, keyed by repo-relative path. */
|
|
58
|
+
function blobsAt(repoRoot, commitish, pathspec) {
|
|
59
|
+
const out = git(repoRoot, ['ls-tree', '-r', `${commitish}^{tree}`, '--', pathspec]);
|
|
60
|
+
const blobs = new Map();
|
|
61
|
+
for (const line of out.split('\n')) {
|
|
62
|
+
if (line.trim() === '')
|
|
63
|
+
continue;
|
|
64
|
+
const [meta, file] = line.split('\t');
|
|
65
|
+
const parts = meta.split(/\s+/);
|
|
66
|
+
if (parts[1] !== 'blob')
|
|
67
|
+
continue; // submodules and trees are not files we can compare
|
|
68
|
+
blobs.set(file, parts[2]);
|
|
69
|
+
}
|
|
70
|
+
return blobs;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* @implements A-SPEC-315
|
|
74
|
+
* Every failure carries a CLASSIFICATION prefix — `unsafe-pathspec`, `destination-not-empty`,
|
|
75
|
+
* `git-ls-tree-failed`, `empty-pathspec`, `archive-or-extract-failed` — followed by the original
|
|
76
|
+
* detail. Two full-suite runs went red here in two different tests and both reported only `false`;
|
|
77
|
+
* isolation, two clean full runs and a 320-call concurrency probe all passed, so the cause is still
|
|
78
|
+
* unknown. Deliberately NO retry: a retry would hide whatever is actually happening.
|
|
79
|
+
*
|
|
80
|
+
* Lay out the tree a commit recorded, WITHOUT touching the working tree or index.
|
|
81
|
+
*
|
|
82
|
+
* `git archive` reads straight from the object database, so no checkout, stash or index write is
|
|
83
|
+
* involved — the caller's uncommitted work is untouched, which is the property this whole slice
|
|
84
|
+
* exists to obtain. The alternative (checkout into the same worktree) would destroy user state.
|
|
85
|
+
*/
|
|
86
|
+
function materializeTreeAt(repoRoot, commitish, pathspec, destDir, opts) {
|
|
87
|
+
if (!pathspecIsSafe(pathspec))
|
|
88
|
+
return { ok: false, reason: `unsafe-pathspec: ${pathspec}` };
|
|
89
|
+
if (!opts?.allowNonEmpty && fs.existsSync(destDir) && fs.readdirSync(destDir).length > 0) {
|
|
90
|
+
// Extracting over existing files yields a tree that is partly one commit and partly another —
|
|
91
|
+
// and it would still score, silently and wrongly.
|
|
92
|
+
return { ok: false, reason: `destination-not-empty: destination is not empty: ${destDir}` };
|
|
93
|
+
}
|
|
94
|
+
fs.mkdirSync(destDir, { recursive: true });
|
|
95
|
+
let blobs;
|
|
96
|
+
try {
|
|
97
|
+
blobs = blobsAt(repoRoot, commitish, pathspec);
|
|
98
|
+
}
|
|
99
|
+
catch (err) {
|
|
100
|
+
// The original message is kept, not replaced: a classification that DISCARDS the detail makes
|
|
101
|
+
// diagnosis worse, not better.
|
|
102
|
+
return { ok: false, reason: `git-ls-tree-failed: unknown or unreadable commit-ish ${commitish}: ${err.message}` };
|
|
103
|
+
}
|
|
104
|
+
if (blobs.size === 0) {
|
|
105
|
+
// Zero files is not a score of zero. Nothing was measured.
|
|
106
|
+
return { ok: false, reason: `empty-pathspec: no files at ${commitish} for pathspec ${pathspec}` };
|
|
107
|
+
}
|
|
108
|
+
// @implements A-SPEC-329
|
|
109
|
+
// The tar goes through a FILE, not through tar's stdin. Feeding a multi-megabyte buffer to
|
|
110
|
+
// `execFileSync('tar', …, { input })` failed intermittently under full-suite load with
|
|
111
|
+
// `spawnSync tar EPIPE` — observed three times, and only nameable because A-SPEC-315 had added
|
|
112
|
+
// failure classifications instead of the retry that would have hidden it. A retry would have
|
|
113
|
+
// masked a fixable bug; removing the pipe removes the failure mode rather than surviving it.
|
|
114
|
+
//
|
|
115
|
+
// The scratch file lives OUTSIDE the destination: inside, it would mix with the extracted tree and
|
|
116
|
+
// the fidelity check would see a file the commit never had.
|
|
117
|
+
const tarPath = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'pit-tar-')), 'tree.tar');
|
|
118
|
+
try {
|
|
119
|
+
const tar = (0, node_child_process_1.execFileSync)('git', ['-C', repoRoot, 'archive', '--format=tar', commitish, '--', pathspec], { maxBuffer: 512 * 1024 * 1024 });
|
|
120
|
+
fs.writeFileSync(tarPath, tar);
|
|
121
|
+
(0, node_child_process_1.execFileSync)('tar', ['-x', '-C', destDir, '-f', tarPath], { maxBuffer: 512 * 1024 * 1024 });
|
|
122
|
+
}
|
|
123
|
+
catch (err) {
|
|
124
|
+
return { ok: false, reason: `archive-or-extract-failed: could not materialize ${commitish}: ${err.message}` };
|
|
125
|
+
}
|
|
126
|
+
finally {
|
|
127
|
+
// Removed on every path — a scratch file that survives a failure is litter in the temp dir.
|
|
128
|
+
fs.rmSync(path.dirname(tarPath), { recursive: true, force: true });
|
|
129
|
+
}
|
|
130
|
+
return { ok: true, files: [...blobs.keys()].sort() };
|
|
131
|
+
}
|
|
132
|
+
/** Git's blob id: sha1 over `blob <byteLength>\0` followed by the content. */
|
|
133
|
+
function blobId(content) {
|
|
134
|
+
return crypto.createHash('sha1').update(`blob ${content.length}\0`).update(content).digest('hex');
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* Confirm the materialized tree really is that commit. The leak this slice removes is invisible by
|
|
138
|
+
* construction, so it must be asserted rather than assumed: a wrong tree still produces numbers.
|
|
139
|
+
*/
|
|
140
|
+
function verifyTreeFidelity(repoRoot, commitish, pathspec, destDir) {
|
|
141
|
+
let blobs;
|
|
142
|
+
try {
|
|
143
|
+
blobs = blobsAt(repoRoot, commitish, pathspec);
|
|
144
|
+
}
|
|
145
|
+
catch {
|
|
146
|
+
return { checked: 0, mismatches: [`<unreadable commit-ish: ${commitish}>`] };
|
|
147
|
+
}
|
|
148
|
+
const mismatches = [];
|
|
149
|
+
let checked = 0;
|
|
150
|
+
for (const [file, expected] of blobs) {
|
|
151
|
+
const onDisk = path.join(destDir, file);
|
|
152
|
+
checked++;
|
|
153
|
+
if (!fs.existsSync(onDisk)) {
|
|
154
|
+
mismatches.push(file);
|
|
155
|
+
continue;
|
|
156
|
+
}
|
|
157
|
+
if (blobId(fs.readFileSync(onDisk)) !== expected)
|
|
158
|
+
mismatches.push(file);
|
|
159
|
+
}
|
|
160
|
+
return { checked, mismatches: mismatches.sort() };
|
|
161
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Why the shipped harness lost on recall (S-404), decomposed.
|
|
3
|
+
*
|
|
4
|
+
* "Recall is low" has two completely different causes and they call for opposite fixes: either the
|
|
5
|
+
* answer never became a candidate (admission — widen the pool, add a retrieval channel), or it was
|
|
6
|
+
* a candidate and something else outranked it (ordering — change the scoring, and widening the pool
|
|
7
|
+
* does nothing). Thirteen rejected retrieval mechanisms were all attempts at the first one. This
|
|
8
|
+
* module measures which it actually is, so the next attempt is aimed at the measured cause.
|
|
9
|
+
*
|
|
10
|
+
* On the eight blind jarvis cases of A-SPEC-404 the answer is unambiguous, and it is the second:
|
|
11
|
+
* every truth file was already in the pool.
|
|
12
|
+
*/
|
|
13
|
+
export interface PooledTruth {
|
|
14
|
+
file: string;
|
|
15
|
+
/** Position in the full candidate pool, best first. `-1` when the file never entered it. */
|
|
16
|
+
rank: number;
|
|
17
|
+
/** Lexical/spec score before the history re-rank. Zero means only commit history reached it. */
|
|
18
|
+
lexical: number;
|
|
19
|
+
}
|
|
20
|
+
export interface RankDiagnosis {
|
|
21
|
+
truthTotal: number;
|
|
22
|
+
/** Answers the pipeline never proposed at any depth — the admission failure. */
|
|
23
|
+
notInPool: number;
|
|
24
|
+
/** Answers that landed inside the reported cut. */
|
|
25
|
+
insideCut: number;
|
|
26
|
+
/** Answers present in the pool but ranked past the cut — the ordering failure. */
|
|
27
|
+
pooledBelowCut: number;
|
|
28
|
+
/** Pooled answers the lexical/spec layer scored at zero (reached only through commit history). */
|
|
29
|
+
pooledWithZeroLexical: number;
|
|
30
|
+
/** Recall the cut could reach if ordering were perfect. 1 means ranking is the only obstacle. */
|
|
31
|
+
cutRecallCeiling: number;
|
|
32
|
+
}
|
|
33
|
+
export declare function diagnoseRanking(cases: ReadonlyArray<{
|
|
34
|
+
truth: readonly PooledTruth[];
|
|
35
|
+
}>, cut: number): RankDiagnosis;
|
|
36
|
+
/**
|
|
37
|
+
* Measured 2026-08-29 on the eight A-SPEC-404 blind jarvis cases, through the shipped pipeline
|
|
38
|
+
* (localizeIssue at pool 500, then the recency x commit-text re-rank of A-SPEC-378/388).
|
|
39
|
+
*/
|
|
40
|
+
export declare const S405_POOLED_TRUTH: ReadonlyArray<{
|
|
41
|
+
truth: readonly PooledTruth[];
|
|
42
|
+
}>;
|
|
43
|
+
export declare const S405_DIAGNOSIS: RankDiagnosis;
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.S405_DIAGNOSIS = exports.S405_POOLED_TRUTH = void 0;
|
|
4
|
+
exports.diagnoseRanking = diagnoseRanking;
|
|
5
|
+
function diagnoseRanking(cases, cut) {
|
|
6
|
+
let truthTotal = 0;
|
|
7
|
+
let notInPool = 0;
|
|
8
|
+
let insideCut = 0;
|
|
9
|
+
let pooledBelowCut = 0;
|
|
10
|
+
let pooledWithZeroLexical = 0;
|
|
11
|
+
for (const c of cases) {
|
|
12
|
+
for (const t of c.truth) {
|
|
13
|
+
truthTotal += 1;
|
|
14
|
+
if (t.rank < 0) {
|
|
15
|
+
notInPool += 1;
|
|
16
|
+
continue;
|
|
17
|
+
}
|
|
18
|
+
if (t.rank < cut)
|
|
19
|
+
insideCut += 1;
|
|
20
|
+
else
|
|
21
|
+
pooledBelowCut += 1;
|
|
22
|
+
if (t.lexical === 0)
|
|
23
|
+
pooledWithZeroLexical += 1;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
return {
|
|
27
|
+
truthTotal,
|
|
28
|
+
notInPool,
|
|
29
|
+
insideCut,
|
|
30
|
+
pooledBelowCut,
|
|
31
|
+
pooledWithZeroLexical,
|
|
32
|
+
cutRecallCeiling: truthTotal === 0 ? 0 : (truthTotal - notInPool) / truthTotal,
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Measured 2026-08-29 on the eight A-SPEC-404 blind jarvis cases, through the shipped pipeline
|
|
37
|
+
* (localizeIssue at pool 500, then the recency x commit-text re-rank of A-SPEC-378/388).
|
|
38
|
+
*/
|
|
39
|
+
exports.S405_POOLED_TRUTH = [
|
|
40
|
+
{
|
|
41
|
+
"truth": [
|
|
42
|
+
{
|
|
43
|
+
"file": "src/gateway/handlers.py",
|
|
44
|
+
"rank": 6,
|
|
45
|
+
"lexical": 0
|
|
46
|
+
},
|
|
47
|
+
{
|
|
48
|
+
"file": "src/gateway/persona.py",
|
|
49
|
+
"rank": 87,
|
|
50
|
+
"lexical": 0
|
|
51
|
+
}
|
|
52
|
+
]
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
"truth": [
|
|
56
|
+
{
|
|
57
|
+
"file": "src/structured_logger.py",
|
|
58
|
+
"rank": 93,
|
|
59
|
+
"lexical": 0
|
|
60
|
+
}
|
|
61
|
+
]
|
|
62
|
+
},
|
|
63
|
+
{
|
|
64
|
+
"truth": [
|
|
65
|
+
{
|
|
66
|
+
"file": "src/gateway/self_review.py",
|
|
67
|
+
"rank": 27,
|
|
68
|
+
"lexical": 0
|
|
69
|
+
},
|
|
70
|
+
{
|
|
71
|
+
"file": "src/llm_providers/base_cli.py",
|
|
72
|
+
"rank": 2,
|
|
73
|
+
"lexical": 32.271010116342076
|
|
74
|
+
},
|
|
75
|
+
{
|
|
76
|
+
"file": "src/llm_providers/claude_cli.py",
|
|
77
|
+
"rank": 0,
|
|
78
|
+
"lexical": 40.47872273076396
|
|
79
|
+
},
|
|
80
|
+
{
|
|
81
|
+
"file": "src/main.py",
|
|
82
|
+
"rank": 6,
|
|
83
|
+
"lexical": 8.415456200622677
|
|
84
|
+
},
|
|
85
|
+
{
|
|
86
|
+
"file": "src/web_server.py",
|
|
87
|
+
"rank": 44,
|
|
88
|
+
"lexical": 0
|
|
89
|
+
}
|
|
90
|
+
]
|
|
91
|
+
},
|
|
92
|
+
{
|
|
93
|
+
"truth": [
|
|
94
|
+
{
|
|
95
|
+
"file": "src/llm_providers/base_cli.py",
|
|
96
|
+
"rank": 14,
|
|
97
|
+
"lexical": 0
|
|
98
|
+
}
|
|
99
|
+
]
|
|
100
|
+
},
|
|
101
|
+
{
|
|
102
|
+
"truth": [
|
|
103
|
+
{
|
|
104
|
+
"file": "src/gateway/hardening_apply.py",
|
|
105
|
+
"rank": 24,
|
|
106
|
+
"lexical": 0
|
|
107
|
+
}
|
|
108
|
+
]
|
|
109
|
+
},
|
|
110
|
+
{
|
|
111
|
+
"truth": [
|
|
112
|
+
{
|
|
113
|
+
"file": "src/core/memory_audit.py",
|
|
114
|
+
"rank": 12,
|
|
115
|
+
"lexical": 0
|
|
116
|
+
},
|
|
117
|
+
{
|
|
118
|
+
"file": "src/core/profile.py",
|
|
119
|
+
"rank": 2,
|
|
120
|
+
"lexical": 0
|
|
121
|
+
},
|
|
122
|
+
{
|
|
123
|
+
"file": "src/proactive.py",
|
|
124
|
+
"rank": 0,
|
|
125
|
+
"lexical": 139.77894930506255
|
|
126
|
+
}
|
|
127
|
+
]
|
|
128
|
+
},
|
|
129
|
+
{
|
|
130
|
+
"truth": [
|
|
131
|
+
{
|
|
132
|
+
"file": "src/core/memory_gate.py",
|
|
133
|
+
"rank": 7,
|
|
134
|
+
"lexical": 0.7049978210990919
|
|
135
|
+
},
|
|
136
|
+
{
|
|
137
|
+
"file": "src/graph_memory.py",
|
|
138
|
+
"rank": 0,
|
|
139
|
+
"lexical": 57.809821330125516
|
|
140
|
+
},
|
|
141
|
+
{
|
|
142
|
+
"file": "src/knowledge_distiller.py",
|
|
143
|
+
"rank": 6,
|
|
144
|
+
"lexical": 0
|
|
145
|
+
}
|
|
146
|
+
]
|
|
147
|
+
},
|
|
148
|
+
{
|
|
149
|
+
"truth": [
|
|
150
|
+
{
|
|
151
|
+
"file": "src/tools/capabilities.py",
|
|
152
|
+
"rank": 7,
|
|
153
|
+
"lexical": 0
|
|
154
|
+
},
|
|
155
|
+
{
|
|
156
|
+
"file": "src/tools/changelog.py",
|
|
157
|
+
"rank": 8,
|
|
158
|
+
"lexical": 0
|
|
159
|
+
}
|
|
160
|
+
]
|
|
161
|
+
}
|
|
162
|
+
];
|
|
163
|
+
exports.S405_DIAGNOSIS = diagnoseRanking(exports.S405_POOLED_TRUTH, 10);
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Does the confidence number mean anything?
|
|
3
|
+
*
|
|
4
|
+
* Goal Phase D asks for the closed loop, and the mechanism has been verified for a while — what is
|
|
5
|
+
* missing is data (`.ax/evidence` is n=0). Filling that directory with synthetic records would be
|
|
6
|
+
* worse than leaving it empty: it is a user-visible governance ledger the user deliberately chose
|
|
7
|
+
* not to git-ignore, and calibration computed from replay would read as calibration observed in use.
|
|
8
|
+
* So this measures in a temporary ledger and says plainly what it measured.
|
|
9
|
+
*
|
|
10
|
+
* The question it CAN answer honestly is not the one `computeCalibration` scores. That function
|
|
11
|
+
* grades classification correctness, and point-in-time replay carries no ground truth for "new
|
|
12
|
+
* requirement vs change to an existing one" — inventing that label would be scoring our own guess
|
|
13
|
+
* against our own guess. What replay does carry is the files the commit actually touched, so the
|
|
14
|
+
* answerable question is whether a HIGHER confidence goes with a MORE accurate answer.
|
|
15
|
+
*
|
|
16
|
+
* The prior is not neutral. The confidence formula is
|
|
17
|
+
* `0.55 + 0.15·(direct symbols exist) + 0.15·(tests found) + 0.1·(history found)` — it counts
|
|
18
|
+
* whether evidence is PRESENT, never whether it is any good. A number built that way can be
|
|
19
|
+
* expected to correlate weakly with accuracy, and if it does, saying so is the finding: a confidence
|
|
20
|
+
* that does not track correctness sells unearned certainty to whoever reads it.
|
|
21
|
+
*/
|
|
22
|
+
export interface ConfidenceObservation {
|
|
23
|
+
confidence: number;
|
|
24
|
+
/** Per-case recall of the ranked answer against what the commit really changed. */
|
|
25
|
+
accuracy: number;
|
|
26
|
+
}
|
|
27
|
+
export interface ConfidenceCalibration {
|
|
28
|
+
n: number;
|
|
29
|
+
/** Pearson correlation between confidence and accuracy; null when confidence never varies. */
|
|
30
|
+
correlation: number | null;
|
|
31
|
+
/** Distinct confidence values observed — a formula that emits one value cannot rank anything. */
|
|
32
|
+
distinctConfidences: number;
|
|
33
|
+
/** Mean accuracy among the cases at or above the median confidence, and below it. */
|
|
34
|
+
meanAccuracyHigh: number | null;
|
|
35
|
+
meanAccuracyLow: number | null;
|
|
36
|
+
/** Brier-style mean squared gap between the promise and the delivery. */
|
|
37
|
+
meanSquaredGap: number | null;
|
|
38
|
+
}
|
|
39
|
+
export declare function confidenceCalibration(obs: readonly ConfidenceObservation[]): ConfidenceCalibration;
|
|
40
|
+
/**
|
|
41
|
+
* @implements A-SPEC-359
|
|
42
|
+
* Does ANY quantity the analysis already computes predict how accurate it will be?
|
|
43
|
+
*
|
|
44
|
+
* The confidence field does not (A-SPEC-357: correlation -0.062, and the confident cases were
|
|
45
|
+
* marginally *less* accurate than the ambiguous ones). Before proposing a better formula, the honest
|
|
46
|
+
* step is to find out whether a better formula is possible at all with the signals to hand — a
|
|
47
|
+
* search that can come back empty, and an empty result is publishable.
|
|
48
|
+
*
|
|
49
|
+
* A signal counts only when both corpora agree on its SIGN. A correlation seen in one repository can
|
|
50
|
+
* be a property of that repository, which this work has now been caught by twice.
|
|
51
|
+
*/
|
|
52
|
+
export interface SignalCorrelation {
|
|
53
|
+
signal: string;
|
|
54
|
+
correlation: number | null;
|
|
55
|
+
n: number;
|
|
56
|
+
}
|
|
57
|
+
export declare function correlate(pairs: readonly {
|
|
58
|
+
x: number;
|
|
59
|
+
y: number;
|
|
60
|
+
}[]): number | null;
|
|
61
|
+
/** Signals that agree in sign across corpora AND clear the threshold in both. */
|
|
62
|
+
export declare function agreeingSignals(a: readonly SignalCorrelation[], b: readonly SignalCorrelation[], threshold: number): string[];
|