@holmes-lab/holmes-kit 0.18.0 → 0.19.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +149 -0
- package/README.md +3 -0
- package/dist/.build-id +1 -1
- package/dist/holmes/cli/agents.d.ts +22 -0
- package/dist/holmes/cli/agents.js +76 -1
- package/dist/holmes/cli/approve.js +6 -1
- package/dist/holmes/cli/doctor.d.ts +36 -1
- package/dist/holmes/cli/doctor.js +182 -35
- package/dist/holmes/cli/index.js +7 -1
- package/dist/holmes/cli/init.js +12 -0
- package/dist/holmes/cli/native-deps.d.ts +65 -0
- package/dist/holmes/cli/native-deps.js +131 -0
- package/dist/holmes/cli/release-docs.d.ts +27 -0
- package/dist/holmes/cli/release-docs.js +68 -0
- package/dist/holmes/cpg/arch-observe.d.ts +15 -0
- package/dist/holmes/cpg/arch-observe.js +19 -0
- package/dist/holmes/cpg/cpg-scanner.d.ts +10 -36
- package/dist/holmes/cpg/cpg-scanner.js +27 -3
- package/dist/holmes/cpg/cycle-detect.d.ts +87 -0
- package/dist/holmes/cpg/cycle-detect.js +251 -0
- package/dist/holmes/cpg/cycle-observation.d.ts +65 -0
- package/dist/holmes/cpg/cycle-observation.js +146 -0
- package/dist/holmes/cpg/scan-cache.d.ts +1 -1
- package/dist/holmes/cpg/scanned-file.d.ts +36 -0
- package/dist/holmes/cpg/scanned-file.js +2 -0
- package/dist/holmes/governance/approval-queue.d.ts +23 -4
- package/dist/holmes/governance/approval-queue.js +44 -6
- package/dist/holmes/governance/constitution.d.ts +20 -0
- package/dist/holmes/governance/constitution.js +17 -0
- package/dist/holmes/governance/ledger-store.d.ts +9 -0
- package/dist/holmes/governance/ledger-store.js +47 -0
- package/dist/holmes/governance/provenance-chain.d.ts +16 -1
- package/dist/holmes/governance/provenance-chain.js +5 -3
- package/dist/holmes/hooks/pre-tool-use.js +3 -1
- package/dist/holmes/hooks/stop.d.ts +29 -0
- package/dist/holmes/hooks/stop.js +119 -3
- package/dist/holmes/mcp/defuse-bound.d.ts +1 -0
- package/dist/holmes/mcp/defuse-bound.js +8 -0
- package/dist/holmes/mcp/handlers.d.ts +7 -0
- package/dist/holmes/mcp/handlers.js +132 -6
- package/dist/holmes/mcp/history-admission.d.ts +15 -0
- package/dist/holmes/mcp/history-admission.js +37 -0
- package/dist/holmes/mcp/maintenance-analyze.d.ts +45 -0
- package/dist/holmes/mcp/maintenance-analyze.js +117 -9
- package/dist/holmes/mcp/maintenance-evidence.d.ts +41 -0
- package/dist/holmes/mcp/maintenance-evidence.js +71 -4
- package/dist/holmes/project/install-scripts-policy.d.ts +76 -0
- package/dist/holmes/project/install-scripts-policy.js +131 -0
- package/dist/holmes/project/npx-bin.d.ts +6 -0
- package/dist/holmes/project/npx-bin.js +10 -0
- package/dist/holmes/review/evaluation-metrics.d.ts +6 -0
- package/dist/holmes/review/evaluation-metrics.js +18 -1
- package/dist/holmes/review/failed-test-names.d.ts +19 -0
- package/dist/holmes/review/failed-test-names.js +43 -0
- package/dist/holmes/review/paired-power.d.ts +14 -0
- package/dist/holmes/review/paired-power.js +57 -0
- package/dist/holmes/review/replay-corpus.d.ts +11 -0
- package/dist/holmes/review/replay-corpus.js +34 -0
- package/dist/holmes/review/run-replay.d.ts +23 -0
- package/dist/holmes/review/run-replay.js +90 -4
- package/dist/holmes/review/symbol-truth.d.ts +14 -0
- package/dist/holmes/review/symbol-truth.js +23 -0
- package/dist/holmes/review/test-runner.d.ts +27 -0
- package/dist/holmes/review/test-runner.js +59 -3
- package/dist/holmes/rtm/defuse-symbols.d.ts +17 -0
- package/dist/holmes/rtm/defuse-symbols.js +91 -0
- package/dist/holmes/rtm/incremental.js +5 -0
- package/dist/holmes/rtm/rtm-builder.d.ts +8 -0
- package/dist/holmes/rtm/rtm-builder.js +32 -5
- package/dist/holmes/rtm/rtm-graph.d.ts +11 -0
- package/dist/holmes/rtm/rtm-graph.js +13 -0
- package/dist/holmes/spec/legacy-fields.d.ts +2 -0
- package/dist/holmes/spec/legacy-fields.js +9 -0
- package/dist/holmes/spec/legacy-format.d.ts +1 -1
- package/dist/holmes/spec/legacy-format.js +4 -1
- package/dist/holmes/spec/spec-parser.js +5 -3
- package/docs/install-guide.md +54 -5
- package/package.json +4 -1
- package/playbooks/author-slice/PLAYBOOK.md +14 -0
- package/playbooks/publish/PLAYBOOK.md +32 -0
- package/playbooks/tdd-slice/PLAYBOOK.md +14 -0
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.PRIMARY_TIER_WIDTH = exports.EVIDENCE_DIMENSIONS = exports.MAINTENANCE_ANALYSIS_EXTRACTOR = exports.MAINTENANCE_ANALYSIS_SCHEMA = void 0;
|
|
3
|
+
exports.ANALYZE_TEXT_CAP = exports.ANALYZE_LIST_CAP = exports.PRIMARY_TIER_WIDTH = exports.EVIDENCE_DIMENSIONS = exports.MAINTENANCE_ANALYSIS_EXTRACTOR = exports.MAINTENANCE_ANALYSIS_SCHEMA = void 0;
|
|
4
4
|
exports.unquoteGitPath = unquoteGitPath;
|
|
5
5
|
exports.analyzeMaintenance = analyzeMaintenance;
|
|
6
|
+
exports.boundAnalysis = boundAnalysis;
|
|
6
7
|
// @implements A-SPEC-295
|
|
7
8
|
// @implements A-SPEC-294
|
|
8
9
|
// @implements A-SPEC-293
|
|
@@ -34,6 +35,8 @@ const decision_context_1 = require("../rtm/decision-context");
|
|
|
34
35
|
const taint_1 = require("../rtm/taint");
|
|
35
36
|
const test_evidence_1 = require("../review/test-evidence");
|
|
36
37
|
const language_capability_1 = require("../cpg/language-capability");
|
|
38
|
+
const history_admission_1 = require("./history-admission");
|
|
39
|
+
const defuse_symbols_1 = require("../rtm/defuse-symbols");
|
|
37
40
|
// @implements A-SPEC-409 — the SHAPE gained a field (`rerankPool`), so a consumer can detect it.
|
|
38
41
|
// @implements A-SPEC-405 A-SPEC-407 — the EXTRACTOR changed too: the same request now returns a
|
|
39
42
|
// differently ordered candidate list (keep-head rank fusion, then the graph-hop list), so two
|
|
@@ -181,10 +184,12 @@ function analyzeMaintenance(input) {
|
|
|
181
184
|
// Files the commit prose names but the lexical layer never scored have to be able to ENTER, or the
|
|
182
185
|
// fusion can only reorder what lexical already found — and reaching what it missed is where the
|
|
183
186
|
// measured gain came from (Top-10 recall 0.393 -> 0.601 on the second corpus).
|
|
187
|
+
// @implements A-SPEC-573.1 — but only files that could be source at all. The keys come from git
|
|
188
|
+
// history, so without this the pool also admits ledger JSONL and build artifacts that cannot be
|
|
189
|
+
// the answer (measured: 63.3% of emitted slots). The lexical hits are untouched, and a file the
|
|
190
|
+
// scan has never seen still enters — that is the boundary contract directly above.
|
|
184
191
|
const seeded = hasBoost
|
|
185
|
-
? [...localization.hits, ...Object.keys(boost)
|
|
186
|
-
.filter((file) => !localization.hits.some((h) => h.file === file))
|
|
187
|
-
.map((file) => ({ file, score: 0, matchedSymbols: [], viaSpecs: [], why: ['reached through commit history'] }))]
|
|
192
|
+
? [...localization.hits, ...(0, history_admission_1.admitHistoryFiles)(Object.keys(boost).filter((file) => !localization.hits.some((h) => h.file === file))).map((file) => ({ file, score: 0, matchedSymbols: [], viaSpecs: [], why: ['reached through commit history'] }))]
|
|
188
193
|
: localization.hits;
|
|
189
194
|
const reranked = !hasPrior && !hasBoost
|
|
190
195
|
? localization.hits
|
|
@@ -202,7 +207,13 @@ function analyzeMaintenance(input) {
|
|
|
202
207
|
// than prose spread evenly. Kept, with the cost named rather than defended.
|
|
203
208
|
+ (hasBoost ? COMMIT_TEXT_WEIGHT * (boost[hit.file] ?? 0) * Math.max(1, hit.score) : 0),
|
|
204
209
|
}))
|
|
205
|
-
.
|
|
210
|
+
// @implements A-SPEC-573.2 — vendored last, exactly as the lexical path orders it
|
|
211
|
+
// (localize.ts). A history-seeded file scores 0 lexically, so the boost term alone can carry
|
|
212
|
+
// it to 2.0 and hand it the head — measured, a vendored file took the head in 2 of 12 real
|
|
213
|
+
// requests. This is the ordering rule that path already applies; the channel was bypassing
|
|
214
|
+
// it. Demotion, not exclusion: vendored code is the answer in some projects.
|
|
215
|
+
.sort((a, b) => Number((0, history_admission_1.isVendorPath)(a.file)) - Number((0, history_admission_1.isVendorPath)(b.file))
|
|
216
|
+
|| b.score - a.score || a.file.localeCompare(b.file));
|
|
206
217
|
// @implements A-SPEC-405
|
|
207
218
|
// S-405 decomposed the recall loss measured in S-404: on the eight blind cases 18 of 18 truth
|
|
208
219
|
// files were already in this pool and 7 sat at ranks 12-93, so widening retrieval could not have
|
|
@@ -294,7 +305,10 @@ function analyzeMaintenance(input) {
|
|
|
294
305
|
;
|
|
295
306
|
const lists = [
|
|
296
307
|
desc((f) => byFile.get(f).score),
|
|
297
|
-
|
|
308
|
+
// @implements A-SPEC-573.2 — vendored last inside this list. Ordering `seeded` was not enough:
|
|
309
|
+
// measured, the head is decided HERE, and a vendored file with a high boost took it twice in
|
|
310
|
+
// twelve requests. The boost VALUES are untouched, so the lexical scoring above is unchanged.
|
|
311
|
+
hasBoost ? desc((f) => boost[f] ?? 0).sort((a, b) => Number((0, history_admission_1.isVendorPath)(a)) - Number((0, history_admission_1.isVendorPath)(b))) : [],
|
|
298
312
|
hasPrior ? desc((f) => (prior[f] ?? 1) - 1) : [],
|
|
299
313
|
cited.length > 0 ? [] : hops,
|
|
300
314
|
];
|
|
@@ -345,10 +359,23 @@ function analyzeMaintenance(input) {
|
|
|
345
359
|
// @implements A-SPEC-486 — the S-502/503 calibration point for gemini cosines. Display only:
|
|
346
360
|
// nothing in this module compares against it to drop or reorder anything.
|
|
347
361
|
const SEM_VERIFY_TAU = 0.65;
|
|
362
|
+
// @implements A-SPEC-573.4 — def-use rides BEHIND the lexical match, never in front of it: the
|
|
363
|
+
// symbols the request actually named keep the front, and the data-flow neighbours are appended.
|
|
364
|
+
const defUseTerms = input.defUse === undefined ? [] : (0, localize_1.significantTerms)(input.request);
|
|
365
|
+
const symbolsByFile = new Map(scanned.map((f) => [f.sourcePath, f.symbols.map((sy) => sy.qualifiedName)]));
|
|
366
|
+
const symbolsOf = (file, matched) => {
|
|
367
|
+
const facts = input.defUse?.[file];
|
|
368
|
+
if (facts === undefined)
|
|
369
|
+
return matched;
|
|
370
|
+
const inFile = symbolsByFile.get(file) ?? [];
|
|
371
|
+
if (inFile.length === 0)
|
|
372
|
+
return matched;
|
|
373
|
+
return (0, defuse_symbols_1.enrichCandidateSymbols)(matched, (0, defuse_symbols_1.rankSymbolsByDefUse)(facts, defUseTerms, inFile));
|
|
374
|
+
};
|
|
348
375
|
const shape = (hit) => ({
|
|
349
376
|
file: hit.file,
|
|
350
377
|
score: hit.score,
|
|
351
|
-
symbols: sortedUnique(hit.matchedSymbols),
|
|
378
|
+
symbols: symbolsOf(hit.file, sortedUnique(hit.matchedSymbols)),
|
|
352
379
|
evidence: hit.why,
|
|
353
380
|
});
|
|
354
381
|
// @implements A-SPEC-428
|
|
@@ -376,10 +403,15 @@ function analyzeMaintenance(input) {
|
|
|
376
403
|
const rescuedFiles = new Set(rescued.map((h) => h.file));
|
|
377
404
|
const rerankPool = [...rescued, ...exposedTail.filter((h) => !rescuedFiles.has(h.file))]
|
|
378
405
|
.slice(0, RERANK_POOL_N).map(shape);
|
|
406
|
+
const lexicalSymbolsOf = new Map(ordered.slice(0, LOCALIZATION_TOP_N).map((hit) => [hit.file, sortedUnique(hit.matchedSymbols)]));
|
|
379
407
|
const candidates = ordered.slice(0, LOCALIZATION_TOP_N).map((hit) => ({
|
|
380
408
|
file: hit.file,
|
|
381
409
|
score: hit.score,
|
|
382
|
-
|
|
410
|
+
// @implements A-SPEC-573.4 — the emitted list, which is what a caller reads and what the
|
|
411
|
+
// benchmark scores on the symbol axis. `shape` above serves the rerank pool only; wiring the
|
|
412
|
+
// enrichment there and not here would have enriched a surface nobody grades (caught by the
|
|
413
|
+
// symbol axis reporting NO movement — the instrument earned its keep on its first use).
|
|
414
|
+
symbols: symbolsOf(hit.file, sortedUnique(hit.matchedSymbols)),
|
|
383
415
|
evidence: [...hit.why],
|
|
384
416
|
}));
|
|
385
417
|
// @implements A-SPEC-572.3
|
|
@@ -460,7 +492,12 @@ function analyzeMaintenance(input) {
|
|
|
460
492
|
.flatMap((f) => f.symbols.map((sym) => sym.qualifiedName));
|
|
461
493
|
// A file you edited is not unaffected by your edit, so these are direct impact and not merely a
|
|
462
494
|
// traversal starting point.
|
|
463
|
-
|
|
495
|
+
// @implements A-SPEC-573.4 — the impact axis seeds from what the LEXICAL layer matched, not from
|
|
496
|
+
// the def-use symbols appended for emission. Measured: seeding the enriched list moved the union
|
|
497
|
+
// axis on both corpora (precision 0.1667->0.2075 / 0.1476->0.1689, recall 0.8755->0.8698 /
|
|
498
|
+
// 0.8302->0.7996) — a different axis than this slice targets, and axis transfer is exactly what
|
|
499
|
+
// this repository's record forbids. The emitted symbols stay enriched; the seeds do not.
|
|
500
|
+
const direct = sortedUnique([...lexicalSymbolsOf.values()].flat().concat(changedSymbols));
|
|
464
501
|
const transitiveIds = new Set();
|
|
465
502
|
const maxCallDepth = 3;
|
|
466
503
|
const hubInDegree = 12;
|
|
@@ -1012,3 +1049,74 @@ function buildAblation(a) {
|
|
|
1012
1049
|
], truth, a.primaryTier, a.impactedTier, a.importedByTier);
|
|
1013
1050
|
return { graphOff, current, enhanced };
|
|
1014
1051
|
}
|
|
1052
|
+
// @implements A-SPEC-578.5
|
|
1053
|
+
/**
|
|
1054
|
+
* Items a response lists before it starts counting instead.
|
|
1055
|
+
*
|
|
1056
|
+
* TEN, because that is the unit this product is measured in: localization emits Top-10 and every
|
|
1057
|
+
* recall figure in this repository is recall@10. Aligning the response cap with the emission unit
|
|
1058
|
+
* is a reason; picking a number that happens to hit a size target is a knob. Measured on the real
|
|
1059
|
+
* 187,174-character response: cap 20 gave 34,811 with `intent` alone at 11,037 (32%), cap 10 fits.
|
|
1060
|
+
*/
|
|
1061
|
+
exports.ANALYZE_LIST_CAP = 10;
|
|
1062
|
+
/** Characters one listed item keeps. */
|
|
1063
|
+
exports.ANALYZE_TEXT_CAP = 400;
|
|
1064
|
+
/** Dropped whole: another tool answers this question, and it is 41,619 characters of the answer. */
|
|
1065
|
+
const DROPPED_FIELDS = ['contextBundle'];
|
|
1066
|
+
/**
|
|
1067
|
+
* @implements A-SPEC-578.5
|
|
1068
|
+
* The RESPONSE-EDGE projection. The artifact on disk keeps everything.
|
|
1069
|
+
*
|
|
1070
|
+
* Measured 2026-09-09: one `maintenance_analyze` call returns 187,174 characters (~47k tokens) and
|
|
1071
|
+
* the harness refuses it outright. AGENTS.md step 3 instructs every agent to make that call before
|
|
1072
|
+
* editing source — and across 2,779 spec approvals `persist:true` had never been used once. An
|
|
1073
|
+
* instruction nobody can afford to follow is not an instruction.
|
|
1074
|
+
*
|
|
1075
|
+
* The cap is applied RECURSIVELY, which the first cut got wrong. Capping only the top level took
|
|
1076
|
+
* the response from 187,174 to 142,701 and no further, because the weight is in NESTED DUPLICATES:
|
|
1077
|
+
* `impacts.test` and `testScope.impactedTestFiles` carry the same 158 paths, and `impacts.contract`
|
|
1078
|
+
* and `testScope.impactedSpecs` the same 160 spec ids. A cap that stops at depth one caps the
|
|
1079
|
+
* cheapest arrays in the document.
|
|
1080
|
+
*
|
|
1081
|
+
* NEVER CALLED FROM `analyzeMaintenance`. The design-time advisory named the replay benchmark
|
|
1082
|
+
* (`run-replay.ts`, `replay-corpus.ts`, `commit-text.ts`, `temporal-prior.ts`) as readers of this
|
|
1083
|
+
* type, and among their anchors sits A-SPEC-402 — "the numbers reported are not what the product
|
|
1084
|
+
* does". Bounding inside the analysis would make every pinned benchmark score a truncated
|
|
1085
|
+
* pipeline, which is the most expensive instrument failure this repository has recorded. The
|
|
1086
|
+
* benchmark keeps seeing exactly what it sees today; only the conversation gets the short form.
|
|
1087
|
+
*
|
|
1088
|
+
* What is dropped is COUNTED, at whatever depth it was dropped. A response that quietly shrank
|
|
1089
|
+
* would be a worse lie than a long one.
|
|
1090
|
+
*/
|
|
1091
|
+
function boundAnalysis(a) {
|
|
1092
|
+
const src = a;
|
|
1093
|
+
const out = boundValue(src, 0);
|
|
1094
|
+
for (const field of DROPPED_FIELDS)
|
|
1095
|
+
delete out[field];
|
|
1096
|
+
return out;
|
|
1097
|
+
}
|
|
1098
|
+
/** Depth bound: deep enough for the shapes measured, shallow enough to terminate on any input. */
|
|
1099
|
+
const BOUND_MAX_DEPTH = 6;
|
|
1100
|
+
function boundValue(value, depth) {
|
|
1101
|
+
if (depth > BOUND_MAX_DEPTH || value === null || typeof value !== 'object')
|
|
1102
|
+
return trimItem(value);
|
|
1103
|
+
if (Array.isArray(value)) {
|
|
1104
|
+
return value.slice(0, exports.ANALYZE_LIST_CAP).map((v) => boundValue(v, depth + 1));
|
|
1105
|
+
}
|
|
1106
|
+
const src = value;
|
|
1107
|
+
const out = {};
|
|
1108
|
+
for (const [k, v] of Object.entries(src)) {
|
|
1109
|
+
out[k] = boundValue(v, depth + 1);
|
|
1110
|
+
// The counter sits BESIDE the array it describes, at the depth it was cut, so a nested trim is
|
|
1111
|
+
// as visible as a top-level one.
|
|
1112
|
+
if (Array.isArray(v) && v.length > exports.ANALYZE_LIST_CAP)
|
|
1113
|
+
out[`${k}Omitted`] = v.length - exports.ANALYZE_LIST_CAP;
|
|
1114
|
+
}
|
|
1115
|
+
return out;
|
|
1116
|
+
}
|
|
1117
|
+
/** Trim one listed item. A long item is CUT, never discarded — the item's existence is information. */
|
|
1118
|
+
function trimItem(item) {
|
|
1119
|
+
if (typeof item !== 'string' || item.length <= exports.ANALYZE_TEXT_CAP)
|
|
1120
|
+
return item;
|
|
1121
|
+
return `${item.slice(0, exports.ANALYZE_TEXT_CAP)}… [+${item.length - exports.ANALYZE_TEXT_CAP}]`;
|
|
1122
|
+
}
|
|
@@ -100,8 +100,49 @@ export interface OutcomeInput {
|
|
|
100
100
|
/**
|
|
101
101
|
* Attach what actually happened to a prediction that was actually made. An unknown digest is
|
|
102
102
|
* refused: inventing the prediction alongside the outcome would let the record score itself.
|
|
103
|
+
*
|
|
104
|
+
* @implements A-SPEC-578.6
|
|
105
|
+
* MERGES into any outcome already there. It used to REPLACE, and the defect was found by following
|
|
106
|
+
* this repository's own instruction: A-SPEC-578.4 made `test_run` attach the 12 files a slice
|
|
107
|
+
* actually changed, AGENTS.md then said "call maintenance_outcome yourself only to add a judged
|
|
108
|
+
* classification" — and that call left `actualFiles: []`. The documented workflow destroyed the
|
|
109
|
+
* data it existed to complete.
|
|
110
|
+
*
|
|
111
|
+
* Omission and erasure are DIFFERENT ACTS: a field left out keeps its value, a field given as an
|
|
112
|
+
* empty array is cleared. Without the second, a wrong record could never be corrected; without the
|
|
113
|
+
* first, completing a record destroys it.
|
|
103
114
|
*/
|
|
104
115
|
export declare function recordOutcome(dir: string, digest: string, outcome: OutcomeInput, recordedAt: string): EvidenceArtifact;
|
|
116
|
+
/**
|
|
117
|
+
* Predictions still waiting to be scored — the loop's missing second half.
|
|
118
|
+
*
|
|
119
|
+
* `since` bounds the window: an analysis recorded before the previous evidence run belonged to a
|
|
120
|
+
* previous slice, and attributing today's changes to it would be contamination rather than
|
|
121
|
+
* measurement. The boundary is EXCLUSIVE (`> since`) because the previous run already saw that
|
|
122
|
+
* instant.
|
|
123
|
+
*/
|
|
124
|
+
export declare function openArtifacts(dir: string, since?: string): EvidenceArtifact[];
|
|
125
|
+
/**
|
|
126
|
+
* @implements A-SPEC-578.4
|
|
127
|
+
* Attach what a slice ACTUALLY touched to the predictions it made.
|
|
128
|
+
*
|
|
129
|
+
* Measured 2026-09-09: `maintenance_calibration` had reported `n:0, artifacts:0` since the loop was
|
|
130
|
+
* built, because `maintenance_analyze({persist:true})` is instructed (AGENTS.md step 3) while
|
|
131
|
+
* `maintenance_outcome` is instructed nowhere at all. The write path was healthy the whole time —
|
|
132
|
+
* one call produced an artifact immediately. What was missing was the call.
|
|
133
|
+
*
|
|
134
|
+
* TWO REFUSALS make this a measurement rather than a self-graded exam:
|
|
135
|
+
* - `actualClassification` is never filled in. That is a judgement, and a loop that supplies its
|
|
136
|
+
* own judgement scores itself; the bins stay `insufficient-data` and only the objective half —
|
|
137
|
+
* file-level false positives and negatives — accrues.
|
|
138
|
+
* - An empty change set closes nothing. Attributing "nothing changed" would turn every prediction
|
|
139
|
+
* into a false positive: a measurement of nothing, recorded as a failure of everything.
|
|
140
|
+
*/
|
|
141
|
+
export declare function closeOpenArtifacts(dir: string, actual: {
|
|
142
|
+
files: string[];
|
|
143
|
+
}, recordedAt: string, since?: string): {
|
|
144
|
+
closed: string[];
|
|
145
|
+
};
|
|
105
146
|
export interface CalibrationBin {
|
|
106
147
|
lower: number;
|
|
107
148
|
upper: number;
|
|
@@ -39,6 +39,8 @@ exports.artifactFrom = artifactFrom;
|
|
|
39
39
|
exports.writeArtifact = writeArtifact;
|
|
40
40
|
exports.readArtifacts = readArtifacts;
|
|
41
41
|
exports.recordOutcome = recordOutcome;
|
|
42
|
+
exports.openArtifacts = openArtifacts;
|
|
43
|
+
exports.closeOpenArtifacts = closeOpenArtifacts;
|
|
42
44
|
exports.computeCalibration = computeCalibration;
|
|
43
45
|
// @implements A-SPEC-271
|
|
44
46
|
// @implements A-SPEC-268
|
|
@@ -170,6 +172,17 @@ function readArtifacts(dir) {
|
|
|
170
172
|
/**
|
|
171
173
|
* Attach what actually happened to a prediction that was actually made. An unknown digest is
|
|
172
174
|
* refused: inventing the prediction alongside the outcome would let the record score itself.
|
|
175
|
+
*
|
|
176
|
+
* @implements A-SPEC-578.6
|
|
177
|
+
* MERGES into any outcome already there. It used to REPLACE, and the defect was found by following
|
|
178
|
+
* this repository's own instruction: A-SPEC-578.4 made `test_run` attach the 12 files a slice
|
|
179
|
+
* actually changed, AGENTS.md then said "call maintenance_outcome yourself only to add a judged
|
|
180
|
+
* classification" — and that call left `actualFiles: []`. The documented workflow destroyed the
|
|
181
|
+
* data it existed to complete.
|
|
182
|
+
*
|
|
183
|
+
* Omission and erasure are DIFFERENT ACTS: a field left out keeps its value, a field given as an
|
|
184
|
+
* empty array is cleared. Without the second, a wrong record could never be corrected; without the
|
|
185
|
+
* first, completing a record destroys it.
|
|
173
186
|
*/
|
|
174
187
|
function recordOutcome(dir, digest, outcome, recordedAt) {
|
|
175
188
|
const file = path.join(dir, digestFilename(digest));
|
|
@@ -177,19 +190,73 @@ function recordOutcome(dir, digest, outcome, recordedAt) {
|
|
|
177
190
|
throw new Error(`Refusing to record an outcome for an unknown analysis digest: ${digest}`);
|
|
178
191
|
}
|
|
179
192
|
const artifact = JSON.parse(fs.readFileSync(file, 'utf8'));
|
|
193
|
+
const prev = artifact.outcome;
|
|
194
|
+
const merged = (given, before) => given === undefined ? (before ?? []) : sortedUnique(given);
|
|
195
|
+
const classification = outcome.actualClassification ?? prev?.actualClassification;
|
|
180
196
|
const updated = {
|
|
181
197
|
...artifact,
|
|
182
198
|
outcome: {
|
|
183
199
|
recordedAt,
|
|
184
|
-
actualFiles:
|
|
185
|
-
actualSymbols:
|
|
186
|
-
actualTests:
|
|
187
|
-
...(
|
|
200
|
+
actualFiles: merged(outcome.actualFiles, prev?.actualFiles),
|
|
201
|
+
actualSymbols: merged(outcome.actualSymbols, prev?.actualSymbols),
|
|
202
|
+
actualTests: merged(outcome.actualTests, prev?.actualTests),
|
|
203
|
+
...(classification ? { actualClassification: classification } : {}),
|
|
188
204
|
},
|
|
189
205
|
};
|
|
190
206
|
writeAtomic(file, `${JSON.stringify(updated, null, 2)}\n`);
|
|
191
207
|
return updated;
|
|
192
208
|
}
|
|
209
|
+
// @implements A-SPEC-578.4
|
|
210
|
+
/**
|
|
211
|
+
* Predictions still waiting to be scored — the loop's missing second half.
|
|
212
|
+
*
|
|
213
|
+
* `since` bounds the window: an analysis recorded before the previous evidence run belonged to a
|
|
214
|
+
* previous slice, and attributing today's changes to it would be contamination rather than
|
|
215
|
+
* measurement. The boundary is EXCLUSIVE (`> since`) because the previous run already saw that
|
|
216
|
+
* instant.
|
|
217
|
+
*/
|
|
218
|
+
function openArtifacts(dir, since) {
|
|
219
|
+
const { artifacts } = readArtifacts(dir);
|
|
220
|
+
return artifacts.filter((a) => {
|
|
221
|
+
if (a.outcome !== undefined)
|
|
222
|
+
return false;
|
|
223
|
+
if (since === undefined)
|
|
224
|
+
return true;
|
|
225
|
+
const t = Date.parse(a.recordedAt);
|
|
226
|
+
const s = Date.parse(since);
|
|
227
|
+
return !Number.isFinite(t) || !Number.isFinite(s) ? true : t > s;
|
|
228
|
+
});
|
|
229
|
+
}
|
|
230
|
+
/**
|
|
231
|
+
* @implements A-SPEC-578.4
|
|
232
|
+
* Attach what a slice ACTUALLY touched to the predictions it made.
|
|
233
|
+
*
|
|
234
|
+
* Measured 2026-09-09: `maintenance_calibration` had reported `n:0, artifacts:0` since the loop was
|
|
235
|
+
* built, because `maintenance_analyze({persist:true})` is instructed (AGENTS.md step 3) while
|
|
236
|
+
* `maintenance_outcome` is instructed nowhere at all. The write path was healthy the whole time —
|
|
237
|
+
* one call produced an artifact immediately. What was missing was the call.
|
|
238
|
+
*
|
|
239
|
+
* TWO REFUSALS make this a measurement rather than a self-graded exam:
|
|
240
|
+
* - `actualClassification` is never filled in. That is a judgement, and a loop that supplies its
|
|
241
|
+
* own judgement scores itself; the bins stay `insufficient-data` and only the objective half —
|
|
242
|
+
* file-level false positives and negatives — accrues.
|
|
243
|
+
* - An empty change set closes nothing. Attributing "nothing changed" would turn every prediction
|
|
244
|
+
* into a false positive: a measurement of nothing, recorded as a failure of everything.
|
|
245
|
+
*/
|
|
246
|
+
function closeOpenArtifacts(dir, actual, recordedAt, since) {
|
|
247
|
+
const files = Array.isArray(actual?.files) ? actual.files : [];
|
|
248
|
+
if (files.length === 0)
|
|
249
|
+
return { closed: [] };
|
|
250
|
+
const closed = [];
|
|
251
|
+
for (const a of openArtifacts(dir, since)) {
|
|
252
|
+
try {
|
|
253
|
+
recordOutcome(dir, a.digest, { actualFiles: files }, recordedAt);
|
|
254
|
+
closed.push(a.digest);
|
|
255
|
+
}
|
|
256
|
+
catch { /* an artifact we cannot write is not a reason to abandon the rest */ }
|
|
257
|
+
}
|
|
258
|
+
return { closed };
|
|
259
|
+
}
|
|
193
260
|
const BIN_EDGES = [0, 0.2, 0.4, 0.6, 0.8, 1];
|
|
194
261
|
/**
|
|
195
262
|
* Reliability bins plus a Brier score. `minSamples` is the honesty knob: below it a bin reports
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @implements A-SPEC-579
|
|
3
|
+
* npm 12 blocks dependency install scripts unless the ROOT package.json's `allowScripts` covers
|
|
4
|
+
* them ("Install commands silently skip lifecycle scripts for any dependency that does not have a
|
|
5
|
+
* matching entry" — npm docs, cli/v12/commands/npm-install-scripts). Measured 2026-09-09 on
|
|
6
|
+
* Windows (npm 12.0.1, Node 24.19.0): `npm ci` succeeded, nine install scripts were skipped, and
|
|
7
|
+
* better-sqlite3 had no binary — while the eight tree-sitter packages still loaded, because their
|
|
8
|
+
* `node-gyp-build` runtime loader finds `prebuilds/<platform>-<arch>` without the script ever
|
|
9
|
+
* running. So the minimum approval this repository needs is ONE package, pinned to the lockfile
|
|
10
|
+
* version so a dependency bump forces a fresh review.
|
|
11
|
+
*
|
|
12
|
+
* This module is the pure re-statement of that rule: no I/O, no process, no platform — the same
|
|
13
|
+
* verdict on every OS, shared by the test that pins package.json and by doctor (A-SPEC-580).
|
|
14
|
+
*/
|
|
15
|
+
/** Dependencies whose install script must actually RUN for the module to load. */
|
|
16
|
+
export declare const NATIVE_INSTALL_SCRIPT_DEPS: readonly ["better-sqlite3"];
|
|
17
|
+
/** Native dependencies that ship prebuilds and load with the script blocked (measured). */
|
|
18
|
+
export declare const PREBUILT_NATIVE_DEPS: readonly ["tree-sitter", "tree-sitter-typescript", "tree-sitter-python", "tree-sitter-c-sharp", "tree-sitter-java", "tree-sitter-go", "tree-sitter-rust", "tree-sitter-cpp"];
|
|
19
|
+
export type AllowScripts = Record<string, boolean>;
|
|
20
|
+
export type Coverage = 'approved-pinned' | 'approved-unpinned' | 'denied' | 'uncovered';
|
|
21
|
+
export interface LockLike {
|
|
22
|
+
packages?: Record<string, {
|
|
23
|
+
version?: string;
|
|
24
|
+
hasInstallScript?: boolean;
|
|
25
|
+
}>;
|
|
26
|
+
}
|
|
27
|
+
export type PolicyProblem = {
|
|
28
|
+
kind: 'reapproval-required';
|
|
29
|
+
name: string;
|
|
30
|
+
lockVersion: string;
|
|
31
|
+
approved: string[];
|
|
32
|
+
fix: string;
|
|
33
|
+
} | {
|
|
34
|
+
kind: 'unpinned';
|
|
35
|
+
name: string;
|
|
36
|
+
fix: string;
|
|
37
|
+
} | {
|
|
38
|
+
kind: 'denied';
|
|
39
|
+
name: string;
|
|
40
|
+
fix: string;
|
|
41
|
+
} | {
|
|
42
|
+
kind: 'beyond-minimum';
|
|
43
|
+
key: string;
|
|
44
|
+
fix: string;
|
|
45
|
+
} | {
|
|
46
|
+
kind: 'missing-from-lock';
|
|
47
|
+
name: string;
|
|
48
|
+
};
|
|
49
|
+
/** The exact command npm 12 documents for a pinned approval. */
|
|
50
|
+
export declare function approveCommand(name: string, version: string, npmBin?: string): string;
|
|
51
|
+
/**
|
|
52
|
+
* How `allowScripts` covers one package at one version, under npm 12's rules: a pinned `true`
|
|
53
|
+
* covers that version only, a bare `true` covers every version, and a `false` on either key is
|
|
54
|
+
* a denial that survives any approval (npm: deny "survives `approve --all`").
|
|
55
|
+
*/
|
|
56
|
+
export declare function allowScriptsCoverage(allow: AllowScripts | undefined, name: string, version: string): Coverage;
|
|
57
|
+
/**
|
|
58
|
+
* What `npm install-scripts ls` would list: every lock entry with an install script that
|
|
59
|
+
* `allowScripts` does not approve. The root entry (`""`) is the project itself, never a dependency.
|
|
60
|
+
*/
|
|
61
|
+
export declare function blockedInstallScripts(lock: LockLike, allow: AllowScripts | undefined): {
|
|
62
|
+
name: string;
|
|
63
|
+
version: string;
|
|
64
|
+
}[];
|
|
65
|
+
/**
|
|
66
|
+
* This repository's policy, judged: every script-requiring dependency approved AND pinned to the
|
|
67
|
+
* lock version, and nothing else approved. Pure; never throws.
|
|
68
|
+
*/
|
|
69
|
+
export declare function policyVerdict(pkg: {
|
|
70
|
+
allowScripts?: AllowScripts;
|
|
71
|
+
}, lock: LockLike, opts?: {
|
|
72
|
+
npmBin?: string;
|
|
73
|
+
}): {
|
|
74
|
+
ok: boolean;
|
|
75
|
+
problems: PolicyProblem[];
|
|
76
|
+
};
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// @implements A-SPEC-579
|
|
3
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
+
exports.PREBUILT_NATIVE_DEPS = exports.NATIVE_INSTALL_SCRIPT_DEPS = void 0;
|
|
5
|
+
exports.approveCommand = approveCommand;
|
|
6
|
+
exports.allowScriptsCoverage = allowScriptsCoverage;
|
|
7
|
+
exports.blockedInstallScripts = blockedInstallScripts;
|
|
8
|
+
exports.policyVerdict = policyVerdict;
|
|
9
|
+
/**
|
|
10
|
+
* @implements A-SPEC-579
|
|
11
|
+
* npm 12 blocks dependency install scripts unless the ROOT package.json's `allowScripts` covers
|
|
12
|
+
* them ("Install commands silently skip lifecycle scripts for any dependency that does not have a
|
|
13
|
+
* matching entry" — npm docs, cli/v12/commands/npm-install-scripts). Measured 2026-09-09 on
|
|
14
|
+
* Windows (npm 12.0.1, Node 24.19.0): `npm ci` succeeded, nine install scripts were skipped, and
|
|
15
|
+
* better-sqlite3 had no binary — while the eight tree-sitter packages still loaded, because their
|
|
16
|
+
* `node-gyp-build` runtime loader finds `prebuilds/<platform>-<arch>` without the script ever
|
|
17
|
+
* running. So the minimum approval this repository needs is ONE package, pinned to the lockfile
|
|
18
|
+
* version so a dependency bump forces a fresh review.
|
|
19
|
+
*
|
|
20
|
+
* This module is the pure re-statement of that rule: no I/O, no process, no platform — the same
|
|
21
|
+
* verdict on every OS, shared by the test that pins package.json and by doctor (A-SPEC-580).
|
|
22
|
+
*/
|
|
23
|
+
/** Dependencies whose install script must actually RUN for the module to load. */
|
|
24
|
+
exports.NATIVE_INSTALL_SCRIPT_DEPS = ['better-sqlite3'];
|
|
25
|
+
/** Native dependencies that ship prebuilds and load with the script blocked (measured). */
|
|
26
|
+
exports.PREBUILT_NATIVE_DEPS = [
|
|
27
|
+
'tree-sitter', 'tree-sitter-typescript', 'tree-sitter-python', 'tree-sitter-c-sharp',
|
|
28
|
+
'tree-sitter-java', 'tree-sitter-go', 'tree-sitter-rust', 'tree-sitter-cpp',
|
|
29
|
+
];
|
|
30
|
+
/** The exact command npm 12 documents for a pinned approval. */
|
|
31
|
+
function approveCommand(name, version, npmBin = 'npm') {
|
|
32
|
+
return `${npmBin} install-scripts approve ${name}@${version}`;
|
|
33
|
+
}
|
|
34
|
+
/** Split an allowScripts key into its package name and optional pinned version (scopes kept). */
|
|
35
|
+
function splitKey(key) {
|
|
36
|
+
const at = key.lastIndexOf('@');
|
|
37
|
+
if (at <= 0)
|
|
38
|
+
return { name: key }; // '@scope/pkg' or 'pkg'
|
|
39
|
+
return { name: key.slice(0, at), version: key.slice(at + 1) };
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* How `allowScripts` covers one package at one version, under npm 12's rules: a pinned `true`
|
|
43
|
+
* covers that version only, a bare `true` covers every version, and a `false` on either key is
|
|
44
|
+
* a denial that survives any approval (npm: deny "survives `approve --all`").
|
|
45
|
+
*/
|
|
46
|
+
function allowScriptsCoverage(allow, name, version) {
|
|
47
|
+
if (!allow)
|
|
48
|
+
return 'uncovered';
|
|
49
|
+
const pinned = allow[`${name}@${version}`];
|
|
50
|
+
const bare = allow[name];
|
|
51
|
+
if (pinned === false || bare === false)
|
|
52
|
+
return 'denied';
|
|
53
|
+
if (pinned === true)
|
|
54
|
+
return 'approved-pinned';
|
|
55
|
+
if (bare === true)
|
|
56
|
+
return 'approved-unpinned';
|
|
57
|
+
return 'uncovered';
|
|
58
|
+
}
|
|
59
|
+
/** The package name of a lockfile `packages` key: the segment after its LAST `node_modules/`. */
|
|
60
|
+
function lockEntryName(key) {
|
|
61
|
+
const idx = key.lastIndexOf('node_modules/');
|
|
62
|
+
if (idx < 0)
|
|
63
|
+
return null;
|
|
64
|
+
const name = key.slice(idx + 'node_modules/'.length);
|
|
65
|
+
return name === '' ? null : name;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* What `npm install-scripts ls` would list: every lock entry with an install script that
|
|
69
|
+
* `allowScripts` does not approve. The root entry (`""`) is the project itself, never a dependency.
|
|
70
|
+
*/
|
|
71
|
+
function blockedInstallScripts(lock, allow) {
|
|
72
|
+
const out = [];
|
|
73
|
+
for (const [key, entry] of Object.entries(lock.packages ?? {})) {
|
|
74
|
+
if (!entry?.hasInstallScript)
|
|
75
|
+
continue;
|
|
76
|
+
const name = lockEntryName(key);
|
|
77
|
+
if (name === null)
|
|
78
|
+
continue;
|
|
79
|
+
const version = entry.version ?? '';
|
|
80
|
+
const cov = allowScriptsCoverage(allow, name, version);
|
|
81
|
+
if (cov !== 'approved-pinned' && cov !== 'approved-unpinned')
|
|
82
|
+
out.push({ name, version });
|
|
83
|
+
}
|
|
84
|
+
return out;
|
|
85
|
+
}
|
|
86
|
+
/** The top-level lock version of a dependency (`node_modules/<name>`), if present. */
|
|
87
|
+
function lockVersionOf(lock, name) {
|
|
88
|
+
return lock.packages?.[`node_modules/${name}`]?.version;
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* This repository's policy, judged: every script-requiring dependency approved AND pinned to the
|
|
92
|
+
* lock version, and nothing else approved. Pure; never throws.
|
|
93
|
+
*/
|
|
94
|
+
function policyVerdict(pkg, lock, opts) {
|
|
95
|
+
const npmBin = opts?.npmBin ?? 'npm';
|
|
96
|
+
const allow = pkg.allowScripts;
|
|
97
|
+
const problems = [];
|
|
98
|
+
for (const name of exports.NATIVE_INSTALL_SCRIPT_DEPS) {
|
|
99
|
+
const lockVersion = lockVersionOf(lock, name);
|
|
100
|
+
if (lockVersion === undefined) {
|
|
101
|
+
problems.push({ kind: 'missing-from-lock', name });
|
|
102
|
+
continue;
|
|
103
|
+
}
|
|
104
|
+
const fix = approveCommand(name, lockVersion, npmBin);
|
|
105
|
+
switch (allowScriptsCoverage(allow, name, lockVersion)) {
|
|
106
|
+
case 'approved-pinned': break;
|
|
107
|
+
case 'approved-unpinned':
|
|
108
|
+
problems.push({ kind: 'unpinned', name, fix });
|
|
109
|
+
break;
|
|
110
|
+
case 'denied':
|
|
111
|
+
problems.push({ kind: 'denied', name, fix });
|
|
112
|
+
break;
|
|
113
|
+
case 'uncovered': {
|
|
114
|
+
const approved = Object.entries(allow ?? {})
|
|
115
|
+
.filter(([k, v]) => v === true && splitKey(k).name === name && splitKey(k).version !== undefined)
|
|
116
|
+
.map(([k]) => splitKey(k).version);
|
|
117
|
+
problems.push({ kind: 'reapproval-required', name, lockVersion, approved, fix });
|
|
118
|
+
break;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
const minimum = new Set(exports.NATIVE_INSTALL_SCRIPT_DEPS);
|
|
123
|
+
for (const [key, value] of Object.entries(allow ?? {})) {
|
|
124
|
+
if (value !== true)
|
|
125
|
+
continue; // a denial is never "beyond" the minimum
|
|
126
|
+
if (!minimum.has(splitKey(key).name)) {
|
|
127
|
+
problems.push({ kind: 'beyond-minimum', key, fix: `Remove "${key}" from allowScripts in package.json — it loads from its shipped prebuilds without an install script.` });
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
return { ok: problems.length === 0, problems };
|
|
131
|
+
}
|
|
@@ -8,3 +8,9 @@
|
|
|
8
8
|
* platform must never flip a POSIX hint to a Windows-only form.
|
|
9
9
|
*/
|
|
10
10
|
export declare function npxBin(platform?: string): string;
|
|
11
|
+
/**
|
|
12
|
+
* @implements A-SPEC-580
|
|
13
|
+
* The npm twin: the recovery commands doctor emits (`npm install-scripts approve`, `npm rebuild`)
|
|
14
|
+
* hit the same `.ps1` shim on Windows (measured 2026-09-09: npm.ps1 blocked, npm.cmd runs).
|
|
15
|
+
*/
|
|
16
|
+
export declare function npmBin(platform?: string): string;
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
// @implements A-SPEC-542.1
|
|
3
|
+
// @implements A-SPEC-580
|
|
3
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
5
|
exports.npxBin = npxBin;
|
|
6
|
+
exports.npmBin = npmBin;
|
|
5
7
|
/**
|
|
6
8
|
* @implements A-SPEC-542.1
|
|
7
9
|
* The npx binary an OPERATOR can actually run on the platform this process runs on — which is the
|
|
@@ -14,3 +16,11 @@ exports.npxBin = npxBin;
|
|
|
14
16
|
function npxBin(platform = process.platform) {
|
|
15
17
|
return platform === 'win32' ? 'npx.cmd' : 'npx';
|
|
16
18
|
}
|
|
19
|
+
/**
|
|
20
|
+
* @implements A-SPEC-580
|
|
21
|
+
* The npm twin: the recovery commands doctor emits (`npm install-scripts approve`, `npm rebuild`)
|
|
22
|
+
* hit the same `.ps1` shim on Windows (measured 2026-09-09: npm.ps1 blocked, npm.cmd runs).
|
|
23
|
+
*/
|
|
24
|
+
function npmBin(platform = process.platform) {
|
|
25
|
+
return platform === 'win32' ? 'npm.cmd' : 'npm';
|
|
26
|
+
}
|
|
@@ -20,6 +20,10 @@ export interface ReplayOutcome {
|
|
|
20
20
|
truthFiles: string[];
|
|
21
21
|
selectedTests: string[];
|
|
22
22
|
truthTests: string[];
|
|
23
|
+
/** Symbols the change actually touched (parent-time ranges ∩ changed lines). */
|
|
24
|
+
truthSymbols?: string[];
|
|
25
|
+
/** Predicted symbols, best first — the qualified names the emitted candidates carried. */
|
|
26
|
+
rankedSymbols?: string[];
|
|
23
27
|
}
|
|
24
28
|
export interface CutMetrics {
|
|
25
29
|
/** Share of cases where at least one truth file appears in the first K. */
|
|
@@ -33,6 +37,8 @@ export interface EvaluationMetrics {
|
|
|
33
37
|
scorableCases: number;
|
|
34
38
|
topK: Record<number, CutMetrics>;
|
|
35
39
|
testRecall: number | null;
|
|
40
|
+
symbolTopK: Record<number, CutMetrics>;
|
|
41
|
+
symbolCases: number;
|
|
36
42
|
}
|
|
37
43
|
export declare function evaluationMetrics(outcomes: readonly ReplayOutcome[]): EvaluationMetrics;
|
|
38
44
|
/**
|
|
@@ -42,7 +42,24 @@ function evaluationMetrics(outcomes) {
|
|
|
42
42
|
}
|
|
43
43
|
const testScorable = outcomes.filter((o) => o.truthTests.length > 0);
|
|
44
44
|
const testRecall = mean(testScorable.map((o) => o.truthTests.filter((t) => o.selectedTests.includes(t)).length / o.truthTests.length));
|
|
45
|
-
|
|
45
|
+
// @implements A-SPEC-573.3
|
|
46
|
+
const symbolScorable = outcomes.filter((o) => (o.truthSymbols?.length ?? 0) > 0);
|
|
47
|
+
const symbolTopK = {};
|
|
48
|
+
for (const k of exports.TOP_K) {
|
|
49
|
+
const hits = [];
|
|
50
|
+
const recalls = [];
|
|
51
|
+
const precisions = [];
|
|
52
|
+
for (const o of symbolScorable) {
|
|
53
|
+
const cut = (o.rankedSymbols ?? []).slice(0, k);
|
|
54
|
+
const found = cut.filter((s2) => o.truthSymbols.includes(s2)).length;
|
|
55
|
+
hits.push(found > 0 ? 1 : 0);
|
|
56
|
+
recalls.push(found / o.truthSymbols.length);
|
|
57
|
+
precisions.push(cut.length === 0 ? 0 : found / cut.length);
|
|
58
|
+
}
|
|
59
|
+
symbolTopK[k] = { hitRate: mean(hits), recall: mean(recalls), precision: mean(precisions) };
|
|
60
|
+
}
|
|
61
|
+
return { cases: outcomes.length, scorableCases: scorable.length, topK, testRecall,
|
|
62
|
+
symbolTopK, symbolCases: symbolScorable.length };
|
|
46
63
|
}
|
|
47
64
|
/**
|
|
48
65
|
* Impact had no benchmark until this existed, so an impact regression could not fail anything.
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The names behind a red release gate.
|
|
3
|
+
*
|
|
4
|
+
* Measured 2026-09-10: `npm publish` refused with "스위트가 붉습니다" and nothing else, and finding
|
|
5
|
+
* out which test had failed cost two more full-suite runs. `test_run` learned to name its failures
|
|
6
|
+
* in REQ-577; the release gate had not. A gate that says "red" without saying what is a gate that
|
|
7
|
+
* makes the next person guess.
|
|
8
|
+
*/
|
|
9
|
+
/** How many names a refusal lists before it counts the rest. */
|
|
10
|
+
export declare const FAILED_NAME_CAP = 10;
|
|
11
|
+
/**
|
|
12
|
+
* Pull the failing test names out of a jest run's output.
|
|
13
|
+
*
|
|
14
|
+
* PURE, and forgiving: output it cannot read yields an empty list so the caller keeps its existing
|
|
15
|
+
* refusal rather than replacing a working message with an empty one. Jest prints the summary block
|
|
16
|
+
* twice on some configurations, so names are de-duplicated — the same failure listed twice is one
|
|
17
|
+
* failure.
|
|
18
|
+
*/
|
|
19
|
+
export declare function failedTestNames(output: string): string[];
|