@holmes-lab/holmes-kit 0.19.0 → 0.19.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +80 -0
- package/dist/.build-id +1 -1
- package/dist/holmes/cli/agents.d.ts +22 -0
- package/dist/holmes/cli/agents.js +76 -1
- package/dist/holmes/cli/approve.js +6 -1
- package/dist/holmes/cli/doctor.d.ts +36 -1
- package/dist/holmes/cli/doctor.js +182 -35
- package/dist/holmes/cli/index.js +7 -1
- package/dist/holmes/cli/init.js +12 -0
- package/dist/holmes/cli/native-deps.d.ts +65 -0
- package/dist/holmes/cli/native-deps.js +131 -0
- package/dist/holmes/cpg/cycle-observation.d.ts +65 -0
- package/dist/holmes/cpg/cycle-observation.js +146 -0
- package/dist/holmes/governance/approval-queue.d.ts +23 -4
- package/dist/holmes/governance/approval-queue.js +44 -6
- package/dist/holmes/hooks/stop.d.ts +15 -0
- package/dist/holmes/hooks/stop.js +46 -3
- package/dist/holmes/mcp/handlers.d.ts +2 -0
- package/dist/holmes/mcp/handlers.js +29 -2
- package/dist/holmes/mcp/maintenance-analyze.d.ts +37 -0
- package/dist/holmes/mcp/maintenance-analyze.js +73 -1
- package/dist/holmes/mcp/maintenance-evidence.d.ts +41 -0
- package/dist/holmes/mcp/maintenance-evidence.js +71 -4
- package/dist/holmes/project/install-scripts-policy.d.ts +76 -0
- package/dist/holmes/project/install-scripts-policy.js +131 -0
- package/dist/holmes/project/npx-bin.d.ts +6 -0
- package/dist/holmes/project/npx-bin.js +10 -0
- package/dist/holmes/review/failed-test-names.d.ts +19 -0
- package/dist/holmes/review/failed-test-names.js +43 -0
- package/dist/holmes/review/run-replay.d.ts +23 -0
- package/dist/holmes/review/run-replay.js +30 -0
- package/dist/holmes/review/test-runner.d.ts +27 -0
- package/dist/holmes/review/test-runner.js +59 -3
- package/docs/install-guide.md +54 -5
- package/package.json +4 -1
|
@@ -490,6 +490,7 @@ declare function makeRawHandlers(store: SpecStore, opts?: ElicitOpts): {
|
|
|
490
490
|
survivors: import("../spec/kills").Mutation[];
|
|
491
491
|
};
|
|
492
492
|
} | {
|
|
493
|
+
calibrationClosed?: string[] | undefined;
|
|
493
494
|
baselineRecorded?: string | undefined;
|
|
494
495
|
scopeFallback?: "full" | undefined;
|
|
495
496
|
tier: import("../rtm/test-scope").RegressionTier;
|
|
@@ -518,6 +519,7 @@ declare function makeRawHandlers(store: SpecStore, opts?: ElicitOpts): {
|
|
|
518
519
|
changedFiles?: string[];
|
|
519
520
|
}): Promise<MaintenanceAnalysis & {
|
|
520
521
|
persistedTo?: string;
|
|
522
|
+
bounded?: boolean;
|
|
521
523
|
}>;
|
|
522
524
|
maintenance_outcome(a: {
|
|
523
525
|
root: string;
|
|
@@ -2367,6 +2367,9 @@ function makeRawHandlers(store, opts) {
|
|
|
2367
2367
|
// Record ONLY an actually-executed, GREEN run (review C4/C7): a red suite must not stand as
|
|
2368
2368
|
// coverage evidence at the moment the code is broken, and a no-op run must not overwrite a real
|
|
2369
2369
|
// record with a dishonest `passed: true` over an empty result.
|
|
2370
|
+
// @implements A-SPEC-578.4 — read BEFORE the write below replaces it: the calibration window
|
|
2371
|
+
// is "since the previous evidence run", and this line is the only moment that value exists.
|
|
2372
|
+
const previousEvidenceTs = (0, test_evidence_1.readTestEvidence)(root)?.ts;
|
|
2370
2373
|
const verified = (0, baseline_1.shouldRecordBaseline)(result);
|
|
2371
2374
|
if (verified) {
|
|
2372
2375
|
(0, test_evidence_1.writeTestEvidence)(root, { ts: new Date().toISOString(), head, tier: testScope.tier, passed: true, executedByAspec });
|
|
@@ -2382,6 +2385,19 @@ function makeRawHandlers(store, opts) {
|
|
|
2382
2385
|
// actually executed and passed. A red or skipped run must never become the reference point for
|
|
2383
2386
|
// "since the last verified state" — that would silently narrow every later scope against a
|
|
2384
2387
|
// state nobody verified.
|
|
2388
|
+
// @implements A-SPEC-578.4 — close the calibration loop on real work.
|
|
2389
|
+
// The window is the PREVIOUS evidence run's timestamp, read before this run overwrote it —
|
|
2390
|
+
// an analysis older than that belonged to a previous slice, and attributing today's changes
|
|
2391
|
+
// to it would be contamination rather than measurement. Its own try: an observation that
|
|
2392
|
+
// could change `passed` would be a gate wearing another name.
|
|
2393
|
+
let calibrationClosed;
|
|
2394
|
+
try {
|
|
2395
|
+
const closed = (0, maintenance_evidence_1.closeOpenArtifacts)(path.join(root, maintenance_evidence_1.EVIDENCE_DIR), { files: changedFiles }, new Date().toISOString(), previousEvidenceTs).closed;
|
|
2396
|
+
if (closed.length > 0)
|
|
2397
|
+
calibrationClosed = closed;
|
|
2398
|
+
}
|
|
2399
|
+
catch { /* the loop is observation; it never touches the verdict */ }
|
|
2400
|
+
// @implements A-SPEC-128
|
|
2385
2401
|
let baseline;
|
|
2386
2402
|
if (verified) {
|
|
2387
2403
|
baseline = a.mark ?? DEFAULT_BASELINE;
|
|
@@ -2393,7 +2409,8 @@ function makeRawHandlers(store, opts) {
|
|
|
2393
2409
|
ranFiles: result.ranFiles, executedByAspec, tail: result.tail,
|
|
2394
2410
|
// @implements A-SPEC-130 — the remediation rides in the answer: these are the files to anchor.
|
|
2395
2411
|
unresolvedFiles: testScope.unresolvedFiles,
|
|
2396
|
-
changeSource, ...(scopeFallback ? { scopeFallback } : {}), ...(baseline ? { baselineRecorded: baseline } : {})
|
|
2412
|
+
changeSource, ...(scopeFallback ? { scopeFallback } : {}), ...(baseline ? { baselineRecorded: baseline } : {}),
|
|
2413
|
+
...(calibrationClosed ? { calibrationClosed } : {}) };
|
|
2397
2414
|
},
|
|
2398
2415
|
async issue_localize(a) {
|
|
2399
2416
|
assertSpecStoreReachable('issue_localize', store, a.root); // @implements A-SPEC-419
|
|
@@ -2643,9 +2660,19 @@ function makeRawHandlers(store, opts) {
|
|
|
2643
2660
|
if (digest)
|
|
2644
2661
|
fileDigests[file] = digest;
|
|
2645
2662
|
}
|
|
2663
|
+
// The artifact takes the WHOLE analysis, never the bounded one below: a file has no
|
|
2664
|
+
// context window, and the calibration that scores this prediction must score what the
|
|
2665
|
+
// product actually predicted.
|
|
2646
2666
|
result.persistedTo = (0, maintenance_evidence_1.writeArtifact)(path.join(root, maintenance_evidence_1.EVIDENCE_DIR), (0, maintenance_evidence_1.artifactFrom)(analysis, new Date().toISOString(), fileDigests));
|
|
2647
2667
|
}
|
|
2648
|
-
|
|
2668
|
+
// @implements A-SPEC-578.5 — the response, and only the response, is bounded. Measured
|
|
2669
|
+
// 2026-09-09: the full shape is 187,174 characters (~47k tokens) and the harness refuses
|
|
2670
|
+
// it, which is why AGENTS.md step 3 had never once been obeyed with `persist: true`.
|
|
2671
|
+
// `persistedTo` rides along so the caller knows where the whole thing is.
|
|
2672
|
+
const bounded = (0, maintenance_analyze_1.boundAnalysis)(result);
|
|
2673
|
+
if (JSON.stringify(bounded).length !== JSON.stringify(result).length)
|
|
2674
|
+
bounded.bounded = true;
|
|
2675
|
+
return bounded;
|
|
2649
2676
|
}
|
|
2650
2677
|
finally {
|
|
2651
2678
|
graph.close();
|
|
@@ -444,3 +444,40 @@ export declare function unquoteGitPath(line: string): string;
|
|
|
444
444
|
* reproduce and compare an analysis from the exact same captured inputs.
|
|
445
445
|
*/
|
|
446
446
|
export declare function analyzeMaintenance(input: MaintenanceAnalysisInput): MaintenanceAnalysis;
|
|
447
|
+
/**
|
|
448
|
+
* Items a response lists before it starts counting instead.
|
|
449
|
+
*
|
|
450
|
+
* TEN, because that is the unit this product is measured in: localization emits Top-10 and every
|
|
451
|
+
* recall figure in this repository is recall@10. Aligning the response cap with the emission unit
|
|
452
|
+
* is a reason; picking a number that happens to hit a size target is a knob. Measured on the real
|
|
453
|
+
* 187,174-character response: cap 20 gave 34,811 with `intent` alone at 11,037 (32%), cap 10 fits.
|
|
454
|
+
*/
|
|
455
|
+
export declare const ANALYZE_LIST_CAP = 10;
|
|
456
|
+
/** Characters one listed item keeps. */
|
|
457
|
+
export declare const ANALYZE_TEXT_CAP = 400;
|
|
458
|
+
/**
|
|
459
|
+
* @implements A-SPEC-578.5
|
|
460
|
+
* The RESPONSE-EDGE projection. The artifact on disk keeps everything.
|
|
461
|
+
*
|
|
462
|
+
* Measured 2026-09-09: one `maintenance_analyze` call returns 187,174 characters (~47k tokens) and
|
|
463
|
+
* the harness refuses it outright. AGENTS.md step 3 instructs every agent to make that call before
|
|
464
|
+
* editing source — and across 2,779 spec approvals `persist:true` had never been used once. An
|
|
465
|
+
* instruction nobody can afford to follow is not an instruction.
|
|
466
|
+
*
|
|
467
|
+
* The cap is applied RECURSIVELY, which the first cut got wrong. Capping only the top level took
|
|
468
|
+
* the response from 187,174 to 142,701 and no further, because the weight is in NESTED DUPLICATES:
|
|
469
|
+
* `impacts.test` and `testScope.impactedTestFiles` carry the same 158 paths, and `impacts.contract`
|
|
470
|
+
* and `testScope.impactedSpecs` the same 160 spec ids. A cap that stops at depth one caps the
|
|
471
|
+
* cheapest arrays in the document.
|
|
472
|
+
*
|
|
473
|
+
* NEVER CALLED FROM `analyzeMaintenance`. The design-time advisory named the replay benchmark
|
|
474
|
+
* (`run-replay.ts`, `replay-corpus.ts`, `commit-text.ts`, `temporal-prior.ts`) as readers of this
|
|
475
|
+
* type, and among their anchors sits A-SPEC-402 — "the numbers reported are not what the product
|
|
476
|
+
* does". Bounding inside the analysis would make every pinned benchmark score a truncated
|
|
477
|
+
* pipeline, which is the most expensive instrument failure this repository has recorded. The
|
|
478
|
+
* benchmark keeps seeing exactly what it sees today; only the conversation gets the short form.
|
|
479
|
+
*
|
|
480
|
+
* What is dropped is COUNTED, at whatever depth it was dropped. A response that quietly shrank
|
|
481
|
+
* would be a worse lie than a long one.
|
|
482
|
+
*/
|
|
483
|
+
export declare function boundAnalysis(a: MaintenanceAnalysis): MaintenanceAnalysis;
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.PRIMARY_TIER_WIDTH = exports.EVIDENCE_DIMENSIONS = exports.MAINTENANCE_ANALYSIS_EXTRACTOR = exports.MAINTENANCE_ANALYSIS_SCHEMA = void 0;
|
|
3
|
+
exports.ANALYZE_TEXT_CAP = exports.ANALYZE_LIST_CAP = exports.PRIMARY_TIER_WIDTH = exports.EVIDENCE_DIMENSIONS = exports.MAINTENANCE_ANALYSIS_EXTRACTOR = exports.MAINTENANCE_ANALYSIS_SCHEMA = void 0;
|
|
4
4
|
exports.unquoteGitPath = unquoteGitPath;
|
|
5
5
|
exports.analyzeMaintenance = analyzeMaintenance;
|
|
6
|
+
exports.boundAnalysis = boundAnalysis;
|
|
6
7
|
// @implements A-SPEC-295
|
|
7
8
|
// @implements A-SPEC-294
|
|
8
9
|
// @implements A-SPEC-293
|
|
@@ -1048,3 +1049,74 @@ function buildAblation(a) {
|
|
|
1048
1049
|
], truth, a.primaryTier, a.impactedTier, a.importedByTier);
|
|
1049
1050
|
return { graphOff, current, enhanced };
|
|
1050
1051
|
}
|
|
1052
|
+
// @implements A-SPEC-578.5
|
|
1053
|
+
/**
|
|
1054
|
+
* Items a response lists before it starts counting instead.
|
|
1055
|
+
*
|
|
1056
|
+
* TEN, because that is the unit this product is measured in: localization emits Top-10 and every
|
|
1057
|
+
* recall figure in this repository is recall@10. Aligning the response cap with the emission unit
|
|
1058
|
+
* is a reason; picking a number that happens to hit a size target is a knob. Measured on the real
|
|
1059
|
+
* 187,174-character response: cap 20 gave 34,811 with `intent` alone at 11,037 (32%), cap 10 fits.
|
|
1060
|
+
*/
|
|
1061
|
+
exports.ANALYZE_LIST_CAP = 10;
|
|
1062
|
+
/** Characters one listed item keeps. */
|
|
1063
|
+
exports.ANALYZE_TEXT_CAP = 400;
|
|
1064
|
+
/** Dropped whole: another tool answers this question, and it is 41,619 characters of the answer. */
|
|
1065
|
+
const DROPPED_FIELDS = ['contextBundle'];
|
|
1066
|
+
/**
|
|
1067
|
+
* @implements A-SPEC-578.5
|
|
1068
|
+
* The RESPONSE-EDGE projection. The artifact on disk keeps everything.
|
|
1069
|
+
*
|
|
1070
|
+
* Measured 2026-09-09: one `maintenance_analyze` call returns 187,174 characters (~47k tokens) and
|
|
1071
|
+
* the harness refuses it outright. AGENTS.md step 3 instructs every agent to make that call before
|
|
1072
|
+
* editing source — and across 2,779 spec approvals `persist:true` had never been used once. An
|
|
1073
|
+
* instruction nobody can afford to follow is not an instruction.
|
|
1074
|
+
*
|
|
1075
|
+
* The cap is applied RECURSIVELY, which the first cut got wrong. Capping only the top level took
|
|
1076
|
+
* the response from 187,174 to 142,701 and no further, because the weight is in NESTED DUPLICATES:
|
|
1077
|
+
* `impacts.test` and `testScope.impactedTestFiles` carry the same 158 paths, and `impacts.contract`
|
|
1078
|
+
* and `testScope.impactedSpecs` the same 160 spec ids. A cap that stops at depth one caps the
|
|
1079
|
+
* cheapest arrays in the document.
|
|
1080
|
+
*
|
|
1081
|
+
* NEVER CALLED FROM `analyzeMaintenance`. The design-time advisory named the replay benchmark
|
|
1082
|
+
* (`run-replay.ts`, `replay-corpus.ts`, `commit-text.ts`, `temporal-prior.ts`) as readers of this
|
|
1083
|
+
* type, and among their anchors sits A-SPEC-402 — "the numbers reported are not what the product
|
|
1084
|
+
* does". Bounding inside the analysis would make every pinned benchmark score a truncated
|
|
1085
|
+
* pipeline, which is the most expensive instrument failure this repository has recorded. The
|
|
1086
|
+
* benchmark keeps seeing exactly what it sees today; only the conversation gets the short form.
|
|
1087
|
+
*
|
|
1088
|
+
* What is dropped is COUNTED, at whatever depth it was dropped. A response that quietly shrank
|
|
1089
|
+
* would be a worse lie than a long one.
|
|
1090
|
+
*/
|
|
1091
|
+
function boundAnalysis(a) {
|
|
1092
|
+
const src = a;
|
|
1093
|
+
const out = boundValue(src, 0);
|
|
1094
|
+
for (const field of DROPPED_FIELDS)
|
|
1095
|
+
delete out[field];
|
|
1096
|
+
return out;
|
|
1097
|
+
}
|
|
1098
|
+
/** Depth bound: deep enough for the shapes measured, shallow enough to terminate on any input. */
|
|
1099
|
+
const BOUND_MAX_DEPTH = 6;
|
|
1100
|
+
function boundValue(value, depth) {
|
|
1101
|
+
if (depth > BOUND_MAX_DEPTH || value === null || typeof value !== 'object')
|
|
1102
|
+
return trimItem(value);
|
|
1103
|
+
if (Array.isArray(value)) {
|
|
1104
|
+
return value.slice(0, exports.ANALYZE_LIST_CAP).map((v) => boundValue(v, depth + 1));
|
|
1105
|
+
}
|
|
1106
|
+
const src = value;
|
|
1107
|
+
const out = {};
|
|
1108
|
+
for (const [k, v] of Object.entries(src)) {
|
|
1109
|
+
out[k] = boundValue(v, depth + 1);
|
|
1110
|
+
// The counter sits BESIDE the array it describes, at the depth it was cut, so a nested trim is
|
|
1111
|
+
// as visible as a top-level one.
|
|
1112
|
+
if (Array.isArray(v) && v.length > exports.ANALYZE_LIST_CAP)
|
|
1113
|
+
out[`${k}Omitted`] = v.length - exports.ANALYZE_LIST_CAP;
|
|
1114
|
+
}
|
|
1115
|
+
return out;
|
|
1116
|
+
}
|
|
1117
|
+
/** Trim one listed item. A long item is CUT, never discarded — the item's existence is information. */
|
|
1118
|
+
function trimItem(item) {
|
|
1119
|
+
if (typeof item !== 'string' || item.length <= exports.ANALYZE_TEXT_CAP)
|
|
1120
|
+
return item;
|
|
1121
|
+
return `${item.slice(0, exports.ANALYZE_TEXT_CAP)}… [+${item.length - exports.ANALYZE_TEXT_CAP}]`;
|
|
1122
|
+
}
|
|
@@ -100,8 +100,49 @@ export interface OutcomeInput {
|
|
|
100
100
|
/**
|
|
101
101
|
* Attach what actually happened to a prediction that was actually made. An unknown digest is
|
|
102
102
|
* refused: inventing the prediction alongside the outcome would let the record score itself.
|
|
103
|
+
*
|
|
104
|
+
* @implements A-SPEC-578.6
|
|
105
|
+
* MERGES into any outcome already there. It used to REPLACE, and the defect was found by following
|
|
106
|
+
* this repository's own instruction: A-SPEC-578.4 made `test_run` attach the 12 files a slice
|
|
107
|
+
* actually changed, AGENTS.md then said "call maintenance_outcome yourself only to add a judged
|
|
108
|
+
* classification" — and that call left `actualFiles: []`. The documented workflow destroyed the
|
|
109
|
+
* data it existed to complete.
|
|
110
|
+
*
|
|
111
|
+
* Omission and erasure are DIFFERENT ACTS: a field left out keeps its value, a field given as an
|
|
112
|
+
* empty array is cleared. Without the second, a wrong record could never be corrected; without the
|
|
113
|
+
* first, completing a record destroys it.
|
|
103
114
|
*/
|
|
104
115
|
export declare function recordOutcome(dir: string, digest: string, outcome: OutcomeInput, recordedAt: string): EvidenceArtifact;
|
|
116
|
+
/**
|
|
117
|
+
* Predictions still waiting to be scored — the loop's missing second half.
|
|
118
|
+
*
|
|
119
|
+
* `since` bounds the window: an analysis recorded before the previous evidence run belonged to a
|
|
120
|
+
* previous slice, and attributing today's changes to it would be contamination rather than
|
|
121
|
+
* measurement. The boundary is EXCLUSIVE (`> since`) because the previous run already saw that
|
|
122
|
+
* instant.
|
|
123
|
+
*/
|
|
124
|
+
export declare function openArtifacts(dir: string, since?: string): EvidenceArtifact[];
|
|
125
|
+
/**
|
|
126
|
+
* @implements A-SPEC-578.4
|
|
127
|
+
* Attach what a slice ACTUALLY touched to the predictions it made.
|
|
128
|
+
*
|
|
129
|
+
* Measured 2026-09-09: `maintenance_calibration` had reported `n:0, artifacts:0` since the loop was
|
|
130
|
+
* built, because `maintenance_analyze({persist:true})` is instructed (AGENTS.md step 3) while
|
|
131
|
+
* `maintenance_outcome` is instructed nowhere at all. The write path was healthy the whole time —
|
|
132
|
+
* one call produced an artifact immediately. What was missing was the call.
|
|
133
|
+
*
|
|
134
|
+
* TWO REFUSALS make this a measurement rather than a self-graded exam:
|
|
135
|
+
* - `actualClassification` is never filled in. That is a judgement, and a loop that supplies its
|
|
136
|
+
* own judgement scores itself; the bins stay `insufficient-data` and only the objective half —
|
|
137
|
+
* file-level false positives and negatives — accrues.
|
|
138
|
+
* - An empty change set closes nothing. Attributing "nothing changed" would turn every prediction
|
|
139
|
+
* into a false positive: a measurement of nothing, recorded as a failure of everything.
|
|
140
|
+
*/
|
|
141
|
+
export declare function closeOpenArtifacts(dir: string, actual: {
|
|
142
|
+
files: string[];
|
|
143
|
+
}, recordedAt: string, since?: string): {
|
|
144
|
+
closed: string[];
|
|
145
|
+
};
|
|
105
146
|
export interface CalibrationBin {
|
|
106
147
|
lower: number;
|
|
107
148
|
upper: number;
|
|
@@ -39,6 +39,8 @@ exports.artifactFrom = artifactFrom;
|
|
|
39
39
|
exports.writeArtifact = writeArtifact;
|
|
40
40
|
exports.readArtifacts = readArtifacts;
|
|
41
41
|
exports.recordOutcome = recordOutcome;
|
|
42
|
+
exports.openArtifacts = openArtifacts;
|
|
43
|
+
exports.closeOpenArtifacts = closeOpenArtifacts;
|
|
42
44
|
exports.computeCalibration = computeCalibration;
|
|
43
45
|
// @implements A-SPEC-271
|
|
44
46
|
// @implements A-SPEC-268
|
|
@@ -170,6 +172,17 @@ function readArtifacts(dir) {
|
|
|
170
172
|
/**
|
|
171
173
|
* Attach what actually happened to a prediction that was actually made. An unknown digest is
|
|
172
174
|
* refused: inventing the prediction alongside the outcome would let the record score itself.
|
|
175
|
+
*
|
|
176
|
+
* @implements A-SPEC-578.6
|
|
177
|
+
* MERGES into any outcome already there. It used to REPLACE, and the defect was found by following
|
|
178
|
+
* this repository's own instruction: A-SPEC-578.4 made `test_run` attach the 12 files a slice
|
|
179
|
+
* actually changed, AGENTS.md then said "call maintenance_outcome yourself only to add a judged
|
|
180
|
+
* classification" — and that call left `actualFiles: []`. The documented workflow destroyed the
|
|
181
|
+
* data it existed to complete.
|
|
182
|
+
*
|
|
183
|
+
* Omission and erasure are DIFFERENT ACTS: a field left out keeps its value, a field given as an
|
|
184
|
+
* empty array is cleared. Without the second, a wrong record could never be corrected; without the
|
|
185
|
+
* first, completing a record destroys it.
|
|
173
186
|
*/
|
|
174
187
|
function recordOutcome(dir, digest, outcome, recordedAt) {
|
|
175
188
|
const file = path.join(dir, digestFilename(digest));
|
|
@@ -177,19 +190,73 @@ function recordOutcome(dir, digest, outcome, recordedAt) {
|
|
|
177
190
|
throw new Error(`Refusing to record an outcome for an unknown analysis digest: ${digest}`);
|
|
178
191
|
}
|
|
179
192
|
const artifact = JSON.parse(fs.readFileSync(file, 'utf8'));
|
|
193
|
+
const prev = artifact.outcome;
|
|
194
|
+
const merged = (given, before) => given === undefined ? (before ?? []) : sortedUnique(given);
|
|
195
|
+
const classification = outcome.actualClassification ?? prev?.actualClassification;
|
|
180
196
|
const updated = {
|
|
181
197
|
...artifact,
|
|
182
198
|
outcome: {
|
|
183
199
|
recordedAt,
|
|
184
|
-
actualFiles:
|
|
185
|
-
actualSymbols:
|
|
186
|
-
actualTests:
|
|
187
|
-
...(
|
|
200
|
+
actualFiles: merged(outcome.actualFiles, prev?.actualFiles),
|
|
201
|
+
actualSymbols: merged(outcome.actualSymbols, prev?.actualSymbols),
|
|
202
|
+
actualTests: merged(outcome.actualTests, prev?.actualTests),
|
|
203
|
+
...(classification ? { actualClassification: classification } : {}),
|
|
188
204
|
},
|
|
189
205
|
};
|
|
190
206
|
writeAtomic(file, `${JSON.stringify(updated, null, 2)}\n`);
|
|
191
207
|
return updated;
|
|
192
208
|
}
|
|
209
|
+
// @implements A-SPEC-578.4
|
|
210
|
+
/**
|
|
211
|
+
* Predictions still waiting to be scored — the loop's missing second half.
|
|
212
|
+
*
|
|
213
|
+
* `since` bounds the window: an analysis recorded before the previous evidence run belonged to a
|
|
214
|
+
* previous slice, and attributing today's changes to it would be contamination rather than
|
|
215
|
+
* measurement. The boundary is EXCLUSIVE (`> since`) because the previous run already saw that
|
|
216
|
+
* instant.
|
|
217
|
+
*/
|
|
218
|
+
function openArtifacts(dir, since) {
|
|
219
|
+
const { artifacts } = readArtifacts(dir);
|
|
220
|
+
return artifacts.filter((a) => {
|
|
221
|
+
if (a.outcome !== undefined)
|
|
222
|
+
return false;
|
|
223
|
+
if (since === undefined)
|
|
224
|
+
return true;
|
|
225
|
+
const t = Date.parse(a.recordedAt);
|
|
226
|
+
const s = Date.parse(since);
|
|
227
|
+
return !Number.isFinite(t) || !Number.isFinite(s) ? true : t > s;
|
|
228
|
+
});
|
|
229
|
+
}
|
|
230
|
+
/**
|
|
231
|
+
* @implements A-SPEC-578.4
|
|
232
|
+
* Attach what a slice ACTUALLY touched to the predictions it made.
|
|
233
|
+
*
|
|
234
|
+
* Measured 2026-09-09: `maintenance_calibration` had reported `n:0, artifacts:0` since the loop was
|
|
235
|
+
* built, because `maintenance_analyze({persist:true})` is instructed (AGENTS.md step 3) while
|
|
236
|
+
* `maintenance_outcome` is instructed nowhere at all. The write path was healthy the whole time —
|
|
237
|
+
* one call produced an artifact immediately. What was missing was the call.
|
|
238
|
+
*
|
|
239
|
+
* TWO REFUSALS make this a measurement rather than a self-graded exam:
|
|
240
|
+
* - `actualClassification` is never filled in. That is a judgement, and a loop that supplies its
|
|
241
|
+
* own judgement scores itself; the bins stay `insufficient-data` and only the objective half —
|
|
242
|
+
* file-level false positives and negatives — accrues.
|
|
243
|
+
* - An empty change set closes nothing. Attributing "nothing changed" would turn every prediction
|
|
244
|
+
* into a false positive: a measurement of nothing, recorded as a failure of everything.
|
|
245
|
+
*/
|
|
246
|
+
function closeOpenArtifacts(dir, actual, recordedAt, since) {
|
|
247
|
+
const files = Array.isArray(actual?.files) ? actual.files : [];
|
|
248
|
+
if (files.length === 0)
|
|
249
|
+
return { closed: [] };
|
|
250
|
+
const closed = [];
|
|
251
|
+
for (const a of openArtifacts(dir, since)) {
|
|
252
|
+
try {
|
|
253
|
+
recordOutcome(dir, a.digest, { actualFiles: files }, recordedAt);
|
|
254
|
+
closed.push(a.digest);
|
|
255
|
+
}
|
|
256
|
+
catch { /* an artifact we cannot write is not a reason to abandon the rest */ }
|
|
257
|
+
}
|
|
258
|
+
return { closed };
|
|
259
|
+
}
|
|
193
260
|
const BIN_EDGES = [0, 0.2, 0.4, 0.6, 0.8, 1];
|
|
194
261
|
/**
|
|
195
262
|
* Reliability bins plus a Brier score. `minSamples` is the honesty knob: below it a bin reports
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @implements A-SPEC-579
|
|
3
|
+
* npm 12 blocks dependency install scripts unless the ROOT package.json's `allowScripts` covers
|
|
4
|
+
* them ("Install commands silently skip lifecycle scripts for any dependency that does not have a
|
|
5
|
+
* matching entry" — npm docs, cli/v12/commands/npm-install-scripts). Measured 2026-09-09 on
|
|
6
|
+
* Windows (npm 12.0.1, Node 24.19.0): `npm ci` succeeded, nine install scripts were skipped, and
|
|
7
|
+
* better-sqlite3 had no binary — while the eight tree-sitter packages still loaded, because their
|
|
8
|
+
* `node-gyp-build` runtime loader finds `prebuilds/<platform>-<arch>` without the script ever
|
|
9
|
+
* running. So the minimum approval this repository needs is ONE package, pinned to the lockfile
|
|
10
|
+
* version so a dependency bump forces a fresh review.
|
|
11
|
+
*
|
|
12
|
+
* This module is the pure re-statement of that rule: no I/O, no process, no platform — the same
|
|
13
|
+
* verdict on every OS, shared by the test that pins package.json and by doctor (A-SPEC-580).
|
|
14
|
+
*/
|
|
15
|
+
/** Dependencies whose install script must actually RUN for the module to load. */
|
|
16
|
+
export declare const NATIVE_INSTALL_SCRIPT_DEPS: readonly ["better-sqlite3"];
|
|
17
|
+
/** Native dependencies that ship prebuilds and load with the script blocked (measured). */
|
|
18
|
+
export declare const PREBUILT_NATIVE_DEPS: readonly ["tree-sitter", "tree-sitter-typescript", "tree-sitter-python", "tree-sitter-c-sharp", "tree-sitter-java", "tree-sitter-go", "tree-sitter-rust", "tree-sitter-cpp"];
|
|
19
|
+
export type AllowScripts = Record<string, boolean>;
|
|
20
|
+
export type Coverage = 'approved-pinned' | 'approved-unpinned' | 'denied' | 'uncovered';
|
|
21
|
+
export interface LockLike {
|
|
22
|
+
packages?: Record<string, {
|
|
23
|
+
version?: string;
|
|
24
|
+
hasInstallScript?: boolean;
|
|
25
|
+
}>;
|
|
26
|
+
}
|
|
27
|
+
export type PolicyProblem = {
|
|
28
|
+
kind: 'reapproval-required';
|
|
29
|
+
name: string;
|
|
30
|
+
lockVersion: string;
|
|
31
|
+
approved: string[];
|
|
32
|
+
fix: string;
|
|
33
|
+
} | {
|
|
34
|
+
kind: 'unpinned';
|
|
35
|
+
name: string;
|
|
36
|
+
fix: string;
|
|
37
|
+
} | {
|
|
38
|
+
kind: 'denied';
|
|
39
|
+
name: string;
|
|
40
|
+
fix: string;
|
|
41
|
+
} | {
|
|
42
|
+
kind: 'beyond-minimum';
|
|
43
|
+
key: string;
|
|
44
|
+
fix: string;
|
|
45
|
+
} | {
|
|
46
|
+
kind: 'missing-from-lock';
|
|
47
|
+
name: string;
|
|
48
|
+
};
|
|
49
|
+
/** The exact command npm 12 documents for a pinned approval. */
|
|
50
|
+
export declare function approveCommand(name: string, version: string, npmBin?: string): string;
|
|
51
|
+
/**
|
|
52
|
+
* How `allowScripts` covers one package at one version, under npm 12's rules: a pinned `true`
|
|
53
|
+
* covers that version only, a bare `true` covers every version, and a `false` on either key is
|
|
54
|
+
* a denial that survives any approval (npm: deny "survives `approve --all`").
|
|
55
|
+
*/
|
|
56
|
+
export declare function allowScriptsCoverage(allow: AllowScripts | undefined, name: string, version: string): Coverage;
|
|
57
|
+
/**
|
|
58
|
+
* What `npm install-scripts ls` would list: every lock entry with an install script that
|
|
59
|
+
* `allowScripts` does not approve. The root entry (`""`) is the project itself, never a dependency.
|
|
60
|
+
*/
|
|
61
|
+
export declare function blockedInstallScripts(lock: LockLike, allow: AllowScripts | undefined): {
|
|
62
|
+
name: string;
|
|
63
|
+
version: string;
|
|
64
|
+
}[];
|
|
65
|
+
/**
|
|
66
|
+
* This repository's policy, judged: every script-requiring dependency approved AND pinned to the
|
|
67
|
+
* lock version, and nothing else approved. Pure; never throws.
|
|
68
|
+
*/
|
|
69
|
+
export declare function policyVerdict(pkg: {
|
|
70
|
+
allowScripts?: AllowScripts;
|
|
71
|
+
}, lock: LockLike, opts?: {
|
|
72
|
+
npmBin?: string;
|
|
73
|
+
}): {
|
|
74
|
+
ok: boolean;
|
|
75
|
+
problems: PolicyProblem[];
|
|
76
|
+
};
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// @implements A-SPEC-579
|
|
3
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
+
exports.PREBUILT_NATIVE_DEPS = exports.NATIVE_INSTALL_SCRIPT_DEPS = void 0;
|
|
5
|
+
exports.approveCommand = approveCommand;
|
|
6
|
+
exports.allowScriptsCoverage = allowScriptsCoverage;
|
|
7
|
+
exports.blockedInstallScripts = blockedInstallScripts;
|
|
8
|
+
exports.policyVerdict = policyVerdict;
|
|
9
|
+
/**
|
|
10
|
+
* @implements A-SPEC-579
|
|
11
|
+
* npm 12 blocks dependency install scripts unless the ROOT package.json's `allowScripts` covers
|
|
12
|
+
* them ("Install commands silently skip lifecycle scripts for any dependency that does not have a
|
|
13
|
+
* matching entry" — npm docs, cli/v12/commands/npm-install-scripts). Measured 2026-09-09 on
|
|
14
|
+
* Windows (npm 12.0.1, Node 24.19.0): `npm ci` succeeded, nine install scripts were skipped, and
|
|
15
|
+
* better-sqlite3 had no binary — while the eight tree-sitter packages still loaded, because their
|
|
16
|
+
* `node-gyp-build` runtime loader finds `prebuilds/<platform>-<arch>` without the script ever
|
|
17
|
+
* running. So the minimum approval this repository needs is ONE package, pinned to the lockfile
|
|
18
|
+
* version so a dependency bump forces a fresh review.
|
|
19
|
+
*
|
|
20
|
+
* This module is the pure re-statement of that rule: no I/O, no process, no platform — the same
|
|
21
|
+
* verdict on every OS, shared by the test that pins package.json and by doctor (A-SPEC-580).
|
|
22
|
+
*/
|
|
23
|
+
/** Dependencies whose install script must actually RUN for the module to load. */
|
|
24
|
+
exports.NATIVE_INSTALL_SCRIPT_DEPS = ['better-sqlite3'];
|
|
25
|
+
/** Native dependencies that ship prebuilds and load with the script blocked (measured). */
|
|
26
|
+
exports.PREBUILT_NATIVE_DEPS = [
|
|
27
|
+
'tree-sitter', 'tree-sitter-typescript', 'tree-sitter-python', 'tree-sitter-c-sharp',
|
|
28
|
+
'tree-sitter-java', 'tree-sitter-go', 'tree-sitter-rust', 'tree-sitter-cpp',
|
|
29
|
+
];
|
|
30
|
+
/** The exact command npm 12 documents for a pinned approval. */
|
|
31
|
+
function approveCommand(name, version, npmBin = 'npm') {
|
|
32
|
+
return `${npmBin} install-scripts approve ${name}@${version}`;
|
|
33
|
+
}
|
|
34
|
+
/** Split an allowScripts key into its package name and optional pinned version (scopes kept). */
|
|
35
|
+
function splitKey(key) {
|
|
36
|
+
const at = key.lastIndexOf('@');
|
|
37
|
+
if (at <= 0)
|
|
38
|
+
return { name: key }; // '@scope/pkg' or 'pkg'
|
|
39
|
+
return { name: key.slice(0, at), version: key.slice(at + 1) };
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* How `allowScripts` covers one package at one version, under npm 12's rules: a pinned `true`
|
|
43
|
+
* covers that version only, a bare `true` covers every version, and a `false` on either key is
|
|
44
|
+
* a denial that survives any approval (npm: deny "survives `approve --all`").
|
|
45
|
+
*/
|
|
46
|
+
function allowScriptsCoverage(allow, name, version) {
|
|
47
|
+
if (!allow)
|
|
48
|
+
return 'uncovered';
|
|
49
|
+
const pinned = allow[`${name}@${version}`];
|
|
50
|
+
const bare = allow[name];
|
|
51
|
+
if (pinned === false || bare === false)
|
|
52
|
+
return 'denied';
|
|
53
|
+
if (pinned === true)
|
|
54
|
+
return 'approved-pinned';
|
|
55
|
+
if (bare === true)
|
|
56
|
+
return 'approved-unpinned';
|
|
57
|
+
return 'uncovered';
|
|
58
|
+
}
|
|
59
|
+
/** The package name of a lockfile `packages` key: the segment after its LAST `node_modules/`. */
|
|
60
|
+
function lockEntryName(key) {
|
|
61
|
+
const idx = key.lastIndexOf('node_modules/');
|
|
62
|
+
if (idx < 0)
|
|
63
|
+
return null;
|
|
64
|
+
const name = key.slice(idx + 'node_modules/'.length);
|
|
65
|
+
return name === '' ? null : name;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* What `npm install-scripts ls` would list: every lock entry with an install script that
|
|
69
|
+
* `allowScripts` does not approve. The root entry (`""`) is the project itself, never a dependency.
|
|
70
|
+
*/
|
|
71
|
+
function blockedInstallScripts(lock, allow) {
|
|
72
|
+
const out = [];
|
|
73
|
+
for (const [key, entry] of Object.entries(lock.packages ?? {})) {
|
|
74
|
+
if (!entry?.hasInstallScript)
|
|
75
|
+
continue;
|
|
76
|
+
const name = lockEntryName(key);
|
|
77
|
+
if (name === null)
|
|
78
|
+
continue;
|
|
79
|
+
const version = entry.version ?? '';
|
|
80
|
+
const cov = allowScriptsCoverage(allow, name, version);
|
|
81
|
+
if (cov !== 'approved-pinned' && cov !== 'approved-unpinned')
|
|
82
|
+
out.push({ name, version });
|
|
83
|
+
}
|
|
84
|
+
return out;
|
|
85
|
+
}
|
|
86
|
+
/** The top-level lock version of a dependency (`node_modules/<name>`), if present. */
|
|
87
|
+
function lockVersionOf(lock, name) {
|
|
88
|
+
return lock.packages?.[`node_modules/${name}`]?.version;
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* This repository's policy, judged: every script-requiring dependency approved AND pinned to the
|
|
92
|
+
* lock version, and nothing else approved. Pure; never throws.
|
|
93
|
+
*/
|
|
94
|
+
function policyVerdict(pkg, lock, opts) {
|
|
95
|
+
const npmBin = opts?.npmBin ?? 'npm';
|
|
96
|
+
const allow = pkg.allowScripts;
|
|
97
|
+
const problems = [];
|
|
98
|
+
for (const name of exports.NATIVE_INSTALL_SCRIPT_DEPS) {
|
|
99
|
+
const lockVersion = lockVersionOf(lock, name);
|
|
100
|
+
if (lockVersion === undefined) {
|
|
101
|
+
problems.push({ kind: 'missing-from-lock', name });
|
|
102
|
+
continue;
|
|
103
|
+
}
|
|
104
|
+
const fix = approveCommand(name, lockVersion, npmBin);
|
|
105
|
+
switch (allowScriptsCoverage(allow, name, lockVersion)) {
|
|
106
|
+
case 'approved-pinned': break;
|
|
107
|
+
case 'approved-unpinned':
|
|
108
|
+
problems.push({ kind: 'unpinned', name, fix });
|
|
109
|
+
break;
|
|
110
|
+
case 'denied':
|
|
111
|
+
problems.push({ kind: 'denied', name, fix });
|
|
112
|
+
break;
|
|
113
|
+
case 'uncovered': {
|
|
114
|
+
const approved = Object.entries(allow ?? {})
|
|
115
|
+
.filter(([k, v]) => v === true && splitKey(k).name === name && splitKey(k).version !== undefined)
|
|
116
|
+
.map(([k]) => splitKey(k).version);
|
|
117
|
+
problems.push({ kind: 'reapproval-required', name, lockVersion, approved, fix });
|
|
118
|
+
break;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
const minimum = new Set(exports.NATIVE_INSTALL_SCRIPT_DEPS);
|
|
123
|
+
for (const [key, value] of Object.entries(allow ?? {})) {
|
|
124
|
+
if (value !== true)
|
|
125
|
+
continue; // a denial is never "beyond" the minimum
|
|
126
|
+
if (!minimum.has(splitKey(key).name)) {
|
|
127
|
+
problems.push({ kind: 'beyond-minimum', key, fix: `Remove "${key}" from allowScripts in package.json — it loads from its shipped prebuilds without an install script.` });
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
return { ok: problems.length === 0, problems };
|
|
131
|
+
}
|
|
@@ -8,3 +8,9 @@
|
|
|
8
8
|
* platform must never flip a POSIX hint to a Windows-only form.
|
|
9
9
|
*/
|
|
10
10
|
export declare function npxBin(platform?: string): string;
|
|
11
|
+
/**
|
|
12
|
+
* @implements A-SPEC-580
|
|
13
|
+
* The npm twin: the recovery commands doctor emits (`npm install-scripts approve`, `npm rebuild`)
|
|
14
|
+
* hit the same `.ps1` shim on Windows (measured 2026-09-09: npm.ps1 blocked, npm.cmd runs).
|
|
15
|
+
*/
|
|
16
|
+
export declare function npmBin(platform?: string): string;
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
// @implements A-SPEC-542.1
|
|
3
|
+
// @implements A-SPEC-580
|
|
3
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
5
|
exports.npxBin = npxBin;
|
|
6
|
+
exports.npmBin = npmBin;
|
|
5
7
|
/**
|
|
6
8
|
* @implements A-SPEC-542.1
|
|
7
9
|
* The npx binary an OPERATOR can actually run on the platform this process runs on — which is the
|
|
@@ -14,3 +16,11 @@ exports.npxBin = npxBin;
|
|
|
14
16
|
function npxBin(platform = process.platform) {
|
|
15
17
|
return platform === 'win32' ? 'npx.cmd' : 'npx';
|
|
16
18
|
}
|
|
19
|
+
/**
|
|
20
|
+
* @implements A-SPEC-580
|
|
21
|
+
* The npm twin: the recovery commands doctor emits (`npm install-scripts approve`, `npm rebuild`)
|
|
22
|
+
* hit the same `.ps1` shim on Windows (measured 2026-09-09: npm.ps1 blocked, npm.cmd runs).
|
|
23
|
+
*/
|
|
24
|
+
function npmBin(platform = process.platform) {
|
|
25
|
+
return platform === 'win32' ? 'npm.cmd' : 'npm';
|
|
26
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The names behind a red release gate.
|
|
3
|
+
*
|
|
4
|
+
* Measured 2026-09-10: `npm publish` refused with "스위트가 붉습니다" and nothing else, and finding
|
|
5
|
+
* out which test had failed cost two more full-suite runs. `test_run` learned to name its failures
|
|
6
|
+
* in REQ-577; the release gate had not. A gate that says "red" without saying what is a gate that
|
|
7
|
+
* makes the next person guess.
|
|
8
|
+
*/
|
|
9
|
+
/** How many names a refusal lists before it counts the rest. */
|
|
10
|
+
export declare const FAILED_NAME_CAP = 10;
|
|
11
|
+
/**
|
|
12
|
+
* Pull the failing test names out of a jest run's output.
|
|
13
|
+
*
|
|
14
|
+
* PURE, and forgiving: output it cannot read yields an empty list so the caller keeps its existing
|
|
15
|
+
* refusal rather than replacing a working message with an empty one. Jest prints the summary block
|
|
16
|
+
* twice on some configurations, so names are de-duplicated — the same failure listed twice is one
|
|
17
|
+
* failure.
|
|
18
|
+
*/
|
|
19
|
+
export declare function failedTestNames(output: string): string[];
|