@holmes-lab/holmes-kit 0.31.1 → 0.32.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +69 -0
- package/README.md +1 -0
- package/dist/.build-id +1 -1
- package/dist/holmes/cli/codex-legacy.d.ts +16 -0
- package/dist/holmes/cli/codex-legacy.js +129 -0
- package/dist/holmes/cli/doctor.js +32 -1
- package/dist/holmes/cli/index.js +21 -0
- package/dist/holmes/mcp/handlers/maintenance-analysis.js +16 -3
- package/dist/holmes/mcp/handlers/maintenance-evidence.d.ts +6 -0
- package/dist/holmes/mcp/handlers/requirement-triage.d.ts +163 -4
- package/dist/holmes/mcp/handlers/requirement-triage.js +26 -18
- package/dist/holmes/mcp/handlers.d.ts +169 -4
- package/dist/holmes/mcp/maintenance-analyze.d.ts +19 -0
- package/dist/holmes/mcp/maintenance-analyze.js +113 -7
- package/dist/holmes/mcp/maintenance-evidence.d.ts +25 -0
- package/dist/holmes/mcp/maintenance-evidence.js +52 -1
- package/dist/holmes/mcp/tool-schemas.js +3 -1
- package/dist/holmes/review/requirement-triage.d.ts +19 -1
- package/dist/holmes/review/requirement-triage.js +113 -12
- package/dist/holmes/rtm/localize.d.ts +7 -0
- package/dist/holmes/rtm/localize.js +10 -0
- package/package.json +1 -1
|
@@ -34,7 +34,7 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
34
34
|
})();
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
36
|
exports.createRequirementTriageHandlers = createRequirementTriageHandlers;
|
|
37
|
-
// @implements A-SPEC-729, A-SPEC-723, A-SPEC-719, A-SPEC-734
|
|
37
|
+
// @implements A-SPEC-743, A-SPEC-742, A-SPEC-729, A-SPEC-723, A-SPEC-719, A-SPEC-734
|
|
38
38
|
// @implements A-SPEC-713
|
|
39
39
|
/** Read-only MCP projection of the pure requirement lifecycle triage model. */
|
|
40
40
|
const node_child_process_1 = require("node:child_process");
|
|
@@ -163,31 +163,32 @@ function createRequirementTriageHandlers(context) {
|
|
|
163
163
|
const runtime = suppressed ? null : (0, embedder_1.makeTierEmbedder)(tier, new vector_cache_1.VectorCache(root));
|
|
164
164
|
const query = runtime ? await runtime.embedQuery(a.prompt) : null;
|
|
165
165
|
const semanticCandidates = [];
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
}
|
|
166
|
+
// @implements A-SPEC-742 — counted over documents that can carry a vector (a REQ lineage);
|
|
167
|
+
// the unlineaged are reported, not counted against the layer.
|
|
168
|
+
const coverage = (0, requirement_triage_1.requirementSemanticCoverage)(documents, (doc, reqId) => {
|
|
169
|
+
if (!runtime || !query)
|
|
170
|
+
return false;
|
|
171
|
+
const vector = runtime.cachedDocVector((0, requirement_triage_1.canonicalRequirementDocument)(doc, reqId));
|
|
172
|
+
if (vector)
|
|
173
|
+
semanticCandidates.push({ id: doc.id, score: cosine(query, vector) });
|
|
174
|
+
return vector !== null && vector !== undefined;
|
|
175
|
+
});
|
|
176
|
+
const covered = coverage.covered;
|
|
178
177
|
const semanticState = suppressed ? 'not-adopted'
|
|
179
178
|
: runtime === null || query === null
|
|
180
179
|
? (tier.tier === 'none' && !tier.localRuntimePresent ? 'not-adopted' : 'unavailable')
|
|
181
|
-
:
|
|
180
|
+
: coverage.state;
|
|
182
181
|
const readiness = tier.tier === 'none' && tier.localRuntimePresent ? (0, model_cache_1.readLocalModelReadiness)() : null;
|
|
183
182
|
const availability = semanticState === 'covered' ? 'available'
|
|
184
|
-
: semanticState === 'cold' || tier.tier === 'none' && tier.localRuntimePresent ? 'degraded'
|
|
183
|
+
: semanticState === 'cold' || semanticState === 'partial' || tier.tier === 'none' && tier.localRuntimePresent ? 'degraded'
|
|
185
184
|
: 'unavailable';
|
|
186
|
-
const semantic = { tier: tier.tier, covered, total:
|
|
185
|
+
const semantic = { tier: tier.tier, covered, total: coverage.eligible, unlineaged: coverage.unlineaged, state: semanticState,
|
|
186
|
+
...(semanticState === 'partial' ? { missingLineages: coverage.missingLineages } : {}),
|
|
187
187
|
availability, ...(suppressed ? { suppressed: true } : {}),
|
|
188
188
|
...(readiness ? { localState: readiness.state, recovery: readiness.recovery } : {}) };
|
|
189
189
|
// lexicalOnly describes THIS run: no semantic score contributed to its ranking.
|
|
190
|
-
|
|
190
|
+
// A-SPEC-742 review round 1: a vector whose cosine was filtered out (<= 0) ranked nothing.
|
|
191
|
+
const lexical = !semanticCandidates.some((c) => Number.isFinite(c.score) && c.score > 0);
|
|
191
192
|
const ranked = (0, requirement_triage_1.triageRequirements)({ prompt: a.prompt, documents, evidence: lineageEvidence,
|
|
192
193
|
semanticCandidates, semantic, graph: { state: graph.status }, topK: a.topK });
|
|
193
194
|
const result = freshnessCurrent ? ranked : {
|
|
@@ -206,7 +207,14 @@ function createRequirementTriageHandlers(context) {
|
|
|
206
207
|
graphProvenance: Object.fromEntries(candidate.documents.map((document) => [document.id, graph.provenanceBySpec[document.id] ?? null])),
|
|
207
208
|
executionEvidence: { state: executionState, recordedHead: evidence?.head ?? null, currentHead },
|
|
208
209
|
}));
|
|
209
|
-
|
|
210
|
+
// @implements A-SPEC-743 — documents as summaries unless the caller asks for their text: the
|
|
211
|
+
// bodies were 72% of a 201 KB median answer, past the harness tool-result limit. Every decision
|
|
212
|
+
// field (verdict, reasons, uncertainty, candidate order and scores) is the same either way.
|
|
213
|
+
const shown = a.detail === 'full' ? result : { ...result, candidates: result.candidates.map((candidate) => ({
|
|
214
|
+
...candidate,
|
|
215
|
+
documents: candidate.documents.map((d) => ({ id: d.id, type: d.type, title: d.title, status: d.status, dependsOn: d.dependsOn ?? [] })),
|
|
216
|
+
})) };
|
|
217
|
+
const answer = { schema: 'requirement-triage/1', ...shown, ...(lexical ? { lexicalOnly: true } : {}), promptDigest, semantic, provenance,
|
|
210
218
|
graph: { status: graph.status, sourceCommit: graph.sourceCommit, graphAsOf: graph.graphAsOf, resolutionLimits: graph.resolutionLimits },
|
|
211
219
|
graphBasis: { head: currentHead, governedDocumentCount: documents.length, scannedSourceCount: scanned.length },
|
|
212
220
|
freshness: freshness === null ? { state: 'absent' } : freshness,
|
|
@@ -634,17 +634,20 @@ declare function makeRawHandlers(store: SpecStore, opts?: ElicitOpts): {
|
|
|
634
634
|
prompt: string;
|
|
635
635
|
topK?: number;
|
|
636
636
|
persist?: boolean;
|
|
637
|
+
detail?: string;
|
|
637
638
|
}) => Promise<{
|
|
638
639
|
promptDigest: string;
|
|
639
640
|
semantic: {
|
|
640
641
|
localState?: import("../semantic/model-cache").LocalModelState | undefined;
|
|
641
642
|
recovery?: string | undefined;
|
|
642
643
|
suppressed?: true | undefined;
|
|
644
|
+
availability: string;
|
|
645
|
+
missingLineages?: string[] | undefined;
|
|
643
646
|
tier: "local" | "none" | "cloud";
|
|
644
647
|
covered: number;
|
|
645
648
|
total: number;
|
|
646
|
-
|
|
647
|
-
|
|
649
|
+
unlineaged: number;
|
|
650
|
+
state: "covered" | "partial" | "unavailable" | "not-adopted" | "cold";
|
|
648
651
|
};
|
|
649
652
|
provenance: {
|
|
650
653
|
reqId: string;
|
|
@@ -690,6 +693,80 @@ declare function makeRawHandlers(store: SpecStore, opts?: ElicitOpts): {
|
|
|
690
693
|
reasons: string[];
|
|
691
694
|
uncertainty: string[];
|
|
692
695
|
schema: "requirement-triage/1";
|
|
696
|
+
} | {
|
|
697
|
+
promptDigest: string;
|
|
698
|
+
semantic: {
|
|
699
|
+
localState?: import("../semantic/model-cache").LocalModelState | undefined;
|
|
700
|
+
recovery?: string | undefined;
|
|
701
|
+
suppressed?: true | undefined;
|
|
702
|
+
availability: string;
|
|
703
|
+
missingLineages?: string[] | undefined;
|
|
704
|
+
tier: "local" | "none" | "cloud";
|
|
705
|
+
covered: number;
|
|
706
|
+
total: number;
|
|
707
|
+
unlineaged: number;
|
|
708
|
+
state: "covered" | "partial" | "unavailable" | "not-adopted" | "cold";
|
|
709
|
+
};
|
|
710
|
+
provenance: {
|
|
711
|
+
reqId: string;
|
|
712
|
+
documents: {
|
|
713
|
+
id: string;
|
|
714
|
+
type: import("../review/requirement-triage").RequirementDocumentKind;
|
|
715
|
+
status: string;
|
|
716
|
+
dependsOn: readonly string[];
|
|
717
|
+
}[];
|
|
718
|
+
implementationAnchors: string[];
|
|
719
|
+
graphProvenance: {
|
|
720
|
+
[k: string]: {} | null;
|
|
721
|
+
};
|
|
722
|
+
executionEvidence: {
|
|
723
|
+
state: string;
|
|
724
|
+
recordedHead: string | null;
|
|
725
|
+
currentHead: string;
|
|
726
|
+
};
|
|
727
|
+
}[];
|
|
728
|
+
graph: {
|
|
729
|
+
status: "current" | "empty" | "absent" | "unreadable" | "stale" | "unverified";
|
|
730
|
+
sourceCommit: string | null;
|
|
731
|
+
graphAsOf: string | null;
|
|
732
|
+
resolutionLimits: string[];
|
|
733
|
+
};
|
|
734
|
+
graphBasis: {
|
|
735
|
+
head: string;
|
|
736
|
+
governedDocumentCount: number;
|
|
737
|
+
scannedSourceCount: number;
|
|
738
|
+
};
|
|
739
|
+
freshness: import("../refresh/freshness-coordinator").FreshnessReceipt | {
|
|
740
|
+
state: string;
|
|
741
|
+
};
|
|
742
|
+
refresh: {
|
|
743
|
+
status: "spawned" | "coalesced" | "failed";
|
|
744
|
+
generation?: string;
|
|
745
|
+
} | {
|
|
746
|
+
status: "unavailable" | "not-needed";
|
|
747
|
+
};
|
|
748
|
+
lexicalOnly?: true | undefined;
|
|
749
|
+
candidates: {
|
|
750
|
+
documents: {
|
|
751
|
+
id: string;
|
|
752
|
+
type: import("../review/requirement-triage").RequirementDocumentKind;
|
|
753
|
+
title: string;
|
|
754
|
+
status: string;
|
|
755
|
+
dependsOn: readonly string[];
|
|
756
|
+
}[];
|
|
757
|
+
reqId: string;
|
|
758
|
+
lexicalScore: number;
|
|
759
|
+
semanticScore?: number;
|
|
760
|
+
contentTerms: string[];
|
|
761
|
+
approved: boolean;
|
|
762
|
+
implementationAnchors: string[];
|
|
763
|
+
freshGreenTests: boolean;
|
|
764
|
+
reasons: string[];
|
|
765
|
+
}[];
|
|
766
|
+
verdict: import("../review/requirement-triage").RequirementVerdict;
|
|
767
|
+
reasons: string[];
|
|
768
|
+
uncertainty: string[];
|
|
769
|
+
schema: "requirement-triage/1";
|
|
693
770
|
} | {
|
|
694
771
|
evidenceDigest: string;
|
|
695
772
|
persistedTo: string;
|
|
@@ -702,11 +779,13 @@ declare function makeRawHandlers(store: SpecStore, opts?: ElicitOpts): {
|
|
|
702
779
|
localState?: import("../semantic/model-cache").LocalModelState | undefined;
|
|
703
780
|
recovery?: string | undefined;
|
|
704
781
|
suppressed?: true | undefined;
|
|
782
|
+
availability: string;
|
|
783
|
+
missingLineages?: string[] | undefined;
|
|
705
784
|
tier: "local" | "none" | "cloud";
|
|
706
785
|
covered: number;
|
|
707
786
|
total: number;
|
|
708
|
-
|
|
709
|
-
|
|
787
|
+
unlineaged: number;
|
|
788
|
+
state: "covered" | "partial" | "unavailable" | "not-adopted" | "cold";
|
|
710
789
|
};
|
|
711
790
|
provenance: {
|
|
712
791
|
reqId: string;
|
|
@@ -752,6 +831,86 @@ declare function makeRawHandlers(store: SpecStore, opts?: ElicitOpts): {
|
|
|
752
831
|
reasons: string[];
|
|
753
832
|
uncertainty: string[];
|
|
754
833
|
schema: "requirement-triage/1";
|
|
834
|
+
} | {
|
|
835
|
+
evidenceDigest: string;
|
|
836
|
+
persistedTo: string;
|
|
837
|
+
artifact: {
|
|
838
|
+
digest: string;
|
|
839
|
+
path: string;
|
|
840
|
+
};
|
|
841
|
+
promptDigest: string;
|
|
842
|
+
semantic: {
|
|
843
|
+
localState?: import("../semantic/model-cache").LocalModelState | undefined;
|
|
844
|
+
recovery?: string | undefined;
|
|
845
|
+
suppressed?: true | undefined;
|
|
846
|
+
availability: string;
|
|
847
|
+
missingLineages?: string[] | undefined;
|
|
848
|
+
tier: "local" | "none" | "cloud";
|
|
849
|
+
covered: number;
|
|
850
|
+
total: number;
|
|
851
|
+
unlineaged: number;
|
|
852
|
+
state: "covered" | "partial" | "unavailable" | "not-adopted" | "cold";
|
|
853
|
+
};
|
|
854
|
+
provenance: {
|
|
855
|
+
reqId: string;
|
|
856
|
+
documents: {
|
|
857
|
+
id: string;
|
|
858
|
+
type: import("../review/requirement-triage").RequirementDocumentKind;
|
|
859
|
+
status: string;
|
|
860
|
+
dependsOn: readonly string[];
|
|
861
|
+
}[];
|
|
862
|
+
implementationAnchors: string[];
|
|
863
|
+
graphProvenance: {
|
|
864
|
+
[k: string]: {} | null;
|
|
865
|
+
};
|
|
866
|
+
executionEvidence: {
|
|
867
|
+
state: string;
|
|
868
|
+
recordedHead: string | null;
|
|
869
|
+
currentHead: string;
|
|
870
|
+
};
|
|
871
|
+
}[];
|
|
872
|
+
graph: {
|
|
873
|
+
status: "current" | "empty" | "absent" | "unreadable" | "stale" | "unverified";
|
|
874
|
+
sourceCommit: string | null;
|
|
875
|
+
graphAsOf: string | null;
|
|
876
|
+
resolutionLimits: string[];
|
|
877
|
+
};
|
|
878
|
+
graphBasis: {
|
|
879
|
+
head: string;
|
|
880
|
+
governedDocumentCount: number;
|
|
881
|
+
scannedSourceCount: number;
|
|
882
|
+
};
|
|
883
|
+
freshness: import("../refresh/freshness-coordinator").FreshnessReceipt | {
|
|
884
|
+
state: string;
|
|
885
|
+
};
|
|
886
|
+
refresh: {
|
|
887
|
+
status: "spawned" | "coalesced" | "failed";
|
|
888
|
+
generation?: string;
|
|
889
|
+
} | {
|
|
890
|
+
status: "unavailable" | "not-needed";
|
|
891
|
+
};
|
|
892
|
+
lexicalOnly?: true | undefined;
|
|
893
|
+
candidates: {
|
|
894
|
+
documents: {
|
|
895
|
+
id: string;
|
|
896
|
+
type: import("../review/requirement-triage").RequirementDocumentKind;
|
|
897
|
+
title: string;
|
|
898
|
+
status: string;
|
|
899
|
+
dependsOn: readonly string[];
|
|
900
|
+
}[];
|
|
901
|
+
reqId: string;
|
|
902
|
+
lexicalScore: number;
|
|
903
|
+
semanticScore?: number;
|
|
904
|
+
contentTerms: string[];
|
|
905
|
+
approved: boolean;
|
|
906
|
+
implementationAnchors: string[];
|
|
907
|
+
freshGreenTests: boolean;
|
|
908
|
+
reasons: string[];
|
|
909
|
+
}[];
|
|
910
|
+
verdict: import("../review/requirement-triage").RequirementVerdict;
|
|
911
|
+
reasons: string[];
|
|
912
|
+
uncertainty: string[];
|
|
913
|
+
schema: "requirement-triage/1";
|
|
755
914
|
}>;
|
|
756
915
|
ledger_reconcile: (a: {
|
|
757
916
|
root?: string;
|
|
@@ -828,6 +987,12 @@ declare function makeRawHandlers(store: SpecStore, opts?: ElicitOpts): {
|
|
|
828
987
|
falseNegatives: number;
|
|
829
988
|
minSamples: number;
|
|
830
989
|
status: "measured" | "insufficient-data";
|
|
990
|
+
scopeCauses: {
|
|
991
|
+
full: number;
|
|
992
|
+
narrow: number;
|
|
993
|
+
unrecorded: number;
|
|
994
|
+
causes: Record<string, number>;
|
|
995
|
+
};
|
|
831
996
|
}>;
|
|
832
997
|
spec_create: (a: any) => Promise<{
|
|
833
998
|
ok: boolean;
|
|
@@ -9,6 +9,7 @@ import { type TestEvidence } from '../review/test-evidence';
|
|
|
9
9
|
import { type LanguageGap } from '../cpg/language-capability';
|
|
10
10
|
export declare const MAINTENANCE_ANALYSIS_SCHEMA = "maintenance-analysis/1.13";
|
|
11
11
|
export declare const MAINTENANCE_ANALYSIS_EXTRACTOR = "holmes-graph-fusion/1.13";
|
|
12
|
+
export declare const CONFIDENCE_NOTE: string;
|
|
12
13
|
/**
|
|
13
14
|
* The evidence kinds an ablation arm can actually produce. Completeness is measured by asking each
|
|
14
15
|
* arm which of these it FILLED — the first cut hardcoded 1 / 2 / 2+n, so "enhanced beats current"
|
|
@@ -159,6 +160,14 @@ export interface MaintenanceAnalysisInput {
|
|
|
159
160
|
scannedFiles: number;
|
|
160
161
|
skippedFiles: readonly string[];
|
|
161
162
|
unsupportedLanguages: readonly string[];
|
|
163
|
+
/**
|
|
164
|
+
* @implements A-SPEC-741 — paths of the files behind `unsupportedLanguages`, so a language counts
|
|
165
|
+
* against THIS request when one of its unreadable files shares a request term (the Ruby
|
|
166
|
+
* `legacy_session.rb` for "isolate the session"). Absent: only changed files and request paths count.
|
|
167
|
+
*/
|
|
168
|
+
unsupportedFiles?: readonly string[];
|
|
169
|
+
/** @implements A-SPEC-741 — the handler capped `unsupportedFiles`; every language then counts. */
|
|
170
|
+
unsupportedFilesTruncated?: boolean;
|
|
162
171
|
historyStatus?: 'available' | 'unavailable' | 'not-requested';
|
|
163
172
|
};
|
|
164
173
|
}
|
|
@@ -177,6 +186,16 @@ export interface MaintenanceAnalysis {
|
|
|
177
186
|
* nobody has a hypothesis for why it inverts.
|
|
178
187
|
*/
|
|
179
188
|
confidence: number;
|
|
189
|
+
/**
|
|
190
|
+
* @implements A-SPEC-740
|
|
191
|
+
* The standing of `confidence`, stated where the number is read — the tool description says it
|
|
192
|
+
* once, but a caller reading this object or a persisted artifact never sees that. A constant
|
|
193
|
+
* `false`: lifting the request-unrelated coverage gates turns 120/120 replay cases into 0.95 at
|
|
194
|
+
* 0.467/0.383 top-1 accuracy (Phase A, 2026-09-30). Only a passing calibration measurement,
|
|
195
|
+
* shipped as its own requirement, may turn it true.
|
|
196
|
+
*/
|
|
197
|
+
confidenceCalibrated: false;
|
|
198
|
+
confidenceNote: string;
|
|
180
199
|
/**
|
|
181
200
|
* @implements A-SPEC-375
|
|
182
201
|
* A-SPECs worth citing, when the request cited nothing. Empty when it already did.
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.ANALYZE_TEXT_CAP = exports.ANALYZE_LIST_CAP = exports.PRIMARY_TIER_WIDTH = exports.EVIDENCE_DIMENSIONS = exports.MAINTENANCE_ANALYSIS_EXTRACTOR = exports.MAINTENANCE_ANALYSIS_SCHEMA = void 0;
|
|
3
|
+
exports.ANALYZE_TEXT_CAP = exports.ANALYZE_LIST_CAP = exports.PRIMARY_TIER_WIDTH = exports.EVIDENCE_DIMENSIONS = exports.CONFIDENCE_NOTE = exports.MAINTENANCE_ANALYSIS_EXTRACTOR = exports.MAINTENANCE_ANALYSIS_SCHEMA = void 0;
|
|
4
4
|
exports.unquoteGitPath = unquoteGitPath;
|
|
5
5
|
exports.analyzeMaintenance = analyzeMaintenance;
|
|
6
6
|
exports.boundAnalysis = boundAnalysis;
|
|
7
|
-
// @implements A-SPEC-725, A-SPEC-310
|
|
7
|
+
// @implements A-SPEC-745, A-SPEC-741, A-SPEC-740, A-SPEC-725, A-SPEC-310
|
|
8
8
|
// @implements A-SPEC-295
|
|
9
9
|
// @implements A-SPEC-294
|
|
10
10
|
// @implements A-SPEC-293
|
|
@@ -48,6 +48,19 @@ const defuse_symbols_1 = require("../rtm/defuse-symbols");
|
|
|
48
48
|
// artifacts are only comparable when this string matches.
|
|
49
49
|
exports.MAINTENANCE_ANALYSIS_SCHEMA = 'maintenance-analysis/1.13';
|
|
50
50
|
exports.MAINTENANCE_ANALYSIS_EXTRACTOR = 'holmes-graph-fusion/1.13';
|
|
51
|
+
// @implements A-SPEC-741 — tokens for the unreadable-file relevance rule: split on non-alphanumerics
|
|
52
|
+
// and camelCase, lower-cased, length >= 2, NO stop list (a file named `test.sh` or `db.rb` is named).
|
|
53
|
+
function rawTokens(text) {
|
|
54
|
+
return text.replace(/([a-z0-9])([A-Z])/g, '$1 $2').toLowerCase().split(/[^a-z0-9]+/).filter((t) => t.length >= 2);
|
|
55
|
+
}
|
|
56
|
+
/** @implements A-SPEC-741 — words a request uses when it is about a language the scanner cannot read. */
|
|
57
|
+
const LANGUAGE_NAMES = {
|
|
58
|
+
'.sh': ['shell', 'bash', 'sh'], '.rb': ['ruby', 'rails'], '.php': ['php'], '.kt': ['kotlin'], '.kts': ['kotlin'],
|
|
59
|
+
'.swift': ['swift'], '.scala': ['scala'], '.ex': ['elixir'], '.exs': ['elixir'], '.lua': ['lua'],
|
|
60
|
+
};
|
|
61
|
+
// @implements A-SPEC-740
|
|
62
|
+
exports.CONFIDENCE_NOTE = 'confidence is not calibrated: it counts which kinds of evidence were present, '
|
|
63
|
+
+ 'does not predict accuracy, and must not be used to weight answers or to narrow a test scope.';
|
|
51
64
|
/**
|
|
52
65
|
* The evidence kinds an ablation arm can actually produce. Completeness is measured by asking each
|
|
53
66
|
* arm which of these it FILLED — the first cut hardcoded 1 / 2 / 2+n, so "enhanced beats current"
|
|
@@ -589,6 +602,10 @@ function analyzeMaintenance(input) {
|
|
|
589
602
|
}
|
|
590
603
|
frontier = next;
|
|
591
604
|
}
|
|
605
|
+
// Every frontier node is recorded as a stop, including those with no further callers (about half,
|
|
606
|
+
// measured 2026-09-30). A-SPEC-741 first recorded only the ones with unexplored callers; the replay
|
|
607
|
+
// then narrowed 12/120 scopes and 10 of them missed a test the commit changed — the conservative
|
|
608
|
+
// record was masking narrowing from anchors that do not cover the tests, so it stays.
|
|
592
609
|
for (const id of frontier)
|
|
593
610
|
stoppedAt.push({ id, reason: 'depth' });
|
|
594
611
|
const transitive = sortedUnique([...transitiveIds].map(codeQualifiedName).filter((qn) => !direct.includes(qn)));
|
|
@@ -740,6 +757,73 @@ function analyzeMaintenance(input) {
|
|
|
740
757
|
// coexisted with tier `unit`, empty coverage gaps, and no `unknowns` line at all.
|
|
741
758
|
const truncationReasons = sortedUnique(stoppedAt.map((stop) => stop.reason));
|
|
742
759
|
const truncationMarkers = sortedUnique(stoppedAt.map((stop) => `impact-truncated:${stop.reason}:${stop.id}`));
|
|
760
|
+
// @implements A-SPEC-741 — an unsupported language is a gap about THIS change only when the change
|
|
761
|
+
// names it: a changed file with that extension, or a path in the request that ends with it. A CI
|
|
762
|
+
// `.sh` script elsewhere in the tree pinned 455/455 analyses here to `ambiguous` + `full` from
|
|
763
|
+
// 2026-09-03 on. An entry that is not an extension (`not-enumerated(non-git)`) means the analysis
|
|
764
|
+
// could not look at all, so it always counts.
|
|
765
|
+
// A third way in: an unreadable file whose path shares a request term may be the very target the
|
|
766
|
+
// analysis cannot see — `src/legacy_session.rb` for "isolate the session".
|
|
767
|
+
// Review round 1 (2026-09-30): where the rule cannot judge it leans to "relevant" — a false
|
|
768
|
+
// "unrelated" is false confidence. Raw tokens (length >= 2, no stop list: `ci`, `db`, `test` are
|
|
769
|
+
// names), a prefix match from 4 characters (`session`/`sessions`), trailing punctuation stripped
|
|
770
|
+
// from request paths (`ci.sh.`), any non-ASCII letter in the request (a Korean or Japanese request
|
|
771
|
+
// cannot be compared with ASCII paths), and a capped path list all count as "about the change".
|
|
772
|
+
const requestLower = input.request.toLowerCase();
|
|
773
|
+
// Review round 2: a Set, capped — a 500 KB request cost 2.9 s against 5,000 paths when the tokens were
|
|
774
|
+
// an array compared inside the per-file loop.
|
|
775
|
+
// Round 3: de-duplicate FIRST, then cap — a pasted log of repeated lines pushed the file's name past
|
|
776
|
+
// a cap taken over raw tokens.
|
|
777
|
+
// Round 4: tokens over 64 characters (a pasted hex blob, a minified line) are not words.
|
|
778
|
+
const requestTokens = new Set([...new Set(rawTokens(input.request).filter((t) => t.length <= 64))].slice(0, 4096));
|
|
779
|
+
const requestStems = new Set([...requestTokens].filter((t) => t.length >= 5).map((t) => t.slice(0, 5)));
|
|
780
|
+
// Every prefix (>= 4 chars) of every request token, so "a path token is a prefix of a request word"
|
|
781
|
+
// (`auth` / `authentication`) is one lookup instead of a scan.
|
|
782
|
+
// Built on first use only — most analyses never reach the path condition (round 4).
|
|
783
|
+
let requestPrefixes = null;
|
|
784
|
+
const prefixes = () => {
|
|
785
|
+
if (requestPrefixes === null) {
|
|
786
|
+
requestPrefixes = new Set();
|
|
787
|
+
for (const t of requestTokens)
|
|
788
|
+
for (let n = 4; n <= t.length; n++)
|
|
789
|
+
requestPrefixes.add(t.slice(0, n));
|
|
790
|
+
}
|
|
791
|
+
return requestPrefixes;
|
|
792
|
+
};
|
|
793
|
+
const tokenMeetsRequest = (token) => {
|
|
794
|
+
if (requestTokens.has(token))
|
|
795
|
+
return true;
|
|
796
|
+
if (token.length >= 4 && prefixes().has(token))
|
|
797
|
+
return true; // path token ⊑ request word
|
|
798
|
+
for (let n = 4; n < token.length; n++)
|
|
799
|
+
if (requestTokens.has(token.slice(0, n)))
|
|
800
|
+
return true; // request word ⊑ path token
|
|
801
|
+
return token.length >= 5 && requestStems.has(token.slice(0, 5));
|
|
802
|
+
};
|
|
803
|
+
// A LETTER outside ASCII (Hangul, Kana, …) — not an em dash or a curly quote, which every commit subject here uses.
|
|
804
|
+
const requestIsNonAscii = /(?![\x00-\x7f])\p{L}/u.test(input.request);
|
|
805
|
+
const pathsCapped = input.coverage.unsupportedFilesTruncated === true;
|
|
806
|
+
const relevantUnsupported = input.coverage.unsupportedLanguages.filter((language) => {
|
|
807
|
+
if (!language.startsWith('.'))
|
|
808
|
+
return true;
|
|
809
|
+
const ext = language.toLowerCase();
|
|
810
|
+
if ((input.changedFiles ?? []).some((file) => file.toLowerCase().endsWith(ext)))
|
|
811
|
+
return true;
|
|
812
|
+
// A path token ending with the extension: `scripts/ci-local.sh`, not a bare word `sh`.
|
|
813
|
+
if (requestLower.split(/[^\w./-]+/).map((token) => token.replace(/[.,;:!?]+$/, ''))
|
|
814
|
+
.some((token) => token.length > ext.length && token.endsWith(ext)))
|
|
815
|
+
return true;
|
|
816
|
+
// Review round 2: a request that names the language itself (`.sh`, `shell`, `kotlin`) is about it.
|
|
817
|
+
if (requestTokens.has(ext.slice(1)) || (LANGUAGE_NAMES[ext] ?? []).some((name) => requestTokens.has(name)))
|
|
818
|
+
return true;
|
|
819
|
+
const files = (input.coverage.unsupportedFiles ?? []).filter((file) => file.toLowerCase().endsWith(ext));
|
|
820
|
+
if (pathsCapped || (requestIsNonAscii && files.length > 0))
|
|
821
|
+
return true;
|
|
822
|
+
// Every token of the path — folders too (`app/session/store.rb`) — exact, or sharing a 5-letter stem.
|
|
823
|
+
return files.some((file) => rawTokens(file.slice(0, -ext.length)).some(tokenMeetsRequest));
|
|
824
|
+
});
|
|
825
|
+
// The TEST SCOPE keeps every language (A-SPEC-741 revision): relevance decides the classification
|
|
826
|
+
// only. Dropping the unrelated ones from the scope let anchor-based narrowing miss tests on replay.
|
|
743
827
|
const unresolvedCoverage = [
|
|
744
828
|
...input.coverage.skippedFiles,
|
|
745
829
|
...input.coverage.unsupportedLanguages.map((language) => `unsupported:${language}`),
|
|
@@ -759,16 +843,23 @@ function analyzeMaintenance(input) {
|
|
|
759
843
|
.sort((a, b) => b.date.localeCompare(a.date) || a.commit.localeCompare(b.commit));
|
|
760
844
|
const skippedFiles = sortedUnique(input.coverage.skippedFiles);
|
|
761
845
|
const unsupportedLanguages = sortedUnique(input.coverage.unsupportedLanguages);
|
|
762
|
-
|
|
763
|
-
|
|
846
|
+
// @implements A-SPEC-741 — the classification gate holds only gaps about the change. A truncated
|
|
847
|
+
// walk bounds how far the TEST SCOPE can be trusted (it stays in `unresolvedCoverage`, which forces
|
|
848
|
+
// `full`); it says nothing about whether the request maps to an existing requirement.
|
|
849
|
+
const coverageGaps = skippedFiles.length > 0 || relevantUnsupported.length > 0
|
|
850
|
+
|| input.coverage.scannedFiles === 0;
|
|
764
851
|
const stale = input.basis.loadedBuild !== input.basis.diskBuild;
|
|
765
852
|
const unknowns = [];
|
|
766
853
|
if (input.coverage.scannedFiles === 0)
|
|
767
854
|
unknowns.push('No source files were represented in the analyzed graph.');
|
|
768
855
|
if (skippedFiles.length > 0)
|
|
769
856
|
unknowns.push(`Skipped files are outside the evidence basis: ${skippedFiles.join(', ')}`);
|
|
770
|
-
if (unsupportedLanguages.length > 0)
|
|
771
|
-
|
|
857
|
+
if (unsupportedLanguages.length > 0) {
|
|
858
|
+
const outside = unsupportedLanguages.filter((language) => !relevantUnsupported.includes(language));
|
|
859
|
+
unknowns.push(`Unsupported languages are outside the evidence basis: ${unsupportedLanguages.join(', ')}`
|
|
860
|
+
+ (outside.length > 0 ? ` (${outside.join(', ')} judged outside this change for the classification: no changed file, request path or path-sharing file names it; the test scope still counts it)` : '')
|
|
861
|
+
+ (pathsCapped ? ' (the list of unreadable files was capped, so every language counts as about this change)' : ''));
|
|
862
|
+
}
|
|
772
863
|
if (truncationMarkers.length > 0) {
|
|
773
864
|
unknowns.push(`Impact closure was truncated (${truncationReasons.join(', ')}) at ${stoppedAt.length} node(s): callers beyond the bound were never explored, so the transitive impact is a lower bound, not a complete set. Stopped at ${stoppedAt.map((stop) => stop.id).join(', ')}.`);
|
|
774
865
|
}
|
|
@@ -936,17 +1027,32 @@ function analyzeMaintenance(input) {
|
|
|
936
1027
|
+ `scan (libraries and builtins) and ${edgeAmbiguous} are ambiguous. Impact traversal uses only `
|
|
937
1028
|
+ `the resolved ones.`);
|
|
938
1029
|
}
|
|
1030
|
+
// @implements A-SPEC-745 — the same set as `specIds`, emitted in relevance order: the response
|
|
1031
|
+
// bound keeps ten, and id order made those ten A-SPEC-100 and its neighbours for every request.
|
|
1032
|
+
// Cited first, then by the rank of the first candidate file that anchors the spec, then by id.
|
|
1033
|
+
const specRank = new Map();
|
|
1034
|
+
for (const id of (0, localize_1.citationsIn)(input.request, new Set(specIds)).cited)
|
|
1035
|
+
specRank.set(id, -1);
|
|
1036
|
+
const anchorsOf = new Map(scanned.map((file) => [file.sourcePath, file.implementsSpecs]));
|
|
1037
|
+
candidates.forEach((candidate, index) => {
|
|
1038
|
+
for (const id of anchorsOf.get(candidate.file) ?? [])
|
|
1039
|
+
if (!specRank.has(id))
|
|
1040
|
+
specRank.set(id, index);
|
|
1041
|
+
});
|
|
1042
|
+
const relevanceOrderedSpecs = [...specIds].sort((a, b) => (specRank.get(a) ?? Infinity) - (specRank.get(b) ?? Infinity) || a.localeCompare(b));
|
|
939
1043
|
const evidence = {
|
|
940
1044
|
schemaVersion: exports.MAINTENANCE_ANALYSIS_SCHEMA,
|
|
941
1045
|
extractorVersion: exports.MAINTENANCE_ANALYSIS_EXTRACTOR,
|
|
942
1046
|
classification,
|
|
943
1047
|
confidence,
|
|
1048
|
+
confidenceCalibrated: false,
|
|
1049
|
+
confidenceNote: exports.CONFIDENCE_NOTE,
|
|
944
1050
|
citationCandidates,
|
|
945
1051
|
callEdges,
|
|
946
1052
|
candidates,
|
|
947
1053
|
...(semanticAlternates !== undefined ? { semanticAlternates } : {}),
|
|
948
1054
|
rerankPool,
|
|
949
|
-
candidateSpecs:
|
|
1055
|
+
candidateSpecs: relevanceOrderedSpecs,
|
|
950
1056
|
impacts: {
|
|
951
1057
|
files: localizedFiles,
|
|
952
1058
|
tiers: { primary: primaryTier, secondary: secondaryTier },
|
|
@@ -27,6 +27,11 @@ export interface EvidenceArtifact {
|
|
|
27
27
|
extractorVersion: string;
|
|
28
28
|
classification: string;
|
|
29
29
|
confidence: number;
|
|
30
|
+
/**
|
|
31
|
+
* @implements A-SPEC-740 — optional on purpose: an artifact recorded before the marker existed is
|
|
32
|
+
* just as uncalibrated, and the schema string stays 1.4 so calibration still reads it.
|
|
33
|
+
*/
|
|
34
|
+
confidenceCalibrated?: false;
|
|
30
35
|
graphBasis: MaintenanceAnalysis['graphBasis'];
|
|
31
36
|
predicted: {
|
|
32
37
|
files: string[];
|
|
@@ -58,6 +63,9 @@ export interface EvidenceArtifact {
|
|
|
58
63
|
/** The caller walk stopped at a bound: the transitive set is a lower bound, not a complete one. */
|
|
59
64
|
impactTruncated: boolean;
|
|
60
65
|
testScopeTier: string;
|
|
66
|
+
/** @implements A-SPEC-744 — why the scope was what it was; optional so older artifacts still read. */
|
|
67
|
+
testScopeReason?: string;
|
|
68
|
+
testScopeCauses?: string[];
|
|
61
69
|
/** Files reachable only through a call edge — affected, but never analysed as edit candidates. */
|
|
62
70
|
impactedFiles: string[];
|
|
63
71
|
/**
|
|
@@ -170,7 +178,24 @@ export interface Calibration {
|
|
|
170
178
|
falseNegatives: number;
|
|
171
179
|
minSamples: number;
|
|
172
180
|
status: 'measured' | 'insufficient-data';
|
|
181
|
+
/**
|
|
182
|
+
* @implements A-SPEC-744 — how often each cause made a persisted scope `full`. `unrecorded` counts
|
|
183
|
+
* artifacts written before the causes were kept: they are not evidence for any cause.
|
|
184
|
+
*/
|
|
185
|
+
scopeCauses: {
|
|
186
|
+
full: number;
|
|
187
|
+
narrow: number;
|
|
188
|
+
unrecorded: number;
|
|
189
|
+
causes: Record<string, number>;
|
|
190
|
+
};
|
|
173
191
|
}
|
|
192
|
+
/**
|
|
193
|
+
* @implements A-SPEC-744
|
|
194
|
+
* The named reasons a test scope is `full`, read from its structured fields — never parsed out of the
|
|
195
|
+
* sentence. 455 persisted analyses were all `full` and none said why; the investigation that followed
|
|
196
|
+
* needed a 120-commit replay to answer a question the analysis had already known.
|
|
197
|
+
*/
|
|
198
|
+
export declare function scopeCauses(scope: MaintenanceAnalysis['testScope']): string[];
|
|
174
199
|
/**
|
|
175
200
|
* Reliability bins plus a Brier score. `minSamples` is the honesty knob: below it a bin reports
|
|
176
201
|
* `insufficient-data` and a null accuracy, because one sample in a bin yields a 0 or a 1 that looks
|
|
@@ -41,8 +41,9 @@ exports.readArtifacts = readArtifacts;
|
|
|
41
41
|
exports.recordOutcome = recordOutcome;
|
|
42
42
|
exports.openArtifacts = openArtifacts;
|
|
43
43
|
exports.closeOpenArtifacts = closeOpenArtifacts;
|
|
44
|
+
exports.scopeCauses = scopeCauses;
|
|
44
45
|
exports.computeCalibration = computeCalibration;
|
|
45
|
-
// @implements A-SPEC-724.1, A-SPEC-271
|
|
46
|
+
// @implements A-SPEC-744, A-SPEC-740, A-SPEC-724.1, A-SPEC-271
|
|
46
47
|
// @implements A-SPEC-268
|
|
47
48
|
const fs = __importStar(require("node:fs"));
|
|
48
49
|
const path = __importStar(require("node:path"));
|
|
@@ -85,6 +86,7 @@ function artifactFrom(analysis, recordedAt, fileDigests = {}) {
|
|
|
85
86
|
extractorVersion: analysis.extractorVersion,
|
|
86
87
|
classification: analysis.classification,
|
|
87
88
|
confidence: analysis.confidence,
|
|
89
|
+
confidenceCalibrated: false,
|
|
88
90
|
graphBasis: analysis.graphBasis,
|
|
89
91
|
predicted: {
|
|
90
92
|
files: sortedUnique(analysis.impacts.files),
|
|
@@ -103,6 +105,8 @@ function artifactFrom(analysis, recordedAt, fileDigests = {}) {
|
|
|
103
105
|
},
|
|
104
106
|
impactTruncated: analysis.impactBounds.stoppedAt.length > 0,
|
|
105
107
|
testScopeTier: analysis.testScope.tier,
|
|
108
|
+
testScopeReason: analysis.testScope.reason,
|
|
109
|
+
testScopeCauses: scopeCauses(analysis.testScope),
|
|
106
110
|
impactedFiles: [...analysis.impacts.impactedFiles],
|
|
107
111
|
securityAssessed: analysis.constraints.security.assessed,
|
|
108
112
|
compatibilityAssessed: analysis.constraints.compatibility.assessed,
|
|
@@ -257,7 +261,53 @@ function closeOpenArtifacts(dir, actual, recordedAt, since) {
|
|
|
257
261
|
}
|
|
258
262
|
return { closed };
|
|
259
263
|
}
|
|
264
|
+
/**
|
|
265
|
+
* @implements A-SPEC-744
|
|
266
|
+
* The named reasons a test scope is `full`, read from its structured fields — never parsed out of the
|
|
267
|
+
* sentence. 455 persisted analyses were all `full` and none said why; the investigation that followed
|
|
268
|
+
* needed a 120-commit replay to answer a question the analysis had already known.
|
|
269
|
+
*/
|
|
270
|
+
function scopeCauses(scope) {
|
|
271
|
+
if (scope.tier !== 'full')
|
|
272
|
+
return [];
|
|
273
|
+
const out = new Set();
|
|
274
|
+
for (const entry of scope.unresolvedFiles ?? []) {
|
|
275
|
+
out.add(entry.startsWith('unsupported:') ? 'unsupported-language'
|
|
276
|
+
: entry.startsWith('impact-truncated:') ? 'impact-truncated' : 'unresolved-file');
|
|
277
|
+
}
|
|
278
|
+
if ((scope.architectureTouched ?? []).length > 0)
|
|
279
|
+
out.add('architecture-document');
|
|
280
|
+
if ((scope.obligationGaps ?? []).length > 0)
|
|
281
|
+
out.add('obligation-gap');
|
|
282
|
+
if (scope.coverageGaps.length > 0)
|
|
283
|
+
out.add('coverage-gap');
|
|
284
|
+
if (scope.hitsFoundational)
|
|
285
|
+
out.add('foundational-requirement');
|
|
286
|
+
// Review round 1: the scope computation's "impacted specs but no anchored tests" branch was being
|
|
287
|
+
// named blast-radius, pointing remediation at the wrong knob.
|
|
288
|
+
if (out.size === 0)
|
|
289
|
+
out.add(scope.impactedTestFiles.length === 0 ? 'no-anchored-tests' : 'blast-radius');
|
|
290
|
+
return [...out].sort();
|
|
291
|
+
}
|
|
260
292
|
const BIN_EDGES = [0, 0.2, 0.4, 0.6, 0.8, 1];
|
|
293
|
+
function scopeCauseTally(artifacts) {
|
|
294
|
+
const tally = { full: 0, narrow: 0, unrecorded: 0, causes: {} };
|
|
295
|
+
for (const artifact of artifacts) {
|
|
296
|
+
const recorded = artifact.uncertainty?.testScopeCauses;
|
|
297
|
+
if (!Array.isArray(recorded)) {
|
|
298
|
+
tally.unrecorded++;
|
|
299
|
+
continue;
|
|
300
|
+
}
|
|
301
|
+
if (artifact.uncertainty.testScopeTier !== 'full') {
|
|
302
|
+
tally.narrow++;
|
|
303
|
+
continue;
|
|
304
|
+
}
|
|
305
|
+
tally.full++;
|
|
306
|
+
for (const cause of new Set(recorded))
|
|
307
|
+
tally.causes[cause] = (tally.causes[cause] ?? 0) + 1;
|
|
308
|
+
}
|
|
309
|
+
return tally;
|
|
310
|
+
}
|
|
261
311
|
/**
|
|
262
312
|
* Reliability bins plus a Brier score. `minSamples` is the honesty knob: below it a bin reports
|
|
263
313
|
* `insufficient-data` and a null accuracy, because one sample in a bin yields a 0 or a 1 that looks
|
|
@@ -316,5 +366,6 @@ function computeCalibration(artifacts, opts = {}) {
|
|
|
316
366
|
falseNegatives,
|
|
317
367
|
minSamples,
|
|
318
368
|
status: correctness.length >= minSamples ? 'measured' : 'insufficient-data',
|
|
369
|
+
scopeCauses: scopeCauseTally(artifacts),
|
|
319
370
|
};
|
|
320
371
|
}
|