@dailephd/my-dev-kit-lab 0.8.0 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/README.md +7 -4
- package/dist/src/cli/help.js +35 -1
- package/dist/src/commands/runExperimentDescribeCommand.js +15 -1
- package/dist/src/commands/runExperimentRunCommand.js +177 -4
- package/dist/src/evaluation/retrievalQuality/metrics.js +27 -2
- package/dist/src/evaluation/retrievalQueryStrategies.js +26 -0
- package/dist/src/evaluation/retrievalQueryStrategyEvidence.js +58 -0
- package/dist/src/evaluation/runMyDevKitRetrieval.js +60 -51
- package/dist/src/evaluation/runSemanticRetrievalStrategy.js +303 -0
- package/dist/src/experiments/defaultRegistry.js +4 -0
- package/dist/src/experiments/plugins/contextPackGeneration/analysis.js +157 -0
- package/dist/src/experiments/plugins/contextPackGeneration/analysisArtifact.js +22 -0
- package/dist/src/experiments/plugins/contextPackGeneration/analysisTypes.js +1 -0
- package/dist/src/experiments/plugins/contextPackGeneration/buildContextPack.js +84 -0
- package/dist/src/experiments/plugins/contextPackGeneration/config.js +39 -0
- package/dist/src/experiments/plugins/contextPackGeneration/execution.js +361 -0
- package/dist/src/experiments/plugins/contextPackGeneration/executionArtifact.js +17 -0
- package/dist/src/experiments/plugins/contextPackGeneration/executionTypes.js +11 -0
- package/dist/src/experiments/plugins/contextPackGeneration/identityEvidence.js +71 -0
- package/dist/src/experiments/plugins/contextPackGeneration/index.js +19 -0
- package/dist/src/experiments/plugins/contextPackGeneration/localSubjectExecution.js +146 -0
- package/dist/src/experiments/plugins/contextPackGeneration/localSubjectPrivacy.js +276 -0
- package/dist/src/experiments/plugins/contextPackGeneration/metadata.js +24 -0
- package/dist/src/experiments/plugins/contextPackGeneration/metrics.js +46 -0
- package/dist/src/experiments/plugins/contextPackGeneration/packArtifact.js +17 -0
- package/dist/src/experiments/plugins/contextPackGeneration/packEvidence.js +152 -0
- package/dist/src/experiments/plugins/contextPackGeneration/packSelectionPolicy.js +305 -0
- package/dist/src/experiments/plugins/contextPackGeneration/plugin.js +355 -0
- package/dist/src/experiments/plugins/contextPackGeneration/renderContextPack.js +61 -0
- package/dist/src/experiments/plugins/contextPackGeneration/types.js +29 -0
- package/dist/src/experiments/plugins/contextWindowScaling/localSubjectPrivacy.js +2 -0
- package/dist/src/experiments/plugins/index.js +1 -0
- package/dist/src/experiments/plugins/retrievalQueryStrategyComparison/analysis.js +172 -0
- package/dist/src/experiments/plugins/retrievalQueryStrategyComparison/analysisArtifact.js +28 -0
- package/dist/src/experiments/plugins/retrievalQueryStrategyComparison/analysisTypes.js +2 -0
- package/dist/src/experiments/plugins/retrievalQueryStrategyComparison/config.js +39 -0
- package/dist/src/experiments/plugins/retrievalQueryStrategyComparison/execution.js +225 -0
- package/dist/src/experiments/plugins/retrievalQueryStrategyComparison/executionArtifact.js +15 -0
- package/dist/src/experiments/plugins/retrievalQueryStrategyComparison/index.js +12 -0
- package/dist/src/experiments/plugins/retrievalQueryStrategyComparison/localSubjectExecution.js +124 -0
- package/dist/src/experiments/plugins/retrievalQueryStrategyComparison/localSubjectPrivacy.js +176 -0
- package/dist/src/experiments/plugins/retrievalQueryStrategyComparison/metadata.js +51 -0
- package/dist/src/experiments/plugins/retrievalQueryStrategyComparison/metrics.js +60 -0
- package/dist/src/experiments/plugins/retrievalQueryStrategyComparison/plugin.js +292 -0
- package/dist/src/experiments/plugins/retrievalQueryStrategyComparison/types.js +1 -0
- package/dist/src/report/experiments/buildContextPackGenerationReport.js +171 -0
- package/dist/src/report/experiments/buildPluginExperimentReport.js +47 -2
- package/dist/src/report/experiments/buildRetrievalQueryStrategyComparisonReport.js +91 -0
- package/dist/src/report/experiments/contextPackGenerationReportModel.js +9 -0
- package/dist/src/report/experiments/index.js +9 -0
- package/dist/src/report/experiments/loadContextPackArtifacts.js +43 -0
- package/dist/src/report/experiments/renderContextPackGenerationHtml.js +122 -0
- package/dist/src/report/experiments/renderContextPackGenerationText.js +101 -0
- package/dist/src/report/experiments/renderPluginExperimentReportHtml.js +6 -0
- package/dist/src/report/experiments/renderPluginExperimentReportText.js +16 -0
- package/dist/src/report/experiments/renderRetrievalQueryStrategyComparisonHtml.js +96 -0
- package/dist/src/report/experiments/renderRetrievalQueryStrategyComparisonText.js +58 -0
- package/dist/src/report/experiments/retrievalQueryStrategyComparisonReportModel.js +1 -0
- package/dist/src/report/experiments/writePluginExperimentReports.js +2 -0
- package/docs/METRICS.md +149 -2
- package/package.json +1 -1
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
import { compareCodeUnits } from "../../../evaluation/indexSnapshot.js";
|
|
2
|
+
import { normalizeRetrievedRepositoryPath } from "../../../evaluation/retrievalQuality/buildRetrievalEvidence.js";
|
|
3
|
+
/** Frozen v0.8.2 selection policy. These are experimental constants, never user configuration; do not tune them. */
|
|
4
|
+
export const CONTEXT_PACK_SELECTION_POLICY_ID = "bounded-multiseed-v1";
|
|
5
|
+
export const SEARCH_RESULT_LIMIT = 12;
|
|
6
|
+
export const MAX_SEED_NODES = 8;
|
|
7
|
+
export const GRAPH_DEPTH = 1;
|
|
8
|
+
export const MAX_FILES = 12;
|
|
9
|
+
export const MAX_SYMBOLS = 16;
|
|
10
|
+
export const MAX_SOURCE_SLICES = 8;
|
|
11
|
+
export const MAX_SOURCE_LINES_PER_SLICE = 160;
|
|
12
|
+
export const MAX_TOTAL_SOURCE_LINES = 1280;
|
|
13
|
+
export const MAX_TEST_FILES = 8;
|
|
14
|
+
export const MAX_CALL_RELATIONSHIPS = 32;
|
|
15
|
+
export function buildContextPackSelectionPolicy(myDevKitVersion) {
|
|
16
|
+
return {
|
|
17
|
+
id: CONTEXT_PACK_SELECTION_POLICY_ID,
|
|
18
|
+
searchResultLimit: SEARCH_RESULT_LIMIT,
|
|
19
|
+
maxSeedNodes: MAX_SEED_NODES,
|
|
20
|
+
graphDepth: GRAPH_DEPTH,
|
|
21
|
+
maxFiles: MAX_FILES,
|
|
22
|
+
maxSymbols: MAX_SYMBOLS,
|
|
23
|
+
maxSourceSlices: MAX_SOURCE_SLICES,
|
|
24
|
+
maxSourceLinesPerSlice: MAX_SOURCE_LINES_PER_SLICE,
|
|
25
|
+
maxTotalSourceLines: MAX_TOTAL_SOURCE_LINES,
|
|
26
|
+
maxTestFiles: MAX_TEST_FILES,
|
|
27
|
+
maxCallRelationships: MAX_CALL_RELATIONSHIPS,
|
|
28
|
+
myDevKitVersion
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
const ORIGIN_PRIORITY = { search: 0, lookup: 1, graph: 2 };
|
|
32
|
+
const ORIGIN_COMMAND = {
|
|
33
|
+
search: "search",
|
|
34
|
+
lookup: "lookup",
|
|
35
|
+
graph: "slice"
|
|
36
|
+
};
|
|
37
|
+
const FILE_REASON = {
|
|
38
|
+
search: "search-candidate",
|
|
39
|
+
lookup: "lookup-evidence",
|
|
40
|
+
graph: "graph-neighbor"
|
|
41
|
+
};
|
|
42
|
+
const compareNullableRank = (left, right) => left === right ? 0 : left === null ? 1 : right === null ? -1 : left - right;
|
|
43
|
+
const compareNullableText = (left, right) => compareCodeUnits(left ?? "", right ?? "");
|
|
44
|
+
const provenanceOf = (origin, nodeId, rank) => ({
|
|
45
|
+
command: ORIGIN_COMMAND[origin],
|
|
46
|
+
nodeId,
|
|
47
|
+
rank
|
|
48
|
+
});
|
|
49
|
+
function mergeProvenance(entries) {
|
|
50
|
+
const unique = new Map();
|
|
51
|
+
for (const entry of entries)
|
|
52
|
+
unique.set(`${entry.command}\0${entry.nodeId ?? ""}\0${entry.rank ?? ""}`, entry);
|
|
53
|
+
return [...unique.values()].sort((a, b) => compareCodeUnits(a.command, b.command) || compareNullableText(a.nodeId, b.nodeId) || compareNullableRank(a.rank, b.rank));
|
|
54
|
+
}
|
|
55
|
+
function uniqueSortedNotes(notes) {
|
|
56
|
+
return [...new Set(notes)].sort(compareCodeUnits);
|
|
57
|
+
}
|
|
58
|
+
/** Pure repository test-file classification: tests/, test/, __tests__/ directories and *.test.* / *.spec.* files. */
|
|
59
|
+
export function isTestFilePath(path) {
|
|
60
|
+
const segments = path.split("/");
|
|
61
|
+
const directories = segments.slice(0, -1);
|
|
62
|
+
if (directories.some((segment) => segment === "tests" || segment === "test" || segment === "__tests__"))
|
|
63
|
+
return true;
|
|
64
|
+
return /\.(test|spec)\.[^/.]+$/.test(segments[segments.length - 1] ?? "");
|
|
65
|
+
}
|
|
66
|
+
/** Direct search evidence, then lookup, then depth-1 graph; lower rank first; code-unit ties. Deduplicated before the cap. */
|
|
67
|
+
export function selectSeedNodes(candidates) {
|
|
68
|
+
const best = new Map();
|
|
69
|
+
for (const candidate of candidates) {
|
|
70
|
+
const current = best.get(candidate.nodeId);
|
|
71
|
+
if (!current || compareSeeds(candidate, current) < 0)
|
|
72
|
+
best.set(candidate.nodeId, { ...candidate });
|
|
73
|
+
}
|
|
74
|
+
const ordered = [...best.values()].sort(compareSeeds);
|
|
75
|
+
const items = ordered.slice(0, MAX_SEED_NODES);
|
|
76
|
+
return { items, notes: ordered.length > MAX_SEED_NODES ? ["seed-cap-reached"] : [] };
|
|
77
|
+
}
|
|
78
|
+
function compareSeeds(a, b) {
|
|
79
|
+
return ORIGIN_PRIORITY[a.origin] - ORIGIN_PRIORITY[b.origin] || a.rank - b.rank || compareCodeUnits(a.nodeId, b.nodeId);
|
|
80
|
+
}
|
|
81
|
+
export function selectRelevantFiles(candidates) {
|
|
82
|
+
const notes = [];
|
|
83
|
+
const byPath = new Map();
|
|
84
|
+
for (const candidate of candidates) {
|
|
85
|
+
const path = normalizeRetrievedRepositoryPath(candidate.path);
|
|
86
|
+
if (path === null) {
|
|
87
|
+
notes.push("invalid-repository-path-ignored");
|
|
88
|
+
continue;
|
|
89
|
+
}
|
|
90
|
+
const entry = byPath.get(path);
|
|
91
|
+
const provenance = provenanceOf(candidate.origin, null, candidate.rank);
|
|
92
|
+
const normalized = { ...candidate, path };
|
|
93
|
+
if (!entry)
|
|
94
|
+
byPath.set(path, { best: normalized, provenance: [provenance] });
|
|
95
|
+
else {
|
|
96
|
+
entry.provenance.push(provenance);
|
|
97
|
+
if (compareEvidence(normalized, entry.best) < 0)
|
|
98
|
+
entry.best = normalized;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
const ordered = [...byPath.values()].sort((a, b) => compareEvidence(a.best, b.best) || compareCodeUnits(a.best.path, b.best.path));
|
|
102
|
+
if (ordered.length > MAX_FILES)
|
|
103
|
+
notes.push("file-cap-reached");
|
|
104
|
+
return {
|
|
105
|
+
items: ordered.slice(0, MAX_FILES).map(({ best, provenance }) => ({
|
|
106
|
+
path: best.path,
|
|
107
|
+
rank: best.rank,
|
|
108
|
+
reason: isTestFilePath(best.path) ? "test-candidate" : FILE_REASON[best.origin],
|
|
109
|
+
provenance: mergeProvenance(provenance)
|
|
110
|
+
})),
|
|
111
|
+
notes: uniqueSortedNotes(notes)
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
function compareEvidence(a, b) {
|
|
115
|
+
return ORIGIN_PRIORITY[a.origin] - ORIGIN_PRIORITY[b.origin] || a.rank - b.rank;
|
|
116
|
+
}
|
|
117
|
+
export function selectRelevantSymbols(candidates) {
|
|
118
|
+
const notes = [];
|
|
119
|
+
const byKey = new Map();
|
|
120
|
+
for (const candidate of candidates) {
|
|
121
|
+
let file = null;
|
|
122
|
+
if (candidate.file !== null) {
|
|
123
|
+
file = normalizeRetrievedRepositoryPath(candidate.file);
|
|
124
|
+
if (file === null) {
|
|
125
|
+
notes.push("invalid-repository-path-ignored");
|
|
126
|
+
continue;
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
const key = candidate.nodeId !== null ? `n\0${candidate.nodeId}` : `f\0${file ?? ""}\0${candidate.name}`;
|
|
130
|
+
const normalized = { ...candidate, file };
|
|
131
|
+
const provenance = provenanceOf(candidate.origin, candidate.nodeId, candidate.rank);
|
|
132
|
+
const entry = byKey.get(key);
|
|
133
|
+
if (!entry)
|
|
134
|
+
byKey.set(key, { best: normalized, provenance: [provenance] });
|
|
135
|
+
else {
|
|
136
|
+
entry.provenance.push(provenance);
|
|
137
|
+
if (compareEvidence(normalized, entry.best) < 0)
|
|
138
|
+
entry.best = normalized;
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
const ordered = [...byKey.values()].sort((a, b) => compareEvidence(a.best, b.best) ||
|
|
142
|
+
compareNullableText(a.best.file, b.best.file) ||
|
|
143
|
+
compareCodeUnits(a.best.name, b.best.name) ||
|
|
144
|
+
compareNullableText(a.best.nodeId, b.best.nodeId));
|
|
145
|
+
if (ordered.length > MAX_SYMBOLS)
|
|
146
|
+
notes.push("symbol-cap-reached");
|
|
147
|
+
return {
|
|
148
|
+
items: ordered.slice(0, MAX_SYMBOLS).map(({ best, provenance }) => ({
|
|
149
|
+
name: best.name,
|
|
150
|
+
nodeId: best.nodeId,
|
|
151
|
+
file: best.file,
|
|
152
|
+
rank: best.rank,
|
|
153
|
+
line: best.line ?? null,
|
|
154
|
+
provenance: mergeProvenance(provenance)
|
|
155
|
+
})),
|
|
156
|
+
notes: uniqueSortedNotes(notes)
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
/** Only `calls` edges qualify; `defines`, `exports` and every other kind are ignored. Canonical order: from, then to. */
|
|
160
|
+
export function selectCallRelationships(candidates) {
|
|
161
|
+
const unique = new Map();
|
|
162
|
+
for (const candidate of candidates) {
|
|
163
|
+
if (candidate.kind !== "calls")
|
|
164
|
+
continue;
|
|
165
|
+
unique.set(`${candidate.fromNodeId}\0${candidate.toNodeId}`, {
|
|
166
|
+
fromNodeId: candidate.fromNodeId,
|
|
167
|
+
toNodeId: candidate.toNodeId,
|
|
168
|
+
kind: "calls",
|
|
169
|
+
provenance: [{ command: "slice", nodeId: candidate.fromNodeId, rank: null }]
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
const ordered = [...unique.values()].sort((a, b) => compareCodeUnits(a.fromNodeId, b.fromNodeId) || compareCodeUnits(a.toNodeId, b.toNodeId));
|
|
173
|
+
return {
|
|
174
|
+
items: ordered.slice(0, MAX_CALL_RELATIONSHIPS),
|
|
175
|
+
notes: ordered.length > MAX_CALL_RELATIONSHIPS ? ["call-relationship-cap-reached"] : []
|
|
176
|
+
};
|
|
177
|
+
}
|
|
178
|
+
const TEST_ORIGIN_PRIORITY = { "search-candidate": 0, "graph-neighbor": 1 };
|
|
179
|
+
/** Descriptive test evidence only; non-test paths are ignored. No coverage score exists. */
|
|
180
|
+
export function selectTestFiles(candidates) {
|
|
181
|
+
const notes = [];
|
|
182
|
+
const byPath = new Map();
|
|
183
|
+
const compare = (a, b) => TEST_ORIGIN_PRIORITY[a.origin] - TEST_ORIGIN_PRIORITY[b.origin] || compareNullableRank(a.rank, b.rank);
|
|
184
|
+
for (const candidate of candidates) {
|
|
185
|
+
const path = normalizeRetrievedRepositoryPath(candidate.path);
|
|
186
|
+
if (path === null) {
|
|
187
|
+
notes.push("invalid-repository-path-ignored");
|
|
188
|
+
continue;
|
|
189
|
+
}
|
|
190
|
+
if (!isTestFilePath(path))
|
|
191
|
+
continue;
|
|
192
|
+
const rank = candidate.rank ?? null;
|
|
193
|
+
const provenance = { command: candidate.origin === "search-candidate" ? "search" : "slice", nodeId: null, rank };
|
|
194
|
+
const normalized = { path, origin: candidate.origin, rank };
|
|
195
|
+
const entry = byPath.get(path);
|
|
196
|
+
if (!entry)
|
|
197
|
+
byPath.set(path, { best: normalized, provenance: [provenance] });
|
|
198
|
+
else {
|
|
199
|
+
entry.provenance.push(provenance);
|
|
200
|
+
if (compare(normalized, entry.best) < 0)
|
|
201
|
+
entry.best = normalized;
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
const ordered = [...byPath.values()].sort((a, b) => compare(a.best, b.best) || compareCodeUnits(a.best.path, b.best.path));
|
|
205
|
+
if (ordered.length > MAX_TEST_FILES)
|
|
206
|
+
notes.push("test-file-cap-reached");
|
|
207
|
+
return {
|
|
208
|
+
items: ordered.slice(0, MAX_TEST_FILES).map(({ best, provenance }) => ({
|
|
209
|
+
path: best.path,
|
|
210
|
+
rank: best.rank,
|
|
211
|
+
how: best.origin,
|
|
212
|
+
provenance: mergeProvenance(provenance)
|
|
213
|
+
})),
|
|
214
|
+
notes: uniqueSortedNotes(notes)
|
|
215
|
+
};
|
|
216
|
+
}
|
|
217
|
+
const normalizeLineEndings = (text) => text.replace(/\r\n?/g, "\n");
|
|
218
|
+
function splitLines(text) {
|
|
219
|
+
const normalized = normalizeLineEndings(text);
|
|
220
|
+
if (normalized.length === 0)
|
|
221
|
+
return [];
|
|
222
|
+
const lines = normalized.split("\n");
|
|
223
|
+
if (lines[lines.length - 1] === "")
|
|
224
|
+
lines.pop();
|
|
225
|
+
return lines;
|
|
226
|
+
}
|
|
227
|
+
const FROZEN_SOURCE_SLICE_LIMITS = {
|
|
228
|
+
maxSourceSlices: MAX_SOURCE_SLICES,
|
|
229
|
+
maxSourceLinesPerSlice: MAX_SOURCE_LINES_PER_SLICE,
|
|
230
|
+
maxTotalSourceLines: MAX_TOTAL_SOURCE_LINES
|
|
231
|
+
};
|
|
232
|
+
/**
|
|
233
|
+
* Applies the source caps to already-captured slices; performs no source command and no continuation.
|
|
234
|
+
* Slices are consumed in (rank, file, startLine, endLine, nodeId) order. A slice longer than the per-slice cap keeps its
|
|
235
|
+
* first lines. The total cap truncates the slice that crosses it and drops every later slice; earlier slices are never
|
|
236
|
+
* reordered. The returned slices are in canonical render order: file, startLine, endLine, nodeId.
|
|
237
|
+
*/
|
|
238
|
+
export function selectSourceSlices(candidates,
|
|
239
|
+
/** Test seam only: with the frozen constants the slice-count cap (8 x 160 = 1280) is reached before the total cap can be crossed. */
|
|
240
|
+
limits = FROZEN_SOURCE_SLICE_LIMITS) {
|
|
241
|
+
const notes = [];
|
|
242
|
+
const prepared = [];
|
|
243
|
+
for (const candidate of candidates) {
|
|
244
|
+
const file = normalizeRetrievedRepositoryPath(candidate.file);
|
|
245
|
+
if (file === null) {
|
|
246
|
+
notes.push("invalid-repository-path-ignored");
|
|
247
|
+
continue;
|
|
248
|
+
}
|
|
249
|
+
const lines = splitLines(candidate.text);
|
|
250
|
+
if (!Number.isInteger(candidate.startLine) || candidate.startLine < 1 || lines.length === 0) {
|
|
251
|
+
notes.push("invalid-source-slice-ignored");
|
|
252
|
+
continue;
|
|
253
|
+
}
|
|
254
|
+
prepared.push({ ...candidate, file, nodeId: candidate.nodeId ?? null, lines });
|
|
255
|
+
}
|
|
256
|
+
const byRange = (a, b) => compareCodeUnits(a.file, b.file) || a.startLine - b.startLine || a.lines.length - b.lines.length || compareNullableText(a.nodeId, b.nodeId);
|
|
257
|
+
prepared.sort((a, b) => a.rank - b.rank || byRange(a, b));
|
|
258
|
+
const selected = [];
|
|
259
|
+
const seen = new Set();
|
|
260
|
+
let usedLines = 0;
|
|
261
|
+
for (const candidate of prepared) {
|
|
262
|
+
const key = `${candidate.file}\0${candidate.startLine}\0${candidate.lines.length}\0${candidate.nodeId ?? ""}`;
|
|
263
|
+
if (seen.has(key))
|
|
264
|
+
continue;
|
|
265
|
+
seen.add(key);
|
|
266
|
+
if (selected.length >= limits.maxSourceSlices) {
|
|
267
|
+
notes.push("source-slice-count-cap-reached");
|
|
268
|
+
break;
|
|
269
|
+
}
|
|
270
|
+
if (usedLines >= limits.maxTotalSourceLines) {
|
|
271
|
+
notes.push("total-source-line-cap-reached");
|
|
272
|
+
break;
|
|
273
|
+
}
|
|
274
|
+
let keep = candidate.lines.length;
|
|
275
|
+
let truncated = candidate.truncated === true;
|
|
276
|
+
if (keep > limits.maxSourceLinesPerSlice) {
|
|
277
|
+
keep = limits.maxSourceLinesPerSlice;
|
|
278
|
+
truncated = true;
|
|
279
|
+
notes.push("source-slice-cap-reached");
|
|
280
|
+
}
|
|
281
|
+
if (usedLines + keep > limits.maxTotalSourceLines) {
|
|
282
|
+
keep = limits.maxTotalSourceLines - usedLines;
|
|
283
|
+
truncated = true;
|
|
284
|
+
notes.push("total-source-line-cap-reached");
|
|
285
|
+
}
|
|
286
|
+
usedLines += keep;
|
|
287
|
+
if (!candidate.boundaryKnown)
|
|
288
|
+
notes.push("symbol-end-unknown");
|
|
289
|
+
selected.push({
|
|
290
|
+
file: candidate.file,
|
|
291
|
+
nodeId: candidate.nodeId,
|
|
292
|
+
symbolName: candidate.symbolName ?? null,
|
|
293
|
+
startLine: candidate.startLine,
|
|
294
|
+
endLine: candidate.startLine + keep - 1,
|
|
295
|
+
lineCount: keep,
|
|
296
|
+
text: candidate.lines.slice(0, keep).join("\n"),
|
|
297
|
+
boundaryKnown: candidate.boundaryKnown,
|
|
298
|
+
truncated,
|
|
299
|
+
continuationAvailable: candidate.continuationAvailable === true,
|
|
300
|
+
provenance: [{ command: "source", nodeId: candidate.nodeId, rank: candidate.rank }]
|
|
301
|
+
});
|
|
302
|
+
}
|
|
303
|
+
selected.sort((a, b) => compareCodeUnits(a.file, b.file) || a.startLine - b.startLine || a.endLine - b.endLine || compareNullableText(a.nodeId, b.nodeId));
|
|
304
|
+
return { items: selected, notes: uniqueSortedNotes(notes) };
|
|
305
|
+
}
|
|
@@ -0,0 +1,355 @@
|
|
|
1
|
+
import { access, mkdir, rm, writeFile } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { resolveWithinRoot } from "../../../core/pathSafety.js";
|
|
4
|
+
import { serializeLocalRepositorySubjectManifest } from "../../../evaluation/localRepositorySubject/index.js";
|
|
5
|
+
import { summarizeExperimentRun } from "../../results.js";
|
|
6
|
+
import { LocalSubjectExecutionError } from "../contextWindowScaling/localSubjectErrors.js";
|
|
7
|
+
import { projectExternalLocalTarget } from "../contextWindowScaling/localSubjectPrivacy.js";
|
|
8
|
+
import { isSamePhysicalDirectory } from "../contextWindowScaling/localSubjectScratch.js";
|
|
9
|
+
import { selectWarmIndexCases } from "../warmIndexReuse/selection.js";
|
|
10
|
+
import { analyzeContextPackGeneration } from "./analysis.js";
|
|
11
|
+
import { buildContextPackGenerationAnalysisArtifact, CONTEXT_PACK_GENERATION_ANALYSIS_ARTIFACT_FILE } from "./analysisArtifact.js";
|
|
12
|
+
import { contextPackGenerationConfigDefinition, defaultContextPackGenerationConfig, validateContextPackGenerationConfig } from "./config.js";
|
|
13
|
+
import { executeContextPackGeneration } from "./execution.js";
|
|
14
|
+
import { buildContextPackGenerationExecutionArtifact, CONTEXT_PACK_GENERATION_EXECUTION_ARTIFACT_FILE } from "./executionArtifact.js";
|
|
15
|
+
import { CONTEXT_PACK_GENERATION_PLUGIN_ID, CONTEXT_PACK_GENERATION_VARIANTS, contextPackGenerationMetadata } from "./metadata.js";
|
|
16
|
+
import { toContextPackGenerationCaseComparisonMetrics, toContextPackGenerationOutcomeMetrics, toContextPackGenerationRunMetrics } from "./metrics.js";
|
|
17
|
+
import { executeLocalRepositorySubjectContextPackGeneration } from "./localSubjectExecution.js";
|
|
18
|
+
import { assertContextPackExternalProjectionIsPrivate, collectContextPackExternalPrivateValues, CONTEXT_PACK_EXTERNAL_PERSISTENCE_FAILURE_MESSAGE, CONTEXT_PACK_PRIVACY_FAILURE_MESSAGE, CONTEXT_PACK_UNEXPECTED_EXTERNAL_FAILURE_MESSAGE, describeContextPackLocalSubjectFailureForPersistence, projectContextPackAnalysisForExternalLocalPersistence, projectContextPackExecutionForExternalLocalPersistence } from "./localSubjectPrivacy.js";
|
|
19
|
+
import { contextPackArtifactRelativePath, serializeContextPackArtifact } from "./packArtifact.js";
|
|
20
|
+
/** An external target must always arrive with a loaded local repository subject; the bundled corpus never runs against it. */
|
|
21
|
+
export const CONTEXT_PACK_GENERATION_SELF_ONLY_MESSAGE = "External-local context-pack-generation targets require a loaded local repository subject (--local-subject-config).";
|
|
22
|
+
/** Privacy-safe subject manifest persisted beside the artifacts for external-local runs (same file as the other local-subject plugins). */
|
|
23
|
+
export const CONTEXT_PACK_GENERATION_LOCAL_SUBJECT_MANIFEST_FILE = "local-repository-subject-manifest.json";
|
|
24
|
+
export const CONTEXT_PACK_GENERATION_PERSISTENCE_FAILURE_MESSAGE = "Context pack generation artifact persistence failed.";
|
|
25
|
+
export const CONTEXT_PACK_GENERATION_ARTIFACT_STATE_MESSAGE = "Context pack generation artifact state was inconsistent; nothing was written.";
|
|
26
|
+
const defaultArtifactIo = {
|
|
27
|
+
ensureDirectory: async (directory) => {
|
|
28
|
+
await mkdir(directory, { recursive: true });
|
|
29
|
+
},
|
|
30
|
+
exists: async (filePath) => {
|
|
31
|
+
try {
|
|
32
|
+
await access(filePath);
|
|
33
|
+
return true;
|
|
34
|
+
}
|
|
35
|
+
catch {
|
|
36
|
+
return false;
|
|
37
|
+
}
|
|
38
|
+
},
|
|
39
|
+
writeFile: (filePath, content) => writeFile(filePath, content, "utf8"),
|
|
40
|
+
removeFile: (filePath) => rm(filePath, { force: true })
|
|
41
|
+
};
|
|
42
|
+
export const contextPackGenerationPlugin = {
|
|
43
|
+
metadata: contextPackGenerationMetadata,
|
|
44
|
+
defaultConfig: defaultContextPackGenerationConfig,
|
|
45
|
+
configDefinition: contextPackGenerationConfigDefinition,
|
|
46
|
+
supportedVariants: CONTEXT_PACK_GENERATION_VARIANTS.map((variant) => variant.id),
|
|
47
|
+
validateConfig: validateContextPackGenerationConfig,
|
|
48
|
+
async run(context) {
|
|
49
|
+
const startedAt = context.startedAt.toISOString();
|
|
50
|
+
const inputs = context.inputs;
|
|
51
|
+
const localSubject = readLocalSubjectInput(inputs);
|
|
52
|
+
if (localSubject)
|
|
53
|
+
return runLocalSubject(context, localSubject, startedAt);
|
|
54
|
+
// The runner does not enforce supportedTargets; never run the bundled corpus against another target.
|
|
55
|
+
if (context.target.kind !== "self" || context.target.isSelf !== true) {
|
|
56
|
+
throw new Error(CONTEXT_PACK_GENERATION_SELF_ONLY_MESSAGE);
|
|
57
|
+
}
|
|
58
|
+
const cases = selectWarmIndexCases(readCasesInput(inputs), context.config);
|
|
59
|
+
const outDir = context.outputRoot ?? path.resolve(context.toolRoot, context.config.outDir);
|
|
60
|
+
const io = readArtifactIo(inputs);
|
|
61
|
+
await io.ensureDirectory(outDir);
|
|
62
|
+
// 1-3. Execute every case and construct all packs in memory; nothing durable is written yet.
|
|
63
|
+
const results = await executeContextPackGeneration({
|
|
64
|
+
cases,
|
|
65
|
+
kitCommand: context.config.kitCommand,
|
|
66
|
+
outputRoot: outDir,
|
|
67
|
+
dependencies: readDependenciesInput(inputs)
|
|
68
|
+
});
|
|
69
|
+
const caseEvidence = results.map((result) => result.evidence);
|
|
70
|
+
// Scientific analysis is calculated in memory before persistence; reports later consume it, never recalculate it.
|
|
71
|
+
const analysis = analyzeContextPackGeneration(cases, caseEvidence);
|
|
72
|
+
const completedAt = new Date().toISOString();
|
|
73
|
+
// 4-6. Build and validate the complete artifact state.
|
|
74
|
+
const common = { runId: context.runId, pluginId: contextPackGenerationMetadata.id, pluginSchemaVersion: contextPackGenerationMetadata.schemaVersion, startedAt, completedAt };
|
|
75
|
+
const executionArtifact = buildContextPackGenerationExecutionArtifact({ ...common, cases: caseEvidence });
|
|
76
|
+
const analysisArtifact = buildContextPackGenerationAnalysisArtifact({ ...common, analysis });
|
|
77
|
+
const packFiles = collectPackFiles(results, outDir);
|
|
78
|
+
// 7-9. Packs in corpus order, then the execution artifact, then the analysis artifact last.
|
|
79
|
+
const executionArtifactPath = resolveWithinRoot(outDir, CONTEXT_PACK_GENERATION_EXECUTION_ARTIFACT_FILE);
|
|
80
|
+
const analysisArtifactPath = resolveWithinRoot(outDir, CONTEXT_PACK_GENERATION_ANALYSIS_ARTIFACT_FILE);
|
|
81
|
+
const created = [];
|
|
82
|
+
const write = async (filePath, content) => {
|
|
83
|
+
const existed = await io.exists(filePath);
|
|
84
|
+
await io.writeFile(filePath, content);
|
|
85
|
+
if (!existed)
|
|
86
|
+
created.push(filePath);
|
|
87
|
+
};
|
|
88
|
+
try {
|
|
89
|
+
if (packFiles.length > 0)
|
|
90
|
+
await io.ensureDirectory(resolveWithinRoot(outDir, "packs"));
|
|
91
|
+
for (const packFile of packFiles)
|
|
92
|
+
await write(packFile.absolutePath, packFile.content);
|
|
93
|
+
await write(executionArtifactPath, `${JSON.stringify(executionArtifact, null, 2)}\n`);
|
|
94
|
+
await write(analysisArtifactPath, `${JSON.stringify(analysisArtifact, null, 2)}\n`);
|
|
95
|
+
}
|
|
96
|
+
catch {
|
|
97
|
+
// Remove only files this attempt created; never touch anything else.
|
|
98
|
+
for (const filePath of created.reverse()) {
|
|
99
|
+
try {
|
|
100
|
+
await io.removeFile(filePath);
|
|
101
|
+
}
|
|
102
|
+
catch {
|
|
103
|
+
// best effort: the failure below is the outcome
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
throw new Error(CONTEXT_PACK_GENERATION_PERSISTENCE_FAILURE_MESSAGE);
|
|
107
|
+
}
|
|
108
|
+
return mapContextPackGenerationToRun({
|
|
109
|
+
runId: context.runId,
|
|
110
|
+
startedAt,
|
|
111
|
+
completedAt,
|
|
112
|
+
target: context.target,
|
|
113
|
+
caseEvidence,
|
|
114
|
+
analysis,
|
|
115
|
+
executionArtifactPath,
|
|
116
|
+
analysisArtifactPath,
|
|
117
|
+
packArtifactPaths: packFiles.map((packFile) => ({ caseId: packFile.caseId, path: packFile.relativePath }))
|
|
118
|
+
});
|
|
119
|
+
},
|
|
120
|
+
summarize(result) {
|
|
121
|
+
return result.summary ?? summarizeExperimentRun(result);
|
|
122
|
+
}
|
|
123
|
+
};
|
|
124
|
+
function readLocalSubjectInput(inputs) {
|
|
125
|
+
const value = inputs?.localSubject;
|
|
126
|
+
return value && typeof value === "object" ? value : undefined;
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* External-local path. The safe seam owns execution, immutability and scratch cleanup and either returns real in-memory
|
|
130
|
+
* results or throws. Science is calculated from the real, unprojected identities; only then is the evidence
|
|
131
|
+
* privacy-projected and asserted private. Nothing durable is written before every gate has passed, and then in this
|
|
132
|
+
* order: execution artifact, analysis artifact, manifest. No context-pack body is ever written.
|
|
133
|
+
*/
|
|
134
|
+
async function runLocalSubject(context, subject, startedAt) {
|
|
135
|
+
if (context.target.isSelf || context.target.kind !== "external-local") {
|
|
136
|
+
throw new Error("Local-repository subject mode requires an external-local target.");
|
|
137
|
+
}
|
|
138
|
+
if (context.config.caseIds !== undefined || context.config.benchmarkProjects !== undefined) {
|
|
139
|
+
throw new Error("Case and benchmark-project filters are not supported for an external local repository subject; the subject config owns the case set.");
|
|
140
|
+
}
|
|
141
|
+
if (!(await isSamePhysicalDirectory(context.target.targetRoot, subject.repositoryRoot))) {
|
|
142
|
+
throw new Error("The selected --target is not the repository the local subject was loaded from.");
|
|
143
|
+
}
|
|
144
|
+
const outDir = context.outputRoot;
|
|
145
|
+
if (!outDir)
|
|
146
|
+
throw new Error("Local-repository subject mode requires an experiment output directory.");
|
|
147
|
+
const io = readArtifactIo(context.inputs);
|
|
148
|
+
let local;
|
|
149
|
+
try {
|
|
150
|
+
local = await executeLocalRepositorySubjectContextPackGeneration({
|
|
151
|
+
subject,
|
|
152
|
+
kitCommand: context.config.kitCommand,
|
|
153
|
+
workRoot: outDir,
|
|
154
|
+
dependencies: readDependenciesInput(context.inputs)
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
catch (error) {
|
|
158
|
+
if (error instanceof LocalSubjectExecutionError)
|
|
159
|
+
throw new Error(describeContextPackLocalSubjectFailureForPersistence(error));
|
|
160
|
+
throw new Error(CONTEXT_PACK_UNEXPECTED_EXTERNAL_FAILURE_MESSAGE);
|
|
161
|
+
}
|
|
162
|
+
// Science first, on real identities; placeholders are never scientific identities.
|
|
163
|
+
const realEvidence = local.results.map((result) => result.evidence);
|
|
164
|
+
let analysis;
|
|
165
|
+
try {
|
|
166
|
+
analysis = analyzeContextPackGeneration(subject.evaluationCases, realEvidence);
|
|
167
|
+
}
|
|
168
|
+
catch {
|
|
169
|
+
throw new Error(CONTEXT_PACK_UNEXPECTED_EXTERNAL_FAILURE_MESSAGE);
|
|
170
|
+
}
|
|
171
|
+
let projectedExecution;
|
|
172
|
+
let projectedAnalysis;
|
|
173
|
+
try {
|
|
174
|
+
projectedExecution = projectContextPackExecutionForExternalLocalPersistence(local.results);
|
|
175
|
+
projectedAnalysis = projectContextPackAnalysisForExternalLocalPersistence(analysis);
|
|
176
|
+
}
|
|
177
|
+
catch {
|
|
178
|
+
throw new Error(CONTEXT_PACK_PRIVACY_FAILURE_MESSAGE);
|
|
179
|
+
}
|
|
180
|
+
const privateValues = collectContextPackExternalPrivateValues({ subject, results: local.results, outputRoot: outDir });
|
|
181
|
+
const manifestText = serializeLocalRepositorySubjectManifest(subject.manifest);
|
|
182
|
+
assertContextPackExternalProjectionIsPrivate(projectedExecution, privateValues);
|
|
183
|
+
assertContextPackExternalProjectionIsPrivate(projectedAnalysis, privateValues);
|
|
184
|
+
assertContextPackExternalProjectionIsPrivate(manifestText, privateValues);
|
|
185
|
+
const completedAt = new Date().toISOString();
|
|
186
|
+
const common = { runId: context.runId, pluginId: contextPackGenerationMetadata.id, pluginSchemaVersion: contextPackGenerationMetadata.schemaVersion, startedAt, completedAt };
|
|
187
|
+
const executionArtifact = buildContextPackGenerationExecutionArtifact({ ...common, cases: projectedExecution });
|
|
188
|
+
const analysisArtifact = buildContextPackGenerationAnalysisArtifact({ ...common, analysis: projectedAnalysis });
|
|
189
|
+
const executionText = `${JSON.stringify(executionArtifact, null, 2)}\n`;
|
|
190
|
+
const analysisText = `${JSON.stringify(analysisArtifact, null, 2)}\n`;
|
|
191
|
+
// The run carries only relative artifact names and the projected logical target; pack bodies never exist externally.
|
|
192
|
+
const run = mapContextPackGenerationToRun({
|
|
193
|
+
runId: context.runId,
|
|
194
|
+
startedAt,
|
|
195
|
+
completedAt,
|
|
196
|
+
target: projectExternalLocalTarget(subject.manifest),
|
|
197
|
+
caseEvidence: projectedExecution,
|
|
198
|
+
analysis: projectedAnalysis,
|
|
199
|
+
executionArtifactPath: CONTEXT_PACK_GENERATION_EXECUTION_ARTIFACT_FILE,
|
|
200
|
+
analysisArtifactPath: CONTEXT_PACK_GENERATION_ANALYSIS_ARTIFACT_FILE,
|
|
201
|
+
packArtifactPaths: []
|
|
202
|
+
});
|
|
203
|
+
run.artifacts.push({
|
|
204
|
+
id: "local-repository-subject-manifest",
|
|
205
|
+
label: "Privacy-safe local repository subject manifest",
|
|
206
|
+
path: CONTEXT_PACK_GENERATION_LOCAL_SUBJECT_MANIFEST_FILE,
|
|
207
|
+
kind: "artifact",
|
|
208
|
+
mimeType: "application/json",
|
|
209
|
+
description: "Privacy-safe logical subject, Git, safety and aggregate inventory metadata without source or private identity lists."
|
|
210
|
+
});
|
|
211
|
+
for (const payload of [executionText, analysisText, run])
|
|
212
|
+
assertContextPackExternalProjectionIsPrivate(payload, privateValues);
|
|
213
|
+
// Durable phase: execution, analysis, then the manifest last. Only files this attempt created are removed on failure.
|
|
214
|
+
const created = [];
|
|
215
|
+
const write = async (relativePath, content) => {
|
|
216
|
+
const filePath = resolveWithinRoot(outDir, relativePath);
|
|
217
|
+
const existed = await io.exists(filePath);
|
|
218
|
+
await io.writeFile(filePath, content);
|
|
219
|
+
if (!existed)
|
|
220
|
+
created.push(filePath);
|
|
221
|
+
};
|
|
222
|
+
try {
|
|
223
|
+
await io.ensureDirectory(outDir);
|
|
224
|
+
await write(CONTEXT_PACK_GENERATION_EXECUTION_ARTIFACT_FILE, executionText);
|
|
225
|
+
await write(CONTEXT_PACK_GENERATION_ANALYSIS_ARTIFACT_FILE, analysisText);
|
|
226
|
+
await write(CONTEXT_PACK_GENERATION_LOCAL_SUBJECT_MANIFEST_FILE, manifestText);
|
|
227
|
+
}
|
|
228
|
+
catch {
|
|
229
|
+
for (const filePath of created.reverse()) {
|
|
230
|
+
try {
|
|
231
|
+
await io.removeFile(filePath);
|
|
232
|
+
}
|
|
233
|
+
catch {
|
|
234
|
+
// best effort: the failure below is the outcome
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
throw new Error(CONTEXT_PACK_EXTERNAL_PERSISTENCE_FAILURE_MESSAGE);
|
|
238
|
+
}
|
|
239
|
+
return run;
|
|
240
|
+
}
|
|
241
|
+
/** Validates that every produced pack has exactly the path its evidence promises, then serializes in corpus order. */
|
|
242
|
+
function collectPackFiles(results, outDir) {
|
|
243
|
+
const files = [];
|
|
244
|
+
const seen = new Set();
|
|
245
|
+
for (const result of results) {
|
|
246
|
+
const packTreatment = result.evidence.treatments.find((treatment) => treatment.treatmentId === "context-pack");
|
|
247
|
+
if (!result.pack) {
|
|
248
|
+
if (packTreatment?.packArtifactPath)
|
|
249
|
+
throw new Error(CONTEXT_PACK_GENERATION_ARTIFACT_STATE_MESSAGE);
|
|
250
|
+
continue;
|
|
251
|
+
}
|
|
252
|
+
const relativePath = contextPackArtifactRelativePath(result.evidence.caseId);
|
|
253
|
+
if (packTreatment?.packArtifactPath !== relativePath || seen.has(relativePath) || result.pack.caseId !== result.evidence.caseId) {
|
|
254
|
+
throw new Error(CONTEXT_PACK_GENERATION_ARTIFACT_STATE_MESSAGE);
|
|
255
|
+
}
|
|
256
|
+
seen.add(relativePath);
|
|
257
|
+
files.push({ caseId: result.evidence.caseId, relativePath, absolutePath: resolveWithinRoot(outDir, relativePath), content: serializeContextPackArtifact(result.pack) });
|
|
258
|
+
}
|
|
259
|
+
return files;
|
|
260
|
+
}
|
|
261
|
+
export function mapContextPackGenerationToRun(args) {
|
|
262
|
+
const cases = args.caseEvidence.map((entry, caseIndex) => ({
|
|
263
|
+
id: entry.caseId,
|
|
264
|
+
name: entry.caseName,
|
|
265
|
+
outcomes: entry.treatments.map((treatment, treatmentIndex) => buildOutcome(entry, treatment, args.analysis.cases[caseIndex].treatments[treatmentIndex])),
|
|
266
|
+
metadata: { benchmarkProject: entry.benchmarkProject, taskLocality: entry.taskLocality }
|
|
267
|
+
}));
|
|
268
|
+
const run = {
|
|
269
|
+
runId: args.runId,
|
|
270
|
+
pluginId: CONTEXT_PACK_GENERATION_PLUGIN_ID,
|
|
271
|
+
startedAt: args.startedAt,
|
|
272
|
+
completedAt: args.completedAt,
|
|
273
|
+
status: aggregateTreatmentStatus(args.caseEvidence.flatMap((entry) => entry.treatments.map((treatment) => treatment.status))),
|
|
274
|
+
target: args.target,
|
|
275
|
+
variants: CONTEXT_PACK_GENERATION_VARIANTS.map((variant) => ({ ...variant })),
|
|
276
|
+
cases,
|
|
277
|
+
metrics: [...toContextPackGenerationRunMetrics(args.analysis), ...args.analysis.cases.flatMap(toContextPackGenerationCaseComparisonMetrics)],
|
|
278
|
+
artifacts: [
|
|
279
|
+
{
|
|
280
|
+
id: "context-pack-generation-execution",
|
|
281
|
+
label: "Context pack generation execution evidence",
|
|
282
|
+
path: args.executionArtifactPath,
|
|
283
|
+
kind: "artifact",
|
|
284
|
+
mimeType: "application/json",
|
|
285
|
+
description: "Matched per-case execution evidence for raw-full-file and context-pack without source text or raw command output."
|
|
286
|
+
},
|
|
287
|
+
{
|
|
288
|
+
id: "context-pack-generation-analysis",
|
|
289
|
+
label: "Context pack generation scientific analysis",
|
|
290
|
+
path: args.analysisArtifactPath,
|
|
291
|
+
kind: "artifact",
|
|
292
|
+
mimeType: "application/json",
|
|
293
|
+
description: "Retrieval-quality metrics, size comparison and matched-case scope aggregates for raw-full-file versus context-pack."
|
|
294
|
+
},
|
|
295
|
+
...args.packArtifactPaths.map((pack) => ({
|
|
296
|
+
id: `context-pack:${pack.caseId}`,
|
|
297
|
+
label: `Context pack for ${pack.caseId}`,
|
|
298
|
+
path: pack.path,
|
|
299
|
+
kind: "artifact",
|
|
300
|
+
mimeType: "application/json",
|
|
301
|
+
description: "Complete bundled experimental context pack (my-dev-kit-lab-context-pack-experiment-v1).",
|
|
302
|
+
caseId: pack.caseId,
|
|
303
|
+
variantId: "context-pack"
|
|
304
|
+
}))
|
|
305
|
+
],
|
|
306
|
+
warnings: [],
|
|
307
|
+
// Outcome failures are the authoritative per-treatment failure location.
|
|
308
|
+
failures: [],
|
|
309
|
+
metadata: { executionArtifactPath: args.executionArtifactPath, analysisArtifactPath: args.analysisArtifactPath },
|
|
310
|
+
caseExecutionEvidence: structuredClone(args.caseEvidence),
|
|
311
|
+
analysis: structuredClone(args.analysis)
|
|
312
|
+
};
|
|
313
|
+
run.summary = summarizeExperimentRun(run);
|
|
314
|
+
return run;
|
|
315
|
+
}
|
|
316
|
+
function buildOutcome(caseEvidence, treatment, treatmentAnalysis) {
|
|
317
|
+
const { caseId, benchmarkProject, taskLocality } = caseEvidence;
|
|
318
|
+
const { treatmentId } = treatment;
|
|
319
|
+
return {
|
|
320
|
+
id: `${caseId}:${treatmentId}`,
|
|
321
|
+
caseId,
|
|
322
|
+
variantId: treatmentId,
|
|
323
|
+
status: treatment.status,
|
|
324
|
+
metrics: toContextPackGenerationOutcomeMetrics(treatmentAnalysis, caseId),
|
|
325
|
+
artifacts: [],
|
|
326
|
+
warnings: [],
|
|
327
|
+
failures: treatment.errors.map((error) => ({ code: error.code, message: error.message, variantId: treatmentId, caseId, recoverable: true })),
|
|
328
|
+
metadata: { benchmarkProject, taskLocality, evidenceAvailability: treatment.availability }
|
|
329
|
+
};
|
|
330
|
+
}
|
|
331
|
+
function aggregateTreatmentStatus(statuses) {
|
|
332
|
+
if (statuses.length === 0)
|
|
333
|
+
return "skipped";
|
|
334
|
+
if (statuses.every((status) => status === "completed"))
|
|
335
|
+
return "completed";
|
|
336
|
+
if (statuses.every((status) => status === "skipped"))
|
|
337
|
+
return "skipped";
|
|
338
|
+
if (statuses.every((status) => status === "failed"))
|
|
339
|
+
return "failed";
|
|
340
|
+
return "partial";
|
|
341
|
+
}
|
|
342
|
+
function readCasesInput(inputs) {
|
|
343
|
+
const value = inputs?.cases;
|
|
344
|
+
if (!Array.isArray(value))
|
|
345
|
+
throw new Error("Context pack generation requires cases input.");
|
|
346
|
+
return value;
|
|
347
|
+
}
|
|
348
|
+
function readDependenciesInput(inputs) {
|
|
349
|
+
const value = inputs?.contextPackDependencies;
|
|
350
|
+
return value && typeof value === "object" && !Array.isArray(value) ? value : undefined;
|
|
351
|
+
}
|
|
352
|
+
function readArtifactIo(inputs) {
|
|
353
|
+
const value = inputs?.contextPackArtifactIo;
|
|
354
|
+
return value && typeof value === "object" && !Array.isArray(value) ? { ...defaultArtifactIo, ...value } : defaultArtifactIo;
|
|
355
|
+
}
|