@dailephd/my-dev-kit-lab 0.6.1 → 0.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/README.md +9 -7
- package/benchmarks/contracts/incremental-change-staleness-scenarios.json +360 -0
- package/dist/src/cli/help.js +16 -9
- package/dist/src/commands/runExperimentDescribeCommand.js +5 -2
- package/dist/src/commands/runExperimentRunCommand.js +4 -7
- package/dist/src/evaluation/affectedNeighborhoodMetrics.js +51 -0
- package/dist/src/evaluation/benchmarkMetadata.js +5 -1
- package/dist/src/evaluation/parseAgentAnswer.js +172 -24
- package/dist/src/evaluation/runMyDevKitRetrieval.js +120 -7
- package/dist/src/experiments/defaultRegistry.js +2 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/comparison.js +101 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/comparisonV2.js +49 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/config.js +67 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/disposableTarget.js +125 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/execution.js +430 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/executionArtifact.js +160 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/executionArtifactV2.js +289 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/executionV2.js +195 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/index.js +13 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/indexClone.js +76 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/lifecycle.js +308 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/lifecyclePolicy.js +154 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/lifecyclePolicyV2.js +78 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/lifecycleV2.js +298 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/mutationExecution.js +144 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/plugin.js +475 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/scenarioCatalog.js +42 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/scenarioMutation.js +116 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/scenarioSelection.js +79 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/scenarioTypes.js +37 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/scenarioValidation.js +168 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/sourceState.js +145 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/treatmentSession.js +6 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/treatmentSessionV2.js +5 -0
- package/dist/src/experiments/plugins/index.js +1 -0
- package/dist/src/experiments/plugins/warmIndexReuse/metrics.js +17 -20
- package/dist/src/experiments/plugins/warmIndexReuse/warmIndexSession.js +3 -1
- package/dist/src/mobile/android/advancedSecurity/candidateEvidence.js +9 -2
- package/dist/src/mobile/android/advancedSecurity/redaction.js +19 -10
- package/dist/src/mobile/android/advancedSecurity/secretCandidates/analyzeSecretCandidates.js +12 -2
- package/dist/src/mobile/android/advancedSecurity/signingConfiguration/analyzeSigningConfiguration.js +1 -1
- package/dist/src/mobile/android/advancedSecurity/signingConfiguration/extractSigningConfigurations.js +7 -7
- package/dist/src/report/experiments/buildIncrementalChangeStalenessPluginReport.js +33 -0
- package/dist/src/report/experiments/buildIncrementalChangeStalenessReport.js +112 -0
- package/dist/src/report/experiments/buildIncrementalChangeStalenessReportV2.js +80 -0
- package/dist/src/report/experiments/buildPluginExperimentReport.js +3 -0
- package/dist/src/report/experiments/incrementalChangeStalenessReportModel.js +4 -0
- package/dist/src/report/experiments/incrementalChangeStalenessReportModelV2.js +3 -0
- package/dist/src/report/experiments/incrementalChangeStalenessReportRowsV2.js +146 -0
- package/dist/src/report/experiments/index.js +6 -0
- package/dist/src/report/experiments/renderIncrementalChangeStalenessHtml.js +243 -0
- package/dist/src/report/experiments/renderIncrementalChangeStalenessHtmlV2.js +111 -0
- package/dist/src/report/experiments/renderIncrementalChangeStalenessTextV2.js +96 -0
- package/dist/src/report/experiments/renderPluginExperimentReportHtml.js +3 -0
- package/dist/src/report/experiments/renderPluginExperimentReportText.js +144 -0
- package/dist/src/securityValidation/staticScans/codeql.js +8 -8
- package/docs/METRICS.md +67 -1
- package/examples/tutorial-browser/prepare.mjs +13 -13
- package/examples/tutorial-browser/server.mjs +13 -13
- package/package.json +1 -1
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
const FIELD_LABELS = {
|
|
2
|
+
changedFileCount: "the changed indexed file count",
|
|
3
|
+
changedSymbolCount: "the changed baseline symbol count",
|
|
4
|
+
affectedNodeCount: "the affected graph node count",
|
|
5
|
+
affectedEdgeCount: "the affected graph edge count",
|
|
6
|
+
taskOverlapCount: "the task-overlap node count",
|
|
7
|
+
taskOverlapPercent: "the task-overlap percent"
|
|
8
|
+
};
|
|
9
|
+
/**
|
|
10
|
+
* Converts a persisted (or in-memory) affected-neighborhood assessment into the six frozen v0.6.1
|
|
11
|
+
* metrics, once. A finite value (including zero) is available; a missing assessment or a null
|
|
12
|
+
* field is unavailable with a bounded reason, never zero. Pure; this is the ONLY owner of this
|
|
13
|
+
* conversion, reused unchanged by every caller.
|
|
14
|
+
*/
|
|
15
|
+
export function toAffectedNeighborhoodMetricValues(assessment, noAssessmentReason) {
|
|
16
|
+
const one = (field) => {
|
|
17
|
+
const unit = field === "taskOverlapPercent" ? "percent" : "count";
|
|
18
|
+
if (!assessment) {
|
|
19
|
+
return { availability: "unavailable", value: null, unit, reason: noAssessmentReason };
|
|
20
|
+
}
|
|
21
|
+
const value = assessment[field];
|
|
22
|
+
if (typeof value === "number") {
|
|
23
|
+
return { availability: "available", value, unit, reason: null };
|
|
24
|
+
}
|
|
25
|
+
const context = `assessment ${assessment.status}; seed mapping ${assessment.seedMappingStatus}; ` +
|
|
26
|
+
`graph ${assessment.graphEvidenceStatus}; neighborhood ${assessment.neighborhoodStatus}; task mapping ${assessment.taskMapping.status}` +
|
|
27
|
+
(field === "taskOverlapPercent" ? `; resolvable task nodes ${assessment.taskMapping.resolvableTaskNodeCount}` : "");
|
|
28
|
+
return {
|
|
29
|
+
availability: "unavailable",
|
|
30
|
+
value: null,
|
|
31
|
+
unit,
|
|
32
|
+
reason: `Affected-neighborhood evidence did not establish ${FIELD_LABELS[field]} (${context}).`
|
|
33
|
+
};
|
|
34
|
+
};
|
|
35
|
+
return {
|
|
36
|
+
changedFileCount: one("changedFileCount"),
|
|
37
|
+
changedSymbolCount: one("changedSymbolCount"),
|
|
38
|
+
affectedNodeCount: one("affectedNodeCount"),
|
|
39
|
+
affectedEdgeCount: one("affectedEdgeCount"),
|
|
40
|
+
taskOverlapCount: one("taskOverlapCount"),
|
|
41
|
+
taskOverlapPercent: one("taskOverlapPercent")
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
export const AFFECTED_NEIGHBORHOOD_METRIC_FIELDS = [
|
|
45
|
+
"changedFileCount",
|
|
46
|
+
"changedSymbolCount",
|
|
47
|
+
"affectedNodeCount",
|
|
48
|
+
"affectedEdgeCount",
|
|
49
|
+
"taskOverlapCount",
|
|
50
|
+
"taskOverlapPercent"
|
|
51
|
+
];
|
|
@@ -280,7 +280,11 @@ function validateWarmIndexSourceRoots(sourceRoots, profile, label) {
|
|
|
280
280
|
return errors;
|
|
281
281
|
}
|
|
282
282
|
function normalizeRelativeRoot(value) {
|
|
283
|
-
|
|
283
|
+
const normalized = path.posix.normalize(value.replaceAll("\\", "/"));
|
|
284
|
+
let end = normalized.length;
|
|
285
|
+
while (end > 0 && normalized[end - 1] === "/")
|
|
286
|
+
end -= 1;
|
|
287
|
+
return normalized.slice(0, end);
|
|
284
288
|
}
|
|
285
289
|
function sameOrderedValues(left, right) {
|
|
286
290
|
return left.length === right.length && left.every((value, index) => value === right[index]);
|
|
@@ -77,11 +77,24 @@ function collectJsonCandidates(text) {
|
|
|
77
77
|
if (trimmed.startsWith("{") && trimmed.endsWith("}")) {
|
|
78
78
|
candidates.push(trimmed);
|
|
79
79
|
}
|
|
80
|
-
|
|
81
|
-
|
|
80
|
+
let cursor = 0;
|
|
81
|
+
while (cursor < text.length) {
|
|
82
|
+
const opening = text.indexOf("```", cursor);
|
|
83
|
+
if (opening < 0)
|
|
84
|
+
break;
|
|
85
|
+
let bodyStart = opening + 3;
|
|
86
|
+
if (text.slice(bodyStart, bodyStart + 4).toLowerCase() === "json")
|
|
87
|
+
bodyStart += 4;
|
|
88
|
+
while (bodyStart < text.length && isWhitespace(text[bodyStart]))
|
|
89
|
+
bodyStart += 1;
|
|
90
|
+
const closing = text.indexOf("```", bodyStart);
|
|
91
|
+
if (closing < 0)
|
|
92
|
+
break;
|
|
93
|
+
const body = text.slice(bodyStart, closing).trim();
|
|
82
94
|
if (body?.startsWith("{")) {
|
|
83
95
|
candidates.push(body);
|
|
84
96
|
}
|
|
97
|
+
cursor = closing + 3;
|
|
85
98
|
}
|
|
86
99
|
return candidates;
|
|
87
100
|
}
|
|
@@ -89,39 +102,103 @@ function collectFieldValues(text) {
|
|
|
89
102
|
const fields = new Map();
|
|
90
103
|
let currentKey;
|
|
91
104
|
for (const line of text.split(/\r?\n/)) {
|
|
92
|
-
const
|
|
93
|
-
if (
|
|
94
|
-
currentKey = normalizeKey(
|
|
105
|
+
const field = parseStructuredFieldLine(line);
|
|
106
|
+
if (field) {
|
|
107
|
+
currentKey = normalizeKey(field.label);
|
|
95
108
|
const current = fields.get(currentKey) ?? [];
|
|
96
|
-
current.push(stripMarkupOnlyValue(
|
|
109
|
+
current.push(stripMarkupOnlyValue(field.value));
|
|
97
110
|
fields.set(currentKey, current);
|
|
98
111
|
continue;
|
|
99
112
|
}
|
|
100
|
-
if (currentKey
|
|
101
|
-
|
|
113
|
+
if (currentKey) {
|
|
114
|
+
const continuation = parseFieldContinuation(line);
|
|
115
|
+
if (continuation !== undefined)
|
|
116
|
+
fields.get(currentKey)?.push(continuation);
|
|
102
117
|
}
|
|
103
118
|
}
|
|
104
119
|
return fields;
|
|
105
120
|
}
|
|
121
|
+
function isWhitespace(character) {
|
|
122
|
+
return character.trim().length === 0;
|
|
123
|
+
}
|
|
124
|
+
function parseStructuredFieldLine(line) {
|
|
125
|
+
let start = 0;
|
|
126
|
+
while (start < line.length && isWhitespace(line[start]))
|
|
127
|
+
start += 1;
|
|
128
|
+
if (line[start] === "-" || line[start] === "*") {
|
|
129
|
+
start += 1;
|
|
130
|
+
while (start < line.length && isWhitespace(line[start]))
|
|
131
|
+
start += 1;
|
|
132
|
+
}
|
|
133
|
+
let leadingMarkup = 0;
|
|
134
|
+
while (start < line.length && leadingMarkup < 2 && "*_`".includes(line[start])) {
|
|
135
|
+
start += 1;
|
|
136
|
+
leadingMarkup += 1;
|
|
137
|
+
}
|
|
138
|
+
const colon = line.indexOf(":", start);
|
|
139
|
+
if (colon < 0)
|
|
140
|
+
return undefined;
|
|
141
|
+
let labelEnd = colon;
|
|
142
|
+
let trailingMarkup = 0;
|
|
143
|
+
while (labelEnd > start && trailingMarkup < 2 && "*_`".includes(line[labelEnd - 1])) {
|
|
144
|
+
labelEnd -= 1;
|
|
145
|
+
trailingMarkup += 1;
|
|
146
|
+
}
|
|
147
|
+
const label = line.slice(start, labelEnd);
|
|
148
|
+
if (label.length < 2 || label.length > 41 || !isAsciiLetter(label[0]))
|
|
149
|
+
return undefined;
|
|
150
|
+
for (let index = 1; index < label.length; index += 1) {
|
|
151
|
+
const character = label[index];
|
|
152
|
+
if (!isAsciiLetter(character) && !(character >= "0" && character <= "9") && character !== " " && character !== "_" && character !== "-")
|
|
153
|
+
return undefined;
|
|
154
|
+
}
|
|
155
|
+
let valueStart = colon + 1;
|
|
156
|
+
while (valueStart < line.length && isWhitespace(line[valueStart]))
|
|
157
|
+
valueStart += 1;
|
|
158
|
+
return { label, value: line.slice(valueStart) };
|
|
159
|
+
}
|
|
160
|
+
function isAsciiLetter(character) {
|
|
161
|
+
return (character >= "a" && character <= "z") || (character >= "A" && character <= "Z");
|
|
162
|
+
}
|
|
163
|
+
function parseFieldContinuation(line) {
|
|
164
|
+
let index = 0;
|
|
165
|
+
while (index < line.length && isWhitespace(line[index]))
|
|
166
|
+
index += 1;
|
|
167
|
+
const indented = index > 0;
|
|
168
|
+
if (index === line.length)
|
|
169
|
+
return undefined;
|
|
170
|
+
if (line[index] === "-" || line[index] === "*") {
|
|
171
|
+
index += 1;
|
|
172
|
+
const whitespaceStart = index;
|
|
173
|
+
while (index < line.length && isWhitespace(line[index]))
|
|
174
|
+
index += 1;
|
|
175
|
+
if ((!indented && index === whitespaceStart) || index === line.length)
|
|
176
|
+
return undefined;
|
|
177
|
+
return line.slice(index).trim();
|
|
178
|
+
}
|
|
179
|
+
if (!indented)
|
|
180
|
+
return undefined;
|
|
181
|
+
return line.slice(index).trim();
|
|
182
|
+
}
|
|
106
183
|
function parseMarkdownSection(text, headings) {
|
|
107
184
|
const lines = text.split(/\r?\n/);
|
|
108
185
|
const values = [];
|
|
109
186
|
let inSection = false;
|
|
110
187
|
for (const line of lines) {
|
|
111
|
-
const heading = line
|
|
112
|
-
if (heading) {
|
|
113
|
-
inSection = headings.some((candidate) => normalizeKey(candidate) === normalizeKey(heading
|
|
188
|
+
const heading = parseMarkdownHeading(line);
|
|
189
|
+
if (heading !== undefined) {
|
|
190
|
+
inSection = headings.some((candidate) => normalizeKey(candidate) === normalizeKey(heading));
|
|
114
191
|
continue;
|
|
115
192
|
}
|
|
116
193
|
if (!inSection) {
|
|
117
194
|
continue;
|
|
118
195
|
}
|
|
119
|
-
if (
|
|
196
|
+
if (isMarkdownHeading(line)) {
|
|
120
197
|
break;
|
|
121
198
|
}
|
|
122
|
-
const bullet = line
|
|
123
|
-
if (bullet) {
|
|
124
|
-
values.push(bullet
|
|
199
|
+
const bullet = parseMarkdownBullet(line);
|
|
200
|
+
if (bullet !== undefined) {
|
|
201
|
+
values.push(bullet);
|
|
125
202
|
}
|
|
126
203
|
else if (line.includes(",")) {
|
|
127
204
|
values.push(...splitListValues([line]));
|
|
@@ -129,6 +206,41 @@ function parseMarkdownSection(text, headings) {
|
|
|
129
206
|
}
|
|
130
207
|
return unique(values);
|
|
131
208
|
}
|
|
209
|
+
function parseMarkdownHeading(line) {
|
|
210
|
+
let index = 0;
|
|
211
|
+
while (index < line.length && isWhitespace(line[index]))
|
|
212
|
+
index += 1;
|
|
213
|
+
const hashStart = index;
|
|
214
|
+
while (index < line.length && line[index] === "#" && index - hashStart < 6)
|
|
215
|
+
index += 1;
|
|
216
|
+
if (index === hashStart || (index < line.length && line[index] === "#"))
|
|
217
|
+
return undefined;
|
|
218
|
+
const heading = line.slice(index).trim();
|
|
219
|
+
return heading.length > 0 ? heading : undefined;
|
|
220
|
+
}
|
|
221
|
+
function isMarkdownHeading(line) {
|
|
222
|
+
let index = 0;
|
|
223
|
+
while (index < line.length && isWhitespace(line[index]))
|
|
224
|
+
index += 1;
|
|
225
|
+
const start = index;
|
|
226
|
+
while (index < line.length && line[index] === "#" && index - start < 6)
|
|
227
|
+
index += 1;
|
|
228
|
+
return index > start && index < line.length && isWhitespace(line[index]);
|
|
229
|
+
}
|
|
230
|
+
function parseMarkdownBullet(line) {
|
|
231
|
+
let index = 0;
|
|
232
|
+
while (index < line.length && isWhitespace(line[index]))
|
|
233
|
+
index += 1;
|
|
234
|
+
if (line[index] !== "-" && line[index] !== "*")
|
|
235
|
+
return undefined;
|
|
236
|
+
index += 1;
|
|
237
|
+
if (index >= line.length || !isWhitespace(line[index]))
|
|
238
|
+
return undefined;
|
|
239
|
+
while (index < line.length && isWhitespace(line[index]))
|
|
240
|
+
index += 1;
|
|
241
|
+
const content = line.slice(index).trim();
|
|
242
|
+
return content.length > 0 ? content : undefined;
|
|
243
|
+
}
|
|
132
244
|
function enrichFacts(parsed, text, answerKey) {
|
|
133
245
|
parsed.expectedFactsFound = normalizeFactMatches(parsed.expectedFactsFound, text, answerKey);
|
|
134
246
|
if (parsed.answerText.length === 0) {
|
|
@@ -168,15 +280,51 @@ function splitListValues(values) {
|
|
|
168
280
|
.filter(Boolean));
|
|
169
281
|
}
|
|
170
282
|
function cleanListItem(value) {
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
.
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
.
|
|
178
|
-
|
|
179
|
-
|
|
283
|
+
let source = value;
|
|
284
|
+
let searchFrom = 0;
|
|
285
|
+
while (searchFrom < value.length) {
|
|
286
|
+
const open = value.indexOf("`", searchFrom);
|
|
287
|
+
if (open < 0)
|
|
288
|
+
break;
|
|
289
|
+
const close = value.indexOf("`", open + 1);
|
|
290
|
+
if (close < 0)
|
|
291
|
+
break;
|
|
292
|
+
if (close > open + 1) {
|
|
293
|
+
source = value.slice(open + 1, close);
|
|
294
|
+
break;
|
|
295
|
+
}
|
|
296
|
+
searchFrom = close + 1;
|
|
297
|
+
}
|
|
298
|
+
let start = 0;
|
|
299
|
+
let end = source.length;
|
|
300
|
+
while (start < end && isWhitespace(source[start]))
|
|
301
|
+
start += 1;
|
|
302
|
+
while (end > start && isWhitespace(source[end - 1]))
|
|
303
|
+
end -= 1;
|
|
304
|
+
if (source[start] === "-" || source[start] === "*") {
|
|
305
|
+
start += 1;
|
|
306
|
+
while (start < end && isWhitespace(source[start]))
|
|
307
|
+
start += 1;
|
|
308
|
+
}
|
|
309
|
+
while (start < end && (source[start] === '"' || source[start] === "'" || source[start] === "`"))
|
|
310
|
+
start += 1;
|
|
311
|
+
while (end > start && (source[end - 1] === '"' || source[end - 1] === "'" || source[end - 1] === "`" || source[end - 1] === "."))
|
|
312
|
+
end -= 1;
|
|
313
|
+
let whitespaceStart = -1;
|
|
314
|
+
for (let index = start; index < end; index += 1) {
|
|
315
|
+
const character = source[index];
|
|
316
|
+
if (isWhitespace(character)) {
|
|
317
|
+
if (whitespaceStart < 0)
|
|
318
|
+
whitespaceStart = index;
|
|
319
|
+
continue;
|
|
320
|
+
}
|
|
321
|
+
if ((character === "-" || character === "–" || character === "—") && whitespaceStart >= start && index + 1 < end && isWhitespace(source[index + 1])) {
|
|
322
|
+
end = whitespaceStart;
|
|
323
|
+
break;
|
|
324
|
+
}
|
|
325
|
+
whitespaceStart = -1;
|
|
326
|
+
}
|
|
327
|
+
return source.slice(start, end).trim();
|
|
180
328
|
}
|
|
181
329
|
function stripMarkupOnlyValue(value) {
|
|
182
330
|
return /^[*_`\s]+$/.test(value) ? "" : value;
|
|
@@ -55,13 +55,118 @@ function skippedRetrieval(caseId, warnings, commands, durationMs) {
|
|
|
55
55
|
durationMs
|
|
56
56
|
};
|
|
57
57
|
}
|
|
58
|
+
const APPLIED_SCOPES = ["none", "changed-files", "affected-neighborhood", "full"];
|
|
59
|
+
const SELECTION_STATUSES = ["not-needed", "applied", "fallback-full"];
|
|
60
|
+
const NEIGHBORHOOD_COUNT_KEYS = ["seedFileCount", "seedSymbolCount", "affectedNodeCount", "affectedEdgeCount"];
|
|
61
|
+
function isNonNegativeInteger(value) {
|
|
62
|
+
return typeof value === "number" && Number.isInteger(value) && value >= 0;
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Validates the per-invocation `incrementalRefresh` object from `my-dev-kit index --incremental --json`
|
|
66
|
+
* stdout against the scope Lab requested. Returns validated evidence or an explicit error; never coerces.
|
|
67
|
+
* The fallback reason is intentionally an open string: upstream reason codes may evolve additively.
|
|
68
|
+
*/
|
|
69
|
+
export function parseIncrementalRefreshEvidence(stdout, requestedScope) {
|
|
70
|
+
const payload = parseJsonIfPossible(stdout);
|
|
71
|
+
if (payload === undefined)
|
|
72
|
+
return { ok: false, error: "index stdout is not valid JSON" };
|
|
73
|
+
if (!payload || typeof payload !== "object" || Array.isArray(payload)) {
|
|
74
|
+
return { ok: false, error: "index JSON result is not an object" };
|
|
75
|
+
}
|
|
76
|
+
const raw = payload.incrementalRefresh;
|
|
77
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
|
|
78
|
+
return { ok: false, error: "index JSON result has no incrementalRefresh object" };
|
|
79
|
+
}
|
|
80
|
+
const r = raw;
|
|
81
|
+
if (r.requestedScope !== requestedScope) {
|
|
82
|
+
return {
|
|
83
|
+
ok: false,
|
|
84
|
+
error: `incrementalRefresh.requestedScope ${JSON.stringify(r.requestedScope)} does not match requested scope ${requestedScope}`
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
if (!APPLIED_SCOPES.includes(r.appliedScope)) {
|
|
88
|
+
return { ok: false, error: `incrementalRefresh.appliedScope ${JSON.stringify(r.appliedScope)} is not a known value` };
|
|
89
|
+
}
|
|
90
|
+
if (!SELECTION_STATUSES.includes(r.selectionStatus)) {
|
|
91
|
+
return { ok: false, error: `incrementalRefresh.selectionStatus ${JSON.stringify(r.selectionStatus)} is not a known value` };
|
|
92
|
+
}
|
|
93
|
+
if (r.fallbackReason !== null && typeof r.fallbackReason !== "string") {
|
|
94
|
+
return { ok: false, error: "incrementalRefresh.fallbackReason must be a string or null" };
|
|
95
|
+
}
|
|
96
|
+
for (const key of NEIGHBORHOOD_COUNT_KEYS) {
|
|
97
|
+
if (r[key] !== null && !isNonNegativeInteger(r[key])) {
|
|
98
|
+
return { ok: false, error: `incrementalRefresh.${key} must be a non-negative integer or null` };
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
for (const key of ["forcedNeighborReanalysisFileCount", "freshExtractionFileCount", "reusedFileCount"]) {
|
|
102
|
+
if (!isNonNegativeInteger(r[key])) {
|
|
103
|
+
return { ok: false, error: `incrementalRefresh.${key} must be a non-negative integer` };
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
if (!Array.isArray(r.forcedNeighborSample) || r.forcedNeighborSample.some((item) => typeof item !== "string")) {
|
|
107
|
+
return { ok: false, error: "incrementalRefresh.forcedNeighborSample must be an array of strings" };
|
|
108
|
+
}
|
|
109
|
+
const appliedScope = r.appliedScope;
|
|
110
|
+
const selectionStatus = r.selectionStatus;
|
|
111
|
+
const fallbackReason = r.fallbackReason;
|
|
112
|
+
if (selectionStatus === "applied") {
|
|
113
|
+
if (appliedScope !== requestedScope) {
|
|
114
|
+
return { ok: false, error: "incrementalRefresh applied status requires appliedScope equal to requestedScope" };
|
|
115
|
+
}
|
|
116
|
+
if (fallbackReason !== null)
|
|
117
|
+
return { ok: false, error: "incrementalRefresh applied status requires a null fallbackReason" };
|
|
118
|
+
}
|
|
119
|
+
else if (selectionStatus === "fallback-full") {
|
|
120
|
+
if (appliedScope !== "full")
|
|
121
|
+
return { ok: false, error: "incrementalRefresh fallback-full status requires appliedScope full" };
|
|
122
|
+
if (fallbackReason === null || fallbackReason.length === 0) {
|
|
123
|
+
return { ok: false, error: "incrementalRefresh fallback-full status requires a nonempty fallbackReason" };
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
else {
|
|
127
|
+
if (appliedScope !== "none")
|
|
128
|
+
return { ok: false, error: "incrementalRefresh not-needed status requires appliedScope none" };
|
|
129
|
+
if (fallbackReason !== null)
|
|
130
|
+
return { ok: false, error: "incrementalRefresh not-needed status requires a null fallbackReason" };
|
|
131
|
+
}
|
|
132
|
+
if (requestedScope === "affected-neighborhood" && selectionStatus === "applied") {
|
|
133
|
+
for (const key of NEIGHBORHOOD_COUNT_KEYS) {
|
|
134
|
+
if (r[key] === null) {
|
|
135
|
+
return { ok: false, error: `incrementalRefresh.${key} must be numeric for an applied affected-neighborhood refresh` };
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
return {
|
|
140
|
+
ok: true,
|
|
141
|
+
evidence: {
|
|
142
|
+
requestedScope,
|
|
143
|
+
appliedScope,
|
|
144
|
+
selectionStatus,
|
|
145
|
+
fallbackReason,
|
|
146
|
+
seedFileCount: r.seedFileCount,
|
|
147
|
+
seedSymbolCount: r.seedSymbolCount,
|
|
148
|
+
affectedNodeCount: r.affectedNodeCount,
|
|
149
|
+
affectedEdgeCount: r.affectedEdgeCount,
|
|
150
|
+
forcedNeighborReanalysisFileCount: r.forcedNeighborReanalysisFileCount,
|
|
151
|
+
forcedNeighborSample: [...r.forcedNeighborSample],
|
|
152
|
+
freshExtractionFileCount: r.freshExtractionFileCount,
|
|
153
|
+
reusedFileCount: r.reusedFileCount
|
|
154
|
+
}
|
|
155
|
+
};
|
|
156
|
+
}
|
|
58
157
|
/**
|
|
59
158
|
* Runs exactly one my-dev-kit `index` command for the target and writes the index only to
|
|
60
159
|
* `indexDir`. Performs no search/lookup/slice/source work. With `requireKit` a failed index
|
|
61
160
|
* throws; otherwise the failure is returned as `ok: false` with a warning.
|
|
161
|
+
*
|
|
162
|
+
* `mode` defaults to a full index. An incremental mode adds `--incremental --refresh-scope <scope>`
|
|
163
|
+
* and requires valid per-invocation `incrementalRefresh` JSON evidence on stdout; a successful
|
|
164
|
+
* process without that evidence is not a usable incremental build. Upstream full-fallback and
|
|
165
|
+
* no-change outcomes are valid evidence and are returned as-is.
|
|
62
166
|
*/
|
|
63
167
|
export async function buildMyDevKitIndex(options) {
|
|
64
168
|
const warnings = [];
|
|
169
|
+
const mode = options.mode ?? { kind: "full" };
|
|
65
170
|
const command = await runMeasuredCommand({
|
|
66
171
|
commandId: "index",
|
|
67
172
|
commandString: options.kitCommand,
|
|
@@ -74,22 +179,30 @@ export async function buildMyDevKitIndex(options) {
|
|
|
74
179
|
...options.target.sourceRoots.flatMap((sourceRoot) => ["--src", sourceRoot]),
|
|
75
180
|
"--out",
|
|
76
181
|
options.indexDir,
|
|
182
|
+
...(mode.kind === "incremental" ? ["--incremental", "--refresh-scope", mode.refreshScope] : []),
|
|
77
183
|
"--json"
|
|
78
184
|
]
|
|
79
185
|
});
|
|
186
|
+
const result = { indexDir: options.indexDir, durationMs: command.durationMs, command, mode };
|
|
80
187
|
if (!command.ok) {
|
|
81
188
|
if (options.requireKit) {
|
|
82
189
|
throw new Error(command.error || `my-dev-kit index failed with exit code ${command.exitCode}`);
|
|
83
190
|
}
|
|
84
191
|
warnings.push("my-dev-kit index command was unavailable or failed.");
|
|
192
|
+
return { ...result, ok: false, warnings, incrementalRefresh: null };
|
|
85
193
|
}
|
|
86
|
-
|
|
87
|
-
ok:
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
194
|
+
if (mode.kind === "full") {
|
|
195
|
+
return { ...result, ok: true, warnings, incrementalRefresh: null };
|
|
196
|
+
}
|
|
197
|
+
const parsed = parseIncrementalRefreshEvidence(command.stdout, mode.refreshScope);
|
|
198
|
+
if (!parsed.ok) {
|
|
199
|
+
const message = `my-dev-kit incremental index (${mode.refreshScope}) returned no valid incrementalRefresh evidence: ${parsed.error}`;
|
|
200
|
+
if (options.requireKit)
|
|
201
|
+
throw new Error(message);
|
|
202
|
+
warnings.push(message);
|
|
203
|
+
return { ...result, ok: false, warnings, incrementalRefresh: null };
|
|
204
|
+
}
|
|
205
|
+
return { ...result, ok: true, warnings, incrementalRefresh: parsed.evidence };
|
|
93
206
|
}
|
|
94
207
|
/**
|
|
95
208
|
* Runs `<kit-command> --version` once through the shared measured-command runner and returns
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import { ExperimentPluginRegistry } from "./registry.js";
|
|
2
2
|
import { contextStrategyComparisonPlugin } from "./plugins/contextStrategyComparison/index.js";
|
|
3
|
+
import { incrementalChangeStalenessPlugin } from "./plugins/incrementalChangeStaleness/index.js";
|
|
3
4
|
import { warmIndexReusePlugin } from "./plugins/warmIndexReuse/index.js";
|
|
4
5
|
export function createDefaultExperimentPluginRegistry() {
|
|
5
6
|
const registry = new ExperimentPluginRegistry();
|
|
6
7
|
registry.register(contextStrategyComparisonPlugin);
|
|
7
8
|
registry.register(warmIndexReusePlugin);
|
|
9
|
+
registry.register(incrementalChangeStalenessPlugin);
|
|
8
10
|
return registry;
|
|
9
11
|
}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
// v0.6.2 Batch 4 -- pure, deterministic comparison/classification owner.
|
|
2
|
+
//
|
|
3
|
+
// This module owns exactly:
|
|
4
|
+
// - CorrectnessRelationV1;
|
|
5
|
+
// - RequiredFileEvidenceStatusV1 / RequiredFileEvidenceV1;
|
|
6
|
+
// - RequiredFileEvidenceRelationV1;
|
|
7
|
+
// - StaleRiskClassificationV1;
|
|
8
|
+
// - the pure functions that compute them.
|
|
9
|
+
//
|
|
10
|
+
// Every function here is a pure, side-effect-free transform over already
|
|
11
|
+
// collected evidence. No retrieval, no filesystem access, no my-dev-kit
|
|
12
|
+
// invocation, and no numeric risk score: see the frozen batch prompt
|
|
13
|
+
// sections 22-35 for the exact rules this file implements.
|
|
14
|
+
/** Section 23: exact correctness comparison rules. */
|
|
15
|
+
export function compareCorrectness(stale, fullRefresh) {
|
|
16
|
+
if (!stale.available || !fullRefresh.available) {
|
|
17
|
+
return "unknown";
|
|
18
|
+
}
|
|
19
|
+
if (stale.score < fullRefresh.score)
|
|
20
|
+
return "stale-worse";
|
|
21
|
+
if (stale.score > fullRefresh.score)
|
|
22
|
+
return "stale-better";
|
|
23
|
+
return "same";
|
|
24
|
+
}
|
|
25
|
+
/** Repository-standard forward-slash path normalization, matching existing lifecycle policy. */
|
|
26
|
+
export function normalizeRequiredFilePath(value) {
|
|
27
|
+
return value.replace(/\\/g, "/").replace(/^(\.\/)+/, "");
|
|
28
|
+
}
|
|
29
|
+
function sortedUnique(values) {
|
|
30
|
+
return [...new Set(values.map(normalizeRequiredFilePath))].sort((left, right) => (left < right ? -1 : left > right ? 1 : 0));
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Section 27: builds one treatment's required-file evidence.
|
|
34
|
+
*
|
|
35
|
+
* - `observedFilesComplete: false` means the treatment's actual-files-read set could not be
|
|
36
|
+
* established completely (retrieval failed/partial or the contract could not prove completeness);
|
|
37
|
+
* the result is always `unknown` in that case, regardless of `observedFiles` contents.
|
|
38
|
+
* - Otherwise: every normalized required file present in `observedFiles` is `present`; any
|
|
39
|
+
* absence is `missing`.
|
|
40
|
+
*/
|
|
41
|
+
export function buildRequiredFileEvidence(args) {
|
|
42
|
+
const requiredFiles = sortedUnique(args.requiredFiles);
|
|
43
|
+
const observedFiles = sortedUnique(args.observedFiles);
|
|
44
|
+
if (!args.observedFilesComplete) {
|
|
45
|
+
return {
|
|
46
|
+
status: "unknown",
|
|
47
|
+
requiredFiles,
|
|
48
|
+
observedFiles,
|
|
49
|
+
missingFiles: [],
|
|
50
|
+
reason: args.unknownReason ?? "Retrieval evidence did not establish a complete actual-files-read set."
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
const observedSet = new Set(observedFiles);
|
|
54
|
+
const missingFiles = requiredFiles.filter((file) => !observedSet.has(file));
|
|
55
|
+
return {
|
|
56
|
+
status: missingFiles.length === 0 ? "present" : "missing",
|
|
57
|
+
requiredFiles,
|
|
58
|
+
observedFiles,
|
|
59
|
+
missingFiles,
|
|
60
|
+
reason: null
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
/** Section 30: exact required-file comparison rules. */
|
|
64
|
+
export function compareRequiredFileEvidence(stale, fullRefresh) {
|
|
65
|
+
if (stale.status === "unknown" || fullRefresh.status === "unknown") {
|
|
66
|
+
return "unknown";
|
|
67
|
+
}
|
|
68
|
+
if (stale.status === "missing" && fullRefresh.status === "present")
|
|
69
|
+
return "stale-worse";
|
|
70
|
+
if (stale.status === "present" && fullRefresh.status === "missing")
|
|
71
|
+
return "stale-better";
|
|
72
|
+
return "same";
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Sections 31-35: the frozen precedence algorithm. Correctness and required-file evidence are the
|
|
76
|
+
* only two inputs; affected-neighborhood evidence and reindexRecommendation are never inputs here.
|
|
77
|
+
*/
|
|
78
|
+
export function classifyStaleRisk(correctnessRelation, requiredFileEvidenceRelation) {
|
|
79
|
+
const reasonCodes = [];
|
|
80
|
+
// RULE 1 -- observed regression wins even if the other dimension is unknown.
|
|
81
|
+
if (correctnessRelation === "stale-worse" || requiredFileEvidenceRelation === "stale-worse") {
|
|
82
|
+
if (correctnessRelation === "stale-worse")
|
|
83
|
+
reasonCodes.push("stale-correctness-lower");
|
|
84
|
+
if (requiredFileEvidenceRelation === "stale-worse")
|
|
85
|
+
reasonCodes.push("stale-missing-required-file");
|
|
86
|
+
return { staleRiskClassification: "observed-stale-regression", reasonCodes: dedupe(reasonCodes) };
|
|
87
|
+
}
|
|
88
|
+
// RULE 2 -- inconclusive.
|
|
89
|
+
if (correctnessRelation === "unknown" || requiredFileEvidenceRelation === "unknown") {
|
|
90
|
+
if (correctnessRelation === "unknown")
|
|
91
|
+
reasonCodes.push("correctness-unavailable");
|
|
92
|
+
if (requiredFileEvidenceRelation === "unknown")
|
|
93
|
+
reasonCodes.push("required-file-evidence-unavailable");
|
|
94
|
+
return { staleRiskClassification: "inconclusive", reasonCodes: dedupe(reasonCodes) };
|
|
95
|
+
}
|
|
96
|
+
// RULE 3 -- no observed regression.
|
|
97
|
+
return { staleRiskClassification: "no-observed-stale-regression", reasonCodes: ["no-stale-specific-difference-observed"] };
|
|
98
|
+
}
|
|
99
|
+
function dedupe(values) {
|
|
100
|
+
return [...new Set(values)];
|
|
101
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { compareCorrectness, compareRequiredFileEvidence } from "./comparison.js";
|
|
2
|
+
function neutral(relation) {
|
|
3
|
+
if (relation === "stale-worse")
|
|
4
|
+
return "candidate-worse";
|
|
5
|
+
if (relation === "stale-better")
|
|
6
|
+
return "candidate-better";
|
|
7
|
+
return relation;
|
|
8
|
+
}
|
|
9
|
+
/** Same rules as the v0.6.2 correctness comparison, candidate vs reference. */
|
|
10
|
+
export function compareCandidateCorrectness(candidate, reference) {
|
|
11
|
+
return neutral(compareCorrectness(candidate, reference));
|
|
12
|
+
}
|
|
13
|
+
/** Same rules as the v0.6.2 required-file comparison, candidate vs reference. */
|
|
14
|
+
export function compareCandidateRequiredFileEvidence(candidate, reference) {
|
|
15
|
+
return neutral(compareRequiredFileEvidence(candidate, reference));
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Frozen precedence. Rule 0 (realization gate) first: a full fallback is never evidence about
|
|
19
|
+
* partial-refresh performance. Then observed regression (either dimension worse), then inconclusive
|
|
20
|
+
* (a dimension unknown), then no observed regression (including candidate-better, which is not a
|
|
21
|
+
* winner claim). affected-neighborhood evidence and reindexRecommendation are never inputs.
|
|
22
|
+
*/
|
|
23
|
+
export function classifyPartialRefreshAgainstFull(args) {
|
|
24
|
+
const { realization, correctnessRelation, requiredFileEvidenceRelation } = args;
|
|
25
|
+
// RULE 0 -- realization gate.
|
|
26
|
+
if (realization === "FALLBACK_FULL") {
|
|
27
|
+
return { classification: "not-comparable-as-partial-refresh", reasonCodes: ["candidate-fell-back-to-full"] };
|
|
28
|
+
}
|
|
29
|
+
// RULE 1 -- observed regression wins even if the other dimension is unknown.
|
|
30
|
+
if (correctnessRelation === "candidate-worse" || requiredFileEvidenceRelation === "candidate-worse") {
|
|
31
|
+
const reasonCodes = [];
|
|
32
|
+
if (correctnessRelation === "candidate-worse")
|
|
33
|
+
reasonCodes.push("candidate-correctness-lower");
|
|
34
|
+
if (requiredFileEvidenceRelation === "candidate-worse")
|
|
35
|
+
reasonCodes.push("candidate-missing-required-file");
|
|
36
|
+
return { classification: "observed-regression-relative-to-full", reasonCodes };
|
|
37
|
+
}
|
|
38
|
+
// RULE 2 -- inconclusive.
|
|
39
|
+
if (correctnessRelation === "unknown" || requiredFileEvidenceRelation === "unknown") {
|
|
40
|
+
const reasonCodes = [];
|
|
41
|
+
if (correctnessRelation === "unknown")
|
|
42
|
+
reasonCodes.push("correctness-unavailable");
|
|
43
|
+
if (requiredFileEvidenceRelation === "unknown")
|
|
44
|
+
reasonCodes.push("required-file-evidence-unavailable");
|
|
45
|
+
return { classification: "inconclusive", reasonCodes };
|
|
46
|
+
}
|
|
47
|
+
// RULE 3 -- no observed regression.
|
|
48
|
+
return { classification: "no-observed-regression-relative-to-full", reasonCodes: ["no-reference-regression-observed"] };
|
|
49
|
+
}
|