@dailephd/my-dev-kit-lab 0.6.2 → 0.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/README.md +9 -8
- package/dist/src/cli/help.js +16 -9
- package/dist/src/commands/runExperimentDescribeCommand.js +5 -2
- package/dist/src/commands/runExperimentRunCommand.js +0 -4
- package/dist/src/evaluation/benchmarkMetadata.js +5 -1
- package/dist/src/evaluation/parseAgentAnswer.js +172 -24
- package/dist/src/evaluation/runMyDevKitRetrieval.js +120 -7
- package/dist/src/experiments/plugins/incrementalChangeStaleness/comparisonV2.js +49 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/config.js +5 -4
- package/dist/src/experiments/plugins/incrementalChangeStaleness/disposableTarget.js +20 -2
- package/dist/src/experiments/plugins/incrementalChangeStaleness/execution.js +42 -16
- package/dist/src/experiments/plugins/incrementalChangeStaleness/executionArtifact.js +4 -4
- package/dist/src/experiments/plugins/incrementalChangeStaleness/executionArtifactV2.js +289 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/executionV2.js +195 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/indexClone.js +76 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/lifecycle.js +64 -32
- package/dist/src/experiments/plugins/incrementalChangeStaleness/lifecyclePolicy.js +6 -4
- package/dist/src/experiments/plugins/incrementalChangeStaleness/lifecyclePolicyV2.js +78 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/lifecycleV2.js +298 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/plugin.js +213 -12
- package/dist/src/experiments/plugins/incrementalChangeStaleness/treatmentSessionV2.js +5 -0
- package/dist/src/experiments/plugins/warmIndexReuse/warmIndexSession.js +3 -1
- package/dist/src/mobile/android/advancedSecurity/candidateEvidence.js +9 -2
- package/dist/src/mobile/android/advancedSecurity/redaction.js +19 -10
- package/dist/src/mobile/android/advancedSecurity/secretCandidates/analyzeSecretCandidates.js +12 -2
- package/dist/src/mobile/android/advancedSecurity/signingConfiguration/analyzeSigningConfiguration.js +1 -1
- package/dist/src/mobile/android/advancedSecurity/signingConfiguration/extractSigningConfigurations.js +7 -7
- package/dist/src/report/experiments/buildIncrementalChangeStalenessPluginReport.js +33 -0
- package/dist/src/report/experiments/buildIncrementalChangeStalenessReport.js +1 -1
- package/dist/src/report/experiments/buildIncrementalChangeStalenessReportV2.js +80 -0
- package/dist/src/report/experiments/buildPluginExperimentReport.js +2 -2
- package/dist/src/report/experiments/incrementalChangeStalenessReportModelV2.js +3 -0
- package/dist/src/report/experiments/incrementalChangeStalenessReportRowsV2.js +146 -0
- package/dist/src/report/experiments/index.js +3 -0
- package/dist/src/report/experiments/renderIncrementalChangeStalenessHtml.js +4 -0
- package/dist/src/report/experiments/renderIncrementalChangeStalenessHtmlV2.js +111 -0
- package/dist/src/report/experiments/renderIncrementalChangeStalenessTextV2.js +96 -0
- package/dist/src/report/experiments/renderPluginExperimentReportText.js +6 -0
- package/dist/src/securityValidation/staticScans/codeql.js +8 -8
- package/docs/METRICS.md +32 -0
- package/examples/tutorial-browser/prepare.mjs +13 -13
- package/examples/tutorial-browser/server.mjs +13 -13
- package/package.json +1 -1
|
@@ -0,0 +1,298 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
import { createDisposableTreatmentTarget, INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_IDS, INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_INTENTS } from "./disposableTarget.js";
|
|
3
|
+
import { cloneIncrementalChangeStalenessIndexDirectory } from "./indexClone.js";
|
|
4
|
+
import { defaultIncrementalChangeStalenessLifecycleDeps, fail, incrementalChangeStalenessIndexDir, incrementalChangeStalenessTargetsRoot, LifecycleFailure, prepareTreatmentIndexEvidence } from "./lifecycle.js";
|
|
5
|
+
import { evaluateBaselineStaleFreshness, evaluateChangedPathSymmetry, evaluateRefreshedFreshness, evaluateToolIdentity, normalizeRelativePath, normalizeRootForComparison } from "./lifecyclePolicy.js";
|
|
6
|
+
import { evaluateBaselineBootstrapEvidence, evaluatePartialRefreshRealization, evaluateV2InvocationCounts } from "./lifecyclePolicyV2.js";
|
|
7
|
+
import { compareIncrementalChangeStalenessSourceStates } from "./sourceState.js";
|
|
8
|
+
import { INCREMENTAL_CHANGE_STALENESS_LIFECYCLE_SESSION_V2_SCHEMA_VERSION } from "./treatmentSessionV2.js";
|
|
9
|
+
const PARTIAL_REFRESH_SCOPES = {
|
|
10
|
+
"changed-files-refresh": "changed-files",
|
|
11
|
+
"affected-neighborhood-refresh": "affected-neighborhood"
|
|
12
|
+
};
|
|
13
|
+
/** Every baseline is bootstrapped incrementally so upstream writes the cache metadata a later partial refresh needs. */
|
|
14
|
+
const BOOTSTRAP_MODE = { kind: "incremental", refreshScope: "changed-files" };
|
|
15
|
+
const V2_TREATMENT_IDS = INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_IDS;
|
|
16
|
+
function withRefreshEvidence(evidence, buildMode, incrementalRefresh) {
|
|
17
|
+
return Object.freeze({ ...evidence, buildMode, incrementalRefresh });
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* v0.6.3 internal four-treatment lifecycle for ONE frozen scenario (runtime only; not yet wired into the
|
|
21
|
+
* public plugin, execution artifact, comparison, or reports):
|
|
22
|
+
*
|
|
23
|
+
* 1. four independent disposable copies (stale-index, changed-files-refresh, affected-neighborhood-refresh,
|
|
24
|
+
* full-refresh), all derived from the same immutable canonical benchmark;
|
|
25
|
+
* 2. pre-mutation controlled source-state equivalence of every treatment against the stale-index reference;
|
|
26
|
+
* 3. all four baselines bootstrapped with `--incremental --refresh-scope changed-files` into brand-new
|
|
27
|
+
* index directories (expected bootstrap evidence: full fallback, `cache-missing`);
|
|
28
|
+
* 4. each partial treatment's untouched baseline is cloned byte-for-byte into its own refreshed directory;
|
|
29
|
+
* 5. BARRIER: only after every baseline + clone is ready is the same frozen mutation applied to all four;
|
|
30
|
+
* 6. post-mutation source-state equivalence and symmetric complete-stale baseline freshness;
|
|
31
|
+
* 7. stale-index keeps its baseline; each partial treatment refreshes its clone in place and full-refresh
|
|
32
|
+
* builds a new full index; every refreshed index must be complete `fresh`.
|
|
33
|
+
*
|
|
34
|
+
* Seven index invocations and seven freshness assessments per scenario. Performs no retrieval,
|
|
35
|
+
* affected-neighborhood task traversal, or comparison classification. Sequential by design; deletes nothing.
|
|
36
|
+
*/
|
|
37
|
+
export async function prepareIncrementalChangeStalenessScenarioLifecycleV2(options) {
|
|
38
|
+
const cloneIndexDirectory = options.deps?.cloneIndexDirectory ?? cloneIncrementalChangeStalenessIndexDirectory;
|
|
39
|
+
const deps = { ...defaultIncrementalChangeStalenessLifecycleDeps, ...options.deps };
|
|
40
|
+
const { repoRoot, scenario, baseCase, kitCommand } = options;
|
|
41
|
+
const runOwnedRoot = path.resolve(options.runOwnedRoot);
|
|
42
|
+
const events = [];
|
|
43
|
+
const indexInvocationCounts = Object.fromEntries(V2_TREATMENT_IDS.map((id) => [id, 0]));
|
|
44
|
+
const freshnessAssessmentCounts = Object.fromEntries(V2_TREATMENT_IDS.map((id) => [id, 0]));
|
|
45
|
+
const sourceRoots = baseCase.sourceRoots;
|
|
46
|
+
const controlledChangedPaths = scenario.mutation.files.map((file) => normalizeRelativePath(file.path)).sort();
|
|
47
|
+
async function prepareIndex(target, role, indexDir, mode, existingIndexDir = false) {
|
|
48
|
+
const treatmentId = target.treatmentId;
|
|
49
|
+
indexInvocationCounts[treatmentId] += 1;
|
|
50
|
+
const { evidence, build } = await prepareTreatmentIndexEvidence({
|
|
51
|
+
deps,
|
|
52
|
+
runOwnedRoot,
|
|
53
|
+
scenarioId: scenario.id,
|
|
54
|
+
kitCommand,
|
|
55
|
+
sourceRoots,
|
|
56
|
+
target,
|
|
57
|
+
role,
|
|
58
|
+
indexDir,
|
|
59
|
+
mode,
|
|
60
|
+
existingIndexDir,
|
|
61
|
+
onIndexPrepared: () => events.push(`${treatmentId}:${role}-index-built`, `${treatmentId}:${role}-snapshot-captured`, `${treatmentId}:${role}-graph-loaded`)
|
|
62
|
+
});
|
|
63
|
+
events.push(`${treatmentId}:${role}-identity-verified`);
|
|
64
|
+
return { evidence, build };
|
|
65
|
+
}
|
|
66
|
+
async function captureState(target) {
|
|
67
|
+
return deps.captureSourceState(target.targetRoot, sourceRoots, target.treatmentId, baseCase.benchmarkProjectId);
|
|
68
|
+
}
|
|
69
|
+
async function mutate(target) {
|
|
70
|
+
const receipt = await deps.executeMutation(scenario, target, sourceRoots, repoRoot);
|
|
71
|
+
if (receipt.status !== "applied") {
|
|
72
|
+
fail({ code: "mutation-failed", message: `${target.treatmentId} mutation was rejected: ${receipt.errors.join(" ")}`, treatmentId: target.treatmentId });
|
|
73
|
+
}
|
|
74
|
+
events.push(`${target.treatmentId}:mutation-applied`);
|
|
75
|
+
return receipt;
|
|
76
|
+
}
|
|
77
|
+
async function assess(index, target) {
|
|
78
|
+
freshnessAssessmentCounts[target.treatmentId] += 1;
|
|
79
|
+
return deps.assessIndexFreshness({ snapshot: index.snapshot, targetRoot: target.targetRoot, sourceRoots });
|
|
80
|
+
}
|
|
81
|
+
function compareAgainstReference(reference, states, code, phase) {
|
|
82
|
+
const comparisons = {};
|
|
83
|
+
for (const treatmentId of V2_TREATMENT_IDS) {
|
|
84
|
+
if (treatmentId === "stale-index")
|
|
85
|
+
continue;
|
|
86
|
+
const comparison = compareIncrementalChangeStalenessSourceStates(reference, states[treatmentId]);
|
|
87
|
+
comparisons[treatmentId] = comparison;
|
|
88
|
+
if (comparison.result !== "equivalent") {
|
|
89
|
+
fail({
|
|
90
|
+
code,
|
|
91
|
+
message: `${treatmentId} is not ${phase} equivalent to the stale-index reference (${comparison.result}): ${comparison.reasons.join("; ")}`,
|
|
92
|
+
treatmentId
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
return comparisons;
|
|
97
|
+
}
|
|
98
|
+
try {
|
|
99
|
+
const baselineDirs = {};
|
|
100
|
+
const refreshedDirs = {};
|
|
101
|
+
for (const treatmentId of V2_TREATMENT_IDS) {
|
|
102
|
+
baselineDirs[treatmentId] = incrementalChangeStalenessIndexDir(runOwnedRoot, scenario.id, treatmentId, "baseline");
|
|
103
|
+
refreshedDirs[treatmentId] = incrementalChangeStalenessIndexDir(runOwnedRoot, scenario.id, treatmentId, "refreshed");
|
|
104
|
+
}
|
|
105
|
+
const allDirs = [...Object.values(baselineDirs), ...Object.values(refreshedDirs)].map(normalizeRootForComparison);
|
|
106
|
+
if (new Set(allDirs).size !== allDirs.length) {
|
|
107
|
+
fail({ code: "index-directory-collision", message: "Treatment/phase index directories are not distinct.", treatmentId: null });
|
|
108
|
+
}
|
|
109
|
+
// 1. Four independent disposable copies through the established owner, in fixed order.
|
|
110
|
+
const targets = {};
|
|
111
|
+
for (const treatmentId of V2_TREATMENT_IDS) {
|
|
112
|
+
try {
|
|
113
|
+
targets[treatmentId] = await createDisposableTreatmentTarget({
|
|
114
|
+
repoRoot,
|
|
115
|
+
runRoot: incrementalChangeStalenessTargetsRoot(runOwnedRoot),
|
|
116
|
+
scenarioId: scenario.id,
|
|
117
|
+
treatmentId,
|
|
118
|
+
benchmarkProjectId: baseCase.benchmarkProjectId,
|
|
119
|
+
canonicalProjectRootRelative: baseCase.canonicalProjectRootRelative
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
catch (error) {
|
|
123
|
+
fail({ code: "treatment-target-creation-failed", message: `${treatmentId}: ${error.message}`, treatmentId });
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
const targetRootKeys = V2_TREATMENT_IDS.map((id) => normalizeRootForComparison(targets[id].targetRoot));
|
|
127
|
+
if (new Set(targetRootKeys).size !== targetRootKeys.length) {
|
|
128
|
+
fail({ code: "treatment-target-creation-failed", message: "Treatment target roots are not distinct.", treatmentId: null });
|
|
129
|
+
}
|
|
130
|
+
events.push("treatment-targets-created");
|
|
131
|
+
// 2. Pre-mutation equivalence gates all indexing and mutation; stale-index is only the comparison reference.
|
|
132
|
+
const preStates = {};
|
|
133
|
+
for (const treatmentId of V2_TREATMENT_IDS)
|
|
134
|
+
preStates[treatmentId] = await captureState(targets[treatmentId]);
|
|
135
|
+
events.push("pre-mutation-source-state-captured");
|
|
136
|
+
const preMutationEquivalence = compareAgainstReference(preStates["stale-index"], preStates, "pre-mutation-not-equivalent", "pre-mutation");
|
|
137
|
+
events.push("pre-mutation-equivalence-proven");
|
|
138
|
+
// 3. Baselines: every treatment bootstraps incrementally into a brand-new index directory.
|
|
139
|
+
const baselines = {};
|
|
140
|
+
const bootstraps = {};
|
|
141
|
+
for (const treatmentId of V2_TREATMENT_IDS) {
|
|
142
|
+
const { evidence, build } = await prepareIndex(targets[treatmentId], "baseline", baselineDirs[treatmentId], BOOTSTRAP_MODE);
|
|
143
|
+
fail(evaluateBaselineBootstrapEvidence(build, treatmentId));
|
|
144
|
+
bootstraps[treatmentId] = build.incrementalRefresh;
|
|
145
|
+
baselines[treatmentId] = withRefreshEvidence(evidence, build.mode, build.incrementalRefresh);
|
|
146
|
+
events.push(`${treatmentId}:baseline-bootstrap-verified`);
|
|
147
|
+
}
|
|
148
|
+
const baselineTools = evaluateToolIdentity(V2_TREATMENT_IDS.map((id) => baselines[id]));
|
|
149
|
+
fail(baselineTools.failure);
|
|
150
|
+
events.push("baseline-tool-identity-verified");
|
|
151
|
+
// 4. Preserve each partial treatment's immutable baseline; the refresh runs in place against a byte-for-byte clone.
|
|
152
|
+
for (const treatmentId of ["changed-files-refresh", "affected-neighborhood-refresh"]) {
|
|
153
|
+
try {
|
|
154
|
+
await cloneIndexDirectory(runOwnedRoot, baselineDirs[treatmentId], refreshedDirs[treatmentId]);
|
|
155
|
+
}
|
|
156
|
+
catch (error) {
|
|
157
|
+
fail({ code: "index-clone-failed", message: `${treatmentId} baseline index clone failed: ${error.message}`, treatmentId });
|
|
158
|
+
}
|
|
159
|
+
events.push(`${treatmentId}:baseline-index-cloned`);
|
|
160
|
+
}
|
|
161
|
+
// 5. Barrier: all four baselines are index + snapshot + graph READY and clones exist. Only now may mutation begin.
|
|
162
|
+
events.push("baseline-barrier-ready");
|
|
163
|
+
const receipts = {};
|
|
164
|
+
for (const treatmentId of V2_TREATMENT_IDS)
|
|
165
|
+
receipts[treatmentId] = await mutate(targets[treatmentId]);
|
|
166
|
+
// 6. Post-mutation equivalence.
|
|
167
|
+
const postStates = {};
|
|
168
|
+
for (const treatmentId of V2_TREATMENT_IDS)
|
|
169
|
+
postStates[treatmentId] = await captureState(targets[treatmentId]);
|
|
170
|
+
events.push("post-mutation-source-state-captured");
|
|
171
|
+
const postMutationEquivalence = compareAgainstReference(postStates["stale-index"], postStates, "post-mutation-not-equivalent", "post-mutation");
|
|
172
|
+
events.push("post-mutation-equivalence-proven");
|
|
173
|
+
// 7. Baseline (change-authority) freshness against each mutated target, symmetric with the reference.
|
|
174
|
+
const baselineFreshness = {};
|
|
175
|
+
for (const treatmentId of V2_TREATMENT_IDS) {
|
|
176
|
+
baselineFreshness[treatmentId] = await assess(baselines[treatmentId], targets[treatmentId]);
|
|
177
|
+
fail(evaluateBaselineStaleFreshness(baselineFreshness[treatmentId], controlledChangedPaths, treatmentId));
|
|
178
|
+
events.push(`${treatmentId}:baseline-freshness-stale`);
|
|
179
|
+
}
|
|
180
|
+
for (const treatmentId of V2_TREATMENT_IDS) {
|
|
181
|
+
if (treatmentId === "stale-index")
|
|
182
|
+
continue;
|
|
183
|
+
fail(evaluateChangedPathSymmetry(baselineFreshness["stale-index"], baselineFreshness[treatmentId], { reference: "stale-index", other: treatmentId }));
|
|
184
|
+
}
|
|
185
|
+
events.push("baseline-changed-paths-symmetric");
|
|
186
|
+
// 8. stale-index: no rebuild; baseline stays the active retrieval index.
|
|
187
|
+
events.push("stale-index:baseline-retained-as-active");
|
|
188
|
+
// 9. Partial treatments: refresh the cloned working index in place through the shared build owner.
|
|
189
|
+
const refreshed = {};
|
|
190
|
+
const refreshedFreshness = {};
|
|
191
|
+
const realizations = {};
|
|
192
|
+
for (const treatmentId of ["changed-files-refresh", "affected-neighborhood-refresh"]) {
|
|
193
|
+
const scope = PARTIAL_REFRESH_SCOPES[treatmentId];
|
|
194
|
+
const { evidence, build } = await prepareIndex(targets[treatmentId], "refreshed", refreshedDirs[treatmentId], { kind: "incremental", refreshScope: scope }, true);
|
|
195
|
+
const realization = evaluatePartialRefreshRealization(build, scope, treatmentId);
|
|
196
|
+
fail(realization.failure);
|
|
197
|
+
realizations[treatmentId] = realization.realization;
|
|
198
|
+
events.push(`${treatmentId}:refresh-realization-${realization.realization === "APPLIED_PARTIAL" ? "applied-partial" : "fallback-full"}`);
|
|
199
|
+
refreshed[treatmentId] = withRefreshEvidence(evidence, build.mode, build.incrementalRefresh);
|
|
200
|
+
refreshedFreshness[treatmentId] = await assess(evidence, targets[treatmentId]);
|
|
201
|
+
fail(evaluateRefreshedFreshness(refreshedFreshness[treatmentId], treatmentId));
|
|
202
|
+
events.push(`${treatmentId}:refreshed-freshness-fresh`);
|
|
203
|
+
}
|
|
204
|
+
// 10. full-refresh: a NEW complete post-mutation index in a separate directory (baseline is not cloned).
|
|
205
|
+
{
|
|
206
|
+
const { evidence, build } = await prepareIndex(targets["full-refresh"], "refreshed", refreshedDirs["full-refresh"], undefined);
|
|
207
|
+
refreshed["full-refresh"] = withRefreshEvidence(evidence, build.mode, build.incrementalRefresh);
|
|
208
|
+
refreshedFreshness["full-refresh"] = await assess(evidence, targets["full-refresh"]);
|
|
209
|
+
fail(evaluateRefreshedFreshness(refreshedFreshness["full-refresh"], "full-refresh"));
|
|
210
|
+
events.push("full-refresh:refreshed-freshness-fresh");
|
|
211
|
+
}
|
|
212
|
+
// 11. Scenario-level proofs.
|
|
213
|
+
const allTools = evaluateToolIdentity([...V2_TREATMENT_IDS.map((id) => baselines[id]), ...V2_TREATMENT_IDS.flatMap((id) => (refreshed[id] ? [refreshed[id]] : []))]);
|
|
214
|
+
fail(allTools.failure);
|
|
215
|
+
events.push("tool-identity-consistent");
|
|
216
|
+
fail(evaluateV2InvocationCounts(indexInvocationCounts, freshnessAssessmentCounts));
|
|
217
|
+
events.push("invocation-counts-verified");
|
|
218
|
+
events.push("lifecycle-ready");
|
|
219
|
+
const common = (treatmentId) => ({
|
|
220
|
+
treatmentIntent: INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_INTENTS[treatmentId],
|
|
221
|
+
target: targets[treatmentId],
|
|
222
|
+
preMutationSourceState: preStates[treatmentId],
|
|
223
|
+
postMutationSourceState: postStates[treatmentId],
|
|
224
|
+
mutationReceipt: receipts[treatmentId],
|
|
225
|
+
changeAuthority: Object.freeze({
|
|
226
|
+
baselineIndex: baselines[treatmentId],
|
|
227
|
+
baselineBootstrap: bootstraps[treatmentId],
|
|
228
|
+
postMutationBaselineFreshness: baselineFreshness[treatmentId]
|
|
229
|
+
}),
|
|
230
|
+
indexInvocationCount: indexInvocationCounts[treatmentId],
|
|
231
|
+
freshnessAssessmentCount: freshnessAssessmentCounts[treatmentId],
|
|
232
|
+
lifecycleStatus: "ready",
|
|
233
|
+
warnings: Object.freeze([])
|
|
234
|
+
});
|
|
235
|
+
const partialSession = (treatmentId) => Object.freeze({
|
|
236
|
+
...common(treatmentId),
|
|
237
|
+
treatmentId,
|
|
238
|
+
activeRetrieval: Object.freeze({ role: "refreshed", index: refreshed[treatmentId] }),
|
|
239
|
+
postMutationIndexBuilt: true,
|
|
240
|
+
refreshedIndex: refreshed[treatmentId],
|
|
241
|
+
refreshedFreshness: refreshedFreshness[treatmentId],
|
|
242
|
+
refreshRealization: realizations[treatmentId]
|
|
243
|
+
});
|
|
244
|
+
const staleSession = Object.freeze({
|
|
245
|
+
...common("stale-index"),
|
|
246
|
+
treatmentId: "stale-index",
|
|
247
|
+
activeRetrieval: Object.freeze({ role: "baseline", index: baselines["stale-index"] }),
|
|
248
|
+
postMutationIndexBuilt: false,
|
|
249
|
+
refreshedIndex: null,
|
|
250
|
+
refreshedFreshness: null,
|
|
251
|
+
refreshRealization: null
|
|
252
|
+
});
|
|
253
|
+
const fullSession = Object.freeze({
|
|
254
|
+
...common("full-refresh"),
|
|
255
|
+
treatmentId: "full-refresh",
|
|
256
|
+
activeRetrieval: Object.freeze({ role: "refreshed", index: refreshed["full-refresh"] }),
|
|
257
|
+
postMutationIndexBuilt: true,
|
|
258
|
+
refreshedIndex: refreshed["full-refresh"],
|
|
259
|
+
refreshedFreshness: refreshedFreshness["full-refresh"],
|
|
260
|
+
refreshRealization: null
|
|
261
|
+
});
|
|
262
|
+
const session = Object.freeze({
|
|
263
|
+
schemaVersion: INCREMENTAL_CHANGE_STALENESS_LIFECYCLE_SESSION_V2_SCHEMA_VERSION,
|
|
264
|
+
scenarioId: scenario.id,
|
|
265
|
+
scenario,
|
|
266
|
+
baseCase,
|
|
267
|
+
runOwnedRoot,
|
|
268
|
+
kitCommand,
|
|
269
|
+
toolIdentity: allTools.tool,
|
|
270
|
+
controlledChangedPaths: Object.freeze(controlledChangedPaths),
|
|
271
|
+
preMutationEquivalence: Object.freeze(preMutationEquivalence),
|
|
272
|
+
postMutationEquivalence: Object.freeze(postMutationEquivalence),
|
|
273
|
+
treatments: Object.freeze({
|
|
274
|
+
"stale-index": staleSession,
|
|
275
|
+
"changed-files-refresh": partialSession("changed-files-refresh"),
|
|
276
|
+
"affected-neighborhood-refresh": partialSession("affected-neighborhood-refresh"),
|
|
277
|
+
"full-refresh": fullSession
|
|
278
|
+
}),
|
|
279
|
+
totalIndexInvocationCount: V2_TREATMENT_IDS.reduce((sum, id) => sum + indexInvocationCounts[id], 0),
|
|
280
|
+
totalFreshnessAssessmentCount: V2_TREATMENT_IDS.reduce((sum, id) => sum + freshnessAssessmentCounts[id], 0),
|
|
281
|
+
lifecycleEvents: Object.freeze([...events]),
|
|
282
|
+
status: "ready"
|
|
283
|
+
});
|
|
284
|
+
return { status: "ready", session };
|
|
285
|
+
}
|
|
286
|
+
catch (error) {
|
|
287
|
+
const failure = error instanceof LifecycleFailure
|
|
288
|
+
? error.failure
|
|
289
|
+
: { code: "lifecycle-error", message: `Lifecycle failed unexpectedly: ${error instanceof Error ? error.message : String(error)}`, treatmentId: null };
|
|
290
|
+
return {
|
|
291
|
+
status: "failed",
|
|
292
|
+
scenarioId: scenario.id,
|
|
293
|
+
failure,
|
|
294
|
+
lifecycleEvents: Object.freeze([...events]),
|
|
295
|
+
indexInvocationCounts: Object.freeze({ ...indexInvocationCounts })
|
|
296
|
+
};
|
|
297
|
+
}
|
|
298
|
+
}
|
|
@@ -7,10 +7,10 @@ import { toAffectedNeighborhoodMetricValues } from "../../../evaluation/affected
|
|
|
7
7
|
import { sanitizePathSegment } from "../../outputPaths.js";
|
|
8
8
|
import { summarizeExperimentRun } from "../../results.js";
|
|
9
9
|
import { defaultIncrementalChangeStalenessConfig, incrementalChangeStalenessConfigDefinition, validateIncrementalChangeStalenessConfig } from "./config.js";
|
|
10
|
-
import { DEFAULT_INCREMENTAL_CHANGE_STALENESS_RUNTIME_ROOT_RELATIVE,
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
10
|
+
import { DEFAULT_INCREMENTAL_CHANGE_STALENESS_RUNTIME_ROOT_RELATIVE, INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_IDS, INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_INTENTS, removeIncrementalChangeStalenessRuntimeRoot } from "./disposableTarget.js";
|
|
11
|
+
import { buildIncrementalChangeStalenessExecutionArtifactV2, writeIncrementalChangeStalenessExecutionArtifactV2 } from "./executionArtifactV2.js";
|
|
12
|
+
import { executeIncrementalChangeStalenessScenarioV2 } from "./executionV2.js";
|
|
13
|
+
import { prepareIncrementalChangeStalenessScenarioLifecycleV2 } from "./lifecycleV2.js";
|
|
14
14
|
import { BENCHMARK_PROJECT_PROFILES_PATH } from "./scenarioCatalog.js";
|
|
15
15
|
import { resolveIncrementalChangeStalenessScenarios } from "./scenarioSelection.js";
|
|
16
16
|
export const INCREMENTAL_CHANGE_STALENESS_PLUGIN_ID = "incremental-change-staleness";
|
|
@@ -21,13 +21,39 @@ export const INCREMENTAL_CHANGE_STALENESS_RUN_OWNER_MARKER = ".incremental-chang
|
|
|
21
21
|
export const incrementalChangeStalenessMetadata = {
|
|
22
22
|
id: INCREMENTAL_CHANGE_STALENESS_PLUGIN_ID,
|
|
23
23
|
name: "Incremental Change Staleness",
|
|
24
|
-
description: "Compares
|
|
24
|
+
description: "Compares no refresh, my-dev-kit changed-files incremental refresh, my-dev-kit affected-neighborhood incremental refresh, and full refresh after the same deterministic controlled source change. Comparisons are scoped to the observed evidence (correctness and required-file presence); full refresh is a reference treatment, not an asserted winner, and a partial treatment may truthfully fall back to a full rebuild.",
|
|
25
25
|
schemaVersion: "1.0.0",
|
|
26
26
|
status: "experimental",
|
|
27
27
|
supportedTargets: ["self"],
|
|
28
28
|
supportedOutputs: ["json", "html"]
|
|
29
29
|
};
|
|
30
|
+
export const CHANGED_FILES_REFRESH_VARIANT_ID = "changed-files-refresh";
|
|
31
|
+
export const AFFECTED_NEIGHBORHOOD_REFRESH_VARIANT_ID = "affected-neighborhood-refresh";
|
|
32
|
+
/** Public v0.6.3 treatments, in the fixed lifecycle order. Factual descriptions only; no treatment is called best or preferred. */
|
|
30
33
|
const TREATMENT_VARIANTS = [
|
|
34
|
+
{
|
|
35
|
+
id: STALE_INDEX_VARIANT_ID,
|
|
36
|
+
name: "No refresh (stale index)",
|
|
37
|
+
description: "Keeps the pre-mutation baseline index and performs no post-mutation refresh."
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
id: CHANGED_FILES_REFRESH_VARIANT_ID,
|
|
41
|
+
name: "Changed-files refresh",
|
|
42
|
+
description: "Uses my-dev-kit's incremental changed-files refresh after the controlled mutation; upstream may truthfully fall back to a full rebuild."
|
|
43
|
+
},
|
|
44
|
+
{
|
|
45
|
+
id: AFFECTED_NEIGHBORHOOD_REFRESH_VARIANT_ID,
|
|
46
|
+
name: "Affected-neighborhood refresh",
|
|
47
|
+
description: "Uses my-dev-kit's affected-neighborhood incremental refresh after the controlled mutation; upstream may truthfully fall back to a full rebuild."
|
|
48
|
+
},
|
|
49
|
+
{
|
|
50
|
+
id: FULL_REFRESH_VARIANT_ID,
|
|
51
|
+
name: "Full refresh",
|
|
52
|
+
description: "Builds a complete post-mutation index and uses it as the comparison reference."
|
|
53
|
+
}
|
|
54
|
+
];
|
|
55
|
+
/** The released v0.6.2 two-treatment variants, kept only for the historical V1 run mapper. */
|
|
56
|
+
const V1_TREATMENT_VARIANTS = [
|
|
31
57
|
{
|
|
32
58
|
id: STALE_INDEX_VARIANT_ID,
|
|
33
59
|
name: "Stale index",
|
|
@@ -78,7 +104,7 @@ export const incrementalChangeStalenessPlugin = {
|
|
|
78
104
|
metadata: incrementalChangeStalenessMetadata,
|
|
79
105
|
defaultConfig: defaultIncrementalChangeStalenessConfig,
|
|
80
106
|
configDefinition: incrementalChangeStalenessConfigDefinition,
|
|
81
|
-
supportedVariants: [...
|
|
107
|
+
supportedVariants: [...INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_IDS],
|
|
82
108
|
validateConfig: validateIncrementalChangeStalenessConfig,
|
|
83
109
|
async run(context) {
|
|
84
110
|
if (!context.target.isSelf) {
|
|
@@ -94,14 +120,14 @@ export const incrementalChangeStalenessPlugin = {
|
|
|
94
120
|
const projectProfiles = await readBenchmarkProjectProfiles(path.resolve(context.toolRoot, BENCHMARK_PROJECT_PROFILES_PATH), context.toolRoot);
|
|
95
121
|
const executions = [];
|
|
96
122
|
for (const resolved of resolvedScenarios) {
|
|
97
|
-
const lifecycle = await
|
|
123
|
+
const lifecycle = await prepareIncrementalChangeStalenessScenarioLifecycleV2({
|
|
98
124
|
repoRoot: context.toolRoot,
|
|
99
125
|
runOwnedRoot,
|
|
100
126
|
scenario: resolved.scenario,
|
|
101
127
|
baseCase: resolved.baseCase,
|
|
102
128
|
kitCommand: context.config.kitCommand
|
|
103
129
|
});
|
|
104
|
-
const execution = await
|
|
130
|
+
const execution = await executeIncrementalChangeStalenessScenarioV2({
|
|
105
131
|
repoRoot: context.toolRoot,
|
|
106
132
|
scenario: resolved.scenario,
|
|
107
133
|
lifecycle,
|
|
@@ -113,15 +139,15 @@ export const incrementalChangeStalenessPlugin = {
|
|
|
113
139
|
executions.push(execution);
|
|
114
140
|
}
|
|
115
141
|
const outDir = context.outputRoot ?? path.resolve(context.toolRoot, context.config.outDir);
|
|
116
|
-
const artifact =
|
|
142
|
+
const artifact = buildIncrementalChangeStalenessExecutionArtifactV2({
|
|
117
143
|
runId: context.runId,
|
|
118
144
|
pluginId: INCREMENTAL_CHANGE_STALENESS_PLUGIN_ID,
|
|
119
145
|
executions
|
|
120
146
|
});
|
|
121
147
|
// The artifact is written before the framework's separate `cleanup` phase runs, so it is
|
|
122
148
|
// written before the run-owned ephemeral index/target state is ever destroyed.
|
|
123
|
-
const artifactPath = await
|
|
124
|
-
return
|
|
149
|
+
const artifactPath = await writeIncrementalChangeStalenessExecutionArtifactV2(outDir, artifact);
|
|
150
|
+
return mapIncrementalChangeStalenessExecutionsToRunV2({
|
|
125
151
|
runId: context.runId,
|
|
126
152
|
startedAt,
|
|
127
153
|
completedAt: new Date().toISOString(),
|
|
@@ -239,7 +265,7 @@ export function mapIncrementalChangeStalenessExecutionsToRun(args) {
|
|
|
239
265
|
completedAt: args.completedAt,
|
|
240
266
|
status,
|
|
241
267
|
target: args.target,
|
|
242
|
-
variants:
|
|
268
|
+
variants: V1_TREATMENT_VARIANTS.map((variant) => ({ ...variant })),
|
|
243
269
|
cases,
|
|
244
270
|
metrics: [
|
|
245
271
|
{ id: "incremental-change-staleness-scenario-count", name: "Scenario count", value: args.executions.length, unit: "count", description: "Selected scenarios in this run." },
|
|
@@ -272,3 +298,178 @@ export function mapIncrementalChangeStalenessExecutionsToRun(args) {
|
|
|
272
298
|
run.summary = summarizeExperimentRun(run);
|
|
273
299
|
return run;
|
|
274
300
|
}
|
|
301
|
+
// ---------------------------------------------------------------------------
|
|
302
|
+
// v0.6.3 four-treatment mapping. Maps already-executed V2 evidence; recomputes nothing.
|
|
303
|
+
// ---------------------------------------------------------------------------
|
|
304
|
+
const UPSTREAM_REFRESH_METRIC_SPECS = [
|
|
305
|
+
{ id: "upstream-refresh-fresh-extraction-file-count", name: "Upstream fresh-extraction files", field: "freshExtractionFileCount", description: "Files my-dev-kit freshly re-extracted during the refresh invocation (upstream execution evidence, not Lab task overlap)." },
|
|
306
|
+
{ id: "upstream-refresh-reused-file-count", name: "Upstream reused files", field: "reusedFileCount", description: "Files my-dev-kit reused from the previous index during the refresh invocation (upstream execution evidence)." },
|
|
307
|
+
{ id: "upstream-refresh-forced-neighbor-reanalysis-file-count", name: "Upstream forced-neighbor reanalysis files", field: "forcedNeighborReanalysisFileCount", description: "Neighbor files my-dev-kit forced to re-analyze during the refresh invocation (upstream execution evidence)." },
|
|
308
|
+
{ id: "upstream-refresh-seed-file-count", name: "Upstream seed files", field: "seedFileCount", description: "Seed files my-dev-kit selected for the affected-neighborhood refresh (upstream execution evidence; absent when upstream reported none)." },
|
|
309
|
+
{ id: "upstream-refresh-seed-symbol-count", name: "Upstream seed symbols", field: "seedSymbolCount", description: "Seed symbols my-dev-kit selected for the affected-neighborhood refresh (upstream execution evidence; absent when upstream reported none)." },
|
|
310
|
+
{ id: "upstream-refresh-affected-node-count", name: "Upstream affected nodes", field: "affectedNodeCount", description: "Graph nodes my-dev-kit treated as affected during the refresh (upstream execution evidence, distinct from the Lab affected-neighborhood metrics)." },
|
|
311
|
+
{ id: "upstream-refresh-affected-edge-count", name: "Upstream affected edges", field: "affectedEdgeCount", description: "Graph edges my-dev-kit treated as affected during the refresh (upstream execution evidence, distinct from the Lab affected-neighborhood metrics)." }
|
|
312
|
+
];
|
|
313
|
+
function upstreamRefreshMetrics(treatment, caseId) {
|
|
314
|
+
const refresh = treatment.refreshExecution;
|
|
315
|
+
if (refresh.kind !== "incremental")
|
|
316
|
+
return [];
|
|
317
|
+
const metrics = [];
|
|
318
|
+
for (const spec of UPSTREAM_REFRESH_METRIC_SPECS) {
|
|
319
|
+
const value = refresh.incrementalRefresh[spec.field];
|
|
320
|
+
// Null upstream fields are unavailable evidence, never a zero-valued metric.
|
|
321
|
+
if (typeof value === "number")
|
|
322
|
+
metrics.push(metric(spec.id, spec.name, spec.description, "count", value, treatment.treatmentId, caseId));
|
|
323
|
+
}
|
|
324
|
+
return metrics;
|
|
325
|
+
}
|
|
326
|
+
function comparisonMetadata(execution, treatmentId) {
|
|
327
|
+
const comparison = execution.referenceComparisons.find((entry) => entry.candidateTreatmentId === treatmentId);
|
|
328
|
+
if (!comparison)
|
|
329
|
+
return {};
|
|
330
|
+
if (comparison.kind === "stale-risk") {
|
|
331
|
+
return {
|
|
332
|
+
comparisonCorrectnessRelation: comparison.comparison.correctnessRelation,
|
|
333
|
+
comparisonRequiredFileEvidenceRelation: comparison.comparison.requiredFileEvidenceRelation,
|
|
334
|
+
staleRiskClassification: comparison.comparison.staleRiskClassification
|
|
335
|
+
};
|
|
336
|
+
}
|
|
337
|
+
return {
|
|
338
|
+
referenceComparisonCorrectnessRelation: comparison.correctnessRelation,
|
|
339
|
+
referenceComparisonRequiredFileEvidenceRelation: comparison.requiredFileEvidenceRelation,
|
|
340
|
+
referenceComparisonClassification: comparison.classification
|
|
341
|
+
};
|
|
342
|
+
}
|
|
343
|
+
function treatmentOutcomeV2(execution, treatmentId) {
|
|
344
|
+
const caseId = execution.scenarioId;
|
|
345
|
+
const treatmentIntent = INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_INTENTS[treatmentId];
|
|
346
|
+
const base = { id: `${caseId}:${treatmentId}`, caseId, variantId: treatmentId, artifacts: [] };
|
|
347
|
+
const treatment = execution.treatments.find((entry) => entry.treatmentId === treatmentId);
|
|
348
|
+
if (execution.status === "failed" || !treatment) {
|
|
349
|
+
return {
|
|
350
|
+
...base,
|
|
351
|
+
status: "failed",
|
|
352
|
+
metrics: [],
|
|
353
|
+
warnings: [],
|
|
354
|
+
failures: [
|
|
355
|
+
{
|
|
356
|
+
code: "incremental-change-staleness-scenario-failed",
|
|
357
|
+
message: execution.failureReason ?? "Scenario execution failed before treatment evidence was produced.",
|
|
358
|
+
variantId: treatmentId,
|
|
359
|
+
caseId,
|
|
360
|
+
recoverable: false
|
|
361
|
+
}
|
|
362
|
+
],
|
|
363
|
+
metadata: { treatmentId, treatmentIntent, scenarioStatus: execution.status }
|
|
364
|
+
};
|
|
365
|
+
}
|
|
366
|
+
const status = treatment.status === "completed" ? "completed" : treatment.status === "failed" ? "failed" : "partial";
|
|
367
|
+
const refresh = treatment.refreshExecution;
|
|
368
|
+
const incremental = refresh.kind === "incremental" ? refresh.incrementalRefresh : null;
|
|
369
|
+
return {
|
|
370
|
+
...base,
|
|
371
|
+
status,
|
|
372
|
+
metrics: [...treatmentMetrics(treatment, caseId), ...upstreamRefreshMetrics(treatment, caseId)],
|
|
373
|
+
warnings: treatment.retrievalStatus === "failed" || treatment.fakeAgent?.status === "failed"
|
|
374
|
+
? [{ code: "incremental-change-staleness-treatment-partial", message: treatment.failureReason ?? "Treatment retrieval or fake-agent evaluation did not fully complete.", variantId: treatmentId, caseId }]
|
|
375
|
+
: [],
|
|
376
|
+
failures: [],
|
|
377
|
+
metadata: {
|
|
378
|
+
treatmentId,
|
|
379
|
+
treatmentIntent,
|
|
380
|
+
activeIndexPhase: treatment.activeIndexPhase,
|
|
381
|
+
treatmentStatus: treatment.status,
|
|
382
|
+
retrievalStatus: treatment.retrievalStatus,
|
|
383
|
+
fakeAgentStatus: treatment.fakeAgent?.status ?? "not-run",
|
|
384
|
+
requiredFileEvidenceStatus: treatment.requiredFileEvidence.status,
|
|
385
|
+
affectedNeighborhoodRelationship: treatment.affectedNeighborhood.relationship,
|
|
386
|
+
affectedNeighborhoodReindexRecommendation: treatment.affectedNeighborhood.reindexRecommendation,
|
|
387
|
+
refreshKind: refresh.kind,
|
|
388
|
+
refreshRealization: refresh.realization,
|
|
389
|
+
// Upstream categorical evidence only for incremental treatments; never invented for the others.
|
|
390
|
+
...(incremental
|
|
391
|
+
? {
|
|
392
|
+
requestedScope: incremental.requestedScope,
|
|
393
|
+
appliedScope: incremental.appliedScope,
|
|
394
|
+
selectionStatus: incremental.selectionStatus,
|
|
395
|
+
fallbackReason: incremental.fallbackReason
|
|
396
|
+
}
|
|
397
|
+
: {}),
|
|
398
|
+
...comparisonMetadata(execution, treatmentId)
|
|
399
|
+
}
|
|
400
|
+
};
|
|
401
|
+
}
|
|
402
|
+
function countByRealization(executions, treatmentId, realization) {
|
|
403
|
+
return executions.filter((execution) => execution.treatments.find((treatment) => treatment.treatmentId === treatmentId)?.refreshExecution.realization === realization).length;
|
|
404
|
+
}
|
|
405
|
+
export function mapIncrementalChangeStalenessExecutionsToRunV2(args) {
|
|
406
|
+
const cases = args.executions.map((execution) => {
|
|
407
|
+
const partialClass = (candidate) => {
|
|
408
|
+
const comparison = execution.referenceComparisons.find((entry) => entry.candidateTreatmentId === candidate);
|
|
409
|
+
return comparison && comparison.kind === "partial-refresh" ? comparison.classification : null;
|
|
410
|
+
};
|
|
411
|
+
const stale = execution.referenceComparisons.find((entry) => entry.candidateTreatmentId === "stale-index");
|
|
412
|
+
return {
|
|
413
|
+
id: execution.scenarioId,
|
|
414
|
+
name: `${execution.scenarioId} (${execution.category})`,
|
|
415
|
+
// Exactly four outcomes per scenario, in the fixed lifecycle order (never a "raw" outcome).
|
|
416
|
+
outcomes: INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_IDS.map((treatmentId) => treatmentOutcomeV2(execution, treatmentId)),
|
|
417
|
+
metadata: {
|
|
418
|
+
category: execution.category,
|
|
419
|
+
benchmarkProject: execution.benchmarkProjectId,
|
|
420
|
+
baseCaseId: execution.baseCaseId,
|
|
421
|
+
scenarioStatus: execution.status,
|
|
422
|
+
staleRiskClassification: stale && stale.kind === "stale-risk" ? stale.comparison.staleRiskClassification : null,
|
|
423
|
+
changedFilesReferenceClassification: partialClass("changed-files-refresh"),
|
|
424
|
+
affectedNeighborhoodReferenceClassification: partialClass("affected-neighborhood-refresh")
|
|
425
|
+
}
|
|
426
|
+
};
|
|
427
|
+
});
|
|
428
|
+
const outcomeStatuses = cases.flatMap((experimentCase) => experimentCase.outcomes.map((outcome) => outcome.status));
|
|
429
|
+
const status = outcomeStatuses.length > 0 && outcomeStatuses.every((value) => value === "failed")
|
|
430
|
+
? "failed"
|
|
431
|
+
: outcomeStatuses.every((value) => value === "completed")
|
|
432
|
+
? "completed"
|
|
433
|
+
: "partial";
|
|
434
|
+
const count = (id, name, value, description) => ({ id, name, value, unit: "count", description });
|
|
435
|
+
const run = {
|
|
436
|
+
runId: args.runId,
|
|
437
|
+
pluginId: INCREMENTAL_CHANGE_STALENESS_PLUGIN_ID,
|
|
438
|
+
startedAt: args.startedAt,
|
|
439
|
+
completedAt: args.completedAt,
|
|
440
|
+
status,
|
|
441
|
+
target: args.target,
|
|
442
|
+
variants: TREATMENT_VARIANTS.map((variant) => ({ ...variant })),
|
|
443
|
+
cases,
|
|
444
|
+
metrics: [
|
|
445
|
+
count("incremental-change-staleness-scenario-count", "Scenario count", args.executions.length, "Selected scenarios in this run."),
|
|
446
|
+
count("incremental-change-staleness-observed-regression-count", "Observed stale regression count", args.executions.filter((execution) => {
|
|
447
|
+
const stale = execution.referenceComparisons.find((entry) => entry.candidateTreatmentId === "stale-index");
|
|
448
|
+
return stale?.kind === "stale-risk" && stale.comparison.staleRiskClassification === "observed-stale-regression";
|
|
449
|
+
}).length, "Scenarios whose no-refresh treatment was classified observed-stale-regression against full refresh."),
|
|
450
|
+
count("incremental-change-staleness-changed-files-applied-partial-count", "Changed-files applied-partial count", countByRealization(args.executions, "changed-files-refresh", "APPLIED_PARTIAL"), "Scenarios where the changed-files refresh was applied as a partial refresh."),
|
|
451
|
+
count("incremental-change-staleness-changed-files-fallback-full-count", "Changed-files fallback-full count", countByRealization(args.executions, "changed-files-refresh", "FALLBACK_FULL"), "Scenarios where the changed-files refresh fell back to a full rebuild."),
|
|
452
|
+
count("incremental-change-staleness-affected-neighborhood-applied-partial-count", "Affected-neighborhood applied-partial count", countByRealization(args.executions, "affected-neighborhood-refresh", "APPLIED_PARTIAL"), "Scenarios where the affected-neighborhood refresh was applied as a partial refresh."),
|
|
453
|
+
count("incremental-change-staleness-affected-neighborhood-fallback-full-count", "Affected-neighborhood fallback-full count", countByRealization(args.executions, "affected-neighborhood-refresh", "FALLBACK_FULL"), "Scenarios where the affected-neighborhood refresh fell back to a full rebuild.")
|
|
454
|
+
],
|
|
455
|
+
artifacts: [
|
|
456
|
+
{
|
|
457
|
+
id: "incremental-change-staleness-execution",
|
|
458
|
+
label: "Incremental-change-staleness execution evidence",
|
|
459
|
+
path: args.artifactPath,
|
|
460
|
+
kind: "artifact",
|
|
461
|
+
mimeType: "application/json",
|
|
462
|
+
description: "Per-scenario four-treatment execution, refresh-realization, and full-refresh reference-comparison evidence, without raw context bodies."
|
|
463
|
+
}
|
|
464
|
+
],
|
|
465
|
+
warnings: [],
|
|
466
|
+
failures: [],
|
|
467
|
+
metadata: {
|
|
468
|
+
kitCommand: args.kitCommand,
|
|
469
|
+
scenarioIds: args.executions.map((execution) => execution.scenarioId),
|
|
470
|
+
executionArtifactPath: args.artifactPath
|
|
471
|
+
}
|
|
472
|
+
};
|
|
473
|
+
run.summary = summarizeExperimentRun(run);
|
|
474
|
+
return run;
|
|
475
|
+
}
|
|
@@ -5,7 +5,9 @@ import { loadAffectedNeighborhoodGraphEvidence } from "../../../evaluation/affec
|
|
|
5
5
|
import { captureIndexSnapshot } from "../../../evaluation/indexSnapshot.js";
|
|
6
6
|
/**
|
|
7
7
|
* Builds the index once through the shared `buildMyDevKitIndex` owner. A build that reports
|
|
8
|
-
* success but leaves no index directory is not treated as a prepared session.
|
|
8
|
+
* success but leaves no index directory is not treated as a prepared session. `mode` is optional
|
|
9
|
+
* and defaults to an ordinary full build; only callers that explicitly request an incremental
|
|
10
|
+
* build mode get one.
|
|
9
11
|
*/
|
|
10
12
|
export async function prepareWarmIndexSession(options) {
|
|
11
13
|
const build = await buildMyDevKitIndex(options);
|
|
@@ -19,14 +19,21 @@ function buildCandidateEvidenceId(ruleId, location, fingerprint) {
|
|
|
19
19
|
const locationPart = [location.path, location.line !== undefined ? `l${location.line}` : undefined, location.column !== undefined ? `c${location.column}` : undefined]
|
|
20
20
|
.filter((part) => Boolean(part))
|
|
21
21
|
.join(":");
|
|
22
|
-
|
|
22
|
+
const payloadStart = fingerprint.indexOf(":") + 1;
|
|
23
|
+
return [ruleId, slug(locationPart), fingerprint.slice(payloadStart, payloadStart + 16)].join("--");
|
|
23
24
|
}
|
|
24
25
|
// Never accepts a raw secret value as a plain string parameter name to
|
|
25
26
|
// discourage accidental logging at call sites — `rawValue` is redacted and
|
|
26
27
|
// fingerprinted here, once, and the raw value is never stored on the
|
|
27
28
|
// returned object.
|
|
28
29
|
export function makeCandidateEvidence(input) {
|
|
29
|
-
const fingerprint = fingerprintCandidateValue(input.rawValue
|
|
30
|
+
const fingerprint = fingerprintCandidateValue(input.rawValue, {
|
|
31
|
+
ruleId: input.ruleId,
|
|
32
|
+
sourcePath: input.location.path,
|
|
33
|
+
line: input.location.line,
|
|
34
|
+
column: input.location.column,
|
|
35
|
+
purpose: "candidate-evidence"
|
|
36
|
+
});
|
|
30
37
|
const redactedPreview = redactedPreviewForCandidate(input.rawValue);
|
|
31
38
|
const id = buildCandidateEvidenceId(input.ruleId, input.location, fingerprint);
|
|
32
39
|
return {
|