@dailephd/my-dev-kit-lab 0.6.2 → 0.6.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/CHANGELOG.md +18 -0
  2. package/README.md +9 -8
  3. package/dist/src/cli/help.js +16 -9
  4. package/dist/src/commands/runExperimentDescribeCommand.js +5 -2
  5. package/dist/src/commands/runExperimentRunCommand.js +0 -4
  6. package/dist/src/evaluation/benchmarkMetadata.js +5 -1
  7. package/dist/src/evaluation/parseAgentAnswer.js +172 -24
  8. package/dist/src/evaluation/runMyDevKitRetrieval.js +120 -7
  9. package/dist/src/experiments/plugins/incrementalChangeStaleness/comparisonV2.js +49 -0
  10. package/dist/src/experiments/plugins/incrementalChangeStaleness/config.js +5 -4
  11. package/dist/src/experiments/plugins/incrementalChangeStaleness/disposableTarget.js +20 -2
  12. package/dist/src/experiments/plugins/incrementalChangeStaleness/execution.js +42 -16
  13. package/dist/src/experiments/plugins/incrementalChangeStaleness/executionArtifact.js +4 -4
  14. package/dist/src/experiments/plugins/incrementalChangeStaleness/executionArtifactV2.js +289 -0
  15. package/dist/src/experiments/plugins/incrementalChangeStaleness/executionV2.js +195 -0
  16. package/dist/src/experiments/plugins/incrementalChangeStaleness/indexClone.js +76 -0
  17. package/dist/src/experiments/plugins/incrementalChangeStaleness/lifecycle.js +64 -32
  18. package/dist/src/experiments/plugins/incrementalChangeStaleness/lifecyclePolicy.js +6 -4
  19. package/dist/src/experiments/plugins/incrementalChangeStaleness/lifecyclePolicyV2.js +78 -0
  20. package/dist/src/experiments/plugins/incrementalChangeStaleness/lifecycleV2.js +298 -0
  21. package/dist/src/experiments/plugins/incrementalChangeStaleness/plugin.js +213 -12
  22. package/dist/src/experiments/plugins/incrementalChangeStaleness/treatmentSessionV2.js +5 -0
  23. package/dist/src/experiments/plugins/warmIndexReuse/warmIndexSession.js +3 -1
  24. package/dist/src/mobile/android/advancedSecurity/candidateEvidence.js +9 -2
  25. package/dist/src/mobile/android/advancedSecurity/redaction.js +19 -10
  26. package/dist/src/mobile/android/advancedSecurity/secretCandidates/analyzeSecretCandidates.js +12 -2
  27. package/dist/src/mobile/android/advancedSecurity/signingConfiguration/analyzeSigningConfiguration.js +1 -1
  28. package/dist/src/mobile/android/advancedSecurity/signingConfiguration/extractSigningConfigurations.js +7 -7
  29. package/dist/src/report/experiments/buildIncrementalChangeStalenessPluginReport.js +33 -0
  30. package/dist/src/report/experiments/buildIncrementalChangeStalenessReport.js +1 -1
  31. package/dist/src/report/experiments/buildIncrementalChangeStalenessReportV2.js +80 -0
  32. package/dist/src/report/experiments/buildPluginExperimentReport.js +2 -2
  33. package/dist/src/report/experiments/incrementalChangeStalenessReportModelV2.js +3 -0
  34. package/dist/src/report/experiments/incrementalChangeStalenessReportRowsV2.js +146 -0
  35. package/dist/src/report/experiments/index.js +3 -0
  36. package/dist/src/report/experiments/renderIncrementalChangeStalenessHtml.js +4 -0
  37. package/dist/src/report/experiments/renderIncrementalChangeStalenessHtmlV2.js +111 -0
  38. package/dist/src/report/experiments/renderIncrementalChangeStalenessTextV2.js +96 -0
  39. package/dist/src/report/experiments/renderPluginExperimentReportText.js +6 -0
  40. package/dist/src/securityValidation/staticScans/codeql.js +8 -8
  41. package/docs/METRICS.md +32 -0
  42. package/examples/tutorial-browser/prepare.mjs +13 -13
  43. package/examples/tutorial-browser/server.mjs +13 -13
  44. package/package.json +1 -1
@@ -0,0 +1,298 @@
1
+ import path from "node:path";
2
+ import { createDisposableTreatmentTarget, INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_IDS, INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_INTENTS } from "./disposableTarget.js";
3
+ import { cloneIncrementalChangeStalenessIndexDirectory } from "./indexClone.js";
4
+ import { defaultIncrementalChangeStalenessLifecycleDeps, fail, incrementalChangeStalenessIndexDir, incrementalChangeStalenessTargetsRoot, LifecycleFailure, prepareTreatmentIndexEvidence } from "./lifecycle.js";
5
+ import { evaluateBaselineStaleFreshness, evaluateChangedPathSymmetry, evaluateRefreshedFreshness, evaluateToolIdentity, normalizeRelativePath, normalizeRootForComparison } from "./lifecyclePolicy.js";
6
+ import { evaluateBaselineBootstrapEvidence, evaluatePartialRefreshRealization, evaluateV2InvocationCounts } from "./lifecyclePolicyV2.js";
7
+ import { compareIncrementalChangeStalenessSourceStates } from "./sourceState.js";
8
+ import { INCREMENTAL_CHANGE_STALENESS_LIFECYCLE_SESSION_V2_SCHEMA_VERSION } from "./treatmentSessionV2.js";
9
+ const PARTIAL_REFRESH_SCOPES = {
10
+ "changed-files-refresh": "changed-files",
11
+ "affected-neighborhood-refresh": "affected-neighborhood"
12
+ };
13
+ /** Every baseline is bootstrapped incrementally so upstream writes the cache metadata a later partial refresh needs. */
14
+ const BOOTSTRAP_MODE = { kind: "incremental", refreshScope: "changed-files" };
15
+ const V2_TREATMENT_IDS = INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_IDS;
16
+ function withRefreshEvidence(evidence, buildMode, incrementalRefresh) {
17
+ return Object.freeze({ ...evidence, buildMode, incrementalRefresh });
18
+ }
19
+ /**
20
+ * v0.6.3 internal four-treatment lifecycle for ONE frozen scenario (runtime only; not yet wired into the
21
+ * public plugin, execution artifact, comparison, or reports):
22
+ *
23
+ * 1. four independent disposable copies (stale-index, changed-files-refresh, affected-neighborhood-refresh,
24
+ * full-refresh), all derived from the same immutable canonical benchmark;
25
+ * 2. pre-mutation controlled source-state equivalence of every treatment against the stale-index reference;
26
+ * 3. all four baselines bootstrapped with `--incremental --refresh-scope changed-files` into brand-new
27
+ * index directories (expected bootstrap evidence: full fallback, `cache-missing`);
28
+ * 4. each partial treatment's untouched baseline is cloned byte-for-byte into its own refreshed directory;
29
+ * 5. BARRIER: only after every baseline + clone is ready is the same frozen mutation applied to all four;
30
+ * 6. post-mutation source-state equivalence and symmetric complete-stale baseline freshness;
31
+ * 7. stale-index keeps its baseline; each partial treatment refreshes its clone in place and full-refresh
32
+ * builds a new full index; every refreshed index must be complete `fresh`.
33
+ *
34
+ * Seven index invocations and seven freshness assessments per scenario. Performs no retrieval,
35
+ * affected-neighborhood task traversal, or comparison classification. Sequential by design; deletes nothing.
36
+ */
37
+ export async function prepareIncrementalChangeStalenessScenarioLifecycleV2(options) {
38
+ const cloneIndexDirectory = options.deps?.cloneIndexDirectory ?? cloneIncrementalChangeStalenessIndexDirectory;
39
+ const deps = { ...defaultIncrementalChangeStalenessLifecycleDeps, ...options.deps };
40
+ const { repoRoot, scenario, baseCase, kitCommand } = options;
41
+ const runOwnedRoot = path.resolve(options.runOwnedRoot);
42
+ const events = [];
43
+ const indexInvocationCounts = Object.fromEntries(V2_TREATMENT_IDS.map((id) => [id, 0]));
44
+ const freshnessAssessmentCounts = Object.fromEntries(V2_TREATMENT_IDS.map((id) => [id, 0]));
45
+ const sourceRoots = baseCase.sourceRoots;
46
+ const controlledChangedPaths = scenario.mutation.files.map((file) => normalizeRelativePath(file.path)).sort();
47
+ async function prepareIndex(target, role, indexDir, mode, existingIndexDir = false) {
48
+ const treatmentId = target.treatmentId;
49
+ indexInvocationCounts[treatmentId] += 1;
50
+ const { evidence, build } = await prepareTreatmentIndexEvidence({
51
+ deps,
52
+ runOwnedRoot,
53
+ scenarioId: scenario.id,
54
+ kitCommand,
55
+ sourceRoots,
56
+ target,
57
+ role,
58
+ indexDir,
59
+ mode,
60
+ existingIndexDir,
61
+ onIndexPrepared: () => events.push(`${treatmentId}:${role}-index-built`, `${treatmentId}:${role}-snapshot-captured`, `${treatmentId}:${role}-graph-loaded`)
62
+ });
63
+ events.push(`${treatmentId}:${role}-identity-verified`);
64
+ return { evidence, build };
65
+ }
66
+ async function captureState(target) {
67
+ return deps.captureSourceState(target.targetRoot, sourceRoots, target.treatmentId, baseCase.benchmarkProjectId);
68
+ }
69
+ async function mutate(target) {
70
+ const receipt = await deps.executeMutation(scenario, target, sourceRoots, repoRoot);
71
+ if (receipt.status !== "applied") {
72
+ fail({ code: "mutation-failed", message: `${target.treatmentId} mutation was rejected: ${receipt.errors.join(" ")}`, treatmentId: target.treatmentId });
73
+ }
74
+ events.push(`${target.treatmentId}:mutation-applied`);
75
+ return receipt;
76
+ }
77
+ async function assess(index, target) {
78
+ freshnessAssessmentCounts[target.treatmentId] += 1;
79
+ return deps.assessIndexFreshness({ snapshot: index.snapshot, targetRoot: target.targetRoot, sourceRoots });
80
+ }
81
+ function compareAgainstReference(reference, states, code, phase) {
82
+ const comparisons = {};
83
+ for (const treatmentId of V2_TREATMENT_IDS) {
84
+ if (treatmentId === "stale-index")
85
+ continue;
86
+ const comparison = compareIncrementalChangeStalenessSourceStates(reference, states[treatmentId]);
87
+ comparisons[treatmentId] = comparison;
88
+ if (comparison.result !== "equivalent") {
89
+ fail({
90
+ code,
91
+ message: `${treatmentId} is not ${phase} equivalent to the stale-index reference (${comparison.result}): ${comparison.reasons.join("; ")}`,
92
+ treatmentId
93
+ });
94
+ }
95
+ }
96
+ return comparisons;
97
+ }
98
+ try {
99
+ const baselineDirs = {};
100
+ const refreshedDirs = {};
101
+ for (const treatmentId of V2_TREATMENT_IDS) {
102
+ baselineDirs[treatmentId] = incrementalChangeStalenessIndexDir(runOwnedRoot, scenario.id, treatmentId, "baseline");
103
+ refreshedDirs[treatmentId] = incrementalChangeStalenessIndexDir(runOwnedRoot, scenario.id, treatmentId, "refreshed");
104
+ }
105
+ const allDirs = [...Object.values(baselineDirs), ...Object.values(refreshedDirs)].map(normalizeRootForComparison);
106
+ if (new Set(allDirs).size !== allDirs.length) {
107
+ fail({ code: "index-directory-collision", message: "Treatment/phase index directories are not distinct.", treatmentId: null });
108
+ }
109
+ // 1. Four independent disposable copies through the established owner, in fixed order.
110
+ const targets = {};
111
+ for (const treatmentId of V2_TREATMENT_IDS) {
112
+ try {
113
+ targets[treatmentId] = await createDisposableTreatmentTarget({
114
+ repoRoot,
115
+ runRoot: incrementalChangeStalenessTargetsRoot(runOwnedRoot),
116
+ scenarioId: scenario.id,
117
+ treatmentId,
118
+ benchmarkProjectId: baseCase.benchmarkProjectId,
119
+ canonicalProjectRootRelative: baseCase.canonicalProjectRootRelative
120
+ });
121
+ }
122
+ catch (error) {
123
+ fail({ code: "treatment-target-creation-failed", message: `${treatmentId}: ${error.message}`, treatmentId });
124
+ }
125
+ }
126
+ const targetRootKeys = V2_TREATMENT_IDS.map((id) => normalizeRootForComparison(targets[id].targetRoot));
127
+ if (new Set(targetRootKeys).size !== targetRootKeys.length) {
128
+ fail({ code: "treatment-target-creation-failed", message: "Treatment target roots are not distinct.", treatmentId: null });
129
+ }
130
+ events.push("treatment-targets-created");
131
+ // 2. Pre-mutation equivalence gates all indexing and mutation; stale-index is only the comparison reference.
132
+ const preStates = {};
133
+ for (const treatmentId of V2_TREATMENT_IDS)
134
+ preStates[treatmentId] = await captureState(targets[treatmentId]);
135
+ events.push("pre-mutation-source-state-captured");
136
+ const preMutationEquivalence = compareAgainstReference(preStates["stale-index"], preStates, "pre-mutation-not-equivalent", "pre-mutation");
137
+ events.push("pre-mutation-equivalence-proven");
138
+ // 3. Baselines: every treatment bootstraps incrementally into a brand-new index directory.
139
+ const baselines = {};
140
+ const bootstraps = {};
141
+ for (const treatmentId of V2_TREATMENT_IDS) {
142
+ const { evidence, build } = await prepareIndex(targets[treatmentId], "baseline", baselineDirs[treatmentId], BOOTSTRAP_MODE);
143
+ fail(evaluateBaselineBootstrapEvidence(build, treatmentId));
144
+ bootstraps[treatmentId] = build.incrementalRefresh;
145
+ baselines[treatmentId] = withRefreshEvidence(evidence, build.mode, build.incrementalRefresh);
146
+ events.push(`${treatmentId}:baseline-bootstrap-verified`);
147
+ }
148
+ const baselineTools = evaluateToolIdentity(V2_TREATMENT_IDS.map((id) => baselines[id]));
149
+ fail(baselineTools.failure);
150
+ events.push("baseline-tool-identity-verified");
151
+ // 4. Preserve each partial treatment's immutable baseline; the refresh runs in place against a byte-for-byte clone.
152
+ for (const treatmentId of ["changed-files-refresh", "affected-neighborhood-refresh"]) {
153
+ try {
154
+ await cloneIndexDirectory(runOwnedRoot, baselineDirs[treatmentId], refreshedDirs[treatmentId]);
155
+ }
156
+ catch (error) {
157
+ fail({ code: "index-clone-failed", message: `${treatmentId} baseline index clone failed: ${error.message}`, treatmentId });
158
+ }
159
+ events.push(`${treatmentId}:baseline-index-cloned`);
160
+ }
161
+ // 5. Barrier: all four baselines are index + snapshot + graph READY and clones exist. Only now may mutation begin.
162
+ events.push("baseline-barrier-ready");
163
+ const receipts = {};
164
+ for (const treatmentId of V2_TREATMENT_IDS)
165
+ receipts[treatmentId] = await mutate(targets[treatmentId]);
166
+ // 6. Post-mutation equivalence.
167
+ const postStates = {};
168
+ for (const treatmentId of V2_TREATMENT_IDS)
169
+ postStates[treatmentId] = await captureState(targets[treatmentId]);
170
+ events.push("post-mutation-source-state-captured");
171
+ const postMutationEquivalence = compareAgainstReference(postStates["stale-index"], postStates, "post-mutation-not-equivalent", "post-mutation");
172
+ events.push("post-mutation-equivalence-proven");
173
+ // 7. Baseline (change-authority) freshness against each mutated target, symmetric with the reference.
174
+ const baselineFreshness = {};
175
+ for (const treatmentId of V2_TREATMENT_IDS) {
176
+ baselineFreshness[treatmentId] = await assess(baselines[treatmentId], targets[treatmentId]);
177
+ fail(evaluateBaselineStaleFreshness(baselineFreshness[treatmentId], controlledChangedPaths, treatmentId));
178
+ events.push(`${treatmentId}:baseline-freshness-stale`);
179
+ }
180
+ for (const treatmentId of V2_TREATMENT_IDS) {
181
+ if (treatmentId === "stale-index")
182
+ continue;
183
+ fail(evaluateChangedPathSymmetry(baselineFreshness["stale-index"], baselineFreshness[treatmentId], { reference: "stale-index", other: treatmentId }));
184
+ }
185
+ events.push("baseline-changed-paths-symmetric");
186
+ // 8. stale-index: no rebuild; baseline stays the active retrieval index.
187
+ events.push("stale-index:baseline-retained-as-active");
188
+ // 9. Partial treatments: refresh the cloned working index in place through the shared build owner.
189
+ const refreshed = {};
190
+ const refreshedFreshness = {};
191
+ const realizations = {};
192
+ for (const treatmentId of ["changed-files-refresh", "affected-neighborhood-refresh"]) {
193
+ const scope = PARTIAL_REFRESH_SCOPES[treatmentId];
194
+ const { evidence, build } = await prepareIndex(targets[treatmentId], "refreshed", refreshedDirs[treatmentId], { kind: "incremental", refreshScope: scope }, true);
195
+ const realization = evaluatePartialRefreshRealization(build, scope, treatmentId);
196
+ fail(realization.failure);
197
+ realizations[treatmentId] = realization.realization;
198
+ events.push(`${treatmentId}:refresh-realization-${realization.realization === "APPLIED_PARTIAL" ? "applied-partial" : "fallback-full"}`);
199
+ refreshed[treatmentId] = withRefreshEvidence(evidence, build.mode, build.incrementalRefresh);
200
+ refreshedFreshness[treatmentId] = await assess(evidence, targets[treatmentId]);
201
+ fail(evaluateRefreshedFreshness(refreshedFreshness[treatmentId], treatmentId));
202
+ events.push(`${treatmentId}:refreshed-freshness-fresh`);
203
+ }
204
+ // 10. full-refresh: a NEW complete post-mutation index in a separate directory (baseline is not cloned).
205
+ {
206
+ const { evidence, build } = await prepareIndex(targets["full-refresh"], "refreshed", refreshedDirs["full-refresh"], undefined);
207
+ refreshed["full-refresh"] = withRefreshEvidence(evidence, build.mode, build.incrementalRefresh);
208
+ refreshedFreshness["full-refresh"] = await assess(evidence, targets["full-refresh"]);
209
+ fail(evaluateRefreshedFreshness(refreshedFreshness["full-refresh"], "full-refresh"));
210
+ events.push("full-refresh:refreshed-freshness-fresh");
211
+ }
212
+ // 11. Scenario-level proofs.
213
+ const allTools = evaluateToolIdentity([...V2_TREATMENT_IDS.map((id) => baselines[id]), ...V2_TREATMENT_IDS.flatMap((id) => (refreshed[id] ? [refreshed[id]] : []))]);
214
+ fail(allTools.failure);
215
+ events.push("tool-identity-consistent");
216
+ fail(evaluateV2InvocationCounts(indexInvocationCounts, freshnessAssessmentCounts));
217
+ events.push("invocation-counts-verified");
218
+ events.push("lifecycle-ready");
219
+ const common = (treatmentId) => ({
220
+ treatmentIntent: INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_INTENTS[treatmentId],
221
+ target: targets[treatmentId],
222
+ preMutationSourceState: preStates[treatmentId],
223
+ postMutationSourceState: postStates[treatmentId],
224
+ mutationReceipt: receipts[treatmentId],
225
+ changeAuthority: Object.freeze({
226
+ baselineIndex: baselines[treatmentId],
227
+ baselineBootstrap: bootstraps[treatmentId],
228
+ postMutationBaselineFreshness: baselineFreshness[treatmentId]
229
+ }),
230
+ indexInvocationCount: indexInvocationCounts[treatmentId],
231
+ freshnessAssessmentCount: freshnessAssessmentCounts[treatmentId],
232
+ lifecycleStatus: "ready",
233
+ warnings: Object.freeze([])
234
+ });
235
+ const partialSession = (treatmentId) => Object.freeze({
236
+ ...common(treatmentId),
237
+ treatmentId,
238
+ activeRetrieval: Object.freeze({ role: "refreshed", index: refreshed[treatmentId] }),
239
+ postMutationIndexBuilt: true,
240
+ refreshedIndex: refreshed[treatmentId],
241
+ refreshedFreshness: refreshedFreshness[treatmentId],
242
+ refreshRealization: realizations[treatmentId]
243
+ });
244
+ const staleSession = Object.freeze({
245
+ ...common("stale-index"),
246
+ treatmentId: "stale-index",
247
+ activeRetrieval: Object.freeze({ role: "baseline", index: baselines["stale-index"] }),
248
+ postMutationIndexBuilt: false,
249
+ refreshedIndex: null,
250
+ refreshedFreshness: null,
251
+ refreshRealization: null
252
+ });
253
+ const fullSession = Object.freeze({
254
+ ...common("full-refresh"),
255
+ treatmentId: "full-refresh",
256
+ activeRetrieval: Object.freeze({ role: "refreshed", index: refreshed["full-refresh"] }),
257
+ postMutationIndexBuilt: true,
258
+ refreshedIndex: refreshed["full-refresh"],
259
+ refreshedFreshness: refreshedFreshness["full-refresh"],
260
+ refreshRealization: null
261
+ });
262
+ const session = Object.freeze({
263
+ schemaVersion: INCREMENTAL_CHANGE_STALENESS_LIFECYCLE_SESSION_V2_SCHEMA_VERSION,
264
+ scenarioId: scenario.id,
265
+ scenario,
266
+ baseCase,
267
+ runOwnedRoot,
268
+ kitCommand,
269
+ toolIdentity: allTools.tool,
270
+ controlledChangedPaths: Object.freeze(controlledChangedPaths),
271
+ preMutationEquivalence: Object.freeze(preMutationEquivalence),
272
+ postMutationEquivalence: Object.freeze(postMutationEquivalence),
273
+ treatments: Object.freeze({
274
+ "stale-index": staleSession,
275
+ "changed-files-refresh": partialSession("changed-files-refresh"),
276
+ "affected-neighborhood-refresh": partialSession("affected-neighborhood-refresh"),
277
+ "full-refresh": fullSession
278
+ }),
279
+ totalIndexInvocationCount: V2_TREATMENT_IDS.reduce((sum, id) => sum + indexInvocationCounts[id], 0),
280
+ totalFreshnessAssessmentCount: V2_TREATMENT_IDS.reduce((sum, id) => sum + freshnessAssessmentCounts[id], 0),
281
+ lifecycleEvents: Object.freeze([...events]),
282
+ status: "ready"
283
+ });
284
+ return { status: "ready", session };
285
+ }
286
+ catch (error) {
287
+ const failure = error instanceof LifecycleFailure
288
+ ? error.failure
289
+ : { code: "lifecycle-error", message: `Lifecycle failed unexpectedly: ${error instanceof Error ? error.message : String(error)}`, treatmentId: null };
290
+ return {
291
+ status: "failed",
292
+ scenarioId: scenario.id,
293
+ failure,
294
+ lifecycleEvents: Object.freeze([...events]),
295
+ indexInvocationCounts: Object.freeze({ ...indexInvocationCounts })
296
+ };
297
+ }
298
+ }
@@ -7,10 +7,10 @@ import { toAffectedNeighborhoodMetricValues } from "../../../evaluation/affected
7
7
  import { sanitizePathSegment } from "../../outputPaths.js";
8
8
  import { summarizeExperimentRun } from "../../results.js";
9
9
  import { defaultIncrementalChangeStalenessConfig, incrementalChangeStalenessConfigDefinition, validateIncrementalChangeStalenessConfig } from "./config.js";
10
- import { DEFAULT_INCREMENTAL_CHANGE_STALENESS_RUNTIME_ROOT_RELATIVE, INCREMENTAL_CHANGE_STALENESS_TREATMENT_IDS, removeIncrementalChangeStalenessRuntimeRoot } from "./disposableTarget.js";
11
- import { executeIncrementalChangeStalenessScenario } from "./execution.js";
12
- import { buildIncrementalChangeStalenessExecutionArtifact, writeIncrementalChangeStalenessExecutionArtifact } from "./executionArtifact.js";
13
- import { prepareIncrementalChangeStalenessScenarioLifecycle } from "./lifecycle.js";
10
+ import { DEFAULT_INCREMENTAL_CHANGE_STALENESS_RUNTIME_ROOT_RELATIVE, INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_IDS, INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_INTENTS, removeIncrementalChangeStalenessRuntimeRoot } from "./disposableTarget.js";
11
+ import { buildIncrementalChangeStalenessExecutionArtifactV2, writeIncrementalChangeStalenessExecutionArtifactV2 } from "./executionArtifactV2.js";
12
+ import { executeIncrementalChangeStalenessScenarioV2 } from "./executionV2.js";
13
+ import { prepareIncrementalChangeStalenessScenarioLifecycleV2 } from "./lifecycleV2.js";
14
14
  import { BENCHMARK_PROJECT_PROFILES_PATH } from "./scenarioCatalog.js";
15
15
  import { resolveIncrementalChangeStalenessScenarios } from "./scenarioSelection.js";
16
16
  export const INCREMENTAL_CHANGE_STALENESS_PLUGIN_ID = "incremental-change-staleness";
@@ -21,13 +21,39 @@ export const INCREMENTAL_CHANGE_STALENESS_RUN_OWNER_MARKER = ".incremental-chang
21
21
  export const incrementalChangeStalenessMetadata = {
22
22
  id: INCREMENTAL_CHANGE_STALENESS_PLUGIN_ID,
23
23
  name: "Incremental Change Staleness",
24
- description: "Compares stale-index and full-refresh my-dev-kit retrieval after the same frozen controlled source change: matched correctness, required-file evidence, and a conservative stale-risk classification.",
24
+ description: "Compares no refresh, my-dev-kit changed-files incremental refresh, my-dev-kit affected-neighborhood incremental refresh, and full refresh after the same deterministic controlled source change. Comparisons are scoped to the observed evidence (correctness and required-file presence); full refresh is a reference treatment, not an asserted winner, and a partial treatment may truthfully fall back to a full rebuild.",
25
25
  schemaVersion: "1.0.0",
26
26
  status: "experimental",
27
27
  supportedTargets: ["self"],
28
28
  supportedOutputs: ["json", "html"]
29
29
  };
30
+ export const CHANGED_FILES_REFRESH_VARIANT_ID = "changed-files-refresh";
31
+ export const AFFECTED_NEIGHBORHOOD_REFRESH_VARIANT_ID = "affected-neighborhood-refresh";
32
+ /** Public v0.6.3 treatments, in the fixed lifecycle order. Factual descriptions only; no treatment is called best or preferred. */
30
33
  const TREATMENT_VARIANTS = [
34
+ {
35
+ id: STALE_INDEX_VARIANT_ID,
36
+ name: "No refresh (stale index)",
37
+ description: "Keeps the pre-mutation baseline index and performs no post-mutation refresh."
38
+ },
39
+ {
40
+ id: CHANGED_FILES_REFRESH_VARIANT_ID,
41
+ name: "Changed-files refresh",
42
+ description: "Uses my-dev-kit's incremental changed-files refresh after the controlled mutation; upstream may truthfully fall back to a full rebuild."
43
+ },
44
+ {
45
+ id: AFFECTED_NEIGHBORHOOD_REFRESH_VARIANT_ID,
46
+ name: "Affected-neighborhood refresh",
47
+ description: "Uses my-dev-kit's affected-neighborhood incremental refresh after the controlled mutation; upstream may truthfully fall back to a full rebuild."
48
+ },
49
+ {
50
+ id: FULL_REFRESH_VARIANT_ID,
51
+ name: "Full refresh",
52
+ description: "Builds a complete post-mutation index and uses it as the comparison reference."
53
+ }
54
+ ];
55
+ /** The released v0.6.2 two-treatment variants, kept only for the historical V1 run mapper. */
56
+ const V1_TREATMENT_VARIANTS = [
31
57
  {
32
58
  id: STALE_INDEX_VARIANT_ID,
33
59
  name: "Stale index",
@@ -78,7 +104,7 @@ export const incrementalChangeStalenessPlugin = {
78
104
  metadata: incrementalChangeStalenessMetadata,
79
105
  defaultConfig: defaultIncrementalChangeStalenessConfig,
80
106
  configDefinition: incrementalChangeStalenessConfigDefinition,
81
- supportedVariants: [...INCREMENTAL_CHANGE_STALENESS_TREATMENT_IDS],
107
+ supportedVariants: [...INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_IDS],
82
108
  validateConfig: validateIncrementalChangeStalenessConfig,
83
109
  async run(context) {
84
110
  if (!context.target.isSelf) {
@@ -94,14 +120,14 @@ export const incrementalChangeStalenessPlugin = {
94
120
  const projectProfiles = await readBenchmarkProjectProfiles(path.resolve(context.toolRoot, BENCHMARK_PROJECT_PROFILES_PATH), context.toolRoot);
95
121
  const executions = [];
96
122
  for (const resolved of resolvedScenarios) {
97
- const lifecycle = await prepareIncrementalChangeStalenessScenarioLifecycle({
123
+ const lifecycle = await prepareIncrementalChangeStalenessScenarioLifecycleV2({
98
124
  repoRoot: context.toolRoot,
99
125
  runOwnedRoot,
100
126
  scenario: resolved.scenario,
101
127
  baseCase: resolved.baseCase,
102
128
  kitCommand: context.config.kitCommand
103
129
  });
104
- const execution = await executeIncrementalChangeStalenessScenario({
130
+ const execution = await executeIncrementalChangeStalenessScenarioV2({
105
131
  repoRoot: context.toolRoot,
106
132
  scenario: resolved.scenario,
107
133
  lifecycle,
@@ -113,15 +139,15 @@ export const incrementalChangeStalenessPlugin = {
113
139
  executions.push(execution);
114
140
  }
115
141
  const outDir = context.outputRoot ?? path.resolve(context.toolRoot, context.config.outDir);
116
- const artifact = buildIncrementalChangeStalenessExecutionArtifact({
142
+ const artifact = buildIncrementalChangeStalenessExecutionArtifactV2({
117
143
  runId: context.runId,
118
144
  pluginId: INCREMENTAL_CHANGE_STALENESS_PLUGIN_ID,
119
145
  executions
120
146
  });
121
147
  // The artifact is written before the framework's separate `cleanup` phase runs, so it is
122
148
  // written before the run-owned ephemeral index/target state is ever destroyed.
123
- const artifactPath = await writeIncrementalChangeStalenessExecutionArtifact(outDir, artifact);
124
- return mapIncrementalChangeStalenessExecutionsToRun({
149
+ const artifactPath = await writeIncrementalChangeStalenessExecutionArtifactV2(outDir, artifact);
150
+ return mapIncrementalChangeStalenessExecutionsToRunV2({
125
151
  runId: context.runId,
126
152
  startedAt,
127
153
  completedAt: new Date().toISOString(),
@@ -239,7 +265,7 @@ export function mapIncrementalChangeStalenessExecutionsToRun(args) {
239
265
  completedAt: args.completedAt,
240
266
  status,
241
267
  target: args.target,
242
- variants: TREATMENT_VARIANTS.map((variant) => ({ ...variant })),
268
+ variants: V1_TREATMENT_VARIANTS.map((variant) => ({ ...variant })),
243
269
  cases,
244
270
  metrics: [
245
271
  { id: "incremental-change-staleness-scenario-count", name: "Scenario count", value: args.executions.length, unit: "count", description: "Selected scenarios in this run." },
@@ -272,3 +298,178 @@ export function mapIncrementalChangeStalenessExecutionsToRun(args) {
272
298
  run.summary = summarizeExperimentRun(run);
273
299
  return run;
274
300
  }
301
+ // ---------------------------------------------------------------------------
302
+ // v0.6.3 four-treatment mapping. Maps already-executed V2 evidence; recomputes nothing.
303
+ // ---------------------------------------------------------------------------
304
+ const UPSTREAM_REFRESH_METRIC_SPECS = [
305
+ { id: "upstream-refresh-fresh-extraction-file-count", name: "Upstream fresh-extraction files", field: "freshExtractionFileCount", description: "Files my-dev-kit freshly re-extracted during the refresh invocation (upstream execution evidence, not Lab task overlap)." },
306
+ { id: "upstream-refresh-reused-file-count", name: "Upstream reused files", field: "reusedFileCount", description: "Files my-dev-kit reused from the previous index during the refresh invocation (upstream execution evidence)." },
307
+ { id: "upstream-refresh-forced-neighbor-reanalysis-file-count", name: "Upstream forced-neighbor reanalysis files", field: "forcedNeighborReanalysisFileCount", description: "Neighbor files my-dev-kit forced to re-analyze during the refresh invocation (upstream execution evidence)." },
308
+ { id: "upstream-refresh-seed-file-count", name: "Upstream seed files", field: "seedFileCount", description: "Seed files my-dev-kit selected for the affected-neighborhood refresh (upstream execution evidence; absent when upstream reported none)." },
309
+ { id: "upstream-refresh-seed-symbol-count", name: "Upstream seed symbols", field: "seedSymbolCount", description: "Seed symbols my-dev-kit selected for the affected-neighborhood refresh (upstream execution evidence; absent when upstream reported none)." },
310
+ { id: "upstream-refresh-affected-node-count", name: "Upstream affected nodes", field: "affectedNodeCount", description: "Graph nodes my-dev-kit treated as affected during the refresh (upstream execution evidence, distinct from the Lab affected-neighborhood metrics)." },
311
+ { id: "upstream-refresh-affected-edge-count", name: "Upstream affected edges", field: "affectedEdgeCount", description: "Graph edges my-dev-kit treated as affected during the refresh (upstream execution evidence, distinct from the Lab affected-neighborhood metrics)." }
312
+ ];
313
+ function upstreamRefreshMetrics(treatment, caseId) {
314
+ const refresh = treatment.refreshExecution;
315
+ if (refresh.kind !== "incremental")
316
+ return [];
317
+ const metrics = [];
318
+ for (const spec of UPSTREAM_REFRESH_METRIC_SPECS) {
319
+ const value = refresh.incrementalRefresh[spec.field];
320
+ // Null upstream fields are unavailable evidence, never a zero-valued metric.
321
+ if (typeof value === "number")
322
+ metrics.push(metric(spec.id, spec.name, spec.description, "count", value, treatment.treatmentId, caseId));
323
+ }
324
+ return metrics;
325
+ }
326
+ function comparisonMetadata(execution, treatmentId) {
327
+ const comparison = execution.referenceComparisons.find((entry) => entry.candidateTreatmentId === treatmentId);
328
+ if (!comparison)
329
+ return {};
330
+ if (comparison.kind === "stale-risk") {
331
+ return {
332
+ comparisonCorrectnessRelation: comparison.comparison.correctnessRelation,
333
+ comparisonRequiredFileEvidenceRelation: comparison.comparison.requiredFileEvidenceRelation,
334
+ staleRiskClassification: comparison.comparison.staleRiskClassification
335
+ };
336
+ }
337
+ return {
338
+ referenceComparisonCorrectnessRelation: comparison.correctnessRelation,
339
+ referenceComparisonRequiredFileEvidenceRelation: comparison.requiredFileEvidenceRelation,
340
+ referenceComparisonClassification: comparison.classification
341
+ };
342
+ }
343
+ function treatmentOutcomeV2(execution, treatmentId) {
344
+ const caseId = execution.scenarioId;
345
+ const treatmentIntent = INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_INTENTS[treatmentId];
346
+ const base = { id: `${caseId}:${treatmentId}`, caseId, variantId: treatmentId, artifacts: [] };
347
+ const treatment = execution.treatments.find((entry) => entry.treatmentId === treatmentId);
348
+ if (execution.status === "failed" || !treatment) {
349
+ return {
350
+ ...base,
351
+ status: "failed",
352
+ metrics: [],
353
+ warnings: [],
354
+ failures: [
355
+ {
356
+ code: "incremental-change-staleness-scenario-failed",
357
+ message: execution.failureReason ?? "Scenario execution failed before treatment evidence was produced.",
358
+ variantId: treatmentId,
359
+ caseId,
360
+ recoverable: false
361
+ }
362
+ ],
363
+ metadata: { treatmentId, treatmentIntent, scenarioStatus: execution.status }
364
+ };
365
+ }
366
+ const status = treatment.status === "completed" ? "completed" : treatment.status === "failed" ? "failed" : "partial";
367
+ const refresh = treatment.refreshExecution;
368
+ const incremental = refresh.kind === "incremental" ? refresh.incrementalRefresh : null;
369
+ return {
370
+ ...base,
371
+ status,
372
+ metrics: [...treatmentMetrics(treatment, caseId), ...upstreamRefreshMetrics(treatment, caseId)],
373
+ warnings: treatment.retrievalStatus === "failed" || treatment.fakeAgent?.status === "failed"
374
+ ? [{ code: "incremental-change-staleness-treatment-partial", message: treatment.failureReason ?? "Treatment retrieval or fake-agent evaluation did not fully complete.", variantId: treatmentId, caseId }]
375
+ : [],
376
+ failures: [],
377
+ metadata: {
378
+ treatmentId,
379
+ treatmentIntent,
380
+ activeIndexPhase: treatment.activeIndexPhase,
381
+ treatmentStatus: treatment.status,
382
+ retrievalStatus: treatment.retrievalStatus,
383
+ fakeAgentStatus: treatment.fakeAgent?.status ?? "not-run",
384
+ requiredFileEvidenceStatus: treatment.requiredFileEvidence.status,
385
+ affectedNeighborhoodRelationship: treatment.affectedNeighborhood.relationship,
386
+ affectedNeighborhoodReindexRecommendation: treatment.affectedNeighborhood.reindexRecommendation,
387
+ refreshKind: refresh.kind,
388
+ refreshRealization: refresh.realization,
389
+ // Upstream categorical evidence only for incremental treatments; never invented for the others.
390
+ ...(incremental
391
+ ? {
392
+ requestedScope: incremental.requestedScope,
393
+ appliedScope: incremental.appliedScope,
394
+ selectionStatus: incremental.selectionStatus,
395
+ fallbackReason: incremental.fallbackReason
396
+ }
397
+ : {}),
398
+ ...comparisonMetadata(execution, treatmentId)
399
+ }
400
+ };
401
+ }
402
+ function countByRealization(executions, treatmentId, realization) {
403
+ return executions.filter((execution) => execution.treatments.find((treatment) => treatment.treatmentId === treatmentId)?.refreshExecution.realization === realization).length;
404
+ }
405
+ export function mapIncrementalChangeStalenessExecutionsToRunV2(args) {
406
+ const cases = args.executions.map((execution) => {
407
+ const partialClass = (candidate) => {
408
+ const comparison = execution.referenceComparisons.find((entry) => entry.candidateTreatmentId === candidate);
409
+ return comparison && comparison.kind === "partial-refresh" ? comparison.classification : null;
410
+ };
411
+ const stale = execution.referenceComparisons.find((entry) => entry.candidateTreatmentId === "stale-index");
412
+ return {
413
+ id: execution.scenarioId,
414
+ name: `${execution.scenarioId} (${execution.category})`,
415
+ // Exactly four outcomes per scenario, in the fixed lifecycle order (never a "raw" outcome).
416
+ outcomes: INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_IDS.map((treatmentId) => treatmentOutcomeV2(execution, treatmentId)),
417
+ metadata: {
418
+ category: execution.category,
419
+ benchmarkProject: execution.benchmarkProjectId,
420
+ baseCaseId: execution.baseCaseId,
421
+ scenarioStatus: execution.status,
422
+ staleRiskClassification: stale && stale.kind === "stale-risk" ? stale.comparison.staleRiskClassification : null,
423
+ changedFilesReferenceClassification: partialClass("changed-files-refresh"),
424
+ affectedNeighborhoodReferenceClassification: partialClass("affected-neighborhood-refresh")
425
+ }
426
+ };
427
+ });
428
+ const outcomeStatuses = cases.flatMap((experimentCase) => experimentCase.outcomes.map((outcome) => outcome.status));
429
+ const status = outcomeStatuses.length > 0 && outcomeStatuses.every((value) => value === "failed")
430
+ ? "failed"
431
+ : outcomeStatuses.every((value) => value === "completed")
432
+ ? "completed"
433
+ : "partial";
434
+ const count = (id, name, value, description) => ({ id, name, value, unit: "count", description });
435
+ const run = {
436
+ runId: args.runId,
437
+ pluginId: INCREMENTAL_CHANGE_STALENESS_PLUGIN_ID,
438
+ startedAt: args.startedAt,
439
+ completedAt: args.completedAt,
440
+ status,
441
+ target: args.target,
442
+ variants: TREATMENT_VARIANTS.map((variant) => ({ ...variant })),
443
+ cases,
444
+ metrics: [
445
+ count("incremental-change-staleness-scenario-count", "Scenario count", args.executions.length, "Selected scenarios in this run."),
446
+ count("incremental-change-staleness-observed-regression-count", "Observed stale regression count", args.executions.filter((execution) => {
447
+ const stale = execution.referenceComparisons.find((entry) => entry.candidateTreatmentId === "stale-index");
448
+ return stale?.kind === "stale-risk" && stale.comparison.staleRiskClassification === "observed-stale-regression";
449
+ }).length, "Scenarios whose no-refresh treatment was classified observed-stale-regression against full refresh."),
450
+ count("incremental-change-staleness-changed-files-applied-partial-count", "Changed-files applied-partial count", countByRealization(args.executions, "changed-files-refresh", "APPLIED_PARTIAL"), "Scenarios where the changed-files refresh was applied as a partial refresh."),
451
+ count("incremental-change-staleness-changed-files-fallback-full-count", "Changed-files fallback-full count", countByRealization(args.executions, "changed-files-refresh", "FALLBACK_FULL"), "Scenarios where the changed-files refresh fell back to a full rebuild."),
452
+ count("incremental-change-staleness-affected-neighborhood-applied-partial-count", "Affected-neighborhood applied-partial count", countByRealization(args.executions, "affected-neighborhood-refresh", "APPLIED_PARTIAL"), "Scenarios where the affected-neighborhood refresh was applied as a partial refresh."),
453
+ count("incremental-change-staleness-affected-neighborhood-fallback-full-count", "Affected-neighborhood fallback-full count", countByRealization(args.executions, "affected-neighborhood-refresh", "FALLBACK_FULL"), "Scenarios where the affected-neighborhood refresh fell back to a full rebuild.")
454
+ ],
455
+ artifacts: [
456
+ {
457
+ id: "incremental-change-staleness-execution",
458
+ label: "Incremental-change-staleness execution evidence",
459
+ path: args.artifactPath,
460
+ kind: "artifact",
461
+ mimeType: "application/json",
462
+ description: "Per-scenario four-treatment execution, refresh-realization, and full-refresh reference-comparison evidence, without raw context bodies."
463
+ }
464
+ ],
465
+ warnings: [],
466
+ failures: [],
467
+ metadata: {
468
+ kitCommand: args.kitCommand,
469
+ scenarioIds: args.executions.map((execution) => execution.scenarioId),
470
+ executionArtifactPath: args.artifactPath
471
+ }
472
+ };
473
+ run.summary = summarizeExperimentRun(run);
474
+ return run;
475
+ }
@@ -0,0 +1,5 @@
1
+ /**
2
+ * v0.6.3 in-memory (runtime-only) four-treatment lifecycle session schema version. This is NOT the
3
+ * persisted execution artifact; persistence is a later batch.
4
+ */
5
+ export const INCREMENTAL_CHANGE_STALENESS_LIFECYCLE_SESSION_V2_SCHEMA_VERSION = "2.0.0";
@@ -5,7 +5,9 @@ import { loadAffectedNeighborhoodGraphEvidence } from "../../../evaluation/affec
5
5
  import { captureIndexSnapshot } from "../../../evaluation/indexSnapshot.js";
6
6
  /**
7
7
  * Builds the index once through the shared `buildMyDevKitIndex` owner. A build that reports
8
- * success but leaves no index directory is not treated as a prepared session.
8
+ * success but leaves no index directory is not treated as a prepared session. `mode` is optional
9
+ * and defaults to an ordinary full build; only callers that explicitly request an incremental
10
+ * build mode get one.
9
11
  */
10
12
  export async function prepareWarmIndexSession(options) {
11
13
  const build = await buildMyDevKitIndex(options);
@@ -19,14 +19,21 @@ function buildCandidateEvidenceId(ruleId, location, fingerprint) {
19
19
  const locationPart = [location.path, location.line !== undefined ? `l${location.line}` : undefined, location.column !== undefined ? `c${location.column}` : undefined]
20
20
  .filter((part) => Boolean(part))
21
21
  .join(":");
22
- return [ruleId, slug(locationPart), fingerprint.slice(0, 16)].join("--");
22
+ const payloadStart = fingerprint.indexOf(":") + 1;
23
+ return [ruleId, slug(locationPart), fingerprint.slice(payloadStart, payloadStart + 16)].join("--");
23
24
  }
24
25
  // Never accepts a raw secret value as a plain string parameter name to
25
26
  // discourage accidental logging at call sites — `rawValue` is redacted and
26
27
  // fingerprinted here, once, and the raw value is never stored on the
27
28
  // returned object.
28
29
  export function makeCandidateEvidence(input) {
29
- const fingerprint = fingerprintCandidateValue(input.rawValue);
30
+ const fingerprint = fingerprintCandidateValue(input.rawValue, {
31
+ ruleId: input.ruleId,
32
+ sourcePath: input.location.path,
33
+ line: input.location.line,
34
+ column: input.location.column,
35
+ purpose: "candidate-evidence"
36
+ });
30
37
  const redactedPreview = redactedPreviewForCandidate(input.rawValue);
31
38
  const id = buildCandidateEvidenceId(input.ruleId, input.location, fingerprint);
32
39
  return {