@dailephd/my-dev-kit-lab 0.6.0 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/README.md +7 -4
- package/benchmarks/contracts/incremental-change-staleness-scenarios.json +360 -0
- package/dist/src/commands/runExperimentRunCommand.js +4 -3
- package/dist/src/evaluation/affectedNeighborhood.js +712 -0
- package/dist/src/evaluation/affectedNeighborhoodMetrics.js +51 -0
- package/dist/src/evaluation/indexSnapshot.js +1 -1
- package/dist/src/experiments/defaultRegistry.js +2 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/comparison.js +101 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/config.js +66 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/disposableTarget.js +107 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/execution.js +404 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/executionArtifact.js +160 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/index.js +13 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/lifecycle.js +276 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/lifecyclePolicy.js +152 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/mutationExecution.js +144 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/plugin.js +274 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/scenarioCatalog.js +42 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/scenarioMutation.js +116 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/scenarioSelection.js +79 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/scenarioTypes.js +37 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/scenarioValidation.js +168 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/sourceState.js +145 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/treatmentSession.js +6 -0
- package/dist/src/experiments/plugins/index.js +1 -0
- package/dist/src/experiments/plugins/warmIndexReuse/execution.js +21 -0
- package/dist/src/experiments/plugins/warmIndexReuse/executionArtifact.js +1 -0
- package/dist/src/experiments/plugins/warmIndexReuse/metrics.js +68 -0
- package/dist/src/experiments/plugins/warmIndexReuse/warmIndexSession.js +9 -1
- package/dist/src/report/experiments/buildIncrementalChangeStalenessReport.js +112 -0
- package/dist/src/report/experiments/buildPluginExperimentReport.js +17 -2
- package/dist/src/report/experiments/buildWarmIndexReuseReport.js +80 -3
- package/dist/src/report/experiments/incrementalChangeStalenessReportModel.js +4 -0
- package/dist/src/report/experiments/index.js +3 -0
- package/dist/src/report/experiments/renderIncrementalChangeStalenessHtml.js +239 -0
- package/dist/src/report/experiments/renderPluginExperimentReportHtml.js +3 -0
- package/dist/src/report/experiments/renderPluginExperimentReportText.js +222 -0
- package/dist/src/report/experiments/renderWarmIndexReuseHtml.js +98 -0
- package/dist/src/report/experiments/warmIndexReuseReportModel.js +5 -0
- package/docs/METRICS.md +79 -3
- package/examples/tutorial-browser/prepare.mjs +13 -13
- package/examples/tutorial-browser/server.mjs +13 -13
- package/package.json +1 -1
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
import { existsSync } from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { resolveWithinRoot } from "../../../core/pathSafety.js";
|
|
4
|
+
import { assessIndexFreshness } from "../../../evaluation/indexFreshness.js";
|
|
5
|
+
import { prepareWarmIndexSession } from "../warmIndexReuse/warmIndexSession.js";
|
|
6
|
+
import { createDisposableTreatmentTarget } from "./disposableTarget.js";
|
|
7
|
+
import { evaluateBaselineStaleFreshness, evaluateChangedPathSymmetry, evaluateIndexEvidence, evaluateRefreshedFreshness, evaluateToolIdentity, normalizeRelativePath, normalizeRootForComparison } from "./lifecyclePolicy.js";
|
|
8
|
+
import { executeIncrementalChangeStalenessMutation } from "./mutationExecution.js";
|
|
9
|
+
import { captureIncrementalChangeStalenessSourceState, compareIncrementalChangeStalenessSourceStates } from "./sourceState.js";
|
|
10
|
+
import { INCREMENTAL_CHANGE_STALENESS_LIFECYCLE_SESSION_SCHEMA_VERSION } from "./treatmentSession.js";
|
|
11
|
+
export const defaultIncrementalChangeStalenessLifecycleDeps = {
|
|
12
|
+
prepareIndexSession: prepareWarmIndexSession,
|
|
13
|
+
assessIndexFreshness,
|
|
14
|
+
executeMutation: executeIncrementalChangeStalenessMutation,
|
|
15
|
+
captureSourceState: captureIncrementalChangeStalenessSourceState
|
|
16
|
+
};
|
|
17
|
+
/** Deterministic run-owned layout: targets/, indexes/, and commands/ are separate sibling trees. */
|
|
18
|
+
export function incrementalChangeStalenessTargetsRoot(runOwnedRoot) {
|
|
19
|
+
return path.join(path.resolve(runOwnedRoot), "targets");
|
|
20
|
+
}
|
|
21
|
+
export function incrementalChangeStalenessIndexDir(runOwnedRoot, scenarioId, treatmentId, role) {
|
|
22
|
+
return resolveWithinRoot(path.resolve(runOwnedRoot), path.join("indexes", scenarioId, treatmentId, role));
|
|
23
|
+
}
|
|
24
|
+
export function incrementalChangeStalenessCommandsDir(runOwnedRoot, scenarioId, treatmentId, role) {
|
|
25
|
+
return resolveWithinRoot(path.resolve(runOwnedRoot), path.join("commands", scenarioId, treatmentId, role));
|
|
26
|
+
}
|
|
27
|
+
class LifecycleFailure extends Error {
|
|
28
|
+
failure;
|
|
29
|
+
constructor(failure) {
|
|
30
|
+
super(failure.message);
|
|
31
|
+
this.failure = failure;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
function fail(failure) {
|
|
35
|
+
if (failure)
|
|
36
|
+
throw new LifecycleFailure(failure);
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Owns the matched Batch 3 lifecycle for ONE frozen scenario:
|
|
40
|
+
*
|
|
41
|
+
* 1. two independent disposable copies (stale-index, full-refresh);
|
|
42
|
+
* 2. pre-mutation controlled source-state equivalence;
|
|
43
|
+
* 3. stale-index baseline: index -> snapshot -> graph, then identity proof;
|
|
44
|
+
* 4. full-refresh baseline: index -> snapshot -> graph, then identity proof;
|
|
45
|
+
* 5. BARRIER: only after both baselines are fully ready, the same frozen
|
|
46
|
+
* mutation is applied independently to both copies;
|
|
47
|
+
* 6. post-mutation controlled source-state equivalence;
|
|
48
|
+
* 7. each baseline snapshot assessed against its mutated target (complete
|
|
49
|
+
* `stale` on exactly the scenario-controlled paths, symmetric across
|
|
50
|
+
* treatments);
|
|
51
|
+
* 8. stale-index keeps its baseline index as active retrieval authority
|
|
52
|
+
* (no rebuild);
|
|
53
|
+
* 9. full-refresh builds a NEW complete index into a separate directory,
|
|
54
|
+
* captures its snapshot/graph, and proves it complete `fresh`.
|
|
55
|
+
*
|
|
56
|
+
* Exactly three full index builds and three freshness assessments happen per
|
|
57
|
+
* scenario. Performs no retrieval, affected-neighborhood traversal, metric
|
|
58
|
+
* computation, correctness evaluation, or comparison classification, and
|
|
59
|
+
* never acts on a reindex recommendation. Sequential by design. Does not
|
|
60
|
+
* delete anything: the caller owns and cleans the run-owned root after the
|
|
61
|
+
* session has been consumed.
|
|
62
|
+
*/
|
|
63
|
+
export async function prepareIncrementalChangeStalenessScenarioLifecycle(options) {
|
|
64
|
+
const deps = { ...defaultIncrementalChangeStalenessLifecycleDeps, ...options.deps };
|
|
65
|
+
const { repoRoot, scenario, baseCase, kitCommand } = options;
|
|
66
|
+
const runOwnedRoot = path.resolve(options.runOwnedRoot);
|
|
67
|
+
const events = [];
|
|
68
|
+
const indexBuildCounts = { "stale-index": 0, "full-refresh": 0 };
|
|
69
|
+
const sourceRoots = baseCase.sourceRoots;
|
|
70
|
+
const controlledChangedPaths = scenario.mutation.files.map((file) => normalizeRelativePath(file.path)).sort();
|
|
71
|
+
async function prepareIndex(target, role, indexDir) {
|
|
72
|
+
const treatmentId = target.treatmentId;
|
|
73
|
+
if (existsSync(indexDir)) {
|
|
74
|
+
fail({ code: "index-directory-collision", message: `Refusing to overwrite existing ${treatmentId} ${role} index directory: ${indexDir}.`, treatmentId });
|
|
75
|
+
}
|
|
76
|
+
const commandsDir = incrementalChangeStalenessCommandsDir(runOwnedRoot, scenario.id, treatmentId, role);
|
|
77
|
+
indexBuildCounts[treatmentId] += 1;
|
|
78
|
+
const prepared = await deps.prepareIndexSession({
|
|
79
|
+
target: { absoluteTargetRoot: target.targetRoot, sourceRoots: [...sourceRoots] },
|
|
80
|
+
kitCommand,
|
|
81
|
+
indexDir,
|
|
82
|
+
commandsDir,
|
|
83
|
+
requireKit: false
|
|
84
|
+
});
|
|
85
|
+
if (!prepared.ok) {
|
|
86
|
+
fail({
|
|
87
|
+
code: role === "baseline" ? "baseline-index-build-failed" : "refresh-index-build-failed",
|
|
88
|
+
message: `${treatmentId} ${role} my-dev-kit index build failed: ${prepared.warnings.join(" ") || `exit code ${prepared.build.command.exitCode}`}`,
|
|
89
|
+
treatmentId
|
|
90
|
+
});
|
|
91
|
+
throw new Error("unreachable");
|
|
92
|
+
}
|
|
93
|
+
// prepareIndexSession resolves only after the index command, snapshot capture, and graph
|
|
94
|
+
// loading have all completed, in that order.
|
|
95
|
+
events.push(`${treatmentId}:${role}-index-built`, `${treatmentId}:${role}-snapshot-captured`, `${treatmentId}:${role}-graph-loaded`);
|
|
96
|
+
const evidence = Object.freeze({
|
|
97
|
+
role,
|
|
98
|
+
treatmentId,
|
|
99
|
+
builtRelativeToMutation: role === "baseline" ? "pre-mutation" : "post-mutation",
|
|
100
|
+
indexDir: prepared.session.indexDir,
|
|
101
|
+
commandsDir,
|
|
102
|
+
targetRoot: prepared.session.targetRoot,
|
|
103
|
+
sourceRoots: prepared.session.sourceRoots,
|
|
104
|
+
buildDurationMs: prepared.session.buildDurationMs,
|
|
105
|
+
buildCommand: prepared.session.buildCommand,
|
|
106
|
+
snapshot: prepared.session.indexSnapshot,
|
|
107
|
+
graph: prepared.session.affectedNeighborhoodGraph
|
|
108
|
+
});
|
|
109
|
+
fail(evaluateIndexEvidence(evidence, { targetRoot: target.targetRoot, sourceRoots, indexDir }));
|
|
110
|
+
events.push(`${treatmentId}:${role}-identity-verified`);
|
|
111
|
+
return evidence;
|
|
112
|
+
}
|
|
113
|
+
async function captureState(target) {
|
|
114
|
+
return deps.captureSourceState(target.targetRoot, sourceRoots, target.treatmentId, baseCase.benchmarkProjectId);
|
|
115
|
+
}
|
|
116
|
+
async function mutate(target) {
|
|
117
|
+
const receipt = await deps.executeMutation(scenario, target, sourceRoots, repoRoot);
|
|
118
|
+
if (receipt.status !== "applied") {
|
|
119
|
+
fail({ code: "mutation-failed", message: `${target.treatmentId} mutation was rejected: ${receipt.errors.join(" ")}`, treatmentId: target.treatmentId });
|
|
120
|
+
}
|
|
121
|
+
events.push(`${target.treatmentId}:mutation-applied`);
|
|
122
|
+
return receipt;
|
|
123
|
+
}
|
|
124
|
+
async function assess(index, target) {
|
|
125
|
+
return deps.assessIndexFreshness({ snapshot: index.snapshot, targetRoot: target.targetRoot, sourceRoots });
|
|
126
|
+
}
|
|
127
|
+
try {
|
|
128
|
+
const indexDirs = {
|
|
129
|
+
staleBaseline: incrementalChangeStalenessIndexDir(runOwnedRoot, scenario.id, "stale-index", "baseline"),
|
|
130
|
+
fullBaseline: incrementalChangeStalenessIndexDir(runOwnedRoot, scenario.id, "full-refresh", "baseline"),
|
|
131
|
+
fullRefreshed: incrementalChangeStalenessIndexDir(runOwnedRoot, scenario.id, "full-refresh", "refreshed")
|
|
132
|
+
};
|
|
133
|
+
if (normalizeRootForComparison(indexDirs.staleBaseline) === normalizeRootForComparison(indexDirs.fullBaseline) ||
|
|
134
|
+
normalizeRootForComparison(indexDirs.fullBaseline) === normalizeRootForComparison(indexDirs.fullRefreshed) ||
|
|
135
|
+
normalizeRootForComparison(indexDirs.staleBaseline) === normalizeRootForComparison(indexDirs.fullRefreshed)) {
|
|
136
|
+
fail({ code: "index-directory-collision", message: "Treatment/phase index directories are not distinct.", treatmentId: null });
|
|
137
|
+
}
|
|
138
|
+
// 1. Two independent disposable copies through the Batch 2 owner.
|
|
139
|
+
const targets = {};
|
|
140
|
+
for (const treatmentId of ["stale-index", "full-refresh"]) {
|
|
141
|
+
try {
|
|
142
|
+
targets[treatmentId] = await createDisposableTreatmentTarget({
|
|
143
|
+
repoRoot,
|
|
144
|
+
runRoot: incrementalChangeStalenessTargetsRoot(runOwnedRoot),
|
|
145
|
+
scenarioId: scenario.id,
|
|
146
|
+
treatmentId,
|
|
147
|
+
benchmarkProjectId: baseCase.benchmarkProjectId,
|
|
148
|
+
canonicalProjectRootRelative: baseCase.canonicalProjectRootRelative
|
|
149
|
+
});
|
|
150
|
+
}
|
|
151
|
+
catch (error) {
|
|
152
|
+
fail({ code: "treatment-target-creation-failed", message: `${treatmentId}: ${error.message}`, treatmentId });
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
const stale = targets["stale-index"];
|
|
156
|
+
const full = targets["full-refresh"];
|
|
157
|
+
events.push("treatment-targets-created");
|
|
158
|
+
// 2. Pre-mutation equivalence gates all indexing and mutation.
|
|
159
|
+
const stalePre = await captureState(stale);
|
|
160
|
+
const fullPre = await captureState(full);
|
|
161
|
+
events.push("pre-mutation-source-state-captured");
|
|
162
|
+
const preMutationEquivalence = compareIncrementalChangeStalenessSourceStates(stalePre, fullPre);
|
|
163
|
+
if (preMutationEquivalence.result !== "equivalent") {
|
|
164
|
+
fail({
|
|
165
|
+
code: "pre-mutation-not-equivalent",
|
|
166
|
+
message: `Treatment copies are not pre-mutation equivalent (${preMutationEquivalence.result}): ${preMutationEquivalence.reasons.join("; ")}`,
|
|
167
|
+
treatmentId: null
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
events.push("pre-mutation-equivalence-proven");
|
|
171
|
+
// 3-4. Baselines: each index -> snapshot -> graph -> identity, sequentially.
|
|
172
|
+
const staleBaseline = await prepareIndex(stale, "baseline", indexDirs.staleBaseline);
|
|
173
|
+
const fullBaseline = await prepareIndex(full, "baseline", indexDirs.fullBaseline);
|
|
174
|
+
const baselineTools = evaluateToolIdentity([staleBaseline, fullBaseline]);
|
|
175
|
+
fail(baselineTools.failure);
|
|
176
|
+
events.push("baseline-tool-identity-verified");
|
|
177
|
+
// 5. Barrier: both baselines are index + snapshot + graph READY. Only now may mutation begin.
|
|
178
|
+
events.push("baseline-barrier-ready");
|
|
179
|
+
const staleReceipt = await mutate(stale);
|
|
180
|
+
const fullReceipt = await mutate(full);
|
|
181
|
+
// 6. Post-mutation equivalence.
|
|
182
|
+
const stalePost = await captureState(stale);
|
|
183
|
+
const fullPost = await captureState(full);
|
|
184
|
+
events.push("post-mutation-source-state-captured");
|
|
185
|
+
const postMutationEquivalence = compareIncrementalChangeStalenessSourceStates(stalePost, fullPost);
|
|
186
|
+
if (postMutationEquivalence.result !== "equivalent") {
|
|
187
|
+
fail({
|
|
188
|
+
code: "post-mutation-not-equivalent",
|
|
189
|
+
message: `Treatment copies are not post-mutation equivalent (${postMutationEquivalence.result}): ${postMutationEquivalence.reasons.join("; ")}`,
|
|
190
|
+
treatmentId: null
|
|
191
|
+
});
|
|
192
|
+
}
|
|
193
|
+
events.push("post-mutation-equivalence-proven");
|
|
194
|
+
// 7. Baseline (change-authority) freshness against each mutated target.
|
|
195
|
+
const staleBaselineFreshness = await assess(staleBaseline, stale);
|
|
196
|
+
fail(evaluateBaselineStaleFreshness(staleBaselineFreshness, controlledChangedPaths, "stale-index"));
|
|
197
|
+
events.push("stale-index:baseline-freshness-stale");
|
|
198
|
+
const fullBaselineFreshness = await assess(fullBaseline, full);
|
|
199
|
+
fail(evaluateBaselineStaleFreshness(fullBaselineFreshness, controlledChangedPaths, "full-refresh"));
|
|
200
|
+
events.push("full-refresh:baseline-freshness-stale");
|
|
201
|
+
fail(evaluateChangedPathSymmetry(staleBaselineFreshness, fullBaselineFreshness));
|
|
202
|
+
events.push("baseline-changed-paths-symmetric");
|
|
203
|
+
// 8. stale-index: no rebuild; baseline stays the active retrieval index.
|
|
204
|
+
events.push("stale-index:baseline-retained-as-active");
|
|
205
|
+
// 9. full-refresh: new complete post-mutation index in a separate directory.
|
|
206
|
+
const refreshed = await prepareIndex(full, "refreshed", indexDirs.fullRefreshed);
|
|
207
|
+
const refreshedFreshness = await assess(refreshed, full);
|
|
208
|
+
fail(evaluateRefreshedFreshness(refreshedFreshness));
|
|
209
|
+
events.push("full-refresh:refreshed-freshness-fresh");
|
|
210
|
+
const allTools = evaluateToolIdentity([staleBaseline, fullBaseline, refreshed]);
|
|
211
|
+
fail(allTools.failure);
|
|
212
|
+
events.push("tool-identity-consistent");
|
|
213
|
+
events.push("full-refresh:refreshed-active");
|
|
214
|
+
events.push("lifecycle-ready");
|
|
215
|
+
const staleSession = Object.freeze({
|
|
216
|
+
treatmentId: "stale-index",
|
|
217
|
+
target: stale,
|
|
218
|
+
preMutationSourceState: stalePre,
|
|
219
|
+
postMutationSourceState: stalePost,
|
|
220
|
+
mutationReceipt: staleReceipt,
|
|
221
|
+
changeAuthority: Object.freeze({ baselineIndex: staleBaseline, postMutationBaselineFreshness: staleBaselineFreshness }),
|
|
222
|
+
activeRetrieval: Object.freeze({ role: "baseline", index: staleBaseline }),
|
|
223
|
+
postMutationIndexBuilt: false,
|
|
224
|
+
refreshedIndex: null,
|
|
225
|
+
refreshedFreshness: null,
|
|
226
|
+
indexBuildCount: indexBuildCounts["stale-index"],
|
|
227
|
+
freshnessAssessmentCount: 1,
|
|
228
|
+
lifecycleStatus: "ready",
|
|
229
|
+
warnings: Object.freeze([])
|
|
230
|
+
});
|
|
231
|
+
const fullSession = Object.freeze({
|
|
232
|
+
treatmentId: "full-refresh",
|
|
233
|
+
target: full,
|
|
234
|
+
preMutationSourceState: fullPre,
|
|
235
|
+
postMutationSourceState: fullPost,
|
|
236
|
+
mutationReceipt: fullReceipt,
|
|
237
|
+
changeAuthority: Object.freeze({ baselineIndex: fullBaseline, postMutationBaselineFreshness: fullBaselineFreshness }),
|
|
238
|
+
activeRetrieval: Object.freeze({ role: "refreshed", index: refreshed }),
|
|
239
|
+
postMutationIndexBuilt: true,
|
|
240
|
+
refreshedIndex: refreshed,
|
|
241
|
+
refreshedFreshness,
|
|
242
|
+
indexBuildCount: indexBuildCounts["full-refresh"],
|
|
243
|
+
freshnessAssessmentCount: 2,
|
|
244
|
+
lifecycleStatus: "ready",
|
|
245
|
+
warnings: Object.freeze([])
|
|
246
|
+
});
|
|
247
|
+
const session = Object.freeze({
|
|
248
|
+
schemaVersion: INCREMENTAL_CHANGE_STALENESS_LIFECYCLE_SESSION_SCHEMA_VERSION,
|
|
249
|
+
scenarioId: scenario.id,
|
|
250
|
+
scenario,
|
|
251
|
+
baseCase,
|
|
252
|
+
runOwnedRoot,
|
|
253
|
+
kitCommand,
|
|
254
|
+
toolIdentity: allTools.tool,
|
|
255
|
+
controlledChangedPaths: Object.freeze(controlledChangedPaths),
|
|
256
|
+
preMutationEquivalence,
|
|
257
|
+
postMutationEquivalence,
|
|
258
|
+
treatments: Object.freeze({ "stale-index": staleSession, "full-refresh": fullSession }),
|
|
259
|
+
lifecycleEvents: Object.freeze([...events]),
|
|
260
|
+
status: "ready"
|
|
261
|
+
});
|
|
262
|
+
return { status: "ready", session };
|
|
263
|
+
}
|
|
264
|
+
catch (error) {
|
|
265
|
+
const failure = error instanceof LifecycleFailure
|
|
266
|
+
? error.failure
|
|
267
|
+
: { code: "lifecycle-error", message: `Lifecycle failed unexpectedly: ${error instanceof Error ? error.message : String(error)}`, treatmentId: null };
|
|
268
|
+
return {
|
|
269
|
+
status: "failed",
|
|
270
|
+
scenarioId: scenario.id,
|
|
271
|
+
failure,
|
|
272
|
+
lifecycleEvents: Object.freeze([...events]),
|
|
273
|
+
indexBuildCounts: Object.freeze({ ...indexBuildCounts })
|
|
274
|
+
};
|
|
275
|
+
}
|
|
276
|
+
}
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
/**
|
|
3
|
+
* Pure v0.6.2 Batch 3 lifecycle-readiness policy. Every function here only
|
|
4
|
+
* inspects already-produced evidence and returns an explicit failure or
|
|
5
|
+
* null; none touches the filesystem, runs my-dev-kit, mutates, or retrieves.
|
|
6
|
+
* These rules are scoped to this experiment's matched production lifecycle
|
|
7
|
+
* and deliberately do not change the released v0.6.0/v0.6.1 semantics of the
|
|
8
|
+
* evidence they read.
|
|
9
|
+
*/
|
|
10
|
+
export function normalizeRootForComparison(root) {
|
|
11
|
+
const resolved = path.resolve(root);
|
|
12
|
+
return process.platform === "win32" ? resolved.toLowerCase() : resolved;
|
|
13
|
+
}
|
|
14
|
+
export function normalizeRelativePath(relativePath) {
|
|
15
|
+
return relativePath.replace(/\\/g, "/");
|
|
16
|
+
}
|
|
17
|
+
function normalizeSourceRoot(root) {
|
|
18
|
+
return root.replace(/\\/g, "/").replace(/\/+$/, "");
|
|
19
|
+
}
|
|
20
|
+
/** Exact, order-preserving source-root equality after separator/trailing-slash normalization. */
|
|
21
|
+
export function sameSourceRoots(left, right) {
|
|
22
|
+
return left.length === right.length && left.every((root, index) => normalizeSourceRoot(root) === normalizeSourceRoot(right[index]));
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Graph evidence is usable for this lifecycle only when an indexed-file claim
|
|
26
|
+
* exists (not `unavailable`) AND the code graph was actually loaded, since a
|
|
27
|
+
* null code graph makes later node mapping impossible.
|
|
28
|
+
*/
|
|
29
|
+
export function isUsableGraphEvidence(graph) {
|
|
30
|
+
return graph.status !== "unavailable" && graph.codeGraph !== null;
|
|
31
|
+
}
|
|
32
|
+
function failure(code, message, treatmentId) {
|
|
33
|
+
return { code, message, treatmentId };
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Proves one prepared index belongs to the intended treatment target, uses
|
|
37
|
+
* the canonical source roots, carries a complete IndexSnapshotV1 (whose
|
|
38
|
+
* manifest projectRoot/sourceRoots were already checked against the target by
|
|
39
|
+
* captureIndexSnapshot), and carries usable graph evidence loaded from the
|
|
40
|
+
* same index directory. Never relies on directory naming alone.
|
|
41
|
+
*/
|
|
42
|
+
export function evaluateIndexEvidence(evidence, expected) {
|
|
43
|
+
const refreshed = evidence.role === "refreshed";
|
|
44
|
+
const label = `${evidence.treatmentId} ${evidence.role} index`;
|
|
45
|
+
const targetMismatchCode = refreshed ? "active-index-target-mismatch" : "index-target-mismatch";
|
|
46
|
+
if (normalizeRootForComparison(evidence.targetRoot) !== normalizeRootForComparison(expected.targetRoot)) {
|
|
47
|
+
return failure(targetMismatchCode, `${label} target root ${evidence.targetRoot} does not match treatment target ${expected.targetRoot}.`, evidence.treatmentId);
|
|
48
|
+
}
|
|
49
|
+
if (normalizeRootForComparison(evidence.indexDir) !== normalizeRootForComparison(expected.indexDir)) {
|
|
50
|
+
return failure(targetMismatchCode, `${label} directory ${evidence.indexDir} does not match the owned index directory ${expected.indexDir}.`, evidence.treatmentId);
|
|
51
|
+
}
|
|
52
|
+
if (!sameSourceRoots(evidence.sourceRoots, expected.sourceRoots)) {
|
|
53
|
+
return failure("source-root-mismatch", `${label} source roots [${evidence.sourceRoots.join(", ")}] do not match canonical base-case source roots [${expected.sourceRoots.join(", ")}].`, evidence.treatmentId);
|
|
54
|
+
}
|
|
55
|
+
const snapshot = evidence.snapshot;
|
|
56
|
+
if (snapshot.status === "unavailable" && snapshot.unavailable?.code === "manifest-target-mismatch") {
|
|
57
|
+
return failure(targetMismatchCode, `${label} manifest does not belong to the treatment target: ${snapshot.unavailable.message}`, evidence.treatmentId);
|
|
58
|
+
}
|
|
59
|
+
if (snapshot.status !== "complete") {
|
|
60
|
+
const reason = snapshot.unavailable ? `${snapshot.unavailable.code}: ${snapshot.unavailable.message}` : `${snapshot.unresolvedFileCount} unresolved indexed file(s)`;
|
|
61
|
+
return failure(refreshed ? "refreshed-snapshot-unavailable" : "baseline-snapshot-unavailable", `${label} IndexSnapshotV1 is ${snapshot.status} (${reason}).`, evidence.treatmentId);
|
|
62
|
+
}
|
|
63
|
+
if (normalizeRootForComparison(evidence.graph.indexRoot) !== normalizeRootForComparison(expected.indexDir)) {
|
|
64
|
+
return failure(targetMismatchCode, `${label} graph evidence was loaded from ${evidence.graph.indexRoot}, not ${expected.indexDir}.`, evidence.treatmentId);
|
|
65
|
+
}
|
|
66
|
+
if (!isUsableGraphEvidence(evidence.graph)) {
|
|
67
|
+
const reason = evidence.graph.unavailable ? `${evidence.graph.unavailable.code}: ${evidence.graph.unavailable.message}` : "code graph was not loaded";
|
|
68
|
+
return failure(refreshed ? "refreshed-graph-unavailable" : "baseline-graph-unavailable", `${label} graph evidence is not usable (${evidence.graph.status}; ${reason}).`, evidence.treatmentId);
|
|
69
|
+
}
|
|
70
|
+
return null;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Every index of one matched scenario must come from the same configured
|
|
74
|
+
* my-dev-kit command and the same probed tool version. An unavailable
|
|
75
|
+
* version is never guessed: identity that cannot be established blocks
|
|
76
|
+
* readiness explicitly.
|
|
77
|
+
*/
|
|
78
|
+
export function evaluateToolIdentity(indexes) {
|
|
79
|
+
for (const index of indexes) {
|
|
80
|
+
if (index.snapshot.tool.availability !== "available" || index.snapshot.tool.version === null) {
|
|
81
|
+
return {
|
|
82
|
+
failure: failure("tool-identity-unavailable", `${index.treatmentId} ${index.role} index my-dev-kit version could not be established: ${index.snapshot.tool.reason ?? "unavailable"}.`, index.treatmentId),
|
|
83
|
+
tool: null
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
const versions = [...new Set(indexes.map((index) => index.snapshot.tool.version))];
|
|
88
|
+
if (versions.length > 1) {
|
|
89
|
+
return {
|
|
90
|
+
failure: failure("tool-identity-mismatch", `Matched scenario indexes were built with different my-dev-kit versions: ${indexes.map((index) => `${index.treatmentId}/${index.role}=${index.snapshot.tool.version}`).join(", ")}.`, null),
|
|
91
|
+
tool: null
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
const commands = [...new Set(indexes.map((index) => index.snapshot.indexCommand.commandString))];
|
|
95
|
+
if (commands.length > 1) {
|
|
96
|
+
return {
|
|
97
|
+
failure: failure("tool-identity-mismatch", `Matched scenario indexes were built with different my-dev-kit commands: ${commands.join(" | ")}.`, null),
|
|
98
|
+
tool: null
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
return { failure: null, tool: indexes.length > 0 ? { ...indexes[0].snapshot.tool } : null };
|
|
102
|
+
}
|
|
103
|
+
function changedPaths(assessment) {
|
|
104
|
+
return assessment.changes.map((change) => normalizeRelativePath(change.path)).sort();
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* A frozen production scenario must be observed as COMPLETE `stale` against
|
|
108
|
+
* the pre-mutation baseline snapshot, and the confirmed changed set must be
|
|
109
|
+
* exactly the scenario-declared controlled file(s) (all modifications, none
|
|
110
|
+
* missing). `fresh`, `unknown`, and `partially-stale` never count as a proven
|
|
111
|
+
* controlled stale state.
|
|
112
|
+
*/
|
|
113
|
+
export function evaluateBaselineStaleFreshness(assessment, controlledChangedPaths, treatmentId) {
|
|
114
|
+
if (assessment.status !== "stale") {
|
|
115
|
+
return failure("baseline-freshness-not-stale", `${treatmentId} baseline freshness after mutation is ${assessment.status}, not complete stale.`, treatmentId);
|
|
116
|
+
}
|
|
117
|
+
if (assessment.baselineSnapshotStatus !== "complete" || assessment.unresolvedFileCount !== 0 || assessment.changesTruncated) {
|
|
118
|
+
return failure("baseline-freshness-not-stale", `${treatmentId} baseline freshness comparison is incomplete (snapshot ${assessment.baselineSnapshotStatus}, ${assessment.unresolvedFileCount} unresolved, truncated=${assessment.changesTruncated}).`, treatmentId);
|
|
119
|
+
}
|
|
120
|
+
const expected = [...controlledChangedPaths].map(normalizeRelativePath).sort();
|
|
121
|
+
const observed = changedPaths(assessment);
|
|
122
|
+
const sameSet = observed.length === expected.length && observed.every((value, index) => value === expected[index]);
|
|
123
|
+
if (!sameSet || assessment.missingFileCount !== 0) {
|
|
124
|
+
return failure("baseline-changed-paths-mismatch", `${treatmentId} baseline freshness changed paths [${observed.join(", ")}] (missing=${assessment.missingFileCount}) do not match scenario-controlled paths [${expected.join(", ")}].`, treatmentId);
|
|
125
|
+
}
|
|
126
|
+
return null;
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* Both treatments applied the same mutation to equivalent copies, so their
|
|
130
|
+
* baseline freshness change evidence must agree path-for-path, including
|
|
131
|
+
* baseline and current content identity.
|
|
132
|
+
*/
|
|
133
|
+
export function evaluateChangedPathSymmetry(stale, fullRefresh) {
|
|
134
|
+
const key = (assessment) => assessment.changes
|
|
135
|
+
.map((change) => `${normalizeRelativePath(change.path)}\t${change.changeType}\t${change.baselineSha256}\t${change.currentSha256 ?? "null"}`)
|
|
136
|
+
.sort();
|
|
137
|
+
const left = key(stale);
|
|
138
|
+
const right = key(fullRefresh);
|
|
139
|
+
const same = left.length === right.length && left.every((value, index) => value === right[index]);
|
|
140
|
+
if (!same) {
|
|
141
|
+
return failure("treatment-changed-paths-conflict", `stale-index and full-refresh baseline freshness disagree about the controlled change: stale=[${left.join("; ")}] full-refresh=[${right.join("; ")}].`, null);
|
|
142
|
+
}
|
|
143
|
+
return null;
|
|
144
|
+
}
|
|
145
|
+
/** The refreshed index must be proven complete `fresh` against the unchanged mutated target. */
|
|
146
|
+
export function evaluateRefreshedFreshness(assessment) {
|
|
147
|
+
const complete = assessment.baselineSnapshotStatus === "complete" && assessment.unresolvedFileCount === 0;
|
|
148
|
+
if (assessment.status !== "fresh" || !complete || assessment.changedFileCount !== 0 || assessment.missingFileCount !== 0) {
|
|
149
|
+
return failure("refreshed-freshness-not-fresh", `full-refresh refreshed index freshness is ${assessment.status} (snapshot ${assessment.baselineSnapshotStatus}, ${assessment.changedFileCount} changed, ${assessment.missingFileCount} missing, ${assessment.unresolvedFileCount} unresolved); complete fresh is required.`, "full-refresh");
|
|
150
|
+
}
|
|
151
|
+
return null;
|
|
152
|
+
}
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
import { readFile, writeFile } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { resolveWithinDisposableTarget } from "./disposableTarget.js";
|
|
4
|
+
import { applyMutationOperations, isPathCoveredBySourceRoots, sha256Hex } from "./scenarioMutation.js";
|
|
5
|
+
export const INCREMENTAL_CHANGE_STALENESS_MUTATION_RECEIPT_SCHEMA_VERSION = "1.0.0";
|
|
6
|
+
function canonicalBenchmarkProjectsRoot(repoRoot) {
|
|
7
|
+
return path.resolve(repoRoot, "benchmarks/projects");
|
|
8
|
+
}
|
|
9
|
+
function isWithin(root, candidate) {
|
|
10
|
+
const relative = path.relative(root, candidate);
|
|
11
|
+
return relative !== "" && !relative.startsWith("..") && !path.isAbsolute(relative);
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Executes one scenario's already-validated mutation plan against ONE
|
|
15
|
+
* disposable treatment copy. Re-validates every guard at execution time
|
|
16
|
+
* (path containment/coverage, pre-hash, ordered exact-preimage replacement,
|
|
17
|
+
* post-hash) rather than trusting that Batch 1 catalog validation still
|
|
18
|
+
* holds, because a disposable copy can drift between creation and mutation.
|
|
19
|
+
*
|
|
20
|
+
* Two-phase: every file is fully validated and its post-mutation bytes
|
|
21
|
+
* computed in memory BEFORE any filesystem write begins. Writes only start
|
|
22
|
+
* once the complete plan is internally valid. If a later write/verification
|
|
23
|
+
* fails, files already written and verified remain recorded as such; the
|
|
24
|
+
* overall receipt status is "rejected" and no file is misreported as
|
|
25
|
+
* successfully mutated.
|
|
26
|
+
*/
|
|
27
|
+
export async function executeIncrementalChangeStalenessMutation(scenario, target, caseSourceRoots, repoRoot) {
|
|
28
|
+
const base = () => ({
|
|
29
|
+
schemaVersion: INCREMENTAL_CHANGE_STALENESS_MUTATION_RECEIPT_SCHEMA_VERSION,
|
|
30
|
+
scenarioId: scenario.id,
|
|
31
|
+
treatmentId: target.treatmentId,
|
|
32
|
+
benchmarkProjectId: target.benchmarkProjectId
|
|
33
|
+
});
|
|
34
|
+
const canonicalRoot = canonicalBenchmarkProjectsRoot(repoRoot);
|
|
35
|
+
if (isWithin(canonicalRoot, path.resolve(target.targetRoot)) || path.resolve(target.targetRoot) === canonicalRoot) {
|
|
36
|
+
return {
|
|
37
|
+
...base(),
|
|
38
|
+
status: "rejected",
|
|
39
|
+
files: [],
|
|
40
|
+
errors: [`Refusing to mutate a canonical benchmark target directly: ${target.targetRoot}.`]
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
const fileReceipts = [];
|
|
44
|
+
const plannedWrites = [];
|
|
45
|
+
const errors = [];
|
|
46
|
+
// Phase 1: validate every file and compute post-mutation bytes in memory. No writes yet.
|
|
47
|
+
for (const fileDeclaration of scenario.mutation.files) {
|
|
48
|
+
const label = `${scenario.id}/${target.treatmentId}: ${fileDeclaration.path}`;
|
|
49
|
+
const receipt = {
|
|
50
|
+
path: fileDeclaration.path,
|
|
51
|
+
expectedPreSha256: fileDeclaration.expectedPreSha256,
|
|
52
|
+
observedPreSha256: null,
|
|
53
|
+
operationCount: fileDeclaration.operations.length,
|
|
54
|
+
expectedPostSha256: fileDeclaration.expectedPostSha256,
|
|
55
|
+
observedPostSha256: null,
|
|
56
|
+
written: false
|
|
57
|
+
};
|
|
58
|
+
if (!isPathCoveredBySourceRoots(fileDeclaration.path, caseSourceRoots)) {
|
|
59
|
+
receipt.error = `${label}: path is not covered by the referenced case's indexed source roots.`;
|
|
60
|
+
errors.push(receipt.error);
|
|
61
|
+
fileReceipts.push(receipt);
|
|
62
|
+
return { ...base(), status: "rejected", files: fileReceipts, errors };
|
|
63
|
+
}
|
|
64
|
+
let absolutePath;
|
|
65
|
+
try {
|
|
66
|
+
absolutePath = await resolveWithinDisposableTarget(target, fileDeclaration.path);
|
|
67
|
+
}
|
|
68
|
+
catch (error) {
|
|
69
|
+
receipt.error = `${label}: ${error.message}`;
|
|
70
|
+
errors.push(receipt.error);
|
|
71
|
+
fileReceipts.push(receipt);
|
|
72
|
+
return { ...base(), status: "rejected", files: fileReceipts, errors };
|
|
73
|
+
}
|
|
74
|
+
let buffer;
|
|
75
|
+
try {
|
|
76
|
+
buffer = await readFile(absolutePath);
|
|
77
|
+
}
|
|
78
|
+
catch (error) {
|
|
79
|
+
receipt.error = `${label}: unable to read disposable file: ${error.message}`;
|
|
80
|
+
errors.push(receipt.error);
|
|
81
|
+
fileReceipts.push(receipt);
|
|
82
|
+
return { ...base(), status: "rejected", files: fileReceipts, errors };
|
|
83
|
+
}
|
|
84
|
+
const observedPreSha256 = sha256Hex(buffer);
|
|
85
|
+
receipt.observedPreSha256 = observedPreSha256;
|
|
86
|
+
if (observedPreSha256 !== fileDeclaration.expectedPreSha256) {
|
|
87
|
+
receipt.error = `${label}: expectedPreSha256 ${fileDeclaration.expectedPreSha256} does not match observed ${observedPreSha256}.`;
|
|
88
|
+
errors.push(receipt.error);
|
|
89
|
+
fileReceipts.push(receipt);
|
|
90
|
+
return { ...base(), status: "rejected", files: fileReceipts, errors };
|
|
91
|
+
}
|
|
92
|
+
let mutatedText;
|
|
93
|
+
try {
|
|
94
|
+
mutatedText = applyMutationOperations(buffer.toString("utf8"), fileDeclaration.operations);
|
|
95
|
+
}
|
|
96
|
+
catch (error) {
|
|
97
|
+
receipt.error = `${label}: ${error.message}`;
|
|
98
|
+
errors.push(receipt.error);
|
|
99
|
+
fileReceipts.push(receipt);
|
|
100
|
+
return { ...base(), status: "rejected", files: fileReceipts, errors };
|
|
101
|
+
}
|
|
102
|
+
const mutatedBuffer = Buffer.from(mutatedText, "utf8");
|
|
103
|
+
const computedPostSha256 = sha256Hex(mutatedBuffer);
|
|
104
|
+
if (computedPostSha256 !== fileDeclaration.expectedPostSha256) {
|
|
105
|
+
receipt.error = `${label}: expectedPostSha256 ${fileDeclaration.expectedPostSha256} does not match computed in-memory post-mutation SHA-256 ${computedPostSha256}.`;
|
|
106
|
+
errors.push(receipt.error);
|
|
107
|
+
fileReceipts.push(receipt);
|
|
108
|
+
return { ...base(), status: "rejected", files: fileReceipts, errors };
|
|
109
|
+
}
|
|
110
|
+
fileReceipts.push(receipt);
|
|
111
|
+
plannedWrites.push({ fileDeclaration, absolutePath, mutatedBuffer });
|
|
112
|
+
}
|
|
113
|
+
// Phase 2: every file validated. Write, then re-read/re-hash to verify the actual on-disk result.
|
|
114
|
+
for (const [index, planned] of plannedWrites.entries()) {
|
|
115
|
+
const receipt = fileReceipts[index];
|
|
116
|
+
const label = `${scenario.id}/${target.treatmentId}: ${planned.fileDeclaration.path}`;
|
|
117
|
+
try {
|
|
118
|
+
await writeFile(planned.absolutePath, planned.mutatedBuffer);
|
|
119
|
+
}
|
|
120
|
+
catch (error) {
|
|
121
|
+
receipt.error = `${label}: write failed: ${error.message}`;
|
|
122
|
+
errors.push(receipt.error);
|
|
123
|
+
return { ...base(), status: "rejected", files: fileReceipts, errors };
|
|
124
|
+
}
|
|
125
|
+
let rereadBuffer;
|
|
126
|
+
try {
|
|
127
|
+
rereadBuffer = await readFile(planned.absolutePath);
|
|
128
|
+
}
|
|
129
|
+
catch (error) {
|
|
130
|
+
receipt.error = `${label}: post-write re-read failed: ${error.message}`;
|
|
131
|
+
errors.push(receipt.error);
|
|
132
|
+
return { ...base(), status: "rejected", files: fileReceipts, errors };
|
|
133
|
+
}
|
|
134
|
+
const observedPostSha256 = sha256Hex(rereadBuffer);
|
|
135
|
+
receipt.observedPostSha256 = observedPostSha256;
|
|
136
|
+
if (observedPostSha256 !== planned.fileDeclaration.expectedPostSha256) {
|
|
137
|
+
receipt.error = `${label}: post-write SHA-256 ${observedPostSha256} does not match expected ${planned.fileDeclaration.expectedPostSha256}.`;
|
|
138
|
+
errors.push(receipt.error);
|
|
139
|
+
return { ...base(), status: "rejected", files: fileReceipts, errors };
|
|
140
|
+
}
|
|
141
|
+
receipt.written = true;
|
|
142
|
+
}
|
|
143
|
+
return { ...base(), status: "applied", files: fileReceipts, errors: [] };
|
|
144
|
+
}
|