@dailephd/my-dev-kit-lab 0.6.1 → 0.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/README.md +9 -7
- package/benchmarks/contracts/incremental-change-staleness-scenarios.json +360 -0
- package/dist/src/cli/help.js +16 -9
- package/dist/src/commands/runExperimentDescribeCommand.js +5 -2
- package/dist/src/commands/runExperimentRunCommand.js +4 -7
- package/dist/src/evaluation/affectedNeighborhoodMetrics.js +51 -0
- package/dist/src/evaluation/benchmarkMetadata.js +5 -1
- package/dist/src/evaluation/parseAgentAnswer.js +172 -24
- package/dist/src/evaluation/runMyDevKitRetrieval.js +120 -7
- package/dist/src/experiments/defaultRegistry.js +2 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/comparison.js +101 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/comparisonV2.js +49 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/config.js +67 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/disposableTarget.js +125 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/execution.js +430 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/executionArtifact.js +160 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/executionArtifactV2.js +289 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/executionV2.js +195 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/index.js +13 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/indexClone.js +76 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/lifecycle.js +308 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/lifecyclePolicy.js +154 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/lifecyclePolicyV2.js +78 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/lifecycleV2.js +298 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/mutationExecution.js +144 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/plugin.js +475 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/scenarioCatalog.js +42 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/scenarioMutation.js +116 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/scenarioSelection.js +79 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/scenarioTypes.js +37 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/scenarioValidation.js +168 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/sourceState.js +145 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/treatmentSession.js +6 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/treatmentSessionV2.js +5 -0
- package/dist/src/experiments/plugins/index.js +1 -0
- package/dist/src/experiments/plugins/warmIndexReuse/metrics.js +17 -20
- package/dist/src/experiments/plugins/warmIndexReuse/warmIndexSession.js +3 -1
- package/dist/src/mobile/android/advancedSecurity/candidateEvidence.js +9 -2
- package/dist/src/mobile/android/advancedSecurity/redaction.js +19 -10
- package/dist/src/mobile/android/advancedSecurity/secretCandidates/analyzeSecretCandidates.js +12 -2
- package/dist/src/mobile/android/advancedSecurity/signingConfiguration/analyzeSigningConfiguration.js +1 -1
- package/dist/src/mobile/android/advancedSecurity/signingConfiguration/extractSigningConfigurations.js +7 -7
- package/dist/src/report/experiments/buildIncrementalChangeStalenessPluginReport.js +33 -0
- package/dist/src/report/experiments/buildIncrementalChangeStalenessReport.js +112 -0
- package/dist/src/report/experiments/buildIncrementalChangeStalenessReportV2.js +80 -0
- package/dist/src/report/experiments/buildPluginExperimentReport.js +3 -0
- package/dist/src/report/experiments/incrementalChangeStalenessReportModel.js +4 -0
- package/dist/src/report/experiments/incrementalChangeStalenessReportModelV2.js +3 -0
- package/dist/src/report/experiments/incrementalChangeStalenessReportRowsV2.js +146 -0
- package/dist/src/report/experiments/index.js +6 -0
- package/dist/src/report/experiments/renderIncrementalChangeStalenessHtml.js +243 -0
- package/dist/src/report/experiments/renderIncrementalChangeStalenessHtmlV2.js +111 -0
- package/dist/src/report/experiments/renderIncrementalChangeStalenessTextV2.js +96 -0
- package/dist/src/report/experiments/renderPluginExperimentReportHtml.js +3 -0
- package/dist/src/report/experiments/renderPluginExperimentReportText.js +144 -0
- package/dist/src/securityValidation/staticScans/codeql.js +8 -8
- package/docs/METRICS.md +67 -1
- package/examples/tutorial-browser/prepare.mjs +13 -13
- package/examples/tutorial-browser/server.mjs +13 -13
- package/package.json +1 -1
|
@@ -0,0 +1,298 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
import { createDisposableTreatmentTarget, INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_IDS, INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_INTENTS } from "./disposableTarget.js";
|
|
3
|
+
import { cloneIncrementalChangeStalenessIndexDirectory } from "./indexClone.js";
|
|
4
|
+
import { defaultIncrementalChangeStalenessLifecycleDeps, fail, incrementalChangeStalenessIndexDir, incrementalChangeStalenessTargetsRoot, LifecycleFailure, prepareTreatmentIndexEvidence } from "./lifecycle.js";
|
|
5
|
+
import { evaluateBaselineStaleFreshness, evaluateChangedPathSymmetry, evaluateRefreshedFreshness, evaluateToolIdentity, normalizeRelativePath, normalizeRootForComparison } from "./lifecyclePolicy.js";
|
|
6
|
+
import { evaluateBaselineBootstrapEvidence, evaluatePartialRefreshRealization, evaluateV2InvocationCounts } from "./lifecyclePolicyV2.js";
|
|
7
|
+
import { compareIncrementalChangeStalenessSourceStates } from "./sourceState.js";
|
|
8
|
+
import { INCREMENTAL_CHANGE_STALENESS_LIFECYCLE_SESSION_V2_SCHEMA_VERSION } from "./treatmentSessionV2.js";
|
|
9
|
+
const PARTIAL_REFRESH_SCOPES = {
|
|
10
|
+
"changed-files-refresh": "changed-files",
|
|
11
|
+
"affected-neighborhood-refresh": "affected-neighborhood"
|
|
12
|
+
};
|
|
13
|
+
/** Every baseline is bootstrapped incrementally so upstream writes the cache metadata a later partial refresh needs. */
|
|
14
|
+
const BOOTSTRAP_MODE = { kind: "incremental", refreshScope: "changed-files" };
|
|
15
|
+
const V2_TREATMENT_IDS = INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_IDS;
|
|
16
|
+
function withRefreshEvidence(evidence, buildMode, incrementalRefresh) {
|
|
17
|
+
return Object.freeze({ ...evidence, buildMode, incrementalRefresh });
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* v0.6.3 internal four-treatment lifecycle for ONE frozen scenario (runtime only; not yet wired into the
|
|
21
|
+
* public plugin, execution artifact, comparison, or reports):
|
|
22
|
+
*
|
|
23
|
+
* 1. four independent disposable copies (stale-index, changed-files-refresh, affected-neighborhood-refresh,
|
|
24
|
+
* full-refresh), all derived from the same immutable canonical benchmark;
|
|
25
|
+
* 2. pre-mutation controlled source-state equivalence of every treatment against the stale-index reference;
|
|
26
|
+
* 3. all four baselines bootstrapped with `--incremental --refresh-scope changed-files` into brand-new
|
|
27
|
+
* index directories (expected bootstrap evidence: full fallback, `cache-missing`);
|
|
28
|
+
* 4. each partial treatment's untouched baseline is cloned byte-for-byte into its own refreshed directory;
|
|
29
|
+
* 5. BARRIER: only after every baseline + clone is ready is the same frozen mutation applied to all four;
|
|
30
|
+
* 6. post-mutation source-state equivalence and symmetric complete-stale baseline freshness;
|
|
31
|
+
* 7. stale-index keeps its baseline; each partial treatment refreshes its clone in place and full-refresh
|
|
32
|
+
* builds a new full index; every refreshed index must be complete `fresh`.
|
|
33
|
+
*
|
|
34
|
+
* Seven index invocations and seven freshness assessments per scenario. Performs no retrieval,
|
|
35
|
+
* affected-neighborhood task traversal, or comparison classification. Sequential by design; deletes nothing.
|
|
36
|
+
*/
|
|
37
|
+
export async function prepareIncrementalChangeStalenessScenarioLifecycleV2(options) {
|
|
38
|
+
const cloneIndexDirectory = options.deps?.cloneIndexDirectory ?? cloneIncrementalChangeStalenessIndexDirectory;
|
|
39
|
+
const deps = { ...defaultIncrementalChangeStalenessLifecycleDeps, ...options.deps };
|
|
40
|
+
const { repoRoot, scenario, baseCase, kitCommand } = options;
|
|
41
|
+
const runOwnedRoot = path.resolve(options.runOwnedRoot);
|
|
42
|
+
const events = [];
|
|
43
|
+
const indexInvocationCounts = Object.fromEntries(V2_TREATMENT_IDS.map((id) => [id, 0]));
|
|
44
|
+
const freshnessAssessmentCounts = Object.fromEntries(V2_TREATMENT_IDS.map((id) => [id, 0]));
|
|
45
|
+
const sourceRoots = baseCase.sourceRoots;
|
|
46
|
+
const controlledChangedPaths = scenario.mutation.files.map((file) => normalizeRelativePath(file.path)).sort();
|
|
47
|
+
async function prepareIndex(target, role, indexDir, mode, existingIndexDir = false) {
|
|
48
|
+
const treatmentId = target.treatmentId;
|
|
49
|
+
indexInvocationCounts[treatmentId] += 1;
|
|
50
|
+
const { evidence, build } = await prepareTreatmentIndexEvidence({
|
|
51
|
+
deps,
|
|
52
|
+
runOwnedRoot,
|
|
53
|
+
scenarioId: scenario.id,
|
|
54
|
+
kitCommand,
|
|
55
|
+
sourceRoots,
|
|
56
|
+
target,
|
|
57
|
+
role,
|
|
58
|
+
indexDir,
|
|
59
|
+
mode,
|
|
60
|
+
existingIndexDir,
|
|
61
|
+
onIndexPrepared: () => events.push(`${treatmentId}:${role}-index-built`, `${treatmentId}:${role}-snapshot-captured`, `${treatmentId}:${role}-graph-loaded`)
|
|
62
|
+
});
|
|
63
|
+
events.push(`${treatmentId}:${role}-identity-verified`);
|
|
64
|
+
return { evidence, build };
|
|
65
|
+
}
|
|
66
|
+
async function captureState(target) {
|
|
67
|
+
return deps.captureSourceState(target.targetRoot, sourceRoots, target.treatmentId, baseCase.benchmarkProjectId);
|
|
68
|
+
}
|
|
69
|
+
async function mutate(target) {
|
|
70
|
+
const receipt = await deps.executeMutation(scenario, target, sourceRoots, repoRoot);
|
|
71
|
+
if (receipt.status !== "applied") {
|
|
72
|
+
fail({ code: "mutation-failed", message: `${target.treatmentId} mutation was rejected: ${receipt.errors.join(" ")}`, treatmentId: target.treatmentId });
|
|
73
|
+
}
|
|
74
|
+
events.push(`${target.treatmentId}:mutation-applied`);
|
|
75
|
+
return receipt;
|
|
76
|
+
}
|
|
77
|
+
async function assess(index, target) {
|
|
78
|
+
freshnessAssessmentCounts[target.treatmentId] += 1;
|
|
79
|
+
return deps.assessIndexFreshness({ snapshot: index.snapshot, targetRoot: target.targetRoot, sourceRoots });
|
|
80
|
+
}
|
|
81
|
+
function compareAgainstReference(reference, states, code, phase) {
|
|
82
|
+
const comparisons = {};
|
|
83
|
+
for (const treatmentId of V2_TREATMENT_IDS) {
|
|
84
|
+
if (treatmentId === "stale-index")
|
|
85
|
+
continue;
|
|
86
|
+
const comparison = compareIncrementalChangeStalenessSourceStates(reference, states[treatmentId]);
|
|
87
|
+
comparisons[treatmentId] = comparison;
|
|
88
|
+
if (comparison.result !== "equivalent") {
|
|
89
|
+
fail({
|
|
90
|
+
code,
|
|
91
|
+
message: `${treatmentId} is not ${phase} equivalent to the stale-index reference (${comparison.result}): ${comparison.reasons.join("; ")}`,
|
|
92
|
+
treatmentId
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
return comparisons;
|
|
97
|
+
}
|
|
98
|
+
try {
|
|
99
|
+
const baselineDirs = {};
|
|
100
|
+
const refreshedDirs = {};
|
|
101
|
+
for (const treatmentId of V2_TREATMENT_IDS) {
|
|
102
|
+
baselineDirs[treatmentId] = incrementalChangeStalenessIndexDir(runOwnedRoot, scenario.id, treatmentId, "baseline");
|
|
103
|
+
refreshedDirs[treatmentId] = incrementalChangeStalenessIndexDir(runOwnedRoot, scenario.id, treatmentId, "refreshed");
|
|
104
|
+
}
|
|
105
|
+
const allDirs = [...Object.values(baselineDirs), ...Object.values(refreshedDirs)].map(normalizeRootForComparison);
|
|
106
|
+
if (new Set(allDirs).size !== allDirs.length) {
|
|
107
|
+
fail({ code: "index-directory-collision", message: "Treatment/phase index directories are not distinct.", treatmentId: null });
|
|
108
|
+
}
|
|
109
|
+
// 1. Four independent disposable copies through the established owner, in fixed order.
|
|
110
|
+
const targets = {};
|
|
111
|
+
for (const treatmentId of V2_TREATMENT_IDS) {
|
|
112
|
+
try {
|
|
113
|
+
targets[treatmentId] = await createDisposableTreatmentTarget({
|
|
114
|
+
repoRoot,
|
|
115
|
+
runRoot: incrementalChangeStalenessTargetsRoot(runOwnedRoot),
|
|
116
|
+
scenarioId: scenario.id,
|
|
117
|
+
treatmentId,
|
|
118
|
+
benchmarkProjectId: baseCase.benchmarkProjectId,
|
|
119
|
+
canonicalProjectRootRelative: baseCase.canonicalProjectRootRelative
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
catch (error) {
|
|
123
|
+
fail({ code: "treatment-target-creation-failed", message: `${treatmentId}: ${error.message}`, treatmentId });
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
const targetRootKeys = V2_TREATMENT_IDS.map((id) => normalizeRootForComparison(targets[id].targetRoot));
|
|
127
|
+
if (new Set(targetRootKeys).size !== targetRootKeys.length) {
|
|
128
|
+
fail({ code: "treatment-target-creation-failed", message: "Treatment target roots are not distinct.", treatmentId: null });
|
|
129
|
+
}
|
|
130
|
+
events.push("treatment-targets-created");
|
|
131
|
+
// 2. Pre-mutation equivalence gates all indexing and mutation; stale-index is only the comparison reference.
|
|
132
|
+
const preStates = {};
|
|
133
|
+
for (const treatmentId of V2_TREATMENT_IDS)
|
|
134
|
+
preStates[treatmentId] = await captureState(targets[treatmentId]);
|
|
135
|
+
events.push("pre-mutation-source-state-captured");
|
|
136
|
+
const preMutationEquivalence = compareAgainstReference(preStates["stale-index"], preStates, "pre-mutation-not-equivalent", "pre-mutation");
|
|
137
|
+
events.push("pre-mutation-equivalence-proven");
|
|
138
|
+
// 3. Baselines: every treatment bootstraps incrementally into a brand-new index directory.
|
|
139
|
+
const baselines = {};
|
|
140
|
+
const bootstraps = {};
|
|
141
|
+
for (const treatmentId of V2_TREATMENT_IDS) {
|
|
142
|
+
const { evidence, build } = await prepareIndex(targets[treatmentId], "baseline", baselineDirs[treatmentId], BOOTSTRAP_MODE);
|
|
143
|
+
fail(evaluateBaselineBootstrapEvidence(build, treatmentId));
|
|
144
|
+
bootstraps[treatmentId] = build.incrementalRefresh;
|
|
145
|
+
baselines[treatmentId] = withRefreshEvidence(evidence, build.mode, build.incrementalRefresh);
|
|
146
|
+
events.push(`${treatmentId}:baseline-bootstrap-verified`);
|
|
147
|
+
}
|
|
148
|
+
const baselineTools = evaluateToolIdentity(V2_TREATMENT_IDS.map((id) => baselines[id]));
|
|
149
|
+
fail(baselineTools.failure);
|
|
150
|
+
events.push("baseline-tool-identity-verified");
|
|
151
|
+
// 4. Preserve each partial treatment's immutable baseline; the refresh runs in place against a byte-for-byte clone.
|
|
152
|
+
for (const treatmentId of ["changed-files-refresh", "affected-neighborhood-refresh"]) {
|
|
153
|
+
try {
|
|
154
|
+
await cloneIndexDirectory(runOwnedRoot, baselineDirs[treatmentId], refreshedDirs[treatmentId]);
|
|
155
|
+
}
|
|
156
|
+
catch (error) {
|
|
157
|
+
fail({ code: "index-clone-failed", message: `${treatmentId} baseline index clone failed: ${error.message}`, treatmentId });
|
|
158
|
+
}
|
|
159
|
+
events.push(`${treatmentId}:baseline-index-cloned`);
|
|
160
|
+
}
|
|
161
|
+
// 5. Barrier: all four baselines are index + snapshot + graph READY and clones exist. Only now may mutation begin.
|
|
162
|
+
events.push("baseline-barrier-ready");
|
|
163
|
+
const receipts = {};
|
|
164
|
+
for (const treatmentId of V2_TREATMENT_IDS)
|
|
165
|
+
receipts[treatmentId] = await mutate(targets[treatmentId]);
|
|
166
|
+
// 6. Post-mutation equivalence.
|
|
167
|
+
const postStates = {};
|
|
168
|
+
for (const treatmentId of V2_TREATMENT_IDS)
|
|
169
|
+
postStates[treatmentId] = await captureState(targets[treatmentId]);
|
|
170
|
+
events.push("post-mutation-source-state-captured");
|
|
171
|
+
const postMutationEquivalence = compareAgainstReference(postStates["stale-index"], postStates, "post-mutation-not-equivalent", "post-mutation");
|
|
172
|
+
events.push("post-mutation-equivalence-proven");
|
|
173
|
+
// 7. Baseline (change-authority) freshness against each mutated target, symmetric with the reference.
|
|
174
|
+
const baselineFreshness = {};
|
|
175
|
+
for (const treatmentId of V2_TREATMENT_IDS) {
|
|
176
|
+
baselineFreshness[treatmentId] = await assess(baselines[treatmentId], targets[treatmentId]);
|
|
177
|
+
fail(evaluateBaselineStaleFreshness(baselineFreshness[treatmentId], controlledChangedPaths, treatmentId));
|
|
178
|
+
events.push(`${treatmentId}:baseline-freshness-stale`);
|
|
179
|
+
}
|
|
180
|
+
for (const treatmentId of V2_TREATMENT_IDS) {
|
|
181
|
+
if (treatmentId === "stale-index")
|
|
182
|
+
continue;
|
|
183
|
+
fail(evaluateChangedPathSymmetry(baselineFreshness["stale-index"], baselineFreshness[treatmentId], { reference: "stale-index", other: treatmentId }));
|
|
184
|
+
}
|
|
185
|
+
events.push("baseline-changed-paths-symmetric");
|
|
186
|
+
// 8. stale-index: no rebuild; baseline stays the active retrieval index.
|
|
187
|
+
events.push("stale-index:baseline-retained-as-active");
|
|
188
|
+
// 9. Partial treatments: refresh the cloned working index in place through the shared build owner.
|
|
189
|
+
const refreshed = {};
|
|
190
|
+
const refreshedFreshness = {};
|
|
191
|
+
const realizations = {};
|
|
192
|
+
for (const treatmentId of ["changed-files-refresh", "affected-neighborhood-refresh"]) {
|
|
193
|
+
const scope = PARTIAL_REFRESH_SCOPES[treatmentId];
|
|
194
|
+
const { evidence, build } = await prepareIndex(targets[treatmentId], "refreshed", refreshedDirs[treatmentId], { kind: "incremental", refreshScope: scope }, true);
|
|
195
|
+
const realization = evaluatePartialRefreshRealization(build, scope, treatmentId);
|
|
196
|
+
fail(realization.failure);
|
|
197
|
+
realizations[treatmentId] = realization.realization;
|
|
198
|
+
events.push(`${treatmentId}:refresh-realization-${realization.realization === "APPLIED_PARTIAL" ? "applied-partial" : "fallback-full"}`);
|
|
199
|
+
refreshed[treatmentId] = withRefreshEvidence(evidence, build.mode, build.incrementalRefresh);
|
|
200
|
+
refreshedFreshness[treatmentId] = await assess(evidence, targets[treatmentId]);
|
|
201
|
+
fail(evaluateRefreshedFreshness(refreshedFreshness[treatmentId], treatmentId));
|
|
202
|
+
events.push(`${treatmentId}:refreshed-freshness-fresh`);
|
|
203
|
+
}
|
|
204
|
+
// 10. full-refresh: a NEW complete post-mutation index in a separate directory (baseline is not cloned).
|
|
205
|
+
{
|
|
206
|
+
const { evidence, build } = await prepareIndex(targets["full-refresh"], "refreshed", refreshedDirs["full-refresh"], undefined);
|
|
207
|
+
refreshed["full-refresh"] = withRefreshEvidence(evidence, build.mode, build.incrementalRefresh);
|
|
208
|
+
refreshedFreshness["full-refresh"] = await assess(evidence, targets["full-refresh"]);
|
|
209
|
+
fail(evaluateRefreshedFreshness(refreshedFreshness["full-refresh"], "full-refresh"));
|
|
210
|
+
events.push("full-refresh:refreshed-freshness-fresh");
|
|
211
|
+
}
|
|
212
|
+
// 11. Scenario-level proofs.
|
|
213
|
+
const allTools = evaluateToolIdentity([...V2_TREATMENT_IDS.map((id) => baselines[id]), ...V2_TREATMENT_IDS.flatMap((id) => (refreshed[id] ? [refreshed[id]] : []))]);
|
|
214
|
+
fail(allTools.failure);
|
|
215
|
+
events.push("tool-identity-consistent");
|
|
216
|
+
fail(evaluateV2InvocationCounts(indexInvocationCounts, freshnessAssessmentCounts));
|
|
217
|
+
events.push("invocation-counts-verified");
|
|
218
|
+
events.push("lifecycle-ready");
|
|
219
|
+
const common = (treatmentId) => ({
|
|
220
|
+
treatmentIntent: INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_INTENTS[treatmentId],
|
|
221
|
+
target: targets[treatmentId],
|
|
222
|
+
preMutationSourceState: preStates[treatmentId],
|
|
223
|
+
postMutationSourceState: postStates[treatmentId],
|
|
224
|
+
mutationReceipt: receipts[treatmentId],
|
|
225
|
+
changeAuthority: Object.freeze({
|
|
226
|
+
baselineIndex: baselines[treatmentId],
|
|
227
|
+
baselineBootstrap: bootstraps[treatmentId],
|
|
228
|
+
postMutationBaselineFreshness: baselineFreshness[treatmentId]
|
|
229
|
+
}),
|
|
230
|
+
indexInvocationCount: indexInvocationCounts[treatmentId],
|
|
231
|
+
freshnessAssessmentCount: freshnessAssessmentCounts[treatmentId],
|
|
232
|
+
lifecycleStatus: "ready",
|
|
233
|
+
warnings: Object.freeze([])
|
|
234
|
+
});
|
|
235
|
+
const partialSession = (treatmentId) => Object.freeze({
|
|
236
|
+
...common(treatmentId),
|
|
237
|
+
treatmentId,
|
|
238
|
+
activeRetrieval: Object.freeze({ role: "refreshed", index: refreshed[treatmentId] }),
|
|
239
|
+
postMutationIndexBuilt: true,
|
|
240
|
+
refreshedIndex: refreshed[treatmentId],
|
|
241
|
+
refreshedFreshness: refreshedFreshness[treatmentId],
|
|
242
|
+
refreshRealization: realizations[treatmentId]
|
|
243
|
+
});
|
|
244
|
+
const staleSession = Object.freeze({
|
|
245
|
+
...common("stale-index"),
|
|
246
|
+
treatmentId: "stale-index",
|
|
247
|
+
activeRetrieval: Object.freeze({ role: "baseline", index: baselines["stale-index"] }),
|
|
248
|
+
postMutationIndexBuilt: false,
|
|
249
|
+
refreshedIndex: null,
|
|
250
|
+
refreshedFreshness: null,
|
|
251
|
+
refreshRealization: null
|
|
252
|
+
});
|
|
253
|
+
const fullSession = Object.freeze({
|
|
254
|
+
...common("full-refresh"),
|
|
255
|
+
treatmentId: "full-refresh",
|
|
256
|
+
activeRetrieval: Object.freeze({ role: "refreshed", index: refreshed["full-refresh"] }),
|
|
257
|
+
postMutationIndexBuilt: true,
|
|
258
|
+
refreshedIndex: refreshed["full-refresh"],
|
|
259
|
+
refreshedFreshness: refreshedFreshness["full-refresh"],
|
|
260
|
+
refreshRealization: null
|
|
261
|
+
});
|
|
262
|
+
const session = Object.freeze({
|
|
263
|
+
schemaVersion: INCREMENTAL_CHANGE_STALENESS_LIFECYCLE_SESSION_V2_SCHEMA_VERSION,
|
|
264
|
+
scenarioId: scenario.id,
|
|
265
|
+
scenario,
|
|
266
|
+
baseCase,
|
|
267
|
+
runOwnedRoot,
|
|
268
|
+
kitCommand,
|
|
269
|
+
toolIdentity: allTools.tool,
|
|
270
|
+
controlledChangedPaths: Object.freeze(controlledChangedPaths),
|
|
271
|
+
preMutationEquivalence: Object.freeze(preMutationEquivalence),
|
|
272
|
+
postMutationEquivalence: Object.freeze(postMutationEquivalence),
|
|
273
|
+
treatments: Object.freeze({
|
|
274
|
+
"stale-index": staleSession,
|
|
275
|
+
"changed-files-refresh": partialSession("changed-files-refresh"),
|
|
276
|
+
"affected-neighborhood-refresh": partialSession("affected-neighborhood-refresh"),
|
|
277
|
+
"full-refresh": fullSession
|
|
278
|
+
}),
|
|
279
|
+
totalIndexInvocationCount: V2_TREATMENT_IDS.reduce((sum, id) => sum + indexInvocationCounts[id], 0),
|
|
280
|
+
totalFreshnessAssessmentCount: V2_TREATMENT_IDS.reduce((sum, id) => sum + freshnessAssessmentCounts[id], 0),
|
|
281
|
+
lifecycleEvents: Object.freeze([...events]),
|
|
282
|
+
status: "ready"
|
|
283
|
+
});
|
|
284
|
+
return { status: "ready", session };
|
|
285
|
+
}
|
|
286
|
+
catch (error) {
|
|
287
|
+
const failure = error instanceof LifecycleFailure
|
|
288
|
+
? error.failure
|
|
289
|
+
: { code: "lifecycle-error", message: `Lifecycle failed unexpectedly: ${error instanceof Error ? error.message : String(error)}`, treatmentId: null };
|
|
290
|
+
return {
|
|
291
|
+
status: "failed",
|
|
292
|
+
scenarioId: scenario.id,
|
|
293
|
+
failure,
|
|
294
|
+
lifecycleEvents: Object.freeze([...events]),
|
|
295
|
+
indexInvocationCounts: Object.freeze({ ...indexInvocationCounts })
|
|
296
|
+
};
|
|
297
|
+
}
|
|
298
|
+
}
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
import { readFile, writeFile } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { resolveWithinDisposableTarget } from "./disposableTarget.js";
|
|
4
|
+
import { applyMutationOperations, isPathCoveredBySourceRoots, sha256Hex } from "./scenarioMutation.js";
|
|
5
|
+
export const INCREMENTAL_CHANGE_STALENESS_MUTATION_RECEIPT_SCHEMA_VERSION = "1.0.0";
|
|
6
|
+
function canonicalBenchmarkProjectsRoot(repoRoot) {
|
|
7
|
+
return path.resolve(repoRoot, "benchmarks/projects");
|
|
8
|
+
}
|
|
9
|
+
function isWithin(root, candidate) {
|
|
10
|
+
const relative = path.relative(root, candidate);
|
|
11
|
+
return relative !== "" && !relative.startsWith("..") && !path.isAbsolute(relative);
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Executes one scenario's already-validated mutation plan against ONE
|
|
15
|
+
* disposable treatment copy. Re-validates every guard at execution time
|
|
16
|
+
* (path containment/coverage, pre-hash, ordered exact-preimage replacement,
|
|
17
|
+
* post-hash) rather than trusting that Batch 1 catalog validation still
|
|
18
|
+
* holds, because a disposable copy can drift between creation and mutation.
|
|
19
|
+
*
|
|
20
|
+
* Two-phase: every file is fully validated and its post-mutation bytes
|
|
21
|
+
* computed in memory BEFORE any filesystem write begins. Writes only start
|
|
22
|
+
* once the complete plan is internally valid. If a later write/verification
|
|
23
|
+
* fails, files already written and verified remain recorded as such; the
|
|
24
|
+
* overall receipt status is "rejected" and no file is misreported as
|
|
25
|
+
* successfully mutated.
|
|
26
|
+
*/
|
|
27
|
+
export async function executeIncrementalChangeStalenessMutation(scenario, target, caseSourceRoots, repoRoot) {
|
|
28
|
+
const base = () => ({
|
|
29
|
+
schemaVersion: INCREMENTAL_CHANGE_STALENESS_MUTATION_RECEIPT_SCHEMA_VERSION,
|
|
30
|
+
scenarioId: scenario.id,
|
|
31
|
+
treatmentId: target.treatmentId,
|
|
32
|
+
benchmarkProjectId: target.benchmarkProjectId
|
|
33
|
+
});
|
|
34
|
+
const canonicalRoot = canonicalBenchmarkProjectsRoot(repoRoot);
|
|
35
|
+
if (isWithin(canonicalRoot, path.resolve(target.targetRoot)) || path.resolve(target.targetRoot) === canonicalRoot) {
|
|
36
|
+
return {
|
|
37
|
+
...base(),
|
|
38
|
+
status: "rejected",
|
|
39
|
+
files: [],
|
|
40
|
+
errors: [`Refusing to mutate a canonical benchmark target directly: ${target.targetRoot}.`]
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
const fileReceipts = [];
|
|
44
|
+
const plannedWrites = [];
|
|
45
|
+
const errors = [];
|
|
46
|
+
// Phase 1: validate every file and compute post-mutation bytes in memory. No writes yet.
|
|
47
|
+
for (const fileDeclaration of scenario.mutation.files) {
|
|
48
|
+
const label = `${scenario.id}/${target.treatmentId}: ${fileDeclaration.path}`;
|
|
49
|
+
const receipt = {
|
|
50
|
+
path: fileDeclaration.path,
|
|
51
|
+
expectedPreSha256: fileDeclaration.expectedPreSha256,
|
|
52
|
+
observedPreSha256: null,
|
|
53
|
+
operationCount: fileDeclaration.operations.length,
|
|
54
|
+
expectedPostSha256: fileDeclaration.expectedPostSha256,
|
|
55
|
+
observedPostSha256: null,
|
|
56
|
+
written: false
|
|
57
|
+
};
|
|
58
|
+
if (!isPathCoveredBySourceRoots(fileDeclaration.path, caseSourceRoots)) {
|
|
59
|
+
receipt.error = `${label}: path is not covered by the referenced case's indexed source roots.`;
|
|
60
|
+
errors.push(receipt.error);
|
|
61
|
+
fileReceipts.push(receipt);
|
|
62
|
+
return { ...base(), status: "rejected", files: fileReceipts, errors };
|
|
63
|
+
}
|
|
64
|
+
let absolutePath;
|
|
65
|
+
try {
|
|
66
|
+
absolutePath = await resolveWithinDisposableTarget(target, fileDeclaration.path);
|
|
67
|
+
}
|
|
68
|
+
catch (error) {
|
|
69
|
+
receipt.error = `${label}: ${error.message}`;
|
|
70
|
+
errors.push(receipt.error);
|
|
71
|
+
fileReceipts.push(receipt);
|
|
72
|
+
return { ...base(), status: "rejected", files: fileReceipts, errors };
|
|
73
|
+
}
|
|
74
|
+
let buffer;
|
|
75
|
+
try {
|
|
76
|
+
buffer = await readFile(absolutePath);
|
|
77
|
+
}
|
|
78
|
+
catch (error) {
|
|
79
|
+
receipt.error = `${label}: unable to read disposable file: ${error.message}`;
|
|
80
|
+
errors.push(receipt.error);
|
|
81
|
+
fileReceipts.push(receipt);
|
|
82
|
+
return { ...base(), status: "rejected", files: fileReceipts, errors };
|
|
83
|
+
}
|
|
84
|
+
const observedPreSha256 = sha256Hex(buffer);
|
|
85
|
+
receipt.observedPreSha256 = observedPreSha256;
|
|
86
|
+
if (observedPreSha256 !== fileDeclaration.expectedPreSha256) {
|
|
87
|
+
receipt.error = `${label}: expectedPreSha256 ${fileDeclaration.expectedPreSha256} does not match observed ${observedPreSha256}.`;
|
|
88
|
+
errors.push(receipt.error);
|
|
89
|
+
fileReceipts.push(receipt);
|
|
90
|
+
return { ...base(), status: "rejected", files: fileReceipts, errors };
|
|
91
|
+
}
|
|
92
|
+
let mutatedText;
|
|
93
|
+
try {
|
|
94
|
+
mutatedText = applyMutationOperations(buffer.toString("utf8"), fileDeclaration.operations);
|
|
95
|
+
}
|
|
96
|
+
catch (error) {
|
|
97
|
+
receipt.error = `${label}: ${error.message}`;
|
|
98
|
+
errors.push(receipt.error);
|
|
99
|
+
fileReceipts.push(receipt);
|
|
100
|
+
return { ...base(), status: "rejected", files: fileReceipts, errors };
|
|
101
|
+
}
|
|
102
|
+
const mutatedBuffer = Buffer.from(mutatedText, "utf8");
|
|
103
|
+
const computedPostSha256 = sha256Hex(mutatedBuffer);
|
|
104
|
+
if (computedPostSha256 !== fileDeclaration.expectedPostSha256) {
|
|
105
|
+
receipt.error = `${label}: expectedPostSha256 ${fileDeclaration.expectedPostSha256} does not match computed in-memory post-mutation SHA-256 ${computedPostSha256}.`;
|
|
106
|
+
errors.push(receipt.error);
|
|
107
|
+
fileReceipts.push(receipt);
|
|
108
|
+
return { ...base(), status: "rejected", files: fileReceipts, errors };
|
|
109
|
+
}
|
|
110
|
+
fileReceipts.push(receipt);
|
|
111
|
+
plannedWrites.push({ fileDeclaration, absolutePath, mutatedBuffer });
|
|
112
|
+
}
|
|
113
|
+
// Phase 2: every file validated. Write, then re-read/re-hash to verify the actual on-disk result.
|
|
114
|
+
for (const [index, planned] of plannedWrites.entries()) {
|
|
115
|
+
const receipt = fileReceipts[index];
|
|
116
|
+
const label = `${scenario.id}/${target.treatmentId}: ${planned.fileDeclaration.path}`;
|
|
117
|
+
try {
|
|
118
|
+
await writeFile(planned.absolutePath, planned.mutatedBuffer);
|
|
119
|
+
}
|
|
120
|
+
catch (error) {
|
|
121
|
+
receipt.error = `${label}: write failed: ${error.message}`;
|
|
122
|
+
errors.push(receipt.error);
|
|
123
|
+
return { ...base(), status: "rejected", files: fileReceipts, errors };
|
|
124
|
+
}
|
|
125
|
+
let rereadBuffer;
|
|
126
|
+
try {
|
|
127
|
+
rereadBuffer = await readFile(planned.absolutePath);
|
|
128
|
+
}
|
|
129
|
+
catch (error) {
|
|
130
|
+
receipt.error = `${label}: post-write re-read failed: ${error.message}`;
|
|
131
|
+
errors.push(receipt.error);
|
|
132
|
+
return { ...base(), status: "rejected", files: fileReceipts, errors };
|
|
133
|
+
}
|
|
134
|
+
const observedPostSha256 = sha256Hex(rereadBuffer);
|
|
135
|
+
receipt.observedPostSha256 = observedPostSha256;
|
|
136
|
+
if (observedPostSha256 !== planned.fileDeclaration.expectedPostSha256) {
|
|
137
|
+
receipt.error = `${label}: post-write SHA-256 ${observedPostSha256} does not match expected ${planned.fileDeclaration.expectedPostSha256}.`;
|
|
138
|
+
errors.push(receipt.error);
|
|
139
|
+
return { ...base(), status: "rejected", files: fileReceipts, errors };
|
|
140
|
+
}
|
|
141
|
+
receipt.written = true;
|
|
142
|
+
}
|
|
143
|
+
return { ...base(), status: "applied", files: fileReceipts, errors: [] };
|
|
144
|
+
}
|