@dailephd/my-dev-kit-lab 0.6.1 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/README.md +3 -2
- package/benchmarks/contracts/incremental-change-staleness-scenarios.json +360 -0
- package/dist/src/commands/runExperimentRunCommand.js +4 -3
- package/dist/src/evaluation/affectedNeighborhoodMetrics.js +51 -0
- package/dist/src/experiments/defaultRegistry.js +2 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/comparison.js +101 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/config.js +66 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/disposableTarget.js +107 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/execution.js +404 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/executionArtifact.js +160 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/index.js +13 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/lifecycle.js +276 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/lifecyclePolicy.js +152 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/mutationExecution.js +144 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/plugin.js +274 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/scenarioCatalog.js +42 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/scenarioMutation.js +116 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/scenarioSelection.js +79 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/scenarioTypes.js +37 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/scenarioValidation.js +168 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/sourceState.js +145 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/treatmentSession.js +6 -0
- package/dist/src/experiments/plugins/index.js +1 -0
- package/dist/src/experiments/plugins/warmIndexReuse/metrics.js +17 -20
- package/dist/src/report/experiments/buildIncrementalChangeStalenessReport.js +112 -0
- package/dist/src/report/experiments/buildPluginExperimentReport.js +3 -0
- package/dist/src/report/experiments/incrementalChangeStalenessReportModel.js +4 -0
- package/dist/src/report/experiments/index.js +3 -0
- package/dist/src/report/experiments/renderIncrementalChangeStalenessHtml.js +239 -0
- package/dist/src/report/experiments/renderPluginExperimentReportHtml.js +3 -0
- package/dist/src/report/experiments/renderPluginExperimentReportText.js +138 -0
- package/docs/METRICS.md +35 -1
- package/package.json +1 -1
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* v0.6.2 incremental-change-staleness scenario catalog schema version.
|
|
3
|
+
*
|
|
4
|
+
* This is a separate, versioned contract from the warm-index benchmark case
|
|
5
|
+
* contract (`benchmarks/contracts/warm-index-benchmark-cases.json`), which
|
|
6
|
+
* remains unchanged. Scenario records here reference existing benchmark
|
|
7
|
+
* project/case ids rather than duplicating canonical project/target metadata.
|
|
8
|
+
*/
|
|
9
|
+
export const INCREMENTAL_CHANGE_STALENESS_SCENARIO_SCHEMA_VERSION = "1.0.0";
|
|
10
|
+
/**
|
|
11
|
+
* The experimental dimension a scenario deliberately changes. Categories are
|
|
12
|
+
* not required to be structurally exclusive (for example, an exported
|
|
13
|
+
* function body is still implementation code); the selected category
|
|
14
|
+
* describes the intended experimental contrast, not an exhaustive structural
|
|
15
|
+
* classification.
|
|
16
|
+
*/
|
|
17
|
+
export const INCREMENTAL_CHANGE_STALENESS_SCENARIO_CATEGORIES = [
|
|
18
|
+
"unrelated-file-change",
|
|
19
|
+
"local-implementation-change",
|
|
20
|
+
"exported-symbol-change",
|
|
21
|
+
"public-api-change",
|
|
22
|
+
"import-graph-change",
|
|
23
|
+
"test-only-change"
|
|
24
|
+
];
|
|
25
|
+
/**
|
|
26
|
+
* Answer-key policy for a scenario's follow-up query:
|
|
27
|
+
* - `inherit`: reuse the referenced base case's existing query/answer key
|
|
28
|
+
* because the controlled change does not alter that task's semantic truth.
|
|
29
|
+
* - `scenario`: use a scenario-specific post-mutation query and answer key,
|
|
30
|
+
* expressed with the existing benchmark answer-key structure.
|
|
31
|
+
*
|
|
32
|
+
* Exactly these two policies are supported; no other correctness vocabulary
|
|
33
|
+
* is introduced.
|
|
34
|
+
*/
|
|
35
|
+
export const INCREMENTAL_CHANGE_STALENESS_ANSWER_POLICIES = ["inherit", "scenario"];
|
|
36
|
+
/** The six planner-frozen production scenario ids, in their planned order. */
|
|
37
|
+
export const FROZEN_INCREMENTAL_CHANGE_STALENESS_SCENARIO_IDS = ["U1", "L2", "E1", "P1", "I1", "T1"];
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
import { validateAnswerKey } from "../../../evaluation/benchmarkMetadata.js";
|
|
3
|
+
import { validateMutationFile } from "./scenarioMutation.js";
|
|
4
|
+
import { FROZEN_INCREMENTAL_CHANGE_STALENESS_SCENARIO_IDS, INCREMENTAL_CHANGE_STALENESS_ANSWER_POLICIES, INCREMENTAL_CHANGE_STALENESS_SCENARIO_CATEGORIES, INCREMENTAL_CHANGE_STALENESS_SCENARIO_SCHEMA_VERSION } from "./scenarioTypes.js";
|
|
5
|
+
/** Category each frozen production scenario id is required to declare. */
|
|
6
|
+
export const FROZEN_SCENARIO_CATEGORY_BY_ID = {
|
|
7
|
+
U1: "unrelated-file-change",
|
|
8
|
+
L2: "local-implementation-change",
|
|
9
|
+
E1: "exported-symbol-change",
|
|
10
|
+
P1: "public-api-change",
|
|
11
|
+
I1: "import-graph-change",
|
|
12
|
+
T1: "test-only-change"
|
|
13
|
+
};
|
|
14
|
+
/** Substrings that must never appear in scenario notes/query text (Section 7/9/22.I). */
|
|
15
|
+
const FORBIDDEN_TEXT_PATTERNS = [/partial-refresh/i, /changed-files-refresh/i, /affected-neighborhood-refresh/i, /graph-diff/i];
|
|
16
|
+
/**
|
|
17
|
+
* Strictly validates a parsed incremental-change-staleness scenario catalog.
|
|
18
|
+
* Performs schema/vocabulary checks, benchmark project/case reference checks,
|
|
19
|
+
* answer-policy completeness checks, forbidden-content checks, and full
|
|
20
|
+
* in-memory mutation validation (pre-hash, ordered exact-preimage
|
|
21
|
+
* replacement, post-hash) against the real canonical benchmark files. Never
|
|
22
|
+
* writes to disk.
|
|
23
|
+
*/
|
|
24
|
+
export async function validateIncrementalChangeStalenessCatalog(catalog, profiles, cases, repoRoot, options = {}) {
|
|
25
|
+
const errors = [];
|
|
26
|
+
if (!catalog || typeof catalog !== "object") {
|
|
27
|
+
return ["Scenario catalog must be an object."];
|
|
28
|
+
}
|
|
29
|
+
const candidate = catalog;
|
|
30
|
+
if (candidate.schemaVersion !== INCREMENTAL_CHANGE_STALENESS_SCENARIO_SCHEMA_VERSION) {
|
|
31
|
+
errors.push(`schemaVersion must be ${INCREMENTAL_CHANGE_STALENESS_SCENARIO_SCHEMA_VERSION}; received ${JSON.stringify(candidate.schemaVersion)}.`);
|
|
32
|
+
}
|
|
33
|
+
if (!Array.isArray(candidate.scenarios)) {
|
|
34
|
+
errors.push("scenarios must be an array.");
|
|
35
|
+
return errors;
|
|
36
|
+
}
|
|
37
|
+
const profilesById = new Map(profiles.map((profile) => [profile.projectId, profile]));
|
|
38
|
+
const casesById = new Map(cases.map((benchmarkCase) => [benchmarkCase.id, benchmarkCase]));
|
|
39
|
+
const seenIds = new Set();
|
|
40
|
+
for (const [index, scenario] of candidate.scenarios.entries()) {
|
|
41
|
+
errors.push(...(await validateOneScenario(scenario, index, profilesById, casesById, repoRoot, seenIds)));
|
|
42
|
+
}
|
|
43
|
+
if (options.requireFrozenProductionScenarios === true) {
|
|
44
|
+
const actualIds = candidate.scenarios
|
|
45
|
+
.map((scenario) => (scenario && typeof scenario === "object" ? scenario.id : undefined))
|
|
46
|
+
.filter((id) => typeof id === "string");
|
|
47
|
+
const expected = FROZEN_INCREMENTAL_CHANGE_STALENESS_SCENARIO_IDS;
|
|
48
|
+
const actualSet = new Set(actualIds);
|
|
49
|
+
const missing = expected.filter((id) => !actualSet.has(id));
|
|
50
|
+
const extra = actualIds.filter((id) => !expected.includes(id));
|
|
51
|
+
if (missing.length > 0) {
|
|
52
|
+
errors.push(`Production catalog is missing required frozen scenario(s): ${missing.join(", ")}.`);
|
|
53
|
+
}
|
|
54
|
+
if (extra.length > 0) {
|
|
55
|
+
errors.push(`Production catalog contains unexpected scenario id(s) beyond the six frozen scenarios: ${extra.join(", ")}.`);
|
|
56
|
+
}
|
|
57
|
+
if (actualIds.length !== expected.length || missing.length > 0 || extra.length > 0) {
|
|
58
|
+
errors.push(`Production catalog must contain exactly the six frozen scenarios ${expected.join(", ")}; received ${actualIds.join(", ") || "<none>"}.`);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
return errors;
|
|
62
|
+
}
|
|
63
|
+
async function validateOneScenario(scenario, index, profilesById, casesById, repoRoot, seenIds) {
|
|
64
|
+
const errors = [];
|
|
65
|
+
if (!scenario || typeof scenario !== "object") {
|
|
66
|
+
return [`scenario at index ${index}: must be an object.`];
|
|
67
|
+
}
|
|
68
|
+
const candidate = scenario;
|
|
69
|
+
const id = typeof candidate.id === "string" && candidate.id.length > 0 ? candidate.id : undefined;
|
|
70
|
+
const label = id === undefined ? `scenario at index ${index}` : `scenario ${id}`;
|
|
71
|
+
if (id === undefined) {
|
|
72
|
+
errors.push(`${label}: id must be a nonempty string.`);
|
|
73
|
+
}
|
|
74
|
+
else if (seenIds.has(id)) {
|
|
75
|
+
errors.push(`${label}: duplicate scenario id.`);
|
|
76
|
+
}
|
|
77
|
+
else {
|
|
78
|
+
seenIds.add(id);
|
|
79
|
+
}
|
|
80
|
+
if (typeof candidate.category !== "string" || !INCREMENTAL_CHANGE_STALENESS_SCENARIO_CATEGORIES.includes(candidate.category)) {
|
|
81
|
+
errors.push(`${label}: category must be one of ${INCREMENTAL_CHANGE_STALENESS_SCENARIO_CATEGORIES.join(", ")}; received ${JSON.stringify(candidate.category)}.`);
|
|
82
|
+
}
|
|
83
|
+
else if (id !== undefined && id in FROZEN_SCENARIO_CATEGORY_BY_ID) {
|
|
84
|
+
const expectedCategory = FROZEN_SCENARIO_CATEGORY_BY_ID[id];
|
|
85
|
+
if (candidate.category !== expectedCategory) {
|
|
86
|
+
errors.push(`${label}: category must be ${expectedCategory} for frozen scenario ${id}; received ${candidate.category}.`);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
if (typeof candidate.answerPolicy !== "string" || !INCREMENTAL_CHANGE_STALENESS_ANSWER_POLICIES.includes(candidate.answerPolicy)) {
|
|
90
|
+
errors.push(`${label}: answerPolicy must be one of ${INCREMENTAL_CHANGE_STALENESS_ANSWER_POLICIES.join(", ")}; received ${JSON.stringify(candidate.answerPolicy)}.`);
|
|
91
|
+
}
|
|
92
|
+
const projectId = typeof candidate.benchmarkProjectId === "string" && candidate.benchmarkProjectId.length > 0 ? candidate.benchmarkProjectId : undefined;
|
|
93
|
+
if (projectId === undefined) {
|
|
94
|
+
errors.push(`${label}: benchmarkProjectId must be a nonempty string.`);
|
|
95
|
+
}
|
|
96
|
+
const profile = projectId === undefined ? undefined : profilesById.get(projectId);
|
|
97
|
+
if (projectId !== undefined && profile === undefined) {
|
|
98
|
+
errors.push(`${label}: unknown benchmarkProjectId ${projectId}.`);
|
|
99
|
+
}
|
|
100
|
+
const caseId = typeof candidate.baseCaseId === "string" && candidate.baseCaseId.length > 0 ? candidate.baseCaseId : undefined;
|
|
101
|
+
if (caseId === undefined) {
|
|
102
|
+
errors.push(`${label}: baseCaseId must be a nonempty string.`);
|
|
103
|
+
}
|
|
104
|
+
const baseCase = caseId === undefined ? undefined : casesById.get(caseId);
|
|
105
|
+
if (caseId !== undefined && baseCase === undefined) {
|
|
106
|
+
errors.push(`${label}: unknown baseCaseId ${caseId}.`);
|
|
107
|
+
}
|
|
108
|
+
if (baseCase !== undefined && projectId !== undefined && baseCase.benchmarkProject !== projectId) {
|
|
109
|
+
errors.push(`${label}: baseCaseId ${caseId} belongs to benchmark project ${baseCase.benchmarkProject}, not the referenced benchmarkProjectId ${projectId}.`);
|
|
110
|
+
}
|
|
111
|
+
// Answer-policy completeness (Section 22.H). Reuses the existing answer-key structure/validator.
|
|
112
|
+
if (candidate.answerPolicy === "inherit") {
|
|
113
|
+
if (baseCase !== undefined) {
|
|
114
|
+
if (typeof baseCase.query !== "string" || baseCase.query.length === 0) {
|
|
115
|
+
errors.push(`${label}: inherit policy requires the base case to have a nonempty query.`);
|
|
116
|
+
}
|
|
117
|
+
if (baseCase.answerKey === undefined) {
|
|
118
|
+
errors.push(`${label}: inherit policy requires the base case to have an answerKey.`);
|
|
119
|
+
}
|
|
120
|
+
else {
|
|
121
|
+
errors.push(...validateAnswerKey(baseCase.answerKey, `${label} (inherited answerKey)`).map((message) => message));
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
if (candidate.scenarioQuery !== undefined || candidate.scenarioAnswerKey !== undefined) {
|
|
125
|
+
errors.push(`${label}: inherit policy must not declare scenarioQuery or scenarioAnswerKey.`);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
else if (candidate.answerPolicy === "scenario") {
|
|
129
|
+
if (typeof candidate.scenarioQuery !== "string" || candidate.scenarioQuery.trim().length === 0) {
|
|
130
|
+
errors.push(`${label}: scenario policy requires a nonempty scenarioQuery.`);
|
|
131
|
+
}
|
|
132
|
+
if (candidate.scenarioAnswerKey === undefined) {
|
|
133
|
+
errors.push(`${label}: scenario policy requires a scenarioAnswerKey.`);
|
|
134
|
+
}
|
|
135
|
+
else {
|
|
136
|
+
errors.push(...validateAnswerKey(candidate.scenarioAnswerKey, `${label}.scenarioAnswerKey`));
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
if (candidate.expectedRelationship !== undefined && !["related", "unrelated", "unknown"].includes(candidate.expectedRelationship)) {
|
|
140
|
+
errors.push(`${label}: expectedRelationship must be one of related, unrelated, unknown; received ${JSON.stringify(candidate.expectedRelationship)}.`);
|
|
141
|
+
}
|
|
142
|
+
for (const text of [candidate.notes, candidate.scenarioQuery, candidate.id, candidate.category]) {
|
|
143
|
+
if (typeof text === "string") {
|
|
144
|
+
for (const pattern of FORBIDDEN_TEXT_PATTERNS) {
|
|
145
|
+
if (pattern.test(text)) {
|
|
146
|
+
errors.push(`${label}: forbidden v0.6.3+/graph-diff reference found in scenario text: matched ${pattern}.`);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
if (!candidate.mutation || typeof candidate.mutation !== "object" || !Array.isArray(candidate.mutation.files) || candidate.mutation.files.length === 0) {
|
|
152
|
+
errors.push(`${label}: mutation.files must be a nonempty array.`);
|
|
153
|
+
return errors;
|
|
154
|
+
}
|
|
155
|
+
if (profile !== undefined && baseCase !== undefined) {
|
|
156
|
+
const projectAbsoluteRoot = path.resolve(repoRoot, profile.rootPath);
|
|
157
|
+
const caseSourceRoots = Array.isArray(baseCase.sourceRoots) ? baseCase.sourceRoots : [];
|
|
158
|
+
for (const [fileIndex, file] of candidate.mutation.files.entries()) {
|
|
159
|
+
const fileLabel = `${label}.mutation.files[${fileIndex}]`;
|
|
160
|
+
const result = await validateMutationFile(fileLabel, file, projectAbsoluteRoot, caseSourceRoots);
|
|
161
|
+
errors.push(...result.errors);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
else {
|
|
165
|
+
errors.push(`${label}: mutation files cannot be validated because the benchmark project/case reference is invalid.`);
|
|
166
|
+
}
|
|
167
|
+
return errors;
|
|
168
|
+
}
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { readdir, readFile } from "node:fs/promises";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { resolveWithinRoot } from "../../../core/pathSafety.js";
|
|
5
|
+
import { sha256Hex } from "./scenarioMutation.js";
|
|
6
|
+
/**
|
|
7
|
+
* v0.6.2 Batch 2 controlled-source-state schema version. This is a
|
|
8
|
+
* copy-equivalence guard scoped to a benchmark case's configured indexed
|
|
9
|
+
* source roots; it is NOT the real `IndexSnapshotV1` captured from an actual
|
|
10
|
+
* my-dev-kit index build (that remains Batch 3's responsibility).
|
|
11
|
+
*/
|
|
12
|
+
export const INCREMENTAL_CHANGE_STALENESS_SOURCE_STATE_SCHEMA_VERSION = "1.0.0";
|
|
13
|
+
function normalizeSourceRoot(root) {
|
|
14
|
+
return root.replace(/\\/g, "/").replace(/\/+$/, "");
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Recursively enumerates regular files (symbolic links are skipped, never
|
|
18
|
+
* followed) underneath one project-relative source root, hashing each with
|
|
19
|
+
* SHA-256. Deterministic: results are always returned to the caller sorted.
|
|
20
|
+
*/
|
|
21
|
+
async function collectSourceRootFiles(projectRoot, sourceRoot) {
|
|
22
|
+
const absoluteRoot = resolveWithinRoot(projectRoot, sourceRoot);
|
|
23
|
+
const results = [];
|
|
24
|
+
async function walk(currentAbsoluteDir) {
|
|
25
|
+
const entries = await readdir(currentAbsoluteDir, { withFileTypes: true });
|
|
26
|
+
for (const entry of entries) {
|
|
27
|
+
if (entry.isSymbolicLink()) {
|
|
28
|
+
// Never follow symlinks when building controlled source-state evidence.
|
|
29
|
+
continue;
|
|
30
|
+
}
|
|
31
|
+
const entryAbsolutePath = path.join(currentAbsoluteDir, entry.name);
|
|
32
|
+
if (entry.isDirectory()) {
|
|
33
|
+
await walk(entryAbsolutePath);
|
|
34
|
+
}
|
|
35
|
+
else if (entry.isFile()) {
|
|
36
|
+
const buffer = await readFile(entryAbsolutePath);
|
|
37
|
+
const relativePath = path.relative(projectRoot, entryAbsolutePath).split(path.sep).join("/");
|
|
38
|
+
results.push({ relativePath, sha256: sha256Hex(buffer) });
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
await walk(absoluteRoot);
|
|
43
|
+
return results;
|
|
44
|
+
}
|
|
45
|
+
function computeDigest(files) {
|
|
46
|
+
const serialized = files.map((file) => `${file.relativePath}\t${file.sha256}\n`).join("");
|
|
47
|
+
return createHash("sha256").update(Buffer.from(serialized, "utf8")).digest("hex");
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Captures bounded, deterministic controlled source-state evidence for one
|
|
51
|
+
* disposable treatment copy, scoped to the base case's own configured
|
|
52
|
+
* `sourceRoots` (never invented separately). Never converts an unreadable or
|
|
53
|
+
* missing source root into an empty/zero-file "complete" result: any
|
|
54
|
+
* enumeration failure makes the whole capture "unavailable" with a reason.
|
|
55
|
+
*/
|
|
56
|
+
export async function captureIncrementalChangeStalenessSourceState(projectRoot, sourceRoots, treatmentId, benchmarkProjectId) {
|
|
57
|
+
const normalizedSourceRoots = sourceRoots.map(normalizeSourceRoot);
|
|
58
|
+
if (normalizedSourceRoots.length === 0) {
|
|
59
|
+
return {
|
|
60
|
+
schemaVersion: INCREMENTAL_CHANGE_STALENESS_SOURCE_STATE_SCHEMA_VERSION,
|
|
61
|
+
treatmentId,
|
|
62
|
+
benchmarkProjectId,
|
|
63
|
+
sourceRoots: normalizedSourceRoots,
|
|
64
|
+
status: "unavailable",
|
|
65
|
+
files: [],
|
|
66
|
+
fileCount: 0,
|
|
67
|
+
digest: null,
|
|
68
|
+
reason: "No source roots were configured for this benchmark case."
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
try {
|
|
72
|
+
const perRoot = await Promise.all(normalizedSourceRoots.map((root) => collectSourceRootFiles(projectRoot, root)));
|
|
73
|
+
const files = perRoot.flat().sort((left, right) => left.relativePath.localeCompare(right.relativePath));
|
|
74
|
+
return {
|
|
75
|
+
schemaVersion: INCREMENTAL_CHANGE_STALENESS_SOURCE_STATE_SCHEMA_VERSION,
|
|
76
|
+
treatmentId,
|
|
77
|
+
benchmarkProjectId,
|
|
78
|
+
sourceRoots: normalizedSourceRoots,
|
|
79
|
+
status: "complete",
|
|
80
|
+
files,
|
|
81
|
+
fileCount: files.length,
|
|
82
|
+
digest: computeDigest(files)
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
catch (error) {
|
|
86
|
+
return {
|
|
87
|
+
schemaVersion: INCREMENTAL_CHANGE_STALENESS_SOURCE_STATE_SCHEMA_VERSION,
|
|
88
|
+
treatmentId,
|
|
89
|
+
benchmarkProjectId,
|
|
90
|
+
sourceRoots: normalizedSourceRoots,
|
|
91
|
+
status: "unavailable",
|
|
92
|
+
files: [],
|
|
93
|
+
fileCount: 0,
|
|
94
|
+
digest: null,
|
|
95
|
+
reason: `Unable to capture controlled source state: ${error.message}`
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Deterministically compares two controlled source-state captures.
|
|
101
|
+
* `equivalent` requires both sides complete, matching normalized source-root
|
|
102
|
+
* sets, matching path sets, and matching per-file SHA-256 values.
|
|
103
|
+
* Incomplete/unavailable evidence on either side always yields `unknown`,
|
|
104
|
+
* never `equivalent` and never a false `different`.
|
|
105
|
+
*/
|
|
106
|
+
export function compareIncrementalChangeStalenessSourceStates(left, right) {
|
|
107
|
+
const reasons = [];
|
|
108
|
+
if (left.status !== "complete") {
|
|
109
|
+
reasons.push(`left source state is unavailable${left.reason ? `: ${left.reason}` : "."}`);
|
|
110
|
+
}
|
|
111
|
+
if (right.status !== "complete") {
|
|
112
|
+
reasons.push(`right source state is unavailable${right.reason ? `: ${right.reason}` : "."}`);
|
|
113
|
+
}
|
|
114
|
+
if (reasons.length > 0) {
|
|
115
|
+
return { result: "unknown", reasons };
|
|
116
|
+
}
|
|
117
|
+
const leftRoots = [...left.sourceRoots].sort();
|
|
118
|
+
const rightRoots = [...right.sourceRoots].sort();
|
|
119
|
+
if (leftRoots.length !== rightRoots.length || leftRoots.some((root, index) => root !== rightRoots[index])) {
|
|
120
|
+
return {
|
|
121
|
+
result: "different",
|
|
122
|
+
reasons: [`source-root contract differs: left=${JSON.stringify(leftRoots)} right=${JSON.stringify(rightRoots)}`]
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
const leftByPath = new Map(left.files.map((file) => [file.relativePath, file.sha256]));
|
|
126
|
+
const rightByPath = new Map(right.files.map((file) => [file.relativePath, file.sha256]));
|
|
127
|
+
const differences = [];
|
|
128
|
+
for (const [relativePath, sha256] of leftByPath) {
|
|
129
|
+
if (!rightByPath.has(relativePath)) {
|
|
130
|
+
differences.push(`only on left: ${relativePath}`);
|
|
131
|
+
}
|
|
132
|
+
else if (rightByPath.get(relativePath) !== sha256) {
|
|
133
|
+
differences.push(`content differs: ${relativePath}`);
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
for (const relativePath of rightByPath.keys()) {
|
|
137
|
+
if (!leftByPath.has(relativePath)) {
|
|
138
|
+
differences.push(`only on right: ${relativePath}`);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
if (differences.length > 0) {
|
|
142
|
+
return { result: "different", reasons: differences };
|
|
143
|
+
}
|
|
144
|
+
return { result: "equivalent", reasons: [] };
|
|
145
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* v0.6.2 Batch 3 in-memory lifecycle session schema version. This is the
|
|
3
|
+
* runtime handoff contract Batch 4 consumes for treatment execution; it is
|
|
4
|
+
* NOT a persisted artifact and carries full graph evidence only in memory.
|
|
5
|
+
*/
|
|
6
|
+
export const INCREMENTAL_CHANGE_STALENESS_LIFECYCLE_SESSION_SCHEMA_VERSION = "1.0.0";
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { toAffectedNeighborhoodMetricValues } from "../../../evaluation/affectedNeighborhoodMetrics.js";
|
|
1
2
|
// ---------------------------------------------------------------------------
|
|
2
3
|
// Warm-index metric owner. Every warm-index formula (amortization, cumulative
|
|
3
4
|
// prefix sums, availability) is calculated here exactly once, as a pure
|
|
@@ -194,32 +195,28 @@ function warmDirect(task) {
|
|
|
194
195
|
};
|
|
195
196
|
}
|
|
196
197
|
const NO_AFFECTED_NEIGHBORHOOD_REASON = "No affected-neighborhood assessment was recorded for this task.";
|
|
198
|
+
function wrapAffectedNeighborhoodMetricValue(value) {
|
|
199
|
+
return value.availability === "available"
|
|
200
|
+
? availableMetric(value.value, value.unit, "derived")
|
|
201
|
+
: unavailableMetric(value.unit, "derived", value.reason);
|
|
202
|
+
}
|
|
197
203
|
/**
|
|
198
204
|
* Converts the persisted affected-neighborhood assessment into the six warm-side metrics, once.
|
|
199
205
|
* A finite value (including zero) is available; an absent assessment or a null field is
|
|
200
|
-
* unavailable with a reason, never zero. The
|
|
206
|
+
* unavailable with a reason, never zero. The formula itself is owned by the shared
|
|
207
|
+
* `toAffectedNeighborhoodMetricValues` (src/evaluation/affectedNeighborhoodMetrics.ts, extracted in
|
|
208
|
+
* v0.6.2 Batch 4 for reuse by incremental-change-staleness); this function only wraps its bounded
|
|
209
|
+
* values into the warm-index `WarmIndexNumberMetricV1` envelope.
|
|
201
210
|
*/
|
|
202
211
|
function affectedNeighborhoodMetrics(task) {
|
|
203
|
-
const
|
|
204
|
-
const one = (field, label) => {
|
|
205
|
-
const unit = field === "taskOverlapPercent" ? "percent" : "count";
|
|
206
|
-
if (!assessment)
|
|
207
|
-
return unavailableMetric(unit, "derived", NO_AFFECTED_NEIGHBORHOOD_REASON);
|
|
208
|
-
const value = assessment[field];
|
|
209
|
-
if (typeof value === "number")
|
|
210
|
-
return availableMetric(value, unit, "derived");
|
|
211
|
-
const context = `assessment ${assessment.status}; seed mapping ${assessment.seedMappingStatus}; ` +
|
|
212
|
-
`graph ${assessment.graphEvidenceStatus}; neighborhood ${assessment.neighborhoodStatus}; task mapping ${assessment.taskMapping.status}` +
|
|
213
|
-
(field === "taskOverlapPercent" ? `; resolvable task nodes ${assessment.taskMapping.resolvableTaskNodeCount}` : "");
|
|
214
|
-
return unavailableMetric(unit, "derived", `Affected-neighborhood evidence did not establish ${label} (${context}).`);
|
|
215
|
-
};
|
|
212
|
+
const values = toAffectedNeighborhoodMetricValues(task.affectedNeighborhood, NO_AFFECTED_NEIGHBORHOOD_REASON);
|
|
216
213
|
return {
|
|
217
|
-
changedFileCount:
|
|
218
|
-
changedSymbolCount:
|
|
219
|
-
affectedNodeCount:
|
|
220
|
-
affectedEdgeCount:
|
|
221
|
-
taskOverlapCount:
|
|
222
|
-
taskOverlapPercent:
|
|
214
|
+
changedFileCount: wrapAffectedNeighborhoodMetricValue(values.changedFileCount),
|
|
215
|
+
changedSymbolCount: wrapAffectedNeighborhoodMetricValue(values.changedSymbolCount),
|
|
216
|
+
affectedNodeCount: wrapAffectedNeighborhoodMetricValue(values.affectedNodeCount),
|
|
217
|
+
affectedEdgeCount: wrapAffectedNeighborhoodMetricValue(values.affectedEdgeCount),
|
|
218
|
+
taskOverlapCount: wrapAffectedNeighborhoodMetricValue(values.taskOverlapCount),
|
|
219
|
+
taskOverlapPercent: wrapAffectedNeighborhoodMetricValue(values.taskOverlapPercent)
|
|
223
220
|
};
|
|
224
221
|
}
|
|
225
222
|
/** Pure, order-preserving calculation; ordinals and cumulative state restart for every project. */
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
// v0.6.2 Batch 5 -- projects the persisted Batch 4 incremental-change-staleness-execution.json
|
|
2
|
+
// artifact into the plugin report model. This is a deterministic projection only: no retrieval,
|
|
3
|
+
// no fake-agent evaluation, no correctness scoring, no affected-neighborhood assessment, no
|
|
4
|
+
// freshness assessment, no stale-risk classification, and no source/index/benchmark reads happen
|
|
5
|
+
// here. The persisted artifact is the only plugin-specific evidence source (frozen batch plan
|
|
6
|
+
// Section 4); this file's only I/O is reading that one JSON file.
|
|
7
|
+
import { readFileSync } from "node:fs";
|
|
8
|
+
import { INCREMENTAL_CHANGE_STALENESS_EXECUTION_SCHEMA_VERSION } from "../../experiments/plugins/incrementalChangeStaleness/executionArtifact.js";
|
|
9
|
+
import { INCREMENTAL_CHANGE_STALENESS_PLUGIN_ID } from "../../experiments/plugins/incrementalChangeStaleness/plugin.js";
|
|
10
|
+
import { INCREMENTAL_CHANGE_STALENESS_REPORT_SCHEMA_VERSION } from "./incrementalChangeStalenessReportModel.js";
|
|
11
|
+
// Frozen wording (Section 25). Every point must appear in report.txt and report.html.
|
|
12
|
+
const LIMITATIONS = [
|
|
13
|
+
"Results are scoped to the executed benchmark scenarios, deterministic controlled changes, configured my-dev-kit version, and observed retrieval/evaluation evidence.",
|
|
14
|
+
"\"no-observed-stale-regression\" means only that the frozen comparison did not observe a stale-specific regression in that matched run. It does not prove stale indexes are generally safe.",
|
|
15
|
+
"\"reindexRecommendation\" is separate affected-neighborhood observational evidence; it did not select or trigger either experiment treatment.",
|
|
16
|
+
"Required-file evidence is a bounded answer-key file-presence check, not retrieval precision/recall.",
|
|
17
|
+
"Correctness is deterministic fake-agent/harness evidence unless a future version explicitly adds real-agent evidence.",
|
|
18
|
+
"v0.6.2 compares stale full-index reuse with complete full refresh only.",
|
|
19
|
+
"Partial refresh is not evaluated in v0.6.2.",
|
|
20
|
+
"The experiment does not use graph-diff.",
|
|
21
|
+
"\"changedSymbolCount\" does not mean source-level symbol-diff proof; it retains the released v0.6.1 definition."
|
|
22
|
+
];
|
|
23
|
+
/** Fixed neutral wording per stale-risk classification category (Section 21). Presentation only. */
|
|
24
|
+
export const STALE_RISK_CLASSIFICATION_EXPLANATIONS = {
|
|
25
|
+
"observed-stale-regression": "The persisted matched comparison observed at least one stale-specific regression under the frozen v0.6.2 evidence dimensions: correctness and/or required-file presence.",
|
|
26
|
+
"no-observed-stale-regression": "This matched scenario/run did not observe a stale-specific regression under the frozen correctness and required-file evidence dimensions. This does not establish general stale-index safety and does not imply that reindexing should be skipped.",
|
|
27
|
+
inconclusive: "Available persisted evidence was insufficient to determine whether a stale-specific regression occurred under the frozen comparison rules. Inconclusive is not the same as no regression."
|
|
28
|
+
};
|
|
29
|
+
/**
|
|
30
|
+
* Builds the plugin report section for an incremental-change-staleness run, or `null` for any
|
|
31
|
+
* other plugin. Every field is a deterministic projection of the persisted Batch 4 execution
|
|
32
|
+
* artifact; this function performs no scientific recomputation of its own.
|
|
33
|
+
*/
|
|
34
|
+
export function buildIncrementalChangeStalenessReport(run) {
|
|
35
|
+
if (run.pluginId !== INCREMENTAL_CHANGE_STALENESS_PLUGIN_ID)
|
|
36
|
+
return null;
|
|
37
|
+
const artifact = loadExecutionArtifact(run);
|
|
38
|
+
const scenarios = artifact.scenarios.map(toReportScenario);
|
|
39
|
+
return {
|
|
40
|
+
schemaVersion: INCREMENTAL_CHANGE_STALENESS_REPORT_SCHEMA_VERSION,
|
|
41
|
+
pluginId: artifact.pluginId,
|
|
42
|
+
scenarioCount: artifact.summary.scenarioCount,
|
|
43
|
+
readyScenarioCount: artifact.summary.readyScenarioCount,
|
|
44
|
+
failedScenarioCount: artifact.summary.failedScenarioCount,
|
|
45
|
+
observedStaleRegressionCount: artifact.summary.observedStaleRegressionCount,
|
|
46
|
+
noObservedStaleRegressionCount: artifact.summary.noObservedStaleRegressionCount,
|
|
47
|
+
inconclusiveCount: artifact.summary.inconclusiveCount,
|
|
48
|
+
scenarios,
|
|
49
|
+
limitations: [...LIMITATIONS]
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
function toReportScenario(record) {
|
|
53
|
+
return {
|
|
54
|
+
scenarioId: record.scenarioId,
|
|
55
|
+
category: record.category,
|
|
56
|
+
benchmarkProjectId: record.benchmarkProjectId,
|
|
57
|
+
baseCaseId: record.baseCaseId,
|
|
58
|
+
status: record.status,
|
|
59
|
+
failureReason: record.failureReason,
|
|
60
|
+
query: record.query,
|
|
61
|
+
answerPolicy: record.answerPolicy,
|
|
62
|
+
expectedFiles: [...record.expectedFiles],
|
|
63
|
+
expectedSymbols: [...record.expectedSymbols],
|
|
64
|
+
lifecycle: record.lifecycle
|
|
65
|
+
? {
|
|
66
|
+
...record.lifecycle,
|
|
67
|
+
controlledFilePaths: [...record.lifecycle.controlledFilePaths],
|
|
68
|
+
sourceRoots: [...record.lifecycle.sourceRoots]
|
|
69
|
+
}
|
|
70
|
+
: null,
|
|
71
|
+
// Section 12/34: always stale-index then full-refresh, never combined, never resorted.
|
|
72
|
+
staleTreatment: record.stale ? structuredClone(record.stale) : null,
|
|
73
|
+
fullRefreshTreatment: record.fullRefresh ? structuredClone(record.fullRefresh) : null,
|
|
74
|
+
comparison: { ...record.comparison, reasonCodes: [...record.comparison.reasonCodes] }
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
function readArtifactPath(run) {
|
|
78
|
+
const fromMetadata = run.metadata?.executionArtifactPath;
|
|
79
|
+
if (typeof fromMetadata === "string" && fromMetadata)
|
|
80
|
+
return fromMetadata;
|
|
81
|
+
return run.artifacts.find((artifact) => artifact.id === "incremental-change-staleness-execution")?.path ?? undefined;
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* The persisted execution artifact is the sole plugin-specific evidence source for this report
|
|
85
|
+
* (Section 4/46). A missing, unreadable, malformed, or schema-mismatched artifact is a report
|
|
86
|
+
* construction error, never silently reconstructed from source/index/retrieval state.
|
|
87
|
+
*/
|
|
88
|
+
function loadExecutionArtifact(run) {
|
|
89
|
+
const artifactPath = readArtifactPath(run);
|
|
90
|
+
if (!artifactPath) {
|
|
91
|
+
throw new Error("Invalid incremental-change-staleness report source: no execution artifact path was found on the run.");
|
|
92
|
+
}
|
|
93
|
+
let raw;
|
|
94
|
+
try {
|
|
95
|
+
raw = readFileSync(artifactPath, "utf8");
|
|
96
|
+
}
|
|
97
|
+
catch (error) {
|
|
98
|
+
throw new Error(`Invalid incremental-change-staleness report source: execution artifact could not be read at ${artifactPath}: ${error instanceof Error ? error.message : String(error)}`);
|
|
99
|
+
}
|
|
100
|
+
let parsed;
|
|
101
|
+
try {
|
|
102
|
+
parsed = JSON.parse(raw);
|
|
103
|
+
}
|
|
104
|
+
catch {
|
|
105
|
+
throw new Error(`Invalid incremental-change-staleness report source: execution artifact at ${artifactPath} is not valid JSON.`);
|
|
106
|
+
}
|
|
107
|
+
const candidate = parsed;
|
|
108
|
+
if (candidate.schemaVersion !== INCREMENTAL_CHANGE_STALENESS_EXECUTION_SCHEMA_VERSION) {
|
|
109
|
+
throw new Error(`Invalid incremental-change-staleness report source: unsupported execution artifact schema version ${String(candidate.schemaVersion)}.`);
|
|
110
|
+
}
|
|
111
|
+
return candidate;
|
|
112
|
+
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import path from "node:path";
|
|
2
2
|
import { buildContextStrategyComparisonV043Report } from "./buildContextStrategyComparisonV043Report.js";
|
|
3
3
|
import { buildWarmIndexReuseReport } from "./buildWarmIndexReuseReport.js";
|
|
4
|
+
import { buildIncrementalChangeStalenessReport } from "./buildIncrementalChangeStalenessReport.js";
|
|
4
5
|
const V043_BULK_ARRAY_KEYS = [
|
|
5
6
|
"v043StageContextExecutions",
|
|
6
7
|
"v043StageContextEvaluations",
|
|
@@ -11,6 +12,7 @@ export function buildPluginExperimentReport(args) {
|
|
|
11
12
|
const allOutcomes = args.run.cases.flatMap((experimentCase) => experimentCase.outcomes);
|
|
12
13
|
const contextStrategyComparisonV043 = buildContextStrategyComparisonV043Report(args.run);
|
|
13
14
|
const warmIndexReuse = buildWarmIndexReuseReport(args.run);
|
|
15
|
+
const incrementalChangeStaleness = buildIncrementalChangeStalenessReport(args.run);
|
|
14
16
|
const rawRun = { ...args.run, artifacts: relativizeArtifacts(args.run.artifacts, outputRoot) };
|
|
15
17
|
for (const key of V043_BULK_ARRAY_KEYS) {
|
|
16
18
|
delete rawRun[key];
|
|
@@ -39,6 +41,7 @@ export function buildPluginExperimentReport(args) {
|
|
|
39
41
|
skippedOutcomes: allOutcomes.filter((outcome) => outcome.status === "skipped"),
|
|
40
42
|
findings: buildFindings(args.run),
|
|
41
43
|
warmIndexReuse,
|
|
44
|
+
incrementalChangeStaleness,
|
|
42
45
|
contextStrategyComparisonV043,
|
|
43
46
|
interpretation: buildInterpretation(args.run, contextStrategyComparisonV043, warmIndexReuse),
|
|
44
47
|
rawRun,
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
// v0.6.2 Batch 5 -- plugin-specific presentation model types only. No projection logic, no HTML,
|
|
2
|
+
// no text rendering, and no recomputation live here; see buildIncrementalChangeStalenessReport.ts
|
|
3
|
+
// and renderIncrementalChangeStalenessHtml.ts for those responsibilities.
|
|
4
|
+
export const INCREMENTAL_CHANGE_STALENESS_REPORT_SCHEMA_VERSION = "my-dev-kit-lab-incremental-change-staleness-report-v1";
|
|
@@ -14,3 +14,6 @@ export { buildContextIntegrityReport } from "./buildContextIntegrityReport.js";
|
|
|
14
14
|
export { renderContextIntegrityJsonReport } from "./renderContextIntegrityJsonReport.js";
|
|
15
15
|
export { renderContextIntegrityText } from "./renderContextIntegrityText.js";
|
|
16
16
|
export { renderContextIntegrityHtml } from "./renderContextIntegrityHtml.js";
|
|
17
|
+
export * from "./incrementalChangeStalenessReportModel.js";
|
|
18
|
+
export { buildIncrementalChangeStalenessReport, STALE_RISK_CLASSIFICATION_EXPLANATIONS } from "./buildIncrementalChangeStalenessReport.js";
|
|
19
|
+
export { renderIncrementalChangeStalenessHtml } from "./renderIncrementalChangeStalenessHtml.js";
|