@dailephd/my-dev-kit-lab 0.6.2 → 0.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/README.md +9 -8
- package/dist/src/cli/help.js +16 -9
- package/dist/src/commands/runExperimentDescribeCommand.js +5 -2
- package/dist/src/commands/runExperimentRunCommand.js +0 -4
- package/dist/src/evaluation/benchmarkMetadata.js +5 -1
- package/dist/src/evaluation/parseAgentAnswer.js +172 -24
- package/dist/src/evaluation/runMyDevKitRetrieval.js +120 -7
- package/dist/src/experiments/plugins/incrementalChangeStaleness/comparisonV2.js +49 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/config.js +5 -4
- package/dist/src/experiments/plugins/incrementalChangeStaleness/disposableTarget.js +20 -2
- package/dist/src/experiments/plugins/incrementalChangeStaleness/execution.js +42 -16
- package/dist/src/experiments/plugins/incrementalChangeStaleness/executionArtifact.js +4 -4
- package/dist/src/experiments/plugins/incrementalChangeStaleness/executionArtifactV2.js +289 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/executionV2.js +195 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/indexClone.js +76 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/lifecycle.js +64 -32
- package/dist/src/experiments/plugins/incrementalChangeStaleness/lifecyclePolicy.js +6 -4
- package/dist/src/experiments/plugins/incrementalChangeStaleness/lifecyclePolicyV2.js +78 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/lifecycleV2.js +298 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/plugin.js +213 -12
- package/dist/src/experiments/plugins/incrementalChangeStaleness/treatmentSessionV2.js +5 -0
- package/dist/src/experiments/plugins/warmIndexReuse/warmIndexSession.js +3 -1
- package/dist/src/mobile/android/advancedSecurity/candidateEvidence.js +9 -2
- package/dist/src/mobile/android/advancedSecurity/redaction.js +19 -10
- package/dist/src/mobile/android/advancedSecurity/secretCandidates/analyzeSecretCandidates.js +12 -2
- package/dist/src/mobile/android/advancedSecurity/signingConfiguration/analyzeSigningConfiguration.js +1 -1
- package/dist/src/mobile/android/advancedSecurity/signingConfiguration/extractSigningConfigurations.js +7 -7
- package/dist/src/report/experiments/buildIncrementalChangeStalenessPluginReport.js +33 -0
- package/dist/src/report/experiments/buildIncrementalChangeStalenessReport.js +1 -1
- package/dist/src/report/experiments/buildIncrementalChangeStalenessReportV2.js +80 -0
- package/dist/src/report/experiments/buildPluginExperimentReport.js +2 -2
- package/dist/src/report/experiments/incrementalChangeStalenessReportModelV2.js +3 -0
- package/dist/src/report/experiments/incrementalChangeStalenessReportRowsV2.js +146 -0
- package/dist/src/report/experiments/index.js +3 -0
- package/dist/src/report/experiments/renderIncrementalChangeStalenessHtml.js +4 -0
- package/dist/src/report/experiments/renderIncrementalChangeStalenessHtmlV2.js +111 -0
- package/dist/src/report/experiments/renderIncrementalChangeStalenessTextV2.js +96 -0
- package/dist/src/report/experiments/renderPluginExperimentReportText.js +6 -0
- package/dist/src/securityValidation/staticScans/codeql.js +8 -8
- package/docs/METRICS.md +32 -0
- package/examples/tutorial-browser/prepare.mjs +13 -13
- package/examples/tutorial-browser/server.mjs +13 -13
- package/package.json +1 -1
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { compareCorrectness, compareRequiredFileEvidence } from "./comparison.js";
|
|
2
|
+
function neutral(relation) {
|
|
3
|
+
if (relation === "stale-worse")
|
|
4
|
+
return "candidate-worse";
|
|
5
|
+
if (relation === "stale-better")
|
|
6
|
+
return "candidate-better";
|
|
7
|
+
return relation;
|
|
8
|
+
}
|
|
9
|
+
/** Same rules as the v0.6.2 correctness comparison, candidate vs reference. */
|
|
10
|
+
export function compareCandidateCorrectness(candidate, reference) {
|
|
11
|
+
return neutral(compareCorrectness(candidate, reference));
|
|
12
|
+
}
|
|
13
|
+
/** Same rules as the v0.6.2 required-file comparison, candidate vs reference. */
|
|
14
|
+
export function compareCandidateRequiredFileEvidence(candidate, reference) {
|
|
15
|
+
return neutral(compareRequiredFileEvidence(candidate, reference));
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Frozen precedence. Rule 0 (realization gate) first: a full fallback is never evidence about
|
|
19
|
+
* partial-refresh performance. Then observed regression (either dimension worse), then inconclusive
|
|
20
|
+
* (a dimension unknown), then no observed regression (including candidate-better, which is not a
|
|
21
|
+
* winner claim). affected-neighborhood evidence and reindexRecommendation are never inputs.
|
|
22
|
+
*/
|
|
23
|
+
export function classifyPartialRefreshAgainstFull(args) {
|
|
24
|
+
const { realization, correctnessRelation, requiredFileEvidenceRelation } = args;
|
|
25
|
+
// RULE 0 -- realization gate.
|
|
26
|
+
if (realization === "FALLBACK_FULL") {
|
|
27
|
+
return { classification: "not-comparable-as-partial-refresh", reasonCodes: ["candidate-fell-back-to-full"] };
|
|
28
|
+
}
|
|
29
|
+
// RULE 1 -- observed regression wins even if the other dimension is unknown.
|
|
30
|
+
if (correctnessRelation === "candidate-worse" || requiredFileEvidenceRelation === "candidate-worse") {
|
|
31
|
+
const reasonCodes = [];
|
|
32
|
+
if (correctnessRelation === "candidate-worse")
|
|
33
|
+
reasonCodes.push("candidate-correctness-lower");
|
|
34
|
+
if (requiredFileEvidenceRelation === "candidate-worse")
|
|
35
|
+
reasonCodes.push("candidate-missing-required-file");
|
|
36
|
+
return { classification: "observed-regression-relative-to-full", reasonCodes };
|
|
37
|
+
}
|
|
38
|
+
// RULE 2 -- inconclusive.
|
|
39
|
+
if (correctnessRelation === "unknown" || requiredFileEvidenceRelation === "unknown") {
|
|
40
|
+
const reasonCodes = [];
|
|
41
|
+
if (correctnessRelation === "unknown")
|
|
42
|
+
reasonCodes.push("correctness-unavailable");
|
|
43
|
+
if (requiredFileEvidenceRelation === "unknown")
|
|
44
|
+
reasonCodes.push("required-file-evidence-unavailable");
|
|
45
|
+
return { classification: "inconclusive", reasonCodes };
|
|
46
|
+
}
|
|
47
|
+
// RULE 3 -- no observed regression.
|
|
48
|
+
return { classification: "no-observed-regression-relative-to-full", reasonCodes: ["no-reference-regression-observed"] };
|
|
49
|
+
}
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import { invalidExperimentConfig, isPlainObject, mergeConfig, validExperimentConfig } from "../../config.js";
|
|
2
2
|
import { FROZEN_INCREMENTAL_CHANGE_STALENESS_SCENARIO_IDS } from "./scenarioTypes.js";
|
|
3
3
|
/**
|
|
4
|
-
* The
|
|
5
|
-
*
|
|
4
|
+
* The upstream tool version the four-treatment experiment expects: 1.12.5 is the first published
|
|
5
|
+
* my-dev-kit that exposes the incremental refresh-scope options. Every index of one matched scenario
|
|
6
|
+
* must come from this one configured command.
|
|
6
7
|
*/
|
|
7
|
-
export const INCREMENTAL_CHANGE_STALENESS_DEFAULT_KIT_COMMAND = "npx @dailephd/my-dev-kit@1.12.
|
|
8
|
+
export const INCREMENTAL_CHANGE_STALENESS_DEFAULT_KIT_COMMAND = "npx @dailephd/my-dev-kit@1.12.5";
|
|
8
9
|
export const defaultIncrementalChangeStalenessConfig = {
|
|
9
10
|
outDir: "lab-output/incremental-change-staleness",
|
|
10
11
|
kitCommand: INCREMENTAL_CHANGE_STALENESS_DEFAULT_KIT_COMMAND
|
|
@@ -17,7 +18,7 @@ export const incrementalChangeStalenessConfigDefinition = {
|
|
|
17
18
|
{
|
|
18
19
|
name: "caseIds",
|
|
19
20
|
type: "array",
|
|
20
|
-
description: `Frozen scenario ids to include (${FROZEN_INCREMENTAL_CHANGE_STALENESS_SCENARIO_IDS.join(", ")}); defaults to all.
|
|
21
|
+
description: `Frozen scenario ids to include (${FROZEN_INCREMENTAL_CHANGE_STALENESS_SCENARIO_IDS.join(", ")}); defaults to all. All four treatments run for each selected scenario: stale-index, changed-files-refresh, affected-neighborhood-refresh, and full-refresh.`
|
|
21
22
|
}
|
|
22
23
|
]
|
|
23
24
|
};
|
|
@@ -8,6 +8,24 @@ import { resolveWithinRoot } from "../../../core/pathSafety.js";
|
|
|
8
8
|
* target copies; it never builds a treatment index or runs retrieval.
|
|
9
9
|
*/
|
|
10
10
|
export const INCREMENTAL_CHANGE_STALENESS_TREATMENT_IDS = ["stale-index", "full-refresh"];
|
|
11
|
+
/**
|
|
12
|
+
* The four v0.6.3 internal lifecycle treatment identities in their fixed order
|
|
13
|
+
* (never alphabetical). The released v0.6.2 public constant above is unchanged;
|
|
14
|
+
* this set is used only by the internal v0.6.3 lifecycle.
|
|
15
|
+
*/
|
|
16
|
+
export const INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_IDS = [
|
|
17
|
+
"stale-index",
|
|
18
|
+
"changed-files-refresh",
|
|
19
|
+
"affected-neighborhood-refresh",
|
|
20
|
+
"full-refresh"
|
|
21
|
+
];
|
|
22
|
+
/** Treatment ID and treatment intent are separate runtime evidence. */
|
|
23
|
+
export const INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_INTENTS = {
|
|
24
|
+
"stale-index": "my-dev-kit-no-refresh",
|
|
25
|
+
"changed-files-refresh": "my-dev-kit-changed-files-refresh",
|
|
26
|
+
"affected-neighborhood-refresh": "my-dev-kit-affected-neighborhood-refresh",
|
|
27
|
+
"full-refresh": "my-dev-kit-full-refresh"
|
|
28
|
+
};
|
|
11
29
|
/**
|
|
12
30
|
* Default project-relative, gitignored runtime location for disposable
|
|
13
31
|
* treatment targets. Callers may pass a different `runRoot`, but it must
|
|
@@ -38,8 +56,8 @@ function assertDisposableRootDoesNotOverlapCanonical(targetRoot, canonicalProjec
|
|
|
38
56
|
*/
|
|
39
57
|
export async function createDisposableTreatmentTarget(options) {
|
|
40
58
|
const { repoRoot, scenarioId, treatmentId, benchmarkProjectId, canonicalProjectRootRelative } = options;
|
|
41
|
-
if (!
|
|
42
|
-
throw new Error(`Unknown treatment id: ${treatmentId}. Expected one of ${
|
|
59
|
+
if (!INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_IDS.includes(treatmentId)) {
|
|
60
|
+
throw new Error(`Unknown treatment id: ${treatmentId}. Expected one of ${INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_IDS.join(", ")}.`);
|
|
43
61
|
}
|
|
44
62
|
if (typeof scenarioId !== "string" || scenarioId.length === 0) {
|
|
45
63
|
throw new Error("scenarioId must be a nonempty string.");
|
|
@@ -224,13 +224,13 @@ export async function runIncrementalChangeStalenessFakeAgent(args) {
|
|
|
224
224
|
};
|
|
225
225
|
}
|
|
226
226
|
}
|
|
227
|
-
function correctnessComparable(fakeAgent) {
|
|
227
|
+
export function correctnessComparable(fakeAgent) {
|
|
228
228
|
if (!fakeAgent || !fakeAgent.correctness.available || fakeAgent.correctness.score === null) {
|
|
229
229
|
return { available: false };
|
|
230
230
|
}
|
|
231
231
|
return { available: true, score: fakeAgent.correctness.score };
|
|
232
232
|
}
|
|
233
|
-
const INCONCLUSIVE_NO_EXECUTION = {
|
|
233
|
+
export const INCONCLUSIVE_NO_EXECUTION = {
|
|
234
234
|
correctnessRelation: "unknown",
|
|
235
235
|
requiredFileEvidenceRelation: "unknown",
|
|
236
236
|
staleRiskClassification: "inconclusive",
|
|
@@ -240,10 +240,12 @@ export const defaultIncrementalChangeStalenessExecutionDeps = {
|
|
|
240
240
|
runRetrieval: runIncrementalChangeStalenessTreatmentRetrieval,
|
|
241
241
|
runFakeAgent: runIncrementalChangeStalenessFakeAgent
|
|
242
242
|
};
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
243
|
+
/**
|
|
244
|
+
* Retrieval + deterministic fake-agent + required-file evidence for ONE treatment against its own active
|
|
245
|
+
* index. Shared by the v0.6.2 two-treatment path and the v0.6.3 four-treatment path; takes only the
|
|
246
|
+
* lifecycle-provided facts it needs so it never reaches into another treatment's state.
|
|
247
|
+
*/
|
|
248
|
+
export async function executeTreatmentEvaluation(args) {
|
|
247
249
|
let retrieval = null;
|
|
248
250
|
let retrievalStatus = "not-run";
|
|
249
251
|
let failureReason = null;
|
|
@@ -251,7 +253,7 @@ async function executeOneTreatment(args) {
|
|
|
251
253
|
retrieval = await args.deps.runRetrieval({
|
|
252
254
|
derivedTask: args.derivedTask,
|
|
253
255
|
kitCommand: args.kitCommand,
|
|
254
|
-
indexDir:
|
|
256
|
+
indexDir: args.activeIndexDir,
|
|
255
257
|
commandsDir: args.retrievalCommandsDir
|
|
256
258
|
});
|
|
257
259
|
retrievalStatus = classifyIncrementalChangeStalenessRetrieval(retrieval);
|
|
@@ -284,8 +286,8 @@ async function executeOneTreatment(args) {
|
|
|
284
286
|
treatmentId: args.treatmentId,
|
|
285
287
|
status,
|
|
286
288
|
failureReason,
|
|
287
|
-
activeIndexPhase,
|
|
288
|
-
baselineFreshness,
|
|
289
|
+
activeIndexPhase: args.activeIndexPhase,
|
|
290
|
+
baselineFreshness: args.baselineFreshness,
|
|
289
291
|
affectedNeighborhood: args.affectedNeighborhood,
|
|
290
292
|
retrieval,
|
|
291
293
|
retrievalStatus,
|
|
@@ -293,6 +295,36 @@ async function executeOneTreatment(args) {
|
|
|
293
295
|
requiredFileEvidence
|
|
294
296
|
};
|
|
295
297
|
}
|
|
298
|
+
async function executeOneTreatment(args) {
|
|
299
|
+
const treatment = args.session.treatments[args.treatmentId];
|
|
300
|
+
return executeTreatmentEvaluation({
|
|
301
|
+
treatmentId: args.treatmentId,
|
|
302
|
+
baselineFreshness: treatment.changeAuthority.postMutationBaselineFreshness,
|
|
303
|
+
activeIndexPhase: treatment.activeRetrieval.role,
|
|
304
|
+
activeIndexDir: treatment.activeRetrieval.index.indexDir,
|
|
305
|
+
derivedTask: args.derivedTask,
|
|
306
|
+
affectedNeighborhood: args.affectedNeighborhood,
|
|
307
|
+
requiredFiles: args.requiredFiles,
|
|
308
|
+
kitCommand: args.kitCommand,
|
|
309
|
+
retrievalCommandsDir: args.retrievalCommandsDir,
|
|
310
|
+
agentOutDir: args.agentOutDir,
|
|
311
|
+
projectProfiles: args.projectProfiles,
|
|
312
|
+
cwd: args.cwd,
|
|
313
|
+
env: args.env,
|
|
314
|
+
deps: args.deps
|
|
315
|
+
});
|
|
316
|
+
}
|
|
317
|
+
/** Resolves the scenario's query/answer and derives the one in-memory task shared by every matched treatment. */
|
|
318
|
+
export async function resolveIncrementalChangeStalenessScenarioTask(options) {
|
|
319
|
+
const { scenario } = options;
|
|
320
|
+
const resolved = await resolveIncrementalChangeStalenessQueryAndAnswer({ scenario, repoRoot: options.repoRoot });
|
|
321
|
+
const cases = await readEvaluationCases(path.resolve(options.repoRoot, WARM_INDEX_BENCHMARK_CASES_PATH), options.repoRoot);
|
|
322
|
+
const baseEvaluationCase = cases.find((candidate) => candidate.id === scenario.baseCaseId);
|
|
323
|
+
if (!baseEvaluationCase) {
|
|
324
|
+
throw new Error(`Scenario ${scenario.id}: base case ${scenario.baseCaseId} was not found for task-descriptor construction.`);
|
|
325
|
+
}
|
|
326
|
+
return { resolved, derivedTask: buildIncrementalChangeStalenessDerivedTask({ scenario, baseEvaluationCase, resolved }) };
|
|
327
|
+
}
|
|
296
328
|
/**
|
|
297
329
|
* Executes the frozen Batch 4 order (section 36) for one ready lifecycle session: resolve query and
|
|
298
330
|
* answer, derive the task descriptor, assess both treatments' affected neighborhoods from BASELINE
|
|
@@ -323,13 +355,7 @@ export async function executeIncrementalChangeStalenessScenario(options) {
|
|
|
323
355
|
};
|
|
324
356
|
}
|
|
325
357
|
const session = options.lifecycle.session;
|
|
326
|
-
const resolved = await
|
|
327
|
-
const cases = await readEvaluationCases(path.resolve(options.repoRoot, WARM_INDEX_BENCHMARK_CASES_PATH), options.repoRoot);
|
|
328
|
-
const baseEvaluationCase = cases.find((candidate) => candidate.id === scenario.baseCaseId);
|
|
329
|
-
if (!baseEvaluationCase) {
|
|
330
|
-
throw new Error(`Scenario ${scenario.id}: base case ${scenario.baseCaseId} was not found for task-descriptor construction.`);
|
|
331
|
-
}
|
|
332
|
-
const derivedTask = buildIncrementalChangeStalenessDerivedTask({ scenario, baseEvaluationCase, resolved });
|
|
358
|
+
const { resolved, derivedTask } = await resolveIncrementalChangeStalenessScenarioTask({ repoRoot: options.repoRoot, scenario });
|
|
333
359
|
const staleAffected = assessIncrementalChangeStalenessAffectedNeighborhood(session.treatments["stale-index"], derivedTask);
|
|
334
360
|
const fullAffected = assessIncrementalChangeStalenessAffectedNeighborhood(session.treatments["full-refresh"], derivedTask);
|
|
335
361
|
const symmetry = checkIncrementalChangeStalenessAffectedNeighborhoodSymmetry(staleAffected, fullAffected);
|
|
@@ -9,7 +9,7 @@ import { resolveWithinRoot } from "../../../core/pathSafety.js";
|
|
|
9
9
|
// ---------------------------------------------------------------------------
|
|
10
10
|
export const INCREMENTAL_CHANGE_STALENESS_EXECUTION_ARTIFACT_FILE = "incremental-change-staleness-execution.json";
|
|
11
11
|
export const INCREMENTAL_CHANGE_STALENESS_EXECUTION_SCHEMA_VERSION = "my-dev-kit-lab-incremental-change-staleness-execution-v1";
|
|
12
|
-
function sortedUnique(values) {
|
|
12
|
+
export function sortedUnique(values) {
|
|
13
13
|
return [...new Set(values)].sort((left, right) => (left < right ? -1 : left > right ? 1 : 0));
|
|
14
14
|
}
|
|
15
15
|
function summarizeCommand(command) {
|
|
@@ -24,7 +24,7 @@ function summarizeCommand(command) {
|
|
|
24
24
|
error: command.error ?? null
|
|
25
25
|
};
|
|
26
26
|
}
|
|
27
|
-
function summarizeRetrieval(treatment) {
|
|
27
|
+
export function summarizeRetrieval(treatment) {
|
|
28
28
|
const retrieval = treatment.retrieval;
|
|
29
29
|
if (!retrieval) {
|
|
30
30
|
return {
|
|
@@ -56,7 +56,7 @@ function summarizeRetrieval(treatment) {
|
|
|
56
56
|
commands: retrieval.commands.map(summarizeCommand)
|
|
57
57
|
};
|
|
58
58
|
}
|
|
59
|
-
function summarizeFakeAgent(treatment) {
|
|
59
|
+
export function summarizeFakeAgent(treatment) {
|
|
60
60
|
if (!treatment.fakeAgent)
|
|
61
61
|
return null;
|
|
62
62
|
const agent = treatment.fakeAgent;
|
|
@@ -69,7 +69,7 @@ function summarizeFakeAgent(treatment) {
|
|
|
69
69
|
errors: [...agent.errors]
|
|
70
70
|
};
|
|
71
71
|
}
|
|
72
|
-
function summarizeRequiredFileEvidence(evidence) {
|
|
72
|
+
export function summarizeRequiredFileEvidence(evidence) {
|
|
73
73
|
return {
|
|
74
74
|
status: evidence.status,
|
|
75
75
|
requiredFiles: sortedUnique(evidence.requiredFiles),
|
|
@@ -0,0 +1,289 @@
|
|
|
1
|
+
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
2
|
+
import { resolveWithinRoot } from "../../../core/pathSafety.js";
|
|
3
|
+
import { classifyStaleRisk } from "./comparison.js";
|
|
4
|
+
import { classifyPartialRefreshAgainstFull } from "./comparisonV2.js";
|
|
5
|
+
import { INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_IDS, INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_INTENTS } from "./disposableTarget.js";
|
|
6
|
+
import { INCREMENTAL_CHANGE_STALENESS_EXECUTION_ARTIFACT_FILE, INCREMENTAL_CHANGE_STALENESS_EXECUTION_SCHEMA_VERSION, sortedUnique, summarizeFakeAgent, summarizeRequiredFileEvidence, summarizeRetrieval } from "./executionArtifact.js";
|
|
7
|
+
// ---------------------------------------------------------------------------
|
|
8
|
+
// v0.6.3 Batch 3 -- persisted four-treatment execution artifact. A distinct schema from V1 written to the
|
|
9
|
+
// same canonical filename; the schemaVersion is the discriminator. Bounded structured evidence only:
|
|
10
|
+
// never raw retrieved context bodies, graph artifacts, or file contents. Descriptive counts, no ranking.
|
|
11
|
+
// ---------------------------------------------------------------------------
|
|
12
|
+
export const INCREMENTAL_CHANGE_STALENESS_EXECUTION_SCHEMA_VERSION_V2 = "my-dev-kit-lab-incremental-change-staleness-execution-v2";
|
|
13
|
+
const PARTIAL_IDS = ["changed-files-refresh", "affected-neighborhood-refresh"];
|
|
14
|
+
const REQUESTED_SCOPE = {
|
|
15
|
+
"changed-files-refresh": "changed-files",
|
|
16
|
+
"affected-neighborhood-refresh": "affected-neighborhood"
|
|
17
|
+
};
|
|
18
|
+
const COMPARISON_CANDIDATE_ORDER = ["stale-index", "changed-files-refresh", "affected-neighborhood-refresh"];
|
|
19
|
+
function record(build) {
|
|
20
|
+
return Object.fromEntries(INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_IDS.map((id) => [id, build(id)]));
|
|
21
|
+
}
|
|
22
|
+
function summarizeTreatmentV2(treatment) {
|
|
23
|
+
return {
|
|
24
|
+
treatmentId: treatment.treatmentId,
|
|
25
|
+
treatmentIntent: treatment.treatmentIntent,
|
|
26
|
+
status: treatment.status,
|
|
27
|
+
failureReason: treatment.failureReason,
|
|
28
|
+
activeIndexPhase: treatment.activeIndexPhase,
|
|
29
|
+
baselineFreshness: structuredClone(treatment.baselineFreshness),
|
|
30
|
+
refreshedFreshness: treatment.refreshedFreshness ? structuredClone(treatment.refreshedFreshness) : null,
|
|
31
|
+
refreshExecution: structuredClone(treatment.refreshExecution),
|
|
32
|
+
affectedNeighborhood: structuredClone(treatment.affectedNeighborhood),
|
|
33
|
+
// Bounded status/telemetry only; never embeds retrieval.contextText.
|
|
34
|
+
retrieval: summarizeRetrieval(treatment),
|
|
35
|
+
fakeAgent: summarizeFakeAgent(treatment),
|
|
36
|
+
requiredFileEvidence: summarizeRequiredFileEvidence(treatment.requiredFileEvidence)
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
function summarizeLifecycleV2(execution) {
|
|
40
|
+
const session = execution.session;
|
|
41
|
+
if (!session)
|
|
42
|
+
return null;
|
|
43
|
+
const refreshedStatus = {};
|
|
44
|
+
for (const id of INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_IDS) {
|
|
45
|
+
const freshness = session.treatments[id].refreshedFreshness;
|
|
46
|
+
if (freshness)
|
|
47
|
+
refreshedStatus[id] = freshness.status;
|
|
48
|
+
}
|
|
49
|
+
return {
|
|
50
|
+
controlledFilePaths: [...session.controlledChangedPaths],
|
|
51
|
+
treatmentIntents: record((id) => INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_INTENTS[id]),
|
|
52
|
+
preMutationEquivalence: {
|
|
53
|
+
"changed-files-refresh": session.preMutationEquivalence["changed-files-refresh"].result,
|
|
54
|
+
"affected-neighborhood-refresh": session.preMutationEquivalence["affected-neighborhood-refresh"].result,
|
|
55
|
+
"full-refresh": session.preMutationEquivalence["full-refresh"].result
|
|
56
|
+
},
|
|
57
|
+
postMutationEquivalence: {
|
|
58
|
+
"changed-files-refresh": session.postMutationEquivalence["changed-files-refresh"].result,
|
|
59
|
+
"affected-neighborhood-refresh": session.postMutationEquivalence["affected-neighborhood-refresh"].result,
|
|
60
|
+
"full-refresh": session.postMutationEquivalence["full-refresh"].result
|
|
61
|
+
},
|
|
62
|
+
baselineFreshnessStatus: record((id) => session.treatments[id].changeAuthority.postMutationBaselineFreshness.status),
|
|
63
|
+
refreshedFreshnessStatus: refreshedStatus,
|
|
64
|
+
partialRealization: {
|
|
65
|
+
"changed-files-refresh": session.treatments["changed-files-refresh"].refreshRealization,
|
|
66
|
+
"affected-neighborhood-refresh": session.treatments["affected-neighborhood-refresh"].refreshRealization
|
|
67
|
+
},
|
|
68
|
+
indexInvocationCounts: record((id) => session.treatments[id].indexInvocationCount),
|
|
69
|
+
freshnessAssessmentCounts: record((id) => session.treatments[id].freshnessAssessmentCount),
|
|
70
|
+
totalIndexInvocationCount: session.totalIndexInvocationCount,
|
|
71
|
+
totalFreshnessAssessmentCount: session.totalFreshnessAssessmentCount,
|
|
72
|
+
myDevKitVersion: session.toolIdentity.version ?? null,
|
|
73
|
+
sourceRoots: [...session.baseCase.sourceRoots]
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
function summarizeScenarioV2(execution) {
|
|
77
|
+
return {
|
|
78
|
+
scenarioId: execution.scenarioId,
|
|
79
|
+
category: execution.category,
|
|
80
|
+
benchmarkProjectId: execution.benchmarkProjectId,
|
|
81
|
+
baseCaseId: execution.baseCaseId,
|
|
82
|
+
answerPolicy: execution.answerPolicy,
|
|
83
|
+
query: execution.query,
|
|
84
|
+
expectedFiles: sortedUnique(execution.expectedFiles),
|
|
85
|
+
expectedSymbols: sortedUnique(execution.expectedSymbols),
|
|
86
|
+
status: execution.status,
|
|
87
|
+
failureReason: execution.failureReason,
|
|
88
|
+
lifecycle: summarizeLifecycleV2(execution),
|
|
89
|
+
treatments: execution.treatments.map(summarizeTreatmentV2),
|
|
90
|
+
referenceComparisons: structuredClone(execution.referenceComparisons)
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
function partialComparison(scenario, candidate) {
|
|
94
|
+
const found = scenario.referenceComparisons.find((comparison) => comparison.candidateTreatmentId === candidate);
|
|
95
|
+
return found && found.kind === "partial-refresh" ? found : null;
|
|
96
|
+
}
|
|
97
|
+
function countPartial(scenarios, candidate, classification) {
|
|
98
|
+
return scenarios.filter((scenario) => partialComparison(scenario, candidate)?.classification === classification).length;
|
|
99
|
+
}
|
|
100
|
+
function countRealization(scenarios, candidate, realization) {
|
|
101
|
+
return scenarios.filter((scenario) => scenario.treatments.find((treatment) => treatment.treatmentId === candidate)?.refreshExecution.realization === realization).length;
|
|
102
|
+
}
|
|
103
|
+
function summarizeV2(scenarios) {
|
|
104
|
+
const staleClass = (scenario) => {
|
|
105
|
+
const stale = scenario.referenceComparisons.find((comparison) => comparison.candidateTreatmentId === "stale-index");
|
|
106
|
+
return stale && stale.kind === "stale-risk" ? stale.comparison.staleRiskClassification : null;
|
|
107
|
+
};
|
|
108
|
+
return {
|
|
109
|
+
scenarioCount: scenarios.length,
|
|
110
|
+
readyScenarioCount: scenarios.filter((scenario) => scenario.status === "ready").length,
|
|
111
|
+
failedScenarioCount: scenarios.filter((scenario) => scenario.status === "failed").length,
|
|
112
|
+
staleObservedRegressionCount: scenarios.filter((scenario) => staleClass(scenario) === "observed-stale-regression").length,
|
|
113
|
+
staleNoObservedRegressionCount: scenarios.filter((scenario) => staleClass(scenario) === "no-observed-stale-regression").length,
|
|
114
|
+
staleInconclusiveCount: scenarios.filter((scenario) => staleClass(scenario) === "inconclusive").length,
|
|
115
|
+
changedFilesAppliedPartialCount: countRealization(scenarios, "changed-files-refresh", "APPLIED_PARTIAL"),
|
|
116
|
+
changedFilesFallbackFullCount: countRealization(scenarios, "changed-files-refresh", "FALLBACK_FULL"),
|
|
117
|
+
affectedNeighborhoodAppliedPartialCount: countRealization(scenarios, "affected-neighborhood-refresh", "APPLIED_PARTIAL"),
|
|
118
|
+
affectedNeighborhoodFallbackFullCount: countRealization(scenarios, "affected-neighborhood-refresh", "FALLBACK_FULL"),
|
|
119
|
+
changedFilesObservedRegressionRelativeToFullCount: countPartial(scenarios, "changed-files-refresh", "observed-regression-relative-to-full"),
|
|
120
|
+
changedFilesNoObservedRegressionRelativeToFullCount: countPartial(scenarios, "changed-files-refresh", "no-observed-regression-relative-to-full"),
|
|
121
|
+
changedFilesInconclusiveCount: countPartial(scenarios, "changed-files-refresh", "inconclusive"),
|
|
122
|
+
changedFilesNotComparableCount: countPartial(scenarios, "changed-files-refresh", "not-comparable-as-partial-refresh"),
|
|
123
|
+
affectedNeighborhoodObservedRegressionRelativeToFullCount: countPartial(scenarios, "affected-neighborhood-refresh", "observed-regression-relative-to-full"),
|
|
124
|
+
affectedNeighborhoodNoObservedRegressionRelativeToFullCount: countPartial(scenarios, "affected-neighborhood-refresh", "no-observed-regression-relative-to-full"),
|
|
125
|
+
affectedNeighborhoodInconclusiveCount: countPartial(scenarios, "affected-neighborhood-refresh", "inconclusive"),
|
|
126
|
+
affectedNeighborhoodNotComparableCount: countPartial(scenarios, "affected-neighborhood-refresh", "not-comparable-as-partial-refresh")
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
function contradiction(scenarioId, message) {
|
|
130
|
+
throw new Error(`Contradictory v2 execution evidence for scenario ${scenarioId}: ${message}`);
|
|
131
|
+
}
|
|
132
|
+
function sameList(left, right) {
|
|
133
|
+
return left.length === right.length && left.every((value, index) => value === right[index]);
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Rejects internally contradictory evidence; called before anything is built or written. Unavailable/null
|
|
137
|
+
* evidence is preserved, never coerced.
|
|
138
|
+
*/
|
|
139
|
+
export function validateIncrementalChangeStalenessScenarioRecordV2(scenario) {
|
|
140
|
+
const id = scenario.scenarioId;
|
|
141
|
+
// Comparisons: exactly three, fixed order, explicit identities, reference is full-refresh, never full vs full.
|
|
142
|
+
if (!sameList(scenario.referenceComparisons.map((comparison) => comparison.candidateTreatmentId), COMPARISON_CANDIDATE_ORDER)) {
|
|
143
|
+
contradiction(id, "reference comparisons must be exactly stale-index, changed-files-refresh, affected-neighborhood-refresh in order.");
|
|
144
|
+
}
|
|
145
|
+
for (const comparison of scenario.referenceComparisons) {
|
|
146
|
+
if (comparison.referenceTreatmentId !== "full-refresh")
|
|
147
|
+
contradiction(id, `${comparison.candidateTreatmentId} comparison references ${comparison.referenceTreatmentId}, not full-refresh.`);
|
|
148
|
+
if ((comparison.candidateTreatmentId === "stale-index") !== (comparison.kind === "stale-risk")) {
|
|
149
|
+
contradiction(id, `${comparison.candidateTreatmentId} comparison uses the wrong comparison kind (${comparison.kind}).`);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
// Stale comparison must use the unchanged stale-risk semantics.
|
|
153
|
+
const stale = scenario.referenceComparisons[0];
|
|
154
|
+
if (stale.kind === "stale-risk") {
|
|
155
|
+
const expected = classifyStaleRisk(stale.comparison.correctnessRelation, stale.comparison.requiredFileEvidenceRelation);
|
|
156
|
+
if (expected.staleRiskClassification !== stale.comparison.staleRiskClassification || !sameList(expected.reasonCodes, stale.comparison.reasonCodes)) {
|
|
157
|
+
contradiction(id, "stale-index comparison does not follow the stale-risk classification rules.");
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
// Treatment collection.
|
|
161
|
+
if (scenario.treatments.length === 0) {
|
|
162
|
+
if (scenario.status === "ready")
|
|
163
|
+
contradiction(id, "a ready scenario must carry four treatment records.");
|
|
164
|
+
for (const comparison of scenario.referenceComparisons) {
|
|
165
|
+
if (comparison.kind === "partial-refresh" && comparison.refreshRealization !== null) {
|
|
166
|
+
contradiction(id, `${comparison.candidateTreatmentId} carries a refresh realization although no treatment executed.`);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
return;
|
|
170
|
+
}
|
|
171
|
+
if (scenario.status !== "ready")
|
|
172
|
+
contradiction(id, "a failed scenario must not carry treatment records.");
|
|
173
|
+
if (!sameList(scenario.treatments.map((treatment) => treatment.treatmentId), INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_IDS)) {
|
|
174
|
+
contradiction(id, "treatments must be exactly stale-index, changed-files-refresh, affected-neighborhood-refresh, full-refresh, each once, in order.");
|
|
175
|
+
}
|
|
176
|
+
for (const treatment of scenario.treatments) {
|
|
177
|
+
if (treatment.treatmentIntent !== INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_INTENTS[treatment.treatmentId]) {
|
|
178
|
+
contradiction(id, `${treatment.treatmentId} carries treatment intent ${treatment.treatmentIntent}.`);
|
|
179
|
+
}
|
|
180
|
+
const refresh = treatment.refreshExecution;
|
|
181
|
+
if (treatment.treatmentId === "stale-index") {
|
|
182
|
+
if (refresh.kind !== "no-refresh" || refresh.realization !== "NO_REFRESH" || refresh.incrementalRefresh !== null)
|
|
183
|
+
contradiction(id, "stale-index must be no-refresh without incrementalRefresh evidence.");
|
|
184
|
+
if (treatment.activeIndexPhase !== "baseline")
|
|
185
|
+
contradiction(id, "stale-index active index must be the baseline.");
|
|
186
|
+
if (treatment.refreshedFreshness !== null)
|
|
187
|
+
contradiction(id, "stale-index has no refreshed freshness.");
|
|
188
|
+
}
|
|
189
|
+
else {
|
|
190
|
+
if (treatment.activeIndexPhase !== "refreshed")
|
|
191
|
+
contradiction(id, `${treatment.treatmentId} active index must be the refreshed index.`);
|
|
192
|
+
if (treatment.refreshedFreshness === null)
|
|
193
|
+
contradiction(id, `${treatment.treatmentId} is missing refreshed freshness.`);
|
|
194
|
+
}
|
|
195
|
+
if (treatment.treatmentId === "full-refresh") {
|
|
196
|
+
if (refresh.kind !== "full" || refresh.realization !== "FULL_REFRESH" || refresh.incrementalRefresh !== null)
|
|
197
|
+
contradiction(id, "full-refresh must be a full refresh without incrementalRefresh evidence.");
|
|
198
|
+
}
|
|
199
|
+
if (treatment.treatmentId === "changed-files-refresh" || treatment.treatmentId === "affected-neighborhood-refresh") {
|
|
200
|
+
if (refresh.kind !== "incremental")
|
|
201
|
+
contradiction(id, `${treatment.treatmentId} must be an incremental refresh.`);
|
|
202
|
+
if (refresh.kind === "incremental") {
|
|
203
|
+
const upstream = refresh.incrementalRefresh;
|
|
204
|
+
if (upstream.requestedScope !== REQUESTED_SCOPE[treatment.treatmentId])
|
|
205
|
+
contradiction(id, `${treatment.treatmentId} requested scope ${upstream.requestedScope} does not match the treatment.`);
|
|
206
|
+
if (refresh.realization === "APPLIED_PARTIAL" && (upstream.selectionStatus !== "applied" || upstream.appliedScope !== upstream.requestedScope)) {
|
|
207
|
+
contradiction(id, `${treatment.treatmentId} is APPLIED_PARTIAL but upstream reported status=${upstream.selectionStatus} applied=${upstream.appliedScope}.`);
|
|
208
|
+
}
|
|
209
|
+
if (refresh.realization === "FALLBACK_FULL" && (upstream.selectionStatus !== "fallback-full" || upstream.appliedScope !== "full")) {
|
|
210
|
+
contradiction(id, `${treatment.treatmentId} is FALLBACK_FULL but upstream reported status=${upstream.selectionStatus} applied=${upstream.appliedScope}.`);
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
// Partial comparisons must agree with their treatment's realization and the frozen precedence.
|
|
216
|
+
for (const candidate of PARTIAL_IDS) {
|
|
217
|
+
const comparison = partialComparison(scenario, candidate);
|
|
218
|
+
const treatment = scenario.treatments.find((entry) => entry.treatmentId === candidate);
|
|
219
|
+
if (!comparison || !treatment || treatment.refreshExecution.kind !== "incremental")
|
|
220
|
+
contradiction(id, `${candidate} comparison or treatment is missing.`);
|
|
221
|
+
else {
|
|
222
|
+
if (comparison.refreshRealization !== treatment.refreshExecution.realization) {
|
|
223
|
+
contradiction(id, `${candidate} comparison realization ${comparison.refreshRealization} differs from the treatment realization ${treatment.refreshExecution.realization}.`);
|
|
224
|
+
}
|
|
225
|
+
const expected = classifyPartialRefreshAgainstFull({
|
|
226
|
+
realization: treatment.refreshExecution.realization,
|
|
227
|
+
correctnessRelation: comparison.correctnessRelation,
|
|
228
|
+
requiredFileEvidenceRelation: comparison.requiredFileEvidenceRelation
|
|
229
|
+
});
|
|
230
|
+
if (expected.classification !== comparison.classification || !sameList(expected.reasonCodes, comparison.reasonCodes)) {
|
|
231
|
+
contradiction(id, `${candidate} comparison does not follow the partial-refresh reference rules.`);
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
if (scenario.lifecycle) {
|
|
236
|
+
for (const count of [...Object.values(scenario.lifecycle.indexInvocationCounts), ...Object.values(scenario.lifecycle.freshnessAssessmentCounts)]) {
|
|
237
|
+
if (!Number.isInteger(count) || count < 0)
|
|
238
|
+
contradiction(id, "lifecycle counts must be non-negative integers.");
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
else {
|
|
242
|
+
contradiction(id, "a ready scenario must carry a lifecycle summary.");
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
export function validateIncrementalChangeStalenessExecutionArtifactV2(artifact) {
|
|
246
|
+
if (artifact.schemaVersion !== INCREMENTAL_CHANGE_STALENESS_EXECUTION_SCHEMA_VERSION_V2) {
|
|
247
|
+
throw new Error(`Unexpected execution artifact schema ${String(artifact.schemaVersion)}.`);
|
|
248
|
+
}
|
|
249
|
+
for (const scenario of artifact.scenarios)
|
|
250
|
+
validateIncrementalChangeStalenessScenarioRecordV2(scenario);
|
|
251
|
+
}
|
|
252
|
+
/**
|
|
253
|
+
* Builds and validates the v2 artifact from already-executed scenario evidence, in the caller's canonical
|
|
254
|
+
* scenario order. Throws on internal contradiction instead of serializing it.
|
|
255
|
+
*/
|
|
256
|
+
export function buildIncrementalChangeStalenessExecutionArtifactV2(args) {
|
|
257
|
+
const scenarios = args.executions.map(summarizeScenarioV2);
|
|
258
|
+
const artifact = {
|
|
259
|
+
schemaVersion: INCREMENTAL_CHANGE_STALENESS_EXECUTION_SCHEMA_VERSION_V2,
|
|
260
|
+
runId: args.runId,
|
|
261
|
+
pluginId: args.pluginId,
|
|
262
|
+
scenarios,
|
|
263
|
+
summary: summarizeV2(scenarios)
|
|
264
|
+
};
|
|
265
|
+
validateIncrementalChangeStalenessExecutionArtifactV2(artifact);
|
|
266
|
+
return artifact;
|
|
267
|
+
}
|
|
268
|
+
/** Writes to the canonical filename shared with v1; validates first so contradictory evidence is never serialized. */
|
|
269
|
+
export async function writeIncrementalChangeStalenessExecutionArtifactV2(outDir, artifact) {
|
|
270
|
+
validateIncrementalChangeStalenessExecutionArtifactV2(artifact);
|
|
271
|
+
await mkdir(outDir, { recursive: true });
|
|
272
|
+
const artifactPath = resolveWithinRoot(outDir, INCREMENTAL_CHANGE_STALENESS_EXECUTION_ARTIFACT_FILE);
|
|
273
|
+
await writeFile(artifactPath, `${JSON.stringify(artifact, null, 2)}\n`, "utf8");
|
|
274
|
+
return artifactPath;
|
|
275
|
+
}
|
|
276
|
+
/** Discriminates V1 from V2 by schemaVersion only; historical V1 evidence is never converted to V2. */
|
|
277
|
+
export function parseIncrementalChangeStalenessExecutionArtifact(json) {
|
|
278
|
+
const schemaVersion = json && typeof json === "object" ? json.schemaVersion : undefined;
|
|
279
|
+
if (schemaVersion === INCREMENTAL_CHANGE_STALENESS_EXECUTION_SCHEMA_VERSION) {
|
|
280
|
+
return { version: "v1", artifact: json };
|
|
281
|
+
}
|
|
282
|
+
if (schemaVersion === INCREMENTAL_CHANGE_STALENESS_EXECUTION_SCHEMA_VERSION_V2) {
|
|
283
|
+
return { version: "v2", artifact: json };
|
|
284
|
+
}
|
|
285
|
+
throw new Error(`Unrecognized incremental-change-staleness execution artifact schema: ${String(schemaVersion)}.`);
|
|
286
|
+
}
|
|
287
|
+
export async function readIncrementalChangeStalenessExecutionArtifact(artifactPath) {
|
|
288
|
+
return parseIncrementalChangeStalenessExecutionArtifact(JSON.parse(await readFile(artifactPath, "utf8")));
|
|
289
|
+
}
|