@dailephd/my-dev-kit-lab 0.6.2 → 0.6.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/CHANGELOG.md +18 -0
  2. package/README.md +9 -8
  3. package/dist/src/cli/help.js +16 -9
  4. package/dist/src/commands/runExperimentDescribeCommand.js +5 -2
  5. package/dist/src/commands/runExperimentRunCommand.js +0 -4
  6. package/dist/src/evaluation/benchmarkMetadata.js +5 -1
  7. package/dist/src/evaluation/parseAgentAnswer.js +172 -24
  8. package/dist/src/evaluation/runMyDevKitRetrieval.js +120 -7
  9. package/dist/src/experiments/plugins/incrementalChangeStaleness/comparisonV2.js +49 -0
  10. package/dist/src/experiments/plugins/incrementalChangeStaleness/config.js +5 -4
  11. package/dist/src/experiments/plugins/incrementalChangeStaleness/disposableTarget.js +20 -2
  12. package/dist/src/experiments/plugins/incrementalChangeStaleness/execution.js +42 -16
  13. package/dist/src/experiments/plugins/incrementalChangeStaleness/executionArtifact.js +4 -4
  14. package/dist/src/experiments/plugins/incrementalChangeStaleness/executionArtifactV2.js +289 -0
  15. package/dist/src/experiments/plugins/incrementalChangeStaleness/executionV2.js +195 -0
  16. package/dist/src/experiments/plugins/incrementalChangeStaleness/indexClone.js +76 -0
  17. package/dist/src/experiments/plugins/incrementalChangeStaleness/lifecycle.js +64 -32
  18. package/dist/src/experiments/plugins/incrementalChangeStaleness/lifecyclePolicy.js +6 -4
  19. package/dist/src/experiments/plugins/incrementalChangeStaleness/lifecyclePolicyV2.js +78 -0
  20. package/dist/src/experiments/plugins/incrementalChangeStaleness/lifecycleV2.js +298 -0
  21. package/dist/src/experiments/plugins/incrementalChangeStaleness/plugin.js +213 -12
  22. package/dist/src/experiments/plugins/incrementalChangeStaleness/treatmentSessionV2.js +5 -0
  23. package/dist/src/experiments/plugins/warmIndexReuse/warmIndexSession.js +3 -1
  24. package/dist/src/mobile/android/advancedSecurity/candidateEvidence.js +9 -2
  25. package/dist/src/mobile/android/advancedSecurity/redaction.js +19 -10
  26. package/dist/src/mobile/android/advancedSecurity/secretCandidates/analyzeSecretCandidates.js +12 -2
  27. package/dist/src/mobile/android/advancedSecurity/signingConfiguration/analyzeSigningConfiguration.js +1 -1
  28. package/dist/src/mobile/android/advancedSecurity/signingConfiguration/extractSigningConfigurations.js +7 -7
  29. package/dist/src/report/experiments/buildIncrementalChangeStalenessPluginReport.js +33 -0
  30. package/dist/src/report/experiments/buildIncrementalChangeStalenessReport.js +1 -1
  31. package/dist/src/report/experiments/buildIncrementalChangeStalenessReportV2.js +80 -0
  32. package/dist/src/report/experiments/buildPluginExperimentReport.js +2 -2
  33. package/dist/src/report/experiments/incrementalChangeStalenessReportModelV2.js +3 -0
  34. package/dist/src/report/experiments/incrementalChangeStalenessReportRowsV2.js +146 -0
  35. package/dist/src/report/experiments/index.js +3 -0
  36. package/dist/src/report/experiments/renderIncrementalChangeStalenessHtml.js +4 -0
  37. package/dist/src/report/experiments/renderIncrementalChangeStalenessHtmlV2.js +111 -0
  38. package/dist/src/report/experiments/renderIncrementalChangeStalenessTextV2.js +96 -0
  39. package/dist/src/report/experiments/renderPluginExperimentReportText.js +6 -0
  40. package/dist/src/securityValidation/staticScans/codeql.js +8 -8
  41. package/docs/METRICS.md +32 -0
  42. package/examples/tutorial-browser/prepare.mjs +13 -13
  43. package/examples/tutorial-browser/server.mjs +13 -13
  44. package/package.json +1 -1
@@ -0,0 +1,49 @@
1
+ import { compareCorrectness, compareRequiredFileEvidence } from "./comparison.js";
2
+ function neutral(relation) {
3
+ if (relation === "stale-worse")
4
+ return "candidate-worse";
5
+ if (relation === "stale-better")
6
+ return "candidate-better";
7
+ return relation;
8
+ }
9
+ /** Same rules as the v0.6.2 correctness comparison, candidate vs reference. */
10
+ export function compareCandidateCorrectness(candidate, reference) {
11
+ return neutral(compareCorrectness(candidate, reference));
12
+ }
13
+ /** Same rules as the v0.6.2 required-file comparison, candidate vs reference. */
14
+ export function compareCandidateRequiredFileEvidence(candidate, reference) {
15
+ return neutral(compareRequiredFileEvidence(candidate, reference));
16
+ }
17
+ /**
18
+ * Frozen precedence. Rule 0 (realization gate) first: a full fallback is never evidence about
19
+ * partial-refresh performance. Then observed regression (either dimension worse), then inconclusive
20
+ * (a dimension unknown), then no observed regression (including candidate-better, which is not a
21
+ * winner claim). affected-neighborhood evidence and reindexRecommendation are never inputs.
22
+ */
23
+ export function classifyPartialRefreshAgainstFull(args) {
24
+ const { realization, correctnessRelation, requiredFileEvidenceRelation } = args;
25
+ // RULE 0 -- realization gate.
26
+ if (realization === "FALLBACK_FULL") {
27
+ return { classification: "not-comparable-as-partial-refresh", reasonCodes: ["candidate-fell-back-to-full"] };
28
+ }
29
+ // RULE 1 -- observed regression wins even if the other dimension is unknown.
30
+ if (correctnessRelation === "candidate-worse" || requiredFileEvidenceRelation === "candidate-worse") {
31
+ const reasonCodes = [];
32
+ if (correctnessRelation === "candidate-worse")
33
+ reasonCodes.push("candidate-correctness-lower");
34
+ if (requiredFileEvidenceRelation === "candidate-worse")
35
+ reasonCodes.push("candidate-missing-required-file");
36
+ return { classification: "observed-regression-relative-to-full", reasonCodes };
37
+ }
38
+ // RULE 2 -- inconclusive.
39
+ if (correctnessRelation === "unknown" || requiredFileEvidenceRelation === "unknown") {
40
+ const reasonCodes = [];
41
+ if (correctnessRelation === "unknown")
42
+ reasonCodes.push("correctness-unavailable");
43
+ if (requiredFileEvidenceRelation === "unknown")
44
+ reasonCodes.push("required-file-evidence-unavailable");
45
+ return { classification: "inconclusive", reasonCodes };
46
+ }
47
+ // RULE 3 -- no observed regression.
48
+ return { classification: "no-observed-regression-relative-to-full", reasonCodes: ["no-reference-regression-observed"] };
49
+ }
@@ -1,10 +1,11 @@
1
1
  import { invalidExperimentConfig, isPlainObject, mergeConfig, validExperimentConfig } from "../../config.js";
2
2
  import { FROZEN_INCREMENTAL_CHANGE_STALENESS_SCENARIO_IDS } from "./scenarioTypes.js";
3
3
  /**
4
- * The frozen v0.6.2 compatibility baseline for the upstream tool. Every index
5
- * of one matched scenario must come from this one configured command.
4
+ * The upstream tool version the four-treatment experiment expects: 1.12.5 is the first published
5
+ * my-dev-kit that exposes the incremental refresh-scope options. Every index of one matched scenario
6
+ * must come from this one configured command.
6
7
  */
7
- export const INCREMENTAL_CHANGE_STALENESS_DEFAULT_KIT_COMMAND = "npx @dailephd/my-dev-kit@1.12.4";
8
+ export const INCREMENTAL_CHANGE_STALENESS_DEFAULT_KIT_COMMAND = "npx @dailephd/my-dev-kit@1.12.5";
8
9
  export const defaultIncrementalChangeStalenessConfig = {
9
10
  outDir: "lab-output/incremental-change-staleness",
10
11
  kitCommand: INCREMENTAL_CHANGE_STALENESS_DEFAULT_KIT_COMMAND
@@ -17,7 +18,7 @@ export const incrementalChangeStalenessConfigDefinition = {
17
18
  {
18
19
  name: "caseIds",
19
20
  type: "array",
20
- description: `Frozen scenario ids to include (${FROZEN_INCREMENTAL_CHANGE_STALENESS_SCENARIO_IDS.join(", ")}); defaults to all. Both stale-index and full-refresh always run for each selected scenario.`
21
+ description: `Frozen scenario ids to include (${FROZEN_INCREMENTAL_CHANGE_STALENESS_SCENARIO_IDS.join(", ")}); defaults to all. All four treatments run for each selected scenario: stale-index, changed-files-refresh, affected-neighborhood-refresh, and full-refresh.`
21
22
  }
22
23
  ]
23
24
  };
@@ -8,6 +8,24 @@ import { resolveWithinRoot } from "../../../core/pathSafety.js";
8
8
  * target copies; it never builds a treatment index or runs retrieval.
9
9
  */
10
10
  export const INCREMENTAL_CHANGE_STALENESS_TREATMENT_IDS = ["stale-index", "full-refresh"];
11
+ /**
12
+ * The four v0.6.3 internal lifecycle treatment identities in their fixed order
13
+ * (never alphabetical). The released v0.6.2 public constant above is unchanged;
14
+ * this set is used only by the internal v0.6.3 lifecycle.
15
+ */
16
+ export const INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_IDS = [
17
+ "stale-index",
18
+ "changed-files-refresh",
19
+ "affected-neighborhood-refresh",
20
+ "full-refresh"
21
+ ];
22
+ /** Treatment ID and treatment intent are separate runtime evidence. */
23
+ export const INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_INTENTS = {
24
+ "stale-index": "my-dev-kit-no-refresh",
25
+ "changed-files-refresh": "my-dev-kit-changed-files-refresh",
26
+ "affected-neighborhood-refresh": "my-dev-kit-affected-neighborhood-refresh",
27
+ "full-refresh": "my-dev-kit-full-refresh"
28
+ };
11
29
  /**
12
30
  * Default project-relative, gitignored runtime location for disposable
13
31
  * treatment targets. Callers may pass a different `runRoot`, but it must
@@ -38,8 +56,8 @@ function assertDisposableRootDoesNotOverlapCanonical(targetRoot, canonicalProjec
38
56
  */
39
57
  export async function createDisposableTreatmentTarget(options) {
40
58
  const { repoRoot, scenarioId, treatmentId, benchmarkProjectId, canonicalProjectRootRelative } = options;
41
- if (!INCREMENTAL_CHANGE_STALENESS_TREATMENT_IDS.includes(treatmentId)) {
42
- throw new Error(`Unknown treatment id: ${treatmentId}. Expected one of ${INCREMENTAL_CHANGE_STALENESS_TREATMENT_IDS.join(", ")}.`);
59
+ if (!INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_IDS.includes(treatmentId)) {
60
+ throw new Error(`Unknown treatment id: ${treatmentId}. Expected one of ${INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_IDS.join(", ")}.`);
43
61
  }
44
62
  if (typeof scenarioId !== "string" || scenarioId.length === 0) {
45
63
  throw new Error("scenarioId must be a nonempty string.");
@@ -224,13 +224,13 @@ export async function runIncrementalChangeStalenessFakeAgent(args) {
224
224
  };
225
225
  }
226
226
  }
227
- function correctnessComparable(fakeAgent) {
227
+ export function correctnessComparable(fakeAgent) {
228
228
  if (!fakeAgent || !fakeAgent.correctness.available || fakeAgent.correctness.score === null) {
229
229
  return { available: false };
230
230
  }
231
231
  return { available: true, score: fakeAgent.correctness.score };
232
232
  }
233
- const INCONCLUSIVE_NO_EXECUTION = {
233
+ export const INCONCLUSIVE_NO_EXECUTION = {
234
234
  correctnessRelation: "unknown",
235
235
  requiredFileEvidenceRelation: "unknown",
236
236
  staleRiskClassification: "inconclusive",
@@ -240,10 +240,12 @@ export const defaultIncrementalChangeStalenessExecutionDeps = {
240
240
  runRetrieval: runIncrementalChangeStalenessTreatmentRetrieval,
241
241
  runFakeAgent: runIncrementalChangeStalenessFakeAgent
242
242
  };
243
- async function executeOneTreatment(args) {
244
- const treatment = args.session.treatments[args.treatmentId];
245
- const baselineFreshness = treatment.changeAuthority.postMutationBaselineFreshness;
246
- const activeIndexPhase = treatment.activeRetrieval.role;
243
+ /**
244
+ * Retrieval + deterministic fake-agent + required-file evidence for ONE treatment against its own active
245
+ * index. Shared by the v0.6.2 two-treatment path and the v0.6.3 four-treatment path; takes only the
246
+ * lifecycle-provided facts it needs so it never reaches into another treatment's state.
247
+ */
248
+ export async function executeTreatmentEvaluation(args) {
247
249
  let retrieval = null;
248
250
  let retrievalStatus = "not-run";
249
251
  let failureReason = null;
@@ -251,7 +253,7 @@ async function executeOneTreatment(args) {
251
253
  retrieval = await args.deps.runRetrieval({
252
254
  derivedTask: args.derivedTask,
253
255
  kitCommand: args.kitCommand,
254
- indexDir: treatment.activeRetrieval.index.indexDir,
256
+ indexDir: args.activeIndexDir,
255
257
  commandsDir: args.retrievalCommandsDir
256
258
  });
257
259
  retrievalStatus = classifyIncrementalChangeStalenessRetrieval(retrieval);
@@ -284,8 +286,8 @@ async function executeOneTreatment(args) {
284
286
  treatmentId: args.treatmentId,
285
287
  status,
286
288
  failureReason,
287
- activeIndexPhase,
288
- baselineFreshness,
289
+ activeIndexPhase: args.activeIndexPhase,
290
+ baselineFreshness: args.baselineFreshness,
289
291
  affectedNeighborhood: args.affectedNeighborhood,
290
292
  retrieval,
291
293
  retrievalStatus,
@@ -293,6 +295,36 @@ async function executeOneTreatment(args) {
293
295
  requiredFileEvidence
294
296
  };
295
297
  }
298
+ async function executeOneTreatment(args) {
299
+ const treatment = args.session.treatments[args.treatmentId];
300
+ return executeTreatmentEvaluation({
301
+ treatmentId: args.treatmentId,
302
+ baselineFreshness: treatment.changeAuthority.postMutationBaselineFreshness,
303
+ activeIndexPhase: treatment.activeRetrieval.role,
304
+ activeIndexDir: treatment.activeRetrieval.index.indexDir,
305
+ derivedTask: args.derivedTask,
306
+ affectedNeighborhood: args.affectedNeighborhood,
307
+ requiredFiles: args.requiredFiles,
308
+ kitCommand: args.kitCommand,
309
+ retrievalCommandsDir: args.retrievalCommandsDir,
310
+ agentOutDir: args.agentOutDir,
311
+ projectProfiles: args.projectProfiles,
312
+ cwd: args.cwd,
313
+ env: args.env,
314
+ deps: args.deps
315
+ });
316
+ }
317
+ /** Resolves the scenario's query/answer and derives the one in-memory task shared by every matched treatment. */
318
+ export async function resolveIncrementalChangeStalenessScenarioTask(options) {
319
+ const { scenario } = options;
320
+ const resolved = await resolveIncrementalChangeStalenessQueryAndAnswer({ scenario, repoRoot: options.repoRoot });
321
+ const cases = await readEvaluationCases(path.resolve(options.repoRoot, WARM_INDEX_BENCHMARK_CASES_PATH), options.repoRoot);
322
+ const baseEvaluationCase = cases.find((candidate) => candidate.id === scenario.baseCaseId);
323
+ if (!baseEvaluationCase) {
324
+ throw new Error(`Scenario ${scenario.id}: base case ${scenario.baseCaseId} was not found for task-descriptor construction.`);
325
+ }
326
+ return { resolved, derivedTask: buildIncrementalChangeStalenessDerivedTask({ scenario, baseEvaluationCase, resolved }) };
327
+ }
296
328
  /**
297
329
  * Executes the frozen Batch 4 order (section 36) for one ready lifecycle session: resolve query and
298
330
  * answer, derive the task descriptor, assess both treatments' affected neighborhoods from BASELINE
@@ -323,13 +355,7 @@ export async function executeIncrementalChangeStalenessScenario(options) {
323
355
  };
324
356
  }
325
357
  const session = options.lifecycle.session;
326
- const resolved = await resolveIncrementalChangeStalenessQueryAndAnswer({ scenario, repoRoot: options.repoRoot });
327
- const cases = await readEvaluationCases(path.resolve(options.repoRoot, WARM_INDEX_BENCHMARK_CASES_PATH), options.repoRoot);
328
- const baseEvaluationCase = cases.find((candidate) => candidate.id === scenario.baseCaseId);
329
- if (!baseEvaluationCase) {
330
- throw new Error(`Scenario ${scenario.id}: base case ${scenario.baseCaseId} was not found for task-descriptor construction.`);
331
- }
332
- const derivedTask = buildIncrementalChangeStalenessDerivedTask({ scenario, baseEvaluationCase, resolved });
358
+ const { resolved, derivedTask } = await resolveIncrementalChangeStalenessScenarioTask({ repoRoot: options.repoRoot, scenario });
333
359
  const staleAffected = assessIncrementalChangeStalenessAffectedNeighborhood(session.treatments["stale-index"], derivedTask);
334
360
  const fullAffected = assessIncrementalChangeStalenessAffectedNeighborhood(session.treatments["full-refresh"], derivedTask);
335
361
  const symmetry = checkIncrementalChangeStalenessAffectedNeighborhoodSymmetry(staleAffected, fullAffected);
@@ -9,7 +9,7 @@ import { resolveWithinRoot } from "../../../core/pathSafety.js";
9
9
  // ---------------------------------------------------------------------------
10
10
  export const INCREMENTAL_CHANGE_STALENESS_EXECUTION_ARTIFACT_FILE = "incremental-change-staleness-execution.json";
11
11
  export const INCREMENTAL_CHANGE_STALENESS_EXECUTION_SCHEMA_VERSION = "my-dev-kit-lab-incremental-change-staleness-execution-v1";
12
- function sortedUnique(values) {
12
+ export function sortedUnique(values) {
13
13
  return [...new Set(values)].sort((left, right) => (left < right ? -1 : left > right ? 1 : 0));
14
14
  }
15
15
  function summarizeCommand(command) {
@@ -24,7 +24,7 @@ function summarizeCommand(command) {
24
24
  error: command.error ?? null
25
25
  };
26
26
  }
27
- function summarizeRetrieval(treatment) {
27
+ export function summarizeRetrieval(treatment) {
28
28
  const retrieval = treatment.retrieval;
29
29
  if (!retrieval) {
30
30
  return {
@@ -56,7 +56,7 @@ function summarizeRetrieval(treatment) {
56
56
  commands: retrieval.commands.map(summarizeCommand)
57
57
  };
58
58
  }
59
- function summarizeFakeAgent(treatment) {
59
+ export function summarizeFakeAgent(treatment) {
60
60
  if (!treatment.fakeAgent)
61
61
  return null;
62
62
  const agent = treatment.fakeAgent;
@@ -69,7 +69,7 @@ function summarizeFakeAgent(treatment) {
69
69
  errors: [...agent.errors]
70
70
  };
71
71
  }
72
- function summarizeRequiredFileEvidence(evidence) {
72
+ export function summarizeRequiredFileEvidence(evidence) {
73
73
  return {
74
74
  status: evidence.status,
75
75
  requiredFiles: sortedUnique(evidence.requiredFiles),
@@ -0,0 +1,289 @@
1
+ import { mkdir, readFile, writeFile } from "node:fs/promises";
2
+ import { resolveWithinRoot } from "../../../core/pathSafety.js";
3
+ import { classifyStaleRisk } from "./comparison.js";
4
+ import { classifyPartialRefreshAgainstFull } from "./comparisonV2.js";
5
+ import { INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_IDS, INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_INTENTS } from "./disposableTarget.js";
6
+ import { INCREMENTAL_CHANGE_STALENESS_EXECUTION_ARTIFACT_FILE, INCREMENTAL_CHANGE_STALENESS_EXECUTION_SCHEMA_VERSION, sortedUnique, summarizeFakeAgent, summarizeRequiredFileEvidence, summarizeRetrieval } from "./executionArtifact.js";
7
+ // ---------------------------------------------------------------------------
8
+ // v0.6.3 Batch 3 -- persisted four-treatment execution artifact. A distinct schema from V1 written to the
9
+ // same canonical filename; the schemaVersion is the discriminator. Bounded structured evidence only:
10
+ // never raw retrieved context bodies, graph artifacts, or file contents. Descriptive counts, no ranking.
11
+ // ---------------------------------------------------------------------------
12
+ export const INCREMENTAL_CHANGE_STALENESS_EXECUTION_SCHEMA_VERSION_V2 = "my-dev-kit-lab-incremental-change-staleness-execution-v2";
13
+ const PARTIAL_IDS = ["changed-files-refresh", "affected-neighborhood-refresh"];
14
+ const REQUESTED_SCOPE = {
15
+ "changed-files-refresh": "changed-files",
16
+ "affected-neighborhood-refresh": "affected-neighborhood"
17
+ };
18
+ const COMPARISON_CANDIDATE_ORDER = ["stale-index", "changed-files-refresh", "affected-neighborhood-refresh"];
19
+ function record(build) {
20
+ return Object.fromEntries(INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_IDS.map((id) => [id, build(id)]));
21
+ }
22
+ function summarizeTreatmentV2(treatment) {
23
+ return {
24
+ treatmentId: treatment.treatmentId,
25
+ treatmentIntent: treatment.treatmentIntent,
26
+ status: treatment.status,
27
+ failureReason: treatment.failureReason,
28
+ activeIndexPhase: treatment.activeIndexPhase,
29
+ baselineFreshness: structuredClone(treatment.baselineFreshness),
30
+ refreshedFreshness: treatment.refreshedFreshness ? structuredClone(treatment.refreshedFreshness) : null,
31
+ refreshExecution: structuredClone(treatment.refreshExecution),
32
+ affectedNeighborhood: structuredClone(treatment.affectedNeighborhood),
33
+ // Bounded status/telemetry only; never embeds retrieval.contextText.
34
+ retrieval: summarizeRetrieval(treatment),
35
+ fakeAgent: summarizeFakeAgent(treatment),
36
+ requiredFileEvidence: summarizeRequiredFileEvidence(treatment.requiredFileEvidence)
37
+ };
38
+ }
39
+ function summarizeLifecycleV2(execution) {
40
+ const session = execution.session;
41
+ if (!session)
42
+ return null;
43
+ const refreshedStatus = {};
44
+ for (const id of INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_IDS) {
45
+ const freshness = session.treatments[id].refreshedFreshness;
46
+ if (freshness)
47
+ refreshedStatus[id] = freshness.status;
48
+ }
49
+ return {
50
+ controlledFilePaths: [...session.controlledChangedPaths],
51
+ treatmentIntents: record((id) => INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_INTENTS[id]),
52
+ preMutationEquivalence: {
53
+ "changed-files-refresh": session.preMutationEquivalence["changed-files-refresh"].result,
54
+ "affected-neighborhood-refresh": session.preMutationEquivalence["affected-neighborhood-refresh"].result,
55
+ "full-refresh": session.preMutationEquivalence["full-refresh"].result
56
+ },
57
+ postMutationEquivalence: {
58
+ "changed-files-refresh": session.postMutationEquivalence["changed-files-refresh"].result,
59
+ "affected-neighborhood-refresh": session.postMutationEquivalence["affected-neighborhood-refresh"].result,
60
+ "full-refresh": session.postMutationEquivalence["full-refresh"].result
61
+ },
62
+ baselineFreshnessStatus: record((id) => session.treatments[id].changeAuthority.postMutationBaselineFreshness.status),
63
+ refreshedFreshnessStatus: refreshedStatus,
64
+ partialRealization: {
65
+ "changed-files-refresh": session.treatments["changed-files-refresh"].refreshRealization,
66
+ "affected-neighborhood-refresh": session.treatments["affected-neighborhood-refresh"].refreshRealization
67
+ },
68
+ indexInvocationCounts: record((id) => session.treatments[id].indexInvocationCount),
69
+ freshnessAssessmentCounts: record((id) => session.treatments[id].freshnessAssessmentCount),
70
+ totalIndexInvocationCount: session.totalIndexInvocationCount,
71
+ totalFreshnessAssessmentCount: session.totalFreshnessAssessmentCount,
72
+ myDevKitVersion: session.toolIdentity.version ?? null,
73
+ sourceRoots: [...session.baseCase.sourceRoots]
74
+ };
75
+ }
76
+ function summarizeScenarioV2(execution) {
77
+ return {
78
+ scenarioId: execution.scenarioId,
79
+ category: execution.category,
80
+ benchmarkProjectId: execution.benchmarkProjectId,
81
+ baseCaseId: execution.baseCaseId,
82
+ answerPolicy: execution.answerPolicy,
83
+ query: execution.query,
84
+ expectedFiles: sortedUnique(execution.expectedFiles),
85
+ expectedSymbols: sortedUnique(execution.expectedSymbols),
86
+ status: execution.status,
87
+ failureReason: execution.failureReason,
88
+ lifecycle: summarizeLifecycleV2(execution),
89
+ treatments: execution.treatments.map(summarizeTreatmentV2),
90
+ referenceComparisons: structuredClone(execution.referenceComparisons)
91
+ };
92
+ }
93
+ function partialComparison(scenario, candidate) {
94
+ const found = scenario.referenceComparisons.find((comparison) => comparison.candidateTreatmentId === candidate);
95
+ return found && found.kind === "partial-refresh" ? found : null;
96
+ }
97
+ function countPartial(scenarios, candidate, classification) {
98
+ return scenarios.filter((scenario) => partialComparison(scenario, candidate)?.classification === classification).length;
99
+ }
100
+ function countRealization(scenarios, candidate, realization) {
101
+ return scenarios.filter((scenario) => scenario.treatments.find((treatment) => treatment.treatmentId === candidate)?.refreshExecution.realization === realization).length;
102
+ }
103
+ function summarizeV2(scenarios) {
104
+ const staleClass = (scenario) => {
105
+ const stale = scenario.referenceComparisons.find((comparison) => comparison.candidateTreatmentId === "stale-index");
106
+ return stale && stale.kind === "stale-risk" ? stale.comparison.staleRiskClassification : null;
107
+ };
108
+ return {
109
+ scenarioCount: scenarios.length,
110
+ readyScenarioCount: scenarios.filter((scenario) => scenario.status === "ready").length,
111
+ failedScenarioCount: scenarios.filter((scenario) => scenario.status === "failed").length,
112
+ staleObservedRegressionCount: scenarios.filter((scenario) => staleClass(scenario) === "observed-stale-regression").length,
113
+ staleNoObservedRegressionCount: scenarios.filter((scenario) => staleClass(scenario) === "no-observed-stale-regression").length,
114
+ staleInconclusiveCount: scenarios.filter((scenario) => staleClass(scenario) === "inconclusive").length,
115
+ changedFilesAppliedPartialCount: countRealization(scenarios, "changed-files-refresh", "APPLIED_PARTIAL"),
116
+ changedFilesFallbackFullCount: countRealization(scenarios, "changed-files-refresh", "FALLBACK_FULL"),
117
+ affectedNeighborhoodAppliedPartialCount: countRealization(scenarios, "affected-neighborhood-refresh", "APPLIED_PARTIAL"),
118
+ affectedNeighborhoodFallbackFullCount: countRealization(scenarios, "affected-neighborhood-refresh", "FALLBACK_FULL"),
119
+ changedFilesObservedRegressionRelativeToFullCount: countPartial(scenarios, "changed-files-refresh", "observed-regression-relative-to-full"),
120
+ changedFilesNoObservedRegressionRelativeToFullCount: countPartial(scenarios, "changed-files-refresh", "no-observed-regression-relative-to-full"),
121
+ changedFilesInconclusiveCount: countPartial(scenarios, "changed-files-refresh", "inconclusive"),
122
+ changedFilesNotComparableCount: countPartial(scenarios, "changed-files-refresh", "not-comparable-as-partial-refresh"),
123
+ affectedNeighborhoodObservedRegressionRelativeToFullCount: countPartial(scenarios, "affected-neighborhood-refresh", "observed-regression-relative-to-full"),
124
+ affectedNeighborhoodNoObservedRegressionRelativeToFullCount: countPartial(scenarios, "affected-neighborhood-refresh", "no-observed-regression-relative-to-full"),
125
+ affectedNeighborhoodInconclusiveCount: countPartial(scenarios, "affected-neighborhood-refresh", "inconclusive"),
126
+ affectedNeighborhoodNotComparableCount: countPartial(scenarios, "affected-neighborhood-refresh", "not-comparable-as-partial-refresh")
127
+ };
128
+ }
129
+ function contradiction(scenarioId, message) {
130
+ throw new Error(`Contradictory v2 execution evidence for scenario ${scenarioId}: ${message}`);
131
+ }
132
+ function sameList(left, right) {
133
+ return left.length === right.length && left.every((value, index) => value === right[index]);
134
+ }
135
+ /**
136
+ * Rejects internally contradictory evidence; called before anything is built or written. Unavailable/null
137
+ * evidence is preserved, never coerced.
138
+ */
139
+ export function validateIncrementalChangeStalenessScenarioRecordV2(scenario) {
140
+ const id = scenario.scenarioId;
141
+ // Comparisons: exactly three, fixed order, explicit identities, reference is full-refresh, never full vs full.
142
+ if (!sameList(scenario.referenceComparisons.map((comparison) => comparison.candidateTreatmentId), COMPARISON_CANDIDATE_ORDER)) {
143
+ contradiction(id, "reference comparisons must be exactly stale-index, changed-files-refresh, affected-neighborhood-refresh in order.");
144
+ }
145
+ for (const comparison of scenario.referenceComparisons) {
146
+ if (comparison.referenceTreatmentId !== "full-refresh")
147
+ contradiction(id, `${comparison.candidateTreatmentId} comparison references ${comparison.referenceTreatmentId}, not full-refresh.`);
148
+ if ((comparison.candidateTreatmentId === "stale-index") !== (comparison.kind === "stale-risk")) {
149
+ contradiction(id, `${comparison.candidateTreatmentId} comparison uses the wrong comparison kind (${comparison.kind}).`);
150
+ }
151
+ }
152
+ // Stale comparison must use the unchanged stale-risk semantics.
153
+ const stale = scenario.referenceComparisons[0];
154
+ if (stale.kind === "stale-risk") {
155
+ const expected = classifyStaleRisk(stale.comparison.correctnessRelation, stale.comparison.requiredFileEvidenceRelation);
156
+ if (expected.staleRiskClassification !== stale.comparison.staleRiskClassification || !sameList(expected.reasonCodes, stale.comparison.reasonCodes)) {
157
+ contradiction(id, "stale-index comparison does not follow the stale-risk classification rules.");
158
+ }
159
+ }
160
+ // Treatment collection.
161
+ if (scenario.treatments.length === 0) {
162
+ if (scenario.status === "ready")
163
+ contradiction(id, "a ready scenario must carry four treatment records.");
164
+ for (const comparison of scenario.referenceComparisons) {
165
+ if (comparison.kind === "partial-refresh" && comparison.refreshRealization !== null) {
166
+ contradiction(id, `${comparison.candidateTreatmentId} carries a refresh realization although no treatment executed.`);
167
+ }
168
+ }
169
+ return;
170
+ }
171
+ if (scenario.status !== "ready")
172
+ contradiction(id, "a failed scenario must not carry treatment records.");
173
+ if (!sameList(scenario.treatments.map((treatment) => treatment.treatmentId), INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_IDS)) {
174
+ contradiction(id, "treatments must be exactly stale-index, changed-files-refresh, affected-neighborhood-refresh, full-refresh, each once, in order.");
175
+ }
176
+ for (const treatment of scenario.treatments) {
177
+ if (treatment.treatmentIntent !== INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_INTENTS[treatment.treatmentId]) {
178
+ contradiction(id, `${treatment.treatmentId} carries treatment intent ${treatment.treatmentIntent}.`);
179
+ }
180
+ const refresh = treatment.refreshExecution;
181
+ if (treatment.treatmentId === "stale-index") {
182
+ if (refresh.kind !== "no-refresh" || refresh.realization !== "NO_REFRESH" || refresh.incrementalRefresh !== null)
183
+ contradiction(id, "stale-index must be no-refresh without incrementalRefresh evidence.");
184
+ if (treatment.activeIndexPhase !== "baseline")
185
+ contradiction(id, "stale-index active index must be the baseline.");
186
+ if (treatment.refreshedFreshness !== null)
187
+ contradiction(id, "stale-index has no refreshed freshness.");
188
+ }
189
+ else {
190
+ if (treatment.activeIndexPhase !== "refreshed")
191
+ contradiction(id, `${treatment.treatmentId} active index must be the refreshed index.`);
192
+ if (treatment.refreshedFreshness === null)
193
+ contradiction(id, `${treatment.treatmentId} is missing refreshed freshness.`);
194
+ }
195
+ if (treatment.treatmentId === "full-refresh") {
196
+ if (refresh.kind !== "full" || refresh.realization !== "FULL_REFRESH" || refresh.incrementalRefresh !== null)
197
+ contradiction(id, "full-refresh must be a full refresh without incrementalRefresh evidence.");
198
+ }
199
+ if (treatment.treatmentId === "changed-files-refresh" || treatment.treatmentId === "affected-neighborhood-refresh") {
200
+ if (refresh.kind !== "incremental")
201
+ contradiction(id, `${treatment.treatmentId} must be an incremental refresh.`);
202
+ if (refresh.kind === "incremental") {
203
+ const upstream = refresh.incrementalRefresh;
204
+ if (upstream.requestedScope !== REQUESTED_SCOPE[treatment.treatmentId])
205
+ contradiction(id, `${treatment.treatmentId} requested scope ${upstream.requestedScope} does not match the treatment.`);
206
+ if (refresh.realization === "APPLIED_PARTIAL" && (upstream.selectionStatus !== "applied" || upstream.appliedScope !== upstream.requestedScope)) {
207
+ contradiction(id, `${treatment.treatmentId} is APPLIED_PARTIAL but upstream reported status=${upstream.selectionStatus} applied=${upstream.appliedScope}.`);
208
+ }
209
+ if (refresh.realization === "FALLBACK_FULL" && (upstream.selectionStatus !== "fallback-full" || upstream.appliedScope !== "full")) {
210
+ contradiction(id, `${treatment.treatmentId} is FALLBACK_FULL but upstream reported status=${upstream.selectionStatus} applied=${upstream.appliedScope}.`);
211
+ }
212
+ }
213
+ }
214
+ }
215
+ // Partial comparisons must agree with their treatment's realization and the frozen precedence.
216
+ for (const candidate of PARTIAL_IDS) {
217
+ const comparison = partialComparison(scenario, candidate);
218
+ const treatment = scenario.treatments.find((entry) => entry.treatmentId === candidate);
219
+ if (!comparison || !treatment || treatment.refreshExecution.kind !== "incremental")
220
+ contradiction(id, `${candidate} comparison or treatment is missing.`);
221
+ else {
222
+ if (comparison.refreshRealization !== treatment.refreshExecution.realization) {
223
+ contradiction(id, `${candidate} comparison realization ${comparison.refreshRealization} differs from the treatment realization ${treatment.refreshExecution.realization}.`);
224
+ }
225
+ const expected = classifyPartialRefreshAgainstFull({
226
+ realization: treatment.refreshExecution.realization,
227
+ correctnessRelation: comparison.correctnessRelation,
228
+ requiredFileEvidenceRelation: comparison.requiredFileEvidenceRelation
229
+ });
230
+ if (expected.classification !== comparison.classification || !sameList(expected.reasonCodes, comparison.reasonCodes)) {
231
+ contradiction(id, `${candidate} comparison does not follow the partial-refresh reference rules.`);
232
+ }
233
+ }
234
+ }
235
+ if (scenario.lifecycle) {
236
+ for (const count of [...Object.values(scenario.lifecycle.indexInvocationCounts), ...Object.values(scenario.lifecycle.freshnessAssessmentCounts)]) {
237
+ if (!Number.isInteger(count) || count < 0)
238
+ contradiction(id, "lifecycle counts must be non-negative integers.");
239
+ }
240
+ }
241
+ else {
242
+ contradiction(id, "a ready scenario must carry a lifecycle summary.");
243
+ }
244
+ }
245
+ export function validateIncrementalChangeStalenessExecutionArtifactV2(artifact) {
246
+ if (artifact.schemaVersion !== INCREMENTAL_CHANGE_STALENESS_EXECUTION_SCHEMA_VERSION_V2) {
247
+ throw new Error(`Unexpected execution artifact schema ${String(artifact.schemaVersion)}.`);
248
+ }
249
+ for (const scenario of artifact.scenarios)
250
+ validateIncrementalChangeStalenessScenarioRecordV2(scenario);
251
+ }
252
+ /**
253
+ * Builds and validates the v2 artifact from already-executed scenario evidence, in the caller's canonical
254
+ * scenario order. Throws on internal contradiction instead of serializing it.
255
+ */
256
+ export function buildIncrementalChangeStalenessExecutionArtifactV2(args) {
257
+ const scenarios = args.executions.map(summarizeScenarioV2);
258
+ const artifact = {
259
+ schemaVersion: INCREMENTAL_CHANGE_STALENESS_EXECUTION_SCHEMA_VERSION_V2,
260
+ runId: args.runId,
261
+ pluginId: args.pluginId,
262
+ scenarios,
263
+ summary: summarizeV2(scenarios)
264
+ };
265
+ validateIncrementalChangeStalenessExecutionArtifactV2(artifact);
266
+ return artifact;
267
+ }
268
+ /** Writes to the canonical filename shared with v1; validates first so contradictory evidence is never serialized. */
269
+ export async function writeIncrementalChangeStalenessExecutionArtifactV2(outDir, artifact) {
270
+ validateIncrementalChangeStalenessExecutionArtifactV2(artifact);
271
+ await mkdir(outDir, { recursive: true });
272
+ const artifactPath = resolveWithinRoot(outDir, INCREMENTAL_CHANGE_STALENESS_EXECUTION_ARTIFACT_FILE);
273
+ await writeFile(artifactPath, `${JSON.stringify(artifact, null, 2)}\n`, "utf8");
274
+ return artifactPath;
275
+ }
276
+ /** Discriminates V1 from V2 by schemaVersion only; historical V1 evidence is never converted to V2. */
277
+ export function parseIncrementalChangeStalenessExecutionArtifact(json) {
278
+ const schemaVersion = json && typeof json === "object" ? json.schemaVersion : undefined;
279
+ if (schemaVersion === INCREMENTAL_CHANGE_STALENESS_EXECUTION_SCHEMA_VERSION) {
280
+ return { version: "v1", artifact: json };
281
+ }
282
+ if (schemaVersion === INCREMENTAL_CHANGE_STALENESS_EXECUTION_SCHEMA_VERSION_V2) {
283
+ return { version: "v2", artifact: json };
284
+ }
285
+ throw new Error(`Unrecognized incremental-change-staleness execution artifact schema: ${String(schemaVersion)}.`);
286
+ }
287
+ export async function readIncrementalChangeStalenessExecutionArtifact(artifactPath) {
288
+ return parseIncrementalChangeStalenessExecutionArtifact(JSON.parse(await readFile(artifactPath, "utf8")));
289
+ }