@dailephd/my-dev-kit-lab 0.6.1 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/README.md +3 -2
- package/benchmarks/contracts/incremental-change-staleness-scenarios.json +360 -0
- package/dist/src/commands/runExperimentRunCommand.js +4 -3
- package/dist/src/evaluation/affectedNeighborhoodMetrics.js +51 -0
- package/dist/src/experiments/defaultRegistry.js +2 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/comparison.js +101 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/config.js +66 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/disposableTarget.js +107 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/execution.js +404 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/executionArtifact.js +160 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/index.js +13 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/lifecycle.js +276 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/lifecyclePolicy.js +152 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/mutationExecution.js +144 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/plugin.js +274 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/scenarioCatalog.js +42 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/scenarioMutation.js +116 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/scenarioSelection.js +79 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/scenarioTypes.js +37 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/scenarioValidation.js +168 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/sourceState.js +145 -0
- package/dist/src/experiments/plugins/incrementalChangeStaleness/treatmentSession.js +6 -0
- package/dist/src/experiments/plugins/index.js +1 -0
- package/dist/src/experiments/plugins/warmIndexReuse/metrics.js +17 -20
- package/dist/src/report/experiments/buildIncrementalChangeStalenessReport.js +112 -0
- package/dist/src/report/experiments/buildPluginExperimentReport.js +3 -0
- package/dist/src/report/experiments/incrementalChangeStalenessReportModel.js +4 -0
- package/dist/src/report/experiments/index.js +3 -0
- package/dist/src/report/experiments/renderIncrementalChangeStalenessHtml.js +239 -0
- package/dist/src/report/experiments/renderPluginExperimentReportHtml.js +3 -0
- package/dist/src/report/experiments/renderPluginExperimentReportText.js +138 -0
- package/docs/METRICS.md +35 -1
- package/package.json +1 -1
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
// v0.6.2 Batch 5 -- plugin-specific HTML section rendering only. Renders exactly the persisted
|
|
2
|
+
// report-model fields; performs no classification, no metric arithmetic, and no percent
|
|
3
|
+
// calculation beyond formatting an already-persisted number.
|
|
4
|
+
import { STALE_RISK_CLASSIFICATION_EXPLANATIONS } from "./buildIncrementalChangeStalenessReport.js";
|
|
5
|
+
import { REINDEX_RECOMMENDATION_EXPLANATIONS } from "./buildWarmIndexReuseReport.js";
|
|
6
|
+
const MAX_DISPLAY_ITEMS = 20;
|
|
7
|
+
export function renderIncrementalChangeStalenessHtml(section) {
|
|
8
|
+
if (section === null) {
|
|
9
|
+
return `<section>
|
|
10
|
+
<h2>Incremental-Change And Staleness Evidence</h2>
|
|
11
|
+
<p>Not applicable to this plugin.</p>
|
|
12
|
+
</section>`;
|
|
13
|
+
}
|
|
14
|
+
return `<section>
|
|
15
|
+
<h2>Incremental-Change And Staleness Evidence</h2>
|
|
16
|
+
<p class="muted">Scoped experimental evidence: results are limited to the executed scenarios below, not a general safety verdict.</p>
|
|
17
|
+
${table(["Field", "Value"], [
|
|
18
|
+
["Scenarios", String(section.scenarioCount)],
|
|
19
|
+
["Ready", String(section.readyScenarioCount)],
|
|
20
|
+
["Failed", String(section.failedScenarioCount)],
|
|
21
|
+
["Observed stale regression", String(section.observedStaleRegressionCount)],
|
|
22
|
+
["No observed stale regression", String(section.noObservedStaleRegressionCount)],
|
|
23
|
+
["Inconclusive", String(section.inconclusiveCount)]
|
|
24
|
+
])}
|
|
25
|
+
<h3>Scenario Comparison</h3>
|
|
26
|
+
${table([
|
|
27
|
+
"Scenario",
|
|
28
|
+
"Category",
|
|
29
|
+
"Status",
|
|
30
|
+
"Relationship",
|
|
31
|
+
"Reindex recommendation",
|
|
32
|
+
"Stale correctness",
|
|
33
|
+
"Full-refresh correctness",
|
|
34
|
+
"Correctness relation",
|
|
35
|
+
"Stale required files",
|
|
36
|
+
"Full-refresh required files",
|
|
37
|
+
"Required-file relation",
|
|
38
|
+
"Stale-risk classification"
|
|
39
|
+
], section.scenarios.map(comparisonRow))}
|
|
40
|
+
${section.scenarios.map(renderScenarioDetail).join("\n")}
|
|
41
|
+
<h3>Limitations</h3>
|
|
42
|
+
${list(section.limitations)}
|
|
43
|
+
</section>`;
|
|
44
|
+
}
|
|
45
|
+
function symmetricAffectedNeighborhood(scenario) {
|
|
46
|
+
const stale = scenario.staleTreatment?.affectedNeighborhood ?? null;
|
|
47
|
+
const fullRefresh = scenario.fullRefreshTreatment?.affectedNeighborhood ?? null;
|
|
48
|
+
if (!stale || !fullRefresh)
|
|
49
|
+
return null;
|
|
50
|
+
if (stale.relationship !== fullRefresh.relationship || stale.reindexRecommendation !== fullRefresh.reindexRecommendation) {
|
|
51
|
+
return null;
|
|
52
|
+
}
|
|
53
|
+
return stale;
|
|
54
|
+
}
|
|
55
|
+
function comparisonRow(scenario) {
|
|
56
|
+
const stale = scenario.staleTreatment?.affectedNeighborhood ?? null;
|
|
57
|
+
const fullRefresh = scenario.fullRefreshTreatment?.affectedNeighborhood ?? null;
|
|
58
|
+
const symmetric = symmetricAffectedNeighborhood(scenario);
|
|
59
|
+
const inconsistent = stale !== null && fullRefresh !== null && symmetric === null;
|
|
60
|
+
return [
|
|
61
|
+
`${scenario.scenarioId}`,
|
|
62
|
+
scenario.category,
|
|
63
|
+
scenario.status,
|
|
64
|
+
inconsistent ? "inconsistent persisted evidence" : symmetric?.relationship ?? "not available",
|
|
65
|
+
inconsistent ? "inconsistent persisted evidence" : symmetric?.reindexRecommendation ?? "not available",
|
|
66
|
+
formatCorrectness(scenario.staleTreatment),
|
|
67
|
+
formatCorrectness(scenario.fullRefreshTreatment),
|
|
68
|
+
scenario.comparison.correctnessRelation,
|
|
69
|
+
scenario.staleTreatment?.requiredFileEvidence.status ?? "not available",
|
|
70
|
+
scenario.fullRefreshTreatment?.requiredFileEvidence.status ?? "not available",
|
|
71
|
+
scenario.comparison.requiredFileEvidenceRelation,
|
|
72
|
+
scenario.comparison.staleRiskClassification
|
|
73
|
+
];
|
|
74
|
+
}
|
|
75
|
+
function formatCorrectness(treatment) {
|
|
76
|
+
if (!treatment || !treatment.fakeAgent)
|
|
77
|
+
return "unavailable";
|
|
78
|
+
const correctness = treatment.fakeAgent.correctness;
|
|
79
|
+
return correctness.available && correctness.score !== null ? String(correctness.score) : "unavailable";
|
|
80
|
+
}
|
|
81
|
+
function renderScenarioDetail(scenario) {
|
|
82
|
+
return `<h3>Scenario: ${escapeHtml(scenario.scenarioId)}</h3>
|
|
83
|
+
${renderDefinition(scenario)}
|
|
84
|
+
${renderLifecycle(scenario)}
|
|
85
|
+
${renderAffectedNeighborhoodSection(scenario)}
|
|
86
|
+
${renderTreatment("Stale-Index Treatment", scenario.staleTreatment)}
|
|
87
|
+
${renderTreatment("Full-Refresh Treatment", scenario.fullRefreshTreatment)}
|
|
88
|
+
${renderComparison(scenario)}`;
|
|
89
|
+
}
|
|
90
|
+
function renderDefinition(scenario) {
|
|
91
|
+
return `<h4>Scenario Definition</h4>
|
|
92
|
+
${table(["Field", "Value"], [
|
|
93
|
+
["Scenario ID", scenario.scenarioId],
|
|
94
|
+
["Category", scenario.category],
|
|
95
|
+
["Benchmark project", scenario.benchmarkProjectId],
|
|
96
|
+
["Base case", scenario.baseCaseId],
|
|
97
|
+
["Answer policy", scenario.answerPolicy],
|
|
98
|
+
["Query", scenario.query ?? "unavailable"],
|
|
99
|
+
["Status", scenario.status],
|
|
100
|
+
["Failure reason", scenario.failureReason ?? "not applicable"]
|
|
101
|
+
])}
|
|
102
|
+
<p>Expected files:</p>
|
|
103
|
+
${boundedList(scenario.expectedFiles)}
|
|
104
|
+
<p>Expected symbols:</p>
|
|
105
|
+
${boundedList(scenario.expectedSymbols)}`;
|
|
106
|
+
}
|
|
107
|
+
function renderLifecycle(scenario) {
|
|
108
|
+
const lifecycle = scenario.lifecycle;
|
|
109
|
+
if (!lifecycle) {
|
|
110
|
+
return `<h4>Lifecycle Integrity</h4><p>Not available for this scenario.</p>`;
|
|
111
|
+
}
|
|
112
|
+
return `<h4>Lifecycle Integrity</h4>
|
|
113
|
+
${table(["Field", "Value"], [
|
|
114
|
+
["Pre-mutation equivalence", lifecycle.preMutationEquivalence],
|
|
115
|
+
["Post-mutation equivalence", lifecycle.postMutationEquivalence],
|
|
116
|
+
["my-dev-kit version", lifecycle.myDevKitVersion ?? "unavailable"],
|
|
117
|
+
["Stale baseline freshness", lifecycle.staleBaselineFreshnessStatus],
|
|
118
|
+
["Full-refresh baseline freshness", lifecycle.fullRefreshBaselineFreshnessStatus],
|
|
119
|
+
["Full-refresh refreshed freshness", lifecycle.fullRefreshRefreshedFreshnessStatus ?? "unavailable"]
|
|
120
|
+
])}
|
|
121
|
+
<p>Controlled file paths:</p>
|
|
122
|
+
${boundedList(lifecycle.controlledFilePaths)}
|
|
123
|
+
<p>Source roots:</p>
|
|
124
|
+
${boundedList(lifecycle.sourceRoots)}`;
|
|
125
|
+
}
|
|
126
|
+
function renderAffectedNeighborhoodSection(scenario) {
|
|
127
|
+
const stale = scenario.staleTreatment?.affectedNeighborhood ?? null;
|
|
128
|
+
const fullRefresh = scenario.fullRefreshTreatment?.affectedNeighborhood ?? null;
|
|
129
|
+
if (!stale && !fullRefresh) {
|
|
130
|
+
return `<h4>Affected-Neighborhood Evidence (Changed Baseline Neighborhood Relative To The Task)</h4><p>Not available.</p>`;
|
|
131
|
+
}
|
|
132
|
+
const symmetric = symmetricAffectedNeighborhood(scenario);
|
|
133
|
+
if (!symmetric) {
|
|
134
|
+
return `<h4>Affected-Neighborhood Evidence (Changed Baseline Neighborhood Relative To The Task)</h4><p>inconsistent persisted evidence</p>`;
|
|
135
|
+
}
|
|
136
|
+
const explanation = REINDEX_RECOMMENDATION_EXPLANATIONS[symmetric.reindexRecommendation];
|
|
137
|
+
return `<h4>Affected-Neighborhood Evidence (Changed Baseline Neighborhood Relative To The Task)</h4>
|
|
138
|
+
${table(["Field", "Value"], [
|
|
139
|
+
["Relationship", symmetric.relationship],
|
|
140
|
+
["Reindex recommendation", symmetric.reindexRecommendation],
|
|
141
|
+
["Changed indexed files", formatAffectedNumber(symmetric.changedFileCount)],
|
|
142
|
+
["Changed baseline symbols", formatAffectedNumber(symmetric.changedSymbolCount)],
|
|
143
|
+
["Affected graph nodes", formatAffectedNumber(symmetric.affectedNodeCount)],
|
|
144
|
+
["Affected graph edges", formatAffectedNumber(symmetric.affectedEdgeCount)],
|
|
145
|
+
["Task-overlap nodes", formatAffectedNumber(symmetric.taskOverlapCount)],
|
|
146
|
+
["Task-overlap percent", formatAffectedNumber(symmetric.taskOverlapPercent)]
|
|
147
|
+
])}
|
|
148
|
+
<p>${escapeHtml(explanation)}</p>`;
|
|
149
|
+
}
|
|
150
|
+
function renderTreatment(title, treatment) {
|
|
151
|
+
if (!treatment) {
|
|
152
|
+
return `<h4>${escapeHtml(title)}</h4><p>Not available.</p>`;
|
|
153
|
+
}
|
|
154
|
+
const retrieval = treatment.retrieval;
|
|
155
|
+
const fakeAgent = treatment.fakeAgent;
|
|
156
|
+
const requiredFileEvidence = treatment.requiredFileEvidence;
|
|
157
|
+
return `<h4>${escapeHtml(title)}</h4>
|
|
158
|
+
${table(["Field", "Value"], [
|
|
159
|
+
["Treatment status", treatment.status],
|
|
160
|
+
["Failure reason", treatment.failureReason ?? "not applicable"],
|
|
161
|
+
["Active index phase", treatment.activeIndexPhase],
|
|
162
|
+
["Baseline freshness status", treatment.baselineFreshness.status]
|
|
163
|
+
])}
|
|
164
|
+
<p><strong>Retrieval</strong> (bounded evidence; full retrieved context body is never rendered here)</p>
|
|
165
|
+
${table(["Field", "Value"], [
|
|
166
|
+
["Status", retrieval.status],
|
|
167
|
+
["Selected node ID", retrieval.selectedNodeId ?? "unavailable"],
|
|
168
|
+
["Selected file", retrieval.selectedFile ?? "unavailable"],
|
|
169
|
+
["Selected symbol", retrieval.selectedSymbol ?? "unavailable"],
|
|
170
|
+
["Context characters", String(retrieval.totalChars)],
|
|
171
|
+
["Estimated context tokens", String(retrieval.totalEstimatedTokens)],
|
|
172
|
+
["Duration (ms)", String(retrieval.durationMs)]
|
|
173
|
+
])}
|
|
174
|
+
<p>Files read:</p>
|
|
175
|
+
${boundedList(retrieval.filesRead)}
|
|
176
|
+
<p>Retrieval warnings:</p>
|
|
177
|
+
${boundedList(retrieval.warnings)}
|
|
178
|
+
<p><strong>Fake-Agent Evidence</strong> (deterministic fake-agent / simulated harness evidence, not a provider or real-model measurement)</p>
|
|
179
|
+
${fakeAgent
|
|
180
|
+
? table(["Field", "Value"], [
|
|
181
|
+
["Status", fakeAgent.status],
|
|
182
|
+
["Correctness score", fakeAgent.correctness.available && fakeAgent.correctness.score !== null ? String(fakeAgent.correctness.score) : "unavailable"],
|
|
183
|
+
["Passed", fakeAgent.correctness.passed === null ? "unavailable" : String(fakeAgent.correctness.passed)],
|
|
184
|
+
["Duration (ms)", fakeAgent.durationMs === null ? "unavailable" : String(fakeAgent.durationMs)]
|
|
185
|
+
])
|
|
186
|
+
: "<p>Not run for this treatment.</p>"}
|
|
187
|
+
<p><strong>Required-File Evidence</strong> (bounded answer-key file-presence check, not retrieval precision/recall)</p>
|
|
188
|
+
${table(["Field", "Value"], [
|
|
189
|
+
["Status", requiredFileEvidence.status],
|
|
190
|
+
["Reason", requiredFileEvidence.reason ?? "not applicable"]
|
|
191
|
+
])}
|
|
192
|
+
<p>Required files:</p>
|
|
193
|
+
${boundedList(requiredFileEvidence.requiredFiles)}
|
|
194
|
+
<p>Observed files:</p>
|
|
195
|
+
${boundedList(requiredFileEvidence.observedFiles)}
|
|
196
|
+
<p>Missing files:</p>
|
|
197
|
+
${boundedList(requiredFileEvidence.missingFiles)}`;
|
|
198
|
+
}
|
|
199
|
+
function renderComparison(scenario) {
|
|
200
|
+
const comparison = scenario.comparison;
|
|
201
|
+
const explanation = STALE_RISK_CLASSIFICATION_EXPLANATIONS[comparison.staleRiskClassification];
|
|
202
|
+
return `<h4>Matched Comparison</h4>
|
|
203
|
+
${table(["Field", "Value"], [
|
|
204
|
+
["Correctness relation", comparison.correctnessRelation],
|
|
205
|
+
["Required-file evidence relation", comparison.requiredFileEvidenceRelation],
|
|
206
|
+
["Stale-risk classification", comparison.staleRiskClassification]
|
|
207
|
+
])}
|
|
208
|
+
<p>Reason codes:</p>
|
|
209
|
+
${list(comparison.reasonCodes)}
|
|
210
|
+
<p>${escapeHtml(explanation)}</p>`;
|
|
211
|
+
}
|
|
212
|
+
function formatAffectedNumber(value) {
|
|
213
|
+
return value === null ? "unavailable" : String(value);
|
|
214
|
+
}
|
|
215
|
+
function boundedList(items) {
|
|
216
|
+
if (items.length === 0)
|
|
217
|
+
return "<p>none</p>";
|
|
218
|
+
const shown = items.slice(0, MAX_DISPLAY_ITEMS);
|
|
219
|
+
const omitted = items.length - shown.length;
|
|
220
|
+
return `${list(shown)}${omitted > 0 ? `<p class="muted">... ${omitted} more</p>` : ""}`;
|
|
221
|
+
}
|
|
222
|
+
function list(items) {
|
|
223
|
+
if (items.length === 0)
|
|
224
|
+
return "<ul><li>none</li></ul>";
|
|
225
|
+
return `<ul>${items.map((item) => `<li>${escapeHtml(item)}</li>`).join("")}</ul>`;
|
|
226
|
+
}
|
|
227
|
+
function table(headers, rows) {
|
|
228
|
+
return `<table><thead><tr>${headers.map((header) => `<th>${escapeHtml(header)}</th>`).join("")}</tr></thead><tbody>${rows
|
|
229
|
+
.map((row) => `<tr>${row.map((cell) => `<td>${escapeHtml(cell)}</td>`).join("")}</tr>`)
|
|
230
|
+
.join("")}</tbody></table>`;
|
|
231
|
+
}
|
|
232
|
+
function escapeHtml(value) {
|
|
233
|
+
return String(value ?? "")
|
|
234
|
+
.replaceAll("&", "&")
|
|
235
|
+
.replaceAll("<", "<")
|
|
236
|
+
.replaceAll(">", ">")
|
|
237
|
+
.replaceAll('"', """)
|
|
238
|
+
.replaceAll("'", "'");
|
|
239
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { renderContextStrategyComparisonV043Html } from "./renderContextStrategyComparisonV043Html.js";
|
|
2
2
|
import { renderWarmIndexReuseHtml } from "./renderWarmIndexReuseHtml.js";
|
|
3
|
+
import { renderIncrementalChangeStalenessHtml } from "./renderIncrementalChangeStalenessHtml.js";
|
|
3
4
|
export function renderPluginExperimentReportHtml(report) {
|
|
4
5
|
return `<!DOCTYPE html>
|
|
5
6
|
<html lang="en">
|
|
@@ -99,6 +100,8 @@ export function renderPluginExperimentReportHtml(report) {
|
|
|
99
100
|
|
|
100
101
|
${renderWarmIndexReuseHtml(report.warmIndexReuse)}
|
|
101
102
|
|
|
103
|
+
${renderIncrementalChangeStalenessHtml(report.incrementalChangeStaleness)}
|
|
104
|
+
|
|
102
105
|
<section>
|
|
103
106
|
<h2>Warnings, Skips, And Failures</h2>
|
|
104
107
|
${report.findings.length ? table(["Severity", "Code", "Message", "Variant", "Case"], report.findings.map((finding) => [
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { STALE_RISK_CLASSIFICATION_EXPLANATIONS } from "./buildIncrementalChangeStalenessReport.js";
|
|
2
|
+
import { REINDEX_RECOMMENDATION_EXPLANATIONS } from "./buildWarmIndexReuseReport.js";
|
|
1
3
|
function sanitizeScalar(value) {
|
|
2
4
|
const text = String(value);
|
|
3
5
|
let result = "";
|
|
@@ -489,6 +491,140 @@ function renderTaskFreshnessDetail(lines, freshness) {
|
|
|
489
491
|
lines.push(`- ${entry.path === null ? "run-level" : sanitizeScalar(entry.path)} [${sanitizeScalar(entry.reasonCode)}] ${sanitizeScalar(entry.message)}`);
|
|
490
492
|
}
|
|
491
493
|
}
|
|
494
|
+
const INCREMENTAL_CHANGE_STALENESS_MAX_DISPLAY_ITEMS = 20;
|
|
495
|
+
function pushBoundedStringList(lines, label, items) {
|
|
496
|
+
lines.push(`${label}:`);
|
|
497
|
+
if (items.length === 0) {
|
|
498
|
+
lines.push("- none");
|
|
499
|
+
return;
|
|
500
|
+
}
|
|
501
|
+
const shown = items.slice(0, INCREMENTAL_CHANGE_STALENESS_MAX_DISPLAY_ITEMS);
|
|
502
|
+
for (const item of shown)
|
|
503
|
+
lines.push(`- ${sanitizeScalar(item)}`);
|
|
504
|
+
const omitted = items.length - shown.length;
|
|
505
|
+
if (omitted > 0)
|
|
506
|
+
lines.push(`... ${omitted} more`);
|
|
507
|
+
}
|
|
508
|
+
function symmetricIncrementalAffectedNeighborhood(scenario) {
|
|
509
|
+
const stale = scenario.staleTreatment?.affectedNeighborhood ?? null;
|
|
510
|
+
const fullRefresh = scenario.fullRefreshTreatment?.affectedNeighborhood ?? null;
|
|
511
|
+
if (!stale || !fullRefresh)
|
|
512
|
+
return null;
|
|
513
|
+
if (stale.relationship !== fullRefresh.relationship || stale.reindexRecommendation !== fullRefresh.reindexRecommendation) {
|
|
514
|
+
return null;
|
|
515
|
+
}
|
|
516
|
+
return stale;
|
|
517
|
+
}
|
|
518
|
+
function renderIncrementalChangeStalenessTreatment(lines, label, treatment) {
|
|
519
|
+
pushSection(lines, label);
|
|
520
|
+
if (!treatment) {
|
|
521
|
+
lines.push("Not available.");
|
|
522
|
+
return;
|
|
523
|
+
}
|
|
524
|
+
lines.push(fieldLine("Treatment Status", treatment.status));
|
|
525
|
+
lines.push(fieldLine("Failure Reason", treatment.failureReason));
|
|
526
|
+
lines.push(fieldLine("Active Index Phase", treatment.activeIndexPhase));
|
|
527
|
+
lines.push(fieldLine("Baseline Freshness Status", treatment.baselineFreshness.status));
|
|
528
|
+
lines.push("Retrieval (deterministic fake-agent / simulated harness evidence label applies to fakeAgent below):");
|
|
529
|
+
lines.push(fieldLine("Retrieval Status", treatment.retrieval.status));
|
|
530
|
+
lines.push(fieldLine("Selected Node ID", treatment.retrieval.selectedNodeId));
|
|
531
|
+
lines.push(fieldLine("Selected File", treatment.retrieval.selectedFile));
|
|
532
|
+
lines.push(fieldLine("Selected Symbol", treatment.retrieval.selectedSymbol));
|
|
533
|
+
lines.push(fieldLine("Context Characters", treatment.retrieval.totalChars));
|
|
534
|
+
lines.push(fieldLine("Estimated Context Tokens", treatment.retrieval.totalEstimatedTokens));
|
|
535
|
+
lines.push(fieldLine("Retrieval Duration (ms)", treatment.retrieval.durationMs));
|
|
536
|
+
pushBoundedStringList(lines, "Files Read", treatment.retrieval.filesRead);
|
|
537
|
+
pushBoundedStringList(lines, "Retrieval Warnings", treatment.retrieval.warnings);
|
|
538
|
+
lines.push("Fake-Agent Evidence (deterministic fake-agent / simulated harness evidence):");
|
|
539
|
+
if (treatment.fakeAgent) {
|
|
540
|
+
lines.push(fieldLine("Fake-Agent Status", treatment.fakeAgent.status));
|
|
541
|
+
lines.push(fieldLine("Correctness Score", treatment.fakeAgent.correctness.available ? treatment.fakeAgent.correctness.score : null));
|
|
542
|
+
lines.push(fieldLine("Passed", treatment.fakeAgent.correctness.passed));
|
|
543
|
+
lines.push(fieldLine("Fake-Agent Duration (ms)", treatment.fakeAgent.durationMs));
|
|
544
|
+
}
|
|
545
|
+
else {
|
|
546
|
+
lines.push("Not run for this treatment.");
|
|
547
|
+
}
|
|
548
|
+
lines.push("Required-File Evidence (bounded answer-key file-presence check, not retrieval precision/recall):");
|
|
549
|
+
lines.push(fieldLine("Required-File Status", treatment.requiredFileEvidence.status));
|
|
550
|
+
lines.push(fieldLine("Required-File Reason", treatment.requiredFileEvidence.reason));
|
|
551
|
+
pushBoundedStringList(lines, "Required Files", treatment.requiredFileEvidence.requiredFiles);
|
|
552
|
+
pushBoundedStringList(lines, "Observed Files", treatment.requiredFileEvidence.observedFiles);
|
|
553
|
+
pushBoundedStringList(lines, "Missing Files", treatment.requiredFileEvidence.missingFiles);
|
|
554
|
+
}
|
|
555
|
+
function renderIncrementalChangeStalenessScenario(lines, scenario) {
|
|
556
|
+
pushSection(lines, `Scenario: ${scenario.scenarioId}`);
|
|
557
|
+
lines.push(fieldLine("Category", scenario.category));
|
|
558
|
+
lines.push(fieldLine("Status", scenario.status));
|
|
559
|
+
lines.push(fieldLine("Failure Reason", scenario.failureReason));
|
|
560
|
+
lines.push(fieldLine("Benchmark Project", scenario.benchmarkProjectId));
|
|
561
|
+
lines.push(fieldLine("Base Case", scenario.baseCaseId));
|
|
562
|
+
lines.push(fieldLine("Answer Policy", scenario.answerPolicy));
|
|
563
|
+
lines.push(fieldLine("Query", scenario.query));
|
|
564
|
+
pushBoundedStringList(lines, "Expected Files", scenario.expectedFiles);
|
|
565
|
+
pushBoundedStringList(lines, "Expected Symbols", scenario.expectedSymbols);
|
|
566
|
+
pushSection(lines, "Lifecycle");
|
|
567
|
+
const lifecycle = scenario.lifecycle;
|
|
568
|
+
if (!lifecycle) {
|
|
569
|
+
lines.push("Not available.");
|
|
570
|
+
}
|
|
571
|
+
else {
|
|
572
|
+
lines.push(fieldLine("Pre-Mutation Equivalence", lifecycle.preMutationEquivalence));
|
|
573
|
+
lines.push(fieldLine("Post-Mutation Equivalence", lifecycle.postMutationEquivalence));
|
|
574
|
+
lines.push(fieldLine("my-dev-kit Version", lifecycle.myDevKitVersion));
|
|
575
|
+
lines.push(fieldLine("Stale Baseline Freshness Status", lifecycle.staleBaselineFreshnessStatus));
|
|
576
|
+
lines.push(fieldLine("Full-Refresh Baseline Freshness Status", lifecycle.fullRefreshBaselineFreshnessStatus));
|
|
577
|
+
lines.push(fieldLine("Full-Refresh Refreshed Freshness Status", lifecycle.fullRefreshRefreshedFreshnessStatus));
|
|
578
|
+
pushBoundedStringList(lines, "Controlled File Paths", lifecycle.controlledFilePaths);
|
|
579
|
+
pushBoundedStringList(lines, "Source Roots", lifecycle.sourceRoots);
|
|
580
|
+
}
|
|
581
|
+
pushSection(lines, "Affected-Neighborhood Evidence (changed baseline neighborhood relative to the task)");
|
|
582
|
+
const symmetric = symmetricIncrementalAffectedNeighborhood(scenario);
|
|
583
|
+
const bothPresent = scenario.staleTreatment?.affectedNeighborhood && scenario.fullRefreshTreatment?.affectedNeighborhood;
|
|
584
|
+
if (!scenario.staleTreatment?.affectedNeighborhood && !scenario.fullRefreshTreatment?.affectedNeighborhood) {
|
|
585
|
+
lines.push("Not available.");
|
|
586
|
+
}
|
|
587
|
+
else if (bothPresent && !symmetric) {
|
|
588
|
+
lines.push("inconsistent persisted evidence");
|
|
589
|
+
}
|
|
590
|
+
else if (symmetric) {
|
|
591
|
+
lines.push(fieldLine("Relationship", symmetric.relationship));
|
|
592
|
+
lines.push(fieldLine("Reindex Recommendation", symmetric.reindexRecommendation));
|
|
593
|
+
lines.push(fieldLine("Reindex Recommendation Explanation", REINDEX_RECOMMENDATION_EXPLANATIONS[symmetric.reindexRecommendation]));
|
|
594
|
+
lines.push(fieldLine("Changed Indexed Files", symmetric.changedFileCount));
|
|
595
|
+
lines.push(fieldLine("Changed Baseline Symbols", symmetric.changedSymbolCount));
|
|
596
|
+
lines.push(fieldLine("Affected Graph Nodes", symmetric.affectedNodeCount));
|
|
597
|
+
lines.push(fieldLine("Affected Graph Edges", symmetric.affectedEdgeCount));
|
|
598
|
+
lines.push(fieldLine("Task-Overlap Nodes", symmetric.taskOverlapCount));
|
|
599
|
+
lines.push(fieldLine("Task-Overlap Percent", symmetric.taskOverlapPercent));
|
|
600
|
+
}
|
|
601
|
+
renderIncrementalChangeStalenessTreatment(lines, "Stale-Index Treatment", scenario.staleTreatment);
|
|
602
|
+
renderIncrementalChangeStalenessTreatment(lines, "Full-Refresh Treatment", scenario.fullRefreshTreatment);
|
|
603
|
+
pushSection(lines, "Matched Comparison");
|
|
604
|
+
lines.push(fieldLine("Correctness Relation", scenario.comparison.correctnessRelation));
|
|
605
|
+
lines.push(fieldLine("Required-File Evidence Relation", scenario.comparison.requiredFileEvidenceRelation));
|
|
606
|
+
lines.push(fieldLine("Stale-Risk Classification", scenario.comparison.staleRiskClassification));
|
|
607
|
+
pushDashList(lines, scenario.comparison.reasonCodes);
|
|
608
|
+
lines.push(STALE_RISK_CLASSIFICATION_EXPLANATIONS[scenario.comparison.staleRiskClassification]);
|
|
609
|
+
}
|
|
610
|
+
function renderIncrementalChangeStalenessSection(lines, section) {
|
|
611
|
+
if (section === null) {
|
|
612
|
+
lines.push("Not applicable to this plugin.");
|
|
613
|
+
return;
|
|
614
|
+
}
|
|
615
|
+
pushSection(lines, "Summary");
|
|
616
|
+
lines.push(fieldLine("Scenarios", section.scenarioCount));
|
|
617
|
+
lines.push(fieldLine("Ready", section.readyScenarioCount));
|
|
618
|
+
lines.push(fieldLine("Failed", section.failedScenarioCount));
|
|
619
|
+
lines.push(fieldLine("Observed Stale Regression", section.observedStaleRegressionCount));
|
|
620
|
+
lines.push(fieldLine("No Observed Stale Regression", section.noObservedStaleRegressionCount));
|
|
621
|
+
lines.push(fieldLine("Inconclusive", section.inconclusiveCount));
|
|
622
|
+
for (const scenario of section.scenarios) {
|
|
623
|
+
renderIncrementalChangeStalenessScenario(lines, scenario);
|
|
624
|
+
}
|
|
625
|
+
pushSection(lines, "Limitations");
|
|
626
|
+
pushDashList(lines, section.limitations);
|
|
627
|
+
}
|
|
492
628
|
export function renderPluginExperimentReportText(report) {
|
|
493
629
|
const lines = [];
|
|
494
630
|
pushSection(lines, "Plugin Experiment Report");
|
|
@@ -519,6 +655,8 @@ export function renderPluginExperimentReportText(report) {
|
|
|
519
655
|
renderV043Section(lines, report.contextStrategyComparisonV043);
|
|
520
656
|
pushSection(lines, "Warm Index Reuse Evidence");
|
|
521
657
|
renderWarmIndexReuseSection(lines, report.warmIndexReuse);
|
|
658
|
+
pushSection(lines, "Incremental-Change And Staleness Evidence");
|
|
659
|
+
renderIncrementalChangeStalenessSection(lines, report.incrementalChangeStaleness);
|
|
522
660
|
pushSection(lines, "Warnings, Skips, And Failures");
|
|
523
661
|
pushDashList(lines, report.findings.map((finding) => `${finding.severity} ${finding.code} ${finding.message} variant=${finding.variantId ?? ""} case=${finding.caseId ?? ""}`));
|
|
524
662
|
pushSection(lines, "Artifacts");
|
package/docs/METRICS.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Metrics
|
|
2
2
|
|
|
3
|
-
This document is the canonical metric glossary for my-dev-kit-lab. It defines every metric that appears in benchmark profiles, prompt variants, controlled experiment artifacts, and rendered reports.
|
|
3
|
+
This document is the canonical metric glossary for my-dev-kit-lab. It defines every metric that appears in benchmark profiles, prompt variants, controlled experiment artifacts, and rendered reports across the registered experiment plugins (`context-strategy-comparison`, `warm-index-reuse`, and `incremental-change-staleness`).
|
|
4
4
|
|
|
5
5
|
Related documentation:
|
|
6
6
|
- [ARCHITECTURE.md](ARCHITECTURE.md) — how metrics flow through the pipeline
|
|
@@ -595,3 +595,37 @@ All six metrics are warm-side only (they are not on the raw-full-file side, beca
|
|
|
595
595
|
**Report presentation.** `report.warmIndexReuse.affectedNeighborhoodSummary` holds presentation counts (assessed and unassessed tasks; complete, partial, and unavailable assessments; related, unrelated, and unknown relationships; recommended, not-indicated, and unknown recommendations). They are counts, not metric formulas. Each task's `affectedNeighborhood` block shows the statuses, the six metric objects, the relationship, the recommendation with a fixed explanation, and bounded lists (at most 20 affected node IDs, 20 edge IDs, 10 unresolved and 10 ambiguous task mappings, and 10 warnings, each with its total and omitted count). The full identity arrays stay in `warm-index-execution.json`.
|
|
596
596
|
|
|
597
597
|
**Observational only.** The evidence never triggers reindexing, suppresses or alters warm retrieval, or changes execution, provider, agent, correctness, or token-evidence status. It uses only the baseline graph of the prepared index: there is no graph-diff, refreshed-index, or stale-versus-refreshed comparison.
|
|
598
|
+
|
|
599
|
+
## Incremental-change and staleness evidence (v0.6.2)
|
|
600
|
+
|
|
601
|
+
The `incremental-change-staleness` plugin reuses the six existing v0.6.1 affected-neighborhood numeric metrics unchanged (`changedFileCount`, `changedSymbolCount`, `affectedNodeCount`, `affectedEdgeCount`, `taskOverlapCount`, `taskOverlapPercent`; see "Affected-neighborhood metrics (v0.6.1)" above) and adds no new numeric formula.
|
|
602
|
+
|
|
603
|
+
**Correctness relation** (per scenario, comparing the `stale-index` and `full-refresh` treatments' deterministic fake-agent correctness):
|
|
604
|
+
|
|
605
|
+
- `stale-worse`: `stale-index` correctness is lower than `full-refresh` correctness.
|
|
606
|
+
- `same`: both treatments have available, equal correctness.
|
|
607
|
+
- `stale-better`: `stale-index` correctness is higher than `full-refresh` correctness.
|
|
608
|
+
- `unknown`: either treatment's correctness is unavailable.
|
|
609
|
+
|
|
610
|
+
**Required-file status** (per treatment, an answer-key file-presence check against that treatment's actual retrieved-files evidence — not retrieval precision, recall, F1, MRR, NDCG, or symbol recall):
|
|
611
|
+
|
|
612
|
+
- `present`: every required file was observed.
|
|
613
|
+
- `missing`: at least one required file was not observed, and the observed-files evidence was complete.
|
|
614
|
+
- `unknown`: the actual-files-read evidence was not complete enough to establish presence or absence.
|
|
615
|
+
|
|
616
|
+
**Required-file relation** (per scenario, comparing the two treatments' required-file status):
|
|
617
|
+
|
|
618
|
+
- `stale-worse`: `stale-index` is `missing` while `full-refresh` is `present`.
|
|
619
|
+
- `same`: both treatments have the same status (including `missing`/`missing`).
|
|
620
|
+
- `stale-better`: `stale-index` is `present` while `full-refresh` is `missing`.
|
|
621
|
+
- `unknown`: either treatment's status is `unknown`.
|
|
622
|
+
|
|
623
|
+
**Stale-risk classification** (per scenario, the frozen precedence over the two relations above):
|
|
624
|
+
|
|
625
|
+
1. `observed-stale-regression` — the correctness relation or the required-file relation is `stale-worse`.
|
|
626
|
+
2. `inconclusive` — otherwise, if the correctness relation or the required-file relation is `unknown`.
|
|
627
|
+
3. `no-observed-stale-regression` — otherwise.
|
|
628
|
+
|
|
629
|
+
`staleRiskClassification` is not a numeric risk score, not a winner, not a ranking, and not an automatic reindex decision. `no-observed-stale-regression` means only that this matched scenario did not observe a stale-specific regression under these two frozen evidence dimensions; it does not establish that stale indexes are generally safe. `reindexRecommendation` (the released v0.6.1 categorical evidence) remains separate observational evidence about the affected-neighborhood analysis; it does not select or trigger either treatment and is not an input to the stale-risk classification above.
|
|
630
|
+
|
|
631
|
+
**Report presentation.** `report.incrementalChangeStaleness` holds `scenarioCount`, `readyScenarioCount`, `failedScenarioCount`, `observedStaleRegressionCount`, `noObservedStaleRegressionCount`, `inconclusiveCount`, per-scenario evidence (lifecycle, both treatments in order, and the comparison), and fixed limitations text. There is no overall score, grade, winner, best-treatment, or safe-to-skip-reindex field anywhere in the report or the persisted execution artifact.
|