@dailephd/my-dev-kit-lab 0.9.2 → 0.9.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -1
- package/README.md +9 -5
- package/dist/src/agents/adapters/codexAdapter.js +4 -1
- package/dist/src/cli/help.js +38 -2
- package/dist/src/cli/runLabCli.js +9 -1
- package/dist/src/commands/buildGalleryCommand.js +16 -3
- package/dist/src/commands/runComparePromptsCommand.js +109 -0
- package/dist/src/commands/runExperimentRunCommand.js +47 -2
- package/dist/src/core/processTree.js +41 -17
- package/dist/src/core/runMeasuredCommand.js +14 -8
- package/dist/src/evaluation/affectedNeighborhood.js +140 -32
- package/dist/src/experiments/plugins/agentSuccessRate/campaignCheckpointStore.js +23 -1
- package/dist/src/experiments/plugins/agentSuccessRate/campaignExecution.js +7 -2
- package/dist/src/experiments/plugins/agentSuccessRate/campaignIdentity.js +29 -4
- package/dist/src/experiments/plugins/agentSuccessRate/campaignResumableExecution.js +1 -1
- package/dist/src/experiments/plugins/agentSuccessRate/config.js +47 -2
- package/dist/src/experiments/plugins/agentSuccessRate/plugin.js +4 -2
- package/dist/src/experiments/plugins/agentSuccessRate/promptContract.js +121 -0
- package/dist/src/experiments/plugins/agentSuccessRate/realAgentExecution.js +32 -2
- package/dist/src/experiments/plugins/agentSuccessRate/realAgentPrompt.js +105 -11
- package/dist/src/experiments/plugins/agentSuccessRate/strictDiffResponse.js +312 -0
- package/dist/src/gallery/galleryPathSafety.js +111 -0
- package/dist/src/gallery/readGallerySources.js +542 -0
- package/dist/src/gallery/writeGalleryManifest.js +57 -3
- package/dist/src/report/experiments/buildPluginExperimentReport.js +4 -16
- package/dist/src/report/experiments/htmlSafety.js +32 -0
- package/dist/src/report/experiments/metricEvidence.js +389 -0
- package/dist/src/report/experiments/metricGlossary.js +227 -0
- package/dist/src/report/experiments/metricGlossaryCatalog.js +199 -0
- package/dist/src/report/experiments/promptTemplateComparisonEvidence.js +377 -0
- package/dist/src/report/experiments/promptTemplateComparisonModel.js +436 -0
- package/dist/src/report/experiments/renderPluginExperimentReportHtml.js +158 -64
- package/dist/src/report/experiments/renderPluginExperimentReportText.js +22 -677
- package/dist/src/report/experiments/renderPromptTemplateComparison.js +192 -0
- package/dist/src/report/experiments/renderSpecializedTextSections.js +633 -0
- package/dist/src/report/experiments/reportSectionRegistry.js +79 -0
- package/dist/src/report/experiments/specializedReportSections.js +142 -0
- package/dist/src/report/experiments/writePluginExperimentReports.js +4 -2
- package/dist/src/securityValidation/attackScenarios/scenarios/secretLeakageScenario.js +146 -133
- package/dist/src/securityValidation/attackScenarios/scenarios/subprocessInjectionScenario.js +54 -36
- package/dist/src/securityValidation/commandRunner.js +13 -6
- package/dist/src/securityValidation/packageChecks/parseNpmPackDryRun.js +46 -21
- package/docs/METRICS.md +11 -1
- package/package.json +3 -3
|
@@ -0,0 +1,633 @@
|
|
|
1
|
+
import { STALE_RISK_CLASSIFICATION_EXPLANATIONS } from "./buildIncrementalChangeStalenessReport.js";
|
|
2
|
+
import { isV2ReportSection } from "./buildIncrementalChangeStalenessReportV2.js";
|
|
3
|
+
import { renderIncrementalChangeStalenessSectionV2 } from "./renderIncrementalChangeStalenessTextV2.js";
|
|
4
|
+
import { REINDEX_RECOMMENDATION_EXPLANATIONS } from "./buildWarmIndexReuseReport.js";
|
|
5
|
+
function sanitizeScalar(value) {
|
|
6
|
+
const text = String(value);
|
|
7
|
+
let result = "";
|
|
8
|
+
for (const ch of text) {
|
|
9
|
+
const code = ch.codePointAt(0) ?? 0;
|
|
10
|
+
if (code === 13 || code === 10 || code === 9) {
|
|
11
|
+
result += " ";
|
|
12
|
+
continue;
|
|
13
|
+
}
|
|
14
|
+
if (code <= 8)
|
|
15
|
+
continue;
|
|
16
|
+
if (code === 11 || code === 12)
|
|
17
|
+
continue;
|
|
18
|
+
if (code >= 14 && code <= 31)
|
|
19
|
+
continue;
|
|
20
|
+
if (code === 127)
|
|
21
|
+
continue;
|
|
22
|
+
result += ch;
|
|
23
|
+
}
|
|
24
|
+
return result;
|
|
25
|
+
}
|
|
26
|
+
export function fieldLine(label, value) {
|
|
27
|
+
if (value === null || value === undefined)
|
|
28
|
+
return `${label}: unavailable`;
|
|
29
|
+
return `${label}: ${sanitizeScalar(value)}`;
|
|
30
|
+
}
|
|
31
|
+
export function pushSection(lines, title) {
|
|
32
|
+
lines.push(title);
|
|
33
|
+
}
|
|
34
|
+
export function pushDashList(lines, items) {
|
|
35
|
+
if (items.length === 0) {
|
|
36
|
+
lines.push("- none");
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
for (const item of items) {
|
|
40
|
+
lines.push(`- ${sanitizeScalar(item)}`);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
function pushBoundedList(lines, list, renderItem) {
|
|
44
|
+
lines.push(`Displayed: ${list.displayedCount} of ${list.totalCount}`);
|
|
45
|
+
lines.push(`Omitted: ${list.omittedCount}`);
|
|
46
|
+
if (list.totalCount === 0) {
|
|
47
|
+
lines.push("- none");
|
|
48
|
+
return;
|
|
49
|
+
}
|
|
50
|
+
for (const item of list.items) {
|
|
51
|
+
lines.push(`- ${sanitizeScalar(renderItem(item))}`);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
function pushRatioMetric(lines, label, metric) {
|
|
55
|
+
lines.push(label);
|
|
56
|
+
lines.push(`Availability: ${metric.availability}`);
|
|
57
|
+
if (metric.availability === "available") {
|
|
58
|
+
lines.push(`Numerator: ${metric.numerator}`);
|
|
59
|
+
lines.push(`Denominator: ${metric.denominator}`);
|
|
60
|
+
lines.push(`Rate: ${metric.rate}`);
|
|
61
|
+
lines.push(`Percentage: ${String(metric.rate * 100)}%`);
|
|
62
|
+
}
|
|
63
|
+
else {
|
|
64
|
+
lines.push(`Reason: ${sanitizeScalar(metric.reason ?? "")}`);
|
|
65
|
+
}
|
|
66
|
+
lines.push("Matched Expectation IDs:");
|
|
67
|
+
pushBoundedList(lines, metric.matchedExpectationIds, (id) => id);
|
|
68
|
+
lines.push("Missing Expectation IDs:");
|
|
69
|
+
pushBoundedList(lines, metric.missingExpectationIds, (id) => id);
|
|
70
|
+
}
|
|
71
|
+
function pushCountMetric(lines, label, metric) {
|
|
72
|
+
lines.push(label);
|
|
73
|
+
lines.push(`Availability: ${metric.availability}`);
|
|
74
|
+
if (metric.availability === "available") {
|
|
75
|
+
lines.push(`Count: ${metric.count}`);
|
|
76
|
+
}
|
|
77
|
+
else {
|
|
78
|
+
lines.push(`Reason: ${sanitizeScalar(metric.reason ?? "")}`);
|
|
79
|
+
}
|
|
80
|
+
lines.push("Evidence Keys:");
|
|
81
|
+
pushBoundedList(lines, metric.evidenceKeys, (key) => key);
|
|
82
|
+
}
|
|
83
|
+
function renderStrategy(lines, strategy, index) {
|
|
84
|
+
pushSection(lines, `Strategy ${index + 1}: ${strategy.strategyId}`);
|
|
85
|
+
pushSection(lines, "Artifacts");
|
|
86
|
+
pushBoundedList(lines, strategy.artifacts, (artifact) => `${artifact.sourceInstance} [${artifact.artifactKind}] path=${artifact.sourcePath} schema=${artifact.schemaVersion}`);
|
|
87
|
+
pushSection(lines, "Execution");
|
|
88
|
+
lines.push(fieldLine("Status", strategy.execution.status));
|
|
89
|
+
lines.push("Issues:");
|
|
90
|
+
pushBoundedList(lines, strategy.execution.issues, (issue) => `${issue.code} ${issue.fieldPath ?? ""} ${issue.message}`);
|
|
91
|
+
pushSection(lines, "Evaluation Metrics");
|
|
92
|
+
lines.push(fieldLine("Status", strategy.evaluation.status));
|
|
93
|
+
lines.push(fieldLine("Reason", strategy.evaluation.reason));
|
|
94
|
+
lines.push("Warnings:");
|
|
95
|
+
pushBoundedList(lines, strategy.evaluation.warnings, (warning) => warning);
|
|
96
|
+
if (strategy.evaluation.metrics) {
|
|
97
|
+
const metrics = strategy.evaluation.metrics;
|
|
98
|
+
pushRatioMetric(lines, "Required Evidence Recall", metrics.requiredEvidenceRecall);
|
|
99
|
+
pushRatioMetric(lines, "Allowed Evidence Coverage", metrics.allowedEvidenceCoverage);
|
|
100
|
+
pushRatioMetric(lines, "Forbidden Evidence Inclusion", metrics.forbiddenEvidenceInclusion);
|
|
101
|
+
pushCountMetric(lines, "Irrelevant File Inclusion", metrics.irrelevantFileInclusion);
|
|
102
|
+
pushCountMetric(lines, "Irrelevant Instruction Inclusion", metrics.irrelevantInstructionInclusion);
|
|
103
|
+
pushRatioMetric(lines, "Required Provenance Recall", metrics.requiredProvenanceRecall);
|
|
104
|
+
pushCountMetric(lines, "Considered But Unselected Reads", metrics.consideredButUnselectedReads);
|
|
105
|
+
pushCountMetric(lines, "Unnecessary Reads", metrics.unnecessaryReads);
|
|
106
|
+
pushCountMetric(lines, "Target Immutability (Evaluation)", metrics.targetImmutability);
|
|
107
|
+
}
|
|
108
|
+
pushSection(lines, "Expectation Matches");
|
|
109
|
+
pushBoundedList(lines, strategy.evaluation.expectationMatches, (match) => `${match.expectationId} inclusion=${match.inclusion} outcome=${match.outcome} sourceArtifact=${match.sourceArtifact} targetKey=${match.targetKey}`);
|
|
110
|
+
pushSection(lines, "Observed Evidence");
|
|
111
|
+
pushBoundedList(lines, strategy.evaluation.observedEvidence, (evidence) => `${evidence.sourceArtifact}.${evidence.sourceInstance} category=${evidence.category} targetKey=${evidence.targetKey} field=${evidence.sourceFieldPath}`);
|
|
112
|
+
pushSection(lines, "State Comparisons");
|
|
113
|
+
if (strategy.evaluation.metrics) {
|
|
114
|
+
pushBoundedList(lines, strategy.evaluation.metrics.stateComparisons, (comparison) => `${comparison.sourceArtifact}.${comparison.sourceInstance} expectationFieldPath=${comparison.expectationFieldPath} artifactFieldPath=${comparison.artifactFieldPath ?? "null"} availability=${comparison.availability} expected=${JSON.stringify(comparison.expected)} actual=${JSON.stringify(comparison.actual)} matched=${comparison.matched}`);
|
|
115
|
+
}
|
|
116
|
+
else {
|
|
117
|
+
lines.push("Displayed: 0 of 0");
|
|
118
|
+
lines.push("Omitted: 0");
|
|
119
|
+
lines.push("- none");
|
|
120
|
+
}
|
|
121
|
+
pushSection(lines, "Responsibility Mapping");
|
|
122
|
+
if (strategy.evaluation.metrics) {
|
|
123
|
+
pushBoundedList(lines, strategy.evaluation.metrics.responsibilityMappingCompleteness, (mapping) => `${mapping.sourceArtifact}.${mapping.sourceInstance} requested=${mapping.requested} operational=${mapping.operational} mapped=${mapping.mappedCount} partiallyMapped=${mapping.partiallyMappedCount} unmapped=${mapping.unmappedCount} notApplicable=${mapping.notApplicableCount} denominator=${mapping.denominator} mappedRate=${mapping.mappedRate}`);
|
|
124
|
+
}
|
|
125
|
+
else {
|
|
126
|
+
lines.push("Displayed: 0 of 0");
|
|
127
|
+
lines.push("Omitted: 0");
|
|
128
|
+
lines.push("- none");
|
|
129
|
+
}
|
|
130
|
+
pushSection(lines, "Context Size");
|
|
131
|
+
if (strategy.evaluation.metrics) {
|
|
132
|
+
const size = strategy.evaluation.metrics.contextSize;
|
|
133
|
+
lines.push(fieldLine("Total Character Count", size.totalCharacterCount));
|
|
134
|
+
lines.push(fieldLine("Total Estimated Token Count (estimate)", size.totalEstimatedTokenCount));
|
|
135
|
+
lines.push(fieldLine("Token Estimate Formula", size.tokenEstimateFormula));
|
|
136
|
+
pushBoundedList(lines, size.sources, (source) => `${source.sourceInstance} kind=${source.sourceKind} characters=${source.characterCount} estimatedTokens=${source.estimatedTokenCount}`);
|
|
137
|
+
}
|
|
138
|
+
else {
|
|
139
|
+
lines.push("Displayed: 0 of 0");
|
|
140
|
+
lines.push("Omitted: 0");
|
|
141
|
+
lines.push("- none");
|
|
142
|
+
}
|
|
143
|
+
pushSection(lines, "Target Immutability");
|
|
144
|
+
pushBoundedList(lines, strategy.assurance.runRecords, (record) => {
|
|
145
|
+
if (record.targetImmutabilityAvailability === "unavailable") {
|
|
146
|
+
return `run=${record.runNumber} availability=unavailable reason=${record.targetImmutabilityReason ?? ""}`;
|
|
147
|
+
}
|
|
148
|
+
return `run=${record.runNumber} availability=available status=${record.targetImmutabilityStatus} newMutationCount=${record.newMutationCount} mutations=${record.mutations.totalCount}`;
|
|
149
|
+
});
|
|
150
|
+
pushSection(lines, "Repeated-Run Determinism");
|
|
151
|
+
const determinism = strategy.assurance.determinism;
|
|
152
|
+
lines.push(fieldLine("Availability", determinism.availability));
|
|
153
|
+
lines.push(fieldLine("Repeat Count", determinism.repeatCount));
|
|
154
|
+
if (determinism.availability === "available") {
|
|
155
|
+
lines.push(fieldLine("Deterministic", determinism.deterministic));
|
|
156
|
+
lines.push(fieldLine("Baseline SHA-256", determinism.baselineSha256));
|
|
157
|
+
lines.push("Run Digests:");
|
|
158
|
+
pushBoundedList(lines, determinism.runDigests, (digest) => `run=${digest.runNumber} sha256=${digest.sha256}`);
|
|
159
|
+
lines.push(`Mismatch Run Numbers: ${determinism.mismatchRunNumbers.join(", ") || "none"}`);
|
|
160
|
+
}
|
|
161
|
+
else {
|
|
162
|
+
lines.push(fieldLine("Reason", determinism.reason));
|
|
163
|
+
}
|
|
164
|
+
pushSection(lines, "Run Assurance");
|
|
165
|
+
lines.push(fieldLine("Status", strategy.assurance.status));
|
|
166
|
+
lines.push(fieldLine("Repeat Count", strategy.assurance.repeatCount));
|
|
167
|
+
lines.push("Issues:");
|
|
168
|
+
pushBoundedList(lines, strategy.assurance.issues, (issue) => `${issue.code} run=${issue.runNumber ?? ""} field=${issue.fieldPath ?? ""} ${issue.message}`);
|
|
169
|
+
if (strategy.producerReadinessBridge) {
|
|
170
|
+
renderProducerReadinessBridge(lines, strategy.producerReadinessBridge);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
function renderProducerReadinessSide(lines, label, side) {
|
|
174
|
+
pushSection(lines, `Producer-Readiness Bridge: ${label}`);
|
|
175
|
+
if (side === null) {
|
|
176
|
+
lines.push("- not supplied");
|
|
177
|
+
return;
|
|
178
|
+
}
|
|
179
|
+
pushRatioMetric(lines, "Expected Owner Present", side.ownerEvaluation.expectedOwnerPresent);
|
|
180
|
+
pushRatioMetric(lines, "Forbidden Owner Present", side.ownerEvaluation.forbiddenOwnerPresent);
|
|
181
|
+
pushCountMetric(lines, "Owner False Positives", side.ownerEvaluation.falsePositiveCount);
|
|
182
|
+
pushCountMetric(lines, "Owner False Negatives", side.ownerEvaluation.falseNegativeCount);
|
|
183
|
+
pushSection(lines, "Truncation Classification");
|
|
184
|
+
pushBoundedList(lines, side.truncationEvaluation, (t) => `group=${t.groupId} cause=${t.cause} availability=${t.availability} limit=${t.limit} used=${t.used}`);
|
|
185
|
+
for (const [agreementLabel, agreement] of [
|
|
186
|
+
["Packet/Raw Agreement", side.packetAgreement],
|
|
187
|
+
["Report/Raw Agreement", side.reportAgreement],
|
|
188
|
+
]) {
|
|
189
|
+
pushSection(lines, agreementLabel);
|
|
190
|
+
if (agreement === null) {
|
|
191
|
+
lines.push("- unavailable");
|
|
192
|
+
continue;
|
|
193
|
+
}
|
|
194
|
+
lines.push(`Producer Parity Preserved: ${agreement.upstreamProducerParityPreserved === null ? "unavailable" : agreement.upstreamProducerParityPreserved}`);
|
|
195
|
+
lines.push("Contradictions:");
|
|
196
|
+
pushDashList(lines, agreement.contradictions.map((c) => `${c.field}: raw=${sanitizeScalar(c.rawValue)} supplemental=${sanitizeScalar(c.supplementalValue)}`));
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
function renderProducerReadinessBridge(lines, bridge) {
|
|
200
|
+
pushSection(lines, "Producer-Readiness Bridge");
|
|
201
|
+
lines.push(fieldLine("Status", bridge.status));
|
|
202
|
+
if (bridge.status === "not-applicable") {
|
|
203
|
+
lines.push(fieldLine("Reason", bridge.reason));
|
|
204
|
+
return;
|
|
205
|
+
}
|
|
206
|
+
renderProducerReadinessSide(lines, "Implementation", bridge.implementation);
|
|
207
|
+
renderProducerReadinessSide(lines, "Test", bridge.testImplementation);
|
|
208
|
+
pushSection(lines, "Readiness Agreement");
|
|
209
|
+
if (bridge.readinessAgreement === null) {
|
|
210
|
+
lines.push("- not supplied");
|
|
211
|
+
}
|
|
212
|
+
else {
|
|
213
|
+
const decision = bridge.readinessAgreement.decisionAgreement;
|
|
214
|
+
lines.push(fieldLine("Observed Decision", decision.observedDecision));
|
|
215
|
+
lines.push(fieldLine("Decision Agreement", decision.decisionAgreement));
|
|
216
|
+
lines.push(fieldLine("Issue-Code Agreement", decision.issueCodesAgreement));
|
|
217
|
+
lines.push("Observed Issue Codes:");
|
|
218
|
+
pushDashList(lines, decision.observedIssueCodes);
|
|
219
|
+
lines.push(fieldLine("Invalid Ready", bridge.readinessAgreement.invalidReady.invalidReady));
|
|
220
|
+
lines.push(fieldLine("Valid Blocked", bridge.readinessAgreement.validBlocked.validBlocked));
|
|
221
|
+
lines.push(fieldLine("Valid Refresh-Required", bridge.readinessAgreement.validBlocked.validRefreshRequired));
|
|
222
|
+
}
|
|
223
|
+
pushSection(lines, "Criticality Evaluation");
|
|
224
|
+
if (bridge.criticalityEvaluation === null) {
|
|
225
|
+
lines.push("- not supplied");
|
|
226
|
+
}
|
|
227
|
+
else {
|
|
228
|
+
pushRatioMetric(lines, "Mapped Critical Completeness", bridge.criticalityEvaluation.mappedCriticalCompleteness);
|
|
229
|
+
lines.push(`Fully Mapped: ${bridge.criticalityEvaluation.fullyMappedCriticalIds.join(", ") || "none"}`);
|
|
230
|
+
lines.push(`Partially Mapped: ${bridge.criticalityEvaluation.partiallyMappedCriticalIds.join(", ") || "none"}`);
|
|
231
|
+
lines.push(`Unmapped: ${bridge.criticalityEvaluation.unmappedCriticalIds.join(", ") || "none"}`);
|
|
232
|
+
lines.push(`Conflicting: ${bridge.criticalityEvaluation.conflictingCriticalityResponsibilityIds.join(", ") || "none"}`);
|
|
233
|
+
}
|
|
234
|
+
pushSection(lines, "Producer-Readiness Bridge Warnings");
|
|
235
|
+
pushBoundedList(lines, bridge.warnings, (w) => w);
|
|
236
|
+
}
|
|
237
|
+
export function renderV043Section(lines, section) {
|
|
238
|
+
if (section === null) {
|
|
239
|
+
lines.push("Not applicable to this plugin.");
|
|
240
|
+
return;
|
|
241
|
+
}
|
|
242
|
+
if (section.summary.strategyCount === 0) {
|
|
243
|
+
lines.push("No v0.4.3 stage-context strategies were selected.");
|
|
244
|
+
return;
|
|
245
|
+
}
|
|
246
|
+
lines.push(fieldLine("Strategy Count", section.summary.strategyCount));
|
|
247
|
+
lines.push(fieldLine("Completed Executions", section.summary.completedExecutionCount));
|
|
248
|
+
lines.push(fieldLine("Invalid-Input Executions", section.summary.invalidInputExecutionCount));
|
|
249
|
+
lines.push(fieldLine("Failed Executions", section.summary.failedExecutionCount));
|
|
250
|
+
lines.push(fieldLine("Completed Evaluations", section.summary.completedEvaluationCount));
|
|
251
|
+
lines.push(fieldLine("Not-Applicable Evaluations", section.summary.notApplicableEvaluationCount));
|
|
252
|
+
lines.push(fieldLine("Failed Evaluations", section.summary.failedEvaluationCount));
|
|
253
|
+
lines.push(fieldLine("Passed Assurance", section.summary.passedAssuranceCount));
|
|
254
|
+
lines.push(fieldLine("Failed Assurance", section.summary.failedAssuranceCount));
|
|
255
|
+
lines.push(fieldLine("Not-Applicable Assurance", section.summary.notApplicableAssuranceCount));
|
|
256
|
+
lines.push(fieldLine("Interpretation Summary", section.interpretation.summary));
|
|
257
|
+
lines.push("Limitations:");
|
|
258
|
+
pushDashList(lines, section.interpretation.limitations);
|
|
259
|
+
for (let index = 0; index < section.strategies.length; index += 1) {
|
|
260
|
+
renderStrategy(lines, section.strategies[index], index);
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
function formatWarmMetric(metric) {
|
|
264
|
+
const method = metric.tokenCountMethod ? `, method ${metric.tokenCountMethod}` : "";
|
|
265
|
+
if (metric.availability === "available") {
|
|
266
|
+
return `${metric.value} ${metric.unit} (${metric.source}${method})`;
|
|
267
|
+
}
|
|
268
|
+
return `${metric.availability} (${metric.reason ?? ""})`;
|
|
269
|
+
}
|
|
270
|
+
function formatWarmAgent(agent) {
|
|
271
|
+
if (agent === null)
|
|
272
|
+
return "not run (no usable context evidence for this side)";
|
|
273
|
+
const passed = agent.passed === null ? "unavailable" : String(agent.passed);
|
|
274
|
+
const issues = [...agent.errors, ...agent.warnings].slice(0, 3).join("; ");
|
|
275
|
+
return `agent ${agent.agentId}, status ${agent.status}, passed ${passed}, token source ${agent.tokenUsageSource}, reliability ${agent.tokenUsageReliability}${issues ? `, notes: ${issues}` : ""}`;
|
|
276
|
+
}
|
|
277
|
+
function renderCampaignSection(lines, campaign) {
|
|
278
|
+
pushSection(lines, "Real-Agent Campaign");
|
|
279
|
+
lines.push(fieldLine("Campaign Preset", campaign.presetId));
|
|
280
|
+
lines.push(fieldLine("Agent ID", campaign.agentId));
|
|
281
|
+
lines.push(fieldLine("Campaign Timeout", `${campaign.timeoutMs} ms`));
|
|
282
|
+
lines.push(fieldLine("Scheduled Agent Sides", campaign.scheduledSideCount));
|
|
283
|
+
lines.push(fieldLine("Executed Agent Sides", campaign.executedSideCount));
|
|
284
|
+
lines.push(fieldLine("Not Run For Missing Context", campaign.notRunForMissingContextCount));
|
|
285
|
+
lines.push(fieldLine("Agent Evidence Status", campaign.agentEvidenceStatus));
|
|
286
|
+
lines.push(fieldLine("Token Evidence Status", campaign.tokenEvidenceStatus));
|
|
287
|
+
lines.push(fieldLine("Completed Agent Sides", campaign.outcomeCounts.completed));
|
|
288
|
+
lines.push(fieldLine("Failed Agent Sides", campaign.outcomeCounts.failed));
|
|
289
|
+
lines.push(fieldLine("Timeout Agent Sides", campaign.outcomeCounts.timeout));
|
|
290
|
+
lines.push(fieldLine("Invalid-Output Agent Sides", campaign.outcomeCounts.invalidOutput));
|
|
291
|
+
lines.push(fieldLine("Agent-Unavailable Sides", campaign.outcomeCounts.agentUnavailable));
|
|
292
|
+
lines.push(fieldLine("Agent-Limit-Reached Sides", campaign.outcomeCounts.agentLimitReached));
|
|
293
|
+
lines.push(fieldLine("Skipped Agent Sides", campaign.outcomeCounts.skipped));
|
|
294
|
+
}
|
|
295
|
+
export function renderWarmIndexReuseSection(lines, section) {
|
|
296
|
+
if (section === null) {
|
|
297
|
+
lines.push("Not applicable to this plugin.");
|
|
298
|
+
return;
|
|
299
|
+
}
|
|
300
|
+
lines.push(fieldLine("Project Count", section.summary.projectCount));
|
|
301
|
+
lines.push(fieldLine("Task Count", section.summary.taskCount));
|
|
302
|
+
lines.push(fieldLine("Prepared Session Project Count", section.summary.preparedSessionProjectCount));
|
|
303
|
+
lines.push(fieldLine("Incomplete Or Failed Project Count", section.summary.incompleteProjectCount));
|
|
304
|
+
lines.push(fieldLine("Agent ID", section.agent.id));
|
|
305
|
+
lines.push(fieldLine("Agent Mode", section.agent.mode));
|
|
306
|
+
lines.push(fieldLine("Agent Correctness Available (task sides)", `${section.summary.agentCorrectnessAvailableCount} of ${section.summary.agentSideCount}`));
|
|
307
|
+
lines.push(fieldLine("Agent Token Totals Available (task sides)", `${section.summary.agentTotalTokensAvailableCount} of ${section.summary.agentSideCount}`));
|
|
308
|
+
renderFreshnessSummary(lines, section);
|
|
309
|
+
renderAffectedNeighborhoodSummary(lines, section);
|
|
310
|
+
if (section.agentCampaign) {
|
|
311
|
+
renderCampaignSection(lines, section.agentCampaign);
|
|
312
|
+
}
|
|
313
|
+
lines.push("Cold Start And Warm Reuse:");
|
|
314
|
+
pushDashList(lines, section.costModel);
|
|
315
|
+
lines.push("Limitations:");
|
|
316
|
+
pushDashList(lines, section.limitations);
|
|
317
|
+
if (section.projects.length === 0) {
|
|
318
|
+
lines.push("No warm-index project evidence was recorded.");
|
|
319
|
+
return;
|
|
320
|
+
}
|
|
321
|
+
section.projects.forEach((project, index) => {
|
|
322
|
+
pushSection(lines, `Warm Index Project ${index + 1}: ${sanitizeScalar(project.benchmarkProject)}`);
|
|
323
|
+
lines.push(fieldLine("Session Key", project.sessionKey));
|
|
324
|
+
lines.push(fieldLine("Status", project.status));
|
|
325
|
+
lines.push(fieldLine("Session Prepared", project.sessionPrepared));
|
|
326
|
+
lines.push(fieldLine("Index Build Duration", formatWarmMetric(project.indexBuildDurationMs)));
|
|
327
|
+
lines.push(fieldLine("Task Count", project.taskCount));
|
|
328
|
+
for (const task of project.tasks) {
|
|
329
|
+
pushSection(lines, `Task ${task.taskOrdinal}: ${sanitizeScalar(task.caseId)}`);
|
|
330
|
+
const rows = [
|
|
331
|
+
["Raw Status", task.rawStatus],
|
|
332
|
+
["Warm Status", task.warmStatus],
|
|
333
|
+
["Index Freshness Status", task.indexFreshness ? task.indexFreshness.status : "not assessed"],
|
|
334
|
+
["Affected Neighborhood Status", task.affectedNeighborhood ? task.affectedNeighborhood.status : "not assessed"],
|
|
335
|
+
["Raw Context Characters", formatWarmMetric(task.raw.contextCharacters)],
|
|
336
|
+
["Warm Context Characters", formatWarmMetric(task.warm.contextCharacters)],
|
|
337
|
+
["Raw Estimated Context Tokens (estimate)", formatWarmMetric(task.raw.contextEstimatedTokens)],
|
|
338
|
+
["Warm Estimated Context Tokens (estimate)", formatWarmMetric(task.warm.contextEstimatedTokens)],
|
|
339
|
+
["Raw Operation Duration", formatWarmMetric(task.raw.operationDurationMs)],
|
|
340
|
+
["Warm Retrieval Duration", formatWarmMetric(task.warm.retrievalDurationMs)],
|
|
341
|
+
["Amortized Index Build Duration", formatWarmMetric(task.warm.amortizedIndexBuildDurationMs)],
|
|
342
|
+
["Cumulative Raw Duration", formatWarmMetric(task.raw.cumulativeDurationMs)],
|
|
343
|
+
["Cumulative Warm Component Duration", formatWarmMetric(task.warm.cumulativeComponentDurationMs)],
|
|
344
|
+
["Cumulative Raw Estimated Context Tokens (estimate)", formatWarmMetric(task.raw.cumulativeEstimatedContextTokens)],
|
|
345
|
+
["Cumulative Warm Estimated Context Tokens (estimate)", formatWarmMetric(task.warm.cumulativeEstimatedContextTokens)],
|
|
346
|
+
["Raw Agent Evaluation", formatWarmAgent(task.rawAgent)],
|
|
347
|
+
["Warm Agent Evaluation", formatWarmAgent(task.warmAgent)],
|
|
348
|
+
["Raw Agent Correctness", formatWarmMetric(task.raw.agentCorrectness)],
|
|
349
|
+
["Warm Agent Correctness", formatWarmMetric(task.warm.agentCorrectness)],
|
|
350
|
+
["Raw Agent Total Tokens", formatWarmMetric(task.raw.agentTotalTokens)],
|
|
351
|
+
["Warm Agent Total Tokens", formatWarmMetric(task.warm.agentTotalTokens)],
|
|
352
|
+
["Cumulative Raw Agent Total Tokens", formatWarmMetric(task.raw.cumulativeAgentTotalTokens)],
|
|
353
|
+
["Cumulative Warm Agent Total Tokens", formatWarmMetric(task.warm.cumulativeAgentTotalTokens)],
|
|
354
|
+
];
|
|
355
|
+
for (const [label, value] of rows) {
|
|
356
|
+
lines.push(fieldLine(label, value));
|
|
357
|
+
}
|
|
358
|
+
renderTaskFreshnessDetail(lines, task.indexFreshness);
|
|
359
|
+
renderTaskAffectedNeighborhoodDetail(lines, task.affectedNeighborhood);
|
|
360
|
+
}
|
|
361
|
+
});
|
|
362
|
+
}
|
|
363
|
+
function renderFreshnessSummary(lines, section) {
|
|
364
|
+
// Reports serialized before v0.6.0 Batch 3 lack the summary; render them as having no assessment.
|
|
365
|
+
const summary = section.indexFreshnessSummary ?? {
|
|
366
|
+
assessedTaskCount: 0,
|
|
367
|
+
unassessedTaskCount: section.summary.taskCount,
|
|
368
|
+
freshTaskCount: 0,
|
|
369
|
+
staleTaskCount: 0,
|
|
370
|
+
partiallyStaleTaskCount: 0,
|
|
371
|
+
unknownTaskCount: 0,
|
|
372
|
+
};
|
|
373
|
+
pushSection(lines, "Index Freshness Summary");
|
|
374
|
+
lines.push(fieldLine("Assessed Tasks", summary.assessedTaskCount));
|
|
375
|
+
lines.push(fieldLine("Unassessed Tasks", summary.unassessedTaskCount));
|
|
376
|
+
lines.push(fieldLine("Fresh Tasks", summary.freshTaskCount));
|
|
377
|
+
lines.push(fieldLine("Stale Tasks", summary.staleTaskCount));
|
|
378
|
+
lines.push(fieldLine("Partially-Stale Tasks", summary.partiallyStaleTaskCount));
|
|
379
|
+
lines.push(fieldLine("Unknown Tasks", summary.unknownTaskCount));
|
|
380
|
+
}
|
|
381
|
+
function renderAffectedNeighborhoodSummary(lines, section) {
|
|
382
|
+
// Reports serialized before v0.6.1 lack the summary; render them as having no assessment.
|
|
383
|
+
const summary = section.affectedNeighborhoodSummary ?? {
|
|
384
|
+
assessedTaskCount: 0,
|
|
385
|
+
unassessedTaskCount: section.summary.taskCount,
|
|
386
|
+
completeAssessmentCount: 0,
|
|
387
|
+
partialAssessmentCount: 0,
|
|
388
|
+
unavailableAssessmentCount: 0,
|
|
389
|
+
relatedTaskCount: 0,
|
|
390
|
+
unrelatedTaskCount: 0,
|
|
391
|
+
unknownRelationshipTaskCount: 0,
|
|
392
|
+
recommendedReindexCount: 0,
|
|
393
|
+
notIndicatedReindexCount: 0,
|
|
394
|
+
unknownReindexRecommendationCount: 0,
|
|
395
|
+
};
|
|
396
|
+
pushSection(lines, "Affected Neighborhood Summary");
|
|
397
|
+
lines.push(fieldLine("Assessed Tasks", summary.assessedTaskCount));
|
|
398
|
+
lines.push(fieldLine("Unassessed Tasks", summary.unassessedTaskCount));
|
|
399
|
+
lines.push(fieldLine("Complete Assessments", summary.completeAssessmentCount));
|
|
400
|
+
lines.push(fieldLine("Partial Assessments", summary.partialAssessmentCount));
|
|
401
|
+
lines.push(fieldLine("Unavailable Assessments", summary.unavailableAssessmentCount));
|
|
402
|
+
lines.push(fieldLine("Related Tasks", summary.relatedTaskCount));
|
|
403
|
+
lines.push(fieldLine("Unrelated Tasks", summary.unrelatedTaskCount));
|
|
404
|
+
lines.push(fieldLine("Unknown-Relationship Tasks", summary.unknownRelationshipTaskCount));
|
|
405
|
+
lines.push(fieldLine("Reindex Recommended", summary.recommendedReindexCount));
|
|
406
|
+
lines.push(fieldLine("Reindex Not Indicated", summary.notIndicatedReindexCount));
|
|
407
|
+
lines.push(fieldLine("Reindex Recommendation Unknown", summary.unknownReindexRecommendationCount));
|
|
408
|
+
}
|
|
409
|
+
/** Renders the persisted affected-neighborhood evidence; numeric lines come from the metric owner. */
|
|
410
|
+
function renderTaskAffectedNeighborhoodDetail(lines, evidence) {
|
|
411
|
+
if (!evidence)
|
|
412
|
+
return;
|
|
413
|
+
lines.push(fieldLine("Affected Neighborhood Relationship", evidence.relationship));
|
|
414
|
+
lines.push(fieldLine("Reindex Recommendation", evidence.reindexRecommendation));
|
|
415
|
+
lines.push(fieldLine("Reindex Recommendation Explanation", evidence.recommendationExplanation));
|
|
416
|
+
lines.push(fieldLine("Affected Neighborhood Freshness Status", evidence.freshnessStatus));
|
|
417
|
+
lines.push(fieldLine("Seed Mapping Status", evidence.seedMappingStatus));
|
|
418
|
+
lines.push(fieldLine("Graph Evidence Status", evidence.graphEvidenceStatus));
|
|
419
|
+
lines.push(fieldLine("Neighborhood Status", evidence.neighborhoodStatus));
|
|
420
|
+
lines.push(fieldLine("Task Mapping Status", evidence.taskMappingStatus));
|
|
421
|
+
lines.push(fieldLine("Resolvable Task Nodes", evidence.resolvableTaskNodeCount));
|
|
422
|
+
lines.push(fieldLine("Changed Indexed Files", formatWarmMetric(evidence.metrics.changedFileCount)));
|
|
423
|
+
lines.push(fieldLine("Changed Baseline Symbols", formatWarmMetric(evidence.metrics.changedSymbolCount)));
|
|
424
|
+
lines.push(fieldLine("Affected Graph Nodes", formatWarmMetric(evidence.metrics.affectedNodeCount)));
|
|
425
|
+
lines.push(fieldLine("Affected Graph Edges", formatWarmMetric(evidence.metrics.affectedEdgeCount)));
|
|
426
|
+
lines.push(fieldLine("Task-Overlap Nodes", formatWarmMetric(evidence.metrics.taskOverlapCount)));
|
|
427
|
+
lines.push(fieldLine("Task-Overlap Percent", formatWarmMetric(evidence.metrics.taskOverlapPercent)));
|
|
428
|
+
lines.push("Affected Node IDs:");
|
|
429
|
+
pushFreshnessBoundedNote(lines, evidence.affectedNodeIds);
|
|
430
|
+
if (evidence.affectedNodeIds.items.length === 0)
|
|
431
|
+
lines.push("- none");
|
|
432
|
+
for (const id of evidence.affectedNodeIds.items)
|
|
433
|
+
lines.push(`- ${sanitizeScalar(id)}`);
|
|
434
|
+
lines.push("Participating Edge IDs:");
|
|
435
|
+
pushFreshnessBoundedNote(lines, evidence.participatingEdgeIds);
|
|
436
|
+
if (evidence.participatingEdgeIds.items.length === 0)
|
|
437
|
+
lines.push("- none");
|
|
438
|
+
for (const id of evidence.participatingEdgeIds.items)
|
|
439
|
+
lines.push(`- ${sanitizeScalar(id)}`);
|
|
440
|
+
lines.push("Unresolved Task Mappings:");
|
|
441
|
+
pushFreshnessBoundedNote(lines, evidence.unresolvedTaskMappings);
|
|
442
|
+
if (evidence.unresolvedTaskMappings.items.length === 0)
|
|
443
|
+
lines.push("- none");
|
|
444
|
+
for (const entry of evidence.unresolvedTaskMappings.items) {
|
|
445
|
+
lines.push(`- ${sanitizeScalar(entry.subject)} ${sanitizeScalar(entry.name)} [${sanitizeScalar(entry.reason)}]`);
|
|
446
|
+
}
|
|
447
|
+
lines.push("Ambiguous Task Symbols:");
|
|
448
|
+
pushFreshnessBoundedNote(lines, evidence.ambiguousTaskSymbols);
|
|
449
|
+
if (evidence.ambiguousTaskSymbols.items.length === 0)
|
|
450
|
+
lines.push("- none");
|
|
451
|
+
for (const entry of evidence.ambiguousTaskSymbols.items) {
|
|
452
|
+
lines.push(`- ${sanitizeScalar(entry.name)}: ${entry.candidateNodeIds.map((id) => sanitizeScalar(id)).join(", ")}`);
|
|
453
|
+
}
|
|
454
|
+
lines.push("Affected Neighborhood Warnings:");
|
|
455
|
+
pushFreshnessBoundedNote(lines, evidence.warnings);
|
|
456
|
+
if (evidence.warnings.items.length === 0)
|
|
457
|
+
lines.push("- none");
|
|
458
|
+
for (const warning of evidence.warnings.items) {
|
|
459
|
+
lines.push(`- [${sanitizeScalar(warning.code)}] ${sanitizeScalar(warning.message)}`);
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
function pushFreshnessBoundedNote(lines, list) {
|
|
463
|
+
if (list.omittedCount > 0) {
|
|
464
|
+
lines.push(`Displayed: ${list.displayedCount} of ${list.totalCount}`);
|
|
465
|
+
lines.push(`Omitted: ${list.omittedCount}`);
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
/** Renders the persisted freshness counts and bounded evidence; hashes are never printed. */
|
|
469
|
+
function renderTaskFreshnessDetail(lines, freshness) {
|
|
470
|
+
if (!freshness)
|
|
471
|
+
return;
|
|
472
|
+
lines.push(fieldLine("Baseline Snapshot Status", freshness.baselineSnapshotStatus));
|
|
473
|
+
lines.push(fieldLine("Indexed Files", freshness.indexedFileCount));
|
|
474
|
+
lines.push(fieldLine("Comparable Files", freshness.comparableFileCount));
|
|
475
|
+
lines.push(fieldLine("Unchanged Files", freshness.unchangedFileCount));
|
|
476
|
+
lines.push(fieldLine("Modified Files", freshness.changedFileCount));
|
|
477
|
+
lines.push(fieldLine("Missing Files", freshness.missingFileCount));
|
|
478
|
+
lines.push(fieldLine("Unresolved Comparisons", freshness.unresolvedFileCount));
|
|
479
|
+
lines.push("Freshness Changes:");
|
|
480
|
+
pushFreshnessBoundedNote(lines, freshness.changes);
|
|
481
|
+
if (freshness.changes.items.length === 0) {
|
|
482
|
+
lines.push("- none");
|
|
483
|
+
}
|
|
484
|
+
for (const change of freshness.changes.items) {
|
|
485
|
+
lines.push(`- ${sanitizeScalar(change.changeType)} ${sanitizeScalar(change.path)}`);
|
|
486
|
+
}
|
|
487
|
+
lines.push("Freshness Unresolved Evidence:");
|
|
488
|
+
pushFreshnessBoundedNote(lines, freshness.unresolved);
|
|
489
|
+
if (freshness.unresolved.items.length === 0) {
|
|
490
|
+
lines.push("- none");
|
|
491
|
+
}
|
|
492
|
+
for (const entry of freshness.unresolved.items) {
|
|
493
|
+
lines.push(`- ${entry.path === null ? "run-level" : sanitizeScalar(entry.path)} [${sanitizeScalar(entry.reasonCode)}] ${sanitizeScalar(entry.message)}`);
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
const INCREMENTAL_CHANGE_STALENESS_MAX_DISPLAY_ITEMS = 20;
|
|
497
|
+
function pushBoundedStringList(lines, label, items) {
|
|
498
|
+
lines.push(`${label}:`);
|
|
499
|
+
if (items.length === 0) {
|
|
500
|
+
lines.push("- none");
|
|
501
|
+
return;
|
|
502
|
+
}
|
|
503
|
+
const shown = items.slice(0, INCREMENTAL_CHANGE_STALENESS_MAX_DISPLAY_ITEMS);
|
|
504
|
+
for (const item of shown)
|
|
505
|
+
lines.push(`- ${sanitizeScalar(item)}`);
|
|
506
|
+
const omitted = items.length - shown.length;
|
|
507
|
+
if (omitted > 0)
|
|
508
|
+
lines.push(`... ${omitted} more`);
|
|
509
|
+
}
|
|
510
|
+
function symmetricIncrementalAffectedNeighborhood(scenario) {
|
|
511
|
+
const stale = scenario.staleTreatment?.affectedNeighborhood ?? null;
|
|
512
|
+
const fullRefresh = scenario.fullRefreshTreatment?.affectedNeighborhood ?? null;
|
|
513
|
+
if (!stale || !fullRefresh)
|
|
514
|
+
return null;
|
|
515
|
+
if (stale.relationship !== fullRefresh.relationship || stale.reindexRecommendation !== fullRefresh.reindexRecommendation) {
|
|
516
|
+
return null;
|
|
517
|
+
}
|
|
518
|
+
return stale;
|
|
519
|
+
}
|
|
520
|
+
function renderIncrementalChangeStalenessTreatment(lines, label, treatment) {
|
|
521
|
+
pushSection(lines, label);
|
|
522
|
+
if (!treatment) {
|
|
523
|
+
lines.push("Not available.");
|
|
524
|
+
return;
|
|
525
|
+
}
|
|
526
|
+
lines.push(fieldLine("Treatment Status", treatment.status));
|
|
527
|
+
lines.push(fieldLine("Failure Reason", treatment.failureReason));
|
|
528
|
+
lines.push(fieldLine("Active Index Phase", treatment.activeIndexPhase));
|
|
529
|
+
lines.push(fieldLine("Baseline Freshness Status", treatment.baselineFreshness.status));
|
|
530
|
+
lines.push("Retrieval (deterministic fake-agent / simulated harness evidence label applies to fakeAgent below):");
|
|
531
|
+
lines.push(fieldLine("Retrieval Status", treatment.retrieval.status));
|
|
532
|
+
lines.push(fieldLine("Selected Node ID", treatment.retrieval.selectedNodeId));
|
|
533
|
+
lines.push(fieldLine("Selected File", treatment.retrieval.selectedFile));
|
|
534
|
+
lines.push(fieldLine("Selected Symbol", treatment.retrieval.selectedSymbol));
|
|
535
|
+
lines.push(fieldLine("Context Characters", treatment.retrieval.totalChars));
|
|
536
|
+
lines.push(fieldLine("Estimated Context Tokens", treatment.retrieval.totalEstimatedTokens));
|
|
537
|
+
lines.push(fieldLine("Retrieval Duration (ms)", treatment.retrieval.durationMs));
|
|
538
|
+
pushBoundedStringList(lines, "Files Read", treatment.retrieval.filesRead);
|
|
539
|
+
pushBoundedStringList(lines, "Retrieval Warnings", treatment.retrieval.warnings);
|
|
540
|
+
lines.push("Fake-Agent Evidence (deterministic fake-agent / simulated harness evidence):");
|
|
541
|
+
if (treatment.fakeAgent) {
|
|
542
|
+
lines.push(fieldLine("Fake-Agent Status", treatment.fakeAgent.status));
|
|
543
|
+
lines.push(fieldLine("Correctness Score", treatment.fakeAgent.correctness.available ? treatment.fakeAgent.correctness.score : null));
|
|
544
|
+
lines.push(fieldLine("Passed", treatment.fakeAgent.correctness.passed));
|
|
545
|
+
lines.push(fieldLine("Fake-Agent Duration (ms)", treatment.fakeAgent.durationMs));
|
|
546
|
+
}
|
|
547
|
+
else {
|
|
548
|
+
lines.push("Not run for this treatment.");
|
|
549
|
+
}
|
|
550
|
+
lines.push("Required-File Evidence (bounded answer-key file-presence check, not retrieval precision/recall):");
|
|
551
|
+
lines.push(fieldLine("Required-File Status", treatment.requiredFileEvidence.status));
|
|
552
|
+
lines.push(fieldLine("Required-File Reason", treatment.requiredFileEvidence.reason));
|
|
553
|
+
pushBoundedStringList(lines, "Required Files", treatment.requiredFileEvidence.requiredFiles);
|
|
554
|
+
pushBoundedStringList(lines, "Observed Files", treatment.requiredFileEvidence.observedFiles);
|
|
555
|
+
pushBoundedStringList(lines, "Missing Files", treatment.requiredFileEvidence.missingFiles);
|
|
556
|
+
}
|
|
557
|
+
function renderIncrementalChangeStalenessScenario(lines, scenario) {
|
|
558
|
+
pushSection(lines, `Scenario: ${scenario.scenarioId}`);
|
|
559
|
+
lines.push(fieldLine("Category", scenario.category));
|
|
560
|
+
lines.push(fieldLine("Status", scenario.status));
|
|
561
|
+
lines.push(fieldLine("Failure Reason", scenario.failureReason));
|
|
562
|
+
lines.push(fieldLine("Benchmark Project", scenario.benchmarkProjectId));
|
|
563
|
+
lines.push(fieldLine("Base Case", scenario.baseCaseId));
|
|
564
|
+
lines.push(fieldLine("Answer Policy", scenario.answerPolicy));
|
|
565
|
+
lines.push(fieldLine("Query", scenario.query));
|
|
566
|
+
pushBoundedStringList(lines, "Expected Files", scenario.expectedFiles);
|
|
567
|
+
pushBoundedStringList(lines, "Expected Symbols", scenario.expectedSymbols);
|
|
568
|
+
pushSection(lines, "Lifecycle");
|
|
569
|
+
const lifecycle = scenario.lifecycle;
|
|
570
|
+
if (!lifecycle) {
|
|
571
|
+
lines.push("Not available.");
|
|
572
|
+
}
|
|
573
|
+
else {
|
|
574
|
+
lines.push(fieldLine("Pre-Mutation Equivalence", lifecycle.preMutationEquivalence));
|
|
575
|
+
lines.push(fieldLine("Post-Mutation Equivalence", lifecycle.postMutationEquivalence));
|
|
576
|
+
lines.push(fieldLine("my-dev-kit Version", lifecycle.myDevKitVersion));
|
|
577
|
+
lines.push(fieldLine("Stale Baseline Freshness Status", lifecycle.staleBaselineFreshnessStatus));
|
|
578
|
+
lines.push(fieldLine("Full-Refresh Baseline Freshness Status", lifecycle.fullRefreshBaselineFreshnessStatus));
|
|
579
|
+
lines.push(fieldLine("Full-Refresh Refreshed Freshness Status", lifecycle.fullRefreshRefreshedFreshnessStatus));
|
|
580
|
+
pushBoundedStringList(lines, "Controlled File Paths", lifecycle.controlledFilePaths);
|
|
581
|
+
pushBoundedStringList(lines, "Source Roots", lifecycle.sourceRoots);
|
|
582
|
+
}
|
|
583
|
+
pushSection(lines, "Affected-Neighborhood Evidence (changed baseline neighborhood relative to the task)");
|
|
584
|
+
const symmetric = symmetricIncrementalAffectedNeighborhood(scenario);
|
|
585
|
+
const bothPresent = scenario.staleTreatment?.affectedNeighborhood && scenario.fullRefreshTreatment?.affectedNeighborhood;
|
|
586
|
+
if (!scenario.staleTreatment?.affectedNeighborhood && !scenario.fullRefreshTreatment?.affectedNeighborhood) {
|
|
587
|
+
lines.push("Not available.");
|
|
588
|
+
}
|
|
589
|
+
else if (bothPresent && !symmetric) {
|
|
590
|
+
lines.push("inconsistent persisted evidence");
|
|
591
|
+
}
|
|
592
|
+
else if (symmetric) {
|
|
593
|
+
lines.push(fieldLine("Relationship", symmetric.relationship));
|
|
594
|
+
lines.push(fieldLine("Reindex Recommendation", symmetric.reindexRecommendation));
|
|
595
|
+
lines.push(fieldLine("Reindex Recommendation Explanation", REINDEX_RECOMMENDATION_EXPLANATIONS[symmetric.reindexRecommendation]));
|
|
596
|
+
lines.push(fieldLine("Changed Indexed Files", symmetric.changedFileCount));
|
|
597
|
+
lines.push(fieldLine("Changed Baseline Symbols", symmetric.changedSymbolCount));
|
|
598
|
+
lines.push(fieldLine("Affected Graph Nodes", symmetric.affectedNodeCount));
|
|
599
|
+
lines.push(fieldLine("Affected Graph Edges", symmetric.affectedEdgeCount));
|
|
600
|
+
lines.push(fieldLine("Task-Overlap Nodes", symmetric.taskOverlapCount));
|
|
601
|
+
lines.push(fieldLine("Task-Overlap Percent", symmetric.taskOverlapPercent));
|
|
602
|
+
}
|
|
603
|
+
renderIncrementalChangeStalenessTreatment(lines, "Stale-Index Treatment", scenario.staleTreatment);
|
|
604
|
+
renderIncrementalChangeStalenessTreatment(lines, "Full-Refresh Treatment", scenario.fullRefreshTreatment);
|
|
605
|
+
pushSection(lines, "Matched Comparison");
|
|
606
|
+
lines.push(fieldLine("Correctness Relation", scenario.comparison.correctnessRelation));
|
|
607
|
+
lines.push(fieldLine("Required-File Evidence Relation", scenario.comparison.requiredFileEvidenceRelation));
|
|
608
|
+
lines.push(fieldLine("Stale-Risk Classification", scenario.comparison.staleRiskClassification));
|
|
609
|
+
pushDashList(lines, scenario.comparison.reasonCodes);
|
|
610
|
+
lines.push(STALE_RISK_CLASSIFICATION_EXPLANATIONS[scenario.comparison.staleRiskClassification]);
|
|
611
|
+
}
|
|
612
|
+
export function renderIncrementalChangeStalenessSection(lines, section) {
|
|
613
|
+
if (section === null) {
|
|
614
|
+
lines.push("Not applicable to this plugin.");
|
|
615
|
+
return;
|
|
616
|
+
}
|
|
617
|
+
if (isV2ReportSection(section)) {
|
|
618
|
+
renderIncrementalChangeStalenessSectionV2(lines, section, { pushSection, fieldLine, pushDashList, pushBoundedStringList });
|
|
619
|
+
return;
|
|
620
|
+
}
|
|
621
|
+
pushSection(lines, "Summary");
|
|
622
|
+
lines.push(fieldLine("Scenarios", section.scenarioCount));
|
|
623
|
+
lines.push(fieldLine("Ready", section.readyScenarioCount));
|
|
624
|
+
lines.push(fieldLine("Failed", section.failedScenarioCount));
|
|
625
|
+
lines.push(fieldLine("Observed Stale Regression", section.observedStaleRegressionCount));
|
|
626
|
+
lines.push(fieldLine("No Observed Stale Regression", section.noObservedStaleRegressionCount));
|
|
627
|
+
lines.push(fieldLine("Inconclusive", section.inconclusiveCount));
|
|
628
|
+
for (const scenario of section.scenarios) {
|
|
629
|
+
renderIncrementalChangeStalenessScenario(lines, scenario);
|
|
630
|
+
}
|
|
631
|
+
pushSection(lines, "Limitations");
|
|
632
|
+
pushDashList(lines, section.limitations);
|
|
633
|
+
}
|