@dailephd/my-dev-kit-lab 0.4.2 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -113
- package/benchmarks/contracts/todo-behavior.md +1 -1
- package/benchmarks/projects/task-analytics-large-mixed/README.md +2 -0
- package/benchmarks/projects/task-workflow-medium-ts/README.md +3 -1
- package/dist/src/evaluation/stageContextDeterminism/calculateStageContextDeterminism.js +52 -0
- package/dist/src/evaluation/stageContextDeterminism/canonicalizeStageContextRun.js +53 -0
- package/dist/src/evaluation/stageContextDeterminism/index.js +2 -0
- package/dist/src/evaluation/stageContextDeterminism/types.js +1 -0
- package/dist/src/evaluation/stageContextExpectations/index.js +2 -0
- package/dist/src/evaluation/stageContextExpectations/producerReadinessExpectationTypes.js +8 -0
- package/dist/src/evaluation/stageContextExpectations/readStageContextExpectationFixtureV1.js +50 -0
- package/dist/src/evaluation/stageContextExpectations/readTypes.js +1 -0
- package/dist/src/evaluation/stageContextExpectations/types.js +1 -0
- package/dist/src/evaluation/stageContextExpectations/validation.js +768 -0
- package/dist/src/evaluation/stageContextMetrics/calculateAllocationMetrics.js +96 -0
- package/dist/src/evaluation/stageContextMetrics/calculateArtifactStateMetrics.js +130 -0
- package/dist/src/evaluation/stageContextMetrics/calculateContextSizeMetrics.js +85 -0
- package/dist/src/evaluation/stageContextMetrics/calculateCriticalityMetrics.js +112 -0
- package/dist/src/evaluation/stageContextMetrics/calculateExpectationMetrics.js +103 -0
- package/dist/src/evaluation/stageContextMetrics/calculateOwnerMetrics.js +84 -0
- package/dist/src/evaluation/stageContextMetrics/calculateReadinessAgreement.js +152 -0
- package/dist/src/evaluation/stageContextMetrics/calculateResponsibilityMappingMetrics.js +36 -0
- package/dist/src/evaluation/stageContextMetrics/calculateSupplementalRawAgreement.js +50 -0
- package/dist/src/evaluation/stageContextMetrics/calculateTruncationClassification.js +44 -0
- package/dist/src/evaluation/stageContextMetrics/collectContextCapsuleEvidence.js +312 -0
- package/dist/src/evaluation/stageContextMetrics/collectRetrievalAuditEvidence.js +35 -0
- package/dist/src/evaluation/stageContextMetrics/collectWorkflowInstructionEvidence.js +137 -0
- package/dist/src/evaluation/stageContextMetrics/evaluateProducerReadinessBridge.js +127 -0
- package/dist/src/evaluation/stageContextMetrics/evaluateV043StageContextExecution.js +247 -0
- package/dist/src/evaluation/stageContextMetrics/index.js +17 -0
- package/dist/src/evaluation/stageContextMetrics/matchStageContextExpectations.js +38 -0
- package/dist/src/evaluation/stageContextMetrics/producerReadinessMetricTypes.js +1 -0
- package/dist/src/evaluation/stageContextMetrics/targetKeys.js +35 -0
- package/dist/src/evaluation/stageContextMetrics/types.js +1 -0
- package/dist/src/evaluation/stageContextSelectors/contextArtifactConsistency.js +89 -0
- package/dist/src/evaluation/stageContextSelectors/contextCapsuleSelectors.js +126 -0
- package/dist/src/evaluation/stageContextSelectors/index.js +4 -0
- package/dist/src/evaluation/stageContextSelectors/retrievalAuditRecordSelectors.js +54 -0
- package/dist/src/evaluation/stageContextSelectors/workflowInstructionPacketSelectors.js +51 -0
- package/dist/src/evaluation/targetImmutability/captureTargetSnapshot.js +208 -0
- package/dist/src/evaluation/targetImmutability/compareTargetSnapshots.js +56 -0
- package/dist/src/evaluation/targetImmutability/index.js +3 -0
- package/dist/src/evaluation/targetImmutability/types.js +1 -0
- package/dist/src/evaluation/targetImmutability/validateTargetImmutabilityConfig.js +130 -0
- package/dist/src/evaluation/upstreamArtifacts/artifactReadTypes.js +1 -0
- package/dist/src/evaluation/upstreamArtifacts/index.js +9 -0
- package/dist/src/evaluation/upstreamArtifacts/jsonTypes.js +1 -0
- package/dist/src/evaluation/upstreamArtifacts/myDevKitClassificationTypesV1.js +1 -0
- package/dist/src/evaluation/upstreamArtifacts/myDevKitContextArtifactsV1.js +1 -0
- package/dist/src/evaluation/upstreamArtifacts/myDevKitSemanticTypesV1.js +1 -0
- package/dist/src/evaluation/upstreamArtifacts/orchestratorContextReadinessResultV1.js +247 -0
- package/dist/src/evaluation/upstreamArtifacts/orchestratorWorkflowInstructionPacketV1.js +1 -0
- package/dist/src/evaluation/upstreamArtifacts/readImplementationContextPacketV1.js +9 -0
- package/dist/src/evaluation/upstreamArtifacts/readImplementationContextRetrievalReportV1.js +9 -0
- package/dist/src/evaluation/upstreamArtifacts/readJsonArtifact.js +53 -0
- package/dist/src/evaluation/upstreamArtifacts/readMyDevKitContextCapsuleV1.js +9 -0
- package/dist/src/evaluation/upstreamArtifacts/readMyDevKitRetrievalAuditRecordV1.js +9 -0
- package/dist/src/evaluation/upstreamArtifacts/readOrchestratorWorkflowInstructionPacketV1.js +9 -0
- package/dist/src/evaluation/upstreamArtifacts/readSupplementalContextTextFile.js +29 -0
- package/dist/src/evaluation/upstreamArtifacts/readTestContextPacketV1.js +9 -0
- package/dist/src/evaluation/upstreamArtifacts/readTestContextRetrievalReportV1.js +9 -0
- package/dist/src/evaluation/upstreamArtifacts/runtimeValidation.js +131 -0
- package/dist/src/evaluation/upstreamArtifacts/schemaVersion.js +32 -0
- package/dist/src/evaluation/upstreamArtifacts/supplementalContextTextParser.js +75 -0
- package/dist/src/evaluation/upstreamArtifacts/supplementalContextTypes.js +152 -0
- package/dist/src/evaluation/upstreamArtifacts/validateMyDevKitContextCapsuleV1.js +1208 -0
- package/dist/src/evaluation/upstreamArtifacts/validateMyDevKitRetrievalAuditRecordV1.js +112 -0
- package/dist/src/evaluation/upstreamArtifacts/validateOrchestratorWorkflowInstructionPacketV1.js +235 -0
- package/dist/src/evaluation/upstreamArtifacts/validateSupplementalContextDocumentV1.js +148 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/executeV043StageContextStrategy.js +46 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/index.js +9 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/loadV043StrategyArtifacts.js +397 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/plugin.js +39 -1
- package/dist/src/experiments/plugins/contextStrategyComparison/readV043FullWorkflowLibraryFixture.js +162 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/resolveV043StrategyInputs.js +43 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/resultMapping.js +3 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/runV043StageContextStrategyWithAssurance.js +205 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/v043FullWorkflowLibraryFixture.js +1 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/v043RunAssuranceTypes.js +1 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/v043StrategyExecutionTypes.js +1 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/v043StrategyIds.js +13 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/v043StrategyInputContracts.js +1 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/validateV043RunAssuranceConfig.js +59 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/validateV043StrategyInput.js +266 -0
- package/dist/src/report/experiments/buildContextStrategyComparisonV043Report.js +520 -0
- package/dist/src/report/experiments/buildPluginExperimentReport.js +21 -6
- package/dist/src/report/experiments/contextStrategyComparisonV043ReportModel.js +1 -0
- package/dist/src/report/experiments/index.js +4 -0
- package/dist/src/report/experiments/renderContextStrategyComparisonV043Html.js +296 -0
- package/dist/src/report/experiments/renderPluginExperimentReportHtml.js +3 -0
- package/dist/src/report/experiments/renderPluginExperimentReportText.js +292 -0
- package/dist/src/report/experiments/writePluginExperimentReports.js +3 -0
- package/dist/src/securityValidation/attackScenarios/scenarios/secretLeakageScenario.js +215 -9
- package/docs/METRICS.md +84 -10
- package/package.json +2 -2
- package/tests/fixtures/full-workflow-library/complete-v1.0.0.json +24 -0
- package/tests/fixtures/full-workflow-library/future-minor-v1.1.0-additive.json +25 -0
- package/tests/fixtures/stage-context-expectations/complete-v1.0.0.json +230 -0
- package/tests/fixtures/stage-context-expectations/future-minor-v1.1.0-additive.json +232 -0
- package/tests/fixtures/stage-context-expectations/invalid/malformed-json.txt +1 -0
- package/tests/fixtures/stage-context-expectations/invalid/non-object-root.json +1 -0
- package/tests/fixtures/stage-context-expectations/minimal-v1.0.0.json +18 -0
- package/tests/fixtures/upstream-artifacts/invalid/malformed-json.txt +1 -0
- package/tests/fixtures/upstream-artifacts/invalid/non-object-root.json +1 -0
- package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.2/context-capsule/complete-v1.0.0.json +504 -0
- package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.2/context-capsule/future-minor-v1.1.0-additive.json +955 -0
- package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.2/retrieval-audit-record/complete-v1.0.0.json +190 -0
- package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.2/retrieval-audit-record/future-minor-v1.1.0-additive.json +327 -0
- package/tests/fixtures/upstream-artifacts/my-dev-kit-1.10.2-identity/PROVENANCE.txt +13 -0
- package/tests/fixtures/upstream-artifacts/my-dev-kit-1.10.2-identity/context-capsule.json +7874 -0
- package/tests/fixtures/upstream-artifacts/my-dev-kit-1.10.2-identity/retrieval-audit-record.json +1076 -0
- package/tests/fixtures/upstream-artifacts/my-dev-kit-orchestrator/1.2.1/workflow-instruction-packet/complete-v1.0.0.json +160 -0
- package/tests/fixtures/upstream-artifacts/my-dev-kit-orchestrator/1.2.1/workflow-instruction-packet/future-minor-v1.1.0-additive.json +186 -0
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
function escapeHtml(value) {
|
|
2
|
+
return String(value ?? "")
|
|
3
|
+
.replaceAll("&", "&")
|
|
4
|
+
.replaceAll("<", "<")
|
|
5
|
+
.replaceAll(">", ">")
|
|
6
|
+
.replaceAll('"', """)
|
|
7
|
+
.replaceAll("'", "'");
|
|
8
|
+
}
|
|
9
|
+
function table(headers, rows) {
|
|
10
|
+
return `<table><thead><tr>${headers.map((header) => `<th>${escapeHtml(header)}</th>`).join("")}</tr></thead><tbody>${rows
|
|
11
|
+
.map((row) => `<tr>${row.map((cell) => `<td>${escapeHtml(cell)}</td>`).join("")}</tr>`)
|
|
12
|
+
.join("")}</tbody></table>`;
|
|
13
|
+
}
|
|
14
|
+
function boundedListNote(list) {
|
|
15
|
+
return `<p class="muted">Displayed ${list.displayedCount} of ${list.totalCount} (omitted ${list.omittedCount}).</p>`;
|
|
16
|
+
}
|
|
17
|
+
function renderRatioMetricTable(label, metric) {
|
|
18
|
+
if (metric.availability !== "available") {
|
|
19
|
+
return `<h4>${escapeHtml(label)}</h4>${table(["Availability", "Reason"], [[metric.availability, metric.reason ?? ""]])}`;
|
|
20
|
+
}
|
|
21
|
+
return `<h4>${escapeHtml(label)}</h4>${table(["Availability", "Numerator", "Denominator", "Rate", "Percentage"], [
|
|
22
|
+
[
|
|
23
|
+
metric.availability,
|
|
24
|
+
String(metric.numerator),
|
|
25
|
+
String(metric.denominator),
|
|
26
|
+
String(metric.rate),
|
|
27
|
+
`${String(metric.rate * 100)}%`,
|
|
28
|
+
],
|
|
29
|
+
])}`;
|
|
30
|
+
}
|
|
31
|
+
function renderCountMetricTable(label, metric) {
|
|
32
|
+
if (metric.availability !== "available") {
|
|
33
|
+
return `<h4>${escapeHtml(label)}</h4>${table(["Availability", "Reason"], [[metric.availability, metric.reason ?? ""]])}`;
|
|
34
|
+
}
|
|
35
|
+
return `<h4>${escapeHtml(label)}</h4>${table(["Availability", "Count"], [[metric.availability, String(metric.count)]])}`;
|
|
36
|
+
}
|
|
37
|
+
function renderStrategySection(strategy, index) {
|
|
38
|
+
const artifactRows = strategy.artifacts.items.map((artifact) => [
|
|
39
|
+
artifact.sourceInstance,
|
|
40
|
+
artifact.artifactKind,
|
|
41
|
+
artifact.sourcePath,
|
|
42
|
+
artifact.schemaVersion,
|
|
43
|
+
artifact.role ?? "",
|
|
44
|
+
]);
|
|
45
|
+
const executionIssueRows = strategy.execution.issues.items.map((issue) => [
|
|
46
|
+
issue.code,
|
|
47
|
+
issue.fieldPath ?? "",
|
|
48
|
+
issue.message,
|
|
49
|
+
]);
|
|
50
|
+
const expectationMatchRows = strategy.evaluation.expectationMatches.items.map((match) => [
|
|
51
|
+
match.expectationId,
|
|
52
|
+
match.inclusion,
|
|
53
|
+
match.sourceArtifact,
|
|
54
|
+
match.category,
|
|
55
|
+
match.targetKey,
|
|
56
|
+
match.outcome,
|
|
57
|
+
]);
|
|
58
|
+
const observedEvidenceRows = strategy.evaluation.observedEvidence.items.map((evidence) => [
|
|
59
|
+
evidence.sourceArtifact,
|
|
60
|
+
evidence.sourceInstance,
|
|
61
|
+
evidence.category,
|
|
62
|
+
evidence.targetKey,
|
|
63
|
+
evidence.sourceFieldPath,
|
|
64
|
+
]);
|
|
65
|
+
const metricsHtml = strategy.evaluation.metrics
|
|
66
|
+
? `
|
|
67
|
+
${renderRatioMetricTable("Required Evidence Recall", strategy.evaluation.metrics.requiredEvidenceRecall)}
|
|
68
|
+
${renderRatioMetricTable("Allowed Evidence Coverage", strategy.evaluation.metrics.allowedEvidenceCoverage)}
|
|
69
|
+
${renderRatioMetricTable("Forbidden Evidence Inclusion", strategy.evaluation.metrics.forbiddenEvidenceInclusion)}
|
|
70
|
+
${renderCountMetricTable("Irrelevant File Inclusion", strategy.evaluation.metrics.irrelevantFileInclusion)}
|
|
71
|
+
${renderCountMetricTable("Irrelevant Instruction Inclusion", strategy.evaluation.metrics.irrelevantInstructionInclusion)}
|
|
72
|
+
${renderRatioMetricTable("Required Provenance Recall", strategy.evaluation.metrics.requiredProvenanceRecall)}
|
|
73
|
+
${renderCountMetricTable("Considered But Unselected Reads", strategy.evaluation.metrics.consideredButUnselectedReads)}
|
|
74
|
+
${renderCountMetricTable("Unnecessary Reads", strategy.evaluation.metrics.unnecessaryReads)}
|
|
75
|
+
${renderCountMetricTable("Target Immutability (Evaluation)", strategy.evaluation.metrics.targetImmutability)}
|
|
76
|
+
<h4>Responsibility Mapping</h4>
|
|
77
|
+
${boundedListNote(strategy.evaluation.metrics.responsibilityMappingCompleteness)}
|
|
78
|
+
${table(["Source", "Instance", "Requested", "Operational", "Mapped", "Partially Mapped", "Unmapped", "Not Applicable", "Denominator", "Mapped Rate"], strategy.evaluation.metrics.responsibilityMappingCompleteness.items.map((mapping) => [
|
|
79
|
+
mapping.sourceArtifact,
|
|
80
|
+
mapping.sourceInstance,
|
|
81
|
+
String(mapping.requested),
|
|
82
|
+
String(mapping.operational),
|
|
83
|
+
String(mapping.mappedCount),
|
|
84
|
+
String(mapping.partiallyMappedCount),
|
|
85
|
+
String(mapping.unmappedCount),
|
|
86
|
+
String(mapping.notApplicableCount),
|
|
87
|
+
String(mapping.denominator),
|
|
88
|
+
String(mapping.mappedRate),
|
|
89
|
+
]))}
|
|
90
|
+
<h4>State Comparisons</h4>
|
|
91
|
+
${boundedListNote(strategy.evaluation.metrics.stateComparisons)}
|
|
92
|
+
${table(["Source", "Instance", "Expectation Field", "Artifact Field", "Availability", "Expected", "Actual", "Matched"], strategy.evaluation.metrics.stateComparisons.items.map((comparison) => [
|
|
93
|
+
comparison.sourceArtifact,
|
|
94
|
+
comparison.sourceInstance,
|
|
95
|
+
comparison.expectationFieldPath,
|
|
96
|
+
comparison.artifactFieldPath ?? "",
|
|
97
|
+
comparison.availability,
|
|
98
|
+
JSON.stringify(comparison.expected),
|
|
99
|
+
JSON.stringify(comparison.actual),
|
|
100
|
+
String(comparison.matched),
|
|
101
|
+
]))}
|
|
102
|
+
<h4>Context Size</h4>
|
|
103
|
+
${table(["Total Characters", "Total Estimated Tokens", "Formula"], [[
|
|
104
|
+
String(strategy.evaluation.metrics.contextSize.totalCharacterCount),
|
|
105
|
+
String(strategy.evaluation.metrics.contextSize.totalEstimatedTokenCount),
|
|
106
|
+
strategy.evaluation.metrics.contextSize.tokenEstimateFormula,
|
|
107
|
+
]])}
|
|
108
|
+
${boundedListNote(strategy.evaluation.metrics.contextSize.sources)}
|
|
109
|
+
${table(["Instance", "Kind", "Characters", "Estimated Tokens"], strategy.evaluation.metrics.contextSize.sources.items.map((source) => [
|
|
110
|
+
source.sourceInstance,
|
|
111
|
+
source.sourceKind,
|
|
112
|
+
String(source.characterCount),
|
|
113
|
+
String(source.estimatedTokenCount),
|
|
114
|
+
]))}`
|
|
115
|
+
: "<p>Metrics unavailable.</p>";
|
|
116
|
+
const targetImmutabilityRows = strategy.assurance.runRecords.items.map((record) => [
|
|
117
|
+
String(record.runNumber),
|
|
118
|
+
record.targetImmutabilityAvailability,
|
|
119
|
+
record.targetImmutabilityStatus ?? "",
|
|
120
|
+
record.newMutationCount === null ? "unavailable" : String(record.newMutationCount),
|
|
121
|
+
record.targetImmutabilityReason ?? "",
|
|
122
|
+
]);
|
|
123
|
+
const mutationRows = strategy.assurance.runRecords.items.flatMap((record) => record.mutations.items.map((mutation) => [
|
|
124
|
+
String(record.runNumber),
|
|
125
|
+
mutation.id,
|
|
126
|
+
mutation.kind,
|
|
127
|
+
mutation.fieldPath,
|
|
128
|
+
JSON.stringify(mutation.before),
|
|
129
|
+
JSON.stringify(mutation.after),
|
|
130
|
+
]));
|
|
131
|
+
const determinism = strategy.assurance.determinism;
|
|
132
|
+
const determinismRows = determinism.availability === "available"
|
|
133
|
+
? [
|
|
134
|
+
[
|
|
135
|
+
determinism.availability,
|
|
136
|
+
String(determinism.repeatCount),
|
|
137
|
+
String(determinism.deterministic),
|
|
138
|
+
determinism.baselineSha256 ?? "",
|
|
139
|
+
determinism.mismatchRunNumbers.join(", "),
|
|
140
|
+
],
|
|
141
|
+
]
|
|
142
|
+
: [[determinism.availability, String(determinism.repeatCount), "", "", determinism.reason ?? ""]];
|
|
143
|
+
const assuranceIssueRows = strategy.assurance.issues.items.map((issue) => [
|
|
144
|
+
issue.code,
|
|
145
|
+
issue.runNumber === null ? "" : String(issue.runNumber),
|
|
146
|
+
issue.fieldPath ?? "",
|
|
147
|
+
issue.message,
|
|
148
|
+
]);
|
|
149
|
+
return `
|
|
150
|
+
<section>
|
|
151
|
+
<h3>Strategy ${index + 1}: ${escapeHtml(strategy.strategyId)}</h3>
|
|
152
|
+
|
|
153
|
+
<h4>Artifacts</h4>
|
|
154
|
+
${boundedListNote(strategy.artifacts)}
|
|
155
|
+
${table(["Instance", "Kind", "Path", "Schema", "Role"], artifactRows)}
|
|
156
|
+
|
|
157
|
+
<h4>Execution</h4>
|
|
158
|
+
${table(["Status"], [[strategy.execution.status]])}
|
|
159
|
+
${boundedListNote(strategy.execution.issues)}
|
|
160
|
+
${table(["Code", "Field Path", "Message"], executionIssueRows)}
|
|
161
|
+
|
|
162
|
+
<h4>Evaluation</h4>
|
|
163
|
+
${table(["Status", "Reason"], [[strategy.evaluation.status, strategy.evaluation.reason ?? ""]])}
|
|
164
|
+
${metricsHtml}
|
|
165
|
+
|
|
166
|
+
<h4>Expectation Matches</h4>
|
|
167
|
+
${boundedListNote(strategy.evaluation.expectationMatches)}
|
|
168
|
+
${table(["Expectation ID", "Inclusion", "Source Artifact", "Category", "Target Key", "Outcome"], expectationMatchRows)}
|
|
169
|
+
|
|
170
|
+
<h4>Observed Evidence</h4>
|
|
171
|
+
${boundedListNote(strategy.evaluation.observedEvidence)}
|
|
172
|
+
${table(["Source Artifact", "Instance", "Category", "Target Key", "Source Field Path"], observedEvidenceRows)}
|
|
173
|
+
|
|
174
|
+
<h4>Target Immutability</h4>
|
|
175
|
+
${boundedListNote(strategy.assurance.runRecords)}
|
|
176
|
+
${table(["Run", "Availability", "Status", "New Mutation Count", "Reason"], targetImmutabilityRows)}
|
|
177
|
+
${table(["Run", "Mutation ID", "Kind", "Field Path", "Before", "After"], mutationRows)}
|
|
178
|
+
|
|
179
|
+
<h4>Repeated-Run Determinism</h4>
|
|
180
|
+
${table(["Availability", "Repeat Count", "Deterministic", "Baseline SHA-256", "Mismatch Run Numbers / Reason"], determinismRows)}
|
|
181
|
+
${determinism.availability === "available" ? boundedListNote(determinism.runDigests) : ""}
|
|
182
|
+
${determinism.availability === "available"
|
|
183
|
+
? table(["Run", "SHA-256"], determinism.runDigests.items.map((digest) => [String(digest.runNumber), digest.sha256]))
|
|
184
|
+
: ""}
|
|
185
|
+
|
|
186
|
+
<h4>Run Assurance</h4>
|
|
187
|
+
${table(["Status", "Repeat Count"], [[strategy.assurance.status, String(strategy.assurance.repeatCount)]])}
|
|
188
|
+
${boundedListNote(strategy.assurance.issues)}
|
|
189
|
+
${table(["Code", "Run", "Field Path", "Message"], assuranceIssueRows)}
|
|
190
|
+
${strategy.producerReadinessBridge ? renderProducerReadinessBridgeSection(strategy.producerReadinessBridge) : ""}
|
|
191
|
+
</section>`;
|
|
192
|
+
}
|
|
193
|
+
function renderProducerReadinessSideSection(label, side) {
|
|
194
|
+
if (side === null) {
|
|
195
|
+
return `<h4>${escapeHtml(label)}</h4><p class="muted">Not supplied.</p>`;
|
|
196
|
+
}
|
|
197
|
+
const truncationRows = side.truncationEvaluation.items.map((t) => [
|
|
198
|
+
t.groupId,
|
|
199
|
+
t.cause,
|
|
200
|
+
t.availability,
|
|
201
|
+
String(t.limit),
|
|
202
|
+
String(t.used),
|
|
203
|
+
t.reason ?? "",
|
|
204
|
+
]);
|
|
205
|
+
function agreementSection(agreementLabel, agreement) {
|
|
206
|
+
if (agreement === null) {
|
|
207
|
+
return `<h5>${escapeHtml(agreementLabel)}</h5><p class="muted">Unavailable.</p>`;
|
|
208
|
+
}
|
|
209
|
+
const rows = agreement.contradictions.map((c) => [c.field, String(c.rawValue), String(c.supplementalValue), c.availability]);
|
|
210
|
+
return `<h5>${escapeHtml(agreementLabel)}</h5>
|
|
211
|
+
<p>Producer parity preserved: ${escapeHtml(agreement.upstreamProducerParityPreserved)}</p>
|
|
212
|
+
${table(["Field", "Raw Value", "Supplemental Value", "Availability"], rows)}`;
|
|
213
|
+
}
|
|
214
|
+
return `<h4>${escapeHtml(label)}</h4>
|
|
215
|
+
${renderRatioMetricTable("Expected Owner Present", side.ownerEvaluation.expectedOwnerPresent)}
|
|
216
|
+
${renderRatioMetricTable("Forbidden Owner Present", side.ownerEvaluation.forbiddenOwnerPresent)}
|
|
217
|
+
${renderCountMetricTable("Owner False Positives", side.ownerEvaluation.falsePositiveCount)}
|
|
218
|
+
${renderCountMetricTable("Owner False Negatives", side.ownerEvaluation.falseNegativeCount)}
|
|
219
|
+
<h5>Truncation Classification</h5>
|
|
220
|
+
${boundedListNote(side.truncationEvaluation)}
|
|
221
|
+
${table(["Group", "Cause", "Availability", "Limit", "Used", "Reason"], truncationRows)}
|
|
222
|
+
${agreementSection("Packet/Raw Agreement", side.packetAgreement)}
|
|
223
|
+
${agreementSection("Report/Raw Agreement", side.reportAgreement)}`;
|
|
224
|
+
}
|
|
225
|
+
function renderProducerReadinessBridgeSection(bridge) {
|
|
226
|
+
if (bridge.status === "not-applicable") {
|
|
227
|
+
return `<h4>Producer-Readiness Bridge</h4><p class="muted">${escapeHtml(bridge.reason)}</p>`;
|
|
228
|
+
}
|
|
229
|
+
const criticality = bridge.criticalityEvaluation;
|
|
230
|
+
return `<h4>Producer-Readiness Bridge</h4>
|
|
231
|
+
${renderProducerReadinessSideSection("Implementation", bridge.implementation)}
|
|
232
|
+
${renderProducerReadinessSideSection("Test", bridge.testImplementation)}
|
|
233
|
+
<h5>Readiness Agreement</h5>
|
|
234
|
+
${bridge.readinessAgreement === null
|
|
235
|
+
? `<p class="muted">Not supplied.</p>`
|
|
236
|
+
: table(["Observed Decision", "Decision Agreement", "Issue-Code Agreement", "Invalid Ready", "Valid Blocked", "Valid Refresh-Required"], [
|
|
237
|
+
[
|
|
238
|
+
String(bridge.readinessAgreement.decisionAgreement.observedDecision),
|
|
239
|
+
String(bridge.readinessAgreement.decisionAgreement.decisionAgreement),
|
|
240
|
+
String(bridge.readinessAgreement.decisionAgreement.issueCodesAgreement),
|
|
241
|
+
String(bridge.readinessAgreement.invalidReady.invalidReady),
|
|
242
|
+
String(bridge.readinessAgreement.validBlocked.validBlocked),
|
|
243
|
+
String(bridge.readinessAgreement.validBlocked.validRefreshRequired),
|
|
244
|
+
],
|
|
245
|
+
])}
|
|
246
|
+
<h5>Criticality Evaluation</h5>
|
|
247
|
+
${criticality === null
|
|
248
|
+
? `<p class="muted">Not supplied.</p>`
|
|
249
|
+
: `${renderRatioMetricTable("Mapped Critical Completeness", criticality.mappedCriticalCompleteness)}
|
|
250
|
+
<p>Partially mapped (excluded from complete): ${escapeHtml(criticality.partiallyMappedCriticalIds.join(", ") || "none")}</p>
|
|
251
|
+
<p>Unmapped: ${escapeHtml(criticality.unmappedCriticalIds.join(", ") || "none")}</p>
|
|
252
|
+
<p>Conflicting: ${escapeHtml(criticality.conflictingCriticalityResponsibilityIds.join(", ") || "none")}</p>`}`;
|
|
253
|
+
}
|
|
254
|
+
export function renderContextStrategyComparisonV043Html(report) {
|
|
255
|
+
if (report === null) {
|
|
256
|
+
return `<section><h2>V0.4.3 Stage-Context Evidence</h2><p>Not applicable to this plugin.</p></section>`;
|
|
257
|
+
}
|
|
258
|
+
if (report.summary.strategyCount === 0) {
|
|
259
|
+
return `<section><h2>V0.4.3 Stage-Context Evidence</h2><p>No v0.4.3 stage-context strategies were selected.</p></section>`;
|
|
260
|
+
}
|
|
261
|
+
const summaryRows = [
|
|
262
|
+
[
|
|
263
|
+
String(report.summary.strategyCount),
|
|
264
|
+
String(report.summary.completedExecutionCount),
|
|
265
|
+
String(report.summary.invalidInputExecutionCount),
|
|
266
|
+
String(report.summary.failedExecutionCount),
|
|
267
|
+
String(report.summary.completedEvaluationCount),
|
|
268
|
+
String(report.summary.notApplicableEvaluationCount),
|
|
269
|
+
String(report.summary.failedEvaluationCount),
|
|
270
|
+
String(report.summary.passedAssuranceCount),
|
|
271
|
+
String(report.summary.failedAssuranceCount),
|
|
272
|
+
String(report.summary.notApplicableAssuranceCount),
|
|
273
|
+
],
|
|
274
|
+
];
|
|
275
|
+
return `
|
|
276
|
+
<section>
|
|
277
|
+
<h2>V0.4.3 Stage-Context Evidence</h2>
|
|
278
|
+
${table([
|
|
279
|
+
"Strategies",
|
|
280
|
+
"Completed Executions",
|
|
281
|
+
"Invalid-Input Executions",
|
|
282
|
+
"Failed Executions",
|
|
283
|
+
"Completed Evaluations",
|
|
284
|
+
"Not-Applicable Evaluations",
|
|
285
|
+
"Failed Evaluations",
|
|
286
|
+
"Passed Assurance",
|
|
287
|
+
"Failed Assurance",
|
|
288
|
+
"Not-Applicable Assurance",
|
|
289
|
+
], summaryRows)}
|
|
290
|
+
<p>${escapeHtml(report.interpretation.summary)}</p>
|
|
291
|
+
<ul>
|
|
292
|
+
${report.interpretation.limitations.map((limitation) => `<li>${escapeHtml(limitation)}</li>`).join("")}
|
|
293
|
+
</ul>
|
|
294
|
+
${report.strategies.map((strategy, index) => renderStrategySection(strategy, index)).join("")}
|
|
295
|
+
</section>`;
|
|
296
|
+
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { renderContextStrategyComparisonV043Html } from "./renderContextStrategyComparisonV043Html.js";
|
|
1
2
|
export function renderPluginExperimentReportHtml(report) {
|
|
2
3
|
return `<!DOCTYPE html>
|
|
3
4
|
<html lang="en">
|
|
@@ -93,6 +94,8 @@ export function renderPluginExperimentReportHtml(report) {
|
|
|
93
94
|
]))}
|
|
94
95
|
</section>
|
|
95
96
|
|
|
97
|
+
${renderContextStrategyComparisonV043Html(report.contextStrategyComparisonV043)}
|
|
98
|
+
|
|
96
99
|
<section>
|
|
97
100
|
<h2>Warnings, Skips, And Failures</h2>
|
|
98
101
|
${report.findings.length ? table(["Severity", "Code", "Message", "Variant", "Case"], report.findings.map((finding) => [
|
|
@@ -0,0 +1,292 @@
|
|
|
1
|
+
function sanitizeScalar(value) {
|
|
2
|
+
const text = String(value);
|
|
3
|
+
let result = "";
|
|
4
|
+
for (const ch of text) {
|
|
5
|
+
const code = ch.codePointAt(0) ?? 0;
|
|
6
|
+
if (code === 13 || code === 10 || code === 9) {
|
|
7
|
+
result += " ";
|
|
8
|
+
continue;
|
|
9
|
+
}
|
|
10
|
+
if (code <= 8)
|
|
11
|
+
continue;
|
|
12
|
+
if (code === 11 || code === 12)
|
|
13
|
+
continue;
|
|
14
|
+
if (code >= 14 && code <= 31)
|
|
15
|
+
continue;
|
|
16
|
+
if (code === 127)
|
|
17
|
+
continue;
|
|
18
|
+
result += ch;
|
|
19
|
+
}
|
|
20
|
+
return result;
|
|
21
|
+
}
|
|
22
|
+
function fieldLine(label, value) {
|
|
23
|
+
if (value === null || value === undefined)
|
|
24
|
+
return `${label}: unavailable`;
|
|
25
|
+
return `${label}: ${sanitizeScalar(value)}`;
|
|
26
|
+
}
|
|
27
|
+
function pushSection(lines, title) {
|
|
28
|
+
lines.push(title);
|
|
29
|
+
}
|
|
30
|
+
function pushDashList(lines, items) {
|
|
31
|
+
if (items.length === 0) {
|
|
32
|
+
lines.push("- none");
|
|
33
|
+
return;
|
|
34
|
+
}
|
|
35
|
+
for (const item of items) {
|
|
36
|
+
lines.push(`- ${sanitizeScalar(item)}`);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
function pushBoundedList(lines, list, renderItem) {
|
|
40
|
+
lines.push(`Displayed: ${list.displayedCount} of ${list.totalCount}`);
|
|
41
|
+
lines.push(`Omitted: ${list.omittedCount}`);
|
|
42
|
+
if (list.totalCount === 0) {
|
|
43
|
+
lines.push("- none");
|
|
44
|
+
return;
|
|
45
|
+
}
|
|
46
|
+
for (const item of list.items) {
|
|
47
|
+
lines.push(`- ${sanitizeScalar(renderItem(item))}`);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
function pushRatioMetric(lines, label, metric) {
|
|
51
|
+
lines.push(label);
|
|
52
|
+
lines.push(`Availability: ${metric.availability}`);
|
|
53
|
+
if (metric.availability === "available") {
|
|
54
|
+
lines.push(`Numerator: ${metric.numerator}`);
|
|
55
|
+
lines.push(`Denominator: ${metric.denominator}`);
|
|
56
|
+
lines.push(`Rate: ${metric.rate}`);
|
|
57
|
+
lines.push(`Percentage: ${String(metric.rate * 100)}%`);
|
|
58
|
+
}
|
|
59
|
+
else {
|
|
60
|
+
lines.push(`Reason: ${sanitizeScalar(metric.reason ?? "")}`);
|
|
61
|
+
}
|
|
62
|
+
lines.push("Matched Expectation IDs:");
|
|
63
|
+
pushBoundedList(lines, metric.matchedExpectationIds, (id) => id);
|
|
64
|
+
lines.push("Missing Expectation IDs:");
|
|
65
|
+
pushBoundedList(lines, metric.missingExpectationIds, (id) => id);
|
|
66
|
+
}
|
|
67
|
+
function pushCountMetric(lines, label, metric) {
|
|
68
|
+
lines.push(label);
|
|
69
|
+
lines.push(`Availability: ${metric.availability}`);
|
|
70
|
+
if (metric.availability === "available") {
|
|
71
|
+
lines.push(`Count: ${metric.count}`);
|
|
72
|
+
}
|
|
73
|
+
else {
|
|
74
|
+
lines.push(`Reason: ${sanitizeScalar(metric.reason ?? "")}`);
|
|
75
|
+
}
|
|
76
|
+
lines.push("Evidence Keys:");
|
|
77
|
+
pushBoundedList(lines, metric.evidenceKeys, (key) => key);
|
|
78
|
+
}
|
|
79
|
+
function renderStrategy(lines, strategy, index) {
|
|
80
|
+
pushSection(lines, `Strategy ${index + 1}: ${strategy.strategyId}`);
|
|
81
|
+
pushSection(lines, "Artifacts");
|
|
82
|
+
pushBoundedList(lines, strategy.artifacts, (artifact) => `${artifact.sourceInstance} [${artifact.artifactKind}] path=${artifact.sourcePath} schema=${artifact.schemaVersion}`);
|
|
83
|
+
pushSection(lines, "Execution");
|
|
84
|
+
lines.push(fieldLine("Status", strategy.execution.status));
|
|
85
|
+
lines.push("Issues:");
|
|
86
|
+
pushBoundedList(lines, strategy.execution.issues, (issue) => `${issue.code} ${issue.fieldPath ?? ""} ${issue.message}`);
|
|
87
|
+
pushSection(lines, "Evaluation Metrics");
|
|
88
|
+
lines.push(fieldLine("Status", strategy.evaluation.status));
|
|
89
|
+
lines.push(fieldLine("Reason", strategy.evaluation.reason));
|
|
90
|
+
lines.push("Warnings:");
|
|
91
|
+
pushBoundedList(lines, strategy.evaluation.warnings, (warning) => warning);
|
|
92
|
+
if (strategy.evaluation.metrics) {
|
|
93
|
+
const metrics = strategy.evaluation.metrics;
|
|
94
|
+
pushRatioMetric(lines, "Required Evidence Recall", metrics.requiredEvidenceRecall);
|
|
95
|
+
pushRatioMetric(lines, "Allowed Evidence Coverage", metrics.allowedEvidenceCoverage);
|
|
96
|
+
pushRatioMetric(lines, "Forbidden Evidence Inclusion", metrics.forbiddenEvidenceInclusion);
|
|
97
|
+
pushCountMetric(lines, "Irrelevant File Inclusion", metrics.irrelevantFileInclusion);
|
|
98
|
+
pushCountMetric(lines, "Irrelevant Instruction Inclusion", metrics.irrelevantInstructionInclusion);
|
|
99
|
+
pushRatioMetric(lines, "Required Provenance Recall", metrics.requiredProvenanceRecall);
|
|
100
|
+
pushCountMetric(lines, "Considered But Unselected Reads", metrics.consideredButUnselectedReads);
|
|
101
|
+
pushCountMetric(lines, "Unnecessary Reads", metrics.unnecessaryReads);
|
|
102
|
+
pushCountMetric(lines, "Target Immutability (Evaluation)", metrics.targetImmutability);
|
|
103
|
+
}
|
|
104
|
+
pushSection(lines, "Expectation Matches");
|
|
105
|
+
pushBoundedList(lines, strategy.evaluation.expectationMatches, (match) => `${match.expectationId} inclusion=${match.inclusion} outcome=${match.outcome} sourceArtifact=${match.sourceArtifact} targetKey=${match.targetKey}`);
|
|
106
|
+
pushSection(lines, "Observed Evidence");
|
|
107
|
+
pushBoundedList(lines, strategy.evaluation.observedEvidence, (evidence) => `${evidence.sourceArtifact}.${evidence.sourceInstance} category=${evidence.category} targetKey=${evidence.targetKey} field=${evidence.sourceFieldPath}`);
|
|
108
|
+
pushSection(lines, "State Comparisons");
|
|
109
|
+
if (strategy.evaluation.metrics) {
|
|
110
|
+
pushBoundedList(lines, strategy.evaluation.metrics.stateComparisons, (comparison) => `${comparison.sourceArtifact}.${comparison.sourceInstance} expectationFieldPath=${comparison.expectationFieldPath} artifactFieldPath=${comparison.artifactFieldPath ?? "null"} availability=${comparison.availability} expected=${JSON.stringify(comparison.expected)} actual=${JSON.stringify(comparison.actual)} matched=${comparison.matched}`);
|
|
111
|
+
}
|
|
112
|
+
else {
|
|
113
|
+
lines.push("Displayed: 0 of 0");
|
|
114
|
+
lines.push("Omitted: 0");
|
|
115
|
+
lines.push("- none");
|
|
116
|
+
}
|
|
117
|
+
pushSection(lines, "Responsibility Mapping");
|
|
118
|
+
if (strategy.evaluation.metrics) {
|
|
119
|
+
pushBoundedList(lines, strategy.evaluation.metrics.responsibilityMappingCompleteness, (mapping) => `${mapping.sourceArtifact}.${mapping.sourceInstance} requested=${mapping.requested} operational=${mapping.operational} mapped=${mapping.mappedCount} partiallyMapped=${mapping.partiallyMappedCount} unmapped=${mapping.unmappedCount} notApplicable=${mapping.notApplicableCount} denominator=${mapping.denominator} mappedRate=${mapping.mappedRate}`);
|
|
120
|
+
}
|
|
121
|
+
else {
|
|
122
|
+
lines.push("Displayed: 0 of 0");
|
|
123
|
+
lines.push("Omitted: 0");
|
|
124
|
+
lines.push("- none");
|
|
125
|
+
}
|
|
126
|
+
pushSection(lines, "Context Size");
|
|
127
|
+
if (strategy.evaluation.metrics) {
|
|
128
|
+
const size = strategy.evaluation.metrics.contextSize;
|
|
129
|
+
lines.push(fieldLine("Total Character Count", size.totalCharacterCount));
|
|
130
|
+
lines.push(fieldLine("Total Estimated Token Count (estimate)", size.totalEstimatedTokenCount));
|
|
131
|
+
lines.push(fieldLine("Token Estimate Formula", size.tokenEstimateFormula));
|
|
132
|
+
pushBoundedList(lines, size.sources, (source) => `${source.sourceInstance} kind=${source.sourceKind} characters=${source.characterCount} estimatedTokens=${source.estimatedTokenCount}`);
|
|
133
|
+
}
|
|
134
|
+
else {
|
|
135
|
+
lines.push("Displayed: 0 of 0");
|
|
136
|
+
lines.push("Omitted: 0");
|
|
137
|
+
lines.push("- none");
|
|
138
|
+
}
|
|
139
|
+
pushSection(lines, "Target Immutability");
|
|
140
|
+
pushBoundedList(lines, strategy.assurance.runRecords, (record) => {
|
|
141
|
+
if (record.targetImmutabilityAvailability === "unavailable") {
|
|
142
|
+
return `run=${record.runNumber} availability=unavailable reason=${record.targetImmutabilityReason ?? ""}`;
|
|
143
|
+
}
|
|
144
|
+
return `run=${record.runNumber} availability=available status=${record.targetImmutabilityStatus} newMutationCount=${record.newMutationCount} mutations=${record.mutations.totalCount}`;
|
|
145
|
+
});
|
|
146
|
+
pushSection(lines, "Repeated-Run Determinism");
|
|
147
|
+
const determinism = strategy.assurance.determinism;
|
|
148
|
+
lines.push(fieldLine("Availability", determinism.availability));
|
|
149
|
+
lines.push(fieldLine("Repeat Count", determinism.repeatCount));
|
|
150
|
+
if (determinism.availability === "available") {
|
|
151
|
+
lines.push(fieldLine("Deterministic", determinism.deterministic));
|
|
152
|
+
lines.push(fieldLine("Baseline SHA-256", determinism.baselineSha256));
|
|
153
|
+
lines.push("Run Digests:");
|
|
154
|
+
pushBoundedList(lines, determinism.runDigests, (digest) => `run=${digest.runNumber} sha256=${digest.sha256}`);
|
|
155
|
+
lines.push(`Mismatch Run Numbers: ${determinism.mismatchRunNumbers.join(", ") || "none"}`);
|
|
156
|
+
}
|
|
157
|
+
else {
|
|
158
|
+
lines.push(fieldLine("Reason", determinism.reason));
|
|
159
|
+
}
|
|
160
|
+
pushSection(lines, "Run Assurance");
|
|
161
|
+
lines.push(fieldLine("Status", strategy.assurance.status));
|
|
162
|
+
lines.push(fieldLine("Repeat Count", strategy.assurance.repeatCount));
|
|
163
|
+
lines.push("Issues:");
|
|
164
|
+
pushBoundedList(lines, strategy.assurance.issues, (issue) => `${issue.code} run=${issue.runNumber ?? ""} field=${issue.fieldPath ?? ""} ${issue.message}`);
|
|
165
|
+
if (strategy.producerReadinessBridge) {
|
|
166
|
+
renderProducerReadinessBridge(lines, strategy.producerReadinessBridge);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
function renderProducerReadinessSide(lines, label, side) {
|
|
170
|
+
pushSection(lines, `Producer-Readiness Bridge: ${label}`);
|
|
171
|
+
if (side === null) {
|
|
172
|
+
lines.push("- not supplied");
|
|
173
|
+
return;
|
|
174
|
+
}
|
|
175
|
+
pushRatioMetric(lines, "Expected Owner Present", side.ownerEvaluation.expectedOwnerPresent);
|
|
176
|
+
pushRatioMetric(lines, "Forbidden Owner Present", side.ownerEvaluation.forbiddenOwnerPresent);
|
|
177
|
+
pushCountMetric(lines, "Owner False Positives", side.ownerEvaluation.falsePositiveCount);
|
|
178
|
+
pushCountMetric(lines, "Owner False Negatives", side.ownerEvaluation.falseNegativeCount);
|
|
179
|
+
pushSection(lines, "Truncation Classification");
|
|
180
|
+
pushBoundedList(lines, side.truncationEvaluation, (t) => `group=${t.groupId} cause=${t.cause} availability=${t.availability} limit=${t.limit} used=${t.used}`);
|
|
181
|
+
for (const [agreementLabel, agreement] of [
|
|
182
|
+
["Packet/Raw Agreement", side.packetAgreement],
|
|
183
|
+
["Report/Raw Agreement", side.reportAgreement],
|
|
184
|
+
]) {
|
|
185
|
+
pushSection(lines, agreementLabel);
|
|
186
|
+
if (agreement === null) {
|
|
187
|
+
lines.push("- unavailable");
|
|
188
|
+
continue;
|
|
189
|
+
}
|
|
190
|
+
lines.push(`Producer Parity Preserved: ${agreement.upstreamProducerParityPreserved === null ? "unavailable" : agreement.upstreamProducerParityPreserved}`);
|
|
191
|
+
lines.push("Contradictions:");
|
|
192
|
+
pushDashList(lines, agreement.contradictions.map((c) => `${c.field}: raw=${sanitizeScalar(c.rawValue)} supplemental=${sanitizeScalar(c.supplementalValue)}`));
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
function renderProducerReadinessBridge(lines, bridge) {
|
|
196
|
+
pushSection(lines, "Producer-Readiness Bridge");
|
|
197
|
+
lines.push(fieldLine("Status", bridge.status));
|
|
198
|
+
if (bridge.status === "not-applicable") {
|
|
199
|
+
lines.push(fieldLine("Reason", bridge.reason));
|
|
200
|
+
return;
|
|
201
|
+
}
|
|
202
|
+
renderProducerReadinessSide(lines, "Implementation", bridge.implementation);
|
|
203
|
+
renderProducerReadinessSide(lines, "Test", bridge.testImplementation);
|
|
204
|
+
pushSection(lines, "Readiness Agreement");
|
|
205
|
+
if (bridge.readinessAgreement === null) {
|
|
206
|
+
lines.push("- not supplied");
|
|
207
|
+
}
|
|
208
|
+
else {
|
|
209
|
+
const decision = bridge.readinessAgreement.decisionAgreement;
|
|
210
|
+
lines.push(fieldLine("Observed Decision", decision.observedDecision));
|
|
211
|
+
lines.push(fieldLine("Decision Agreement", decision.decisionAgreement));
|
|
212
|
+
lines.push(fieldLine("Issue-Code Agreement", decision.issueCodesAgreement));
|
|
213
|
+
lines.push("Observed Issue Codes:");
|
|
214
|
+
pushDashList(lines, decision.observedIssueCodes);
|
|
215
|
+
lines.push(fieldLine("Invalid Ready", bridge.readinessAgreement.invalidReady.invalidReady));
|
|
216
|
+
lines.push(fieldLine("Valid Blocked", bridge.readinessAgreement.validBlocked.validBlocked));
|
|
217
|
+
lines.push(fieldLine("Valid Refresh-Required", bridge.readinessAgreement.validBlocked.validRefreshRequired));
|
|
218
|
+
}
|
|
219
|
+
pushSection(lines, "Criticality Evaluation");
|
|
220
|
+
if (bridge.criticalityEvaluation === null) {
|
|
221
|
+
lines.push("- not supplied");
|
|
222
|
+
}
|
|
223
|
+
else {
|
|
224
|
+
pushRatioMetric(lines, "Mapped Critical Completeness", bridge.criticalityEvaluation.mappedCriticalCompleteness);
|
|
225
|
+
lines.push(`Fully Mapped: ${bridge.criticalityEvaluation.fullyMappedCriticalIds.join(", ") || "none"}`);
|
|
226
|
+
lines.push(`Partially Mapped: ${bridge.criticalityEvaluation.partiallyMappedCriticalIds.join(", ") || "none"}`);
|
|
227
|
+
lines.push(`Unmapped: ${bridge.criticalityEvaluation.unmappedCriticalIds.join(", ") || "none"}`);
|
|
228
|
+
lines.push(`Conflicting: ${bridge.criticalityEvaluation.conflictingCriticalityResponsibilityIds.join(", ") || "none"}`);
|
|
229
|
+
}
|
|
230
|
+
pushSection(lines, "Producer-Readiness Bridge Warnings");
|
|
231
|
+
pushBoundedList(lines, bridge.warnings, (w) => w);
|
|
232
|
+
}
|
|
233
|
+
function renderV043Section(lines, section) {
|
|
234
|
+
if (section === null) {
|
|
235
|
+
lines.push("Not applicable to this plugin.");
|
|
236
|
+
return;
|
|
237
|
+
}
|
|
238
|
+
if (section.summary.strategyCount === 0) {
|
|
239
|
+
lines.push("No v0.4.3 stage-context strategies were selected.");
|
|
240
|
+
return;
|
|
241
|
+
}
|
|
242
|
+
lines.push(fieldLine("Strategy Count", section.summary.strategyCount));
|
|
243
|
+
lines.push(fieldLine("Completed Executions", section.summary.completedExecutionCount));
|
|
244
|
+
lines.push(fieldLine("Invalid-Input Executions", section.summary.invalidInputExecutionCount));
|
|
245
|
+
lines.push(fieldLine("Failed Executions", section.summary.failedExecutionCount));
|
|
246
|
+
lines.push(fieldLine("Completed Evaluations", section.summary.completedEvaluationCount));
|
|
247
|
+
lines.push(fieldLine("Not-Applicable Evaluations", section.summary.notApplicableEvaluationCount));
|
|
248
|
+
lines.push(fieldLine("Failed Evaluations", section.summary.failedEvaluationCount));
|
|
249
|
+
lines.push(fieldLine("Passed Assurance", section.summary.passedAssuranceCount));
|
|
250
|
+
lines.push(fieldLine("Failed Assurance", section.summary.failedAssuranceCount));
|
|
251
|
+
lines.push(fieldLine("Not-Applicable Assurance", section.summary.notApplicableAssuranceCount));
|
|
252
|
+
lines.push(fieldLine("Interpretation Summary", section.interpretation.summary));
|
|
253
|
+
lines.push("Limitations:");
|
|
254
|
+
pushDashList(lines, section.interpretation.limitations);
|
|
255
|
+
for (let index = 0; index < section.strategies.length; index += 1) {
|
|
256
|
+
renderStrategy(lines, section.strategies[index], index);
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
export function renderPluginExperimentReportText(report) {
|
|
260
|
+
const lines = [];
|
|
261
|
+
pushSection(lines, "Plugin Experiment Report");
|
|
262
|
+
pushSection(lines, "Report Metadata");
|
|
263
|
+
lines.push(fieldLine("Generated At", report.metadata.generatedAt));
|
|
264
|
+
lines.push(fieldLine("Run ID", report.metadata.runId));
|
|
265
|
+
lines.push(fieldLine("Started At", report.metadata.startedAt));
|
|
266
|
+
lines.push(fieldLine("Completed At", report.metadata.completedAt));
|
|
267
|
+
lines.push(fieldLine("Status", report.metadata.status));
|
|
268
|
+
lines.push(fieldLine("Output Root", report.metadata.outputRoot));
|
|
269
|
+
pushSection(lines, "Plugin And Target");
|
|
270
|
+
lines.push(fieldLine("Plugin ID", report.plugin.id));
|
|
271
|
+
lines.push(fieldLine("Plugin Name", report.plugin.name));
|
|
272
|
+
lines.push(fieldLine("Plugin Schema Version", report.plugin.schemaVersion));
|
|
273
|
+
lines.push(fieldLine("Mode", report.target.mode));
|
|
274
|
+
lines.push(fieldLine("Tool Root", report.target.toolRoot));
|
|
275
|
+
lines.push(fieldLine("Target Root", report.target.targetRoot));
|
|
276
|
+
pushSection(lines, "Interpretation");
|
|
277
|
+
lines.push(fieldLine("Summary", report.interpretation.summary));
|
|
278
|
+
lines.push(fieldLine("Recommended Next Step", report.interpretation.recommendedNextStep));
|
|
279
|
+
pushSection(lines, "Variants");
|
|
280
|
+
pushDashList(lines, report.variants.map((variant) => `${variant.name} (${variant.id}) completed=${variant.completedOutcomes} partial=${variant.partialOutcomes} failed=${variant.failedOutcomes} skipped=${variant.skippedOutcomes}`));
|
|
281
|
+
pushSection(lines, "Cases");
|
|
282
|
+
pushDashList(lines, report.cases.map((experimentCase) => `${experimentCase.name} (${experimentCase.id}) status=${experimentCase.status} completed=${experimentCase.completedOutcomes} partial=${experimentCase.partialOutcomes} failed=${experimentCase.failedOutcomes} skipped=${experimentCase.skippedOutcomes}`));
|
|
283
|
+
pushSection(lines, "Metrics");
|
|
284
|
+
pushDashList(lines, report.metrics.map((metric) => `${metric.name} (${metric.id}) value=${metric.value ?? "unavailable"} unit=${metric.unit ?? ""}`));
|
|
285
|
+
pushSection(lines, "V0.4.3 Stage-Context Evidence");
|
|
286
|
+
renderV043Section(lines, report.contextStrategyComparisonV043);
|
|
287
|
+
pushSection(lines, "Warnings, Skips, And Failures");
|
|
288
|
+
pushDashList(lines, report.findings.map((finding) => `${finding.severity} ${finding.code} ${finding.message} variant=${finding.variantId ?? ""} case=${finding.caseId ?? ""}`));
|
|
289
|
+
pushSection(lines, "Artifacts");
|
|
290
|
+
pushDashList(lines, report.artifacts.map((artifact) => `${artifact.label} (${artifact.id}) kind=${artifact.kind} path=${artifact.path ?? ""}`));
|
|
291
|
+
return `${lines.join("\n")}\n`;
|
|
292
|
+
}
|
|
@@ -3,6 +3,7 @@ import path from "node:path";
|
|
|
3
3
|
import { resolveWithinRoot } from "../../core/pathSafety.js";
|
|
4
4
|
import { buildPluginExperimentReport } from "./buildPluginExperimentReport.js";
|
|
5
5
|
import { renderPluginExperimentReportHtml } from "./renderPluginExperimentReportHtml.js";
|
|
6
|
+
import { renderPluginExperimentReportText } from "./renderPluginExperimentReportText.js";
|
|
6
7
|
export async function writePluginExperimentReports(args) {
|
|
7
8
|
const rawOutDir = args.outputRoot ?? readString(args.run.metadata?.outputRoot);
|
|
8
9
|
if (!rawOutDir) {
|
|
@@ -13,6 +14,7 @@ export async function writePluginExperimentReports(args) {
|
|
|
13
14
|
outDir,
|
|
14
15
|
jsonPath: resolveWithinRoot(outDir, "report.json"),
|
|
15
16
|
htmlPath: resolveWithinRoot(outDir, "report.html"),
|
|
17
|
+
textPath: resolveWithinRoot(outDir, "report.txt"),
|
|
16
18
|
};
|
|
17
19
|
await mkdir(outputPaths.outDir, { recursive: true });
|
|
18
20
|
const report = buildPluginExperimentReport({
|
|
@@ -23,6 +25,7 @@ export async function writePluginExperimentReports(args) {
|
|
|
23
25
|
});
|
|
24
26
|
await writeFile(outputPaths.jsonPath, `${JSON.stringify({ report, outputPaths }, null, 2)}\n`, "utf8");
|
|
25
27
|
await writeFile(outputPaths.htmlPath, renderPluginExperimentReportHtml(report), "utf8");
|
|
28
|
+
await writeFile(outputPaths.textPath, renderPluginExperimentReportText(report), "utf8");
|
|
26
29
|
return { report, outputPaths };
|
|
27
30
|
}
|
|
28
31
|
function readString(value) {
|