@dailephd/my-dev-kit-lab 0.4.2 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. package/README.md +28 -113
  2. package/benchmarks/contracts/todo-behavior.md +1 -1
  3. package/benchmarks/projects/task-analytics-large-mixed/README.md +2 -0
  4. package/benchmarks/projects/task-workflow-medium-ts/README.md +3 -1
  5. package/dist/src/evaluation/stageContextDeterminism/calculateStageContextDeterminism.js +52 -0
  6. package/dist/src/evaluation/stageContextDeterminism/canonicalizeStageContextRun.js +53 -0
  7. package/dist/src/evaluation/stageContextDeterminism/index.js +2 -0
  8. package/dist/src/evaluation/stageContextDeterminism/types.js +1 -0
  9. package/dist/src/evaluation/stageContextExpectations/index.js +2 -0
  10. package/dist/src/evaluation/stageContextExpectations/producerReadinessExpectationTypes.js +8 -0
  11. package/dist/src/evaluation/stageContextExpectations/readStageContextExpectationFixtureV1.js +50 -0
  12. package/dist/src/evaluation/stageContextExpectations/readTypes.js +1 -0
  13. package/dist/src/evaluation/stageContextExpectations/types.js +1 -0
  14. package/dist/src/evaluation/stageContextExpectations/validation.js +768 -0
  15. package/dist/src/evaluation/stageContextMetrics/calculateAllocationMetrics.js +96 -0
  16. package/dist/src/evaluation/stageContextMetrics/calculateArtifactStateMetrics.js +130 -0
  17. package/dist/src/evaluation/stageContextMetrics/calculateContextSizeMetrics.js +85 -0
  18. package/dist/src/evaluation/stageContextMetrics/calculateCriticalityMetrics.js +112 -0
  19. package/dist/src/evaluation/stageContextMetrics/calculateExpectationMetrics.js +103 -0
  20. package/dist/src/evaluation/stageContextMetrics/calculateOwnerMetrics.js +84 -0
  21. package/dist/src/evaluation/stageContextMetrics/calculateReadinessAgreement.js +152 -0
  22. package/dist/src/evaluation/stageContextMetrics/calculateResponsibilityMappingMetrics.js +36 -0
  23. package/dist/src/evaluation/stageContextMetrics/calculateSupplementalRawAgreement.js +50 -0
  24. package/dist/src/evaluation/stageContextMetrics/calculateTruncationClassification.js +44 -0
  25. package/dist/src/evaluation/stageContextMetrics/collectContextCapsuleEvidence.js +312 -0
  26. package/dist/src/evaluation/stageContextMetrics/collectRetrievalAuditEvidence.js +35 -0
  27. package/dist/src/evaluation/stageContextMetrics/collectWorkflowInstructionEvidence.js +137 -0
  28. package/dist/src/evaluation/stageContextMetrics/evaluateProducerReadinessBridge.js +127 -0
  29. package/dist/src/evaluation/stageContextMetrics/evaluateV043StageContextExecution.js +247 -0
  30. package/dist/src/evaluation/stageContextMetrics/index.js +17 -0
  31. package/dist/src/evaluation/stageContextMetrics/matchStageContextExpectations.js +38 -0
  32. package/dist/src/evaluation/stageContextMetrics/producerReadinessMetricTypes.js +1 -0
  33. package/dist/src/evaluation/stageContextMetrics/targetKeys.js +35 -0
  34. package/dist/src/evaluation/stageContextMetrics/types.js +1 -0
  35. package/dist/src/evaluation/stageContextSelectors/contextArtifactConsistency.js +89 -0
  36. package/dist/src/evaluation/stageContextSelectors/contextCapsuleSelectors.js +126 -0
  37. package/dist/src/evaluation/stageContextSelectors/index.js +4 -0
  38. package/dist/src/evaluation/stageContextSelectors/retrievalAuditRecordSelectors.js +54 -0
  39. package/dist/src/evaluation/stageContextSelectors/workflowInstructionPacketSelectors.js +51 -0
  40. package/dist/src/evaluation/targetImmutability/captureTargetSnapshot.js +208 -0
  41. package/dist/src/evaluation/targetImmutability/compareTargetSnapshots.js +56 -0
  42. package/dist/src/evaluation/targetImmutability/index.js +3 -0
  43. package/dist/src/evaluation/targetImmutability/types.js +1 -0
  44. package/dist/src/evaluation/targetImmutability/validateTargetImmutabilityConfig.js +130 -0
  45. package/dist/src/evaluation/upstreamArtifacts/artifactReadTypes.js +1 -0
  46. package/dist/src/evaluation/upstreamArtifacts/index.js +9 -0
  47. package/dist/src/evaluation/upstreamArtifacts/jsonTypes.js +1 -0
  48. package/dist/src/evaluation/upstreamArtifacts/myDevKitClassificationTypesV1.js +1 -0
  49. package/dist/src/evaluation/upstreamArtifacts/myDevKitContextArtifactsV1.js +1 -0
  50. package/dist/src/evaluation/upstreamArtifacts/myDevKitSemanticTypesV1.js +1 -0
  51. package/dist/src/evaluation/upstreamArtifacts/orchestratorContextReadinessResultV1.js +247 -0
  52. package/dist/src/evaluation/upstreamArtifacts/orchestratorWorkflowInstructionPacketV1.js +1 -0
  53. package/dist/src/evaluation/upstreamArtifacts/readImplementationContextPacketV1.js +9 -0
  54. package/dist/src/evaluation/upstreamArtifacts/readImplementationContextRetrievalReportV1.js +9 -0
  55. package/dist/src/evaluation/upstreamArtifacts/readJsonArtifact.js +53 -0
  56. package/dist/src/evaluation/upstreamArtifacts/readMyDevKitContextCapsuleV1.js +9 -0
  57. package/dist/src/evaluation/upstreamArtifacts/readMyDevKitRetrievalAuditRecordV1.js +9 -0
  58. package/dist/src/evaluation/upstreamArtifacts/readOrchestratorWorkflowInstructionPacketV1.js +9 -0
  59. package/dist/src/evaluation/upstreamArtifacts/readSupplementalContextTextFile.js +29 -0
  60. package/dist/src/evaluation/upstreamArtifacts/readTestContextPacketV1.js +9 -0
  61. package/dist/src/evaluation/upstreamArtifacts/readTestContextRetrievalReportV1.js +9 -0
  62. package/dist/src/evaluation/upstreamArtifacts/runtimeValidation.js +131 -0
  63. package/dist/src/evaluation/upstreamArtifacts/schemaVersion.js +32 -0
  64. package/dist/src/evaluation/upstreamArtifacts/supplementalContextTextParser.js +75 -0
  65. package/dist/src/evaluation/upstreamArtifacts/supplementalContextTypes.js +152 -0
  66. package/dist/src/evaluation/upstreamArtifacts/validateMyDevKitContextCapsuleV1.js +1208 -0
  67. package/dist/src/evaluation/upstreamArtifacts/validateMyDevKitRetrievalAuditRecordV1.js +112 -0
  68. package/dist/src/evaluation/upstreamArtifacts/validateOrchestratorWorkflowInstructionPacketV1.js +235 -0
  69. package/dist/src/evaluation/upstreamArtifacts/validateSupplementalContextDocumentV1.js +148 -0
  70. package/dist/src/experiments/plugins/contextStrategyComparison/executeV043StageContextStrategy.js +46 -0
  71. package/dist/src/experiments/plugins/contextStrategyComparison/index.js +9 -0
  72. package/dist/src/experiments/plugins/contextStrategyComparison/loadV043StrategyArtifacts.js +397 -0
  73. package/dist/src/experiments/plugins/contextStrategyComparison/plugin.js +39 -1
  74. package/dist/src/experiments/plugins/contextStrategyComparison/readV043FullWorkflowLibraryFixture.js +162 -0
  75. package/dist/src/experiments/plugins/contextStrategyComparison/resolveV043StrategyInputs.js +43 -0
  76. package/dist/src/experiments/plugins/contextStrategyComparison/resultMapping.js +3 -0
  77. package/dist/src/experiments/plugins/contextStrategyComparison/runV043StageContextStrategyWithAssurance.js +205 -0
  78. package/dist/src/experiments/plugins/contextStrategyComparison/v043FullWorkflowLibraryFixture.js +1 -0
  79. package/dist/src/experiments/plugins/contextStrategyComparison/v043RunAssuranceTypes.js +1 -0
  80. package/dist/src/experiments/plugins/contextStrategyComparison/v043StrategyExecutionTypes.js +1 -0
  81. package/dist/src/experiments/plugins/contextStrategyComparison/v043StrategyIds.js +13 -0
  82. package/dist/src/experiments/plugins/contextStrategyComparison/v043StrategyInputContracts.js +1 -0
  83. package/dist/src/experiments/plugins/contextStrategyComparison/validateV043RunAssuranceConfig.js +59 -0
  84. package/dist/src/experiments/plugins/contextStrategyComparison/validateV043StrategyInput.js +266 -0
  85. package/dist/src/report/experiments/buildContextStrategyComparisonV043Report.js +520 -0
  86. package/dist/src/report/experiments/buildPluginExperimentReport.js +21 -6
  87. package/dist/src/report/experiments/contextStrategyComparisonV043ReportModel.js +1 -0
  88. package/dist/src/report/experiments/index.js +4 -0
  89. package/dist/src/report/experiments/renderContextStrategyComparisonV043Html.js +296 -0
  90. package/dist/src/report/experiments/renderPluginExperimentReportHtml.js +3 -0
  91. package/dist/src/report/experiments/renderPluginExperimentReportText.js +292 -0
  92. package/dist/src/report/experiments/writePluginExperimentReports.js +3 -0
  93. package/dist/src/securityValidation/attackScenarios/scenarios/secretLeakageScenario.js +215 -9
  94. package/docs/METRICS.md +84 -10
  95. package/package.json +2 -2
  96. package/tests/fixtures/full-workflow-library/complete-v1.0.0.json +24 -0
  97. package/tests/fixtures/full-workflow-library/future-minor-v1.1.0-additive.json +25 -0
  98. package/tests/fixtures/stage-context-expectations/complete-v1.0.0.json +230 -0
  99. package/tests/fixtures/stage-context-expectations/future-minor-v1.1.0-additive.json +232 -0
  100. package/tests/fixtures/stage-context-expectations/invalid/malformed-json.txt +1 -0
  101. package/tests/fixtures/stage-context-expectations/invalid/non-object-root.json +1 -0
  102. package/tests/fixtures/stage-context-expectations/minimal-v1.0.0.json +18 -0
  103. package/tests/fixtures/upstream-artifacts/invalid/malformed-json.txt +1 -0
  104. package/tests/fixtures/upstream-artifacts/invalid/non-object-root.json +1 -0
  105. package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.2/context-capsule/complete-v1.0.0.json +504 -0
  106. package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.2/context-capsule/future-minor-v1.1.0-additive.json +955 -0
  107. package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.2/retrieval-audit-record/complete-v1.0.0.json +190 -0
  108. package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.2/retrieval-audit-record/future-minor-v1.1.0-additive.json +327 -0
  109. package/tests/fixtures/upstream-artifacts/my-dev-kit-1.10.2-identity/PROVENANCE.txt +13 -0
  110. package/tests/fixtures/upstream-artifacts/my-dev-kit-1.10.2-identity/context-capsule.json +7874 -0
  111. package/tests/fixtures/upstream-artifacts/my-dev-kit-1.10.2-identity/retrieval-audit-record.json +1076 -0
  112. package/tests/fixtures/upstream-artifacts/my-dev-kit-orchestrator/1.2.1/workflow-instruction-packet/complete-v1.0.0.json +160 -0
  113. package/tests/fixtures/upstream-artifacts/my-dev-kit-orchestrator/1.2.1/workflow-instruction-packet/future-minor-v1.1.0-additive.json +186 -0
@@ -0,0 +1,520 @@
1
+ import { V043_REPORT_DETAIL_LIMIT, } from "./contextStrategyComparisonV043ReportModel.js";
2
+ const INTERPRETATION_SUMMARY = "Stage-context strategies are reported independently using direct evidence, explicit availability states, target-immutability evidence, and repeated-run determinism. No composite ranking or winning strategy is calculated.";
3
+ const INTERPRETATION_LIMITATIONS = [
4
+ "The published upstream artifacts do not expose considered-but-unselected reads.",
5
+ "The published upstream artifacts do not expose unnecessary-read evidence.",
6
+ "Estimated token counts use ceil(characterCount / 4) per source and are not provider telemetry.",
7
+ "Unavailable and not-applicable values are distinct from zero.",
8
+ ];
9
+ function boundItems(source) {
10
+ const items = source.slice(0, V043_REPORT_DETAIL_LIMIT);
11
+ return {
12
+ totalCount: source.length,
13
+ displayedCount: items.length,
14
+ omittedCount: source.length - items.length,
15
+ items,
16
+ };
17
+ }
18
+ export function buildContextStrategyComparisonV043Report(run) {
19
+ if (run.pluginId !== "context-strategy-comparison")
20
+ return null;
21
+ const candidate = run;
22
+ const executions = candidate.v043StageContextExecutions;
23
+ const evaluations = candidate.v043StageContextEvaluations;
24
+ const assurances = candidate.v043StageContextRunAssurance;
25
+ if (executions === undefined && evaluations === undefined && assurances === undefined) {
26
+ return null;
27
+ }
28
+ if (!Array.isArray(executions) || !Array.isArray(evaluations) || !Array.isArray(assurances)) {
29
+ throw new Error("Invalid v0.4.3 report source: execution, evaluation, and assurance arrays are inconsistent.");
30
+ }
31
+ if (executions.length !== evaluations.length || executions.length !== assurances.length) {
32
+ throw new Error("Invalid v0.4.3 report source: execution, evaluation, and assurance arrays are inconsistent.");
33
+ }
34
+ for (let index = 0; index < executions.length; index += 1) {
35
+ const ids = [
36
+ executions[index].strategyId,
37
+ evaluations[index].strategyId,
38
+ assurances[index].strategyId,
39
+ ].filter((id) => id !== null && id !== undefined);
40
+ const distinct = new Set(ids);
41
+ if (distinct.size > 1) {
42
+ throw new Error("Invalid v0.4.3 report source: execution, evaluation, and assurance arrays are inconsistent.");
43
+ }
44
+ }
45
+ const strategies = executions.map((execution, index) => buildStrategyReport(execution, evaluations[index], assurances[index]));
46
+ return {
47
+ schemaVersion: "1.0.0",
48
+ detailLimit: V043_REPORT_DETAIL_LIMIT,
49
+ summary: buildSummary(executions, evaluations, assurances),
50
+ strategies,
51
+ interpretation: {
52
+ summary: INTERPRETATION_SUMMARY,
53
+ limitations: [...INTERPRETATION_LIMITATIONS],
54
+ },
55
+ };
56
+ }
57
+ function buildSummary(executions, evaluations, assurances) {
58
+ return {
59
+ strategyCount: executions.length,
60
+ completedExecutionCount: executions.filter((e) => e.status === "completed").length,
61
+ invalidInputExecutionCount: executions.filter((e) => e.status === "invalid-input").length,
62
+ failedExecutionCount: executions.filter((e) => e.status === "failed").length,
63
+ completedEvaluationCount: evaluations.filter((e) => e.status === "completed").length,
64
+ notApplicableEvaluationCount: evaluations.filter((e) => e.status === "not-applicable").length,
65
+ failedEvaluationCount: evaluations.filter((e) => e.status === "failed").length,
66
+ passedAssuranceCount: assurances.filter((a) => a.status === "passed").length,
67
+ failedAssuranceCount: assurances.filter((a) => a.status === "failed").length,
68
+ notApplicableAssuranceCount: assurances.filter((a) => a.status === "not-applicable").length,
69
+ };
70
+ }
71
+ function buildStrategyReport(execution, evaluation, assurance) {
72
+ const producerReadinessBridge = buildProducerReadinessBridgeReport(assurance.primaryProducerReadinessBridge);
73
+ return {
74
+ strategyId: assurance.strategyId,
75
+ artifacts: buildArtifactMetadata(execution),
76
+ execution: buildExecutionSummary(execution),
77
+ evaluation: buildEvaluationSummary(evaluation),
78
+ assurance: buildAssuranceSummary(assurance),
79
+ ...(producerReadinessBridge ? { producerReadinessBridge } : {}),
80
+ };
81
+ }
82
+ function buildAllocationFactList(source) {
83
+ return boundItems(source.map((f) => ({ groupId: f.groupId, value: f.value, unit: f.unit, availability: f.availability, reason: f.reason })));
84
+ }
85
+ function buildOwnerEvaluation(owner) {
86
+ return {
87
+ selectedOwnerEvidence: boundItems(owner.selectedOwnerEvidence.map((e) => ({ ownerId: e.ownerId, sourceArtifact: e.sourceArtifact, sourceInstance: e.sourceInstance }))),
88
+ expectedOwnerPresent: buildRatioMetric(owner.expectedOwnerPresent),
89
+ forbiddenOwnerPresent: buildRatioMetric(owner.forbiddenOwnerPresent),
90
+ falsePositiveCount: buildCountMetric(owner.falsePositiveCount),
91
+ falseNegativeCount: buildCountMetric(owner.falseNegativeCount),
92
+ };
93
+ }
94
+ function buildTruncationList(source) {
95
+ return boundItems(source.map((t) => ({
96
+ groupId: t.groupId,
97
+ cause: t.cause,
98
+ availability: t.availability,
99
+ reason: t.reason,
100
+ omittedRequiredEvidenceIds: t.omittedRequiredEvidenceIds,
101
+ limit: t.limit,
102
+ used: t.used,
103
+ available: t.available,
104
+ })));
105
+ }
106
+ function buildSupplementalRawAgreement(agreement) {
107
+ if (agreement === null)
108
+ return null;
109
+ return {
110
+ fields: agreement.fields.map((f) => ({ field: f.field, rawValue: f.rawValue, supplementalValue: f.supplementalValue, agreement: f.agreement, availability: f.availability, reason: f.reason })),
111
+ contradictions: agreement.contradictions.map((f) => ({ field: f.field, rawValue: f.rawValue, supplementalValue: f.supplementalValue, agreement: f.agreement, availability: f.availability, reason: f.reason })),
112
+ upstreamProducerParityPreserved: agreement.upstreamProducerParityPreserved,
113
+ };
114
+ }
115
+ function buildProducerReadinessSide(side) {
116
+ return {
117
+ role: side.role,
118
+ ownerEvaluation: buildOwnerEvaluation(side.ownerEvaluation),
119
+ allocationEvaluation: {
120
+ requiredGroupCapacity: buildAllocationFactList(side.allocationEvaluation.requiredGroupCapacity),
121
+ usedReservation: buildAllocationFactList(side.allocationEvaluation.usedReservation),
122
+ borrowedCapacity: buildAllocationFactList(side.allocationEvaluation.borrowedCapacity),
123
+ unusedCapacity: buildAllocationFactList(side.allocationEvaluation.unusedCapacity),
124
+ },
125
+ truncationEvaluation: buildTruncationList(side.truncationEvaluation),
126
+ packetAgreement: buildSupplementalRawAgreement(side.packetAgreement),
127
+ reportAgreement: buildSupplementalRawAgreement(side.reportAgreement),
128
+ };
129
+ }
130
+ function buildReadinessAgreement(agreement) {
131
+ return {
132
+ structuralAgreement: agreement.structuralAgreement.map((s) => ({
133
+ field: s.field,
134
+ expectedValue: s.expectedValue,
135
+ observedValue: s.observedValue,
136
+ agreement: s.agreement,
137
+ availability: s.availability,
138
+ reason: s.reason,
139
+ })),
140
+ decisionAgreement: {
141
+ availability: agreement.decisionAgreement.availability,
142
+ observedDecision: agreement.decisionAgreement.observedDecision,
143
+ allowedDecisions: agreement.decisionAgreement.allowedDecisions,
144
+ decisionAgreement: agreement.decisionAgreement.decisionAgreement,
145
+ observedIssueCodes: agreement.decisionAgreement.observedIssueCodes,
146
+ expectedIssueCodes: agreement.decisionAgreement.expectedIssueCodes,
147
+ issueCodesAgreement: agreement.decisionAgreement.issueCodesAgreement,
148
+ observedPrimaryIssueCode: agreement.decisionAgreement.observedPrimaryIssueCode,
149
+ expectedPrimaryIssueCode: agreement.decisionAgreement.expectedPrimaryIssueCode,
150
+ primaryIssueAgreement: agreement.decisionAgreement.primaryIssueAgreement,
151
+ reason: agreement.decisionAgreement.reason,
152
+ },
153
+ invalidReady: {
154
+ availability: agreement.invalidReady.availability,
155
+ invalidReady: agreement.invalidReady.invalidReady,
156
+ missingExpectedIssueCodes: agreement.invalidReady.missingExpectedIssueCodes,
157
+ reason: agreement.invalidReady.reason,
158
+ },
159
+ validBlocked: {
160
+ availability: agreement.validBlocked.availability,
161
+ validBlocked: agreement.validBlocked.validBlocked,
162
+ validRefreshRequired: agreement.validBlocked.validRefreshRequired,
163
+ observedDecision: agreement.validBlocked.observedDecision,
164
+ reason: agreement.validBlocked.reason,
165
+ },
166
+ };
167
+ }
168
+ function buildCriticalityEvaluation(criticality) {
169
+ return {
170
+ overlayAgreement: boundItems(criticality.overlayAgreement.map((e) => ({
171
+ responsibilityId: e.responsibilityId,
172
+ expectedCriticality: e.expectedCriticality,
173
+ observedCriticality: e.observedCriticality,
174
+ agreement: e.agreement,
175
+ availability: e.availability,
176
+ reason: e.reason,
177
+ }))),
178
+ missingCriticalityResponsibilityIds: criticality.missingCriticalityResponsibilityIds,
179
+ conflictingCriticalityResponsibilityIds: criticality.conflictingCriticalityResponsibilityIds,
180
+ unexpectedCriticalityResponsibilityIds: criticality.unexpectedCriticalityResponsibilityIds,
181
+ mappedCriticalCompleteness: buildRatioMetric(criticality.mappedCriticalCompleteness),
182
+ fullyMappedCriticalIds: criticality.fullyMappedCriticalIds,
183
+ partiallyMappedCriticalIds: criticality.partiallyMappedCriticalIds,
184
+ unmappedCriticalIds: criticality.unmappedCriticalIds,
185
+ missingMappingEvidenceCriticalIds: criticality.missingMappingEvidenceCriticalIds,
186
+ };
187
+ }
188
+ function buildProducerReadinessBridgeReport(bridge) {
189
+ if (bridge === null || bridge === undefined)
190
+ return null;
191
+ return {
192
+ status: bridge.status,
193
+ reason: bridge.reason,
194
+ implementation: bridge.implementation ? buildProducerReadinessSide(bridge.implementation) : null,
195
+ testImplementation: bridge.testImplementation ? buildProducerReadinessSide(bridge.testImplementation) : null,
196
+ readinessAgreement: bridge.readinessAgreement ? buildReadinessAgreement(bridge.readinessAgreement) : null,
197
+ criticalityEvaluation: bridge.criticalityEvaluation ? buildCriticalityEvaluation(bridge.criticalityEvaluation) : null,
198
+ warnings: boundItems(bridge.warnings),
199
+ evidenceReferences: boundItems(bridge.evidenceReferences),
200
+ };
201
+ }
202
+ function buildExecutionSummary(execution) {
203
+ if (execution.status === "completed") {
204
+ return { status: "completed", issues: boundItems([]) };
205
+ }
206
+ const issues = execution.issues.map(buildIssueFromExecutionIssue);
207
+ return { status: execution.status, issues: boundItems(issues) };
208
+ }
209
+ function buildIssueFromExecutionIssue(issue) {
210
+ return {
211
+ code: issue.code,
212
+ fieldPath: issue.fieldPath ?? null,
213
+ message: issue.message,
214
+ runNumber: null,
215
+ };
216
+ }
217
+ function buildArtifactMetadata(execution) {
218
+ if (execution.status !== "completed") {
219
+ return boundItems([]);
220
+ }
221
+ const entries = [];
222
+ entries.push({
223
+ sourceInstance: "expectations",
224
+ artifactKind: "expectation-fixture",
225
+ sourcePath: execution.expectationsSourcePath,
226
+ schemaVersion: execution.expectations.schemaVersion,
227
+ generatedAt: null,
228
+ producerName: null,
229
+ producerVersion: null,
230
+ role: null,
231
+ workflowId: null,
232
+ stageId: null,
233
+ catalogSchemaVersion: null,
234
+ catalogVersion: null,
235
+ fixtureId: null,
236
+ caseId: execution.expectations.caseId,
237
+ title: execution.expectations.title,
238
+ });
239
+ const payload = execution.payload;
240
+ if ("contextArtifacts" in payload) {
241
+ const contextArtifacts = payload.contextArtifacts;
242
+ contextArtifacts.forEach((pair, index) => {
243
+ entries.push(...buildContextArtifactPairMetadata(pair, `contextArtifacts[${index}]`));
244
+ });
245
+ entries.push(buildWorkflowInstructionPacketMetadata(payload.workflowInstructionPacketSourcePath, payload.workflowInstructionPacket));
246
+ }
247
+ else if ("fullWorkflowLibrary" in payload) {
248
+ entries.push(buildFullWorkflowLibraryMetadata(payload.fullWorkflowLibrarySourcePath, payload.fullWorkflowLibrary));
249
+ }
250
+ else if ("workflowInstructionPacket" in payload) {
251
+ entries.push(buildWorkflowInstructionPacketMetadata(payload.workflowInstructionPacketSourcePath, payload.workflowInstructionPacket));
252
+ }
253
+ else if ("testImplementation" in payload) {
254
+ entries.push(...buildContextArtifactPairMetadata(payload.architecture, "architecture"));
255
+ entries.push(...buildContextArtifactPairMetadata(payload.implementation, "implementation"));
256
+ entries.push(...buildContextArtifactPairMetadata(payload.testImplementation, "testImplementation"));
257
+ }
258
+ else if ("implementation" in payload) {
259
+ entries.push(...buildContextArtifactPairMetadata(payload.architecture, "architecture"));
260
+ entries.push(...buildContextArtifactPairMetadata(payload.implementation, "implementation"));
261
+ }
262
+ else if ("architecture" in payload) {
263
+ entries.push(...buildContextArtifactPairMetadata(payload.architecture, "architecture"));
264
+ }
265
+ return boundItems(entries);
266
+ }
267
+ function buildContextArtifactPairMetadata(pair, sourceInstancePrefix) {
268
+ const entries = [
269
+ {
270
+ sourceInstance: `${sourceInstancePrefix}.contextCapsule`,
271
+ artifactKind: "context-capsule",
272
+ sourcePath: pair.contextCapsuleSourcePath,
273
+ schemaVersion: pair.contextCapsule.schemaVersion,
274
+ generatedAt: pair.contextCapsule.generatedAt,
275
+ producerName: pair.contextCapsule.tool.name,
276
+ producerVersion: pair.contextCapsule.tool.version,
277
+ role: pair.contextCapsule.request.role,
278
+ workflowId: null,
279
+ stageId: null,
280
+ catalogSchemaVersion: null,
281
+ catalogVersion: null,
282
+ fixtureId: null,
283
+ caseId: null,
284
+ title: null,
285
+ },
286
+ ];
287
+ if (pair.retrievalAuditRecord) {
288
+ entries.push({
289
+ sourceInstance: `${sourceInstancePrefix}.retrievalAuditRecord`,
290
+ artifactKind: "retrieval-audit-record",
291
+ sourcePath: pair.retrievalAuditRecordSourcePath ?? "",
292
+ schemaVersion: pair.retrievalAuditRecord.schemaVersion,
293
+ generatedAt: pair.retrievalAuditRecord.generatedAt,
294
+ producerName: pair.retrievalAuditRecord.tool.name,
295
+ producerVersion: pair.retrievalAuditRecord.tool.version,
296
+ role: pair.retrievalAuditRecord.request.role,
297
+ workflowId: null,
298
+ stageId: null,
299
+ catalogSchemaVersion: null,
300
+ catalogVersion: null,
301
+ fixtureId: null,
302
+ caseId: null,
303
+ title: null,
304
+ });
305
+ }
306
+ return entries;
307
+ }
308
+ function buildWorkflowInstructionPacketMetadata(sourcePath, packet) {
309
+ return {
310
+ sourceInstance: "workflowInstructionPacket",
311
+ artifactKind: "workflow-instruction-packet",
312
+ sourcePath,
313
+ schemaVersion: packet.schemaVersion,
314
+ generatedAt: null,
315
+ producerName: null,
316
+ producerVersion: null,
317
+ role: null,
318
+ workflowId: packet.workflowId,
319
+ stageId: packet.stageId,
320
+ catalogSchemaVersion: packet.catalogSchemaVersion,
321
+ catalogVersion: packet.catalogVersion,
322
+ fixtureId: null,
323
+ caseId: null,
324
+ title: null,
325
+ };
326
+ }
327
+ function buildFullWorkflowLibraryMetadata(sourcePath, library) {
328
+ return {
329
+ sourceInstance: "fullWorkflowLibrary",
330
+ artifactKind: "full-workflow-library",
331
+ sourcePath,
332
+ schemaVersion: library.schemaVersion,
333
+ generatedAt: null,
334
+ producerName: null,
335
+ producerVersion: null,
336
+ role: null,
337
+ workflowId: null,
338
+ stageId: null,
339
+ catalogSchemaVersion: null,
340
+ catalogVersion: null,
341
+ fixtureId: library.fixtureId,
342
+ caseId: null,
343
+ title: library.title,
344
+ };
345
+ }
346
+ function buildEvaluationSummary(evaluation) {
347
+ if (evaluation.status !== "completed") {
348
+ return {
349
+ status: evaluation.status,
350
+ reason: evaluation.reason,
351
+ warnings: boundItems([]),
352
+ expectationMatches: boundItems([]),
353
+ observedEvidence: boundItems([]),
354
+ metrics: null,
355
+ };
356
+ }
357
+ return {
358
+ status: "completed",
359
+ reason: null,
360
+ warnings: boundItems(evaluation.warnings),
361
+ expectationMatches: boundItems(evaluation.expectationMatches.map(buildExpectationMatch)),
362
+ observedEvidence: boundItems(evaluation.observedEvidence.map(buildObservedEvidence)),
363
+ metrics: buildMetrics(evaluation.metrics),
364
+ };
365
+ }
366
+ function buildExpectationMatch(match) {
367
+ return {
368
+ expectationId: match.expectationId,
369
+ inclusion: match.inclusion,
370
+ sourceArtifact: match.sourceArtifact,
371
+ category: match.category,
372
+ targetKey: match.targetKey,
373
+ outcome: match.outcome,
374
+ matchedSourceInstances: match.matchedSourceInstances,
375
+ matchedSourceFieldPaths: match.matchedSourceFieldPaths,
376
+ };
377
+ }
378
+ function buildObservedEvidence(evidence) {
379
+ return {
380
+ sourceArtifact: evidence.sourceArtifact,
381
+ sourceInstance: evidence.sourceInstance,
382
+ category: evidence.category,
383
+ targetKey: evidence.targetKey,
384
+ sourceFieldPath: evidence.sourceFieldPath,
385
+ };
386
+ }
387
+ function buildMetrics(metrics) {
388
+ return {
389
+ requiredEvidenceRecall: buildRatioMetric(metrics.requiredEvidenceRecall),
390
+ allowedEvidenceCoverage: buildRatioMetric(metrics.allowedEvidenceCoverage),
391
+ forbiddenEvidenceInclusion: buildRatioMetric(metrics.forbiddenEvidenceInclusion),
392
+ irrelevantFileInclusion: buildCountMetric(metrics.irrelevantFileInclusion),
393
+ irrelevantInstructionInclusion: buildCountMetric(metrics.irrelevantInstructionInclusion),
394
+ requiredProvenanceRecall: buildRatioMetric(metrics.requiredProvenanceRecall),
395
+ responsibilityMappingCompleteness: boundItems(metrics.responsibilityMappingCompleteness.map(buildResponsibilityMapping)),
396
+ stateComparisons: boundItems(metrics.stateComparisons.map(buildStateComparison)),
397
+ contextSize: buildContextSize(metrics.contextSize),
398
+ consideredButUnselectedReads: buildCountMetric(metrics.consideredButUnselectedReads),
399
+ unnecessaryReads: buildCountMetric(metrics.unnecessaryReads),
400
+ targetImmutability: buildCountMetric(metrics.targetImmutability),
401
+ };
402
+ }
403
+ function buildRatioMetric(metric) {
404
+ return {
405
+ availability: metric.availability,
406
+ numerator: metric.numerator,
407
+ denominator: metric.denominator,
408
+ rate: metric.rate,
409
+ matchedExpectationIds: boundItems(metric.matchedExpectationIds),
410
+ missingExpectationIds: boundItems(metric.missingExpectationIds),
411
+ reason: metric.reason,
412
+ };
413
+ }
414
+ function buildCountMetric(metric) {
415
+ return {
416
+ availability: metric.availability,
417
+ count: metric.count,
418
+ evidenceKeys: boundItems(metric.evidenceKeys),
419
+ reason: metric.reason,
420
+ };
421
+ }
422
+ function buildResponsibilityMapping(mapping) {
423
+ return {
424
+ sourceArtifact: mapping.sourceArtifact,
425
+ sourceInstance: mapping.sourceInstance,
426
+ requested: mapping.requested,
427
+ operational: mapping.operational,
428
+ mappedCount: mapping.mappedCount,
429
+ partiallyMappedCount: mapping.partiallyMappedCount,
430
+ unmappedCount: mapping.unmappedCount,
431
+ notApplicableCount: mapping.notApplicableCount,
432
+ denominator: mapping.denominator,
433
+ mappedRate: mapping.mappedRate,
434
+ };
435
+ }
436
+ function buildStateComparison(comparison) {
437
+ return {
438
+ sourceArtifact: comparison.sourceArtifact,
439
+ sourceInstance: comparison.sourceInstance,
440
+ expectationFieldPath: comparison.expectationFieldPath,
441
+ artifactFieldPath: comparison.artifactFieldPath,
442
+ availability: comparison.availability,
443
+ expected: comparison.expected,
444
+ actual: comparison.actual,
445
+ matched: comparison.matched,
446
+ reason: comparison.reason,
447
+ };
448
+ }
449
+ function buildContextSize(size) {
450
+ return {
451
+ sources: boundItems(size.sources.map((source) => ({
452
+ sourceInstance: source.sourceInstance,
453
+ sourceKind: source.sourceKind,
454
+ characterCount: source.characterCount,
455
+ estimatedTokenCount: source.estimatedTokenCount,
456
+ }))),
457
+ totalCharacterCount: size.totalCharacterCount,
458
+ totalEstimatedTokenCount: size.totalEstimatedTokenCount,
459
+ tokenEstimateFormula: "ceil(characterCount / 4) per source",
460
+ };
461
+ }
462
+ function buildAssuranceSummary(assurance) {
463
+ return {
464
+ status: assurance.status,
465
+ repeatCount: assurance.repeatCount,
466
+ runRecords: boundItems(assurance.runRecords.map(buildRunRecord)),
467
+ determinism: buildDeterminism(assurance.determinism),
468
+ issues: boundItems(assurance.issues.map((issue) => ({
469
+ code: issue.code,
470
+ fieldPath: issue.fieldPath,
471
+ message: issue.message,
472
+ runNumber: issue.runNumber,
473
+ }))),
474
+ };
475
+ }
476
+ function buildRunRecord(record) {
477
+ const targetImmutability = record.targetImmutability;
478
+ if (targetImmutability.availability === "available") {
479
+ return {
480
+ runNumber: record.runNumber,
481
+ executionStatus: record.executionStatus,
482
+ evaluationStatus: record.evaluationStatus,
483
+ targetImmutabilityAvailability: "available",
484
+ targetImmutabilityStatus: targetImmutability.comparison.status,
485
+ newMutationCount: targetImmutability.comparison.newMutationCount,
486
+ targetImmutabilityReason: null,
487
+ mutations: boundItems(targetImmutability.comparison.mutations.map(buildTargetMutation)),
488
+ };
489
+ }
490
+ return {
491
+ runNumber: record.runNumber,
492
+ executionStatus: record.executionStatus,
493
+ evaluationStatus: record.evaluationStatus,
494
+ targetImmutabilityAvailability: "unavailable",
495
+ targetImmutabilityStatus: null,
496
+ newMutationCount: null,
497
+ targetImmutabilityReason: targetImmutability.reason,
498
+ mutations: boundItems([]),
499
+ };
500
+ }
501
+ function buildTargetMutation(mutation) {
502
+ return {
503
+ id: mutation.id,
504
+ kind: mutation.kind,
505
+ fieldPath: mutation.fieldPath,
506
+ before: mutation.before,
507
+ after: mutation.after,
508
+ };
509
+ }
510
+ function buildDeterminism(determinism) {
511
+ return {
512
+ availability: determinism.availability,
513
+ repeatCount: determinism.repeatCount,
514
+ deterministic: determinism.deterministic,
515
+ baselineSha256: determinism.baselineSha256,
516
+ runDigests: boundItems(determinism.runDigests.map((digest) => ({ runNumber: digest.runNumber, sha256: digest.sha256 }))),
517
+ mismatchRunNumbers: [...determinism.mismatchRunNumbers],
518
+ reason: determinism.reason,
519
+ };
520
+ }
@@ -1,7 +1,18 @@
1
1
  import path from "node:path";
2
+ import { buildContextStrategyComparisonV043Report } from "./buildContextStrategyComparisonV043Report.js";
3
+ const V043_BULK_ARRAY_KEYS = [
4
+ "v043StageContextExecutions",
5
+ "v043StageContextEvaluations",
6
+ "v043StageContextRunAssurance",
7
+ ];
2
8
  export function buildPluginExperimentReport(args) {
3
9
  const outputRoot = args.outputRoot ?? readString(args.run.metadata?.outputRoot) ?? null;
4
10
  const allOutcomes = args.run.cases.flatMap((experimentCase) => experimentCase.outcomes);
11
+ const contextStrategyComparisonV043 = buildContextStrategyComparisonV043Report(args.run);
12
+ const rawRun = { ...args.run, artifacts: relativizeArtifacts(args.run.artifacts, outputRoot) };
13
+ for (const key of V043_BULK_ARRAY_KEYS) {
14
+ delete rawRun[key];
15
+ }
5
16
  return {
6
17
  metadata: {
7
18
  generatedAt: args.generatedAt ?? new Date().toISOString(),
@@ -25,11 +36,9 @@ export function buildPluginExperimentReport(args) {
25
36
  failures: args.run.failures,
26
37
  skippedOutcomes: allOutcomes.filter((outcome) => outcome.status === "skipped"),
27
38
  findings: buildFindings(args.run),
28
- interpretation: buildInterpretation(args.run),
29
- rawRun: {
30
- ...args.run,
31
- artifacts: relativizeArtifacts(args.run.artifacts, outputRoot),
32
- },
39
+ contextStrategyComparisonV043,
40
+ interpretation: buildInterpretation(args.run, contextStrategyComparisonV043),
41
+ rawRun,
33
42
  };
34
43
  }
35
44
  function buildVariantSummary(variant, outcomes) {
@@ -85,7 +94,13 @@ function buildFindings(run) {
85
94
  ...skipped,
86
95
  ];
87
96
  }
88
- function buildInterpretation(run) {
97
+ function buildInterpretation(run, contextStrategyComparisonV043) {
98
+ if (run.pluginId === "context-strategy-comparison" && (contextStrategyComparisonV043?.summary.strategyCount ?? 0) > 0) {
99
+ return {
100
+ summary: `Stage-context evidence was recorded for ${contextStrategyComparisonV043.summary.strategyCount} strategy executions. Review each strategy independently; this report does not calculate a composite ranking or winning strategy.`,
101
+ recommendedNextStep: "Review required evidence, irrelevant inclusion, state comparisons, target immutability, determinism, and explicit unavailable or not-applicable metrics for each strategy.",
102
+ };
103
+ }
89
104
  if (run.pluginId === "context-strategy-comparison") {
90
105
  const tokenSavings = metricNumber(run.metrics, "average-token-savings-percent");
91
106
  const correctnessDelta = metricNumber(run.metrics, "average-correctness-delta");
@@ -0,0 +1 @@
1
+ export const V043_REPORT_DETAIL_LIMIT = 100;
@@ -2,3 +2,7 @@ export * from "./buildPluginExperimentReport.js";
2
2
  export * from "./experimentReportModel.js";
3
3
  export * from "./renderPluginExperimentReportHtml.js";
4
4
  export * from "./writePluginExperimentReports.js";
5
+ export * from "./contextStrategyComparisonV043ReportModel.js";
6
+ export { buildContextStrategyComparisonV043Report } from "./buildContextStrategyComparisonV043Report.js";
7
+ export { renderContextStrategyComparisonV043Html } from "./renderContextStrategyComparisonV043Html.js";
8
+ export { renderPluginExperimentReportText } from "./renderPluginExperimentReportText.js";