@dailephd/my-dev-kit-lab 0.4.2 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. package/README.md +28 -113
  2. package/benchmarks/contracts/todo-behavior.md +1 -1
  3. package/benchmarks/projects/task-analytics-large-mixed/README.md +2 -0
  4. package/benchmarks/projects/task-workflow-medium-ts/README.md +3 -1
  5. package/dist/src/evaluation/stageContextDeterminism/calculateStageContextDeterminism.js +52 -0
  6. package/dist/src/evaluation/stageContextDeterminism/canonicalizeStageContextRun.js +53 -0
  7. package/dist/src/evaluation/stageContextDeterminism/index.js +2 -0
  8. package/dist/src/evaluation/stageContextDeterminism/types.js +1 -0
  9. package/dist/src/evaluation/stageContextExpectations/index.js +2 -0
  10. package/dist/src/evaluation/stageContextExpectations/producerReadinessExpectationTypes.js +8 -0
  11. package/dist/src/evaluation/stageContextExpectations/readStageContextExpectationFixtureV1.js +50 -0
  12. package/dist/src/evaluation/stageContextExpectations/readTypes.js +1 -0
  13. package/dist/src/evaluation/stageContextExpectations/types.js +1 -0
  14. package/dist/src/evaluation/stageContextExpectations/validation.js +768 -0
  15. package/dist/src/evaluation/stageContextMetrics/calculateAllocationMetrics.js +96 -0
  16. package/dist/src/evaluation/stageContextMetrics/calculateArtifactStateMetrics.js +130 -0
  17. package/dist/src/evaluation/stageContextMetrics/calculateContextSizeMetrics.js +85 -0
  18. package/dist/src/evaluation/stageContextMetrics/calculateCriticalityMetrics.js +112 -0
  19. package/dist/src/evaluation/stageContextMetrics/calculateExpectationMetrics.js +103 -0
  20. package/dist/src/evaluation/stageContextMetrics/calculateOwnerMetrics.js +84 -0
  21. package/dist/src/evaluation/stageContextMetrics/calculateReadinessAgreement.js +152 -0
  22. package/dist/src/evaluation/stageContextMetrics/calculateResponsibilityMappingMetrics.js +36 -0
  23. package/dist/src/evaluation/stageContextMetrics/calculateSupplementalRawAgreement.js +50 -0
  24. package/dist/src/evaluation/stageContextMetrics/calculateTruncationClassification.js +44 -0
  25. package/dist/src/evaluation/stageContextMetrics/collectContextCapsuleEvidence.js +312 -0
  26. package/dist/src/evaluation/stageContextMetrics/collectRetrievalAuditEvidence.js +35 -0
  27. package/dist/src/evaluation/stageContextMetrics/collectWorkflowInstructionEvidence.js +137 -0
  28. package/dist/src/evaluation/stageContextMetrics/evaluateProducerReadinessBridge.js +127 -0
  29. package/dist/src/evaluation/stageContextMetrics/evaluateV043StageContextExecution.js +247 -0
  30. package/dist/src/evaluation/stageContextMetrics/index.js +17 -0
  31. package/dist/src/evaluation/stageContextMetrics/matchStageContextExpectations.js +38 -0
  32. package/dist/src/evaluation/stageContextMetrics/producerReadinessMetricTypes.js +1 -0
  33. package/dist/src/evaluation/stageContextMetrics/targetKeys.js +35 -0
  34. package/dist/src/evaluation/stageContextMetrics/types.js +1 -0
  35. package/dist/src/evaluation/stageContextSelectors/contextArtifactConsistency.js +89 -0
  36. package/dist/src/evaluation/stageContextSelectors/contextCapsuleSelectors.js +126 -0
  37. package/dist/src/evaluation/stageContextSelectors/index.js +4 -0
  38. package/dist/src/evaluation/stageContextSelectors/retrievalAuditRecordSelectors.js +54 -0
  39. package/dist/src/evaluation/stageContextSelectors/workflowInstructionPacketSelectors.js +51 -0
  40. package/dist/src/evaluation/targetImmutability/captureTargetSnapshot.js +208 -0
  41. package/dist/src/evaluation/targetImmutability/compareTargetSnapshots.js +56 -0
  42. package/dist/src/evaluation/targetImmutability/index.js +3 -0
  43. package/dist/src/evaluation/targetImmutability/types.js +1 -0
  44. package/dist/src/evaluation/targetImmutability/validateTargetImmutabilityConfig.js +130 -0
  45. package/dist/src/evaluation/upstreamArtifacts/artifactReadTypes.js +1 -0
  46. package/dist/src/evaluation/upstreamArtifacts/index.js +9 -0
  47. package/dist/src/evaluation/upstreamArtifacts/jsonTypes.js +1 -0
  48. package/dist/src/evaluation/upstreamArtifacts/myDevKitClassificationTypesV1.js +1 -0
  49. package/dist/src/evaluation/upstreamArtifacts/myDevKitContextArtifactsV1.js +1 -0
  50. package/dist/src/evaluation/upstreamArtifacts/myDevKitSemanticTypesV1.js +1 -0
  51. package/dist/src/evaluation/upstreamArtifacts/orchestratorContextReadinessResultV1.js +247 -0
  52. package/dist/src/evaluation/upstreamArtifacts/orchestratorWorkflowInstructionPacketV1.js +1 -0
  53. package/dist/src/evaluation/upstreamArtifacts/readImplementationContextPacketV1.js +9 -0
  54. package/dist/src/evaluation/upstreamArtifacts/readImplementationContextRetrievalReportV1.js +9 -0
  55. package/dist/src/evaluation/upstreamArtifacts/readJsonArtifact.js +53 -0
  56. package/dist/src/evaluation/upstreamArtifacts/readMyDevKitContextCapsuleV1.js +9 -0
  57. package/dist/src/evaluation/upstreamArtifacts/readMyDevKitRetrievalAuditRecordV1.js +9 -0
  58. package/dist/src/evaluation/upstreamArtifacts/readOrchestratorWorkflowInstructionPacketV1.js +9 -0
  59. package/dist/src/evaluation/upstreamArtifacts/readSupplementalContextTextFile.js +29 -0
  60. package/dist/src/evaluation/upstreamArtifacts/readTestContextPacketV1.js +9 -0
  61. package/dist/src/evaluation/upstreamArtifacts/readTestContextRetrievalReportV1.js +9 -0
  62. package/dist/src/evaluation/upstreamArtifacts/runtimeValidation.js +131 -0
  63. package/dist/src/evaluation/upstreamArtifacts/schemaVersion.js +32 -0
  64. package/dist/src/evaluation/upstreamArtifacts/supplementalContextTextParser.js +75 -0
  65. package/dist/src/evaluation/upstreamArtifacts/supplementalContextTypes.js +152 -0
  66. package/dist/src/evaluation/upstreamArtifacts/validateMyDevKitContextCapsuleV1.js +1208 -0
  67. package/dist/src/evaluation/upstreamArtifacts/validateMyDevKitRetrievalAuditRecordV1.js +112 -0
  68. package/dist/src/evaluation/upstreamArtifacts/validateOrchestratorWorkflowInstructionPacketV1.js +235 -0
  69. package/dist/src/evaluation/upstreamArtifacts/validateSupplementalContextDocumentV1.js +148 -0
  70. package/dist/src/experiments/plugins/contextStrategyComparison/executeV043StageContextStrategy.js +46 -0
  71. package/dist/src/experiments/plugins/contextStrategyComparison/index.js +9 -0
  72. package/dist/src/experiments/plugins/contextStrategyComparison/loadV043StrategyArtifacts.js +397 -0
  73. package/dist/src/experiments/plugins/contextStrategyComparison/plugin.js +39 -1
  74. package/dist/src/experiments/plugins/contextStrategyComparison/readV043FullWorkflowLibraryFixture.js +162 -0
  75. package/dist/src/experiments/plugins/contextStrategyComparison/resolveV043StrategyInputs.js +43 -0
  76. package/dist/src/experiments/plugins/contextStrategyComparison/resultMapping.js +3 -0
  77. package/dist/src/experiments/plugins/contextStrategyComparison/runV043StageContextStrategyWithAssurance.js +205 -0
  78. package/dist/src/experiments/plugins/contextStrategyComparison/v043FullWorkflowLibraryFixture.js +1 -0
  79. package/dist/src/experiments/plugins/contextStrategyComparison/v043RunAssuranceTypes.js +1 -0
  80. package/dist/src/experiments/plugins/contextStrategyComparison/v043StrategyExecutionTypes.js +1 -0
  81. package/dist/src/experiments/plugins/contextStrategyComparison/v043StrategyIds.js +13 -0
  82. package/dist/src/experiments/plugins/contextStrategyComparison/v043StrategyInputContracts.js +1 -0
  83. package/dist/src/experiments/plugins/contextStrategyComparison/validateV043RunAssuranceConfig.js +59 -0
  84. package/dist/src/experiments/plugins/contextStrategyComparison/validateV043StrategyInput.js +266 -0
  85. package/dist/src/report/experiments/buildContextStrategyComparisonV043Report.js +520 -0
  86. package/dist/src/report/experiments/buildPluginExperimentReport.js +21 -6
  87. package/dist/src/report/experiments/contextStrategyComparisonV043ReportModel.js +1 -0
  88. package/dist/src/report/experiments/index.js +4 -0
  89. package/dist/src/report/experiments/renderContextStrategyComparisonV043Html.js +296 -0
  90. package/dist/src/report/experiments/renderPluginExperimentReportHtml.js +3 -0
  91. package/dist/src/report/experiments/renderPluginExperimentReportText.js +292 -0
  92. package/dist/src/report/experiments/writePluginExperimentReports.js +3 -0
  93. package/dist/src/securityValidation/attackScenarios/scenarios/secretLeakageScenario.js +215 -9
  94. package/docs/METRICS.md +84 -10
  95. package/package.json +2 -2
  96. package/tests/fixtures/full-workflow-library/complete-v1.0.0.json +24 -0
  97. package/tests/fixtures/full-workflow-library/future-minor-v1.1.0-additive.json +25 -0
  98. package/tests/fixtures/stage-context-expectations/complete-v1.0.0.json +230 -0
  99. package/tests/fixtures/stage-context-expectations/future-minor-v1.1.0-additive.json +232 -0
  100. package/tests/fixtures/stage-context-expectations/invalid/malformed-json.txt +1 -0
  101. package/tests/fixtures/stage-context-expectations/invalid/non-object-root.json +1 -0
  102. package/tests/fixtures/stage-context-expectations/minimal-v1.0.0.json +18 -0
  103. package/tests/fixtures/upstream-artifacts/invalid/malformed-json.txt +1 -0
  104. package/tests/fixtures/upstream-artifacts/invalid/non-object-root.json +1 -0
  105. package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.2/context-capsule/complete-v1.0.0.json +504 -0
  106. package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.2/context-capsule/future-minor-v1.1.0-additive.json +955 -0
  107. package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.2/retrieval-audit-record/complete-v1.0.0.json +190 -0
  108. package/tests/fixtures/upstream-artifacts/my-dev-kit/1.10.2/retrieval-audit-record/future-minor-v1.1.0-additive.json +327 -0
  109. package/tests/fixtures/upstream-artifacts/my-dev-kit-1.10.2-identity/PROVENANCE.txt +13 -0
  110. package/tests/fixtures/upstream-artifacts/my-dev-kit-1.10.2-identity/context-capsule.json +7874 -0
  111. package/tests/fixtures/upstream-artifacts/my-dev-kit-1.10.2-identity/retrieval-audit-record.json +1076 -0
  112. package/tests/fixtures/upstream-artifacts/my-dev-kit-orchestrator/1.2.1/workflow-instruction-packet/complete-v1.0.0.json +160 -0
  113. package/tests/fixtures/upstream-artifacts/my-dev-kit-orchestrator/1.2.1/workflow-instruction-packet/future-minor-v1.1.0-additive.json +186 -0
@@ -0,0 +1,96 @@
1
+ const NO_BORROW_EVIDENCE_REASON = "The frozen my-dev-kit artifacts do not expose cross-group capacity borrowing evidence.";
2
+ export function calculateAllocationFactsForGroups(evidenceGroups, sourceArtifact, sourceInstance) {
3
+ const requiredGroupCapacity = [];
4
+ const usedReservation = [];
5
+ const borrowedCapacity = [];
6
+ const unusedCapacity = [];
7
+ for (const group of evidenceGroups) {
8
+ requiredGroupCapacity.push({
9
+ groupId: group.id,
10
+ sourceArtifact,
11
+ sourceInstance,
12
+ value: group.limit,
13
+ unit: "items",
14
+ availability: group.limit === null ? "unavailable" : "available",
15
+ reason: group.limit === null ? "No hard limit is declared for this evidence group." : null
16
+ });
17
+ usedReservation.push({
18
+ groupId: group.id,
19
+ sourceArtifact,
20
+ sourceInstance,
21
+ value: group.usedCount,
22
+ unit: "items",
23
+ availability: "available",
24
+ reason: null
25
+ });
26
+ borrowedCapacity.push({
27
+ groupId: group.id,
28
+ sourceArtifact,
29
+ sourceInstance,
30
+ value: null,
31
+ unit: "items",
32
+ availability: "unavailable",
33
+ reason: NO_BORROW_EVIDENCE_REASON
34
+ });
35
+ if (group.limit === null) {
36
+ unusedCapacity.push({
37
+ groupId: group.id,
38
+ sourceArtifact,
39
+ sourceInstance,
40
+ value: null,
41
+ unit: "items",
42
+ availability: "unavailable",
43
+ reason: "No hard limit is declared for this evidence group, so unused capacity cannot be computed."
44
+ });
45
+ continue;
46
+ }
47
+ const unused = group.limit - group.usedCount;
48
+ if (unused < 0) {
49
+ unusedCapacity.push({
50
+ groupId: group.id,
51
+ sourceArtifact,
52
+ sourceInstance,
53
+ value: null,
54
+ unit: "items",
55
+ availability: "unavailable",
56
+ reason: `Computed unused capacity was negative (used ${group.usedCount} exceeds limit ${group.limit}); treated as a data inconsistency rather than a valid value.`
57
+ });
58
+ continue;
59
+ }
60
+ unusedCapacity.push({
61
+ groupId: group.id,
62
+ sourceArtifact,
63
+ sourceInstance,
64
+ value: unused,
65
+ unit: "items",
66
+ availability: "available",
67
+ reason: null
68
+ });
69
+ }
70
+ return { requiredGroupCapacity, usedReservation, borrowedCapacity, unusedCapacity };
71
+ }
72
+ export function calculateRequiredEvidenceOmitted(matches, sourceArtifactTruncated) {
73
+ const missing = matches.filter((m) => m.inclusion === "required" && m.outcome === "missing");
74
+ if (missing.length === 0) {
75
+ return {
76
+ metric: { availability: "not-applicable", count: null, evidenceKeys: [], reason: "No required expectation is missing from observed evidence." },
77
+ entries: []
78
+ };
79
+ }
80
+ const entries = missing.map((m) => ({
81
+ evidenceKey: m.targetKey,
82
+ groupId: null,
83
+ truncationReported: sourceArtifactTruncated[m.sourceArtifact] ?? false,
84
+ sourceArtifact: m.sourceArtifact
85
+ }));
86
+ return {
87
+ metric: { availability: "available", count: entries.length, evidenceKeys: entries.map((e) => e.evidenceKey), reason: null },
88
+ entries
89
+ };
90
+ }
91
+ // Not currently consumed by the calculators above, but kept available for callers that
92
+ // want to cross-check allocation expectations directly (bounded, deterministic; the lab
93
+ // still never recalculates the producer's allocation decision).
94
+ export function findAllocationExpectationForGroup(expectations, groupId) {
95
+ return (expectations ?? []).find((e) => e.groupId === groupId);
96
+ }
@@ -0,0 +1,130 @@
1
+ import { isDeepStrictEqual } from "node:util";
2
+ function compareValue(sourceArtifact, sourceInstance, expectationFieldPath, artifactFieldPath, expected, actual, matched) {
3
+ return {
4
+ sourceArtifact,
5
+ sourceInstance,
6
+ expectationFieldPath,
7
+ artifactFieldPath,
8
+ availability: "available",
9
+ expected,
10
+ actual,
11
+ matched,
12
+ reason: null
13
+ };
14
+ }
15
+ function unavailableComparison(sourceArtifact, sourceInstance, expectationFieldPath, expected, reason) {
16
+ return {
17
+ sourceArtifact,
18
+ sourceInstance,
19
+ expectationFieldPath,
20
+ artifactFieldPath: null,
21
+ availability: "unavailable",
22
+ expected,
23
+ actual: null,
24
+ matched: null,
25
+ reason
26
+ };
27
+ }
28
+ export function compareExpectedContextCapsuleState(expected, artifact, sourceInstance) {
29
+ if (expected === undefined)
30
+ return [];
31
+ const comparisons = [];
32
+ const SA = "context-capsule";
33
+ if (expected.contextAdequacyStatus !== undefined) {
34
+ comparisons.push(compareValue(SA, sourceInstance, "contextAdequacyStatus", "contextAdequacy.status", expected.contextAdequacyStatus, artifact.contextAdequacy.status, artifact.contextAdequacy.status === expected.contextAdequacyStatus));
35
+ }
36
+ if (expected.roleAdequacyStatus !== undefined) {
37
+ comparisons.push(compareValue(SA, sourceInstance, "roleAdequacyStatus", "roleAdequacy.status", expected.roleAdequacyStatus, artifact.roleAdequacy.status, artifact.roleAdequacy.status === expected.roleAdequacyStatus));
38
+ }
39
+ if (expected.freshnessState !== undefined) {
40
+ comparisons.push(compareValue(SA, sourceInstance, "freshnessState", "freshness.state", expected.freshnessState, artifact.freshness.state, artifact.freshness.state === expected.freshnessState));
41
+ }
42
+ if (expected.truncated !== undefined) {
43
+ comparisons.push(compareValue(SA, sourceInstance, "truncated", "truncation.truncated", expected.truncated, artifact.truncation.truncated, artifact.truncation.truncated === expected.truncated));
44
+ }
45
+ if (expected.fullFileFallbackUsed !== undefined) {
46
+ comparisons.push(compareValue(SA, sourceInstance, "fullFileFallbackUsed", "fullFileFallback.used", expected.fullFileFallbackUsed, artifact.fullFileFallback.used, artifact.fullFileFallback.used === expected.fullFileFallbackUsed));
47
+ }
48
+ if (expected.unresolvedItemIds !== undefined) {
49
+ comparisons.push(unavailableComparison(SA, sourceInstance, "unresolvedItemIds", expected.unresolvedItemIds, "ContextCapsule unresolvedItems do not expose stable item IDs."));
50
+ }
51
+ if (expected.warningCount !== undefined) {
52
+ comparisons.push(compareValue(SA, sourceInstance, "warningCount", "warnings.length", expected.warningCount, artifact.warnings.length, artifact.warnings.length === expected.warningCount));
53
+ }
54
+ return comparisons;
55
+ }
56
+ export function compareExpectedRetrievalAuditState(expected, artifact, sourceInstance) {
57
+ if (expected === undefined)
58
+ return [];
59
+ const comparisons = [];
60
+ const SA = "retrieval-audit-record";
61
+ if (expected.contextAdequacyStatus !== undefined) {
62
+ comparisons.push(compareValue(SA, sourceInstance, "contextAdequacyStatus", "contextAdequacy.status", expected.contextAdequacyStatus, artifact.contextAdequacy.status, artifact.contextAdequacy.status === expected.contextAdequacyStatus));
63
+ }
64
+ if (expected.roleAdequacyStatus !== undefined) {
65
+ comparisons.push(compareValue(SA, sourceInstance, "roleAdequacyStatus", "roleAdequacy.status", expected.roleAdequacyStatus, artifact.roleAdequacy.status, artifact.roleAdequacy.status === expected.roleAdequacyStatus));
66
+ }
67
+ if (expected.freshnessState !== undefined) {
68
+ comparisons.push(compareValue(SA, sourceInstance, "freshnessState", "freshness.state", expected.freshnessState, artifact.freshness.state, artifact.freshness.state === expected.freshnessState));
69
+ }
70
+ if (expected.truncated !== undefined) {
71
+ comparisons.push(compareValue(SA, sourceInstance, "truncated", "truncation.truncated", expected.truncated, artifact.truncation.truncated, artifact.truncation.truncated === expected.truncated));
72
+ }
73
+ if (expected.fullFileFallbackUsed !== undefined) {
74
+ comparisons.push(compareValue(SA, sourceInstance, "fullFileFallbackUsed", "fullFileFallback.used", expected.fullFileFallbackUsed, artifact.fullFileFallback.used, artifact.fullFileFallback.used === expected.fullFileFallbackUsed));
75
+ }
76
+ if (expected.unresolvedItemIds !== undefined) {
77
+ comparisons.push(unavailableComparison(SA, sourceInstance, "unresolvedItemIds", expected.unresolvedItemIds, "RetrievalAuditRecord does not serialize unresolved item IDs."));
78
+ }
79
+ if (expected.warningCount !== undefined) {
80
+ comparisons.push(compareValue(SA, sourceInstance, "warningCount", "warnings.length", expected.warningCount, artifact.warnings.length, artifact.warnings.length === expected.warningCount));
81
+ }
82
+ return comparisons;
83
+ }
84
+ export function compareExpectedWorkflowInstructionPacketState(expected, artifact, sourceInstance) {
85
+ if (expected === undefined)
86
+ return [];
87
+ const comparisons = [];
88
+ const SA = "workflow-instruction-packet";
89
+ if (expected.adequacyStatus !== undefined) {
90
+ comparisons.push(compareValue(SA, sourceInstance, "adequacyStatus", "adequacy.status", expected.adequacyStatus, artifact.adequacy.status, artifact.adequacy.status === expected.adequacyStatus));
91
+ }
92
+ if (expected.truncated !== undefined) {
93
+ comparisons.push(compareValue(SA, sourceInstance, "truncated", "truncation.truncated", expected.truncated, artifact.truncation.truncated, artifact.truncation.truncated === expected.truncated));
94
+ }
95
+ if (expected.unresolvedReferences !== undefined) {
96
+ comparisons.push(compareValue(SA, sourceInstance, "unresolvedReferences", "unresolvedReferences", expected.unresolvedReferences, artifact.unresolvedReferences, isDeepStrictEqual(artifact.unresolvedReferences, expected.unresolvedReferences)));
97
+ }
98
+ if (expected.warningCount !== undefined) {
99
+ comparisons.push(compareValue(SA, sourceInstance, "warningCount", "warnings.length", expected.warningCount, artifact.warnings.length, artifact.warnings.length === expected.warningCount));
100
+ }
101
+ return comparisons;
102
+ }
103
+ export function compareExpectedTargetImmutabilityState(expected, targetImmutability) {
104
+ if (expected === undefined)
105
+ return [];
106
+ const SA = "target-immutability";
107
+ const expectationFieldPath = "expectedStates.targetImmutability.newMutationCount";
108
+ if (targetImmutability === undefined) {
109
+ return [
110
+ unavailableComparison(SA, "target", expectationFieldPath, expected.newMutationCount, "Target immutability configuration was not supplied for this strategy run.")
111
+ ];
112
+ }
113
+ if (targetImmutability.availability === "unavailable") {
114
+ return [unavailableComparison(SA, "target", expectationFieldPath, expected.newMutationCount, targetImmutability.reason)];
115
+ }
116
+ const comparison = targetImmutability.comparison;
117
+ return [
118
+ {
119
+ sourceArtifact: SA,
120
+ sourceInstance: "target",
121
+ expectationFieldPath,
122
+ artifactFieldPath: "targetImmutability.comparison.newMutationCount",
123
+ availability: "available",
124
+ expected: expected.newMutationCount,
125
+ actual: comparison.newMutationCount,
126
+ matched: expected.newMutationCount === comparison.newMutationCount,
127
+ reason: null
128
+ }
129
+ ];
130
+ }
@@ -0,0 +1,85 @@
1
+ function estimateTokens(characterCount) {
2
+ return Math.ceil(characterCount / 4);
3
+ }
4
+ function pairSources(pair, labelPrefix) {
5
+ const sources = [];
6
+ const capsuleCharacterCount = JSON.stringify(pair.contextCapsule).length;
7
+ sources.push({
8
+ sourceInstance: `${labelPrefix}.contextCapsule`,
9
+ sourceKind: "context-capsule",
10
+ characterCount: capsuleCharacterCount,
11
+ estimatedTokenCount: estimateTokens(capsuleCharacterCount)
12
+ });
13
+ if (pair.retrievalAuditRecord) {
14
+ const auditCharacterCount = JSON.stringify(pair.retrievalAuditRecord).length;
15
+ sources.push({
16
+ sourceInstance: `${labelPrefix}.retrievalAuditRecord`,
17
+ sourceKind: "retrieval-audit-record",
18
+ characterCount: auditCharacterCount,
19
+ estimatedTokenCount: estimateTokens(auditCharacterCount)
20
+ });
21
+ }
22
+ return sources;
23
+ }
24
+ export function calculateV043ExecutionContextSize(execution) {
25
+ const sources = [];
26
+ switch (execution.strategyId) {
27
+ case "architecture-context-only": {
28
+ const payload = execution.payload;
29
+ sources.push(...pairSources(payload.architecture, "architecture"));
30
+ break;
31
+ }
32
+ case "architecture-plus-implementation-refresh": {
33
+ const payload = execution.payload;
34
+ sources.push(...pairSources(payload.architecture, "architecture"));
35
+ sources.push(...pairSources(payload.implementation, "implementation"));
36
+ break;
37
+ }
38
+ case "architecture-plus-implementation-and-test-refresh": {
39
+ const payload = execution.payload;
40
+ sources.push(...pairSources(payload.architecture, "architecture"));
41
+ sources.push(...pairSources(payload.implementation, "implementation"));
42
+ sources.push(...pairSources(payload.testImplementation, "testImplementation"));
43
+ break;
44
+ }
45
+ case "full-workflow-library": {
46
+ const payload = execution.payload;
47
+ const characterCount = payload.fullWorkflowLibrary.rawText.length;
48
+ sources.push({
49
+ sourceInstance: "fullWorkflowLibrary.rawText",
50
+ sourceKind: "full-workflow-library-text",
51
+ characterCount,
52
+ estimatedTokenCount: estimateTokens(characterCount)
53
+ });
54
+ break;
55
+ }
56
+ case "bounded-workflow-instruction-packet": {
57
+ const payload = execution.payload;
58
+ const characterCount = JSON.stringify(payload.workflowInstructionPacket).length;
59
+ sources.push({
60
+ sourceInstance: "workflowInstructionPacket",
61
+ sourceKind: "workflow-instruction-packet",
62
+ characterCount,
63
+ estimatedTokenCount: estimateTokens(characterCount)
64
+ });
65
+ break;
66
+ }
67
+ case "combined-bounded-stage-context": {
68
+ const payload = execution.payload;
69
+ payload.contextArtifacts.forEach((pair, index) => {
70
+ sources.push(...pairSources(pair, `contextArtifacts[${index}]`));
71
+ });
72
+ const characterCount = JSON.stringify(payload.workflowInstructionPacket).length;
73
+ sources.push({
74
+ sourceInstance: "workflowInstructionPacket",
75
+ sourceKind: "workflow-instruction-packet",
76
+ characterCount,
77
+ estimatedTokenCount: estimateTokens(characterCount)
78
+ });
79
+ break;
80
+ }
81
+ }
82
+ const totalCharacterCount = sources.reduce((sum, source) => sum + source.characterCount, 0);
83
+ const totalEstimatedTokenCount = sources.reduce((sum, source) => sum + source.estimatedTokenCount, 0);
84
+ return { availability: "available", sources, totalCharacterCount, totalEstimatedTokenCount };
85
+ }
@@ -0,0 +1,112 @@
1
+ function findMapping(mappings, responsibilityId) {
2
+ return mappings?.find((m) => m.responsibilityId === responsibilityId);
3
+ }
4
+ export function calculateCriticalityMetrics(mappings, expectations) {
5
+ const list = expectations ?? [];
6
+ const overlayAgreement = [];
7
+ const missingCriticalityResponsibilityIds = [];
8
+ const conflictingCriticalityResponsibilityIds = [];
9
+ for (const e of list) {
10
+ if (!e.applicable) {
11
+ overlayAgreement.push({
12
+ responsibilityId: e.responsibilityId,
13
+ expectedCriticality: e.expectedCriticality,
14
+ observedCriticality: findMapping(mappings, e.responsibilityId)?.criticality ?? null,
15
+ agreement: null,
16
+ availability: "not-applicable",
17
+ reason: "Expectation is marked not applicable."
18
+ });
19
+ continue;
20
+ }
21
+ if (mappings === undefined) {
22
+ overlayAgreement.push({
23
+ responsibilityId: e.responsibilityId,
24
+ expectedCriticality: e.expectedCriticality,
25
+ observedCriticality: null,
26
+ agreement: null,
27
+ availability: "unavailable",
28
+ reason: "No raw responsibility-mapping evidence was supplied."
29
+ });
30
+ missingCriticalityResponsibilityIds.push(e.responsibilityId);
31
+ continue;
32
+ }
33
+ const mapping = findMapping(mappings, e.responsibilityId);
34
+ if (mapping === undefined) {
35
+ overlayAgreement.push({
36
+ responsibilityId: e.responsibilityId,
37
+ expectedCriticality: e.expectedCriticality,
38
+ observedCriticality: null,
39
+ agreement: null,
40
+ availability: "unavailable",
41
+ reason: "No raw mapping entry exists for this responsibility ID."
42
+ });
43
+ missingCriticalityResponsibilityIds.push(e.responsibilityId);
44
+ continue;
45
+ }
46
+ const agreement = mapping.criticality === e.expectedCriticality;
47
+ overlayAgreement.push({
48
+ responsibilityId: e.responsibilityId,
49
+ expectedCriticality: e.expectedCriticality,
50
+ observedCriticality: mapping.criticality,
51
+ agreement,
52
+ availability: "available",
53
+ reason: null
54
+ });
55
+ if (!agreement)
56
+ conflictingCriticalityResponsibilityIds.push(e.responsibilityId);
57
+ }
58
+ const unexpectedCriticalityResponsibilityIds = list.length === 0 || mappings === undefined
59
+ ? []
60
+ : mappings.filter((m) => !list.some((e) => e.responsibilityId === m.responsibilityId)).map((m) => m.responsibilityId);
61
+ const fullyMappedCriticalIds = [];
62
+ const partiallyMappedCriticalIds = [];
63
+ const unmappedCriticalIds = [];
64
+ const missingMappingEvidenceCriticalIds = [];
65
+ for (const e of list) {
66
+ if (!e.applicable || !e.requiresFullMapping || e.expectedCriticality !== "critical")
67
+ continue;
68
+ const mapping = findMapping(mappings, e.responsibilityId);
69
+ if (mapping === undefined) {
70
+ missingMappingEvidenceCriticalIds.push(e.responsibilityId);
71
+ continue;
72
+ }
73
+ if (mapping.mappingStatus === "mapped")
74
+ fullyMappedCriticalIds.push(e.responsibilityId);
75
+ else if (mapping.mappingStatus === "partially-mapped")
76
+ partiallyMappedCriticalIds.push(e.responsibilityId);
77
+ else if (mapping.mappingStatus === "unmapped")
78
+ unmappedCriticalIds.push(e.responsibilityId);
79
+ // mapping.mappingStatus === "not-applicable": excluded from the denominator entirely.
80
+ }
81
+ const denominator = fullyMappedCriticalIds.length + partiallyMappedCriticalIds.length + unmappedCriticalIds.length;
82
+ const mappedCriticalCompleteness = denominator === 0
83
+ ? {
84
+ availability: "not-applicable",
85
+ numerator: null,
86
+ denominator: null,
87
+ rate: null,
88
+ matchedExpectationIds: [],
89
+ missingExpectationIds: [],
90
+ reason: "No applicable critical responsibility requiring full mapping has resolvable mapping status."
91
+ }
92
+ : {
93
+ availability: "available",
94
+ numerator: fullyMappedCriticalIds.length,
95
+ denominator,
96
+ rate: fullyMappedCriticalIds.length / denominator,
97
+ matchedExpectationIds: fullyMappedCriticalIds,
98
+ missingExpectationIds: [...partiallyMappedCriticalIds, ...unmappedCriticalIds],
99
+ reason: null
100
+ };
101
+ return {
102
+ overlayAgreement,
103
+ missingCriticalityResponsibilityIds,
104
+ conflictingCriticalityResponsibilityIds,
105
+ unexpectedCriticalityResponsibilityIds,
106
+ mappedCriticalCompleteness,
107
+ fullyMappedCriticalIds,
108
+ partiallyMappedCriticalIds,
109
+ unmappedCriticalIds,
110
+ missingMappingEvidenceCriticalIds
111
+ };
112
+ }
@@ -0,0 +1,103 @@
1
+ import { buildStageContextExpectationTargetKey } from "./targetKeys.js";
2
+ const INSTRUCTION_CATEGORIES = ["workflow", "stage", "command", "rule", "report-contract"];
3
+ function buildRatioMetric(filtered, targetOutcome, notApplicableReason) {
4
+ if (filtered.length === 0) {
5
+ return {
6
+ availability: "not-applicable",
7
+ numerator: null,
8
+ denominator: null,
9
+ rate: null,
10
+ matchedExpectationIds: [],
11
+ missingExpectationIds: [],
12
+ reason: notApplicableReason
13
+ };
14
+ }
15
+ const matchedExpectationIds = [];
16
+ const missingExpectationIds = [];
17
+ for (const match of filtered) {
18
+ if (match.outcome === targetOutcome) {
19
+ matchedExpectationIds.push(match.expectationId);
20
+ }
21
+ else {
22
+ missingExpectationIds.push(match.expectationId);
23
+ }
24
+ }
25
+ return {
26
+ availability: "available",
27
+ numerator: matchedExpectationIds.length,
28
+ denominator: filtered.length,
29
+ rate: matchedExpectationIds.length / filtered.length,
30
+ matchedExpectationIds,
31
+ missingExpectationIds,
32
+ reason: null
33
+ };
34
+ }
35
+ export function calculateRequiredEvidenceRecall(matches) {
36
+ const filtered = matches.filter((match) => match.inclusion === "required");
37
+ return buildRatioMetric(filtered, "matched", "The expectation fixture contains no required evidence items.");
38
+ }
39
+ export function calculateAllowedEvidenceCoverage(matches) {
40
+ const filtered = matches.filter((match) => match.inclusion === "allowed");
41
+ return buildRatioMetric(filtered, "matched", "The expectation fixture contains no allowed evidence items.");
42
+ }
43
+ export function calculateForbiddenEvidenceInclusion(matches) {
44
+ const filtered = matches.filter((match) => match.inclusion === "forbidden");
45
+ return buildRatioMetric(filtered, "violated", "The expectation fixture contains no forbidden evidence items.");
46
+ }
47
+ export function calculateRequiredProvenanceRecall(matches) {
48
+ const filtered = matches.filter((match) => match.inclusion === "required" && match.category === "provenance");
49
+ return buildRatioMetric(filtered, "matched", "The expectation fixture contains no required provenance items.");
50
+ }
51
+ export function calculateIrrelevantFileInclusion(expectations, observedEvidence) {
52
+ const fileEvidence = observedEvidence.filter((evidence) => evidence.sourceArtifact === "context-capsule" && evidence.category === "file");
53
+ if (fileEvidence.length === 0) {
54
+ return {
55
+ availability: "not-applicable",
56
+ count: null,
57
+ evidenceKeys: [],
58
+ reason: "The completed strategy contains no context-capsule file evidence."
59
+ };
60
+ }
61
+ const coveredKeys = new Set(expectations.expectedEvidence
62
+ .filter((item) => item.sourceArtifact === "context-capsule" &&
63
+ item.category === "file" &&
64
+ (item.inclusion === "required" || item.inclusion === "allowed"))
65
+ .map((item) => buildStageContextExpectationTargetKey(item)));
66
+ const seen = new Set();
67
+ const evidenceKeys = [];
68
+ for (const evidence of fileEvidence) {
69
+ if (coveredKeys.has(evidence.targetKey))
70
+ continue;
71
+ if (seen.has(evidence.targetKey))
72
+ continue;
73
+ seen.add(evidence.targetKey);
74
+ evidenceKeys.push(evidence.targetKey);
75
+ }
76
+ return { availability: "available", count: evidenceKeys.length, evidenceKeys, reason: null };
77
+ }
78
+ export function calculateIrrelevantInstructionInclusion(expectations, observedEvidence) {
79
+ const instructionEvidence = observedEvidence.filter((evidence) => (evidence.sourceArtifact === "workflow-instruction-packet" || evidence.sourceArtifact === "full-workflow-library") &&
80
+ INSTRUCTION_CATEGORIES.includes(evidence.category));
81
+ if (instructionEvidence.length === 0) {
82
+ return {
83
+ availability: "not-applicable",
84
+ count: null,
85
+ evidenceKeys: [],
86
+ reason: "The completed strategy contains no workflow instruction evidence."
87
+ };
88
+ }
89
+ const coveredKeys = new Set(expectations.expectedEvidence
90
+ .filter((item) => INSTRUCTION_CATEGORIES.includes(item.category) && (item.inclusion === "required" || item.inclusion === "allowed"))
91
+ .map((item) => buildStageContextExpectationTargetKey(item)));
92
+ const seen = new Set();
93
+ const evidenceKeys = [];
94
+ for (const evidence of instructionEvidence) {
95
+ if (coveredKeys.has(evidence.targetKey))
96
+ continue;
97
+ if (seen.has(evidence.targetKey))
98
+ continue;
99
+ seen.add(evidence.targetKey);
100
+ evidenceKeys.push(evidence.targetKey);
101
+ }
102
+ return { availability: "available", count: evidenceKeys.length, evidenceKeys, reason: null };
103
+ }
@@ -0,0 +1,84 @@
1
+ export function collectSelectedOwnerEvidence(selectedOwners, sourceArtifact, sourceInstance) {
2
+ return selectedOwners.map((item) => ({
3
+ ownerId: item.id,
4
+ sourceArtifact,
5
+ sourceInstance,
6
+ itemKind: item.itemKind,
7
+ path: item.path ?? null,
8
+ symbolId: item.symbolId ?? null,
9
+ sourceFieldPath: `${sourceInstance}.selectedOwners`
10
+ }));
11
+ }
12
+ function notApplicableRatio(reason) {
13
+ return { availability: "not-applicable", numerator: null, denominator: null, rate: null, matchedExpectationIds: [], missingExpectationIds: [], reason };
14
+ }
15
+ function unavailableRatio(reason) {
16
+ return { availability: "unavailable", numerator: null, denominator: null, rate: null, matchedExpectationIds: [], missingExpectationIds: [], reason };
17
+ }
18
+ function notApplicableCount(reason) {
19
+ return { availability: "not-applicable", count: null, evidenceKeys: [], reason };
20
+ }
21
+ function unavailableCount(reason) {
22
+ return { availability: "unavailable", count: null, evidenceKeys: [], reason };
23
+ }
24
+ export function calculateOwnerMetrics(selectedOwnerEvidence, ownerExpectations) {
25
+ const evidence = selectedOwnerEvidence ?? [];
26
+ const evidenceUnavailable = selectedOwnerEvidence === undefined;
27
+ const unavailableReason = "Producer selected-owner evidence was not supplied to this metric.";
28
+ const requiredIds = new Set((ownerExpectations ?? []).filter((e) => e.inclusion === "required").map((e) => e.ownerId));
29
+ const allowedIds = new Set((ownerExpectations ?? []).filter((e) => e.inclusion === "allowed").map((e) => e.ownerId));
30
+ const forbiddenIds = new Set((ownerExpectations ?? []).filter((e) => e.inclusion === "forbidden").map((e) => e.ownerId));
31
+ const hasClosedSet = requiredIds.size > 0 || allowedIds.size > 0;
32
+ const observedIds = new Set(evidence.map((e) => e.ownerId));
33
+ const expectedOwnerPresent = evidenceUnavailable
34
+ ? unavailableRatio(unavailableReason)
35
+ : requiredIds.size === 0
36
+ ? notApplicableRatio("The expectation fixture contains no required owner expectations.")
37
+ : (() => {
38
+ const matched = [...requiredIds].filter((id) => observedIds.has(id));
39
+ const missing = [...requiredIds].filter((id) => !observedIds.has(id));
40
+ return {
41
+ availability: "available",
42
+ numerator: matched.length,
43
+ denominator: requiredIds.size,
44
+ rate: matched.length / requiredIds.size,
45
+ matchedExpectationIds: matched,
46
+ missingExpectationIds: missing,
47
+ reason: null
48
+ };
49
+ })();
50
+ const forbiddenOwnerPresent = evidenceUnavailable
51
+ ? unavailableRatio(unavailableReason)
52
+ : forbiddenIds.size === 0
53
+ ? notApplicableRatio("The expectation fixture contains no forbidden owner expectations.")
54
+ : (() => {
55
+ const present = [...forbiddenIds].filter((id) => observedIds.has(id));
56
+ const absent = [...forbiddenIds].filter((id) => !observedIds.has(id));
57
+ return {
58
+ availability: "available",
59
+ numerator: present.length,
60
+ denominator: forbiddenIds.size,
61
+ rate: present.length / forbiddenIds.size,
62
+ matchedExpectationIds: present,
63
+ missingExpectationIds: absent,
64
+ reason: null
65
+ };
66
+ })();
67
+ const falsePositiveCount = evidenceUnavailable
68
+ ? unavailableCount(unavailableReason)
69
+ : !hasClosedSet && forbiddenIds.size === 0
70
+ ? notApplicableCount("The expectation fixture defines neither a closed owner set nor forbidden owners.")
71
+ : (() => {
72
+ const falsePositives = [...observedIds].filter((id) => forbiddenIds.has(id) || (hasClosedSet && !requiredIds.has(id) && !allowedIds.has(id)));
73
+ return { availability: "available", count: falsePositives.length, evidenceKeys: falsePositives, reason: null };
74
+ })();
75
+ const falseNegativeCount = evidenceUnavailable
76
+ ? unavailableCount(unavailableReason)
77
+ : requiredIds.size === 0
78
+ ? notApplicableCount("The expectation fixture contains no required owner expectations.")
79
+ : (() => {
80
+ const falseNegatives = [...requiredIds].filter((id) => !observedIds.has(id));
81
+ return { availability: "available", count: falseNegatives.length, evidenceKeys: falseNegatives, reason: null };
82
+ })();
83
+ return { selectedOwnerEvidence: [...evidence], expectedOwnerPresent, forbiddenOwnerPresent, falsePositiveCount, falseNegativeCount };
84
+ }