eval-quality 0.2.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. package/README.md +17 -12
  2. package/corpus/dev/README.md +3 -2
  3. package/corpus/dev/compile-seal-example/brief.json +1 -1
  4. package/corpus/dev/compile-seal-example/contract.json +1 -1
  5. package/corpus/dev/contracts/absent-collection-locations.json +1 -1
  6. package/corpus/dev/contracts/absent-sibling-groups.json +1 -1
  7. package/corpus/dev/contracts/absent-success-indicator.json +1 -1
  8. package/corpus/dev/contracts/empty-channel-roles.json +1 -1
  9. package/corpus/dev/contracts/empty-collection-locations.json +1 -1
  10. package/corpus/dev/contracts/empty-request-shapes.json +1 -1
  11. package/corpus/dev/contracts/empty-sibling-groups.json +1 -1
  12. package/corpus/dev/contracts/fragment-selection.json +1 -0
  13. package/corpus/dev/contracts/no-collection-quantifier.json +1 -1
  14. package/corpus/dev/contracts/no-operation-inventory.json +1 -1
  15. package/corpus/dev/contracts/no-read-back-relation.json +1 -1
  16. package/corpus/dev/contracts/no-state-change-marker.json +1 -1
  17. package/corpus/dev/contracts/no-type-violating-step.json +1 -1
  18. package/corpus/dev/contracts/per-key-split-oracles.json +1 -1
  19. package/corpus/dev/contracts/review-corpus.json +1 -0
  20. package/corpus/dev/contracts/satisfied-declarations.json +1 -1
  21. package/corpus/dev/contracts/single-required-response-key.json +1 -1
  22. package/corpus/dev/contracts/split-indicator-oracle.json +1 -1
  23. package/corpus/dev/contracts/unaddressed-parameter-sibling.json +1 -1
  24. package/corpus/dev/contracts/unnamed-reference-set.json +1 -1
  25. package/corpus/dev/contracts/wrong-cardinality-form.json +1 -1
  26. package/corpus/dev/index.json +1 -1
  27. package/dist/adapters/command-line-adapter.d.ts +44 -0
  28. package/dist/adapters/command-line-adapter.js +284 -0
  29. package/dist/adapters/command-target-policy.d.ts +36 -0
  30. package/dist/adapters/command-target-policy.js +33 -0
  31. package/dist/adapters/index.d.ts +2 -0
  32. package/dist/adapters/index.js +1 -0
  33. package/dist/cli/render.d.ts +2 -5
  34. package/dist/cli/render.js +51 -1
  35. package/dist/core/compile/bindings.d.ts +26 -7
  36. package/dist/core/compile/bindings.js +49 -32
  37. package/dist/core/compile/compile.d.ts +12 -0
  38. package/dist/core/compile/compile.js +28 -1
  39. package/dist/core/compile/excluded-content.d.ts +11 -0
  40. package/dist/core/compile/excluded-content.js +42 -0
  41. package/dist/core/compile/expression-legality.d.ts +2 -2
  42. package/dist/core/compile/expression-legality.js +58 -12
  43. package/dist/core/compile/interface-inventory.d.ts +34 -1
  44. package/dist/core/compile/interface-inventory.js +88 -11
  45. package/dist/core/compile/reachability.d.ts +12 -2
  46. package/dist/core/compile/reachability.js +123 -36
  47. package/dist/core/compile/schema-version.d.ts +2 -0
  48. package/dist/core/compile/schema-version.js +25 -0
  49. package/dist/core/compile/sensitivity-witness.d.ts +31 -12
  50. package/dist/core/compile/sensitivity-witness.js +110 -23
  51. package/dist/core/compile/step-reference.d.ts +2 -0
  52. package/dist/core/compile/step-reference.js +49 -0
  53. package/dist/core/coverage/operations.d.ts +62 -0
  54. package/dist/core/coverage/operations.js +57 -0
  55. package/dist/core/coverage/relevance.d.ts +4 -2
  56. package/dist/core/coverage/relevance.js +22 -23
  57. package/dist/core/coverage/satisfaction.d.ts +2 -2
  58. package/dist/core/coverage/satisfaction.js +73 -48
  59. package/dist/core/declared-inputs.d.ts +83 -4
  60. package/dist/core/declared-inputs.js +105 -8
  61. package/dist/core/evaluate/evidence-resolution.d.ts +9 -12
  62. package/dist/core/evaluate/evidence-resolution.js +70 -10
  63. package/dist/core/evaluate/operators.d.ts +22 -0
  64. package/dist/core/evaluate/operators.js +17 -1
  65. package/dist/core/evaluate/resolution.d.ts +14 -3
  66. package/dist/core/evaluate/resolution.js +63 -3
  67. package/dist/core/excluded-content.d.ts +65 -0
  68. package/dist/core/excluded-content.js +113 -0
  69. package/dist/core/failure-codes.d.ts +2 -2
  70. package/dist/core/failure-codes.js +5 -2
  71. package/dist/core/ingest/conditions.d.ts +1 -1
  72. package/dist/core/ingest/ingest.js +8 -0
  73. package/dist/core/preflight/plan.d.ts +7 -5
  74. package/dist/core/preflight/plan.js +75 -32
  75. package/dist/core/preflight/projection.d.ts +6 -3
  76. package/dist/core/preflight/projection.js +22 -2
  77. package/dist/core/preflight/reduce.js +47 -7
  78. package/dist/core/preflight/witness-evidence.d.ts +5 -5
  79. package/dist/core/preflight/witness-evidence.js +63 -20
  80. package/dist/core/schemas/artifact.d.ts +683 -35
  81. package/dist/core/schemas/constraint-ledger.js +11 -0
  82. package/dist/core/schemas/defect-signature.d.ts +438 -16
  83. package/dist/core/schemas/defect-signature.js +58 -10
  84. package/dist/core/schemas/eval-contract.d.ts +517 -12
  85. package/dist/core/schemas/eval-contract.js +18 -2
  86. package/dist/core/schemas/evidence-artifact.d.ts +14 -4
  87. package/dist/core/schemas/evidence-artifact.js +1 -1
  88. package/dist/core/schemas/interface.d.ts +741 -13
  89. package/dist/core/schemas/interface.js +120 -7
  90. package/dist/core/schemas/isolation-manifest.js +16 -9
  91. package/dist/core/schemas/plan.d.ts +217 -3
  92. package/dist/core/schemas/plan.js +22 -1
  93. package/dist/core/schemas/pointer.d.ts +47 -1
  94. package/dist/core/schemas/pointer.js +89 -8
  95. package/dist/core/schemas/port-messages.d.ts +194 -10
  96. package/dist/core/schemas/port-messages.js +73 -6
  97. package/dist/core/schemas/probe-body.d.ts +18 -0
  98. package/dist/core/schemas/probe-body.js +13 -0
  99. package/dist/core/schemas/probe-policy.d.ts +36 -0
  100. package/dist/core/schemas/probe-policy.js +44 -0
  101. package/dist/core/schemas/probe.d.ts +122 -3
  102. package/dist/core/schemas/probe.js +1 -1
  103. package/dist/core/schemas/sealed-run-record.d.ts +131 -9
  104. package/dist/core/schemas/sealed-run-record.js +56 -11
  105. package/dist/core/schemas/sensitivity-witness.d.ts +108 -6
  106. package/dist/core/schemas/sensitivity-witness.js +61 -5
  107. package/dist/core/score/bindings.d.ts +2 -2
  108. package/dist/core/score/bindings.js +25 -13
  109. package/dist/core/score/qualification.d.ts +5 -5
  110. package/dist/core/score/qualification.js +67 -37
  111. package/dist/core/score/quotation.d.ts +1 -24
  112. package/dist/core/score/quotation.js +29 -4
  113. package/dist/core/score/score.js +28 -2
  114. package/dist/core/score/strength.d.ts +9 -0
  115. package/dist/core/score/strength.js +35 -1
  116. package/dist/core/score/witness.d.ts +21 -0
  117. package/dist/core/score/witness.js +9 -4
  118. package/dist/core/seal/derived-reference.d.ts +2 -9
  119. package/dist/core/seal/derived-reference.js +62 -23
  120. package/dist/core/seal/plan-index.d.ts +18 -3
  121. package/dist/core/seal/plan-index.js +44 -8
  122. package/dist/index.d.ts +1 -1
  123. package/dist/index.js +1 -1
  124. package/dist/ports/environment-probe-port.d.ts +60 -4
  125. package/dist/testing/conformance.d.ts +2 -1
  126. package/dist/testing/conformance.js +1 -0
  127. package/dist/testing/index.d.ts +3 -3
  128. package/dist/testing/index.js +1 -1
  129. package/dist/testing/probe-conformance.d.ts +56 -5
  130. package/dist/testing/probe-conformance.js +196 -5
  131. package/package.json +1 -1
  132. package/schemas/eval-contract.schema.json +1356 -464
  133. package/schemas/evidence-artifact.schema.json +59 -29
  134. package/schemas/isolation-manifest.schema.json +17 -10
  135. package/schemas/probe.schema.json +197 -71
  136. package/schemas/rubric.schema.json +1 -1
  137. package/schemas/sealed-run-record.schema.json +296 -39
@@ -4,6 +4,7 @@
4
4
  * unchanged. This module supplies the `Observation` shape they read and a
5
5
  * collection predicate scoped to one operation.
6
6
  */
7
+ import { descriptorArtifactOf, descriptorChannelOf, } from '../declared-inputs.js';
7
8
  import { makeResolveOperand } from '../evaluate/evidence-resolution.js';
8
9
  import { resolveCheck, } from '../evaluate/resolution.js';
9
10
  import { decodeTail, parseEvidenceTarget } from '../seal/plan-index.js';
@@ -17,17 +18,46 @@ const asJsonObject = (value) => value !== null && typeof value === 'object' && !
17
18
  ? value
18
19
  : null;
19
20
  /**
20
- * `ObservedCallInputs` is narrower than `WitnessInputs`, so `body` loses
21
+ * `ObservedCallInputs` is narrower than `ApiWitnessInputs`, so `body` loses
21
22
  * information here: an absent body and a non-object JSON body both map to
22
23
  * `null`, and a relation addressing `/interactions/{legId}/call-inputs/body` on
23
24
  * such a leg resolves `ABSENT`.
24
25
  */
25
- const callInputsOf = (inputs) => ({
26
- path: inputs.path,
27
- query: inputs.query,
28
- header: inputs.header,
29
- body: inputs.body.kind === 'json' ? asJsonObject(inputs.body.value) : null,
30
- });
26
+ const ABSENT_CHANNEL = { kind: 'absent' };
27
+ const bodyValue = (body) => body.kind === 'absent' ? null : body.value;
28
+ /**
29
+ * The leg's supplied inputs as the record spells them: every channel the leg
30
+ * did not use is `null` rather than absent, which is the observation's own
31
+ * convention for an unused channel.
32
+ */
33
+ const callInputsOf = (inputs) => {
34
+ const empty = {
35
+ path: null,
36
+ query: null,
37
+ header: null,
38
+ body: null,
39
+ argument: null,
40
+ option: null,
41
+ environment: null,
42
+ stdin: null,
43
+ };
44
+ if ('body' in inputs) {
45
+ return {
46
+ ...empty,
47
+ path: inputs.path,
48
+ query: inputs.query,
49
+ header: inputs.header,
50
+ body: inputs.body.kind === 'json' ? asJsonObject(inputs.body.value) : null,
51
+ };
52
+ }
53
+ return {
54
+ ...empty,
55
+ argument: inputs.argument,
56
+ option: inputs.option,
57
+ environment: inputs.environment,
58
+ stdin: inputs.stdin.kind === 'json' ? asJsonObject(inputs.stdin.value) : null,
59
+ };
60
+ };
31
61
  /**
32
62
  * One leg as the `Observation` `makeResolveOperand` takes. The relation reads the
33
63
  * **projected** body, which is AD-10's "evaluated over that operation's response
@@ -35,8 +65,10 @@ const callInputsOf = (inputs) => ({
35
65
  * the projection does not carry. `provenance` is `baseline`: a pre-flight leg is
36
66
  * pre-canned by definition.
37
67
  */
38
- export function evidenceOf(projected, observation, inputs) {
68
+ export function evidenceOf(projected, observation, inputs, operation) {
39
69
  const { body } = projected;
70
+ const descriptorChannel = descriptorChannelOf(operation);
71
+ const describedArtifact = descriptorArtifactOf(operation);
40
72
  return {
41
73
  observationId: projected.legId,
42
74
  // A synthetic, single-observation shape built fresh per leg and never
@@ -51,17 +83,28 @@ export function evidenceOf(projected, observation, inputs) {
51
83
  sequence: 1,
52
84
  operationId: projected.operationId,
53
85
  provenance: 'baseline',
86
+ // A pre-flight leg is issued by this package rather than by a harness
87
+ // acting as a declared account, so it names no principal.
88
+ principal: null,
54
89
  callInputs: callInputsOf(inputs),
55
- responseBody: body.kind === 'json'
56
- ? body.value
57
- : body.kind === 'text'
58
- ? body.value
59
- : null,
60
- responseHeaders: observation.headers,
61
- responseStatus: observation.status,
62
- stdout: null,
63
- stderr: null,
64
- exitCode: null,
90
+ // The projected body is whichever channel the operation's descriptor
91
+ // describes, so it lands on the channel a relation addresses. Both are
92
+ // filled from the same projection rather than one being derived from
93
+ // the other, and every channel the leg did not observe is written down
94
+ // as unobserved rather than left to a default.
95
+ responseBody: observation.kind === 'api' ? bodyValue(body) : null,
96
+ responseHeaders: observation.kind === 'api' ? observation.headers : null,
97
+ responseStatus: observation.kind === 'api' ? observation.status : null,
98
+ stdout: observation.kind === 'api' || descriptorChannel !== 'stdout'
99
+ ? ABSENT_CHANNEL
100
+ : body,
101
+ stderr: observation.kind === 'api' || descriptorChannel !== 'stderr'
102
+ ? ABSENT_CHANNEL
103
+ : body,
104
+ exitCode: observation.kind === 'api' ? null : observation.exitCode,
105
+ artifacts: observation.kind === 'api' || describedArtifact === null
106
+ ? {}
107
+ : { [describedArtifact]: body },
65
108
  };
66
109
  }
67
110
  /** the contract's declared reference sets in the shape the resolver wants. */
@@ -93,6 +136,6 @@ export function makeWitnessPointerDenotesCollection(operation) {
93
136
  };
94
137
  }
95
138
  /** Resolves one witness relation over the legs it addresses. */
96
- export function resolveWitnessRelation(relation, legEvidence, operation, referenceSets, artifactPath) {
97
- return resolveCheck(relation, makeResolveOperand(legEvidence, referenceSets), makeWitnessPointerDenotesCollection(operation), PREFLIGHT_REGEX_MATCH_STEP_BUDGET, artifactPath);
139
+ export function resolveWitnessRelation(relation, legEvidence, operation, referenceSets, referenceSetKeys, artifactPath) {
140
+ return resolveCheck(relation, makeResolveOperand(legEvidence, referenceSets), makeWitnessPointerDenotesCollection(operation), referenceSetKeys, PREFLIGHT_REGEX_MATCH_STEP_BUDGET, artifactPath);
98
141
  }