eval-quality 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (179) hide show
  1. package/README.md +142 -67
  2. package/corpus/dev/README.md +20 -13
  3. package/corpus/dev/compile-seal-example/brief.json +1 -1
  4. package/corpus/dev/compile-seal-example/contract.json +1 -1
  5. package/corpus/dev/contracts/absent-collection-locations.json +1 -1
  6. package/corpus/dev/contracts/absent-sibling-groups.json +1 -1
  7. package/corpus/dev/contracts/absent-success-indicator.json +1 -1
  8. package/corpus/dev/contracts/empty-channel-roles.json +1 -1
  9. package/corpus/dev/contracts/empty-collection-locations.json +1 -1
  10. package/corpus/dev/contracts/empty-request-shapes.json +1 -1
  11. package/corpus/dev/contracts/empty-sibling-groups.json +1 -1
  12. package/corpus/dev/contracts/fragment-selection.json +1 -0
  13. package/corpus/dev/contracts/no-collection-quantifier.json +1 -1
  14. package/corpus/dev/contracts/no-operation-inventory.json +1 -1
  15. package/corpus/dev/contracts/no-read-back-relation.json +1 -1
  16. package/corpus/dev/contracts/no-state-change-marker.json +1 -1
  17. package/corpus/dev/contracts/no-type-violating-step.json +1 -1
  18. package/corpus/dev/contracts/per-key-split-oracles.json +1 -1
  19. package/corpus/dev/contracts/review-corpus.json +1 -0
  20. package/corpus/dev/contracts/satisfied-declarations.json +1 -1
  21. package/corpus/dev/contracts/single-required-response-key.json +1 -1
  22. package/corpus/dev/contracts/split-indicator-oracle.json +1 -1
  23. package/corpus/dev/contracts/unaddressed-parameter-sibling.json +1 -1
  24. package/corpus/dev/contracts/unnamed-reference-set.json +1 -1
  25. package/corpus/dev/contracts/wrong-cardinality-form.json +1 -1
  26. package/corpus/dev/index.json +1 -1
  27. package/dist/application/index.d.ts +2 -0
  28. package/dist/application/index.js +1 -0
  29. package/dist/application/score.d.ts +48 -0
  30. package/dist/application/score.js +186 -0
  31. package/dist/cli/arguments.d.ts +7 -3
  32. package/dist/cli/arguments.js +73 -5
  33. package/dist/cli/exit-codes.d.ts +6 -2
  34. package/dist/cli/exit-codes.js +8 -9
  35. package/dist/cli/main.js +9 -0
  36. package/dist/cli/render.d.ts +3 -6
  37. package/dist/cli/render.js +57 -5
  38. package/dist/cli/run.d.ts +13 -3
  39. package/dist/cli/run.js +136 -16
  40. package/dist/core/canonical/canonicalize.d.ts +12 -0
  41. package/dist/core/canonical/canonicalize.js +18 -4
  42. package/dist/core/compile/bindings.d.ts +89 -0
  43. package/dist/core/compile/bindings.js +334 -0
  44. package/dist/core/compile/compile.d.ts +17 -0
  45. package/dist/core/compile/compile.js +37 -1
  46. package/dist/core/compile/excluded-content.d.ts +11 -0
  47. package/dist/core/compile/excluded-content.js +42 -0
  48. package/dist/core/compile/expression-legality.d.ts +49 -0
  49. package/dist/core/compile/expression-legality.js +158 -38
  50. package/dist/core/compile/forbidden-inputs.d.ts +8 -1
  51. package/dist/core/compile/forbidden-inputs.js +16 -3
  52. package/dist/core/compile/interface-inventory.d.ts +57 -2
  53. package/dist/core/compile/interface-inventory.js +119 -13
  54. package/dist/core/compile/reachability.d.ts +28 -1
  55. package/dist/core/compile/reachability.js +167 -35
  56. package/dist/core/compile/schema-version.d.ts +2 -0
  57. package/dist/core/compile/schema-version.js +25 -0
  58. package/dist/core/compile/sensitivity-witness.d.ts +31 -12
  59. package/dist/core/compile/sensitivity-witness.js +110 -23
  60. package/dist/core/compile/step-reference.d.ts +2 -0
  61. package/dist/core/compile/step-reference.js +49 -0
  62. package/dist/core/coverage/operations.d.ts +62 -0
  63. package/dist/core/coverage/operations.js +57 -0
  64. package/dist/core/coverage/relevance.d.ts +4 -2
  65. package/dist/core/coverage/relevance.js +22 -23
  66. package/dist/core/coverage/satisfaction.d.ts +2 -2
  67. package/dist/core/coverage/satisfaction.js +73 -48
  68. package/dist/core/declared-inputs.d.ts +83 -4
  69. package/dist/core/declared-inputs.js +105 -8
  70. package/dist/core/emit/emit.d.ts +9 -0
  71. package/dist/core/emit/emit.js +214 -0
  72. package/dist/core/emit/private-artifact-digest.d.ts +11 -0
  73. package/dist/core/emit/private-artifact-digest.js +31 -0
  74. package/dist/core/evaluate/evidence-resolution.d.ts +10 -3
  75. package/dist/core/evaluate/evidence-resolution.js +74 -11
  76. package/dist/core/evaluate/operators.d.ts +22 -0
  77. package/dist/core/evaluate/operators.js +17 -1
  78. package/dist/core/evaluate/resolution.d.ts +14 -3
  79. package/dist/core/evaluate/resolution.js +63 -3
  80. package/dist/core/excluded-content.d.ts +65 -0
  81. package/dist/core/excluded-content.js +113 -0
  82. package/dist/core/failure-codes.d.ts +2 -2
  83. package/dist/core/failure-codes.js +7 -2
  84. package/dist/core/ingest/conditions.d.ts +201 -0
  85. package/dist/core/ingest/conditions.js +48 -0
  86. package/dist/core/ingest/index.d.ts +13 -0
  87. package/dist/core/ingest/index.js +11 -0
  88. package/dist/core/ingest/ingest.d.ts +68 -0
  89. package/dist/core/ingest/ingest.js +318 -0
  90. package/dist/core/lineage/stage-table.d.ts +21 -4
  91. package/dist/core/lineage/stage-table.js +35 -3
  92. package/dist/core/preflight/plan.d.ts +7 -5
  93. package/dist/core/preflight/plan.js +75 -32
  94. package/dist/core/preflight/projection.d.ts +6 -3
  95. package/dist/core/preflight/projection.js +22 -2
  96. package/dist/core/preflight/reduce.js +47 -7
  97. package/dist/core/preflight/witness-evidence.d.ts +5 -5
  98. package/dist/core/preflight/witness-evidence.js +73 -20
  99. package/dist/core/schemas/artifact.d.ts +1049 -30
  100. package/dist/core/schemas/constraint-ledger.js +51 -0
  101. package/dist/core/schemas/defect-signature.d.ts +695 -0
  102. package/dist/core/schemas/defect-signature.js +175 -0
  103. package/dist/core/schemas/eval-contract.d.ts +550 -12
  104. package/dist/core/schemas/eval-contract.js +32 -2
  105. package/dist/core/schemas/evidence-artifact.d.ts +109 -19
  106. package/dist/core/schemas/evidence-artifact.js +64 -8
  107. package/dist/core/schemas/interface.d.ts +741 -13
  108. package/dist/core/schemas/interface.js +120 -7
  109. package/dist/core/schemas/isolation-manifest.js +16 -9
  110. package/dist/core/schemas/plan.d.ts +329 -3
  111. package/dist/core/schemas/plan.js +89 -3
  112. package/dist/core/schemas/pointer.d.ts +47 -1
  113. package/dist/core/schemas/pointer.js +89 -8
  114. package/dist/core/schemas/port-messages.d.ts +194 -10
  115. package/dist/core/schemas/port-messages.js +73 -6
  116. package/dist/core/schemas/probe-body.d.ts +18 -0
  117. package/dist/core/schemas/probe-body.js +13 -0
  118. package/dist/core/schemas/probe-qualification.d.ts +168 -0
  119. package/dist/core/schemas/probe-qualification.js +106 -0
  120. package/dist/core/schemas/probe.d.ts +431 -0
  121. package/dist/core/schemas/probe.js +6 -2
  122. package/dist/core/schemas/scoring-policy.d.ts +1 -0
  123. package/dist/core/schemas/scoring-policy.js +6 -1
  124. package/dist/core/schemas/sealed-evaluator-brief.d.ts +1 -0
  125. package/dist/core/schemas/sealed-evaluator-brief.js +4 -1
  126. package/dist/core/schemas/sealed-run-record.d.ts +161 -12
  127. package/dist/core/schemas/sealed-run-record.js +92 -15
  128. package/dist/core/schemas/sensitivity-witness.d.ts +108 -6
  129. package/dist/core/schemas/sensitivity-witness.js +61 -5
  130. package/dist/core/score/binding-order.d.ts +31 -0
  131. package/dist/core/score/binding-order.js +88 -0
  132. package/dist/core/score/bindings.d.ts +93 -0
  133. package/dist/core/score/bindings.js +309 -0
  134. package/dist/core/score/ladder-table.d.ts +23 -0
  135. package/dist/core/score/ladder-table.js +109 -0
  136. package/dist/core/score/ladder.d.ts +229 -0
  137. package/dist/core/score/ladder.js +464 -0
  138. package/dist/core/score/mode-agreement.d.ts +30 -0
  139. package/dist/core/score/mode-agreement.js +16 -0
  140. package/dist/core/score/outcome-table.d.ts +17 -0
  141. package/dist/core/score/outcome-table.js +172 -0
  142. package/dist/core/score/outcome.d.ts +466 -0
  143. package/dist/core/score/outcome.js +562 -0
  144. package/dist/core/score/qualification.d.ts +83 -0
  145. package/dist/core/score/qualification.js +645 -0
  146. package/dist/core/score/quotation.d.ts +56 -0
  147. package/dist/core/score/quotation.js +226 -0
  148. package/dist/core/score/reduce-trials.d.ts +80 -0
  149. package/dist/core/score/reduce-trials.js +90 -0
  150. package/dist/core/score/score.d.ts +72 -0
  151. package/dist/core/score/score.js +607 -0
  152. package/dist/core/score/selection.d.ts +80 -0
  153. package/dist/core/score/selection.js +98 -0
  154. package/dist/core/score/strength.d.ts +58 -0
  155. package/dist/core/score/strength.js +227 -0
  156. package/dist/core/score/witness.d.ts +138 -0
  157. package/dist/core/score/witness.js +320 -0
  158. package/dist/core/seal/derived-reference.d.ts +2 -9
  159. package/dist/core/seal/derived-reference.js +293 -46
  160. package/dist/core/seal/plan-index.d.ts +18 -3
  161. package/dist/core/seal/plan-index.js +44 -8
  162. package/dist/core/seal/seal.js +8 -3
  163. package/dist/core/stage-contracts.d.ts +62 -2
  164. package/dist/index.d.ts +1 -1
  165. package/dist/index.js +1 -1
  166. package/dist/ports/environment-probe-port.d.ts +60 -4
  167. package/dist/testing/index.d.ts +2 -0
  168. package/dist/testing/index.js +1 -0
  169. package/dist/testing/probe-conformance.d.ts +29 -0
  170. package/dist/testing/probe-conformance.js +34 -5
  171. package/package.json +12 -3
  172. package/schemas/eval-contract.schema.json +1787 -797
  173. package/schemas/evidence-artifact.schema.json +126 -9
  174. package/schemas/isolation-manifest.schema.json +17 -10
  175. package/schemas/probe.schema.json +636 -5
  176. package/schemas/rubric.schema.json +1 -1
  177. package/schemas/scoring-policy.schema.json +8 -1
  178. package/schemas/sealed-evaluator-brief.schema.json +11 -1
  179. package/schemas/sealed-run-record.schema.json +314 -40
@@ -1,7 +1,9 @@
1
1
  /** the constraints the published JSON Schema cannot carry unaided. */
2
2
  import { INTERCHANGE_ARTIFACTS, } from './artifact.js';
3
+ import { PROBE_BINDING_CHANNEL_NON_EMPTY } from './defect-signature.js';
3
4
  import { TUPLE_ARITY } from './expression.js';
4
5
  import { BINDING_CHANNEL_NON_EMPTY } from './plan.js';
6
+ import { OBSERVATION_SEQUENCE_UNIQUE } from './sealed-run-record.js';
5
7
  // Verified on the pin: z.tuple([a, b]) exports `prefixItems` alone (no
6
8
  // minItems, maxItems, or items), so the published schema accepts the
7
9
  // one-operand and three-operand `equality` that Zod rejects. Arity is the one
@@ -139,5 +141,54 @@ export const CONSTRAINT_LEDGER = [
139
141
  operandTypeEntry,
140
142
  ...lineageEntries,
141
143
  ...secretsProhibitionEntries,
144
+ {
145
+ id: OBSERVATION_SEQUENCE_UNIQUE,
146
+ location: { kind: 'root', artifact: 'sealed-run-record' },
147
+ branch: null,
148
+ field: 'observations',
149
+ statement: "Every observation's `sequence` is unique within the record (ADR-006, owed item 2).",
150
+ disposition: {
151
+ kind: 'not-expressible',
152
+ reason: "Uniqueness of a nested field across sibling array items has no draft-2020-12 keyword: `uniqueItems` compares whole items structurally and nothing in the dialect can name one property path and require distinctness on it alone. Unlike the lineage biconditional above, this constraint IS enforced by Zod (a `.refine()` on `observations`), because owed item 2 requires it schema-enforced and the AD-13 differential corpus never manufactures two observations sharing a `sequence`: the generated mutant walk is keyword-driven (it perturbs one occurrence at a time) and never clones or duplicates an `observations` array item, since that field carries no `maxItems` for the generator's clone-to-exceed step to fire against. The gap is therefore real but inert against the corpus this repository generates, and is stated in `Observation.sequence`'s own description for a non-Zod consumer to reimplement. This safety argument holds only while `observations` carries no `maxItems`: adding one later starts the clone-to-exceed step firing against this field, and this entry must be re-verified then.",
153
+ },
154
+ },
155
+ {
156
+ id: 'evidence-mode-caller-attested',
157
+ location: { kind: 'root', artifact: 'evidence-artifact' },
158
+ branch: null,
159
+ field: 'callerAttestedInputs',
160
+ statement: '`mode` appears in `callerAttestedInputs`. The other five scoring-version inputs may each legitimately be attested or computed, so the list admits any subset of them; `mode` cannot be, since `ScoringVersionInputs.mode` is read from the sealed run record and never re-derived, which makes omitting it a misdeclaration under AD-32 rather than a stricter integration.',
161
+ disposition: {
162
+ kind: 'not-expressible',
163
+ reason: 'A one-member `contains` would express it, and injecting one would fail the same way the observation-sequence entry describes in reverse: the mutant generator manufactures the witness trivially by dropping one array element, so a Zod refinement and an uninjected export would disagree on a corpus member this repository generates. The rule is enforced where the artifact is produced instead: `core/emit` names `mode` in `CALLER_ATTESTED_INPUTS` unconditionally, and the field description states the rule for a caller who authors the artifact by hand. This argument holds only while `emit` is the sole producer in this package.',
164
+ },
165
+ },
166
+ {
167
+ id: 'defect-signature-required',
168
+ location: { kind: 'root', artifact: 'probe' },
169
+ branch: null,
170
+ field: null,
171
+ statement: 'Every probe that is not a known-clean control carries an AD-40 defect signature, except a probe whose class is `canary`, which carries `null`. AD-9\'s "an unqualified probe cannot enter a sealed set" and AD-9\'s route-per-class pairing are the same kind of rule and are enforced beside it.',
172
+ disposition: {
173
+ kind: 'not-expressible',
174
+ reason: 'A union-level `superRefine` is the only Zod spelling, and a refinement exports byte-identically, so Zod would reject a signature-less non-canary probe that ajv accepts. Unlike the observation-sequence entry above, that disagreement is not inert against the corpus this repository generates: the mutant generator manufactures the witness trivially from the accept fixture by nulling one field, and the published-schema differential check exists to fail on exactly such a disagreement. So the schema admits the shape and `qualifyProbe` (`core/score/qualification.ts`) returns a reason code, following `compile/interface-inventory.ts`, which put the declared-principal check in the compiler for the same reason. This argument holds only while the published export drops refinements: a Zod release that emitted a union-level refinement into the document, or a decision to inject a conditional here, makes the gate-side placement a choice rather than a necessity, and this entry must be re-verified then.',
175
+ },
176
+ },
177
+ {
178
+ id: PROBE_BINDING_CHANNEL_NON_EMPTY,
179
+ location: {
180
+ kind: 'definition',
181
+ artifact: 'probe',
182
+ name: 'ProbeInputBindingChannel',
183
+ },
184
+ branch: null,
185
+ field: null,
186
+ statement: "A declared channel of a defect signature's selector names at least one parameter. An unbound channel is `null`, never `{}`. Inject at the definition root, exactly as the contract-side channel does: `minProperties` is ignored on a non-object instance, so it constrains the object branch and leaves the null branch alone.",
187
+ disposition: {
188
+ kind: 'inject',
189
+ dialect: 'draft-2020-12',
190
+ keywords: { minProperties: 1 },
191
+ },
192
+ },
142
193
  ];
143
194
  export const constraintLedgerEntry = (id) => CONSTRAINT_LEDGER.find((entry) => entry.id === id);