eval-quality 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (179) hide show
  1. package/README.md +142 -67
  2. package/corpus/dev/README.md +20 -13
  3. package/corpus/dev/compile-seal-example/brief.json +1 -1
  4. package/corpus/dev/compile-seal-example/contract.json +1 -1
  5. package/corpus/dev/contracts/absent-collection-locations.json +1 -1
  6. package/corpus/dev/contracts/absent-sibling-groups.json +1 -1
  7. package/corpus/dev/contracts/absent-success-indicator.json +1 -1
  8. package/corpus/dev/contracts/empty-channel-roles.json +1 -1
  9. package/corpus/dev/contracts/empty-collection-locations.json +1 -1
  10. package/corpus/dev/contracts/empty-request-shapes.json +1 -1
  11. package/corpus/dev/contracts/empty-sibling-groups.json +1 -1
  12. package/corpus/dev/contracts/fragment-selection.json +1 -0
  13. package/corpus/dev/contracts/no-collection-quantifier.json +1 -1
  14. package/corpus/dev/contracts/no-operation-inventory.json +1 -1
  15. package/corpus/dev/contracts/no-read-back-relation.json +1 -1
  16. package/corpus/dev/contracts/no-state-change-marker.json +1 -1
  17. package/corpus/dev/contracts/no-type-violating-step.json +1 -1
  18. package/corpus/dev/contracts/per-key-split-oracles.json +1 -1
  19. package/corpus/dev/contracts/review-corpus.json +1 -0
  20. package/corpus/dev/contracts/satisfied-declarations.json +1 -1
  21. package/corpus/dev/contracts/single-required-response-key.json +1 -1
  22. package/corpus/dev/contracts/split-indicator-oracle.json +1 -1
  23. package/corpus/dev/contracts/unaddressed-parameter-sibling.json +1 -1
  24. package/corpus/dev/contracts/unnamed-reference-set.json +1 -1
  25. package/corpus/dev/contracts/wrong-cardinality-form.json +1 -1
  26. package/corpus/dev/index.json +1 -1
  27. package/dist/application/index.d.ts +2 -0
  28. package/dist/application/index.js +1 -0
  29. package/dist/application/score.d.ts +48 -0
  30. package/dist/application/score.js +186 -0
  31. package/dist/cli/arguments.d.ts +7 -3
  32. package/dist/cli/arguments.js +73 -5
  33. package/dist/cli/exit-codes.d.ts +6 -2
  34. package/dist/cli/exit-codes.js +8 -9
  35. package/dist/cli/main.js +9 -0
  36. package/dist/cli/render.d.ts +3 -6
  37. package/dist/cli/render.js +57 -5
  38. package/dist/cli/run.d.ts +13 -3
  39. package/dist/cli/run.js +136 -16
  40. package/dist/core/canonical/canonicalize.d.ts +12 -0
  41. package/dist/core/canonical/canonicalize.js +18 -4
  42. package/dist/core/compile/bindings.d.ts +89 -0
  43. package/dist/core/compile/bindings.js +334 -0
  44. package/dist/core/compile/compile.d.ts +17 -0
  45. package/dist/core/compile/compile.js +37 -1
  46. package/dist/core/compile/excluded-content.d.ts +11 -0
  47. package/dist/core/compile/excluded-content.js +42 -0
  48. package/dist/core/compile/expression-legality.d.ts +49 -0
  49. package/dist/core/compile/expression-legality.js +158 -38
  50. package/dist/core/compile/forbidden-inputs.d.ts +8 -1
  51. package/dist/core/compile/forbidden-inputs.js +16 -3
  52. package/dist/core/compile/interface-inventory.d.ts +57 -2
  53. package/dist/core/compile/interface-inventory.js +119 -13
  54. package/dist/core/compile/reachability.d.ts +28 -1
  55. package/dist/core/compile/reachability.js +167 -35
  56. package/dist/core/compile/schema-version.d.ts +2 -0
  57. package/dist/core/compile/schema-version.js +25 -0
  58. package/dist/core/compile/sensitivity-witness.d.ts +31 -12
  59. package/dist/core/compile/sensitivity-witness.js +110 -23
  60. package/dist/core/compile/step-reference.d.ts +2 -0
  61. package/dist/core/compile/step-reference.js +49 -0
  62. package/dist/core/coverage/operations.d.ts +62 -0
  63. package/dist/core/coverage/operations.js +57 -0
  64. package/dist/core/coverage/relevance.d.ts +4 -2
  65. package/dist/core/coverage/relevance.js +22 -23
  66. package/dist/core/coverage/satisfaction.d.ts +2 -2
  67. package/dist/core/coverage/satisfaction.js +73 -48
  68. package/dist/core/declared-inputs.d.ts +83 -4
  69. package/dist/core/declared-inputs.js +105 -8
  70. package/dist/core/emit/emit.d.ts +9 -0
  71. package/dist/core/emit/emit.js +214 -0
  72. package/dist/core/emit/private-artifact-digest.d.ts +11 -0
  73. package/dist/core/emit/private-artifact-digest.js +31 -0
  74. package/dist/core/evaluate/evidence-resolution.d.ts +10 -3
  75. package/dist/core/evaluate/evidence-resolution.js +74 -11
  76. package/dist/core/evaluate/operators.d.ts +22 -0
  77. package/dist/core/evaluate/operators.js +17 -1
  78. package/dist/core/evaluate/resolution.d.ts +14 -3
  79. package/dist/core/evaluate/resolution.js +63 -3
  80. package/dist/core/excluded-content.d.ts +65 -0
  81. package/dist/core/excluded-content.js +113 -0
  82. package/dist/core/failure-codes.d.ts +2 -2
  83. package/dist/core/failure-codes.js +7 -2
  84. package/dist/core/ingest/conditions.d.ts +201 -0
  85. package/dist/core/ingest/conditions.js +48 -0
  86. package/dist/core/ingest/index.d.ts +13 -0
  87. package/dist/core/ingest/index.js +11 -0
  88. package/dist/core/ingest/ingest.d.ts +68 -0
  89. package/dist/core/ingest/ingest.js +318 -0
  90. package/dist/core/lineage/stage-table.d.ts +21 -4
  91. package/dist/core/lineage/stage-table.js +35 -3
  92. package/dist/core/preflight/plan.d.ts +7 -5
  93. package/dist/core/preflight/plan.js +75 -32
  94. package/dist/core/preflight/projection.d.ts +6 -3
  95. package/dist/core/preflight/projection.js +22 -2
  96. package/dist/core/preflight/reduce.js +47 -7
  97. package/dist/core/preflight/witness-evidence.d.ts +5 -5
  98. package/dist/core/preflight/witness-evidence.js +73 -20
  99. package/dist/core/schemas/artifact.d.ts +1049 -30
  100. package/dist/core/schemas/constraint-ledger.js +51 -0
  101. package/dist/core/schemas/defect-signature.d.ts +695 -0
  102. package/dist/core/schemas/defect-signature.js +175 -0
  103. package/dist/core/schemas/eval-contract.d.ts +550 -12
  104. package/dist/core/schemas/eval-contract.js +32 -2
  105. package/dist/core/schemas/evidence-artifact.d.ts +109 -19
  106. package/dist/core/schemas/evidence-artifact.js +64 -8
  107. package/dist/core/schemas/interface.d.ts +741 -13
  108. package/dist/core/schemas/interface.js +120 -7
  109. package/dist/core/schemas/isolation-manifest.js +16 -9
  110. package/dist/core/schemas/plan.d.ts +329 -3
  111. package/dist/core/schemas/plan.js +89 -3
  112. package/dist/core/schemas/pointer.d.ts +47 -1
  113. package/dist/core/schemas/pointer.js +89 -8
  114. package/dist/core/schemas/port-messages.d.ts +194 -10
  115. package/dist/core/schemas/port-messages.js +73 -6
  116. package/dist/core/schemas/probe-body.d.ts +18 -0
  117. package/dist/core/schemas/probe-body.js +13 -0
  118. package/dist/core/schemas/probe-qualification.d.ts +168 -0
  119. package/dist/core/schemas/probe-qualification.js +106 -0
  120. package/dist/core/schemas/probe.d.ts +431 -0
  121. package/dist/core/schemas/probe.js +6 -2
  122. package/dist/core/schemas/scoring-policy.d.ts +1 -0
  123. package/dist/core/schemas/scoring-policy.js +6 -1
  124. package/dist/core/schemas/sealed-evaluator-brief.d.ts +1 -0
  125. package/dist/core/schemas/sealed-evaluator-brief.js +4 -1
  126. package/dist/core/schemas/sealed-run-record.d.ts +161 -12
  127. package/dist/core/schemas/sealed-run-record.js +92 -15
  128. package/dist/core/schemas/sensitivity-witness.d.ts +108 -6
  129. package/dist/core/schemas/sensitivity-witness.js +61 -5
  130. package/dist/core/score/binding-order.d.ts +31 -0
  131. package/dist/core/score/binding-order.js +88 -0
  132. package/dist/core/score/bindings.d.ts +93 -0
  133. package/dist/core/score/bindings.js +309 -0
  134. package/dist/core/score/ladder-table.d.ts +23 -0
  135. package/dist/core/score/ladder-table.js +109 -0
  136. package/dist/core/score/ladder.d.ts +229 -0
  137. package/dist/core/score/ladder.js +464 -0
  138. package/dist/core/score/mode-agreement.d.ts +30 -0
  139. package/dist/core/score/mode-agreement.js +16 -0
  140. package/dist/core/score/outcome-table.d.ts +17 -0
  141. package/dist/core/score/outcome-table.js +172 -0
  142. package/dist/core/score/outcome.d.ts +466 -0
  143. package/dist/core/score/outcome.js +562 -0
  144. package/dist/core/score/qualification.d.ts +83 -0
  145. package/dist/core/score/qualification.js +645 -0
  146. package/dist/core/score/quotation.d.ts +56 -0
  147. package/dist/core/score/quotation.js +226 -0
  148. package/dist/core/score/reduce-trials.d.ts +80 -0
  149. package/dist/core/score/reduce-trials.js +90 -0
  150. package/dist/core/score/score.d.ts +72 -0
  151. package/dist/core/score/score.js +607 -0
  152. package/dist/core/score/selection.d.ts +80 -0
  153. package/dist/core/score/selection.js +98 -0
  154. package/dist/core/score/strength.d.ts +58 -0
  155. package/dist/core/score/strength.js +227 -0
  156. package/dist/core/score/witness.d.ts +138 -0
  157. package/dist/core/score/witness.js +320 -0
  158. package/dist/core/seal/derived-reference.d.ts +2 -9
  159. package/dist/core/seal/derived-reference.js +293 -46
  160. package/dist/core/seal/plan-index.d.ts +18 -3
  161. package/dist/core/seal/plan-index.js +44 -8
  162. package/dist/core/seal/seal.js +8 -3
  163. package/dist/core/stage-contracts.d.ts +62 -2
  164. package/dist/index.d.ts +1 -1
  165. package/dist/index.js +1 -1
  166. package/dist/ports/environment-probe-port.d.ts +60 -4
  167. package/dist/testing/index.d.ts +2 -0
  168. package/dist/testing/index.js +1 -0
  169. package/dist/testing/probe-conformance.d.ts +29 -0
  170. package/dist/testing/probe-conformance.js +34 -5
  171. package/package.json +12 -3
  172. package/schemas/eval-contract.schema.json +1787 -797
  173. package/schemas/evidence-artifact.schema.json +126 -9
  174. package/schemas/isolation-manifest.schema.json +17 -10
  175. package/schemas/probe.schema.json +636 -5
  176. package/schemas/rubric.schema.json +1 -1
  177. package/schemas/scoring-policy.schema.json +8 -1
  178. package/schemas/sealed-evaluator-brief.schema.json +11 -1
  179. package/schemas/sealed-run-record.schema.json +314 -40
@@ -0,0 +1,80 @@
1
+ /**
2
+ * Owed item 2's fix: a step selects observations by `operationId`, ordered by
3
+ * the record's monotonic `sequence` (ADR-006 forbids array position as
4
+ * ordering). `selectObservations` reports a several-match ambiguity as data;
5
+ * no first-match or last-match convention picks a winner. Pure and total:
6
+ * every input produces a defined result, nothing throws, and no AD-6 outcome
7
+ * state is assigned. Assigning outcome state is AD-33's reference decision
8
+ * procedure's job.
9
+ *
10
+ * `resolveTemporalAnchor` is a separate, narrower resolution built on top:
11
+ * picking a concrete point in time for an `any`-cardinality temporal anchor
12
+ * with several matches, by lowest sequence. The policy above still holds
13
+ * here: a single-valued cardinality's several-match ambiguity still comes
14
+ * back unresolved from `resolveTemporalAnchor` too.
15
+ *
16
+ * No stage claims this module yet; `stage-table.ts`'s `score` row stays
17
+ * `module: null` throughout.
18
+ */
19
+ import type { InteractionStep } from '../schemas/plan.ts';
20
+ import type { Observation } from '../schemas/sealed-run-record.ts';
21
+ /** How many observations a step's selector matched. Never a count past two: past one match, only "several" is recorded. */
22
+ export type SelectionCount = 'none' | 'one' | 'several';
23
+ export type StepSelection = {
24
+ readonly result: SelectionCount;
25
+ /**
26
+ * In ascending-`sequence` order, never array order. Empty for `none`, one
27
+ * member for `one`, two or more for `several`.
28
+ */
29
+ readonly matchedObservationIds: readonly Observation['observationId'][];
30
+ };
31
+ /**
32
+ * Matches `step.operationId` against every observation's `operationId`,
33
+ * ordered by `sequence` ascending. Reads no other field of `step`: whether
34
+ * `several` is the named ambiguity condition or a legitimate `any`-cardinality
35
+ * match is a fact about the step's declared cardinality, decided by whoever
36
+ * reads this result, not by this function.
37
+ *
38
+ * Sorts a copy; the input `observations` array is never mutated, and its own
39
+ * order is never read as meaning anything (NFR9: a permutation of the same
40
+ * observations yields byte-identical `matchedObservationIds`). The schema
41
+ * layer enforces per-record `sequence` uniqueness, so a real `SealedRunRecord`
42
+ * never presents a tie. `observationId` is the secondary sort key regardless:
43
+ * this function's own permutation invariance holds even against a hand-built
44
+ * or malformed `observations` array carrying a duplicate or non-finite
45
+ * `sequence`, decided by this comparator alone.
46
+ *
47
+ * Matching on `operationId` alone, ignoring `step.inputBinding`, is a
48
+ * deliberate scope boundary that stays: candidate-tuple resolution lives in
49
+ * `selectWithBindings` (`score/bindings.ts`), which wraps this function and
50
+ * filters its matches against the step's own resolved bindings. Splitting them
51
+ * keeps this function's permutation guarantee provable on its own.
52
+ *
53
+ * `tests/seal/fixtures.ts`'s `irreducibleCollisionPair` was once cited here as
54
+ * a case "distinguishable only by input binding". It is not: its two steps
55
+ * bind nothing in any channel, so a filter over zero bindings separates
56
+ * nothing and both stay `several` even with `selectWithBindings`. The pair
57
+ * that separates is `literalCollisionPair`, whose two steps bind one key to
58
+ * two different literals.
59
+ */
60
+ export declare function selectObservations(step: InteractionStep, observations: readonly Observation[]): StepSelection;
61
+ /**
62
+ * Resolves an `after` temporal clause to the single observation it denotes,
63
+ * built directly from `selectObservations`'s own result over the anchor.
64
+ *
65
+ * Takes the anchor step itself, already resolved from the dependent step's
66
+ * `after` identifier (a one-line lookup against whichever declared plan the
67
+ * caller holds). A step whose clause is `null`, or whose clause names a step
68
+ * the plan does not declare (AD-39's permissive dangling reference), names no
69
+ * anchor and never reaches this function.
70
+ */
71
+ export declare function resolveTemporalAnchor(anchorStep: InteractionStep, observations: readonly Observation[]): TemporalAnchorResolution;
72
+ export type TemporalAnchorResolution = {
73
+ readonly resolved: true;
74
+ readonly observationId: Observation['observationId'];
75
+ readonly matchedObservationIds: readonly Observation['observationId'][];
76
+ } | {
77
+ readonly resolved: false;
78
+ readonly result: Extract<SelectionCount, 'none' | 'several'>;
79
+ readonly matchedObservationIds: readonly Observation['observationId'][];
80
+ };
@@ -0,0 +1,98 @@
1
+ /**
2
+ * Owed item 2's fix: a step selects observations by `operationId`, ordered by
3
+ * the record's monotonic `sequence` (ADR-006 forbids array position as
4
+ * ordering). `selectObservations` reports a several-match ambiguity as data;
5
+ * no first-match or last-match convention picks a winner. Pure and total:
6
+ * every input produces a defined result, nothing throws, and no AD-6 outcome
7
+ * state is assigned. Assigning outcome state is AD-33's reference decision
8
+ * procedure's job.
9
+ *
10
+ * `resolveTemporalAnchor` is a separate, narrower resolution built on top:
11
+ * picking a concrete point in time for an `any`-cardinality temporal anchor
12
+ * with several matches, by lowest sequence. The policy above still holds
13
+ * here: a single-valued cardinality's several-match ambiguity still comes
14
+ * back unresolved from `resolveTemporalAnchor` too.
15
+ *
16
+ * No stage claims this module yet; `stage-table.ts`'s `score` row stays
17
+ * `module: null` throughout.
18
+ */
19
+ /**
20
+ * Matches `step.operationId` against every observation's `operationId`,
21
+ * ordered by `sequence` ascending. Reads no other field of `step`: whether
22
+ * `several` is the named ambiguity condition or a legitimate `any`-cardinality
23
+ * match is a fact about the step's declared cardinality, decided by whoever
24
+ * reads this result, not by this function.
25
+ *
26
+ * Sorts a copy; the input `observations` array is never mutated, and its own
27
+ * order is never read as meaning anything (NFR9: a permutation of the same
28
+ * observations yields byte-identical `matchedObservationIds`). The schema
29
+ * layer enforces per-record `sequence` uniqueness, so a real `SealedRunRecord`
30
+ * never presents a tie. `observationId` is the secondary sort key regardless:
31
+ * this function's own permutation invariance holds even against a hand-built
32
+ * or malformed `observations` array carrying a duplicate or non-finite
33
+ * `sequence`, decided by this comparator alone.
34
+ *
35
+ * Matching on `operationId` alone, ignoring `step.inputBinding`, is a
36
+ * deliberate scope boundary that stays: candidate-tuple resolution lives in
37
+ * `selectWithBindings` (`score/bindings.ts`), which wraps this function and
38
+ * filters its matches against the step's own resolved bindings. Splitting them
39
+ * keeps this function's permutation guarantee provable on its own.
40
+ *
41
+ * `tests/seal/fixtures.ts`'s `irreducibleCollisionPair` was once cited here as
42
+ * a case "distinguishable only by input binding". It is not: its two steps
43
+ * bind nothing in any channel, so a filter over zero bindings separates
44
+ * nothing and both stay `several` even with `selectWithBindings`. The pair
45
+ * that separates is `literalCollisionPair`, whose two steps bind one key to
46
+ * two different literals.
47
+ */
48
+ export function selectObservations(step, observations) {
49
+ const matched = observations
50
+ .filter((observation) => observation.operationId === step.operationId)
51
+ .sort((a, b) => a.sequence - b.sequence || (a.observationId < b.observationId ? -1 : 1));
52
+ const matchedObservationIds = matched.map((observation) => observation.observationId);
53
+ const result = matchedObservationIds.length === 0
54
+ ? 'none'
55
+ : matchedObservationIds.length === 1
56
+ ? 'one'
57
+ : 'several';
58
+ return { result, matchedObservationIds };
59
+ }
60
+ /**
61
+ * Resolves an `after` temporal clause to the single observation it denotes,
62
+ * built directly from `selectObservations`'s own result over the anchor.
63
+ *
64
+ * Takes the anchor step itself, already resolved from the dependent step's
65
+ * `after` identifier (a one-line lookup against whichever declared plan the
66
+ * caller holds). A step whose clause is `null`, or whose clause names a step
67
+ * the plan does not declare (AD-39's permissive dangling reference), names no
68
+ * anchor and never reaches this function.
69
+ */
70
+ export function resolveTemporalAnchor(anchorStep, observations) {
71
+ const selection = selectObservations(anchorStep, observations);
72
+ if (selection.result === 'one') {
73
+ return {
74
+ resolved: true,
75
+ // `result === 'one'` guarantees exactly one member.
76
+ observationId: selection
77
+ .matchedObservationIds[0],
78
+ matchedObservationIds: selection.matchedObservationIds,
79
+ };
80
+ }
81
+ if (selection.result === 'several' && anchorStep.cardinality === 'any') {
82
+ // Ascending-`sequence` order already holds the lowest-sequence match first.
83
+ return {
84
+ resolved: true,
85
+ observationId: selection
86
+ .matchedObservationIds[0],
87
+ matchedObservationIds: selection.matchedObservationIds,
88
+ };
89
+ }
90
+ // `none`, or `several` under a single-valued cardinality: no single
91
+ // observation resolves. Reported as data; routing this to a verdict rung
92
+ // is later work.
93
+ return {
94
+ resolved: false,
95
+ result: selection.result,
96
+ matchedObservationIds: selection.matchedObservationIds,
97
+ };
98
+ }
@@ -0,0 +1,58 @@
1
+ /**
2
+ * AD-7's rate vector and its four-valued dominance relation.
3
+ *
4
+ * `buildStrengthVector` is a pure aggregation over a qualified probe set and
5
+ * the trial-set reducer's per-probe results: unweighted, per probe class,
6
+ * unique qualified probe identifiers over unique qualified probe identifiers
7
+ * exercised, with canary probes and clean controls excluded regardless of
8
+ * class or trial outcome. `compareDominance` takes two already-computed
9
+ * results and never re-derives a vector, reads a port, a corpus, or a clock;
10
+ * comparability is checked first, and the severity-floor override can only
11
+ * push the relation toward `incomparable`.
12
+ */
13
+ import { type Severity } from '../schemas/eval-contract.ts';
14
+ import type { Outcome, Strength, StrengthVector } from '../schemas/evidence-artifact.ts';
15
+ import type { QualifiedProbe } from './qualification.ts';
16
+ import type { TrialSetResult } from './reduce-trials.ts';
17
+ export declare const DOMINANCE_RELATIONS: readonly ['a-dominates-b', 'b-dominates-a', 'equivalent', 'incomparable'];
18
+ export type DominanceRelationValue = (typeof DOMINANCE_RELATIONS)[number];
19
+ /**
20
+ * The slice the dominance comparator reads: the aggregate `Strength` plus the
21
+ * per-probe `outcomes` array the severity-floor override needs, since that
22
+ * identity is lost once probes are aggregated into `ClassStrength` counts,
23
+ * and the `comparabilityKey` the comparator checks before comparing anything
24
+ * else. Every field already lives on `EvidenceArtifact`; this is the read
25
+ * projection the comparator needs from it, not a new artifact shape.
26
+ */
27
+ export type ComparableResult = {
28
+ readonly outcomes: readonly Outcome[];
29
+ readonly strength: Strength;
30
+ readonly comparabilityKey: string;
31
+ };
32
+ /**
33
+ * AD-7's rate vector: per probe class, the catch rate over unique qualified
34
+ * probe identifiers, with raw counts alongside. `admitted` carries each
35
+ * probe's identifier once by construction, so grouping by class needs no
36
+ * deduplication of its own.
37
+ */
38
+ export declare function buildStrengthVector(admitted: readonly QualifiedProbe[], results: ReadonlyMap<string, TrialSetResult>): StrengthVector;
39
+ /**
40
+ * AD-7's four-valued dominance relation. `comparabilityKey` and each side's
41
+ * own `strength.comparable` are checked before any component-wise comparison
42
+ * runs: a key mismatch means the two runs are not measuring a shared probe
43
+ * set, and `comparable: false` means AD-21 already marked that one side's own
44
+ * vector as thinner than the policy's declared minimum, so a `caught`/`rate`
45
+ * on it is not fit to decide a comparison either way. The severity-floor
46
+ * override runs only against the side the raw comparison favoured, and only
47
+ * ever downgrades that result to `incomparable`.
48
+ *
49
+ * That scope is AD-7's own and not an omission: its words are that a contract
50
+ * missing a behaviour at or above the floor "never dominates" one that caught
51
+ * it, which constrains dominance and says nothing about equivalence. Two
52
+ * vectors that are component-wise equal are `equivalent` whatever their
53
+ * severities, because neither is dominating anything for the override to
54
+ * withdraw. Widening it to `equivalent` would be a new rule rather than this
55
+ * one applied more thoroughly, and it is written down here so the asymmetry
56
+ * reads as a decision rather than a gap.
57
+ */
58
+ export declare function compareDominance(a: ComparableResult, b: ComparableResult, severityFloor: Severity): DominanceRelationValue;
@@ -0,0 +1,227 @@
1
+ /**
2
+ * AD-7's rate vector and its four-valued dominance relation.
3
+ *
4
+ * `buildStrengthVector` is a pure aggregation over a qualified probe set and
5
+ * the trial-set reducer's per-probe results: unweighted, per probe class,
6
+ * unique qualified probe identifiers over unique qualified probe identifiers
7
+ * exercised, with canary probes and clean controls excluded regardless of
8
+ * class or trial outcome. `compareDominance` takes two already-computed
9
+ * results and never re-derives a vector, reads a port, a corpus, or a clock;
10
+ * comparability is checked first, and the severity-floor override can only
11
+ * push the relation toward `incomparable`.
12
+ */
13
+ import { SEVERITY_LEVELS } from '../schemas/eval-contract.js';
14
+ export const DOMINANCE_RELATIONS = [
15
+ 'a-dominates-b',
16
+ 'b-dominates-a',
17
+ 'equivalent',
18
+ 'incomparable',
19
+ ];
20
+ const STRENGTH_VECTOR_CLASSES = [
21
+ 'defect',
22
+ 'gameability',
23
+ 'zero-action',
24
+ ];
25
+ /**
26
+ * `admitted`, after excluding `canary` and every `expectedClean: true` probe:
27
+ * AD-7's "canary probes and clean controls never enter the vector" applies
28
+ * regardless of class or trial outcome, and a canary carries `expectedClean:
29
+ * false` on its own schema branch, so both conditions are checked.
30
+ */
31
+ const vectorEligible = (admitted) => admitted.filter(({ probe }) => probe.probeClass !== 'canary' && !probe.expectedClean);
32
+ /**
33
+ * One class's aggregate, or `null` when the eligible set admits no probe of
34
+ * that class. A probe with no `TrialSetResult`, or one that is `exercised:
35
+ * false`, contributes to neither `caught` nor `exercised`, matching the
36
+ * reducer's own "zero valid trials excludes a probe entirely" rule. A class
37
+ * with admitted probes but zero exercised ones is still a present
38
+ * `ClassStrength` of `{ caught: 0, exercised: 0, rate: null }`, never a
39
+ * `null` class: `rate`'s nullability exists specifically for that case, and
40
+ * collapsing the whole class to `null` would make it unobservable.
41
+ */
42
+ const classStrengthOf = (probesInClass, results) => {
43
+ if (probesInClass.length === 0)
44
+ return null;
45
+ let exercised = 0;
46
+ let caught = 0;
47
+ // Counted once per identifier, not once per entry. AD-7's rate is over
48
+ // unique qualified probe identifiers, and nothing in `src/` enforces that
49
+ // `admitted` carries each identifier once: no probe-corpus schema exists,
50
+ // so the uniqueness is inherited from upstream qualification rather than
51
+ // checked. A repeated identifier would otherwise count its trial-set
52
+ // result twice on both sides of the same ratio, which leaves the rate
53
+ // right and the raw counts wrong.
54
+ const counted = new Set();
55
+ for (const { probe } of probesInClass) {
56
+ if (counted.has(probe.probeId))
57
+ continue;
58
+ counted.add(probe.probeId);
59
+ const result = results.get(probe.probeId);
60
+ if (result === undefined || !result.exercised)
61
+ continue;
62
+ exercised += 1;
63
+ if (result.caught)
64
+ caught += 1;
65
+ }
66
+ return {
67
+ exercised,
68
+ caught,
69
+ rate: exercised === 0 ? null : caught / exercised,
70
+ };
71
+ };
72
+ /**
73
+ * AD-7's rate vector: per probe class, the catch rate over unique qualified
74
+ * probe identifiers, with raw counts alongside. `admitted` carries each
75
+ * probe's identifier once by construction, so grouping by class needs no
76
+ * deduplication of its own.
77
+ */
78
+ export function buildStrengthVector(admitted, results) {
79
+ const eligible = vectorEligible(admitted);
80
+ const byClass = Object.fromEntries(STRENGTH_VECTOR_CLASSES.map((probeClass) => [
81
+ probeClass,
82
+ classStrengthOf(eligible.filter(({ probe }) => probe.probeClass === probeClass), results),
83
+ ]));
84
+ return {
85
+ defect: byClass.defect,
86
+ gameability: byClass.gameability,
87
+ 'zero-action': byClass['zero-action'],
88
+ };
89
+ }
90
+ /**
91
+ * A class contributes to the comparison only when it is a non-null
92
+ * `ClassStrength` on both sides and both sides' `rate` is also non-null; a
93
+ * class absent on either side, or present with a `null` rate on either side,
94
+ * carries no comparative evidence and is skipped exactly alike. `equivalent`
95
+ * compares `caught` and `exercised` rather than the derived `rate`, avoiding
96
+ * a floating-point equality check.
97
+ *
98
+ * A class whose two sides tie on `rate` while disagreeing on `caught` or
99
+ * `exercised` contributes to the comparison, blocks `equivalent` (the counts
100
+ * are not equal), and hands neither side a win (neither `rate` is strictly
101
+ * greater). That third possibility has no named outcome of its own in AD-7's
102
+ * three stated cases, and `incomparable` is where a tied vector with no
103
+ * winner on either side belongs: the same value the "no class contributes at
104
+ * all" case already returns.
105
+ */
106
+ function componentComparison(a, b) {
107
+ let aWinsAClass = false;
108
+ let bWinsAClass = false;
109
+ let everyContributingClassEqual = true;
110
+ let contributingClasses = 0;
111
+ for (const key of STRENGTH_VECTOR_CLASSES) {
112
+ const left = a[key];
113
+ const right = b[key];
114
+ if (left === null || right === null)
115
+ continue;
116
+ if (left.rate === null || right.rate === null)
117
+ continue;
118
+ contributingClasses += 1;
119
+ if (left.caught !== right.caught || left.exercised !== right.exercised) {
120
+ everyContributingClassEqual = false;
121
+ }
122
+ if (left.rate > right.rate)
123
+ aWinsAClass = true;
124
+ if (right.rate > left.rate)
125
+ bWinsAClass = true;
126
+ }
127
+ if (contributingClasses === 0)
128
+ return 'incomparable';
129
+ if (everyContributingClassEqual)
130
+ return 'equivalent';
131
+ if (aWinsAClass && !bWinsAClass)
132
+ return 'a-dominates-b';
133
+ if (bWinsAClass && !aWinsAClass)
134
+ return 'b-dominates-a';
135
+ return 'incomparable';
136
+ }
137
+ /**
138
+ * Both operands are looked up before they are compared, because `indexOf`
139
+ * answers `-1` for a value the ladder does not name and `-1 >= -1` reads as
140
+ * "at or above the floor" for two values that are on no ladder at all. A
141
+ * severity outside the closed set is not at or above anything, and a floor
142
+ * outside it bounds nothing, so either one absent is `false` rather than a
143
+ * comparison of two absences.
144
+ */
145
+ const atOrAboveFloor = (severity, floor) => {
146
+ const rank = SEVERITY_LEVELS.indexOf(severity);
147
+ const bound = SEVERITY_LEVELS.indexOf(floor);
148
+ if (rank < 0 || bound < 0)
149
+ return false;
150
+ return rank >= bound;
151
+ };
152
+ /**
153
+ * Keyed by the first outcome carrying each `probeId`, not the last: two
154
+ * `Outcome` entries sharing one `probeId` is itself a defect somewhere
155
+ * upstream (this map has no way to tell which entry is the real one), and a
156
+ * silent last-write-wins overwrite would drop the earlier entry from the
157
+ * severity-floor scan below with no trace it was ever there.
158
+ */
159
+ const outcomesByProbeId = (outcomes) => {
160
+ const byProbeId = new Map();
161
+ for (const outcome of outcomes) {
162
+ if (outcome.probeId === null)
163
+ continue;
164
+ if (byProbeId.has(outcome.probeId))
165
+ continue;
166
+ byProbeId.set(outcome.probeId, outcome);
167
+ }
168
+ return byProbeId;
169
+ };
170
+ /**
171
+ * Whether `favored` failed to catch a probe that `other` caught at or above
172
+ * `severityFloor`: the condition that disqualifies `favored` from dominating,
173
+ * per AD-7's "a contract that missed a behaviour at or above the scoring
174
+ * policy's severity floor never dominates one that caught it, regardless of
175
+ * the rest of the vector". An outcome with `probeId: null` is not tied to any
176
+ * probe and is outside the vector entirely, so both sides skip it.
177
+ */
178
+ function favoredMissesWhatOtherCaught(favored, other, severityFloor) {
179
+ const favoredByProbeId = outcomesByProbeId(favored.outcomes);
180
+ const otherByProbeId = outcomesByProbeId(other.outcomes);
181
+ for (const [probeId, otherOutcome] of otherByProbeId) {
182
+ if (otherOutcome.state !== 'caught')
183
+ continue;
184
+ if (!atOrAboveFloor(otherOutcome.severity, severityFloor))
185
+ continue;
186
+ const favoredOutcome = favoredByProbeId.get(probeId);
187
+ if (favoredOutcome === undefined || favoredOutcome.state !== 'caught') {
188
+ return true;
189
+ }
190
+ }
191
+ return false;
192
+ }
193
+ /**
194
+ * AD-7's four-valued dominance relation. `comparabilityKey` and each side's
195
+ * own `strength.comparable` are checked before any component-wise comparison
196
+ * runs: a key mismatch means the two runs are not measuring a shared probe
197
+ * set, and `comparable: false` means AD-21 already marked that one side's own
198
+ * vector as thinner than the policy's declared minimum, so a `caught`/`rate`
199
+ * on it is not fit to decide a comparison either way. The severity-floor
200
+ * override runs only against the side the raw comparison favoured, and only
201
+ * ever downgrades that result to `incomparable`.
202
+ *
203
+ * That scope is AD-7's own and not an omission: its words are that a contract
204
+ * missing a behaviour at or above the floor "never dominates" one that caught
205
+ * it, which constrains dominance and says nothing about equivalence. Two
206
+ * vectors that are component-wise equal are `equivalent` whatever their
207
+ * severities, because neither is dominating anything for the override to
208
+ * withdraw. Widening it to `equivalent` would be a new rule rather than this
209
+ * one applied more thoroughly, and it is written down here so the asymmetry
210
+ * reads as a decision rather than a gap.
211
+ */
212
+ export function compareDominance(a, b, severityFloor) {
213
+ if (a.comparabilityKey !== b.comparabilityKey)
214
+ return 'incomparable';
215
+ if (!a.strength.comparable || !b.strength.comparable)
216
+ return 'incomparable';
217
+ const raw = componentComparison(a.strength.vector, b.strength.vector);
218
+ if (raw === 'a-dominates-b' &&
219
+ favoredMissesWhatOtherCaught(a, b, severityFloor)) {
220
+ return 'incomparable';
221
+ }
222
+ if (raw === 'b-dominates-a' &&
223
+ favoredMissesWhatOtherCaught(b, a, severityFloor)) {
224
+ return 'incomparable';
225
+ }
226
+ return raw;
227
+ }
@@ -0,0 +1,138 @@
1
+ /**
2
+ * AD-40's witness match: whether a finding cited against a probe actually
3
+ * witnessed the defect that probe seeded.
4
+ *
5
+ * This is the input that makes non-detection reachable. Without it, an oracle
6
+ * that correctly confirmed an untouched behaviour and one that failed to detect
7
+ * the seeded defect present identical inputs, so any table obeying AD-3 must
8
+ * give both the same answer and the catch rate is 1.00 by construction.
9
+ *
10
+ * Pure and total on its ordinary path, and no AD-6 outcome state is assigned
11
+ * here. The result is a fact about evidence; turning it into an outcome state
12
+ * is the decision procedure's.
13
+ *
14
+ * One deliberate non-totality: a `RuntimeFault` from the shipped evaluator
15
+ * propagates undecorated. A fault never becomes a verdict, and AD-28 makes it
16
+ * an invalidating condition under AD-21 rather than a behavioural result, so
17
+ * catching one here would convert AD-21's fault exit into a scored run. The
18
+ * qualification gate rejects every operand class that makes the evaluator's
19
+ * four plain-`Error` sites reachable, which leaves exactly two data-dependent
20
+ * `RuntimeFault`s: an exhausted regex budget and a non-canonicalizable value.
21
+ */
22
+ import type { DefectSignature } from '../schemas/defect-signature.ts';
23
+ import type { PermittedInterface } from '../schemas/interface.ts';
24
+ import type { Probe } from '../schemas/probe.ts';
25
+ import type { SealedRunRecord } from '../schemas/sealed-run-record.ts';
26
+ /**
27
+ * A module constant, following pre-flight's precedent for the same problem:
28
+ * `resolveCheck`'s budget parameter is a bare required number whose declared
29
+ * home is the scoring policy, and a probe is not scored under a policy at match
30
+ * time. Two implementations choosing two budgets would disagree about when the
31
+ * evaluator reports an exhausted budget, which is the disagreement one shared
32
+ * constant removes.
33
+ */
34
+ export declare const PROBE_REGEX_MATCH_STEP_BUDGET = 1000000;
35
+ /**
36
+ * The six results, declared in the evaluation order the match applies, so a
37
+ * reader cannot mistake this constant for a different ordering.
38
+ *
39
+ * - `unexercised`: the evaluator never invoked the home operation.
40
+ * - `unwitnessed-claim`: a defect finding cited this probe and cited home-operation
41
+ * observations, none of which satisfies the condition. An AD-32
42
+ * declared-versus-observed inconsistency, and it outranks a competing
43
+ * `matched` so a broken reporter cannot mask it.
44
+ * - `matched`: a defect finding cited an observation the condition satisfies.
45
+ * - `manifested-unclaimed`: the condition was satisfied and no finding claimed
46
+ * it. This is what makes non-detection reachable.
47
+ * - `not-triggered`: the system was examined and did not manifest the seeded
48
+ * defect. AD-6 calls this the common case on any defect probe. Named for what
49
+ * it is rather than "not manifested", because AD-40 already uses that phrase
50
+ * for the vacuous case and the two route to opposite verdicts.
51
+ * - `vacuous`: every candidate resolved insufficient-evidence, so the corpus
52
+ * presented no defect to detect. A fact about the instrument.
53
+ */
54
+ export declare const PROBE_WITNESS_RESULTS: readonly ['unexercised', 'unwitnessed-claim', 'matched', 'manifested-unclaimed', 'not-triggered', 'vacuous'];
55
+ export type ProbeWitnessResultValue = (typeof PROBE_WITNESS_RESULTS)[number];
56
+ /** A probe on the seeding branch whose signature is present. */
57
+ export type SignedProbe = Extract<Probe, {
58
+ expectedClean: false;
59
+ }> & {
60
+ readonly defectSignature: DefectSignature;
61
+ };
62
+ export type WitnessPartition = {
63
+ /** the candidates the condition resolved `true` over. */
64
+ readonly satisfying: readonly string[];
65
+ /** the candidates it resolved `false` over: the system examined and behaving. */
66
+ readonly refuting: readonly string[];
67
+ /** the candidates it could not examine. Never detection, and never manifestation. */
68
+ readonly inconclusive: readonly string[];
69
+ };
70
+ export type ProbeWitnessMatch = {
71
+ readonly result: ProbeWitnessResultValue;
72
+ /**
73
+ * AD-40 forbids pooling a quotation-reconstructed detection with a measured
74
+ * catch rate, so every result says which it is. The witness match resolves
75
+ * over cited identifiers and is always `measured`.
76
+ */
77
+ readonly basis: 'measured';
78
+ readonly homeOperationResolved: boolean;
79
+ readonly exercised: boolean;
80
+ /** every candidate identifier the match read, ascending by `sequence`. */
81
+ readonly observationIds: readonly string[];
82
+ readonly partition: WitnessPartition;
83
+ readonly partitionSizes: {
84
+ readonly satisfying: number;
85
+ readonly refuting: number;
86
+ readonly inconclusive: number;
87
+ };
88
+ /** the satisfying observations a defect finding actually cited. */
89
+ readonly witnessObservationIds: readonly string[];
90
+ /** the defect findings whose home-operation citations witness nothing. */
91
+ readonly unwitnessedFindingIds: readonly string[];
92
+ };
93
+ /**
94
+ * AD-40's deterministic witness match over one probe and one sealed run record.
95
+ *
96
+ * The candidates are partitioned by AD-4 resolution into satisfying, refuting,
97
+ * and inconclusive, and the result is read off that partition rather than off a
98
+ * precedence list. A partition is exhaustive and totally ordered over the whole
99
+ * input space; a precedence list left mixed refuting/inconclusive with no home
100
+ * and let an empty candidate set satisfy two rows that route to opposite
101
+ * verdicts.
102
+ *
103
+ * The verdict reads cited identifiers alone and never quotation. Quoted evidence
104
+ * that appears in no cited observation is a separate audit, and its invalidation
105
+ * belongs to ingest.
106
+ *
107
+ * No output reads an array's position. Observation identifiers follow the
108
+ * record's `sequence`; finding identifiers are sorted by identifier, because
109
+ * `findings` carries no ordering field of its own.
110
+ */
111
+ export declare function matchProbeWitness(probe: SignedProbe, interfaces: readonly PermittedInterface[], record: Pick<SealedRunRecord, 'observations' | 'findings'>): ProbeWitnessMatch;
112
+ export type MappedFinding = {
113
+ readonly findingId: string;
114
+ readonly probeId: string;
115
+ };
116
+ export type FindingMap = {
117
+ /** cited to a signed probe, and touching that signature's home operation. */
118
+ readonly mapped: readonly MappedFinding[];
119
+ /**
120
+ * cited to a signed probe whose home operation the cited observations never
121
+ * touch. An unexpected real defect under AD-23, never a catch.
122
+ */
123
+ readonly unmapped: readonly MappedFinding[];
124
+ /** cited to a probe identifier the set does not declare: an AD-32 cross-artifact dangling reference. */
125
+ readonly dangling: readonly MappedFinding[];
126
+ /** cited to a canary or a clean control, which carry no signature to map against. */
127
+ readonly signatureless: readonly MappedFinding[];
128
+ };
129
+ /**
130
+ * Sorts every defect finding in the record into the four buckets AD-40 and
131
+ * AD-23 between them require, so no finding is silently dropped and none is
132
+ * counted as a catch it did not earn.
133
+ *
134
+ * Defect findings only: they are the only type that enters a detection measure.
135
+ * The uncited case proper, a finding naming no oracle, is a different rule and
136
+ * a different record.
137
+ */
138
+ export declare function mapFindings(probes: readonly Probe[], interfaces: readonly PermittedInterface[], record: Pick<SealedRunRecord, 'observations' | 'findings'>): FindingMap;