eval-quality 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. package/README.md +142 -67
  2. package/corpus/dev/README.md +17 -11
  3. package/corpus/dev/compile-seal-example/brief.json +1 -1
  4. package/corpus/dev/compile-seal-example/contract.json +1 -1
  5. package/corpus/dev/contracts/absent-collection-locations.json +1 -1
  6. package/corpus/dev/contracts/absent-sibling-groups.json +1 -1
  7. package/corpus/dev/contracts/absent-success-indicator.json +1 -1
  8. package/corpus/dev/contracts/empty-channel-roles.json +1 -1
  9. package/corpus/dev/contracts/empty-collection-locations.json +1 -1
  10. package/corpus/dev/contracts/empty-request-shapes.json +1 -1
  11. package/corpus/dev/contracts/empty-sibling-groups.json +1 -1
  12. package/corpus/dev/contracts/no-collection-quantifier.json +1 -1
  13. package/corpus/dev/contracts/no-operation-inventory.json +1 -1
  14. package/corpus/dev/contracts/no-read-back-relation.json +1 -1
  15. package/corpus/dev/contracts/no-state-change-marker.json +1 -1
  16. package/corpus/dev/contracts/no-type-violating-step.json +1 -1
  17. package/corpus/dev/contracts/per-key-split-oracles.json +1 -1
  18. package/corpus/dev/contracts/satisfied-declarations.json +1 -1
  19. package/corpus/dev/contracts/single-required-response-key.json +1 -1
  20. package/corpus/dev/contracts/split-indicator-oracle.json +1 -1
  21. package/corpus/dev/contracts/unaddressed-parameter-sibling.json +1 -1
  22. package/corpus/dev/contracts/unnamed-reference-set.json +1 -1
  23. package/corpus/dev/contracts/wrong-cardinality-form.json +1 -1
  24. package/corpus/dev/index.json +1 -1
  25. package/dist/application/index.d.ts +2 -0
  26. package/dist/application/index.js +1 -0
  27. package/dist/application/score.d.ts +48 -0
  28. package/dist/application/score.js +186 -0
  29. package/dist/cli/arguments.d.ts +7 -3
  30. package/dist/cli/arguments.js +73 -5
  31. package/dist/cli/exit-codes.d.ts +6 -2
  32. package/dist/cli/exit-codes.js +8 -9
  33. package/dist/cli/main.js +9 -0
  34. package/dist/cli/render.d.ts +1 -1
  35. package/dist/cli/render.js +6 -4
  36. package/dist/cli/run.d.ts +13 -3
  37. package/dist/cli/run.js +136 -16
  38. package/dist/core/canonical/canonicalize.d.ts +12 -0
  39. package/dist/core/canonical/canonicalize.js +18 -4
  40. package/dist/core/compile/bindings.d.ts +70 -0
  41. package/dist/core/compile/bindings.js +317 -0
  42. package/dist/core/compile/compile.d.ts +5 -0
  43. package/dist/core/compile/compile.js +9 -0
  44. package/dist/core/compile/expression-legality.d.ts +49 -0
  45. package/dist/core/compile/expression-legality.js +105 -31
  46. package/dist/core/compile/forbidden-inputs.d.ts +8 -1
  47. package/dist/core/compile/forbidden-inputs.js +16 -3
  48. package/dist/core/compile/interface-inventory.d.ts +23 -1
  49. package/dist/core/compile/interface-inventory.js +32 -3
  50. package/dist/core/compile/reachability.d.ts +18 -1
  51. package/dist/core/compile/reachability.js +46 -1
  52. package/dist/core/compile/sensitivity-witness.d.ts +1 -1
  53. package/dist/core/compile/sensitivity-witness.js +1 -1
  54. package/dist/core/emit/emit.d.ts +9 -0
  55. package/dist/core/emit/emit.js +214 -0
  56. package/dist/core/emit/private-artifact-digest.d.ts +11 -0
  57. package/dist/core/emit/private-artifact-digest.js +31 -0
  58. package/dist/core/evaluate/evidence-resolution.d.ts +11 -1
  59. package/dist/core/evaluate/evidence-resolution.js +4 -1
  60. package/dist/core/failure-codes.d.ts +2 -2
  61. package/dist/core/failure-codes.js +4 -2
  62. package/dist/core/ingest/conditions.d.ts +201 -0
  63. package/dist/core/ingest/conditions.js +48 -0
  64. package/dist/core/ingest/index.d.ts +13 -0
  65. package/dist/core/ingest/index.js +11 -0
  66. package/dist/core/ingest/ingest.d.ts +68 -0
  67. package/dist/core/ingest/ingest.js +310 -0
  68. package/dist/core/lineage/stage-table.d.ts +21 -4
  69. package/dist/core/lineage/stage-table.js +35 -3
  70. package/dist/core/preflight/witness-evidence.js +10 -0
  71. package/dist/core/schemas/artifact.d.ts +385 -14
  72. package/dist/core/schemas/constraint-ledger.js +40 -0
  73. package/dist/core/schemas/defect-signature.d.ts +273 -0
  74. package/dist/core/schemas/defect-signature.js +127 -0
  75. package/dist/core/schemas/eval-contract.d.ts +33 -0
  76. package/dist/core/schemas/eval-contract.js +14 -0
  77. package/dist/core/schemas/evidence-artifact.d.ts +99 -19
  78. package/dist/core/schemas/evidence-artifact.js +64 -8
  79. package/dist/core/schemas/plan.d.ts +112 -0
  80. package/dist/core/schemas/plan.js +67 -2
  81. package/dist/core/schemas/probe-qualification.d.ts +168 -0
  82. package/dist/core/schemas/probe-qualification.js +106 -0
  83. package/dist/core/schemas/probe.d.ts +312 -0
  84. package/dist/core/schemas/probe.js +6 -2
  85. package/dist/core/schemas/scoring-policy.d.ts +1 -0
  86. package/dist/core/schemas/scoring-policy.js +6 -1
  87. package/dist/core/schemas/sealed-evaluator-brief.d.ts +1 -0
  88. package/dist/core/schemas/sealed-evaluator-brief.js +4 -1
  89. package/dist/core/schemas/sealed-run-record.d.ts +30 -3
  90. package/dist/core/schemas/sealed-run-record.js +37 -5
  91. package/dist/core/score/binding-order.d.ts +31 -0
  92. package/dist/core/score/binding-order.js +88 -0
  93. package/dist/core/score/bindings.d.ts +93 -0
  94. package/dist/core/score/bindings.js +297 -0
  95. package/dist/core/score/ladder-table.d.ts +23 -0
  96. package/dist/core/score/ladder-table.js +109 -0
  97. package/dist/core/score/ladder.d.ts +229 -0
  98. package/dist/core/score/ladder.js +464 -0
  99. package/dist/core/score/mode-agreement.d.ts +30 -0
  100. package/dist/core/score/mode-agreement.js +16 -0
  101. package/dist/core/score/outcome-table.d.ts +17 -0
  102. package/dist/core/score/outcome-table.js +172 -0
  103. package/dist/core/score/outcome.d.ts +466 -0
  104. package/dist/core/score/outcome.js +562 -0
  105. package/dist/core/score/qualification.d.ts +83 -0
  106. package/dist/core/score/qualification.js +615 -0
  107. package/dist/core/score/quotation.d.ts +79 -0
  108. package/dist/core/score/quotation.js +201 -0
  109. package/dist/core/score/reduce-trials.d.ts +80 -0
  110. package/dist/core/score/reduce-trials.js +90 -0
  111. package/dist/core/score/score.d.ts +72 -0
  112. package/dist/core/score/score.js +581 -0
  113. package/dist/core/score/selection.d.ts +80 -0
  114. package/dist/core/score/selection.js +98 -0
  115. package/dist/core/score/strength.d.ts +49 -0
  116. package/dist/core/score/strength.js +193 -0
  117. package/dist/core/score/witness.d.ts +117 -0
  118. package/dist/core/score/witness.js +315 -0
  119. package/dist/core/seal/derived-reference.d.ts +1 -1
  120. package/dist/core/seal/derived-reference.js +236 -28
  121. package/dist/core/seal/seal.js +8 -3
  122. package/dist/core/stage-contracts.d.ts +62 -2
  123. package/dist/index.d.ts +1 -1
  124. package/dist/index.js +1 -1
  125. package/dist/testing/index.d.ts +2 -0
  126. package/dist/testing/index.js +1 -0
  127. package/package.json +12 -3
  128. package/schemas/eval-contract.schema.json +100 -2
  129. package/schemas/evidence-artifact.schema.json +96 -9
  130. package/schemas/probe.schema.json +508 -3
  131. package/schemas/scoring-policy.schema.json +8 -1
  132. package/schemas/sealed-evaluator-brief.schema.json +11 -1
  133. package/schemas/sealed-run-record.schema.json +19 -2
@@ -1,9 +1,10 @@
1
1
  /**
2
2
  * AD-24's stage-signature table and AD-29's producer map, as data: six stages
3
- * with their inputs, their one owned output, and the lineage edge each writes;
4
- * twelve interchange artifacts with one producer apiece. The `check:lineage`
5
- * scanner derives its allowlist from `module`, so the registry import stays
6
- * type-only to keep zod off that gate's load path.
3
+ * with their artifact inputs, the non-artifact values they receive, their one
4
+ * owned output, and the lineage edge each writes; twelve interchange artifacts
5
+ * with one producer apiece. The `check:lineage` scanner derives its allowlist
6
+ * from `module`, so the registry import stays type-only to keep zod off that
7
+ * gate's load path.
7
8
  */
8
9
  import type { InterchangeArtifactKey } from '../schemas/artifact.ts';
9
10
  /** AD-24's six stages, in the order the Consistency Conventions list them. */
@@ -16,6 +17,20 @@ export type PipelineStage = (typeof PIPELINE_STAGES)[number];
16
17
  */
17
18
  export declare const INTERNAL_PRODUCTS: readonly ['probe-plan', 'probe-observations', 'validated-observations', 'scored-outcomes-and-verdict'];
18
19
  export type InternalProduct = (typeof INTERNAL_PRODUCTS)[number];
20
+ /**
21
+ * Values a stage reads at its boundary that the `inputs` column does not name.
22
+ * AD-24 writes the table in artifacts, and owed item 6 records the source of
23
+ * run mode as absent from it: a row naming `sealed-run-record` says which bytes
24
+ * arrive and leaves open which value fixes the run's mode. Closed rather than
25
+ * free text, so the column cannot become a notes field.
26
+ *
27
+ * The three digests join `mode` for the same reason: they are exactly what
28
+ * this column exists for, "values a stage reads at its boundary that the
29
+ * inputs column does not name" -- `emit`'s three caller-attested AD-11
30
+ * digests have no artifact source anywhere in this pipeline.
31
+ */
32
+ export declare const STAGE_VALUE_INPUTS: readonly ['mode', 'corpusDigest', 'fixtureDigest', 'evaluatorConfigurationDigest'];
33
+ export type StageValueInput = (typeof STAGE_VALUE_INPUTS)[number];
19
34
  /**
20
35
  * What a stage does to the two AD-29 fields on the artifact it owns. `mints`
21
36
  * covers a root and a revision alike, since both write the fields.
@@ -25,6 +40,8 @@ export type LineageEdge = 'mints' | 'carries-through' | 'none';
25
40
  export type ArtifactProducer = PipelineStage | 'caller' | 'embedded';
26
41
  export type StageSignature = {
27
42
  readonly inputs: readonly (InterchangeArtifactKey | InternalProduct)[];
43
+ /** non-artifact values the stage receives; see STAGE_VALUE_INPUTS. */
44
+ readonly valueInputs: readonly StageValueInput[];
28
45
  readonly owns: InterchangeArtifactKey | InternalProduct;
29
46
  /** the owned output's registry key; null when the output is internal. */
30
47
  readonly ownsInterchange: InterchangeArtifactKey | null;
@@ -18,9 +18,28 @@ export const INTERNAL_PRODUCTS = [
18
18
  'validated-observations',
19
19
  'scored-outcomes-and-verdict',
20
20
  ];
21
+ /**
22
+ * Values a stage reads at its boundary that the `inputs` column does not name.
23
+ * AD-24 writes the table in artifacts, and owed item 6 records the source of
24
+ * run mode as absent from it: a row naming `sealed-run-record` says which bytes
25
+ * arrive and leaves open which value fixes the run's mode. Closed rather than
26
+ * free text, so the column cannot become a notes field.
27
+ *
28
+ * The three digests join `mode` for the same reason: they are exactly what
29
+ * this column exists for, "values a stage reads at its boundary that the
30
+ * inputs column does not name" -- `emit`'s three caller-attested AD-11
31
+ * digests have no artifact source anywhere in this pipeline.
32
+ */
33
+ export const STAGE_VALUE_INPUTS = [
34
+ 'mode',
35
+ 'corpusDigest',
36
+ 'fixtureDigest',
37
+ 'evaluatorConfigurationDigest',
38
+ ];
21
39
  export const STAGE_SIGNATURES = {
22
40
  compile: {
23
41
  inputs: ['eval-contract'],
42
+ valueInputs: [],
24
43
  owns: 'eval-contract',
25
44
  ownsInterchange: 'eval-contract',
26
45
  // The caller authors the contract and its lineage; `compile` validates
@@ -31,6 +50,7 @@ export const STAGE_SIGNATURES = {
31
50
  },
32
51
  seal: {
33
52
  inputs: ['eval-contract'],
53
+ valueInputs: [],
34
54
  owns: 'sealed-evaluator-brief',
35
55
  ownsInterchange: 'sealed-evaluator-brief',
36
56
  lineage: 'mints',
@@ -42,15 +62,21 @@ export const STAGE_SIGNATURES = {
42
62
  'isolation-manifest',
43
63
  'evaluator-configuration',
44
64
  ],
65
+ // Mode arrives on the sealed run record and is named here because owed
66
+ // item 6 asks the table for the source of run mode. `ingest` is the last
67
+ // stage that may read it from the caller and the first that may reject
68
+ // its absence; no later stage derives, recomputes, or defaults it.
69
+ valueInputs: ['mode'],
45
70
  owns: 'validated-observations',
46
71
  ownsInterchange: null,
47
72
  lineage: 'none',
48
- module: null,
73
+ module: 'src/core/ingest/ingest.ts',
49
74
  },
50
75
  preflight: {
51
76
  // AD-34 splits the stage into `plan` and `reduce`; both halves' inputs
52
77
  // are the stage's inputs, and only `reduce` returns an artifact.
53
78
  inputs: ['eval-contract', 'probe', 'probe-plan', 'probe-observations'],
79
+ valueInputs: [],
54
80
  owns: 'preflight-verdict',
55
81
  ownsInterchange: 'preflight-verdict',
56
82
  lineage: 'mints',
@@ -64,19 +90,25 @@ export const STAGE_SIGNATURES = {
64
90
  'preflight-verdict',
65
91
  'scoring-policy',
66
92
  ],
93
+ valueInputs: [],
67
94
  // AD-24: "score produces the outcome and verdict values emit
68
95
  // serializes". Owed item 6 records that type as unnamed; this names it.
69
96
  owns: 'scored-outcomes-and-verdict',
70
97
  ownsInterchange: null,
71
98
  lineage: 'none',
72
- module: null,
99
+ module: 'src/core/score/score.ts',
73
100
  },
74
101
  emit: {
75
102
  inputs: ['scored-outcomes-and-verdict'],
103
+ valueInputs: [
104
+ 'corpusDigest',
105
+ 'fixtureDigest',
106
+ 'evaluatorConfigurationDigest',
107
+ ],
76
108
  owns: 'evidence-artifact',
77
109
  ownsInterchange: 'evidence-artifact',
78
110
  lineage: 'mints',
79
- module: null,
111
+ module: 'src/core/emit/emit.ts',
80
112
  },
81
113
  };
82
114
  export const ARTIFACT_PRODUCERS = {
@@ -39,6 +39,16 @@ export function evidenceOf(projected, observation, inputs) {
39
39
  const { body } = projected;
40
40
  return {
41
41
  observationId: projected.legId,
42
+ // A synthetic, single-observation shape built fresh per leg and never
43
+ // collected alongside a sibling: pre-flight resolves one leg's witness
44
+ // relation at a time, so the schema's ordering and uniqueness concerns
45
+ // (owed item 2) have nothing to apply to here. Constant, since no reader
46
+ // of this value cares which leg it was. This stops being safe if
47
+ // pre-flight ever needs to assemble multiple legs' observations
48
+ // together: AD-40 already names such a future need, pair-defect signing
49
+ // across the monotonic sequence of owed item 2, so revisit this
50
+ // constant then.
51
+ sequence: 1,
42
52
  operationId: projected.operationId,
43
53
  provenance: 'baseline',
44
54
  callInputs: callInputsOf(inputs),