eval-quality 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +142 -67
- package/corpus/dev/README.md +17 -11
- package/corpus/dev/compile-seal-example/brief.json +1 -1
- package/corpus/dev/compile-seal-example/contract.json +1 -1
- package/corpus/dev/contracts/absent-collection-locations.json +1 -1
- package/corpus/dev/contracts/absent-sibling-groups.json +1 -1
- package/corpus/dev/contracts/absent-success-indicator.json +1 -1
- package/corpus/dev/contracts/empty-channel-roles.json +1 -1
- package/corpus/dev/contracts/empty-collection-locations.json +1 -1
- package/corpus/dev/contracts/empty-request-shapes.json +1 -1
- package/corpus/dev/contracts/empty-sibling-groups.json +1 -1
- package/corpus/dev/contracts/no-collection-quantifier.json +1 -1
- package/corpus/dev/contracts/no-operation-inventory.json +1 -1
- package/corpus/dev/contracts/no-read-back-relation.json +1 -1
- package/corpus/dev/contracts/no-state-change-marker.json +1 -1
- package/corpus/dev/contracts/no-type-violating-step.json +1 -1
- package/corpus/dev/contracts/per-key-split-oracles.json +1 -1
- package/corpus/dev/contracts/satisfied-declarations.json +1 -1
- package/corpus/dev/contracts/single-required-response-key.json +1 -1
- package/corpus/dev/contracts/split-indicator-oracle.json +1 -1
- package/corpus/dev/contracts/unaddressed-parameter-sibling.json +1 -1
- package/corpus/dev/contracts/unnamed-reference-set.json +1 -1
- package/corpus/dev/contracts/wrong-cardinality-form.json +1 -1
- package/corpus/dev/index.json +1 -1
- package/dist/application/index.d.ts +2 -0
- package/dist/application/index.js +1 -0
- package/dist/application/score.d.ts +48 -0
- package/dist/application/score.js +186 -0
- package/dist/cli/arguments.d.ts +7 -3
- package/dist/cli/arguments.js +73 -5
- package/dist/cli/exit-codes.d.ts +6 -2
- package/dist/cli/exit-codes.js +8 -9
- package/dist/cli/main.js +9 -0
- package/dist/cli/render.d.ts +1 -1
- package/dist/cli/render.js +6 -4
- package/dist/cli/run.d.ts +13 -3
- package/dist/cli/run.js +136 -16
- package/dist/core/canonical/canonicalize.d.ts +12 -0
- package/dist/core/canonical/canonicalize.js +18 -4
- package/dist/core/compile/bindings.d.ts +70 -0
- package/dist/core/compile/bindings.js +317 -0
- package/dist/core/compile/compile.d.ts +5 -0
- package/dist/core/compile/compile.js +9 -0
- package/dist/core/compile/expression-legality.d.ts +49 -0
- package/dist/core/compile/expression-legality.js +105 -31
- package/dist/core/compile/forbidden-inputs.d.ts +8 -1
- package/dist/core/compile/forbidden-inputs.js +16 -3
- package/dist/core/compile/interface-inventory.d.ts +23 -1
- package/dist/core/compile/interface-inventory.js +32 -3
- package/dist/core/compile/reachability.d.ts +18 -1
- package/dist/core/compile/reachability.js +46 -1
- package/dist/core/compile/sensitivity-witness.d.ts +1 -1
- package/dist/core/compile/sensitivity-witness.js +1 -1
- package/dist/core/emit/emit.d.ts +9 -0
- package/dist/core/emit/emit.js +214 -0
- package/dist/core/emit/private-artifact-digest.d.ts +11 -0
- package/dist/core/emit/private-artifact-digest.js +31 -0
- package/dist/core/evaluate/evidence-resolution.d.ts +11 -1
- package/dist/core/evaluate/evidence-resolution.js +4 -1
- package/dist/core/failure-codes.d.ts +2 -2
- package/dist/core/failure-codes.js +4 -2
- package/dist/core/ingest/conditions.d.ts +201 -0
- package/dist/core/ingest/conditions.js +48 -0
- package/dist/core/ingest/index.d.ts +13 -0
- package/dist/core/ingest/index.js +11 -0
- package/dist/core/ingest/ingest.d.ts +68 -0
- package/dist/core/ingest/ingest.js +310 -0
- package/dist/core/lineage/stage-table.d.ts +21 -4
- package/dist/core/lineage/stage-table.js +35 -3
- package/dist/core/preflight/witness-evidence.js +10 -0
- package/dist/core/schemas/artifact.d.ts +385 -14
- package/dist/core/schemas/constraint-ledger.js +40 -0
- package/dist/core/schemas/defect-signature.d.ts +273 -0
- package/dist/core/schemas/defect-signature.js +127 -0
- package/dist/core/schemas/eval-contract.d.ts +33 -0
- package/dist/core/schemas/eval-contract.js +14 -0
- package/dist/core/schemas/evidence-artifact.d.ts +99 -19
- package/dist/core/schemas/evidence-artifact.js +64 -8
- package/dist/core/schemas/plan.d.ts +112 -0
- package/dist/core/schemas/plan.js +67 -2
- package/dist/core/schemas/probe-qualification.d.ts +168 -0
- package/dist/core/schemas/probe-qualification.js +106 -0
- package/dist/core/schemas/probe.d.ts +312 -0
- package/dist/core/schemas/probe.js +6 -2
- package/dist/core/schemas/scoring-policy.d.ts +1 -0
- package/dist/core/schemas/scoring-policy.js +6 -1
- package/dist/core/schemas/sealed-evaluator-brief.d.ts +1 -0
- package/dist/core/schemas/sealed-evaluator-brief.js +4 -1
- package/dist/core/schemas/sealed-run-record.d.ts +30 -3
- package/dist/core/schemas/sealed-run-record.js +37 -5
- package/dist/core/score/binding-order.d.ts +31 -0
- package/dist/core/score/binding-order.js +88 -0
- package/dist/core/score/bindings.d.ts +93 -0
- package/dist/core/score/bindings.js +297 -0
- package/dist/core/score/ladder-table.d.ts +23 -0
- package/dist/core/score/ladder-table.js +109 -0
- package/dist/core/score/ladder.d.ts +229 -0
- package/dist/core/score/ladder.js +464 -0
- package/dist/core/score/mode-agreement.d.ts +30 -0
- package/dist/core/score/mode-agreement.js +16 -0
- package/dist/core/score/outcome-table.d.ts +17 -0
- package/dist/core/score/outcome-table.js +172 -0
- package/dist/core/score/outcome.d.ts +466 -0
- package/dist/core/score/outcome.js +562 -0
- package/dist/core/score/qualification.d.ts +83 -0
- package/dist/core/score/qualification.js +615 -0
- package/dist/core/score/quotation.d.ts +79 -0
- package/dist/core/score/quotation.js +201 -0
- package/dist/core/score/reduce-trials.d.ts +80 -0
- package/dist/core/score/reduce-trials.js +90 -0
- package/dist/core/score/score.d.ts +72 -0
- package/dist/core/score/score.js +581 -0
- package/dist/core/score/selection.d.ts +80 -0
- package/dist/core/score/selection.js +98 -0
- package/dist/core/score/strength.d.ts +49 -0
- package/dist/core/score/strength.js +193 -0
- package/dist/core/score/witness.d.ts +117 -0
- package/dist/core/score/witness.js +315 -0
- package/dist/core/seal/derived-reference.d.ts +1 -1
- package/dist/core/seal/derived-reference.js +236 -28
- package/dist/core/seal/seal.js +8 -3
- package/dist/core/stage-contracts.d.ts +62 -2
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/testing/index.d.ts +2 -0
- package/dist/testing/index.js +1 -0
- package/package.json +12 -3
- package/schemas/eval-contract.schema.json +100 -2
- package/schemas/evidence-artifact.schema.json +96 -9
- package/schemas/probe.schema.json +508 -3
- package/schemas/scoring-policy.schema.json +8 -1
- package/schemas/sealed-evaluator-brief.schema.json +11 -1
- package/schemas/sealed-run-record.schema.json +19 -2
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* AD-24's stage-signature table and AD-29's producer map, as data: six stages
|
|
3
|
-
* with their inputs,
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* type-only to keep zod off that
|
|
3
|
+
* with their artifact inputs, the non-artifact values they receive, their one
|
|
4
|
+
* owned output, and the lineage edge each writes; twelve interchange artifacts
|
|
5
|
+
* with one producer apiece. The `check:lineage` scanner derives its allowlist
|
|
6
|
+
* from `module`, so the registry import stays type-only to keep zod off that
|
|
7
|
+
* gate's load path.
|
|
7
8
|
*/
|
|
8
9
|
import type { InterchangeArtifactKey } from '../schemas/artifact.ts';
|
|
9
10
|
/** AD-24's six stages, in the order the Consistency Conventions list them. */
|
|
@@ -16,6 +17,20 @@ export type PipelineStage = (typeof PIPELINE_STAGES)[number];
|
|
|
16
17
|
*/
|
|
17
18
|
export declare const INTERNAL_PRODUCTS: readonly ['probe-plan', 'probe-observations', 'validated-observations', 'scored-outcomes-and-verdict'];
|
|
18
19
|
export type InternalProduct = (typeof INTERNAL_PRODUCTS)[number];
|
|
20
|
+
/**
|
|
21
|
+
* Values a stage reads at its boundary that the `inputs` column does not name.
|
|
22
|
+
* AD-24 writes the table in artifacts, and owed item 6 records the source of
|
|
23
|
+
* run mode as absent from it: a row naming `sealed-run-record` says which bytes
|
|
24
|
+
* arrive and leaves open which value fixes the run's mode. Closed rather than
|
|
25
|
+
* free text, so the column cannot become a notes field.
|
|
26
|
+
*
|
|
27
|
+
* The three digests join `mode` for the same reason: they are exactly what
|
|
28
|
+
* this column exists for, "values a stage reads at its boundary that the
|
|
29
|
+
* inputs column does not name" -- `emit`'s three caller-attested AD-11
|
|
30
|
+
* digests have no artifact source anywhere in this pipeline.
|
|
31
|
+
*/
|
|
32
|
+
export declare const STAGE_VALUE_INPUTS: readonly ['mode', 'corpusDigest', 'fixtureDigest', 'evaluatorConfigurationDigest'];
|
|
33
|
+
export type StageValueInput = (typeof STAGE_VALUE_INPUTS)[number];
|
|
19
34
|
/**
|
|
20
35
|
* What a stage does to the two AD-29 fields on the artifact it owns. `mints`
|
|
21
36
|
* covers a root and a revision alike, since both write the fields.
|
|
@@ -25,6 +40,8 @@ export type LineageEdge = 'mints' | 'carries-through' | 'none';
|
|
|
25
40
|
export type ArtifactProducer = PipelineStage | 'caller' | 'embedded';
|
|
26
41
|
export type StageSignature = {
|
|
27
42
|
readonly inputs: readonly (InterchangeArtifactKey | InternalProduct)[];
|
|
43
|
+
/** non-artifact values the stage receives; see STAGE_VALUE_INPUTS. */
|
|
44
|
+
readonly valueInputs: readonly StageValueInput[];
|
|
28
45
|
readonly owns: InterchangeArtifactKey | InternalProduct;
|
|
29
46
|
/** the owned output's registry key; null when the output is internal. */
|
|
30
47
|
readonly ownsInterchange: InterchangeArtifactKey | null;
|
|
@@ -18,9 +18,28 @@ export const INTERNAL_PRODUCTS = [
|
|
|
18
18
|
'validated-observations',
|
|
19
19
|
'scored-outcomes-and-verdict',
|
|
20
20
|
];
|
|
21
|
+
/**
|
|
22
|
+
* Values a stage reads at its boundary that the `inputs` column does not name.
|
|
23
|
+
* AD-24 writes the table in artifacts, and owed item 6 records the source of
|
|
24
|
+
* run mode as absent from it: a row naming `sealed-run-record` says which bytes
|
|
25
|
+
* arrive and leaves open which value fixes the run's mode. Closed rather than
|
|
26
|
+
* free text, so the column cannot become a notes field.
|
|
27
|
+
*
|
|
28
|
+
* The three digests join `mode` for the same reason: they are exactly what
|
|
29
|
+
* this column exists for, "values a stage reads at its boundary that the
|
|
30
|
+
* inputs column does not name" -- `emit`'s three caller-attested AD-11
|
|
31
|
+
* digests have no artifact source anywhere in this pipeline.
|
|
32
|
+
*/
|
|
33
|
+
export const STAGE_VALUE_INPUTS = [
|
|
34
|
+
'mode',
|
|
35
|
+
'corpusDigest',
|
|
36
|
+
'fixtureDigest',
|
|
37
|
+
'evaluatorConfigurationDigest',
|
|
38
|
+
];
|
|
21
39
|
export const STAGE_SIGNATURES = {
|
|
22
40
|
compile: {
|
|
23
41
|
inputs: ['eval-contract'],
|
|
42
|
+
valueInputs: [],
|
|
24
43
|
owns: 'eval-contract',
|
|
25
44
|
ownsInterchange: 'eval-contract',
|
|
26
45
|
// The caller authors the contract and its lineage; `compile` validates
|
|
@@ -31,6 +50,7 @@ export const STAGE_SIGNATURES = {
|
|
|
31
50
|
},
|
|
32
51
|
seal: {
|
|
33
52
|
inputs: ['eval-contract'],
|
|
53
|
+
valueInputs: [],
|
|
34
54
|
owns: 'sealed-evaluator-brief',
|
|
35
55
|
ownsInterchange: 'sealed-evaluator-brief',
|
|
36
56
|
lineage: 'mints',
|
|
@@ -42,15 +62,21 @@ export const STAGE_SIGNATURES = {
|
|
|
42
62
|
'isolation-manifest',
|
|
43
63
|
'evaluator-configuration',
|
|
44
64
|
],
|
|
65
|
+
// Mode arrives on the sealed run record and is named here because owed
|
|
66
|
+
// item 6 asks the table for the source of run mode. `ingest` is the last
|
|
67
|
+
// stage that may read it from the caller and the first that may reject
|
|
68
|
+
// its absence; no later stage derives, recomputes, or defaults it.
|
|
69
|
+
valueInputs: ['mode'],
|
|
45
70
|
owns: 'validated-observations',
|
|
46
71
|
ownsInterchange: null,
|
|
47
72
|
lineage: 'none',
|
|
48
|
-
module:
|
|
73
|
+
module: 'src/core/ingest/ingest.ts',
|
|
49
74
|
},
|
|
50
75
|
preflight: {
|
|
51
76
|
// AD-34 splits the stage into `plan` and `reduce`; both halves' inputs
|
|
52
77
|
// are the stage's inputs, and only `reduce` returns an artifact.
|
|
53
78
|
inputs: ['eval-contract', 'probe', 'probe-plan', 'probe-observations'],
|
|
79
|
+
valueInputs: [],
|
|
54
80
|
owns: 'preflight-verdict',
|
|
55
81
|
ownsInterchange: 'preflight-verdict',
|
|
56
82
|
lineage: 'mints',
|
|
@@ -64,19 +90,25 @@ export const STAGE_SIGNATURES = {
|
|
|
64
90
|
'preflight-verdict',
|
|
65
91
|
'scoring-policy',
|
|
66
92
|
],
|
|
93
|
+
valueInputs: [],
|
|
67
94
|
// AD-24: "score produces the outcome and verdict values emit
|
|
68
95
|
// serializes". Owed item 6 records that type as unnamed; this names it.
|
|
69
96
|
owns: 'scored-outcomes-and-verdict',
|
|
70
97
|
ownsInterchange: null,
|
|
71
98
|
lineage: 'none',
|
|
72
|
-
module:
|
|
99
|
+
module: 'src/core/score/score.ts',
|
|
73
100
|
},
|
|
74
101
|
emit: {
|
|
75
102
|
inputs: ['scored-outcomes-and-verdict'],
|
|
103
|
+
valueInputs: [
|
|
104
|
+
'corpusDigest',
|
|
105
|
+
'fixtureDigest',
|
|
106
|
+
'evaluatorConfigurationDigest',
|
|
107
|
+
],
|
|
76
108
|
owns: 'evidence-artifact',
|
|
77
109
|
ownsInterchange: 'evidence-artifact',
|
|
78
110
|
lineage: 'mints',
|
|
79
|
-
module:
|
|
111
|
+
module: 'src/core/emit/emit.ts',
|
|
80
112
|
},
|
|
81
113
|
};
|
|
82
114
|
export const ARTIFACT_PRODUCERS = {
|
|
@@ -39,6 +39,16 @@ export function evidenceOf(projected, observation, inputs) {
|
|
|
39
39
|
const { body } = projected;
|
|
40
40
|
return {
|
|
41
41
|
observationId: projected.legId,
|
|
42
|
+
// A synthetic, single-observation shape built fresh per leg and never
|
|
43
|
+
// collected alongside a sibling: pre-flight resolves one leg's witness
|
|
44
|
+
// relation at a time, so the schema's ordering and uniqueness concerns
|
|
45
|
+
// (owed item 2) have nothing to apply to here. Constant, since no reader
|
|
46
|
+
// of this value cares which leg it was. This stops being safe if
|
|
47
|
+
// pre-flight ever needs to assemble multiple legs' observations
|
|
48
|
+
// together: AD-40 already names such a future need, pair-defect signing
|
|
49
|
+
// across the monotonic sequence of owed item 2, so revisit this
|
|
50
|
+
// constant then.
|
|
51
|
+
sequence: 1,
|
|
42
52
|
operationId: projected.operationId,
|
|
43
53
|
provenance: 'baseline',
|
|
44
54
|
callInputs: callInputsOf(inputs),
|