eval-quality 0.2.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -12
- package/corpus/dev/README.md +3 -2
- package/corpus/dev/compile-seal-example/brief.json +1 -1
- package/corpus/dev/compile-seal-example/contract.json +1 -1
- package/corpus/dev/contracts/absent-collection-locations.json +1 -1
- package/corpus/dev/contracts/absent-sibling-groups.json +1 -1
- package/corpus/dev/contracts/absent-success-indicator.json +1 -1
- package/corpus/dev/contracts/empty-channel-roles.json +1 -1
- package/corpus/dev/contracts/empty-collection-locations.json +1 -1
- package/corpus/dev/contracts/empty-request-shapes.json +1 -1
- package/corpus/dev/contracts/empty-sibling-groups.json +1 -1
- package/corpus/dev/contracts/fragment-selection.json +1 -0
- package/corpus/dev/contracts/no-collection-quantifier.json +1 -1
- package/corpus/dev/contracts/no-operation-inventory.json +1 -1
- package/corpus/dev/contracts/no-read-back-relation.json +1 -1
- package/corpus/dev/contracts/no-state-change-marker.json +1 -1
- package/corpus/dev/contracts/no-type-violating-step.json +1 -1
- package/corpus/dev/contracts/per-key-split-oracles.json +1 -1
- package/corpus/dev/contracts/review-corpus.json +1 -0
- package/corpus/dev/contracts/satisfied-declarations.json +1 -1
- package/corpus/dev/contracts/single-required-response-key.json +1 -1
- package/corpus/dev/contracts/split-indicator-oracle.json +1 -1
- package/corpus/dev/contracts/unaddressed-parameter-sibling.json +1 -1
- package/corpus/dev/contracts/unnamed-reference-set.json +1 -1
- package/corpus/dev/contracts/wrong-cardinality-form.json +1 -1
- package/corpus/dev/index.json +1 -1
- package/dist/adapters/command-line-adapter.d.ts +44 -0
- package/dist/adapters/command-line-adapter.js +284 -0
- package/dist/adapters/command-target-policy.d.ts +36 -0
- package/dist/adapters/command-target-policy.js +33 -0
- package/dist/adapters/index.d.ts +2 -0
- package/dist/adapters/index.js +1 -0
- package/dist/cli/render.d.ts +2 -5
- package/dist/cli/render.js +51 -1
- package/dist/core/compile/bindings.d.ts +26 -7
- package/dist/core/compile/bindings.js +49 -32
- package/dist/core/compile/compile.d.ts +12 -0
- package/dist/core/compile/compile.js +28 -1
- package/dist/core/compile/excluded-content.d.ts +11 -0
- package/dist/core/compile/excluded-content.js +42 -0
- package/dist/core/compile/expression-legality.d.ts +2 -2
- package/dist/core/compile/expression-legality.js +58 -12
- package/dist/core/compile/interface-inventory.d.ts +34 -1
- package/dist/core/compile/interface-inventory.js +88 -11
- package/dist/core/compile/reachability.d.ts +12 -2
- package/dist/core/compile/reachability.js +123 -36
- package/dist/core/compile/schema-version.d.ts +2 -0
- package/dist/core/compile/schema-version.js +25 -0
- package/dist/core/compile/sensitivity-witness.d.ts +31 -12
- package/dist/core/compile/sensitivity-witness.js +110 -23
- package/dist/core/compile/step-reference.d.ts +2 -0
- package/dist/core/compile/step-reference.js +49 -0
- package/dist/core/coverage/operations.d.ts +62 -0
- package/dist/core/coverage/operations.js +57 -0
- package/dist/core/coverage/relevance.d.ts +4 -2
- package/dist/core/coverage/relevance.js +22 -23
- package/dist/core/coverage/satisfaction.d.ts +2 -2
- package/dist/core/coverage/satisfaction.js +73 -48
- package/dist/core/declared-inputs.d.ts +83 -4
- package/dist/core/declared-inputs.js +105 -8
- package/dist/core/evaluate/evidence-resolution.d.ts +9 -12
- package/dist/core/evaluate/evidence-resolution.js +70 -10
- package/dist/core/evaluate/operators.d.ts +22 -0
- package/dist/core/evaluate/operators.js +17 -1
- package/dist/core/evaluate/resolution.d.ts +14 -3
- package/dist/core/evaluate/resolution.js +63 -3
- package/dist/core/excluded-content.d.ts +65 -0
- package/dist/core/excluded-content.js +113 -0
- package/dist/core/failure-codes.d.ts +2 -2
- package/dist/core/failure-codes.js +5 -2
- package/dist/core/ingest/conditions.d.ts +1 -1
- package/dist/core/ingest/ingest.js +8 -0
- package/dist/core/preflight/plan.d.ts +7 -5
- package/dist/core/preflight/plan.js +75 -32
- package/dist/core/preflight/projection.d.ts +6 -3
- package/dist/core/preflight/projection.js +22 -2
- package/dist/core/preflight/reduce.js +47 -7
- package/dist/core/preflight/witness-evidence.d.ts +5 -5
- package/dist/core/preflight/witness-evidence.js +63 -20
- package/dist/core/schemas/artifact.d.ts +683 -35
- package/dist/core/schemas/constraint-ledger.js +11 -0
- package/dist/core/schemas/defect-signature.d.ts +438 -16
- package/dist/core/schemas/defect-signature.js +58 -10
- package/dist/core/schemas/eval-contract.d.ts +517 -12
- package/dist/core/schemas/eval-contract.js +18 -2
- package/dist/core/schemas/evidence-artifact.d.ts +14 -4
- package/dist/core/schemas/evidence-artifact.js +1 -1
- package/dist/core/schemas/interface.d.ts +741 -13
- package/dist/core/schemas/interface.js +120 -7
- package/dist/core/schemas/isolation-manifest.js +16 -9
- package/dist/core/schemas/plan.d.ts +217 -3
- package/dist/core/schemas/plan.js +22 -1
- package/dist/core/schemas/pointer.d.ts +47 -1
- package/dist/core/schemas/pointer.js +89 -8
- package/dist/core/schemas/port-messages.d.ts +194 -10
- package/dist/core/schemas/port-messages.js +73 -6
- package/dist/core/schemas/probe-body.d.ts +18 -0
- package/dist/core/schemas/probe-body.js +13 -0
- package/dist/core/schemas/probe-policy.d.ts +36 -0
- package/dist/core/schemas/probe-policy.js +44 -0
- package/dist/core/schemas/probe.d.ts +122 -3
- package/dist/core/schemas/probe.js +1 -1
- package/dist/core/schemas/sealed-run-record.d.ts +131 -9
- package/dist/core/schemas/sealed-run-record.js +56 -11
- package/dist/core/schemas/sensitivity-witness.d.ts +108 -6
- package/dist/core/schemas/sensitivity-witness.js +61 -5
- package/dist/core/score/bindings.d.ts +2 -2
- package/dist/core/score/bindings.js +25 -13
- package/dist/core/score/qualification.d.ts +5 -5
- package/dist/core/score/qualification.js +67 -37
- package/dist/core/score/quotation.d.ts +1 -24
- package/dist/core/score/quotation.js +29 -4
- package/dist/core/score/score.js +28 -2
- package/dist/core/score/strength.d.ts +9 -0
- package/dist/core/score/strength.js +35 -1
- package/dist/core/score/witness.d.ts +21 -0
- package/dist/core/score/witness.js +9 -4
- package/dist/core/seal/derived-reference.d.ts +2 -9
- package/dist/core/seal/derived-reference.js +62 -23
- package/dist/core/seal/plan-index.d.ts +18 -3
- package/dist/core/seal/plan-index.js +44 -8
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/ports/environment-probe-port.d.ts +60 -4
- package/dist/testing/conformance.d.ts +2 -1
- package/dist/testing/conformance.js +1 -0
- package/dist/testing/index.d.ts +3 -3
- package/dist/testing/index.js +1 -1
- package/dist/testing/probe-conformance.d.ts +56 -5
- package/dist/testing/probe-conformance.js +196 -5
- package/package.json +1 -1
- package/schemas/eval-contract.schema.json +1356 -464
- package/schemas/evidence-artifact.schema.json +59 -29
- package/schemas/isolation-manifest.schema.json +17 -10
- package/schemas/probe.schema.json +197 -71
- package/schemas/rubric.schema.json +1 -1
- package/schemas/sealed-run-record.schema.json +296 -39
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
* unchanged. This module supplies the `Observation` shape they read and a
|
|
5
5
|
* collection predicate scoped to one operation.
|
|
6
6
|
*/
|
|
7
|
+
import { descriptorArtifactOf, descriptorChannelOf, } from '../declared-inputs.js';
|
|
7
8
|
import { makeResolveOperand } from '../evaluate/evidence-resolution.js';
|
|
8
9
|
import { resolveCheck, } from '../evaluate/resolution.js';
|
|
9
10
|
import { decodeTail, parseEvidenceTarget } from '../seal/plan-index.js';
|
|
@@ -17,17 +18,46 @@ const asJsonObject = (value) => value !== null && typeof value === 'object' && !
|
|
|
17
18
|
? value
|
|
18
19
|
: null;
|
|
19
20
|
/**
|
|
20
|
-
* `ObservedCallInputs` is narrower than `
|
|
21
|
+
* `ObservedCallInputs` is narrower than `ApiWitnessInputs`, so `body` loses
|
|
21
22
|
* information here: an absent body and a non-object JSON body both map to
|
|
22
23
|
* `null`, and a relation addressing `/interactions/{legId}/call-inputs/body` on
|
|
23
24
|
* such a leg resolves `ABSENT`.
|
|
24
25
|
*/
|
|
25
|
-
const
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
26
|
+
const ABSENT_CHANNEL = { kind: 'absent' };
|
|
27
|
+
const bodyValue = (body) => body.kind === 'absent' ? null : body.value;
|
|
28
|
+
/**
|
|
29
|
+
* The leg's supplied inputs as the record spells them: every channel the leg
|
|
30
|
+
* did not use is `null` rather than absent, which is the observation's own
|
|
31
|
+
* convention for an unused channel.
|
|
32
|
+
*/
|
|
33
|
+
const callInputsOf = (inputs) => {
|
|
34
|
+
const empty = {
|
|
35
|
+
path: null,
|
|
36
|
+
query: null,
|
|
37
|
+
header: null,
|
|
38
|
+
body: null,
|
|
39
|
+
argument: null,
|
|
40
|
+
option: null,
|
|
41
|
+
environment: null,
|
|
42
|
+
stdin: null,
|
|
43
|
+
};
|
|
44
|
+
if ('body' in inputs) {
|
|
45
|
+
return {
|
|
46
|
+
...empty,
|
|
47
|
+
path: inputs.path,
|
|
48
|
+
query: inputs.query,
|
|
49
|
+
header: inputs.header,
|
|
50
|
+
body: inputs.body.kind === 'json' ? asJsonObject(inputs.body.value) : null,
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
return {
|
|
54
|
+
...empty,
|
|
55
|
+
argument: inputs.argument,
|
|
56
|
+
option: inputs.option,
|
|
57
|
+
environment: inputs.environment,
|
|
58
|
+
stdin: inputs.stdin.kind === 'json' ? asJsonObject(inputs.stdin.value) : null,
|
|
59
|
+
};
|
|
60
|
+
};
|
|
31
61
|
/**
|
|
32
62
|
* One leg as the `Observation` `makeResolveOperand` takes. The relation reads the
|
|
33
63
|
* **projected** body, which is AD-10's "evaluated over that operation's response
|
|
@@ -35,8 +65,10 @@ const callInputsOf = (inputs) => ({
|
|
|
35
65
|
* the projection does not carry. `provenance` is `baseline`: a pre-flight leg is
|
|
36
66
|
* pre-canned by definition.
|
|
37
67
|
*/
|
|
38
|
-
export function evidenceOf(projected, observation, inputs) {
|
|
68
|
+
export function evidenceOf(projected, observation, inputs, operation) {
|
|
39
69
|
const { body } = projected;
|
|
70
|
+
const descriptorChannel = descriptorChannelOf(operation);
|
|
71
|
+
const describedArtifact = descriptorArtifactOf(operation);
|
|
40
72
|
return {
|
|
41
73
|
observationId: projected.legId,
|
|
42
74
|
// A synthetic, single-observation shape built fresh per leg and never
|
|
@@ -51,17 +83,28 @@ export function evidenceOf(projected, observation, inputs) {
|
|
|
51
83
|
sequence: 1,
|
|
52
84
|
operationId: projected.operationId,
|
|
53
85
|
provenance: 'baseline',
|
|
86
|
+
// A pre-flight leg is issued by this package rather than by a harness
|
|
87
|
+
// acting as a declared account, so it names no principal.
|
|
88
|
+
principal: null,
|
|
54
89
|
callInputs: callInputsOf(inputs),
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
90
|
+
// The projected body is whichever channel the operation's descriptor
|
|
91
|
+
// describes, so it lands on the channel a relation addresses. Both are
|
|
92
|
+
// filled from the same projection rather than one being derived from
|
|
93
|
+
// the other, and every channel the leg did not observe is written down
|
|
94
|
+
// as unobserved rather than left to a default.
|
|
95
|
+
responseBody: observation.kind === 'api' ? bodyValue(body) : null,
|
|
96
|
+
responseHeaders: observation.kind === 'api' ? observation.headers : null,
|
|
97
|
+
responseStatus: observation.kind === 'api' ? observation.status : null,
|
|
98
|
+
stdout: observation.kind === 'api' || descriptorChannel !== 'stdout'
|
|
99
|
+
? ABSENT_CHANNEL
|
|
100
|
+
: body,
|
|
101
|
+
stderr: observation.kind === 'api' || descriptorChannel !== 'stderr'
|
|
102
|
+
? ABSENT_CHANNEL
|
|
103
|
+
: body,
|
|
104
|
+
exitCode: observation.kind === 'api' ? null : observation.exitCode,
|
|
105
|
+
artifacts: observation.kind === 'api' || describedArtifact === null
|
|
106
|
+
? {}
|
|
107
|
+
: { [describedArtifact]: body },
|
|
65
108
|
};
|
|
66
109
|
}
|
|
67
110
|
/** the contract's declared reference sets in the shape the resolver wants. */
|
|
@@ -93,6 +136,6 @@ export function makeWitnessPointerDenotesCollection(operation) {
|
|
|
93
136
|
};
|
|
94
137
|
}
|
|
95
138
|
/** Resolves one witness relation over the legs it addresses. */
|
|
96
|
-
export function resolveWitnessRelation(relation, legEvidence, operation, referenceSets, artifactPath) {
|
|
97
|
-
return resolveCheck(relation, makeResolveOperand(legEvidence, referenceSets), makeWitnessPointerDenotesCollection(operation), PREFLIGHT_REGEX_MATCH_STEP_BUDGET, artifactPath);
|
|
139
|
+
export function resolveWitnessRelation(relation, legEvidence, operation, referenceSets, referenceSetKeys, artifactPath) {
|
|
140
|
+
return resolveCheck(relation, makeResolveOperand(legEvidence, referenceSets), makeWitnessPointerDenotesCollection(operation), referenceSetKeys, PREFLIGHT_REGEX_MATCH_STEP_BUDGET, artifactPath);
|
|
98
141
|
}
|