eval-quality 0.2.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -12
- package/corpus/dev/README.md +3 -2
- package/corpus/dev/compile-seal-example/brief.json +1 -1
- package/corpus/dev/compile-seal-example/contract.json +1 -1
- package/corpus/dev/contracts/absent-collection-locations.json +1 -1
- package/corpus/dev/contracts/absent-sibling-groups.json +1 -1
- package/corpus/dev/contracts/absent-success-indicator.json +1 -1
- package/corpus/dev/contracts/empty-channel-roles.json +1 -1
- package/corpus/dev/contracts/empty-collection-locations.json +1 -1
- package/corpus/dev/contracts/empty-request-shapes.json +1 -1
- package/corpus/dev/contracts/empty-sibling-groups.json +1 -1
- package/corpus/dev/contracts/fragment-selection.json +1 -0
- package/corpus/dev/contracts/no-collection-quantifier.json +1 -1
- package/corpus/dev/contracts/no-operation-inventory.json +1 -1
- package/corpus/dev/contracts/no-read-back-relation.json +1 -1
- package/corpus/dev/contracts/no-state-change-marker.json +1 -1
- package/corpus/dev/contracts/no-type-violating-step.json +1 -1
- package/corpus/dev/contracts/per-key-split-oracles.json +1 -1
- package/corpus/dev/contracts/review-corpus.json +1 -0
- package/corpus/dev/contracts/satisfied-declarations.json +1 -1
- package/corpus/dev/contracts/single-required-response-key.json +1 -1
- package/corpus/dev/contracts/split-indicator-oracle.json +1 -1
- package/corpus/dev/contracts/unaddressed-parameter-sibling.json +1 -1
- package/corpus/dev/contracts/unnamed-reference-set.json +1 -1
- package/corpus/dev/contracts/wrong-cardinality-form.json +1 -1
- package/corpus/dev/index.json +1 -1
- package/dist/adapters/command-line-adapter.d.ts +44 -0
- package/dist/adapters/command-line-adapter.js +284 -0
- package/dist/adapters/command-target-policy.d.ts +36 -0
- package/dist/adapters/command-target-policy.js +33 -0
- package/dist/adapters/index.d.ts +2 -0
- package/dist/adapters/index.js +1 -0
- package/dist/cli/render.d.ts +2 -5
- package/dist/cli/render.js +51 -1
- package/dist/core/compile/bindings.d.ts +26 -7
- package/dist/core/compile/bindings.js +49 -32
- package/dist/core/compile/compile.d.ts +12 -0
- package/dist/core/compile/compile.js +28 -1
- package/dist/core/compile/excluded-content.d.ts +11 -0
- package/dist/core/compile/excluded-content.js +42 -0
- package/dist/core/compile/expression-legality.d.ts +2 -2
- package/dist/core/compile/expression-legality.js +58 -12
- package/dist/core/compile/interface-inventory.d.ts +34 -1
- package/dist/core/compile/interface-inventory.js +88 -11
- package/dist/core/compile/reachability.d.ts +12 -2
- package/dist/core/compile/reachability.js +123 -36
- package/dist/core/compile/schema-version.d.ts +2 -0
- package/dist/core/compile/schema-version.js +25 -0
- package/dist/core/compile/sensitivity-witness.d.ts +31 -12
- package/dist/core/compile/sensitivity-witness.js +110 -23
- package/dist/core/compile/step-reference.d.ts +2 -0
- package/dist/core/compile/step-reference.js +49 -0
- package/dist/core/coverage/operations.d.ts +62 -0
- package/dist/core/coverage/operations.js +57 -0
- package/dist/core/coverage/relevance.d.ts +4 -2
- package/dist/core/coverage/relevance.js +22 -23
- package/dist/core/coverage/satisfaction.d.ts +2 -2
- package/dist/core/coverage/satisfaction.js +73 -48
- package/dist/core/declared-inputs.d.ts +83 -4
- package/dist/core/declared-inputs.js +105 -8
- package/dist/core/evaluate/evidence-resolution.d.ts +9 -12
- package/dist/core/evaluate/evidence-resolution.js +70 -10
- package/dist/core/evaluate/operators.d.ts +22 -0
- package/dist/core/evaluate/operators.js +17 -1
- package/dist/core/evaluate/resolution.d.ts +14 -3
- package/dist/core/evaluate/resolution.js +63 -3
- package/dist/core/excluded-content.d.ts +65 -0
- package/dist/core/excluded-content.js +113 -0
- package/dist/core/failure-codes.d.ts +2 -2
- package/dist/core/failure-codes.js +5 -2
- package/dist/core/ingest/conditions.d.ts +1 -1
- package/dist/core/ingest/ingest.js +8 -0
- package/dist/core/preflight/plan.d.ts +7 -5
- package/dist/core/preflight/plan.js +75 -32
- package/dist/core/preflight/projection.d.ts +6 -3
- package/dist/core/preflight/projection.js +22 -2
- package/dist/core/preflight/reduce.js +47 -7
- package/dist/core/preflight/witness-evidence.d.ts +5 -5
- package/dist/core/preflight/witness-evidence.js +63 -20
- package/dist/core/schemas/artifact.d.ts +683 -35
- package/dist/core/schemas/constraint-ledger.js +11 -0
- package/dist/core/schemas/defect-signature.d.ts +438 -16
- package/dist/core/schemas/defect-signature.js +58 -10
- package/dist/core/schemas/eval-contract.d.ts +517 -12
- package/dist/core/schemas/eval-contract.js +18 -2
- package/dist/core/schemas/evidence-artifact.d.ts +14 -4
- package/dist/core/schemas/evidence-artifact.js +1 -1
- package/dist/core/schemas/interface.d.ts +741 -13
- package/dist/core/schemas/interface.js +120 -7
- package/dist/core/schemas/isolation-manifest.js +16 -9
- package/dist/core/schemas/plan.d.ts +217 -3
- package/dist/core/schemas/plan.js +22 -1
- package/dist/core/schemas/pointer.d.ts +47 -1
- package/dist/core/schemas/pointer.js +89 -8
- package/dist/core/schemas/port-messages.d.ts +194 -10
- package/dist/core/schemas/port-messages.js +73 -6
- package/dist/core/schemas/probe-body.d.ts +18 -0
- package/dist/core/schemas/probe-body.js +13 -0
- package/dist/core/schemas/probe-policy.d.ts +36 -0
- package/dist/core/schemas/probe-policy.js +44 -0
- package/dist/core/schemas/probe.d.ts +122 -3
- package/dist/core/schemas/probe.js +1 -1
- package/dist/core/schemas/sealed-run-record.d.ts +131 -9
- package/dist/core/schemas/sealed-run-record.js +56 -11
- package/dist/core/schemas/sensitivity-witness.d.ts +108 -6
- package/dist/core/schemas/sensitivity-witness.js +61 -5
- package/dist/core/score/bindings.d.ts +2 -2
- package/dist/core/score/bindings.js +25 -13
- package/dist/core/score/qualification.d.ts +5 -5
- package/dist/core/score/qualification.js +67 -37
- package/dist/core/score/quotation.d.ts +1 -24
- package/dist/core/score/quotation.js +29 -4
- package/dist/core/score/score.js +28 -2
- package/dist/core/score/strength.d.ts +9 -0
- package/dist/core/score/strength.js +35 -1
- package/dist/core/score/witness.d.ts +21 -0
- package/dist/core/score/witness.js +9 -4
- package/dist/core/seal/derived-reference.d.ts +2 -9
- package/dist/core/seal/derived-reference.js +62 -23
- package/dist/core/seal/plan-index.d.ts +18 -3
- package/dist/core/seal/plan-index.js +44 -8
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/ports/environment-probe-port.d.ts +60 -4
- package/dist/testing/conformance.d.ts +2 -1
- package/dist/testing/conformance.js +1 -0
- package/dist/testing/index.d.ts +3 -3
- package/dist/testing/index.js +1 -1
- package/dist/testing/probe-conformance.d.ts +56 -5
- package/dist/testing/probe-conformance.js +196 -5
- package/package.json +1 -1
- package/schemas/eval-contract.schema.json +1356 -464
- package/schemas/evidence-artifact.schema.json +59 -29
- package/schemas/isolation-manifest.schema.json +17 -10
- package/schemas/probe.schema.json +197 -71
- package/schemas/rubric.schema.json +1 -1
- package/schemas/sealed-run-record.schema.json +296 -39
|
@@ -1,18 +1,115 @@
|
|
|
1
|
-
import { TRANSPORT_CHANNELS } from './schemas/pointer.js';
|
|
1
|
+
import { COMMAND_CHANNELS, INPUT_CHANNELS, TRANSPORT_CHANNELS, } from './schemas/pointer.js';
|
|
2
|
+
/**
|
|
3
|
+
* Which output channel the operation's one response descriptor describes.
|
|
4
|
+
*
|
|
5
|
+
* AD-19 gives every operation exactly one descriptor, and the channel that
|
|
6
|
+
* descriptor describes is what makes its keys addressable. For an interface
|
|
7
|
+
* that speaks HTTP that channel is the response body; a command operation
|
|
8
|
+
* declares its own. One rule, three roots: every consumer that used to hard-code
|
|
9
|
+
* `response-body` asks this instead.
|
|
10
|
+
*/
|
|
11
|
+
export const descriptorChannelOf = (operation) => {
|
|
12
|
+
if (!isCommandOperation(operation))
|
|
13
|
+
return 'response-body';
|
|
14
|
+
const { descriptorChannel } = operation;
|
|
15
|
+
return descriptorChannel.kind === 'stream'
|
|
16
|
+
? descriptorChannel.channel
|
|
17
|
+
: 'artifact';
|
|
18
|
+
};
|
|
19
|
+
/**
|
|
20
|
+
* Which artifact the descriptor describes, or `null` when it describes a
|
|
21
|
+
* stream. An artifact pointer descends through the descriptor only when it
|
|
22
|
+
* names this one; every other declared artifact is known to exist and declares
|
|
23
|
+
* no structure.
|
|
24
|
+
*/
|
|
25
|
+
export const descriptorArtifactOf = (operation) => {
|
|
26
|
+
if (!isCommandOperation(operation))
|
|
27
|
+
return null;
|
|
28
|
+
const { descriptorChannel } = operation;
|
|
29
|
+
return descriptorChannel.kind === 'artifact'
|
|
30
|
+
? descriptorChannel.artifactId
|
|
31
|
+
: null;
|
|
32
|
+
};
|
|
33
|
+
/**
|
|
34
|
+
* Whether an evidence target addresses the channel this operation's response
|
|
35
|
+
* descriptor describes.
|
|
36
|
+
*
|
|
37
|
+
* The channel name alone is not the answer on the artifact channel: an
|
|
38
|
+
* operation may declare several files while its one descriptor describes one of
|
|
39
|
+
* them, so a pointer at a different file names a channel with no declared
|
|
40
|
+
* structure. Every consumer asks this rather than comparing the channel itself,
|
|
41
|
+
* because comparing only the channel was a defect at three separate sites and a
|
|
42
|
+
* fourth site would have made the same mistake for the same reason.
|
|
43
|
+
*/
|
|
44
|
+
export const targetsDescribedChannel = (operation, target) => {
|
|
45
|
+
if (target.channel !== descriptorChannelOf(operation))
|
|
46
|
+
return false;
|
|
47
|
+
if (target.channel !== 'artifact')
|
|
48
|
+
return true;
|
|
49
|
+
return target.artifactId === descriptorArtifactOf(operation);
|
|
50
|
+
};
|
|
51
|
+
/** Every artifact identifier the operation declares it writes. */
|
|
52
|
+
export const declaredArtifactsOf = (operation) => isCommandOperation(operation) ? operation.artifacts : [];
|
|
53
|
+
/**
|
|
54
|
+
* Which operation shape this is. Reads `invocation` rather than a kind field,
|
|
55
|
+
* because an operation does not carry its interface's kind; the two shapes are
|
|
56
|
+
* distinguished by a required field only one of them declares.
|
|
57
|
+
*/
|
|
58
|
+
export const isCommandOperation = (operation) => 'invocation' in operation;
|
|
59
|
+
/** The same question the other way round, for the callers that filter. */
|
|
60
|
+
export const isApiOperation = (operation) => !isCommandOperation(operation);
|
|
61
|
+
/**
|
|
62
|
+
* The operation's input channels paired with the shapes they declare.
|
|
63
|
+
*
|
|
64
|
+
* Callers take the pairs rather than a channel list they then index the
|
|
65
|
+
* request shape with. Indexing is what breaks under the operation union:
|
|
66
|
+
* TypeScript cannot prove that a channel name drawn from one kind's tuple is a
|
|
67
|
+
* key of the other kind's shape, and both ways around that are casts. Pairing
|
|
68
|
+
* dereferences the union once, here.
|
|
69
|
+
*/
|
|
70
|
+
export function requestChannelsOf(operation) {
|
|
71
|
+
if (isCommandOperation(operation)) {
|
|
72
|
+
return COMMAND_CHANNELS.map((channel) => ({
|
|
73
|
+
channel,
|
|
74
|
+
shape: operation.requestShape[channel],
|
|
75
|
+
}));
|
|
76
|
+
}
|
|
77
|
+
return TRANSPORT_CHANNELS.map((channel) => ({
|
|
78
|
+
channel,
|
|
79
|
+
shape: operation.requestShape[channel],
|
|
80
|
+
}));
|
|
81
|
+
}
|
|
82
|
+
/** The shape one named channel declares, or `undefined` off this kind. */
|
|
83
|
+
export function requestShapeOf(operation, channel) {
|
|
84
|
+
return requestChannelsOf(operation).find((entry) => entry.channel === channel)
|
|
85
|
+
?.shape;
|
|
86
|
+
}
|
|
87
|
+
/** The channel names an operation of this kind may declare inputs on. */
|
|
88
|
+
export const inputChannelsOf = (operation) => isCommandOperation(operation) ? COMMAND_CHANNELS : TRANSPORT_CHANNELS;
|
|
89
|
+
/**
|
|
90
|
+
* A step's bound channels paired with what each binds, for the same reason
|
|
91
|
+
* `requestChannelsOf` pairs: `InputBinding` is a union and a channel name
|
|
92
|
+
* drawn from one branch is not a key of the other.
|
|
93
|
+
*
|
|
94
|
+
* Ordered by `INPUT_CHANNELS` rather than by the parsed object's own key
|
|
95
|
+
* order, so which binding a check reports never depends on how the document
|
|
96
|
+
* happened to be written.
|
|
97
|
+
*/
|
|
98
|
+
export function boundChannelsOf(binding) {
|
|
99
|
+
const present = binding;
|
|
100
|
+
return INPUT_CHANNELS.filter((channel) => channel in binding).map((channel) => ({ channel, bound: present[channel] ?? null }));
|
|
101
|
+
}
|
|
2
102
|
/**
|
|
3
103
|
* AD-10's exemption predicate. Any of a channel's three lists naming a key
|
|
4
104
|
* counts: a permitted-only or types-only channel is still a surface a witness
|
|
5
105
|
* can vary.
|
|
6
106
|
*/
|
|
7
107
|
export function declaresNoRequestKeys(operation) {
|
|
8
|
-
return
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
shape.permittedKeys.length === 0 &&
|
|
12
|
-
Object.keys(shape.types).length === 0);
|
|
13
|
-
});
|
|
108
|
+
return requestChannelsOf(operation).every(({ shape }) => shape.requiredKeys.length === 0 &&
|
|
109
|
+
shape.permittedKeys.length === 0 &&
|
|
110
|
+
Object.keys(shape.types).length === 0);
|
|
14
111
|
}
|
|
15
112
|
/** Whether no channel declares a required key. */
|
|
16
113
|
export function declaresNoRequiredKeys(operation) {
|
|
17
|
-
return
|
|
114
|
+
return requestChannelsOf(operation).every(({ shape }) => shape.requiredKeys.length === 0);
|
|
18
115
|
}
|
|
@@ -9,7 +9,7 @@ import type { EvalContract } from '../schemas/eval-contract.ts';
|
|
|
9
9
|
import type { JsonValue } from '../schemas/primitives.ts';
|
|
10
10
|
import type { Observation } from '../schemas/sealed-run-record.ts';
|
|
11
11
|
import { type EvidenceTarget, type PlanIndex } from '../seal/plan-index.ts';
|
|
12
|
-
import type { PointerDenotesCollection, ResolveOperand } from './resolution.ts';
|
|
12
|
+
import type { PointerDenotesCollection, ReferenceSetKeys, ResolveOperand } from './resolution.ts';
|
|
13
13
|
import { type ResolvedValue } from './resolved-value.ts';
|
|
14
14
|
/**
|
|
15
15
|
* A canonical RFC 6901 array-index token: no leading zero except "0" itself,
|
|
@@ -26,7 +26,7 @@ export declare const ARRAY_INDEX_PATTERN: RegExp;
|
|
|
26
26
|
* falls through to `Object.prototype`. Any miss, type mismatch, or tail
|
|
27
27
|
* running past a scalar collapses to `ABSENT` uniformly (AD-26).
|
|
28
28
|
*/
|
|
29
|
-
export declare function walkTail(root:
|
|
29
|
+
export declare function walkTail(root: ResolvedValue, tail: readonly string[]): ResolvedValue;
|
|
30
30
|
/**
|
|
31
31
|
* Decodes `BoundElementPointer`'s tail. "@/" is the shortest legal pointer
|
|
32
32
|
* form and denotes the element itself. `decodeTail` special-cases only a
|
|
@@ -34,16 +34,7 @@ export declare function walkTail(root: JsonValue, tail: readonly string[]): Reso
|
|
|
34
34
|
* calling it.
|
|
35
35
|
*/
|
|
36
36
|
export declare function decodeBoundElementTail(pointer: string): readonly string[];
|
|
37
|
-
|
|
38
|
-
* Selects the channel `target` names off one `Observation`. `stdout`/`stderr`
|
|
39
|
-
* can carry a tail even though they're bare strings; `walkTail` already
|
|
40
|
-
* resolves any non-empty tail against a string to `ABSENT`, so no special
|
|
41
|
-
* case is needed here.
|
|
42
|
-
*
|
|
43
|
-
* Exported so `core/score/bindings.ts` reads a captured pointer's channel off
|
|
44
|
-
* an observation through this one spelling, keeping the switch in one place.
|
|
45
|
-
*/
|
|
46
|
-
export declare function channelRoot(observation: Observation, target: EvidenceTarget): JsonValue;
|
|
37
|
+
export declare function channelRoot(observation: Observation, target: EvidenceTarget): ResolvedValue;
|
|
47
38
|
/**
|
|
48
39
|
* The `ResolveOperand`. `stepObservations` holds one already-selected
|
|
49
40
|
* `Observation` per interaction step; `referenceSets` mirrors the contract's
|
|
@@ -52,6 +43,12 @@ export declare function channelRoot(observation: Observation, target: EvidenceTa
|
|
|
52
43
|
* prototype-chain gotcha `walkTail` guards against.
|
|
53
44
|
*/
|
|
54
45
|
export declare function makeResolveOperand(stepObservations: Readonly<Record<string, Observation>>, referenceSets: Readonly<Record<string, JsonValue[]>>): ResolveOperand;
|
|
46
|
+
/**
|
|
47
|
+
* The `keys` a contract's reference sets declare. Built from the same
|
|
48
|
+
* `contract.referenceSets` the members map handed to `makeResolveOperand` is
|
|
49
|
+
* built from, so a set that resolves to members always has its keys here.
|
|
50
|
+
*/
|
|
51
|
+
export declare function referenceSetKeysOf(contract: EvalContract): ReferenceSetKeys;
|
|
55
52
|
/**
|
|
56
53
|
* Only `response-body` can ever answer `true` (AD-19: `collectionLocations`
|
|
57
54
|
* is the only declared-collection surface, scoped to the body alone). The
|
|
@@ -5,7 +5,8 @@
|
|
|
5
5
|
* bound-element `@/` form, and walk it into an already-selected
|
|
6
6
|
* `Observation`.
|
|
7
7
|
*/
|
|
8
|
-
import {
|
|
8
|
+
import { targetsDescribedChannel } from '../declared-inputs.js';
|
|
9
|
+
import { anyOperationOf, buildPlanIndex, decodeTail, parseEvidenceTarget, } from '../seal/plan-index.js';
|
|
9
10
|
import { ABSENT } from './resolved-value.js';
|
|
10
11
|
/**
|
|
11
12
|
* A canonical RFC 6901 array-index token: no leading zero except "0" itself,
|
|
@@ -23,6 +24,12 @@ export const ARRAY_INDEX_PATTERN = /^(?:0|[1-9][0-9]*)$/;
|
|
|
23
24
|
* running past a scalar collapses to `ABSENT` uniformly (AD-26).
|
|
24
25
|
*/
|
|
25
26
|
export function walkTail(root, tail) {
|
|
27
|
+
// A root that did not resolve stays unresolved however short the tail is.
|
|
28
|
+
// The bare-pointer case is the one that matters: without this, an absent
|
|
29
|
+
// channel with no tail returned its own root and a caller could not tell it
|
|
30
|
+
// from a resolved value.
|
|
31
|
+
if (root === ABSENT)
|
|
32
|
+
return ABSENT;
|
|
26
33
|
let current = root;
|
|
27
34
|
for (const token of tail) {
|
|
28
35
|
if (current === null || typeof current !== 'object')
|
|
@@ -53,14 +60,29 @@ export function decodeBoundElementTail(pointer) {
|
|
|
53
60
|
return tailSource === '/' ? [] : decodeTail(tailSource);
|
|
54
61
|
}
|
|
55
62
|
/**
|
|
56
|
-
* Selects the channel `target` names off one `Observation`.
|
|
57
|
-
*
|
|
58
|
-
*
|
|
59
|
-
*
|
|
63
|
+
* Selects the channel `target` names off one `Observation`.
|
|
64
|
+
*
|
|
65
|
+
* `stdout`, `stderr`, and each written artifact are tagged, so a channel a
|
|
66
|
+
* harness captured as JSON resolves to the value and a tail walks into it,
|
|
67
|
+
* while one captured as text resolves to the string and a tail over it
|
|
68
|
+
* resolves `ABSENT`. That is what makes a command operation's nominated output
|
|
69
|
+
* channel resolvable at score time on the same terms as a response body.
|
|
60
70
|
*
|
|
61
71
|
* Exported so `core/score/bindings.ts` reads a captured pointer's channel off
|
|
62
72
|
* an observation through this one spelling, keeping the switch in one place.
|
|
63
73
|
*/
|
|
74
|
+
/**
|
|
75
|
+
* The value a tagged observed channel resolves to.
|
|
76
|
+
*
|
|
77
|
+
* A JSON body resolves to the value itself, so a tail walks into it; text
|
|
78
|
+
* resolves to the string, so a tail over it resolves `ABSENT` through
|
|
79
|
+
* `walkTail`, which is the truthful answer for output nothing parsed. An absent
|
|
80
|
+
* channel resolves to `ABSENT` and never to `null`: the tag exists precisely to
|
|
81
|
+
* tell "nothing was observed here" from "a value that was JSON null", and
|
|
82
|
+
* collapsing the two would make `existence` answer true for a file the run
|
|
83
|
+
* never wrote, since AD-26 counts `null` as present.
|
|
84
|
+
*/
|
|
85
|
+
const observedValue = (body) => body.kind === 'absent' ? ABSENT : body.value;
|
|
64
86
|
export function channelRoot(observation, target) {
|
|
65
87
|
switch (target.channel) {
|
|
66
88
|
case 'response-body':
|
|
@@ -70,9 +92,9 @@ export function channelRoot(observation, target) {
|
|
|
70
92
|
case 'response-status':
|
|
71
93
|
return observation.responseStatus;
|
|
72
94
|
case 'stdout':
|
|
73
|
-
return observation.stdout;
|
|
95
|
+
return observedValue(observation.stdout);
|
|
74
96
|
case 'stderr':
|
|
75
|
-
return observation.stderr;
|
|
97
|
+
return observedValue(observation.stderr);
|
|
76
98
|
case 'exit-code':
|
|
77
99
|
return observation.exitCode;
|
|
78
100
|
case 'call-inputs': {
|
|
@@ -84,6 +106,27 @@ export function channelRoot(observation, target) {
|
|
|
84
106
|
}
|
|
85
107
|
return observation.callInputs[transportChannel];
|
|
86
108
|
}
|
|
109
|
+
case 'artifact': {
|
|
110
|
+
const { artifactId } = target;
|
|
111
|
+
if (artifactId === null) {
|
|
112
|
+
// parseEvidenceTarget sets artifactId exactly when the channel
|
|
113
|
+
// is 'artifact', so this throw should never fire.
|
|
114
|
+
throw new TypeError('artifact evidence target names no artifact');
|
|
115
|
+
}
|
|
116
|
+
// `Object.hasOwn` for the reason this module's header gives for
|
|
117
|
+
// `stepObservations` and `referenceSets`: `Identifier` admits
|
|
118
|
+
// `constructor`, `toString` and `valueOf`, so a bare index on an
|
|
119
|
+
// unwritten file of one of those names resolves to an inherited
|
|
120
|
+
// function rather than to nothing.
|
|
121
|
+
//
|
|
122
|
+
// A file the run did not write is absent evidence, not a `null`
|
|
123
|
+
// value. `null` would read as present under AD-26 and invert every
|
|
124
|
+
// oracle asserting the file exists or does not.
|
|
125
|
+
if (!Object.hasOwn(observation.artifacts, artifactId))
|
|
126
|
+
return ABSENT;
|
|
127
|
+
const written = observation.artifacts[artifactId];
|
|
128
|
+
return written === undefined ? ABSENT : observedValue(written);
|
|
129
|
+
}
|
|
87
130
|
}
|
|
88
131
|
}
|
|
89
132
|
/**
|
|
@@ -118,6 +161,17 @@ export function makeResolveOperand(stepObservations, referenceSets) {
|
|
|
118
161
|
return walkTail(channelRoot(observation, target), target.tail);
|
|
119
162
|
};
|
|
120
163
|
}
|
|
164
|
+
/**
|
|
165
|
+
* The `keys` a contract's reference sets declare. Built from the same
|
|
166
|
+
* `contract.referenceSets` the members map handed to `makeResolveOperand` is
|
|
167
|
+
* built from, so a set that resolves to members always has its keys here.
|
|
168
|
+
*/
|
|
169
|
+
export function referenceSetKeysOf(contract) {
|
|
170
|
+
return Object.fromEntries(Object.entries(contract.referenceSets ?? {}).map(([id, set]) => [
|
|
171
|
+
id,
|
|
172
|
+
set.keys,
|
|
173
|
+
]));
|
|
174
|
+
}
|
|
121
175
|
function tokensEqual(a, b) {
|
|
122
176
|
return a.length === b.length && a.every((token, index) => token === b[index]);
|
|
123
177
|
}
|
|
@@ -140,14 +194,20 @@ export function makePointerDenotesCollection(contract, providedIndex) {
|
|
|
140
194
|
if (pointer.startsWith('@'))
|
|
141
195
|
return false;
|
|
142
196
|
const target = parseEvidenceTarget(pointer);
|
|
143
|
-
if (target.channel !== 'response-body')
|
|
144
|
-
return false;
|
|
145
197
|
const step = getIndex().stepOf(target.stepId);
|
|
146
198
|
if (step === undefined)
|
|
147
199
|
return false;
|
|
148
|
-
const operation = getIndex()
|
|
200
|
+
const operation = anyOperationOf(getIndex(), step.operationId);
|
|
149
201
|
if (operation === undefined)
|
|
150
202
|
return false;
|
|
203
|
+
// The channel is tested against the operation's own descriptor rather
|
|
204
|
+
// than against `response-body`. AD-4's empty-collection resolution
|
|
205
|
+
// applies to whichever channel an operation says carries its declared
|
|
206
|
+
// collections, and hard-coding the body left it inapplicable to every
|
|
207
|
+
// command contract: a quantifier over an empty declared collection
|
|
208
|
+
// resolved `false` instead of `insufficient-evidence`.
|
|
209
|
+
if (!targetsDescribedChannel(operation, target))
|
|
210
|
+
return false;
|
|
151
211
|
const { collectionLocations } = operation.responseDescriptor;
|
|
152
212
|
if (collectionLocations === null)
|
|
153
213
|
return false;
|
|
@@ -31,6 +31,18 @@ export declare function setMembership(value: ResolvedValue, set: JsonValue[], ar
|
|
|
31
31
|
* single element to search for: this function receives resolved values only,
|
|
32
32
|
* so it cannot tell a `{ literal }` array from a resolved `{ referenceSet }`.
|
|
33
33
|
* Known, accepted limitation.
|
|
34
|
+
*
|
|
35
|
+
* A separate property, and a separate limitation: a `{ referenceSet }`
|
|
36
|
+
* candidate matches **whole declared members** by structural equality. A
|
|
37
|
+
* container element carrying any field the reference set does not declare
|
|
38
|
+
* therefore never matches, which is the ordinary case for rows read off a
|
|
39
|
+
* response body. `set-membership`'s set position projects a reference set to
|
|
40
|
+
* its single declared key and this position does not, because projecting only
|
|
41
|
+
* where a container happens to hold objects would make the operand's meaning
|
|
42
|
+
* depend on the evidence's runtime shape. An author reconciling rows against a
|
|
43
|
+
* declared set wants `covers-by-key`, which compares on named keys on both
|
|
44
|
+
* sides, or `for-all(rows, set-membership(@/key, { referenceSet }))`, which
|
|
45
|
+
* reads the declared key.
|
|
34
46
|
*/
|
|
35
47
|
export declare function containment(container: ResolvedValue, candidate: ResolvedValue | JsonValue[], artifactPath: string): boolean;
|
|
36
48
|
/**
|
|
@@ -62,6 +74,16 @@ export declare function countTolerance(collection: ResolvedValue, expected: numb
|
|
|
62
74
|
* that is required and not permitted reaches here and fails every value.
|
|
63
75
|
*/
|
|
64
76
|
export declare function shape(value: ResolvedValue, descriptor: KeyedShapeDescriptor, _artifactPath: string): boolean;
|
|
77
|
+
/**
|
|
78
|
+
* Own-property lookup only, so a key like `__proto__` reads as missing
|
|
79
|
+
* rather than inherited. Returns `ABSENT`, never throws (AD-4: a missing key
|
|
80
|
+
* resolves `false`, not an error).
|
|
81
|
+
*
|
|
82
|
+
* Exported so `resolution.ts`'s `set-membership` set-operand projection reads
|
|
83
|
+
* a declared member's key through this one spelling, keeping the own-property
|
|
84
|
+
* guard in one place.
|
|
85
|
+
*/
|
|
86
|
+
export declare function keyValueOf(element: JsonValue, key: string): JsonValue | typeof ABSENT;
|
|
65
87
|
/**
|
|
66
88
|
* AD-4's bijection: equal cardinality and a distinct `actual` match per
|
|
67
89
|
* `expected` element on the named keys. `ABSENT` on either side resolves
|
|
@@ -90,6 +90,18 @@ export function setMembership(value, set, artifactPath) {
|
|
|
90
90
|
* single element to search for: this function receives resolved values only,
|
|
91
91
|
* so it cannot tell a `{ literal }` array from a resolved `{ referenceSet }`.
|
|
92
92
|
* Known, accepted limitation.
|
|
93
|
+
*
|
|
94
|
+
* A separate property, and a separate limitation: a `{ referenceSet }`
|
|
95
|
+
* candidate matches **whole declared members** by structural equality. A
|
|
96
|
+
* container element carrying any field the reference set does not declare
|
|
97
|
+
* therefore never matches, which is the ordinary case for rows read off a
|
|
98
|
+
* response body. `set-membership`'s set position projects a reference set to
|
|
99
|
+
* its single declared key and this position does not, because projecting only
|
|
100
|
+
* where a container happens to hold objects would make the operand's meaning
|
|
101
|
+
* depend on the evidence's runtime shape. An author reconciling rows against a
|
|
102
|
+
* declared set wants `covers-by-key`, which compares on named keys on both
|
|
103
|
+
* sides, or `for-all(rows, set-membership(@/key, { referenceSet }))`, which
|
|
104
|
+
* reads the declared key.
|
|
93
105
|
*/
|
|
94
106
|
export function containment(container, candidate, artifactPath) {
|
|
95
107
|
if (container === ABSENT)
|
|
@@ -287,8 +299,12 @@ export function shape(value, descriptor, _artifactPath) {
|
|
|
287
299
|
* Own-property lookup only, so a key like `__proto__` reads as missing
|
|
288
300
|
* rather than inherited. Returns `ABSENT`, never throws (AD-4: a missing key
|
|
289
301
|
* resolves `false`, not an error).
|
|
302
|
+
*
|
|
303
|
+
* Exported so `resolution.ts`'s `set-membership` set-operand projection reads
|
|
304
|
+
* a declared member's key through this one spelling, keeping the own-property
|
|
305
|
+
* guard in one place.
|
|
290
306
|
*/
|
|
291
|
-
function keyValueOf(element, key) {
|
|
307
|
+
export function keyValueOf(element, key) {
|
|
292
308
|
if (!isPlainObject(element) || !Object.hasOwn(element, key))
|
|
293
309
|
return ABSENT;
|
|
294
310
|
// `Object.hasOwn` above already proves the key is present; `noUncheckedIndexedAccess`
|
|
@@ -3,8 +3,9 @@
|
|
|
3
3
|
* that turns an `Expression` into one `CheckResolutionValue`. Leaf operators,
|
|
4
4
|
* `covers-by-key` included, live in `operators.ts`. Operand resolution, every
|
|
5
5
|
* pointer form including the bound-element `@/` form, is injected;
|
|
6
|
-
* `evidence-resolution.ts` supplies it, and `ResolveOperand
|
|
7
|
-
* `PointerDenotesCollection` are the consumer-side
|
|
6
|
+
* `evidence-resolution.ts` supplies it, and `ResolveOperand`,
|
|
7
|
+
* `PointerDenotesCollection`, and `ReferenceSetKeys` are the consumer-side
|
|
8
|
+
* contract it satisfies.
|
|
8
9
|
*/
|
|
9
10
|
import type { CheckResolutionValue } from '../schemas/evidence-artifact.ts';
|
|
10
11
|
import type { Expression, Operand } from '../schemas/expression.ts';
|
|
@@ -31,10 +32,20 @@ export type ResolveOperand = (operand: Operand, boundElement: ResolvedValue, art
|
|
|
31
32
|
* soft-delete agreement pair (AD-4's own worked example).
|
|
32
33
|
*/
|
|
33
34
|
export type PointerDenotesCollection = (pointer: string) => boolean;
|
|
35
|
+
/**
|
|
36
|
+
* The `keys` each declared reference set names, by identifier.
|
|
37
|
+
* `set-membership`'s set position reads the single declared key off each
|
|
38
|
+
* member (`reference-set.ts`), and the members map the injected
|
|
39
|
+
* `ResolveOperand` closes over has already discarded the keys, so they travel
|
|
40
|
+
* separately. Plain data, because `PreflightPlan` carries this through to
|
|
41
|
+
* its reducer and is compared by value, which a closure fails.
|
|
42
|
+
* `evidence-resolution.ts` builds it from a contract.
|
|
43
|
+
*/
|
|
44
|
+
export type ReferenceSetKeys = Readonly<Record<string, readonly string[]>>;
|
|
34
45
|
/**
|
|
35
46
|
* The public entry point. Walks `expression` and produces one
|
|
36
47
|
* `CheckResolutionValue`, exactly what `Outcome.checkResolution` needs,
|
|
37
48
|
* unmodified. `boundElement` starts `ABSENT` at the root, where no quantifier
|
|
38
49
|
* has bound anything yet.
|
|
39
50
|
*/
|
|
40
|
-
export declare function resolveCheck(expression: Expression, resolveOperand: ResolveOperand, pointerDenotesCollection: PointerDenotesCollection, regexMatchStepBudget: number, artifactPath: string): CheckResolutionValue;
|
|
51
|
+
export declare function resolveCheck(expression: Expression, resolveOperand: ResolveOperand, pointerDenotesCollection: PointerDenotesCollection, referenceSetKeys: ReferenceSetKeys, regexMatchStepBudget: number, artifactPath: string): CheckResolutionValue;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { absence, containment, countTolerance, coversByKey, deepEquality, equality, existence, ordering, regexMatch, setMembership, shape, } from './operators.js';
|
|
1
|
+
import { absence, containment, countTolerance, coversByKey, deepEquality, equality, existence, keyValueOf, ordering, regexMatch, setMembership, shape, } from './operators.js';
|
|
2
2
|
import { ABSENT } from './resolved-value.js';
|
|
3
3
|
/**
|
|
4
4
|
* AD-4's one closed introduction condition, checked per operand and applied
|
|
@@ -242,10 +242,69 @@ function resolveRegexNode(expression, boundElement, ctx) {
|
|
|
242
242
|
const { pattern } = expression;
|
|
243
243
|
return resolveSingleOperand(operand, boundElement, ctx, (resolved) => regexMatch(resolved, pattern, ctx.regexMatchStepBudget, ctx.artifactPath));
|
|
244
244
|
}
|
|
245
|
+
/**
|
|
246
|
+
* Reads the single declared key off each member of a `{ referenceSet }` set
|
|
247
|
+
* operand. `reference-set.ts` declares members as objects so one declaration
|
|
248
|
+
* serves both operators, and states that "a `set-membership` operand against
|
|
249
|
+
* the same set reads the single named key". This is where that reading
|
|
250
|
+
* happens, and without it AD-20 rule 6's injection form can never answer
|
|
251
|
+
* `true`: the value operand resolves to a scalar, and `setMembership` compares
|
|
252
|
+
* whole members by digest, so a scalar against an object is always `false`.
|
|
253
|
+
*
|
|
254
|
+
* Scoped to this one operator position on purpose. `covers-by-key` projects
|
|
255
|
+
* both sides itself on `expectedKey` and `actualKey`, and `containment`
|
|
256
|
+
* matches whole members, so both want the members unprojected. The injected
|
|
257
|
+
* `ResolveOperand` is handed one operand at a time and cannot see which
|
|
258
|
+
* operator position it is filling, which is why the projection lives at the
|
|
259
|
+
* resolution site.
|
|
260
|
+
*
|
|
261
|
+
* The option turned down: an explicit `memberKey` field on `set-membership`
|
|
262
|
+
* mirroring `covers-by-key`'s `expectedKey`. It is symmetric with its sibling
|
|
263
|
+
* and it carries a multi-key set, at the price of a grammar change to a
|
|
264
|
+
* published artifact, an eval-contract `schemaVersion` bump, and
|
|
265
|
+
* published-schema drift, for a case no shipped contract has. Revisit it the
|
|
266
|
+
* first time a real contract wants a multi-key set in this position.
|
|
267
|
+
*
|
|
268
|
+
* Both throws are unreachable for a compiled contract:
|
|
269
|
+
* `checkOperandLegality` rejects a multi-key reference set in this position,
|
|
270
|
+
* and a member missing the declared key, under
|
|
271
|
+
* `malformed-operator-expression`. They throw, because a set this function
|
|
272
|
+
* cannot project carries no membership answer to give.
|
|
273
|
+
*/
|
|
274
|
+
function projectSetOperand(resolvedSet, setOperand, ctx) {
|
|
275
|
+
if (!('referenceSet' in setOperand))
|
|
276
|
+
return resolvedSet;
|
|
277
|
+
const { referenceSet } = setOperand;
|
|
278
|
+
// `Object.hasOwn`, the same prototype-chain guard `makeResolveOperand`
|
|
279
|
+
// applies to its own maps: `Identifier`'s charset admits `constructor`.
|
|
280
|
+
const keys = Object.hasOwn(ctx.referenceSetKeys, referenceSet)
|
|
281
|
+
? ctx.referenceSetKeys[referenceSet]
|
|
282
|
+
: undefined;
|
|
283
|
+
// No declared keys means the contract declares no such reference set,
|
|
284
|
+
// which compilation rejects under `unresolved-reference-set`; a non-array
|
|
285
|
+
// is the array guard below reporting whatever the resolver returned. Both
|
|
286
|
+
// pass through so exactly one guard speaks for each.
|
|
287
|
+
if (keys === undefined || !Array.isArray(resolvedSet))
|
|
288
|
+
return resolvedSet;
|
|
289
|
+
if (keys.length !== 1) {
|
|
290
|
+
throw new Error(`set-membership's set-operand projection: referenceSet "${referenceSet}" declares ${keys.length} keys, and this position reads exactly one. A multi-key set here is malformed-operator-expression, which compilation rejects.`);
|
|
291
|
+
}
|
|
292
|
+
const key = keys[0];
|
|
293
|
+
return resolvedSet.map((member) => {
|
|
294
|
+
const keyValue = keyValueOf(member, key);
|
|
295
|
+
if (keyValue === ABSENT) {
|
|
296
|
+
throw new Error(`set-membership's set-operand projection: a member of referenceSet "${referenceSet}" carries no own property "${key}", the key it declares. That is malformed-operator-expression, which compilation rejects.`);
|
|
297
|
+
}
|
|
298
|
+
return keyValue;
|
|
299
|
+
});
|
|
300
|
+
}
|
|
245
301
|
function resolveSetMembershipNode(expression, boundElement, ctx) {
|
|
246
302
|
const [valueOperand, setOperand] = expression.operands;
|
|
247
303
|
const value = ctx.resolveOperand(valueOperand, boundElement, ctx.artifactPath);
|
|
248
|
-
|
|
304
|
+
// Projected before the empty-collection interception and the array guard
|
|
305
|
+
// below, so both read the one value the operator will see. Projection
|
|
306
|
+
// preserves length and array-ness, so neither answer moves.
|
|
307
|
+
const resolvedSet = projectSetOperand(ctx.resolveOperand(setOperand, boundElement, ctx.artifactPath), setOperand, ctx);
|
|
249
308
|
if (anyOperandEmpty([
|
|
250
309
|
{ operand: valueOperand, resolved: value },
|
|
251
310
|
{ operand: setOperand, resolved: resolvedSet },
|
|
@@ -326,10 +385,11 @@ function resolveNode(expression, boundElement, ctx) {
|
|
|
326
385
|
* unmodified. `boundElement` starts `ABSENT` at the root, where no quantifier
|
|
327
386
|
* has bound anything yet.
|
|
328
387
|
*/
|
|
329
|
-
export function resolveCheck(expression, resolveOperand, pointerDenotesCollection, regexMatchStepBudget, artifactPath) {
|
|
388
|
+
export function resolveCheck(expression, resolveOperand, pointerDenotesCollection, referenceSetKeys, regexMatchStepBudget, artifactPath) {
|
|
330
389
|
return resolveNode(expression, ABSENT, {
|
|
331
390
|
resolveOperand,
|
|
332
391
|
pointerDenotesCollection,
|
|
392
|
+
referenceSetKeys,
|
|
333
393
|
regexMatchStepBudget,
|
|
334
394
|
artifactPath,
|
|
335
395
|
});
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AD-18's excluded categories, as patterns that read a value rather than a name.
|
|
3
|
+
*
|
|
4
|
+
* AD-18 excludes "credentials, tokens, real names, email addresses, account
|
|
5
|
+
* identifiers, and transaction content" from every artifact this package
|
|
6
|
+
* produces or publishes, and binds "published examples and test fixtures as
|
|
7
|
+
* strictly as real runs". Until this module the whole prohibition was carried by
|
|
8
|
+
* one test that scans `corpus/dev`, so a contract carrying a live-shaped token
|
|
9
|
+
* compiled clean and the token travelled into the sealed brief.
|
|
10
|
+
*
|
|
11
|
+
* Only the value-shaped half of that scan ships here, and the split is the point
|
|
12
|
+
* rather than an economy. AD-18's own reading of the rule for the header channel
|
|
13
|
+
* is that a declaration "names the header and its type and never carries a
|
|
14
|
+
* credential value", so a name is what an author is supposed to write: an
|
|
15
|
+
* operation declaring a `password` body key or an `Authorization` header is a
|
|
16
|
+
* contract for an authentication API, and a gate matching `\bpassword\b` would
|
|
17
|
+
* reject exactly the systems this package exists to evaluate. The patterns below
|
|
18
|
+
* therefore match only strings whose shape is decidable without knowing what
|
|
19
|
+
* field they sit in: a PEM header, a token with an issuer prefix, an address, a
|
|
20
|
+
* checkable account number, a card-shaped digit run. The name-shaped patterns
|
|
21
|
+
* stay in `tests/architecture/dev-corpus.test.ts`, where review rather than a
|
|
22
|
+
* compiler decides.
|
|
23
|
+
*
|
|
24
|
+
* Each pattern carries a string it must fire on, and a test asserts every one
|
|
25
|
+
* still does. A pattern that has rotted into a regex matching nothing is worse
|
|
26
|
+
* than no pattern: it reports clean forever.
|
|
27
|
+
*/
|
|
28
|
+
/** AD-18's six categories, in the rule's own order. */
|
|
29
|
+
export declare const EXCLUDED_CATEGORIES: readonly ['credentials', 'tokens', 'real names', 'email addresses', 'account identifiers', 'transaction content'];
|
|
30
|
+
export type ExcludedCategory = (typeof EXCLUDED_CATEGORIES)[number];
|
|
31
|
+
export type ExcludedContentPattern = {
|
|
32
|
+
readonly category: ExcludedCategory;
|
|
33
|
+
readonly regex: RegExp;
|
|
34
|
+
/** a string the pattern must match, so a rotted pattern fails its own test. */
|
|
35
|
+
readonly fires: string;
|
|
36
|
+
};
|
|
37
|
+
/**
|
|
38
|
+
* `real names` has no member here and never will: a personal name has no surface
|
|
39
|
+
* form, so every pattern that reaches one reads the field that carries it, which
|
|
40
|
+
* is a name-shaped test. The category stays in the tuple above because AD-18
|
|
41
|
+
* names six and dropping one from the vocabulary would make the gate look total.
|
|
42
|
+
*/
|
|
43
|
+
export declare const EXCLUDED_VALUE_PATTERNS: readonly ExcludedContentPattern[];
|
|
44
|
+
export type ExcludedContentHit = {
|
|
45
|
+
readonly category: ExcludedCategory;
|
|
46
|
+
/** the artifact-rooted path of the string that matched. */
|
|
47
|
+
readonly path: string;
|
|
48
|
+
/** the matched run, which is short by construction and is what a caller redacts. */
|
|
49
|
+
readonly match: string;
|
|
50
|
+
};
|
|
51
|
+
/**
|
|
52
|
+
* Every string anywhere inside one value, against every value-shaped pattern.
|
|
53
|
+
*
|
|
54
|
+
* Object keys are walked as paths and never scanned as content: a key is a name
|
|
55
|
+
* by construction, which is the same line the pattern set is drawn on. The walk
|
|
56
|
+
* is total over JSON and stops at nothing, so a token nested under a body
|
|
57
|
+
* example twelve levels down is found; the first matching category per string
|
|
58
|
+
* wins, since one string reported six times says nothing more than once.
|
|
59
|
+
*
|
|
60
|
+
* Order is the walk's order, which for an object is its own key order. A caller
|
|
61
|
+
* that needs a stable report over two spellings of the same object sorts what
|
|
62
|
+
* comes back; the compile gate below reads only the first hit and never the
|
|
63
|
+
* order.
|
|
64
|
+
*/
|
|
65
|
+
export declare function scanExcludedContent(value: unknown, path: string): readonly ExcludedContentHit[];
|