eval-quality 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/corpus/dev/README.md +3 -2
- package/corpus/dev/compile-seal-example/brief.json +1 -1
- package/corpus/dev/compile-seal-example/contract.json +1 -1
- package/corpus/dev/contracts/absent-collection-locations.json +1 -1
- package/corpus/dev/contracts/absent-sibling-groups.json +1 -1
- package/corpus/dev/contracts/absent-success-indicator.json +1 -1
- package/corpus/dev/contracts/empty-channel-roles.json +1 -1
- package/corpus/dev/contracts/empty-collection-locations.json +1 -1
- package/corpus/dev/contracts/empty-request-shapes.json +1 -1
- package/corpus/dev/contracts/empty-sibling-groups.json +1 -1
- package/corpus/dev/contracts/fragment-selection.json +1 -0
- package/corpus/dev/contracts/no-collection-quantifier.json +1 -1
- package/corpus/dev/contracts/no-operation-inventory.json +1 -1
- package/corpus/dev/contracts/no-read-back-relation.json +1 -1
- package/corpus/dev/contracts/no-state-change-marker.json +1 -1
- package/corpus/dev/contracts/no-type-violating-step.json +1 -1
- package/corpus/dev/contracts/per-key-split-oracles.json +1 -1
- package/corpus/dev/contracts/review-corpus.json +1 -0
- package/corpus/dev/contracts/satisfied-declarations.json +1 -1
- package/corpus/dev/contracts/single-required-response-key.json +1 -1
- package/corpus/dev/contracts/split-indicator-oracle.json +1 -1
- package/corpus/dev/contracts/unaddressed-parameter-sibling.json +1 -1
- package/corpus/dev/contracts/unnamed-reference-set.json +1 -1
- package/corpus/dev/contracts/wrong-cardinality-form.json +1 -1
- package/corpus/dev/index.json +1 -1
- package/dist/cli/render.d.ts +2 -5
- package/dist/cli/render.js +51 -1
- package/dist/core/compile/bindings.d.ts +26 -7
- package/dist/core/compile/bindings.js +49 -32
- package/dist/core/compile/compile.d.ts +12 -0
- package/dist/core/compile/compile.js +28 -1
- package/dist/core/compile/excluded-content.d.ts +11 -0
- package/dist/core/compile/excluded-content.js +42 -0
- package/dist/core/compile/expression-legality.d.ts +2 -2
- package/dist/core/compile/expression-legality.js +58 -12
- package/dist/core/compile/interface-inventory.d.ts +34 -1
- package/dist/core/compile/interface-inventory.js +88 -11
- package/dist/core/compile/reachability.d.ts +12 -2
- package/dist/core/compile/reachability.js +123 -36
- package/dist/core/compile/schema-version.d.ts +2 -0
- package/dist/core/compile/schema-version.js +25 -0
- package/dist/core/compile/sensitivity-witness.d.ts +31 -12
- package/dist/core/compile/sensitivity-witness.js +110 -23
- package/dist/core/compile/step-reference.d.ts +2 -0
- package/dist/core/compile/step-reference.js +49 -0
- package/dist/core/coverage/operations.d.ts +62 -0
- package/dist/core/coverage/operations.js +57 -0
- package/dist/core/coverage/relevance.d.ts +4 -2
- package/dist/core/coverage/relevance.js +22 -23
- package/dist/core/coverage/satisfaction.d.ts +2 -2
- package/dist/core/coverage/satisfaction.js +73 -48
- package/dist/core/declared-inputs.d.ts +83 -4
- package/dist/core/declared-inputs.js +105 -8
- package/dist/core/evaluate/evidence-resolution.d.ts +9 -12
- package/dist/core/evaluate/evidence-resolution.js +70 -10
- package/dist/core/evaluate/operators.d.ts +22 -0
- package/dist/core/evaluate/operators.js +17 -1
- package/dist/core/evaluate/resolution.d.ts +14 -3
- package/dist/core/evaluate/resolution.js +63 -3
- package/dist/core/excluded-content.d.ts +65 -0
- package/dist/core/excluded-content.js +113 -0
- package/dist/core/failure-codes.d.ts +2 -2
- package/dist/core/failure-codes.js +5 -2
- package/dist/core/ingest/conditions.d.ts +1 -1
- package/dist/core/ingest/ingest.js +8 -0
- package/dist/core/preflight/plan.d.ts +7 -5
- package/dist/core/preflight/plan.js +75 -32
- package/dist/core/preflight/projection.d.ts +6 -3
- package/dist/core/preflight/projection.js +22 -2
- package/dist/core/preflight/reduce.js +47 -7
- package/dist/core/preflight/witness-evidence.d.ts +5 -5
- package/dist/core/preflight/witness-evidence.js +63 -20
- package/dist/core/schemas/artifact.d.ts +683 -35
- package/dist/core/schemas/constraint-ledger.js +11 -0
- package/dist/core/schemas/defect-signature.d.ts +438 -16
- package/dist/core/schemas/defect-signature.js +58 -10
- package/dist/core/schemas/eval-contract.d.ts +517 -12
- package/dist/core/schemas/eval-contract.js +18 -2
- package/dist/core/schemas/evidence-artifact.d.ts +14 -4
- package/dist/core/schemas/evidence-artifact.js +1 -1
- package/dist/core/schemas/interface.d.ts +741 -13
- package/dist/core/schemas/interface.js +120 -7
- package/dist/core/schemas/isolation-manifest.js +16 -9
- package/dist/core/schemas/plan.d.ts +217 -3
- package/dist/core/schemas/plan.js +22 -1
- package/dist/core/schemas/pointer.d.ts +47 -1
- package/dist/core/schemas/pointer.js +89 -8
- package/dist/core/schemas/port-messages.d.ts +194 -10
- package/dist/core/schemas/port-messages.js +73 -6
- package/dist/core/schemas/probe-body.d.ts +18 -0
- package/dist/core/schemas/probe-body.js +13 -0
- package/dist/core/schemas/probe.d.ts +122 -3
- package/dist/core/schemas/probe.js +1 -1
- package/dist/core/schemas/sealed-run-record.d.ts +131 -9
- package/dist/core/schemas/sealed-run-record.js +56 -11
- package/dist/core/schemas/sensitivity-witness.d.ts +108 -6
- package/dist/core/schemas/sensitivity-witness.js +61 -5
- package/dist/core/score/bindings.d.ts +2 -2
- package/dist/core/score/bindings.js +25 -13
- package/dist/core/score/qualification.d.ts +5 -5
- package/dist/core/score/qualification.js +67 -37
- package/dist/core/score/quotation.d.ts +1 -24
- package/dist/core/score/quotation.js +29 -4
- package/dist/core/score/score.js +28 -2
- package/dist/core/score/strength.d.ts +9 -0
- package/dist/core/score/strength.js +35 -1
- package/dist/core/score/witness.d.ts +21 -0
- package/dist/core/score/witness.js +9 -4
- package/dist/core/seal/derived-reference.d.ts +2 -9
- package/dist/core/seal/derived-reference.js +62 -23
- package/dist/core/seal/plan-index.d.ts +18 -3
- package/dist/core/seal/plan-index.js +44 -8
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/ports/environment-probe-port.d.ts +60 -4
- package/dist/testing/probe-conformance.d.ts +29 -0
- package/dist/testing/probe-conformance.js +34 -5
- package/package.json +1 -1
- package/schemas/eval-contract.schema.json +1356 -464
- package/schemas/evidence-artifact.schema.json +59 -29
- package/schemas/isolation-manifest.schema.json +17 -10
- package/schemas/probe.schema.json +197 -71
- package/schemas/rubric.schema.json +1 -1
- package/schemas/sealed-run-record.schema.json +296 -39
|
@@ -1,10 +1,23 @@
|
|
|
1
1
|
import type { EvalContract } from '../schemas/eval-contract.ts';
|
|
2
2
|
import type { InteractionStep } from '../schemas/plan.ts';
|
|
3
|
-
import {
|
|
3
|
+
import type { InputChannelName } from '../schemas/pointer.ts';
|
|
4
4
|
import { type EvidenceTarget } from '../seal/plan-index.ts';
|
|
5
|
+
/**
|
|
6
|
+
* The one channel a captured pointer may name: whichever channel the
|
|
7
|
+
* referenced operation's own response descriptor describes.
|
|
8
|
+
*
|
|
9
|
+
* `ResponseDescriptor` declares `requiredKeys`, `permittedKeys`, `types`,
|
|
10
|
+
* `successIndicator`, `channelRoles`, and `collectionLocations`, and every one
|
|
11
|
+
* of them is about the channel the operation nominates. Off an interface that
|
|
12
|
+
* speaks HTTP that is the response body; off a command it is the stream the
|
|
13
|
+
* operation names. `response-headers` and `response-status` are never it,
|
|
14
|
+
* because `Observation.responseHeaders` admits objects, arrays, numbers, and
|
|
15
|
+
* `null`, so a header capture compiled as a `string` could resolve to an
|
|
16
|
+
* object at score time.
|
|
17
|
+
*/
|
|
5
18
|
/** One `{ captured }` binding, resolved to the pointer target it addresses. */
|
|
6
19
|
export type CapturedBinding = {
|
|
7
|
-
readonly transportChannel:
|
|
20
|
+
readonly transportChannel: InputChannelName;
|
|
8
21
|
readonly key: string;
|
|
9
22
|
readonly pointer: string;
|
|
10
23
|
readonly target: EvidenceTarget;
|
|
@@ -45,11 +58,17 @@ export declare function capturedBindings(step: InteractionStep): readonly Captur
|
|
|
45
58
|
export declare function checkBindingCycle(contract: EvalContract): void;
|
|
46
59
|
/**
|
|
47
60
|
* `captured-channel-undeclared`: a captured pointer naming any AD-26 channel
|
|
48
|
-
* but
|
|
49
|
-
*
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
*
|
|
61
|
+
* but the one the referenced operation's response descriptor describes. That
|
|
62
|
+
* is `response-body` off an interface that speaks HTTP and whichever channel
|
|
63
|
+
* `descriptorChannel` nominates off one that runs behind a command, which is
|
|
64
|
+
* the same rule read against the declaration rather than assumed.
|
|
65
|
+
*
|
|
66
|
+
* Every other channel is refused for a reason that does not depend on the
|
|
67
|
+
* kind: `call-inputs` addresses a step's own request, and `response-headers`,
|
|
68
|
+
* `response-status`, and `exit-code` have no declared structure to give a
|
|
69
|
+
* captured value a type. On the `artifact` channel the identifier is compared
|
|
70
|
+
* too, so a capture from a file the operation writes but does not describe is
|
|
71
|
+
* refused alongside one from a file it never writes.
|
|
53
72
|
*/
|
|
54
73
|
export declare function checkCapturedChannel(contract: EvalContract): void;
|
|
55
74
|
/**
|
|
@@ -15,25 +15,12 @@
|
|
|
15
15
|
* scalar-descent half. The scalar determination, the one-segment tail rule, and
|
|
16
16
|
* type equality are this module's own.
|
|
17
17
|
*/
|
|
18
|
+
import { boundChannelsOf, descriptorChannelOf, requestShapeOf, targetsDescribedChannel, } from '../declared-inputs.js';
|
|
18
19
|
import { ARRAY_INDEX_PATTERN } from '../evaluate/evidence-resolution.js';
|
|
19
20
|
import { StructuralFailure } from '../failure-codes.js';
|
|
20
|
-
import { TRANSPORT_CHANNELS, } from '../schemas/pointer.js';
|
|
21
21
|
import { JsonTypeName } from '../schemas/primitives.js';
|
|
22
|
-
import { buildPlanIndex, parseEvidenceTarget,
|
|
22
|
+
import { anyOperationOf, buildPlanIndex, parseEvidenceTarget, resolveStep, } from '../seal/plan-index.js';
|
|
23
23
|
import { evaluatePointerReachability } from './reachability.js';
|
|
24
|
-
/**
|
|
25
|
-
* The one channel a captured pointer may name. `ResponseDescriptor` declares
|
|
26
|
-
* `requiredKeys`, `permittedKeys`, `types`, `successIndicator`, `channelRoles`,
|
|
27
|
-
* and `collectionLocations`, every one of them about the body, so the body is
|
|
28
|
-
* the one channel it declares and the criterion's "a channel the referenced
|
|
29
|
-
* operation's response descriptor does not declare" is literally true of the
|
|
30
|
-
* other six. Admitting `response-headers` and `response-status` was tried and
|
|
31
|
-
* dropped: their types would have to be invented by fiat, and
|
|
32
|
-
* `Observation.responseHeaders` admits objects, arrays, numbers, and `null`, so
|
|
33
|
-
* a header capture compiled as a `string` could resolve to an object at score
|
|
34
|
-
* time.
|
|
35
|
-
*/
|
|
36
|
-
const CAPTURABLE_CHANNEL = 'response-body';
|
|
37
24
|
/**
|
|
38
25
|
* Every captured binding one step declares, in fixed transport-channel order
|
|
39
26
|
* then by key name, so which binding a check reports never depends on a
|
|
@@ -42,8 +29,7 @@ const CAPTURABLE_CHANNEL = 'response-body';
|
|
|
42
29
|
*/
|
|
43
30
|
export function capturedBindings(step) {
|
|
44
31
|
const captures = [];
|
|
45
|
-
for (const transportChannel of
|
|
46
|
-
const map = step.inputBinding[transportChannel];
|
|
32
|
+
for (const { channel: transportChannel, bound: map } of boundChannelsOf(step.inputBinding)) {
|
|
47
33
|
if (map === null)
|
|
48
34
|
continue;
|
|
49
35
|
for (const key of Object.keys(map).sort()) {
|
|
@@ -172,17 +158,38 @@ function stronglyConnectedComponents(nodes, edgesFrom) {
|
|
|
172
158
|
// ---- captured-channel-undeclared -----------------------------------------
|
|
173
159
|
/**
|
|
174
160
|
* `captured-channel-undeclared`: a captured pointer naming any AD-26 channel
|
|
175
|
-
* but
|
|
176
|
-
*
|
|
177
|
-
*
|
|
178
|
-
*
|
|
179
|
-
*
|
|
161
|
+
* but the one the referenced operation's response descriptor describes. That
|
|
162
|
+
* is `response-body` off an interface that speaks HTTP and whichever channel
|
|
163
|
+
* `descriptorChannel` nominates off one that runs behind a command, which is
|
|
164
|
+
* the same rule read against the declaration rather than assumed.
|
|
165
|
+
*
|
|
166
|
+
* Every other channel is refused for a reason that does not depend on the
|
|
167
|
+
* kind: `call-inputs` addresses a step's own request, and `response-headers`,
|
|
168
|
+
* `response-status`, and `exit-code` have no declared structure to give a
|
|
169
|
+
* captured value a type. On the `artifact` channel the identifier is compared
|
|
170
|
+
* too, so a capture from a file the operation writes but does not describe is
|
|
171
|
+
* refused alongside one from a file it never writes.
|
|
180
172
|
*/
|
|
181
173
|
export function checkCapturedChannel(contract) {
|
|
174
|
+
let index;
|
|
182
175
|
for (const step of contract.interactionPlan) {
|
|
183
176
|
for (const capture of capturedBindings(step)) {
|
|
184
|
-
|
|
177
|
+
index ??= buildPlanIndex(contract.interactionPlan, contract.permittedInterfaces, { duplicateIds: 'unresolved' });
|
|
178
|
+
const referenced = index.stepOf(capture.target.stepId);
|
|
179
|
+
const operation = referenced === undefined
|
|
180
|
+
? undefined
|
|
181
|
+
: anyOperationOf(index, referenced.operationId);
|
|
182
|
+
// An unresolvable reference is `checkCapturedReachability`'s, at a
|
|
183
|
+
// higher rung. With nothing to ask, this check falls back to the
|
|
184
|
+
// api answer so an unresolvable off-body capture still reports.
|
|
185
|
+
const capturable = operation === undefined
|
|
186
|
+
? 'response-body'
|
|
187
|
+
: descriptorChannelOf(operation);
|
|
188
|
+
if (operation === undefined
|
|
189
|
+
? capture.target.channel === capturable
|
|
190
|
+
: targetsDescribedChannel(operation, capture.target)) {
|
|
185
191
|
continue;
|
|
192
|
+
}
|
|
186
193
|
throw new StructuralFailure('captured-channel-undeclared', bindingPath(step, capture), `captured pointer "${capture.pointer}" names the ${capture.target.channel} channel, which the referenced operation's response descriptor does not declare (AD-26)`);
|
|
187
194
|
}
|
|
188
195
|
}
|
|
@@ -197,10 +204,11 @@ const SCALAR_TYPES = new Set(JsonTypeName.options.filter((name) => name !== 'obj
|
|
|
197
204
|
* `evaluatePointerReachability` admits the index against a root collection.
|
|
198
205
|
*/
|
|
199
206
|
function capturedType(target, index) {
|
|
207
|
+
const channel = target.channel;
|
|
200
208
|
const segments = target.tail.length;
|
|
201
209
|
if (segments !== 1) {
|
|
202
210
|
return {
|
|
203
|
-
reason: `addresses ${segments === 0 ?
|
|
211
|
+
reason: `addresses ${segments === 0 ? `the whole ${channel}` : `a ${channel} path ${segments} segments deep`}, which declares no scalar to capture`,
|
|
204
212
|
};
|
|
205
213
|
}
|
|
206
214
|
const key = target.tail[0];
|
|
@@ -210,20 +218,23 @@ function capturedType(target, index) {
|
|
|
210
218
|
}
|
|
211
219
|
if (ARRAY_INDEX_PATTERN.test(key)) {
|
|
212
220
|
return {
|
|
213
|
-
reason: `addresses
|
|
221
|
+
reason: `addresses ${channel} element ${key}, which no declaration gives a type`,
|
|
214
222
|
};
|
|
215
223
|
}
|
|
216
224
|
const step = resolveStep(index, target.stepId);
|
|
217
|
-
const operation =
|
|
225
|
+
const operation = anyOperationOf(index, step.operationId);
|
|
226
|
+
if (operation === undefined) {
|
|
227
|
+
throw new TypeError(`step names an operation the permitted interfaces do not declare: ${step.operationId}`);
|
|
228
|
+
}
|
|
218
229
|
const declared = operation.responseDescriptor.types[key];
|
|
219
230
|
if (declared === undefined || declared === null) {
|
|
220
231
|
return {
|
|
221
|
-
reason: `addresses
|
|
232
|
+
reason: `addresses ${channel} field "${key}", whose type operation "${operation.operationId}" ${declared === undefined ? 'does not declare' : 'declares indeterminate'}`,
|
|
222
233
|
};
|
|
223
234
|
}
|
|
224
235
|
if (!SCALAR_TYPES.has(declared)) {
|
|
225
236
|
return {
|
|
226
|
-
reason: `addresses
|
|
237
|
+
reason: `addresses ${channel} field "${key}", which operation "${operation.operationId}" declares "${declared}" rather than a scalar`,
|
|
227
238
|
};
|
|
228
239
|
}
|
|
229
240
|
return { type: declared };
|
|
@@ -237,10 +248,15 @@ function capturedType(target, index) {
|
|
|
237
248
|
* cross-field rule.
|
|
238
249
|
*/
|
|
239
250
|
function boundParameterType(step, capture, index) {
|
|
240
|
-
const operation = index
|
|
251
|
+
const operation = anyOperationOf(index, step.operationId);
|
|
241
252
|
if (operation === undefined)
|
|
242
253
|
return null;
|
|
243
|
-
const shape = operation
|
|
254
|
+
const shape = requestShapeOf(operation, capture.transportChannel);
|
|
255
|
+
// A channel the operation does not accept input on declares no type for
|
|
256
|
+
// the key either, so there is nothing to compare and the check abstains
|
|
257
|
+
// for the same reason the undeclared-key branch below does.
|
|
258
|
+
if (shape === undefined)
|
|
259
|
+
return null;
|
|
244
260
|
// A key the operation declares in neither list is an input the contract did
|
|
245
261
|
// not declare, which is `undeclared-mandatory-input`'s and strict-only under
|
|
246
262
|
// AD-4. This check runs unconditionally, so claiming it here would reject a
|
|
@@ -290,10 +306,11 @@ export function checkCapturedReachability(contract) {
|
|
|
290
306
|
if (referenced === undefined) {
|
|
291
307
|
throw new StructuralFailure('unreachable-check-evidence', path, `captured pointer "${capture.pointer}" names a step the interaction plan does not declare`);
|
|
292
308
|
}
|
|
293
|
-
|
|
309
|
+
const operation = anyOperationOf(index, referenced.operationId);
|
|
310
|
+
if (operation === undefined) {
|
|
294
311
|
throw new StructuralFailure('unreachable-check-evidence', path, `captured pointer "${capture.pointer}" names step "${capture.target.stepId}", which names operation "${referenced.operationId}", not declared by any permitted interface`);
|
|
295
312
|
}
|
|
296
|
-
if (capture.target
|
|
313
|
+
if (!targetsDescribedChannel(operation, capture.target))
|
|
297
314
|
continue;
|
|
298
315
|
const reachability = evaluatePointerReachability(capture.pointer, index);
|
|
299
316
|
if (!reachability.reachable) {
|
|
@@ -18,6 +18,18 @@
|
|
|
18
18
|
* Grouping it with the two codes owed item 3 added, at the inserted position
|
|
19
19
|
* below, would let a lower-ranked code win on a contract carrying both defects.
|
|
20
20
|
*
|
|
21
|
+
* `checkArtifactReferences` fires `unresolved-artifact-reference`, the
|
|
22
|
+
* registry's last code, but runs near the front for the reason its own call
|
|
23
|
+
* site records: it names a fault that reachability can only describe the
|
|
24
|
+
* consequence of, and reachability abstains on the same pointer so the two
|
|
25
|
+
* never race.
|
|
26
|
+
*
|
|
27
|
+
* `checkStepReferenceReducibility` runs last, after every other check has had
|
|
28
|
+
* its say. It is the only check that reports what a LATER stage cannot do
|
|
29
|
+
* rather than what this contract declares wrongly, so a contract with an
|
|
30
|
+
* ordinary declaration fault should hear about that fault rather than about
|
|
31
|
+
* the sealing consequence of it.
|
|
32
|
+
*
|
|
21
33
|
* `checkRubricIdentifiers` fires `rubric-unanchored` but runs ahead of
|
|
22
34
|
* `checkRubricReasoningProse`, which outranks it in the registry. A duplicated
|
|
23
35
|
* rubric or criterion id makes every `rubrics[id=...]` path the other three
|
|
@@ -18,26 +18,51 @@
|
|
|
18
18
|
* Grouping it with the two codes owed item 3 added, at the inserted position
|
|
19
19
|
* below, would let a lower-ranked code win on a contract carrying both defects.
|
|
20
20
|
*
|
|
21
|
+
* `checkArtifactReferences` fires `unresolved-artifact-reference`, the
|
|
22
|
+
* registry's last code, but runs near the front for the reason its own call
|
|
23
|
+
* site records: it names a fault that reachability can only describe the
|
|
24
|
+
* consequence of, and reachability abstains on the same pointer so the two
|
|
25
|
+
* never race.
|
|
26
|
+
*
|
|
27
|
+
* `checkStepReferenceReducibility` runs last, after every other check has had
|
|
28
|
+
* its say. It is the only check that reports what a LATER stage cannot do
|
|
29
|
+
* rather than what this contract declares wrongly, so a contract with an
|
|
30
|
+
* ordinary declaration fault should hear about that fault rather than about
|
|
31
|
+
* the sealing consequence of it.
|
|
32
|
+
*
|
|
21
33
|
* `checkRubricIdentifiers` fires `rubric-unanchored` but runs ahead of
|
|
22
34
|
* `checkRubricReasoningProse`, which outranks it in the registry. A duplicated
|
|
23
35
|
* rubric or criterion id makes every `rubrics[id=...]` path the other three
|
|
24
36
|
* rubric checks emit address two things, so identifiers are settled before any
|
|
25
37
|
* of them reports.
|
|
26
38
|
*/
|
|
39
|
+
import { EVAL_CONTRACT_SCHEMA_VERSION } from '../schemas/eval-contract.js';
|
|
27
40
|
import { checkBindingCycle, checkCapturedChannel, checkCapturedReachability, } from './bindings.js';
|
|
28
41
|
import { checkObservableSuccessCriterion, checkRequirementLinkage, } from './declarations.js';
|
|
42
|
+
import { checkExcludedContent } from './excluded-content.js';
|
|
29
43
|
import { checkOperandLegality, checkQuantifierNesting, checkQuantifierOverNonCollection, checkReferenceSetResolution, checkRegexConstructs, } from './expression-legality.js';
|
|
30
44
|
import { checkForbiddenInputFloor, checkScopedResourceReferences, } from './forbidden-inputs.js';
|
|
31
|
-
import { checkDuplicateOperationSignature, checkInterfaceKind, checkUndeclaredMandatoryInput, } from './interface-inventory.js';
|
|
45
|
+
import { checkArtifactReferences, checkDuplicateOperationSignature, checkInterfaceKind, checkUndeclaredMandatoryInput, } from './interface-inventory.js';
|
|
32
46
|
import { checkOracleAlignment, checkOracleChannel } from './oracle-alignment.js';
|
|
33
47
|
import { checkBoundElementScope, checkEvidenceReachability, } from './reachability.js';
|
|
34
48
|
import { checkRubricAnchoring, checkRubricEvidenceReachability, checkRubricIdentifiers, checkRubricReasoningProse, } from './rubrics.js';
|
|
49
|
+
import { checkSchemaVersion } from './schema-version.js';
|
|
35
50
|
import { checkNestedTemporalClause, checkScriptingBound, } from './scripting-bound.js';
|
|
36
51
|
import { checkSensitivityWitnessDeclared, checkWitnessLegality, checkWitnessLegIdentifiers, } from './sensitivity-witness.js';
|
|
52
|
+
import { checkStepReferenceReducibility } from './step-reference.js';
|
|
37
53
|
import { checkWaiverCompleteness } from './waivers.js';
|
|
38
54
|
export function compile(contract, options) {
|
|
55
|
+
// First, and before any check reads a declaration. AD-11 makes an unequal
|
|
56
|
+
// stamp a rejection rather than a degraded read, and every check below is
|
|
57
|
+
// written against this version's field shapes.
|
|
58
|
+
checkSchemaVersion(contract.schemaVersion, EVAL_CONTRACT_SCHEMA_VERSION, 'EvalContract.schemaVersion');
|
|
39
59
|
checkRequirementLinkage(contract);
|
|
40
60
|
checkObservableSuccessCriterion(contract);
|
|
61
|
+
// Ahead of reachability, because a pointer naming an artifact nothing
|
|
62
|
+
// declares has no operation shape to be reachable against, and the code
|
|
63
|
+
// that names the fault directly should not lose the race to the code that
|
|
64
|
+
// describes its consequence.
|
|
65
|
+
checkArtifactReferences(contract);
|
|
41
66
|
checkEvidenceReachability(contract);
|
|
42
67
|
checkCapturedReachability(contract);
|
|
43
68
|
checkBoundElementScope(contract);
|
|
@@ -63,8 +88,10 @@ export function compile(contract, options) {
|
|
|
63
88
|
checkRubricAnchoring(contract);
|
|
64
89
|
checkRubricEvidenceReachability(contract);
|
|
65
90
|
checkForbiddenInputFloor(contract);
|
|
91
|
+
checkExcludedContent(contract);
|
|
66
92
|
checkScopedResourceReferences(contract);
|
|
67
93
|
checkWaiverCompleteness(contract);
|
|
94
|
+
checkStepReferenceReducibility(contract);
|
|
68
95
|
// Identifiers before legality. A duplicated or plan-colliding leg id makes
|
|
69
96
|
// the legality check's question ("does the relation address both legs?")
|
|
70
97
|
// unanswerable, so legality-first reports an unreachable-evidence failure on
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { EvalContract } from '../schemas/eval-contract.ts';
|
|
2
|
+
/**
|
|
3
|
+
* `excluded-content-in-declaration`: the contract carries a value-shaped secret.
|
|
4
|
+
*
|
|
5
|
+
* The first hit is reported and the rest are dropped. A contract carrying one
|
|
6
|
+
* has to be re-authored before it compiles at all, so enumerating the others
|
|
7
|
+
* would print more of the secret material into a failure message that gets
|
|
8
|
+
* pasted into a terminal, an issue, and a log. The message names the category
|
|
9
|
+
* and the path and quotes nothing.
|
|
10
|
+
*/
|
|
11
|
+
export declare function checkExcludedContent(contract: EvalContract): void;
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AD-18's exclusion, read against the contract rather than against a directory.
|
|
3
|
+
*
|
|
4
|
+
* AD-18 excludes credentials, tokens, real names, email addresses, account
|
|
5
|
+
* identifiers, and transaction content from every artifact this package produces
|
|
6
|
+
* or publishes, and says so binds "published examples and test fixtures as
|
|
7
|
+
* strictly as real runs". The only mechanism enforcing it was a test that greps
|
|
8
|
+
* `corpus/dev`, which reaches the corpus this repository ships and no contract an
|
|
9
|
+
* adopter authors. A token written into a contract therefore compiled clean and
|
|
10
|
+
* travelled onward: `seal` copies permitted interfaces onto the sealed brief, so
|
|
11
|
+
* the value reaches the one artifact AD-16 hands to an evaluator, and `emit`
|
|
12
|
+
* copies quoted evidence onto the evidence artifact.
|
|
13
|
+
*
|
|
14
|
+
* The whole contract is scanned, not a chosen subtree. Choosing one would have to
|
|
15
|
+
* be right about where an author pastes a secret, and the answer is wherever the
|
|
16
|
+
* field they were filling in happened to be: a witness input value, a fixture
|
|
17
|
+
* reset payload, an oracle's expected operand, a behaviour description. The
|
|
18
|
+
* scan's cost is one walk of an already-parsed object.
|
|
19
|
+
*
|
|
20
|
+
* Unconditional rather than strict-gated, unlike `undeclared-mandatory-input`
|
|
21
|
+
* next to it. That code is strict-only because AD-4 leaves a contract two
|
|
22
|
+
* legitimate readings of how completely it declares its inputs. AD-18 has one
|
|
23
|
+
* reading and no lenient mode: a credential that reaches a published artifact is
|
|
24
|
+
* unrecoverable, which is the harm the decision names.
|
|
25
|
+
*/
|
|
26
|
+
import { scanExcludedContent } from '../excluded-content.js';
|
|
27
|
+
import { StructuralFailure } from '../failure-codes.js';
|
|
28
|
+
/**
|
|
29
|
+
* `excluded-content-in-declaration`: the contract carries a value-shaped secret.
|
|
30
|
+
*
|
|
31
|
+
* The first hit is reported and the rest are dropped. A contract carrying one
|
|
32
|
+
* has to be re-authored before it compiles at all, so enumerating the others
|
|
33
|
+
* would print more of the secret material into a failure message that gets
|
|
34
|
+
* pasted into a terminal, an issue, and a log. The message names the category
|
|
35
|
+
* and the path and quotes nothing.
|
|
36
|
+
*/
|
|
37
|
+
export function checkExcludedContent(contract) {
|
|
38
|
+
const hit = scanExcludedContent(contract, 'EvalContract')[0];
|
|
39
|
+
if (hit === undefined)
|
|
40
|
+
return;
|
|
41
|
+
throw new StructuralFailure('excluded-content-in-declaration', hit.path, `carries a value shaped like ${hit.category}, which AD-18 excludes from every artifact this package produces; store a digest or an AD-8 opaque reference instead. The matched text is not quoted here, since a failure message travels further than the contract does`);
|
|
42
|
+
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { EvalContract } from '../schemas/eval-contract.ts';
|
|
2
2
|
import type { Expression, Operand, SetOperand } from '../schemas/expression.ts';
|
|
3
|
-
import type {
|
|
3
|
+
import type { AnyOperation } from '../schemas/interface.ts';
|
|
4
4
|
type OperandPosition = 0 | 1 | 'collection';
|
|
5
5
|
type OperandBearingOp = Exclude<Expression['op'], 'not' | 'all' | 'any'>;
|
|
6
6
|
type Visitor = {
|
|
@@ -50,7 +50,7 @@ export declare function checkQuantifierNesting(contract: EvalContract): void;
|
|
|
50
50
|
* disagree with the compiler on a committed fixture.
|
|
51
51
|
*/
|
|
52
52
|
export declare function checkExpressionQuantifierOverNonCollection(expression: Expression, artifactPath: string, scope: {
|
|
53
|
-
readonly operation:
|
|
53
|
+
readonly operation: AnyOperation;
|
|
54
54
|
readonly legIds: readonly string[];
|
|
55
55
|
}): void;
|
|
56
56
|
/** Checks response-body collection pointers, after bound-element substitution. */
|
|
@@ -3,9 +3,11 @@
|
|
|
3
3
|
* reference-set resolution. Each check reports the first structural failure.
|
|
4
4
|
*/
|
|
5
5
|
import { digestArtifact } from '../canonical/digest.js';
|
|
6
|
+
import { targetsDescribedChannel } from '../declared-inputs.js';
|
|
6
7
|
import { StructuralFailure } from '../failure-codes.js';
|
|
8
|
+
import { operationsOf } from '../schemas/interface.js';
|
|
7
9
|
import { JsonTypeName } from '../schemas/primitives.js';
|
|
8
|
-
import { buildPlanIndex, parseEvidenceTarget, } from '../seal/plan-index.js';
|
|
10
|
+
import { anyOperationOf, buildPlanIndex, parseEvidenceTarget, } from '../seal/plan-index.js';
|
|
9
11
|
import { substitutePointer } from './oracle-alignment.js';
|
|
10
12
|
/**
|
|
11
13
|
* The one expression walker in this package. Contract-free and a superset of
|
|
@@ -68,7 +70,7 @@ function forEachContractExpression(contract, visit) {
|
|
|
68
70
|
});
|
|
69
71
|
});
|
|
70
72
|
contract.permittedInterfaces.forEach((iface, interfaceIndex) => {
|
|
71
|
-
iface.
|
|
73
|
+
operationsOf(iface).forEach((operation, operationIndex) => {
|
|
72
74
|
const witness = operation.sensitivityWitness;
|
|
73
75
|
if (witness === null)
|
|
74
76
|
return;
|
|
@@ -180,6 +182,19 @@ export function checkExpressionQuantifierNesting(expression, artifactPath) {
|
|
|
180
182
|
},
|
|
181
183
|
});
|
|
182
184
|
}
|
|
185
|
+
/**
|
|
186
|
+
* The declaration a reference-set operand names, and `undefined` when the
|
|
187
|
+
* contract declares none by that identifier. An undeclared identifier is
|
|
188
|
+
* `checkReferenceSetResolution`'s `unresolved-reference-set`, so the
|
|
189
|
+
* shape-aware checks below stay quiet about it.
|
|
190
|
+
*/
|
|
191
|
+
function declaredReferenceSetOf(contract, referenceSetId) {
|
|
192
|
+
const declarations = contract.referenceSets;
|
|
193
|
+
if (declarations === null || !Object.hasOwn(declarations, referenceSetId)) {
|
|
194
|
+
return undefined;
|
|
195
|
+
}
|
|
196
|
+
return declarations[referenceSetId];
|
|
197
|
+
}
|
|
183
198
|
/** Checks each operator's operands against its position-specific constraints. */
|
|
184
199
|
export function checkOperandLegality(contract) {
|
|
185
200
|
forEachContractExpression(contract, (site) => {
|
|
@@ -187,22 +202,50 @@ export function checkOperandLegality(contract) {
|
|
|
187
202
|
onOperand: (operand, op, position, path) => {
|
|
188
203
|
checkOperandAtPosition(operand, op, position, `${site.artifactPath}${path}`);
|
|
189
204
|
},
|
|
205
|
+
// Fires only for `set-membership`'s set position, which is the one
|
|
206
|
+
// position that reads a reference set by its single declared key
|
|
207
|
+
// (`reference-set.ts`). `keys` is `.min(1)` with no maximum, so a
|
|
208
|
+
// two-key set names no one key to read there, and a member missing
|
|
209
|
+
// the declared key has nothing to read at all. AD-26 assigns
|
|
210
|
+
// `malformed-operator-expression` to an operand type the operator
|
|
211
|
+
// does not accept, which is what each of those is. Rejecting them
|
|
212
|
+
// here is what makes the resolver's projection total.
|
|
213
|
+
onSetOperand: (setOperand, path) => {
|
|
214
|
+
if (!('referenceSet' in setOperand))
|
|
215
|
+
return;
|
|
216
|
+
const { referenceSet } = setOperand;
|
|
217
|
+
const declaration = declaredReferenceSetOf(contract, referenceSet);
|
|
218
|
+
if (declaration === undefined)
|
|
219
|
+
return;
|
|
220
|
+
if (declaration.keys.length !== 1) {
|
|
221
|
+
throw new StructuralFailure('malformed-operator-expression', `${site.artifactPath}${path}`, `referenceSet "${referenceSet}" declares ${declaration.keys.length} keys, and set-membership reads exactly one, so only a single-key set is legal in its set position (AD-4, AD-26)`);
|
|
222
|
+
}
|
|
223
|
+
const key = declaration.keys[0];
|
|
224
|
+
for (const [index, member] of declaration.members.entries()) {
|
|
225
|
+
if (Object.hasOwn(member, key))
|
|
226
|
+
continue;
|
|
227
|
+
throw new StructuralFailure('malformed-operator-expression', `EvalContract.referenceSets[id=${referenceSet}].members[${index}]${artifactKey(key)}`, `referenceSet "${referenceSet}" has a member carrying no key "${key}", which it declares and which set-membership reads (AD-4, AD-26)`);
|
|
228
|
+
}
|
|
229
|
+
},
|
|
190
230
|
onCoversByKey: (expr) => {
|
|
191
231
|
const expected = expr.operands[0];
|
|
192
232
|
if (!('referenceSet' in expected))
|
|
193
233
|
return;
|
|
194
|
-
const
|
|
195
|
-
if (declarations === null ||
|
|
196
|
-
!Object.hasOwn(declarations, expected.referenceSet))
|
|
197
|
-
return;
|
|
198
|
-
const declaration = declarations[expected.referenceSet];
|
|
234
|
+
const declaration = declaredReferenceSetOf(contract, expected.referenceSet);
|
|
199
235
|
if (declaration === undefined)
|
|
200
236
|
return;
|
|
201
237
|
const seen = new Set();
|
|
202
238
|
for (const [index, member] of declaration.members.entries()) {
|
|
203
|
-
if (!Object.hasOwn(member, expr.expectedKey))
|
|
204
|
-
continue;
|
|
205
239
|
const memberPath = `EvalContract.referenceSets[id=${expected.referenceSet}].members[${index}]${artifactKey(expr.expectedKey)}`;
|
|
240
|
+
// `coversByKey` returns false for the whole collection when one
|
|
241
|
+
// declared member lacks `expectedKey`, which reads as a detected
|
|
242
|
+
// defect in the system under test, when it is an authoring
|
|
243
|
+
// mistake. This check used to walk past that member to keep the
|
|
244
|
+
// duplicate-detection loop total, which left the compiler silent
|
|
245
|
+
// about the one shape that makes the operator answer wrongly.
|
|
246
|
+
if (!Object.hasOwn(member, expr.expectedKey)) {
|
|
247
|
+
throw new StructuralFailure('malformed-operator-expression', memberPath, `referenceSet "${expected.referenceSet}" has a member carrying no expectedKey "${expr.expectedKey}", which covers-by-key reads on every member (AD-4)`);
|
|
248
|
+
}
|
|
206
249
|
const digest = digestArtifact(member[expr.expectedKey], memberPath);
|
|
207
250
|
if (seen.has(digest)) {
|
|
208
251
|
throw new StructuralFailure('malformed-operator-expression', memberPath, `referenceSet "${expected.referenceSet}" repeats expectedKey "${expr.expectedKey}" for covers-by-key (AD-4)`);
|
|
@@ -341,11 +384,14 @@ function forEachQuantifierCollection(expr, boundElementRoot, path, visit) {
|
|
|
341
384
|
function checkQuantifiersAgainst(expression, artifactPath, operationFor) {
|
|
342
385
|
forEachQuantifierCollection(expression, null, '', (pointer, path) => {
|
|
343
386
|
const target = parseEvidenceTarget(pointer);
|
|
344
|
-
if (target.channel !== 'response-body')
|
|
345
|
-
return;
|
|
346
387
|
const operation = operationFor(target.stepId);
|
|
347
388
|
if (operation === undefined)
|
|
348
389
|
return;
|
|
390
|
+
// The operation is resolved before the target is tested, because which
|
|
391
|
+
// channel carries the declared structure is the operation's own answer,
|
|
392
|
+
// and on the artifact channel so is which file.
|
|
393
|
+
if (!targetsDescribedChannel(operation, target))
|
|
394
|
+
return;
|
|
349
395
|
const firstToken = target.tail.length === 1 ? target.tail[0] : undefined;
|
|
350
396
|
const declaredType = firstToken === undefined
|
|
351
397
|
? undefined
|
|
@@ -385,7 +431,7 @@ export function checkQuantifierOverNonCollection(contract) {
|
|
|
385
431
|
const step = index.stepOf(stepId);
|
|
386
432
|
if (step === undefined)
|
|
387
433
|
return undefined;
|
|
388
|
-
return index
|
|
434
|
+
return anyOperationOf(index, step.operationId);
|
|
389
435
|
};
|
|
390
436
|
forEachContractExpression(contract, (site) => {
|
|
391
437
|
checkQuantifiersAgainst(site.expression, site.artifactPath, (stepId) => operationFor(site, stepId));
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { EvalContract } from '../schemas/eval-contract.ts';
|
|
2
|
-
|
|
2
|
+
import type { AnyOperation } from '../schemas/interface.ts';
|
|
3
3
|
export declare function checkInterfaceKind(contract: EvalContract): void;
|
|
4
4
|
/**
|
|
5
5
|
* The transport identity AD-40 resolves a defect signature against: the method
|
|
@@ -12,8 +12,41 @@ export declare function operationSignature(operation: {
|
|
|
12
12
|
readonly method: string;
|
|
13
13
|
readonly pathTemplate: string;
|
|
14
14
|
}): string;
|
|
15
|
+
/**
|
|
16
|
+
* The separator between a command's executable and each subcommand segment,
|
|
17
|
+
* declared once so the contract side and AD-40's corpus side cannot disagree
|
|
18
|
+
* about it. A space, matching the way the identity is written on a terminal.
|
|
19
|
+
*/
|
|
20
|
+
export declare const COMMAND_SIGNATURE_SEPARATOR = " ";
|
|
21
|
+
/**
|
|
22
|
+
* The command counterpart of `operationSignature`, compared literally.
|
|
23
|
+
*
|
|
24
|
+
* There is no erasure step. A subcommand path carries no parameters: a
|
|
25
|
+
* command's variable inputs are its arguments and options, which live in the
|
|
26
|
+
* request shape. Erasing a subcommand segment would make `tool review` and
|
|
27
|
+
* `tool report` one signature, which is the opposite of what erasure is for.
|
|
28
|
+
*/
|
|
29
|
+
export declare function commandSignature(operation: {
|
|
30
|
+
readonly invocation: {
|
|
31
|
+
readonly executable: string;
|
|
32
|
+
readonly subcommandPath: readonly string[];
|
|
33
|
+
};
|
|
34
|
+
}): string;
|
|
35
|
+
/** The transport identity of an operation of either kind. */
|
|
36
|
+
export declare const anyOperationSignature: (operation: AnyOperation) => string;
|
|
15
37
|
/** Finds duplicate method and path signatures across the full inventory. */
|
|
16
38
|
export declare function checkDuplicateOperationSignature(contract: EvalContract): void;
|
|
39
|
+
/**
|
|
40
|
+
* `unresolved-artifact-reference`: an artifact identifier nothing declares.
|
|
41
|
+
*
|
|
42
|
+
* Two sites name one: an evidence pointer's identifier segment, and a command
|
|
43
|
+
* operation's own `descriptorChannel` when it nominates an artifact. Both are
|
|
44
|
+
* authoring faults the compiler can see, and both take a code rather than
|
|
45
|
+
* resolving `absent`, on AD-26's own precedent for a dangling reference-set
|
|
46
|
+
* identifier: `absent` is defined over pointers that do not resolve against
|
|
47
|
+
* observed evidence, and a dangling declaration is neither.
|
|
48
|
+
*/
|
|
49
|
+
export declare function checkArtifactReferences(contract: EvalContract): void;
|
|
17
50
|
/**
|
|
18
51
|
* Checks each binding key against its operation's request shape, and each
|
|
19
52
|
* `{ principal }` value against the contract's declared principals.
|