eval-quality 1.4.2 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -6
- package/corpus/dev/README.md +24 -13
- package/corpus/dev/compile-seal-example/brief.json +1 -1
- package/corpus/dev/compile-seal-example/contract.json +1 -1
- package/corpus/dev/contracts/absent-collection-locations.json +1 -1
- package/corpus/dev/contracts/absent-sibling-groups.json +1 -1
- package/corpus/dev/contracts/absent-success-indicator.json +1 -1
- package/corpus/dev/contracts/captured-read-back.json +1 -0
- package/corpus/dev/contracts/checklist-selection.json +1 -0
- package/corpus/dev/contracts/empty-channel-roles.json +1 -1
- package/corpus/dev/contracts/empty-collection-locations.json +1 -1
- package/corpus/dev/contracts/empty-request-shapes.json +1 -1
- package/corpus/dev/contracts/empty-sibling-groups.json +1 -1
- package/corpus/dev/contracts/fragment-selection.json +1 -1
- package/corpus/dev/contracts/no-collection-quantifier.json +1 -1
- package/corpus/dev/contracts/no-operation-inventory.json +1 -1
- package/corpus/dev/contracts/no-read-back-relation.json +1 -1
- package/corpus/dev/contracts/no-state-change-marker.json +1 -1
- package/corpus/dev/contracts/no-type-violating-step.json +1 -1
- package/corpus/dev/contracts/notes-tool-server.json +1 -0
- package/corpus/dev/contracts/per-key-split-oracles.json +1 -1
- package/corpus/dev/contracts/review-corpus.json +1 -1
- package/corpus/dev/contracts/satisfied-declarations.json +1 -1
- package/corpus/dev/contracts/single-required-response-key.json +1 -1
- package/corpus/dev/contracts/split-indicator-oracle.json +1 -1
- package/corpus/dev/contracts/unaddressed-parameter-sibling.json +1 -1
- package/corpus/dev/contracts/unnamed-reference-set.json +1 -1
- package/corpus/dev/contracts/wrong-cardinality-form.json +1 -1
- package/corpus/dev/index.json +1 -1
- package/dist/adapters/command-line-adapter.js +44 -6
- package/dist/adapters/index.d.ts +2 -0
- package/dist/adapters/index.js +1 -0
- package/dist/adapters/mcp-adapter.d.ts +32 -0
- package/dist/adapters/mcp-adapter.js +357 -0
- package/dist/adapters/mcp-target-policy.d.ts +39 -0
- package/dist/adapters/mcp-target-policy.js +30 -0
- package/dist/cli/render.d.ts +8 -3
- package/dist/cli/render.js +13 -6
- package/dist/cli/run.js +2 -1
- package/dist/core/compile/bindings.d.ts +8 -18
- package/dist/core/compile/bindings.js +13 -10
- package/dist/core/compile/compile.js +6 -1
- package/dist/core/compile/interface-inventory.d.ts +81 -3
- package/dist/core/compile/interface-inventory.js +134 -23
- package/dist/core/compile/reachability.d.ts +73 -1
- package/dist/core/compile/reachability.js +134 -19
- package/dist/core/compile/schema-version.d.ts +15 -2
- package/dist/core/compile/schema-version.js +11 -3
- package/dist/core/compile/sensitivity-witness.d.ts +22 -10
- package/dist/core/compile/sensitivity-witness.js +101 -13
- package/dist/core/coverage/operations.d.ts +1 -1
- package/dist/core/coverage/operations.js +1 -1
- package/dist/core/coverage/relevance.d.ts +3 -3
- package/dist/core/coverage/relevance.js +3 -3
- package/dist/core/declared-inputs.d.ts +21 -9
- package/dist/core/declared-inputs.js +51 -16
- package/dist/core/evaluate/evidence-resolution.d.ts +8 -7
- package/dist/core/evaluate/evidence-resolution.js +25 -18
- package/dist/core/preflight/plan.js +43 -6
- package/dist/core/preflight/projection.d.ts +10 -1
- package/dist/core/preflight/projection.js +9 -5
- package/dist/core/preflight/reduce.js +6 -2
- package/dist/core/preflight/witness-evidence.js +31 -9
- package/dist/core/schemas/artifact.d.ts +145 -49
- package/dist/core/schemas/defect-signature.d.ts +280 -23
- package/dist/core/schemas/defect-signature.js +65 -37
- package/dist/core/schemas/eval-contract.d.ts +33 -48
- package/dist/core/schemas/eval-contract.js +3 -3
- package/dist/core/schemas/interface.d.ts +193 -55
- package/dist/core/schemas/interface.js +82 -15
- package/dist/core/schemas/plan.d.ts +45 -1
- package/dist/core/schemas/plan.js +13 -2
- package/dist/core/schemas/pointer.d.ts +23 -9
- package/dist/core/schemas/pointer.js +25 -11
- package/dist/core/schemas/port-messages.d.ts +81 -0
- package/dist/core/schemas/port-messages.js +50 -3
- package/dist/core/schemas/primitives.d.ts +18 -0
- package/dist/core/schemas/primitives.js +29 -0
- package/dist/core/schemas/probe-policy.d.ts +41 -0
- package/dist/core/schemas/probe-policy.js +61 -1
- package/dist/core/schemas/probe.d.ts +156 -2
- package/dist/core/schemas/probe.js +45 -2
- package/dist/core/schemas/sealed-run-record.d.ts +11 -5
- package/dist/core/schemas/sealed-run-record.js +21 -9
- package/dist/core/schemas/sensitivity-witness.d.ts +31 -7
- package/dist/core/schemas/sensitivity-witness.js +32 -9
- package/dist/core/score/bindings.d.ts +1 -1
- package/dist/core/score/bindings.js +4 -4
- package/dist/core/score/qualification.d.ts +7 -5
- package/dist/core/score/qualification.js +92 -23
- package/dist/core/score/score.d.ts +1 -1
- package/dist/core/score/score.js +24 -1
- package/dist/core/seal/derived-reference.js +20 -17
- package/dist/core/seal/plan-index.d.ts +18 -9
- package/dist/core/seal/plan-index.js +79 -37
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/ports/environment-probe-port.d.ts +50 -17
- package/dist/ports/environment-probe-port.js +26 -17
- package/dist/testing/conformance.d.ts +3 -2
- package/dist/testing/conformance.js +2 -1
- package/dist/testing/index.d.ts +3 -3
- package/dist/testing/index.js +1 -1
- package/dist/testing/probe-conformance.d.ts +90 -13
- package/dist/testing/probe-conformance.js +375 -160
- package/package.json +4 -2
- package/schemas/eval-contract.schema.json +571 -18
- package/schemas/probe.schema.json +152 -12
- package/schemas/rubric.schema.json +1 -1
- package/schemas/sealed-run-record.schema.json +24 -17
|
@@ -17,9 +17,17 @@
|
|
|
17
17
|
* happened to notice first.
|
|
18
18
|
*/
|
|
19
19
|
import { RuntimeFault } from '../schemas/faults.js';
|
|
20
|
-
/**
|
|
21
|
-
|
|
20
|
+
/**
|
|
21
|
+
* `schema-version-mismatch`: the stamp is not the version this build reads.
|
|
22
|
+
*
|
|
23
|
+
* An options object rather than four positional arguments. `artifactPath` and
|
|
24
|
+
* `consequence` are both free-form strings, and transposed they typecheck: the
|
|
25
|
+
* fault's `artifactPath`, which callers read programmatically, would then hold
|
|
26
|
+
* a sentence.
|
|
27
|
+
*/
|
|
28
|
+
export function checkSchemaVersion(options) {
|
|
29
|
+
const { stamped, accepted, artifactPath, consequence } = options;
|
|
22
30
|
if (stamped === accepted)
|
|
23
31
|
return;
|
|
24
|
-
throw new RuntimeFault('schema-version-mismatch', artifactPath, `carries "schemaVersion" ${stamped} where this build reads ${accepted};
|
|
32
|
+
throw new RuntimeFault('schema-version-mismatch', artifactPath, `carries "schemaVersion" ${stamped} where this build reads ${accepted}; an artifact written for another version is not read leniently, ${consequence} (AD-11)`);
|
|
25
33
|
}
|
|
@@ -15,8 +15,10 @@ import { declaresNoRequestKeys } from '../declared-inputs.ts';
|
|
|
15
15
|
import type { EvalContract } from '../schemas/eval-contract.ts';
|
|
16
16
|
import type { AnyOperation } from '../schemas/interface.ts';
|
|
17
17
|
import type { InputChannelName } from '../schemas/pointer.ts';
|
|
18
|
-
import { type ApiWitnessInputs, type WitnessChannel, type WitnessInputs } from '../schemas/sensitivity-witness.ts';
|
|
18
|
+
import { type ApiWitnessInputs, type McpWitnessInputs, type WitnessChannel, type WitnessInputs } from '../schemas/sensitivity-witness.ts';
|
|
19
19
|
export { declaresNoRequestKeys };
|
|
20
|
+
/** Which spelling this leg uses. `arguments` is the tool-call shape's one key. */
|
|
21
|
+
export declare const isMcpWitnessInputs: (inputs: WitnessInputs) => inputs is McpWitnessInputs;
|
|
20
22
|
/**
|
|
21
23
|
* The transport spelling, for the consumers that can only send one: the probe
|
|
22
24
|
* port carries a method, a path template, and the four transport channels.
|
|
@@ -41,30 +43,40 @@ export declare function suppliesOpaquely(inputs: WitnessInputs, channel: InputCh
|
|
|
41
43
|
* the keyed shape the request declares.
|
|
42
44
|
*/
|
|
43
45
|
export declare function suppliedKeys(inputs: WitnessInputs, channel: InputChannelName): readonly string[];
|
|
44
|
-
/**
|
|
46
|
+
/**
|
|
47
|
+
* One channel's supplied value, for the differential comparison, or the
|
|
48
|
+
* sentinel above where the leg's shape carries no such channel.
|
|
49
|
+
*
|
|
50
|
+
* Reads the branch first rather than indexing the union: indexing returned
|
|
51
|
+
* `undefined` for both legs of a witness whose channel belongs to a different
|
|
52
|
+
* kind, so the two compared equal and the differential check reported "both
|
|
53
|
+
* legs supply the same value" over legs that were the wrong shape entirely.
|
|
54
|
+
* The mismatch has its own diagnosis and `checkWitnessLegality` gives it one.
|
|
55
|
+
*/
|
|
45
56
|
export declare function suppliedValue(inputs: WitnessInputs, channel: WitnessChannel): unknown;
|
|
46
57
|
/**
|
|
47
58
|
* Every channel of one set of witness inputs against the operation that will
|
|
48
|
-
* receive them
|
|
49
|
-
* copies
|
|
50
|
-
* on an unselected channel is as much an outbound value as one
|
|
51
|
-
* channel.
|
|
59
|
+
* receive them, and every one of them rather than the differential channel
|
|
60
|
+
* alone: `planPreflight` copies each onto the `ProbeRequest` and the port sends
|
|
61
|
+
* them, so a value on an unselected channel is as much an outbound value as one
|
|
62
|
+
* on the selected channel. Four channels off an interface that speaks HTTP,
|
|
63
|
+
* four off a command, one off a tool call.
|
|
52
64
|
*
|
|
53
65
|
* The two directions have different standing, and the difference matters
|
|
54
66
|
* enough to name, because the next reader will diff this against
|
|
55
67
|
* `checkUndeclaredMandatoryInput` and find the two disagreeing.
|
|
56
68
|
*
|
|
57
69
|
* The permitted direction is that function's rule, carried over: it loops the
|
|
58
|
-
* same
|
|
70
|
+
* same channels and rejects a key the operation declares in neither list.
|
|
59
71
|
* That is what makes `WitnessInputs`'s AD-18 promise about `header` more than a
|
|
60
72
|
* comment.
|
|
61
73
|
*
|
|
62
74
|
* The required direction is new here, because the two shapes differ. An
|
|
63
75
|
* `InputBindingChannel` is nullable, and `null` means "this step binds nothing in
|
|
64
76
|
* this channel", so `checkUndeclaredMandatoryInput` skips an unbound channel and
|
|
65
|
-
* asks nothing about required keys. `WitnessInputs`
|
|
66
|
-
* concrete
|
|
67
|
-
* issue. The cost is that a required header forces a literal value into the
|
|
77
|
+
* asks nothing about required keys. A `WitnessInputs` leg carries every channel
|
|
78
|
+
* of its own shape as a concrete value, so a leg omitting a required key is a
|
|
79
|
+
* request the port cannot issue. The cost is that a required header forces a literal value into the
|
|
68
80
|
* contract artifact, the surface AD-18 governs; a placeholder satisfies this
|
|
69
81
|
* check, and what pre-flight probes is a fixture.
|
|
70
82
|
*/
|
|
@@ -11,14 +11,17 @@
|
|
|
11
11
|
* `malformed-operator-expression` is the closest available code because the
|
|
12
12
|
* relation is illegal in its position once its operands cannot be told apart.
|
|
13
13
|
*/
|
|
14
|
-
import { declaresNoRequestKeys, isCommandOperation, requestChannelsOf, } from '../declared-inputs.js';
|
|
14
|
+
import { declaresNoRequestKeys, isCommandOperation, isMcpOperation, requestChannelsOf, } from '../declared-inputs.js';
|
|
15
15
|
import { StructuralFailure } from '../failure-codes.js';
|
|
16
16
|
import { operationsOf } from '../schemas/interface.js';
|
|
17
|
-
import { COMMAND_WITNESS_CHANNELS, } from '../schemas/sensitivity-witness.js';
|
|
17
|
+
import { API_WITNESS_CHANNELS, COMMAND_WITNESS_CHANNELS, MCP_WITNESS_CHANNELS, } from '../schemas/sensitivity-witness.js';
|
|
18
18
|
import { parseEvidenceTarget } from '../seal/plan-index.js';
|
|
19
|
+
import { checkExpressionEvidenceReachability, checkExpressionLegChannel, checkExpressionVolatility, } from './reachability.js';
|
|
19
20
|
export { declaresNoRequestKeys };
|
|
20
21
|
/** Which spelling this leg uses. `stdin` is the command shape's own key. */
|
|
21
22
|
const isCommandWitnessInputs = (inputs) => 'stdin' in inputs;
|
|
23
|
+
/** Which spelling this leg uses. `arguments` is the tool-call shape's one key. */
|
|
24
|
+
export const isMcpWitnessInputs = (inputs) => 'arguments' in inputs;
|
|
22
25
|
/**
|
|
23
26
|
* The transport spelling, for the consumers that can only send one: the probe
|
|
24
27
|
* port carries a method, a path template, and the four transport channels.
|
|
@@ -89,6 +92,8 @@ export function suppliedKeys(inputs, channel) {
|
|
|
89
92
|
return Object.keys(inputs.environment);
|
|
90
93
|
return [];
|
|
91
94
|
}
|
|
95
|
+
if (isMcpWitnessInputs(inputs))
|
|
96
|
+
return channel === 'arguments' ? Object.keys(inputs.arguments) : [];
|
|
92
97
|
if (channel === 'body')
|
|
93
98
|
return bodyKeys(inputs.body);
|
|
94
99
|
if (channel === 'path' || channel === 'query')
|
|
@@ -97,36 +102,87 @@ export function suppliedKeys(inputs, channel) {
|
|
|
97
102
|
return Object.keys(inputs.header);
|
|
98
103
|
return [];
|
|
99
104
|
}
|
|
100
|
-
/**
|
|
105
|
+
/**
|
|
106
|
+
* The sentinel `suppliedValue` returns for a channel the leg's own shape has no
|
|
107
|
+
* key for. Distinct from `undefined`, which a leg may legitimately hold nowhere
|
|
108
|
+
* and which two mismatched legs would otherwise compare equal on.
|
|
109
|
+
*/
|
|
110
|
+
const CHANNEL_NOT_ON_SHAPE = Symbol('channel not on this witness leg shape');
|
|
111
|
+
/** Which of the three input vocabularies a leg spells, named for a message. */
|
|
112
|
+
const witnessInputsKindOf = (inputs) => {
|
|
113
|
+
if (isMcpWitnessInputs(inputs))
|
|
114
|
+
return 'tool-call';
|
|
115
|
+
if (isCommandWitnessInputs(inputs))
|
|
116
|
+
return 'command';
|
|
117
|
+
return 'transport';
|
|
118
|
+
};
|
|
119
|
+
/** The same question about the operation that will receive them. */
|
|
120
|
+
const operationInputsKindOf = (operation) => {
|
|
121
|
+
if (isMcpOperation(operation))
|
|
122
|
+
return 'tool-call';
|
|
123
|
+
if (isCommandOperation(operation))
|
|
124
|
+
return 'command';
|
|
125
|
+
return 'transport';
|
|
126
|
+
};
|
|
127
|
+
/**
|
|
128
|
+
* One channel's supplied value, for the differential comparison, or the
|
|
129
|
+
* sentinel above where the leg's shape carries no such channel.
|
|
130
|
+
*
|
|
131
|
+
* Reads the branch first rather than indexing the union: indexing returned
|
|
132
|
+
* `undefined` for both legs of a witness whose channel belongs to a different
|
|
133
|
+
* kind, so the two compared equal and the differential check reported "both
|
|
134
|
+
* legs supply the same value" over legs that were the wrong shape entirely.
|
|
135
|
+
* The mismatch has its own diagnosis and `checkWitnessLegality` gives it one.
|
|
136
|
+
*/
|
|
101
137
|
export function suppliedValue(inputs, channel) {
|
|
102
|
-
|
|
138
|
+
if (isMcpWitnessInputs(inputs))
|
|
139
|
+
return channel === 'arguments' ? inputs.arguments : CHANNEL_NOT_ON_SHAPE;
|
|
140
|
+
if (isCommandWitnessInputs(inputs))
|
|
141
|
+
return COMMAND_WITNESS_CHANNELS.includes(channel)
|
|
142
|
+
? inputs[channel]
|
|
143
|
+
: CHANNEL_NOT_ON_SHAPE;
|
|
144
|
+
return API_WITNESS_CHANNELS.includes(channel)
|
|
145
|
+
? inputs[channel]
|
|
146
|
+
: CHANNEL_NOT_ON_SHAPE;
|
|
103
147
|
}
|
|
104
148
|
/**
|
|
105
149
|
* Every channel of one set of witness inputs against the operation that will
|
|
106
|
-
* receive them
|
|
107
|
-
* copies
|
|
108
|
-
* on an unselected channel is as much an outbound value as one
|
|
109
|
-
* channel.
|
|
150
|
+
* receive them, and every one of them rather than the differential channel
|
|
151
|
+
* alone: `planPreflight` copies each onto the `ProbeRequest` and the port sends
|
|
152
|
+
* them, so a value on an unselected channel is as much an outbound value as one
|
|
153
|
+
* on the selected channel. Four channels off an interface that speaks HTTP,
|
|
154
|
+
* four off a command, one off a tool call.
|
|
110
155
|
*
|
|
111
156
|
* The two directions have different standing, and the difference matters
|
|
112
157
|
* enough to name, because the next reader will diff this against
|
|
113
158
|
* `checkUndeclaredMandatoryInput` and find the two disagreeing.
|
|
114
159
|
*
|
|
115
160
|
* The permitted direction is that function's rule, carried over: it loops the
|
|
116
|
-
* same
|
|
161
|
+
* same channels and rejects a key the operation declares in neither list.
|
|
117
162
|
* That is what makes `WitnessInputs`'s AD-18 promise about `header` more than a
|
|
118
163
|
* comment.
|
|
119
164
|
*
|
|
120
165
|
* The required direction is new here, because the two shapes differ. An
|
|
121
166
|
* `InputBindingChannel` is nullable, and `null` means "this step binds nothing in
|
|
122
167
|
* this channel", so `checkUndeclaredMandatoryInput` skips an unbound channel and
|
|
123
|
-
* asks nothing about required keys. `WitnessInputs`
|
|
124
|
-
* concrete
|
|
125
|
-
* issue. The cost is that a required header forces a literal value into the
|
|
168
|
+
* asks nothing about required keys. A `WitnessInputs` leg carries every channel
|
|
169
|
+
* of its own shape as a concrete value, so a leg omitting a required key is a
|
|
170
|
+
* request the port cannot issue. The cost is that a required header forces a literal value into the
|
|
126
171
|
* contract artifact, the surface AD-18 governs; a placeholder satisfies this
|
|
127
172
|
* check, and what pre-flight probes is a fixture.
|
|
128
173
|
*/
|
|
129
174
|
export function checkInputsAgainstShape(inputs, operation, owner, artifactPath) {
|
|
175
|
+
// The leg's shape has to be the operation's kind before any key is
|
|
176
|
+
// compared. `WitnessInputs` is a plain union with no discriminator, so a
|
|
177
|
+
// tool call's arguments parse against an operation that speaks HTTP, and
|
|
178
|
+
// the key loop below then reports the first required key as omitted, which
|
|
179
|
+
// describes a consequence of the mismatch and names the wrong field. The
|
|
180
|
+
// port's own arm says the same thing about the same pair.
|
|
181
|
+
const legKind = witnessInputsKindOf(inputs);
|
|
182
|
+
const operationKind = operationInputsKindOf(operation);
|
|
183
|
+
if (legKind !== operationKind) {
|
|
184
|
+
throw new StructuralFailure('undeclared-mandatory-input', artifactPath, `${owner} supplies ${legKind} channels to operation "${operation.operationId}", which accepts ${operationKind} channels (AD-10, AD-19)`);
|
|
185
|
+
}
|
|
130
186
|
for (const { channel, shape } of requestChannelsOf(operation)) {
|
|
131
187
|
if (suppliesOpaquely(inputs, channel)) {
|
|
132
188
|
checkOpaqueStream(shape, channel, operation.operationId, owner, artifactPath);
|
|
@@ -218,10 +274,18 @@ const READ_CHANNELS = ['path', 'query'];
|
|
|
218
274
|
* A command carries its inputs the same way whether or not it changes state,
|
|
219
275
|
* so the marker selects nothing there and all four command channels are
|
|
220
276
|
* admitted. The author picks the one their operation is actually sensitive on.
|
|
277
|
+
*
|
|
278
|
+
* A tool call is the command case with one channel: it carries its arguments
|
|
279
|
+
* the same way whichever the marker says, and there is no second channel for a
|
|
280
|
+
* marker rule to choose between. Without this arm the api rule applied and
|
|
281
|
+
* offered `path` or `query` to an operation that has neither, which left an
|
|
282
|
+
* input-bearing tool call with no legal witness at all.
|
|
221
283
|
*/
|
|
222
284
|
const legalChannels = (operation) => {
|
|
223
285
|
if (isCommandOperation(operation))
|
|
224
286
|
return COMMAND_WITNESS_CHANNELS;
|
|
287
|
+
if (isMcpOperation(operation))
|
|
288
|
+
return MCP_WITNESS_CHANNELS;
|
|
225
289
|
return operation.stateChangeMarker ? MUTATING_CHANNELS : READ_CHANNELS;
|
|
226
290
|
};
|
|
227
291
|
/**
|
|
@@ -236,11 +300,21 @@ export function checkWitnessLegality(contract) {
|
|
|
236
300
|
if (!legal.includes(witness.channel)) {
|
|
237
301
|
throw new StructuralFailure('malformed-operator-expression', `${path}.sensitivityWitness`, `channel "${witness.channel}" contradicts stateChangeMarker ${operation.stateChangeMarker} on operation "${operation.operationId}"; AD-10 selects ${legal.map((name) => `"${name}"`).join(' or ')}`);
|
|
238
302
|
}
|
|
303
|
+
const [first, second] = witness.legs;
|
|
304
|
+
// The leg shape agrees with the operation's kind before anything is
|
|
305
|
+
// compared. A leg of the wrong shape carries no value on the selected
|
|
306
|
+
// channel at all, and comparing two of them found them equal and
|
|
307
|
+
// reported "not a differential", which describes a symptom of the real
|
|
308
|
+
// fault rather than the fault.
|
|
309
|
+
for (const leg of witness.legs) {
|
|
310
|
+
if (suppliedValue(leg.inputs, witness.channel) !== CHANNEL_NOT_ON_SHAPE)
|
|
311
|
+
continue;
|
|
312
|
+
throw new StructuralFailure('malformed-operator-expression', `${path}.sensitivityWitness.legs`, `leg "${leg.legId}" of witness "${witness.witnessId}" supplies inputs whose shape carries no "${witness.channel}" channel, so it cannot vary the channel operation "${operation.operationId}" declares (AD-10, AD-19)`);
|
|
313
|
+
}
|
|
239
314
|
// AD-10's predicate is a differential. Two legs supplying the same values on
|
|
240
315
|
// the selected channel establish nothing, and at run time that surfaces as a
|
|
241
316
|
// `failed` check pointing at the fixture, when the defect is in the
|
|
242
317
|
// declaration.
|
|
243
|
-
const [first, second] = witness.legs;
|
|
244
318
|
if (first !== undefined &&
|
|
245
319
|
second !== undefined &&
|
|
246
320
|
JSON.stringify(suppliedValue(first.inputs, witness.channel)) ===
|
|
@@ -261,6 +335,20 @@ export function checkWitnessLegality(contract) {
|
|
|
261
335
|
if (covered.length !== legIds.length) {
|
|
262
336
|
throw new StructuralFailure('malformed-operator-expression', `${path}.sensitivityWitness.relation`, `the relation addresses ${covered.length} of the two legs of witness "${witness.witnessId}"; a differential that reads one leg establishes no sensitivity (AD-10)`);
|
|
263
337
|
}
|
|
338
|
+
// Last, because both presume the two rules above: every root is one of
|
|
339
|
+
// this witness's own legs, so every pointer answers to this operation's
|
|
340
|
+
// own shape. `checkEvidenceReachability` never reaches a relation, since
|
|
341
|
+
// it walks oracle checks alone, and the descriptor question and the
|
|
342
|
+
// projection question are both open here.
|
|
343
|
+
const relationPath = `${path}.sensitivityWitness.relation`;
|
|
344
|
+
// Carriage first. A tailed pointer at a channel the leg does not carry is
|
|
345
|
+
// unreachable against the descriptor too, and that reason invites the
|
|
346
|
+
// author to declare structure on a channel no depth of tail would
|
|
347
|
+
// reach. The carriage reason is the accurate one at a witness for both
|
|
348
|
+
// spellings, so it is asked first.
|
|
349
|
+
checkExpressionLegChannel(witness.relation, relationPath, operation);
|
|
350
|
+
checkExpressionEvidenceReachability(witness.relation, relationPath, operation);
|
|
351
|
+
checkExpressionVolatility(witness.relation, relationPath, operation);
|
|
264
352
|
});
|
|
265
353
|
const reset = contract.fixtureReset;
|
|
266
354
|
if (reset === null)
|
|
@@ -45,7 +45,7 @@ export type ResolvedOperation = {
|
|
|
45
45
|
* The channels a call's inputs may be keyed by. Names only: the two sites
|
|
46
46
|
* that build candidate pointers need the names and never the shapes.
|
|
47
47
|
*/
|
|
48
|
-
readonly
|
|
48
|
+
readonly inputChannels: readonly InputChannelName[];
|
|
49
49
|
/**
|
|
50
50
|
* The same channels already paired with the shapes they declare. Predicates
|
|
51
51
|
* that read a declared shape take these rather than indexing the request
|
|
@@ -51,7 +51,7 @@ export function resolveOperations(contract) {
|
|
|
51
51
|
logicalId: declared.logicalId,
|
|
52
52
|
descriptor: operation.responseDescriptor,
|
|
53
53
|
descriptorRoot: descriptorRootOf(operation),
|
|
54
|
-
|
|
54
|
+
inputChannels: inputChannelsOf(operation),
|
|
55
55
|
requestChannels: requestChannelsOf(operation),
|
|
56
56
|
})));
|
|
57
57
|
}
|
|
@@ -33,9 +33,9 @@ export declare function successIndicatorSeparationRelevance(contract: EvalContra
|
|
|
33
33
|
export declare function wholeBodyRelevance(contract: EvalContract): RelevanceVerdict;
|
|
34
34
|
/**
|
|
35
35
|
* Rule 3: some operation declares a request key on any of its own input
|
|
36
|
-
* channels
|
|
37
|
-
*
|
|
38
|
-
* channel triple. A key with no
|
|
36
|
+
* channels: AD-19's four transport channels off an interface that speaks HTTP,
|
|
37
|
+
* the four command channels off a command, and the one `arguments` channel off
|
|
38
|
+
* a tool call. The site is the whole channel triple. A key with no
|
|
39
39
|
* `types` entry has an absent type, a key typed `null` has AD-31's
|
|
40
40
|
* indeterminate one, and both are relevant, so declaring an input and leaving
|
|
41
41
|
* it untyped buys no irrelevance.
|
|
@@ -59,9 +59,9 @@ export function wholeBodyRelevance(contract) {
|
|
|
59
59
|
}
|
|
60
60
|
/**
|
|
61
61
|
* Rule 3: some operation declares a request key on any of its own input
|
|
62
|
-
* channels
|
|
63
|
-
*
|
|
64
|
-
* channel triple. A key with no
|
|
62
|
+
* channels: AD-19's four transport channels off an interface that speaks HTTP,
|
|
63
|
+
* the four command channels off a command, and the one `arguments` channel off
|
|
64
|
+
* a tool call. The site is the whole channel triple. A key with no
|
|
65
65
|
* `types` entry has an absent type, a key typed `null` has AD-31's
|
|
66
66
|
* indeterminate one, and both are relevant, so declaring an input and leaving
|
|
67
67
|
* it untyped buys no irrelevance.
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* because AD-10's exemption is read by two modules: the compile check that
|
|
7
7
|
* enforces it and the reducer that records it.
|
|
8
8
|
*/
|
|
9
|
-
import type { AnyOperation, CommandOperation, Operation } from './schemas/interface.ts';
|
|
9
|
+
import type { AnyOperation, CommandOperation, McpOperation, Operation } from './schemas/interface.ts';
|
|
10
10
|
import type { BindingChannel, InputBinding } from './schemas/plan.ts';
|
|
11
11
|
import { type EvidenceChannelName, type InputChannelName } from './schemas/pointer.ts';
|
|
12
12
|
import type { KeyedShapeDescriptor } from './schemas/primitives.ts';
|
|
@@ -15,8 +15,9 @@ import type { KeyedShapeDescriptor } from './schemas/primitives.ts';
|
|
|
15
15
|
*
|
|
16
16
|
* AD-19 gives every operation exactly one descriptor, and the channel that
|
|
17
17
|
* descriptor describes is what makes its keys addressable. For an interface
|
|
18
|
-
* that speaks HTTP that channel is the response body
|
|
19
|
-
*
|
|
18
|
+
* that speaks HTTP that channel is the response body, and a tool call's
|
|
19
|
+
* structured result lands on the same one; a command operation declares its
|
|
20
|
+
* own. One rule, several roots: every consumer that used to hard-code
|
|
20
21
|
* `response-body` asks this instead.
|
|
21
22
|
*/
|
|
22
23
|
export declare const descriptorChannelOf: (operation: AnyOperation) => EvidenceChannelName;
|
|
@@ -50,12 +51,23 @@ export type RequestChannel = {
|
|
|
50
51
|
readonly shape: KeyedShapeDescriptor;
|
|
51
52
|
};
|
|
52
53
|
/**
|
|
53
|
-
* Which operation shape this is.
|
|
54
|
-
* because an operation does not carry its interface's kind
|
|
55
|
-
*
|
|
54
|
+
* Which operation shape this is. Each predicate reads a required field only its
|
|
55
|
+
* own shape declares, because an operation does not carry its interface's kind.
|
|
56
|
+
*
|
|
57
|
+
* Three tests rather than one test and its negation. A negation answered `true`
|
|
58
|
+
* for every shape that was not the one it named, so the day a third shape
|
|
59
|
+
* landed the api arm would have claimed it silently and every dispatch below
|
|
60
|
+
* would have handed a tool call the four transport channels. The cost of
|
|
61
|
+
* spelling all three is that a fourth shape breaks the typecheck at each
|
|
62
|
+
* dispatch, which is where the decision belongs.
|
|
56
63
|
*/
|
|
57
64
|
export declare const isCommandOperation: (operation: AnyOperation) => operation is CommandOperation;
|
|
58
|
-
|
|
65
|
+
export declare const isMcpOperation: (operation: AnyOperation) => operation is McpOperation;
|
|
66
|
+
/**
|
|
67
|
+
* Repaired to ask its own question positively. Its two callers filter a mixed
|
|
68
|
+
* inventory down to the shape that declares a method and a path template, and a
|
|
69
|
+
* negation would have handed them every tool call as well.
|
|
70
|
+
*/
|
|
59
71
|
export declare const isApiOperation: (operation: AnyOperation) => operation is Operation;
|
|
60
72
|
/**
|
|
61
73
|
* The operation's input channels paired with the shapes they declare.
|
|
@@ -63,7 +75,7 @@ export declare const isApiOperation: (operation: AnyOperation) => operation is O
|
|
|
63
75
|
* Callers take the pairs rather than a channel list they then index the
|
|
64
76
|
* request shape with. Indexing is what breaks under the operation union:
|
|
65
77
|
* TypeScript cannot prove that a channel name drawn from one kind's tuple is a
|
|
66
|
-
* key of
|
|
78
|
+
* key of another kind's shape, and both ways around that are casts. Pairing
|
|
67
79
|
* dereferences the union once, here.
|
|
68
80
|
*/
|
|
69
81
|
export declare function requestChannelsOf(operation: AnyOperation): readonly RequestChannel[];
|
|
@@ -92,4 +104,4 @@ export declare function boundChannelsOf(binding: InputBinding): readonly {
|
|
|
92
104
|
export declare function declaresNoRequestKeys(operation: AnyOperation): boolean;
|
|
93
105
|
/** Whether no channel declares a required key. */
|
|
94
106
|
export declare function declaresNoRequiredKeys(operation: AnyOperation): boolean;
|
|
95
|
-
export type { AnyOperation, CommandOperation, Operation };
|
|
107
|
+
export type { AnyOperation, CommandOperation, McpOperation, Operation };
|
|
@@ -1,20 +1,30 @@
|
|
|
1
|
-
import { COMMAND_CHANNELS, INPUT_CHANNELS, TRANSPORT_CHANNELS, } from './schemas/pointer.js';
|
|
1
|
+
import { COMMAND_CHANNELS, INPUT_CHANNELS, MCP_CHANNELS, TRANSPORT_CHANNELS, } from './schemas/pointer.js';
|
|
2
|
+
// The channel each tagged descriptor-channel member names. Keyed on the tag, so
|
|
3
|
+
// admitting the prose half of a tool result later fails the typecheck here.
|
|
4
|
+
const MCP_DESCRIPTOR_CHANNELS = {
|
|
5
|
+
'structured-result': 'response-body',
|
|
6
|
+
};
|
|
2
7
|
/**
|
|
3
8
|
* Which output channel the operation's one response descriptor describes.
|
|
4
9
|
*
|
|
5
10
|
* AD-19 gives every operation exactly one descriptor, and the channel that
|
|
6
11
|
* descriptor describes is what makes its keys addressable. For an interface
|
|
7
|
-
* that speaks HTTP that channel is the response body
|
|
8
|
-
*
|
|
12
|
+
* that speaks HTTP that channel is the response body, and a tool call's
|
|
13
|
+
* structured result lands on the same one; a command operation declares its
|
|
14
|
+
* own. One rule, several roots: every consumer that used to hard-code
|
|
9
15
|
* `response-body` asks this instead.
|
|
10
16
|
*/
|
|
11
17
|
export const descriptorChannelOf = (operation) => {
|
|
12
|
-
if (
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
+
if (isCommandOperation(operation)) {
|
|
19
|
+
const { descriptorChannel } = operation;
|
|
20
|
+
return descriptorChannel.kind === 'stream'
|
|
21
|
+
? descriptorChannel.channel
|
|
22
|
+
: 'artifact';
|
|
23
|
+
}
|
|
24
|
+
if (isMcpOperation(operation)) {
|
|
25
|
+
return MCP_DESCRIPTOR_CHANNELS[operation.descriptorChannel.kind];
|
|
26
|
+
}
|
|
27
|
+
return 'response-body';
|
|
18
28
|
};
|
|
19
29
|
/**
|
|
20
30
|
* Which artifact the descriptor describes, or `null` when it describes a
|
|
@@ -25,6 +35,8 @@ export const descriptorChannelOf = (operation) => {
|
|
|
25
35
|
export const descriptorArtifactOf = (operation) => {
|
|
26
36
|
if (!isCommandOperation(operation))
|
|
27
37
|
return null;
|
|
38
|
+
// Only a command operation nominates a file. A tool call and an HTTP
|
|
39
|
+
// response both describe a channel the transport always produces.
|
|
28
40
|
const { descriptorChannel } = operation;
|
|
29
41
|
return descriptorChannel.kind === 'artifact'
|
|
30
42
|
? descriptorChannel.artifactId
|
|
@@ -51,20 +63,31 @@ export const targetsDescribedChannel = (operation, target) => {
|
|
|
51
63
|
/** Every artifact identifier the operation declares it writes. */
|
|
52
64
|
export const declaredArtifactsOf = (operation) => isCommandOperation(operation) ? operation.artifacts : [];
|
|
53
65
|
/**
|
|
54
|
-
* Which operation shape this is.
|
|
55
|
-
* because an operation does not carry its interface's kind
|
|
56
|
-
*
|
|
66
|
+
* Which operation shape this is. Each predicate reads a required field only its
|
|
67
|
+
* own shape declares, because an operation does not carry its interface's kind.
|
|
68
|
+
*
|
|
69
|
+
* Three tests rather than one test and its negation. A negation answered `true`
|
|
70
|
+
* for every shape that was not the one it named, so the day a third shape
|
|
71
|
+
* landed the api arm would have claimed it silently and every dispatch below
|
|
72
|
+
* would have handed a tool call the four transport channels. The cost of
|
|
73
|
+
* spelling all three is that a fourth shape breaks the typecheck at each
|
|
74
|
+
* dispatch, which is where the decision belongs.
|
|
57
75
|
*/
|
|
58
76
|
export const isCommandOperation = (operation) => 'invocation' in operation;
|
|
59
|
-
|
|
60
|
-
|
|
77
|
+
export const isMcpOperation = (operation) => 'toolName' in operation;
|
|
78
|
+
/**
|
|
79
|
+
* Repaired to ask its own question positively. Its two callers filter a mixed
|
|
80
|
+
* inventory down to the shape that declares a method and a path template, and a
|
|
81
|
+
* negation would have handed them every tool call as well.
|
|
82
|
+
*/
|
|
83
|
+
export const isApiOperation = (operation) => 'method' in operation;
|
|
61
84
|
/**
|
|
62
85
|
* The operation's input channels paired with the shapes they declare.
|
|
63
86
|
*
|
|
64
87
|
* Callers take the pairs rather than a channel list they then index the
|
|
65
88
|
* request shape with. Indexing is what breaks under the operation union:
|
|
66
89
|
* TypeScript cannot prove that a channel name drawn from one kind's tuple is a
|
|
67
|
-
* key of
|
|
90
|
+
* key of another kind's shape, and both ways around that are casts. Pairing
|
|
68
91
|
* dereferences the union once, here.
|
|
69
92
|
*/
|
|
70
93
|
export function requestChannelsOf(operation) {
|
|
@@ -74,6 +97,12 @@ export function requestChannelsOf(operation) {
|
|
|
74
97
|
shape: operation.requestShape[channel],
|
|
75
98
|
}));
|
|
76
99
|
}
|
|
100
|
+
if (isMcpOperation(operation)) {
|
|
101
|
+
return MCP_CHANNELS.map((channel) => ({
|
|
102
|
+
channel,
|
|
103
|
+
shape: operation.requestShape[channel],
|
|
104
|
+
}));
|
|
105
|
+
}
|
|
77
106
|
return TRANSPORT_CHANNELS.map((channel) => ({
|
|
78
107
|
channel,
|
|
79
108
|
shape: operation.requestShape[channel],
|
|
@@ -85,7 +114,13 @@ export function requestShapeOf(operation, channel) {
|
|
|
85
114
|
?.shape;
|
|
86
115
|
}
|
|
87
116
|
/** The channel names an operation of this kind may declare inputs on. */
|
|
88
|
-
export const inputChannelsOf = (operation) =>
|
|
117
|
+
export const inputChannelsOf = (operation) => {
|
|
118
|
+
if (isCommandOperation(operation))
|
|
119
|
+
return COMMAND_CHANNELS;
|
|
120
|
+
if (isMcpOperation(operation))
|
|
121
|
+
return MCP_CHANNELS;
|
|
122
|
+
return TRANSPORT_CHANNELS;
|
|
123
|
+
};
|
|
89
124
|
/**
|
|
90
125
|
* A step's bound channels paired with what each binds, for the same reason
|
|
91
126
|
* `requestChannelsOf` pairs: `InputBinding` is a union and a channel name
|
|
@@ -50,12 +50,13 @@ export declare function makeResolveOperand(stepObservations: Readonly<Record<str
|
|
|
50
50
|
*/
|
|
51
51
|
export declare function referenceSetKeysOf(contract: EvalContract): ReferenceSetKeys;
|
|
52
52
|
/**
|
|
53
|
-
*
|
|
54
|
-
*
|
|
55
|
-
* `PlanIndex` builds lazily on first call unless the caller
|
|
56
|
-
* since a schema-admitted duplicate step or operation id would
|
|
57
|
-
* it eagerly throw before this function ever runs. Uses
|
|
58
|
-
* rather than `resolveStep`/`resolveOperation`: those
|
|
59
|
-
* would break this function's always-returns-a-boolean
|
|
53
|
+
* Answers `true` only for a pointer at the channel the invoked operation's own
|
|
54
|
+
* response descriptor describes, since that is where AD-19 puts a declared
|
|
55
|
+
* collection. The `PlanIndex` builds lazily on first call unless the caller
|
|
56
|
+
* supplies one, since a schema-admitted duplicate step or operation id would
|
|
57
|
+
* make building it eagerly throw before this function ever runs. Uses
|
|
58
|
+
* `stepOf`/`operationOf` rather than `resolveStep`/`resolveOperation`: those
|
|
59
|
+
* throw on a miss, which would break this function's always-returns-a-boolean
|
|
60
|
+
* contract.
|
|
60
61
|
*/
|
|
61
62
|
export declare function makePointerDenotesCollection(contract: EvalContract, providedIndex?: PlanIndex): PointerDenotesCollection;
|
|
@@ -98,13 +98,20 @@ export function channelRoot(observation, target) {
|
|
|
98
98
|
case 'exit-code':
|
|
99
99
|
return observation.exitCode;
|
|
100
100
|
case 'call-inputs': {
|
|
101
|
-
const {
|
|
102
|
-
if (
|
|
103
|
-
// parseEvidenceTarget sets
|
|
101
|
+
const { inputChannel } = target;
|
|
102
|
+
if (inputChannel === null) {
|
|
103
|
+
// parseEvidenceTarget sets inputChannel exactly when the
|
|
104
104
|
// channel is 'call-inputs', so this throw should never fire.
|
|
105
|
-
throw new TypeError('call-inputs evidence target carries no
|
|
105
|
+
throw new TypeError('call-inputs evidence target carries no input channel');
|
|
106
106
|
}
|
|
107
|
-
|
|
107
|
+
// A direct read: `ObservedCallInputs` declares one key per member of
|
|
108
|
+
// `INPUT_CHANNELS`, so every channel the grammar parses has a key
|
|
109
|
+
// here and `null` means the channel carried nothing. A tenth
|
|
110
|
+
// channel joining the vocabulary fails the typecheck at this line.
|
|
111
|
+
// The lookup helper this replaced answered absent for a missing
|
|
112
|
+
// key, which was correct while the two widths disagreed and is a
|
|
113
|
+
// silent answer now that they do not.
|
|
114
|
+
return observation.callInputs[inputChannel];
|
|
108
115
|
}
|
|
109
116
|
case 'artifact': {
|
|
110
117
|
const { artifactId } = target;
|
|
@@ -176,13 +183,14 @@ function tokensEqual(a, b) {
|
|
|
176
183
|
return a.length === b.length && a.every((token, index) => token === b[index]);
|
|
177
184
|
}
|
|
178
185
|
/**
|
|
179
|
-
*
|
|
180
|
-
*
|
|
181
|
-
* `PlanIndex` builds lazily on first call unless the caller
|
|
182
|
-
* since a schema-admitted duplicate step or operation id would
|
|
183
|
-
* it eagerly throw before this function ever runs. Uses
|
|
184
|
-
* rather than `resolveStep`/`resolveOperation`: those
|
|
185
|
-
* would break this function's always-returns-a-boolean
|
|
186
|
+
* Answers `true` only for a pointer at the channel the invoked operation's own
|
|
187
|
+
* response descriptor describes, since that is where AD-19 puts a declared
|
|
188
|
+
* collection. The `PlanIndex` builds lazily on first call unless the caller
|
|
189
|
+
* supplies one, since a schema-admitted duplicate step or operation id would
|
|
190
|
+
* make building it eagerly throw before this function ever runs. Uses
|
|
191
|
+
* `stepOf`/`operationOf` rather than `resolveStep`/`resolveOperation`: those
|
|
192
|
+
* throw on a miss, which would break this function's always-returns-a-boolean
|
|
193
|
+
* contract.
|
|
186
194
|
*/
|
|
187
195
|
export function makePointerDenotesCollection(contract, providedIndex) {
|
|
188
196
|
let index = providedIndex;
|
|
@@ -200,12 +208,11 @@ export function makePointerDenotesCollection(contract, providedIndex) {
|
|
|
200
208
|
const operation = anyOperationOf(getIndex(), step.operationId);
|
|
201
209
|
if (operation === undefined)
|
|
202
210
|
return false;
|
|
203
|
-
// The channel
|
|
204
|
-
//
|
|
205
|
-
//
|
|
206
|
-
//
|
|
207
|
-
//
|
|
208
|
-
// resolved `false` instead of `insufficient-evidence`.
|
|
211
|
+
// The channel has to be the one this operation's descriptor describes.
|
|
212
|
+
// Hard-coding `response-body` here left AD-4's empty-collection
|
|
213
|
+
// resolution inapplicable to every command contract: a quantifier over
|
|
214
|
+
// an empty declared collection resolved `false` where the rule says
|
|
215
|
+
// `insufficient-evidence`.
|
|
209
216
|
if (!targetsDescribedChannel(operation, target))
|
|
210
217
|
return false;
|
|
211
218
|
const { collectionLocations } = operation.responseDescriptor;
|