eval-quality 1.4.2 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -5
- package/corpus/dev/README.md +24 -13
- package/corpus/dev/compile-seal-example/brief.json +1 -1
- package/corpus/dev/compile-seal-example/contract.json +1 -1
- package/corpus/dev/contracts/absent-collection-locations.json +1 -1
- package/corpus/dev/contracts/absent-sibling-groups.json +1 -1
- package/corpus/dev/contracts/absent-success-indicator.json +1 -1
- package/corpus/dev/contracts/captured-read-back.json +1 -0
- package/corpus/dev/contracts/checklist-selection.json +1 -0
- package/corpus/dev/contracts/empty-channel-roles.json +1 -1
- package/corpus/dev/contracts/empty-collection-locations.json +1 -1
- package/corpus/dev/contracts/empty-request-shapes.json +1 -1
- package/corpus/dev/contracts/empty-sibling-groups.json +1 -1
- package/corpus/dev/contracts/fragment-selection.json +1 -1
- package/corpus/dev/contracts/no-collection-quantifier.json +1 -1
- package/corpus/dev/contracts/no-operation-inventory.json +1 -1
- package/corpus/dev/contracts/no-read-back-relation.json +1 -1
- package/corpus/dev/contracts/no-state-change-marker.json +1 -1
- package/corpus/dev/contracts/no-type-violating-step.json +1 -1
- package/corpus/dev/contracts/notes-tool-server.json +1 -0
- package/corpus/dev/contracts/per-key-split-oracles.json +1 -1
- package/corpus/dev/contracts/review-corpus.json +1 -1
- package/corpus/dev/contracts/satisfied-declarations.json +1 -1
- package/corpus/dev/contracts/single-required-response-key.json +1 -1
- package/corpus/dev/contracts/split-indicator-oracle.json +1 -1
- package/corpus/dev/contracts/unaddressed-parameter-sibling.json +1 -1
- package/corpus/dev/contracts/unnamed-reference-set.json +1 -1
- package/corpus/dev/contracts/wrong-cardinality-form.json +1 -1
- package/corpus/dev/index.json +1 -1
- package/dist/adapters/command-line-adapter.js +44 -6
- package/dist/adapters/index.d.ts +2 -0
- package/dist/adapters/index.js +1 -0
- package/dist/adapters/mcp-adapter.d.ts +32 -0
- package/dist/adapters/mcp-adapter.js +357 -0
- package/dist/adapters/mcp-target-policy.d.ts +39 -0
- package/dist/adapters/mcp-target-policy.js +30 -0
- package/dist/cli/render.d.ts +8 -3
- package/dist/cli/render.js +13 -6
- package/dist/cli/run.js +2 -1
- package/dist/core/compile/bindings.d.ts +8 -18
- package/dist/core/compile/bindings.js +13 -10
- package/dist/core/compile/interface-inventory.d.ts +81 -3
- package/dist/core/compile/interface-inventory.js +134 -23
- package/dist/core/compile/reachability.d.ts +73 -1
- package/dist/core/compile/reachability.js +134 -19
- package/dist/core/compile/sensitivity-witness.d.ts +22 -10
- package/dist/core/compile/sensitivity-witness.js +101 -13
- package/dist/core/coverage/operations.d.ts +1 -1
- package/dist/core/coverage/operations.js +1 -1
- package/dist/core/coverage/relevance.d.ts +3 -3
- package/dist/core/coverage/relevance.js +3 -3
- package/dist/core/declared-inputs.d.ts +21 -9
- package/dist/core/declared-inputs.js +51 -16
- package/dist/core/evaluate/evidence-resolution.d.ts +8 -7
- package/dist/core/evaluate/evidence-resolution.js +25 -18
- package/dist/core/preflight/plan.js +28 -6
- package/dist/core/preflight/projection.d.ts +10 -1
- package/dist/core/preflight/projection.js +9 -5
- package/dist/core/preflight/reduce.js +6 -2
- package/dist/core/preflight/witness-evidence.js +31 -9
- package/dist/core/schemas/artifact.d.ts +145 -49
- package/dist/core/schemas/defect-signature.d.ts +280 -23
- package/dist/core/schemas/defect-signature.js +65 -37
- package/dist/core/schemas/eval-contract.d.ts +33 -48
- package/dist/core/schemas/eval-contract.js +3 -3
- package/dist/core/schemas/interface.d.ts +193 -55
- package/dist/core/schemas/interface.js +82 -15
- package/dist/core/schemas/plan.d.ts +45 -1
- package/dist/core/schemas/plan.js +13 -2
- package/dist/core/schemas/pointer.d.ts +23 -9
- package/dist/core/schemas/pointer.js +25 -11
- package/dist/core/schemas/port-messages.d.ts +81 -0
- package/dist/core/schemas/port-messages.js +50 -3
- package/dist/core/schemas/primitives.d.ts +18 -0
- package/dist/core/schemas/primitives.js +29 -0
- package/dist/core/schemas/probe-policy.d.ts +41 -0
- package/dist/core/schemas/probe-policy.js +61 -1
- package/dist/core/schemas/probe.d.ts +115 -2
- package/dist/core/schemas/probe.js +2 -2
- package/dist/core/schemas/sealed-run-record.d.ts +11 -5
- package/dist/core/schemas/sealed-run-record.js +21 -9
- package/dist/core/schemas/sensitivity-witness.d.ts +31 -7
- package/dist/core/schemas/sensitivity-witness.js +32 -9
- package/dist/core/score/bindings.d.ts +1 -1
- package/dist/core/score/bindings.js +4 -4
- package/dist/core/score/qualification.d.ts +7 -5
- package/dist/core/score/qualification.js +92 -23
- package/dist/core/score/score.js +1 -1
- package/dist/core/seal/derived-reference.js +20 -17
- package/dist/core/seal/plan-index.d.ts +18 -9
- package/dist/core/seal/plan-index.js +79 -37
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/ports/environment-probe-port.d.ts +50 -17
- package/dist/ports/environment-probe-port.js +26 -17
- package/dist/testing/conformance.d.ts +3 -2
- package/dist/testing/conformance.js +2 -1
- package/dist/testing/index.d.ts +3 -3
- package/dist/testing/index.js +1 -1
- package/dist/testing/probe-conformance.d.ts +90 -13
- package/dist/testing/probe-conformance.js +375 -160
- package/package.json +4 -2
- package/schemas/eval-contract.schema.json +571 -18
- package/schemas/probe.schema.json +152 -12
- package/schemas/rubric.schema.json +1 -1
- package/schemas/sealed-run-record.schema.json +24 -17
|
@@ -22,7 +22,7 @@ export const ApiWitnessInputs = z
|
|
|
22
22
|
})
|
|
23
23
|
.meta({
|
|
24
24
|
id: 'WitnessInputs',
|
|
25
|
-
description: "One probe leg's supplied inputs, keyed by AD-19 transport channel, in the spelling the environment-probe port accepts.
|
|
25
|
+
description: "One probe leg's supplied inputs, keyed by AD-19 transport channel, in the spelling the environment-probe port accepts. The transport branch of the union every leg shape takes: a sensitivity leg, a manifestation witness, and the fixture reset.",
|
|
26
26
|
});
|
|
27
27
|
/**
|
|
28
28
|
* The same, for a leg supplied to a command-kind operation. `stdin` is a
|
|
@@ -50,7 +50,19 @@ export const CommandWitnessInputs = z.strictObject({
|
|
|
50
50
|
stdin: ProbeRequestStdin,
|
|
51
51
|
});
|
|
52
52
|
/**
|
|
53
|
-
*
|
|
53
|
+
* The same, for a leg supplied to a tool call. One key, because a tool call
|
|
54
|
+
* carries an arguments object and nothing else, and a JSON object rather than a
|
|
55
|
+
* tagged value because an argument-less call supplies `{}` and has no absent
|
|
56
|
+
* spelling to tell apart from a JSON null.
|
|
57
|
+
*/
|
|
58
|
+
// Bare, with no `.meta({ id })`, on the reason the command branch above records:
|
|
59
|
+
// two `$ref`'d definitions under one `anyOf` report `#/required` at the same
|
|
60
|
+
// instance path and AD-13's mutation sweep cannot attribute the deletion.
|
|
61
|
+
export const McpWitnessInputs = z.strictObject({
|
|
62
|
+
arguments: JsonObjectValue,
|
|
63
|
+
});
|
|
64
|
+
/**
|
|
65
|
+
* Any spelling. A plain union rather than a discriminated one for the same
|
|
54
66
|
* reason `InputBinding` is: the leg names an operation and the kind of the
|
|
55
67
|
* interface declaring it lives in another subtree, so no discriminator is
|
|
56
68
|
* available to the schema and the agreement is a compile-time check.
|
|
@@ -60,13 +72,18 @@ export const CommandWitnessInputs = z.strictObject({
|
|
|
60
72
|
* took it from 0.3.0 and the other two followed in 1.3.0, since the transport
|
|
61
73
|
* spelling left a seeded defect against a command-line system under test
|
|
62
74
|
* unrepresentable. The reach is the port's: all three legs are issued through
|
|
63
|
-
* the environment-probe port, whose `ProbeRequest`
|
|
64
|
-
* `
|
|
65
|
-
*
|
|
66
|
-
*
|
|
67
|
-
*
|
|
75
|
+
* the environment-probe port, whose `ProbeRequest` carries one member per kind
|
|
76
|
+
* `isSupportedInterfaceKind` admits, and `preflight/plan.ts` gates on that same
|
|
77
|
+
* predicate. The invariant is the count rather than the list: a leg shape
|
|
78
|
+
* narrower than the port it feeds leaves a kind the adapter can run with no way
|
|
79
|
+
* to declare a leg for it, so this union takes one member per port member and
|
|
80
|
+
* opening a kind adds one to each.
|
|
68
81
|
*/
|
|
69
|
-
export const WitnessInputs = z.union([
|
|
82
|
+
export const WitnessInputs = z.union([
|
|
83
|
+
ApiWitnessInputs,
|
|
84
|
+
CommandWitnessInputs,
|
|
85
|
+
McpWitnessInputs,
|
|
86
|
+
]);
|
|
70
87
|
/**
|
|
71
88
|
* Half a witness pair. `legId` roots the relation's pointers, which address this
|
|
72
89
|
* leg's response as `/interactions/{legId}/response-body/...`, so it shares one
|
|
@@ -92,9 +109,15 @@ export const COMMAND_WITNESS_CHANNELS = [
|
|
|
92
109
|
'environment',
|
|
93
110
|
'stdin',
|
|
94
111
|
];
|
|
112
|
+
// A tool call's one channel. AD-10's marker rule decides nothing here for the
|
|
113
|
+
// reason it decides nothing for a command: a tool call carries its inputs the
|
|
114
|
+
// same way whether or not it changes state. With one channel there is nothing
|
|
115
|
+
// left for the rule to select, so both marker values admit it.
|
|
116
|
+
export const MCP_WITNESS_CHANNELS = ['arguments'];
|
|
95
117
|
export const WITNESS_CHANNELS = [
|
|
96
118
|
...API_WITNESS_CHANNELS,
|
|
97
119
|
...COMMAND_WITNESS_CHANNELS,
|
|
120
|
+
...MCP_WITNESS_CHANNELS,
|
|
98
121
|
];
|
|
99
122
|
export const WitnessChannel = z.enum(WITNESS_CHANNELS);
|
|
100
123
|
/**
|
|
@@ -140,7 +163,7 @@ export const ManifestationWitness = z.strictObject({
|
|
|
140
163
|
legId: Identifier,
|
|
141
164
|
interfaceId: Identifier,
|
|
142
165
|
operationId: Identifier,
|
|
143
|
-
inputs: WitnessInputs,
|
|
166
|
+
inputs: WitnessInputs.describe("The probe's `schemaVersion` 3 -> 4 BREAKING bump under AD-11, whose rule is that removing or retyping is breaking. The union gained a third leg shape, one key over a tool call's arguments, so a witness leg against a tool call is expressible. The matching defect signature is a separate shape and version 5 is where it landed, so a version-4 probe declares a witness leg against a tool call and no signature for the defect that leg exercises. Against version 4 every version-3 probe's own bytes still parse, since the widening adds a branch and narrows none; version 5 narrows two shapes and is where a version-3 or version-4 probe stops parsing."),
|
|
144
167
|
relation: Expression,
|
|
145
168
|
});
|
|
146
169
|
/**
|
|
@@ -30,7 +30,7 @@ export type CapturedResolution = {
|
|
|
30
30
|
* JSON-encoded rather than delimiter-joined, since a parameter key is arbitrary
|
|
31
31
|
* caller-supplied text and could contain any separator.
|
|
32
32
|
*/
|
|
33
|
-
export declare function bindingSiteKey(stepId: string,
|
|
33
|
+
export declare function bindingSiteKey(stepId: string, inputChannel: InputChannelName, key: string): string;
|
|
34
34
|
/**
|
|
35
35
|
* Resolves one captured pointer against the record.
|
|
36
36
|
*
|
|
@@ -27,8 +27,8 @@ const ABSENT_RESOLUTION = { status: 'absent' };
|
|
|
27
27
|
* JSON-encoded rather than delimiter-joined, since a parameter key is arbitrary
|
|
28
28
|
* caller-supplied text and could contain any separator.
|
|
29
29
|
*/
|
|
30
|
-
export function bindingSiteKey(stepId,
|
|
31
|
-
return JSON.stringify([stepId,
|
|
30
|
+
export function bindingSiteKey(stepId, inputChannel, key) {
|
|
31
|
+
return JSON.stringify([stepId, inputChannel, key]);
|
|
32
32
|
}
|
|
33
33
|
/**
|
|
34
34
|
* Resolves one captured pointer against the record.
|
|
@@ -111,7 +111,7 @@ export function resolveCapturedBindings(interactionPlan, index, observations) {
|
|
|
111
111
|
if (step === undefined)
|
|
112
112
|
continue;
|
|
113
113
|
for (const capture of capturedBindings(step)) {
|
|
114
|
-
resolved.set(bindingSiteKey(stepId, capture.
|
|
114
|
+
resolved.set(bindingSiteKey(stepId, capture.inputChannel, capture.key), resolveCapturedValue(capture.pointer, index, observations, resolved));
|
|
115
115
|
}
|
|
116
116
|
}
|
|
117
117
|
}
|
|
@@ -161,7 +161,7 @@ export function jsonTypeOf(value) {
|
|
|
161
161
|
function capturedFloor(step, resolved) {
|
|
162
162
|
let floor = null;
|
|
163
163
|
for (const capture of capturedBindings(step)) {
|
|
164
|
-
const resolution = resolved.get(bindingSiteKey(step.stepId, capture.
|
|
164
|
+
const resolution = resolved.get(bindingSiteKey(step.stepId, capture.inputChannel, capture.key));
|
|
165
165
|
if (resolution === undefined || resolution.status !== 'resolved')
|
|
166
166
|
return null;
|
|
167
167
|
floor =
|
|
@@ -30,11 +30,13 @@ export type QualificationResult = {
|
|
|
30
30
|
};
|
|
31
31
|
/**
|
|
32
32
|
* Resolves a signature's home operation against a contract's operation
|
|
33
|
-
* inventory, comparing
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
33
|
+
* inventory, comparing the transport identity each renders inside its own shape
|
|
34
|
+
* family. Off an interface that speaks HTTP that identity is a method plus a
|
|
35
|
+
* path template with parameter names erased first, so a corpus signature on
|
|
36
|
+
* `/notes/{id}` binds a contract declaring `/notes/{noteId}`. A collision inside
|
|
37
|
+
* one contract and one family has already failed compilation under
|
|
38
|
+
* `duplicate-operation-signature`, so the first match is the only match for any
|
|
39
|
+
* contract that compiled.
|
|
38
40
|
*/
|
|
39
41
|
export declare function resolveHomeOperation(signature: DefectSignature, interfaces: readonly PermittedInterface[]): AnyOperation | null;
|
|
40
42
|
/**
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
* `Error`.
|
|
16
16
|
*/
|
|
17
17
|
import { checkExpressionOperandLegality, checkExpressionQuantifierNesting, checkExpressionQuantifierOverNonCollection, checkExpressionRegexConstructs, walkExpression, } from '../compile/expression-legality.js';
|
|
18
|
-
import { anyOperationSignature, commandSignature, operationSignature, } from '../compile/interface-inventory.js';
|
|
18
|
+
import { anyOperationSignature, commandSignature, isSupportedInterfaceKind, mcpSignature, operationSignature, SUPPORTED_KINDS_CLAUSE, signatureFamilyOf, } from '../compile/interface-inventory.js';
|
|
19
19
|
import { checkExpressionBoundElementScope, checkExpressionEvidenceReachability, forEachExpressionPointer, } from '../compile/reachability.js';
|
|
20
20
|
import { requestShapeOf } from '../declared-inputs.js';
|
|
21
21
|
import { StructuralFailure } from '../failure-codes.js';
|
|
@@ -49,26 +49,46 @@ export const QUALIFICATION_FAILURES = [
|
|
|
49
49
|
'condition-quantifier-over-non-collection',
|
|
50
50
|
'condition-bound-element-outside-quantifier',
|
|
51
51
|
];
|
|
52
|
+
/**
|
|
53
|
+
* The transport identity a signature declares, in its own kind's spelling. All
|
|
54
|
+
* four kinds render one now: `mcp` renders the published tool name its own
|
|
55
|
+
* branch declares, which is what `McpOperation` renders on the contract side.
|
|
56
|
+
*
|
|
57
|
+
* A switch with no default arm, so a fifth kind fails the typecheck here rather
|
|
58
|
+
* than inheriting the api arm the way a two-way test let `mcp` do.
|
|
59
|
+
*/
|
|
60
|
+
const declaredIdentityOf = (signature) => {
|
|
61
|
+
switch (signature.interfaceKind) {
|
|
62
|
+
case 'cli':
|
|
63
|
+
return commandSignature(signature);
|
|
64
|
+
case 'mcp':
|
|
65
|
+
return mcpSignature(signature);
|
|
66
|
+
case 'api':
|
|
67
|
+
case 'web':
|
|
68
|
+
return operationSignature(signature);
|
|
69
|
+
}
|
|
70
|
+
};
|
|
52
71
|
/**
|
|
53
72
|
* Resolves a signature's home operation against a contract's operation
|
|
54
|
-
* inventory, comparing
|
|
55
|
-
*
|
|
56
|
-
*
|
|
57
|
-
*
|
|
58
|
-
*
|
|
73
|
+
* inventory, comparing the transport identity each renders inside its own shape
|
|
74
|
+
* family. Off an interface that speaks HTTP that identity is a method plus a
|
|
75
|
+
* path template with parameter names erased first, so a corpus signature on
|
|
76
|
+
* `/notes/{id}` binds a contract declaring `/notes/{noteId}`. A collision inside
|
|
77
|
+
* one contract and one family has already failed compilation under
|
|
78
|
+
* `duplicate-operation-signature`, so the first match is the only match for any
|
|
79
|
+
* contract that compiled.
|
|
59
80
|
*/
|
|
60
81
|
export function resolveHomeOperation(signature, interfaces) {
|
|
61
|
-
// The identity is compared within its own
|
|
62
|
-
// method and a path template can only name an operation
|
|
63
|
-
// pair,
|
|
64
|
-
// declaring one
|
|
82
|
+
// The identity is compared within its own shape family. A signature
|
|
83
|
+
// declaring a method and a path template can only name an operation
|
|
84
|
+
// declaring the same pair, one declaring an invocation can only name an
|
|
85
|
+
// operation declaring one, and one declaring a tool name can only name a
|
|
86
|
+
// tool; comparing the rendered strings across families would let
|
|
65
87
|
// `GET /notes` collide with an executable literally named that.
|
|
66
|
-
const
|
|
67
|
-
const
|
|
68
|
-
? commandSignature(signature)
|
|
69
|
-
: operationSignature(signature);
|
|
88
|
+
const wanted = declaredIdentityOf(signature);
|
|
89
|
+
const family = signatureFamilyOf(signature.interfaceKind);
|
|
70
90
|
for (const iface of interfaces) {
|
|
71
|
-
if ((iface.kind
|
|
91
|
+
if (signatureFamilyOf(iface.kind) !== family)
|
|
72
92
|
continue;
|
|
73
93
|
for (const operation of operationsOf(iface)) {
|
|
74
94
|
if (anyOperationSignature(operation) === wanted)
|
|
@@ -86,8 +106,46 @@ export function resolveHomeOperation(signature, interfaces) {
|
|
|
86
106
|
// a channel joins the enum, so a new channel would have gone unassigned and
|
|
87
107
|
// silently answered "no" to both questions below.
|
|
88
108
|
const RESPONSE_SIDE = new Set(RESPONSE_SIDE_CHANNELS);
|
|
89
|
-
/**
|
|
90
|
-
|
|
109
|
+
/**
|
|
110
|
+
* The channels a signature of this kind can never manifest in.
|
|
111
|
+
*
|
|
112
|
+
* Three arms rather than two. A tool call produces neither the command response
|
|
113
|
+
* channels nor `response-headers`: it carries its structured result on
|
|
114
|
+
* `response-body` and its error flag on `response-status`, which is the same
|
|
115
|
+
* answer `checkExpressionLegChannel` and `evaluateReachabilityAgainstOperation`
|
|
116
|
+
* give at compile. A two-way ternary put `mcp` on the api arm and left
|
|
117
|
+
* `response-headers` admitted here while compile refused it, which is two
|
|
118
|
+
* sources of truth about what a tool call produces.
|
|
119
|
+
*/
|
|
120
|
+
const foreignChannels = (kind) => {
|
|
121
|
+
switch (kind) {
|
|
122
|
+
case 'cli':
|
|
123
|
+
return new Set(API_RESPONSE_CHANNELS);
|
|
124
|
+
case 'mcp':
|
|
125
|
+
return new Set([...COMMAND_RESPONSE_CHANNELS, 'response-headers']);
|
|
126
|
+
case 'api':
|
|
127
|
+
case 'web':
|
|
128
|
+
return new Set(COMMAND_RESPONSE_CHANNELS);
|
|
129
|
+
}
|
|
130
|
+
};
|
|
131
|
+
// `condition-text-channel-on-api` fires for any channel the declared kind
|
|
132
|
+
// cannot produce, `response-headers` on a tool call included, so the name is
|
|
133
|
+
// narrower than the rule. It stays: renaming moves an AD-5 registry row, a
|
|
134
|
+
// `QUALIFICATION_FAILURES` member, every fixture naming it, and the published
|
|
135
|
+
// census, with no behaviour behind any of it. Recorded here as a known
|
|
136
|
+
// imprecision so a reader of the closed table is not surprised by it.
|
|
137
|
+
/** How a detail string names the sort of interface a signature declares. */
|
|
138
|
+
const interfacePhraseOf = (kind) => {
|
|
139
|
+
switch (kind) {
|
|
140
|
+
case 'cli':
|
|
141
|
+
return 'an interface behind a command';
|
|
142
|
+
case 'mcp':
|
|
143
|
+
return 'a tool call';
|
|
144
|
+
case 'api':
|
|
145
|
+
case 'web':
|
|
146
|
+
return 'an api interface';
|
|
147
|
+
}
|
|
148
|
+
};
|
|
91
149
|
const probePath = (probe, tail) => `Probe[probeId=${probe.probeId}]${tail}`;
|
|
92
150
|
/**
|
|
93
151
|
* Which routes this probe's class and control status admit. An empty list is
|
|
@@ -262,7 +320,7 @@ function checkOperandsAndCollectChannels(probe, signature, failures) {
|
|
|
262
320
|
failures.push({
|
|
263
321
|
code: 'condition-text-channel-on-api',
|
|
264
322
|
artifactPath: `${conditionPath}${path}`,
|
|
265
|
-
detail: `"${pointer}" addresses ${target.channel}, which ${signature.interfaceKind
|
|
323
|
+
detail: `"${pointer}" addresses ${target.channel}, which ${interfacePhraseOf(signature.interfaceKind)} never produces (AD-19, AD-26)`,
|
|
266
324
|
});
|
|
267
325
|
}
|
|
268
326
|
});
|
|
@@ -343,7 +401,8 @@ function checkSelectorKeys(probe, signature, operation, failures) {
|
|
|
343
401
|
* unchecked would make an unchecked field load-bearing. On an `api` signature
|
|
344
402
|
* the three text channels are rejected for the same reason a pointer into one
|
|
345
403
|
* is: an api interface never produces them, which is the same rule the pointer
|
|
346
|
-
* walk applies, read on the declaration too.
|
|
404
|
+
* walk applies, read on the declaration too. A tool call rejects those three and
|
|
405
|
+
* `response-headers`, since it produces neither.
|
|
347
406
|
*/
|
|
348
407
|
function checkObservableChannel(probe, signature, failures) {
|
|
349
408
|
const path = probePath(probe, '.defectSignature.observableChannel');
|
|
@@ -359,7 +418,7 @@ function checkObservableChannel(probe, signature, failures) {
|
|
|
359
418
|
failures.push({
|
|
360
419
|
code: 'condition-text-channel-on-api',
|
|
361
420
|
artifactPath: path,
|
|
362
|
-
detail: `declares observableChannel "${signature.observableChannel}", which ${signature.interfaceKind
|
|
421
|
+
detail: `declares observableChannel "${signature.observableChannel}", which ${interfacePhraseOf(signature.interfaceKind)} never produces (AD-19, AD-26)`,
|
|
363
422
|
});
|
|
364
423
|
}
|
|
365
424
|
}
|
|
@@ -589,12 +648,22 @@ export function qualifyProbe(probe, homeOperation) {
|
|
|
589
648
|
}
|
|
590
649
|
}
|
|
591
650
|
if (signature !== null) {
|
|
592
|
-
|
|
593
|
-
|
|
651
|
+
// The same tuple the compile and pre-flight gates read, so what a
|
|
652
|
+
// contract may declare and what a signature may declare against cannot
|
|
653
|
+
// disagree. This was the fourth transcription of the pair and the last.
|
|
654
|
+
//
|
|
655
|
+
// The coupling is deliberate and it is not free: the two questions can
|
|
656
|
+
// legitimately differ, and they did for one release, when the contract
|
|
657
|
+
// gates opened for `mcp` while this one stayed shut because the kind had
|
|
658
|
+
// no signature branch to declare. Opening a kind contract-side now opens
|
|
659
|
+
// it here too. A future kind that needs the gap back gets its own
|
|
660
|
+
// predicate beside `isSupportedInterfaceKind` rather than a re-spelled
|
|
661
|
+
// condition, so the divergence stays a named decision.
|
|
662
|
+
if (!isSupportedInterfaceKind(signature.interfaceKind)) {
|
|
594
663
|
failures.push({
|
|
595
664
|
code: 'signature-interface-kind-unsupported',
|
|
596
665
|
artifactPath: probePath(probe, '.defectSignature.interfaceKind'),
|
|
597
|
-
detail: `"${signature.interfaceKind}" declares a method and a path template with no per-kind semantics behind them
|
|
666
|
+
detail: `"${signature.interfaceKind}" has no probe semantics in this version: it declares a method and a path template with no per-kind semantics behind them, and no operation shape of its own for a signature to bind. It stays in the enum so unsupported-interface-kind stays fireable contract-side; ${SUPPORTED_KINDS_CLAUSE} admitted here (AD-19)`,
|
|
598
667
|
});
|
|
599
668
|
}
|
|
600
669
|
checkObservableChannel(probe, signature, failures);
|
package/dist/core/score/score.js
CHANGED
|
@@ -203,7 +203,7 @@ function trialSetDisagreementsOf(trials) {
|
|
|
203
203
|
* it sees one trial at a time, and nothing checks whether two DIFFERENT trials
|
|
204
204
|
* of one set reuse an observation, finding, or oracle-disposition identifier.
|
|
205
205
|
* That asymmetry looks like a gap beside `trial-set-field-disagreement`, which
|
|
206
|
-
* does compare `mode` and `
|
|
206
|
+
* does compare `mode`, `evaluatorRecommendation`, and `runId` across trials.
|
|
207
207
|
*
|
|
208
208
|
* It is not one. A trial set is n independent evaluator runs of one contract,
|
|
209
209
|
* each producing its own record, and a harness that names its first observation
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* Resolves pointers through a `PlanIndex` into a phrase, kept apart from
|
|
6
6
|
* `plan-index.ts`'s resolving and `direction-prose.ts`'s relation templates.
|
|
7
7
|
*/
|
|
8
|
-
import { boundChannelsOf, isCommandOperation } from '../declared-inputs.js';
|
|
8
|
+
import { boundChannelsOf, isCommandOperation, isMcpOperation, } from '../declared-inputs.js';
|
|
9
9
|
import { StructuralFailure } from '../failure-codes.js';
|
|
10
10
|
import { anyOperationOf, parseEvidenceTarget, resolveStep, } from './plan-index.js';
|
|
11
11
|
function resolveEvidenceTarget(pointer, index) {
|
|
@@ -35,17 +35,20 @@ function joinWithAnd(items) {
|
|
|
35
35
|
// `buildPlanIndex` already rejects a duplicate `operationId` across
|
|
36
36
|
// interfaces, so two resolved operations never share this phrase. The
|
|
37
37
|
// transport identity is never printed here, whichever kind it is: AD-16
|
|
38
|
-
// withholds the operation inventory from the brief, so
|
|
39
|
-
//
|
|
38
|
+
// withholds the operation inventory from the brief, so a method and a path
|
|
39
|
+
// template, an executable and a subcommand path, and a tool name are all
|
|
40
|
+
// withheld from an evaluator alike.
|
|
40
41
|
//
|
|
41
42
|
// The noun follows the kind. Calling a command an endpoint told the evaluator
|
|
42
43
|
// something false about what it was reading, and the word is the only thing
|
|
43
44
|
// this phrase says beyond the operation's own name.
|
|
44
45
|
function operationReference(operation) {
|
|
45
46
|
const name = operation.operationId.split('-').join(' ');
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
47
|
+
if (isCommandOperation(operation))
|
|
48
|
+
return `the ${name} command`;
|
|
49
|
+
if (isMcpOperation(operation))
|
|
50
|
+
return `the ${name} tool`;
|
|
51
|
+
return `the ${name} endpoint`;
|
|
49
52
|
}
|
|
50
53
|
const ESCALATION_LEVELS = [
|
|
51
54
|
'generic',
|
|
@@ -57,23 +60,23 @@ const ESCALATION_LEVELS = [
|
|
|
57
60
|
// stays permutation-invariant. `boundChannelsOf` supplies the channel order.
|
|
58
61
|
function bindingEntries(step) {
|
|
59
62
|
const entries = [];
|
|
60
|
-
for (const { channel:
|
|
63
|
+
for (const { channel: inputChannel, bound: map } of boundChannelsOf(step.inputBinding)) {
|
|
61
64
|
if (map === null)
|
|
62
65
|
continue;
|
|
63
66
|
for (const key of Object.keys(map).sort()) {
|
|
64
67
|
const value = map[key];
|
|
65
68
|
if (value !== undefined)
|
|
66
|
-
entries.push({
|
|
69
|
+
entries.push({ inputChannel, key, value });
|
|
67
70
|
}
|
|
68
71
|
}
|
|
69
72
|
return entries;
|
|
70
73
|
}
|
|
71
|
-
// The
|
|
74
|
+
// The input channel is part of the rendered name as well as the sort
|
|
72
75
|
// key: two bindings can share a parameter name across channels (path.id and
|
|
73
76
|
// query.id), and without the qualifier both would render as "the supplied
|
|
74
77
|
// id", hiding two different bindings behind identical text.
|
|
75
78
|
function entryName(entry) {
|
|
76
|
-
return `${entry.
|
|
79
|
+
return `${entry.inputChannel} ${entry.key}`;
|
|
77
80
|
}
|
|
78
81
|
function isTypeViolating(value) {
|
|
79
82
|
return 'matcher' in value && value.matcher === 'type-violating';
|
|
@@ -411,17 +414,17 @@ function localTargetPhrase(target) {
|
|
|
411
414
|
case 'exit-code':
|
|
412
415
|
return 'its exit code';
|
|
413
416
|
case 'call-inputs': {
|
|
414
|
-
if (target.
|
|
417
|
+
if (target.inputChannel === null) {
|
|
415
418
|
// Unreachable: `parseEvidenceTarget` sets this exactly when the
|
|
416
419
|
// channel is 'call-inputs'.
|
|
417
|
-
throw new TypeError('call-inputs evidence target carries no
|
|
420
|
+
throw new TypeError('call-inputs evidence target carries no input channel');
|
|
418
421
|
}
|
|
419
|
-
// The
|
|
422
|
+
// The input channel is named here too, not only in the no-tail
|
|
420
423
|
// fallback below: same path.id / query.id collision `entryName`
|
|
421
424
|
// above guards against.
|
|
422
425
|
return field !== null
|
|
423
|
-
? `the ${target.
|
|
424
|
-
: `the ${target.
|
|
426
|
+
? `the ${target.inputChannel} ${field} value you sent`
|
|
427
|
+
: `the ${target.inputChannel} you sent`;
|
|
425
428
|
}
|
|
426
429
|
case 'response-body':
|
|
427
430
|
return field !== null
|
|
@@ -448,7 +451,7 @@ function localTargetPhrase(target) {
|
|
|
448
451
|
// channel is 'artifact'.
|
|
449
452
|
throw new TypeError('artifact evidence target names no artifact');
|
|
450
453
|
}
|
|
451
|
-
// The artifact is named for the same reason the
|
|
454
|
+
// The artifact is named for the same reason the input channel
|
|
452
455
|
// is: two files can carry the same field name, and an evaluator
|
|
453
456
|
// reading "its findings field" twice cannot tell which file it
|
|
454
457
|
// means.
|
|
@@ -472,7 +475,7 @@ function fullTargetPhrase(resolved, siblingsOf, index) {
|
|
|
472
475
|
function channelSignature(target) {
|
|
473
476
|
return JSON.stringify([
|
|
474
477
|
target.channel,
|
|
475
|
-
target.
|
|
478
|
+
target.inputChannel,
|
|
476
479
|
target.artifactId,
|
|
477
480
|
target.tail,
|
|
478
481
|
]);
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/** resolves a pointer to its step and operation; nothing about reachability. */
|
|
2
|
-
import type { AnyOperation, CommandOperation, InterfaceKindName, Operation, PermittedInterface } from '../schemas/interface.ts';
|
|
2
|
+
import type { AnyOperation, CommandOperation, InterfaceKindName, McpOperation, Operation, PermittedInterface } from '../schemas/interface.ts';
|
|
3
3
|
import type { InteractionStep } from '../schemas/plan.ts';
|
|
4
4
|
import { type EvidenceChannelName, type InputChannelName } from '../schemas/pointer.ts';
|
|
5
5
|
/**
|
|
@@ -17,13 +17,13 @@ export declare const decodeTail: (tailSource: string) => readonly string[];
|
|
|
17
17
|
export type EvidenceTarget = {
|
|
18
18
|
stepId: string;
|
|
19
19
|
channel: EvidenceChannelName;
|
|
20
|
-
|
|
20
|
+
inputChannel: InputChannelName | null;
|
|
21
21
|
artifactId: string | null;
|
|
22
22
|
tail: readonly string[];
|
|
23
23
|
};
|
|
24
24
|
/**
|
|
25
25
|
* Parses one `InteractionPointer` string (`INTERACTION_POINTER_PATTERN` in
|
|
26
|
-
* `pointer.ts`) into its step id, channel,
|
|
26
|
+
* `pointer.ts`) into its step id, channel, input channel, and tail, using
|
|
27
27
|
* the schema's own channel partition so this accepts exactly what
|
|
28
28
|
* `InteractionPointer.safeParse` accepts. A should-never-happen precondition
|
|
29
29
|
* violation throws `TypeError`, per `digest.ts`'s precedent.
|
|
@@ -38,14 +38,15 @@ export declare function parseEvidenceTarget(pointer: string): EvidenceTarget;
|
|
|
38
38
|
*
|
|
39
39
|
* `operationOf` stays narrow on purpose. It hands a resolved `Operation` to
|
|
40
40
|
* every downstream consumer, so widening its return type to the operation
|
|
41
|
-
* union would retype seventeen files at once. A command operation
|
|
42
|
-
* `undefined` from it and to a value from
|
|
43
|
-
* caller that has to branch reads the declaring kind first.
|
|
41
|
+
* union would retype seventeen files at once. A command operation and a tool
|
|
42
|
+
* call each resolve to `undefined` from it and to a value from their own
|
|
43
|
+
* accessor, and each caller that has to branch reads the declaring kind first.
|
|
44
44
|
*/
|
|
45
45
|
export type PlanIndex = {
|
|
46
46
|
stepOf: (stepId: string) => InteractionStep | undefined;
|
|
47
47
|
operationOf: (operationId: string) => Operation | undefined;
|
|
48
48
|
commandOperationOf: (operationId: string) => CommandOperation | undefined;
|
|
49
|
+
mcpOperationOf: (operationId: string) => McpOperation | undefined;
|
|
49
50
|
interfaceKindOf: (operationId: string) => InterfaceKindName | undefined;
|
|
50
51
|
stepsUsing: (operationId: string) => readonly InteractionStep[];
|
|
51
52
|
};
|
|
@@ -70,9 +71,17 @@ export declare function buildPlanIndex(interactionPlan: readonly InteractionStep
|
|
|
70
71
|
export declare function resolveStep(index: PlanIndex, stepId: string): InteractionStep;
|
|
71
72
|
/**
|
|
72
73
|
* The declared operation of whichever kind, for the callers that read only
|
|
73
|
-
* fields
|
|
74
|
+
* fields all three shapes carry. Callers reading a kind-specific field ask
|
|
74
75
|
* `interfaceKindOf` first and then take the matching accessor.
|
|
75
76
|
*/
|
|
76
77
|
export declare const anyOperationOf: (index: PlanIndex, operationId: string) => AnyOperation | undefined;
|
|
77
|
-
/**
|
|
78
|
-
|
|
78
|
+
/**
|
|
79
|
+
* Resolves an operation id through the index or throws. See `resolveStep`.
|
|
80
|
+
*
|
|
81
|
+
* Reads all three maps. It read `operationOf` alone, so it threw for a command
|
|
82
|
+
* operation and then for a tool call with the message "the permitted interfaces
|
|
83
|
+
* do not declare it", which is false: they declare it, and a different accessor
|
|
84
|
+
* holds it. Callers narrow with the operation predicates in
|
|
85
|
+
* `core/declared-inputs.ts`.
|
|
86
|
+
*/
|
|
87
|
+
export declare function resolveOperation(index: PlanIndex, operationId: string): AnyOperation;
|