eval-quality 1.4.2 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -5
- package/corpus/dev/README.md +24 -13
- package/corpus/dev/compile-seal-example/brief.json +1 -1
- package/corpus/dev/compile-seal-example/contract.json +1 -1
- package/corpus/dev/contracts/absent-collection-locations.json +1 -1
- package/corpus/dev/contracts/absent-sibling-groups.json +1 -1
- package/corpus/dev/contracts/absent-success-indicator.json +1 -1
- package/corpus/dev/contracts/captured-read-back.json +1 -0
- package/corpus/dev/contracts/checklist-selection.json +1 -0
- package/corpus/dev/contracts/empty-channel-roles.json +1 -1
- package/corpus/dev/contracts/empty-collection-locations.json +1 -1
- package/corpus/dev/contracts/empty-request-shapes.json +1 -1
- package/corpus/dev/contracts/empty-sibling-groups.json +1 -1
- package/corpus/dev/contracts/fragment-selection.json +1 -1
- package/corpus/dev/contracts/no-collection-quantifier.json +1 -1
- package/corpus/dev/contracts/no-operation-inventory.json +1 -1
- package/corpus/dev/contracts/no-read-back-relation.json +1 -1
- package/corpus/dev/contracts/no-state-change-marker.json +1 -1
- package/corpus/dev/contracts/no-type-violating-step.json +1 -1
- package/corpus/dev/contracts/notes-tool-server.json +1 -0
- package/corpus/dev/contracts/per-key-split-oracles.json +1 -1
- package/corpus/dev/contracts/review-corpus.json +1 -1
- package/corpus/dev/contracts/satisfied-declarations.json +1 -1
- package/corpus/dev/contracts/single-required-response-key.json +1 -1
- package/corpus/dev/contracts/split-indicator-oracle.json +1 -1
- package/corpus/dev/contracts/unaddressed-parameter-sibling.json +1 -1
- package/corpus/dev/contracts/unnamed-reference-set.json +1 -1
- package/corpus/dev/contracts/wrong-cardinality-form.json +1 -1
- package/corpus/dev/index.json +1 -1
- package/dist/adapters/command-line-adapter.js +44 -6
- package/dist/adapters/index.d.ts +2 -0
- package/dist/adapters/index.js +1 -0
- package/dist/adapters/mcp-adapter.d.ts +32 -0
- package/dist/adapters/mcp-adapter.js +357 -0
- package/dist/adapters/mcp-target-policy.d.ts +39 -0
- package/dist/adapters/mcp-target-policy.js +30 -0
- package/dist/cli/render.d.ts +8 -3
- package/dist/cli/render.js +13 -6
- package/dist/cli/run.js +2 -1
- package/dist/core/compile/bindings.d.ts +8 -18
- package/dist/core/compile/bindings.js +13 -10
- package/dist/core/compile/interface-inventory.d.ts +81 -3
- package/dist/core/compile/interface-inventory.js +134 -23
- package/dist/core/compile/reachability.d.ts +73 -1
- package/dist/core/compile/reachability.js +134 -19
- package/dist/core/compile/sensitivity-witness.d.ts +22 -10
- package/dist/core/compile/sensitivity-witness.js +101 -13
- package/dist/core/coverage/operations.d.ts +1 -1
- package/dist/core/coverage/operations.js +1 -1
- package/dist/core/coverage/relevance.d.ts +3 -3
- package/dist/core/coverage/relevance.js +3 -3
- package/dist/core/declared-inputs.d.ts +21 -9
- package/dist/core/declared-inputs.js +51 -16
- package/dist/core/evaluate/evidence-resolution.d.ts +8 -7
- package/dist/core/evaluate/evidence-resolution.js +25 -18
- package/dist/core/preflight/plan.js +28 -6
- package/dist/core/preflight/projection.d.ts +10 -1
- package/dist/core/preflight/projection.js +9 -5
- package/dist/core/preflight/reduce.js +6 -2
- package/dist/core/preflight/witness-evidence.js +31 -9
- package/dist/core/schemas/artifact.d.ts +145 -49
- package/dist/core/schemas/defect-signature.d.ts +280 -23
- package/dist/core/schemas/defect-signature.js +65 -37
- package/dist/core/schemas/eval-contract.d.ts +33 -48
- package/dist/core/schemas/eval-contract.js +3 -3
- package/dist/core/schemas/interface.d.ts +193 -55
- package/dist/core/schemas/interface.js +82 -15
- package/dist/core/schemas/plan.d.ts +45 -1
- package/dist/core/schemas/plan.js +13 -2
- package/dist/core/schemas/pointer.d.ts +23 -9
- package/dist/core/schemas/pointer.js +25 -11
- package/dist/core/schemas/port-messages.d.ts +81 -0
- package/dist/core/schemas/port-messages.js +50 -3
- package/dist/core/schemas/primitives.d.ts +18 -0
- package/dist/core/schemas/primitives.js +29 -0
- package/dist/core/schemas/probe-policy.d.ts +41 -0
- package/dist/core/schemas/probe-policy.js +61 -1
- package/dist/core/schemas/probe.d.ts +115 -2
- package/dist/core/schemas/probe.js +2 -2
- package/dist/core/schemas/sealed-run-record.d.ts +11 -5
- package/dist/core/schemas/sealed-run-record.js +21 -9
- package/dist/core/schemas/sensitivity-witness.d.ts +31 -7
- package/dist/core/schemas/sensitivity-witness.js +32 -9
- package/dist/core/score/bindings.d.ts +1 -1
- package/dist/core/score/bindings.js +4 -4
- package/dist/core/score/qualification.d.ts +7 -5
- package/dist/core/score/qualification.js +92 -23
- package/dist/core/score/score.js +1 -1
- package/dist/core/seal/derived-reference.js +20 -17
- package/dist/core/seal/plan-index.d.ts +18 -9
- package/dist/core/seal/plan-index.js +79 -37
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/ports/environment-probe-port.d.ts +50 -17
- package/dist/ports/environment-probe-port.js +26 -17
- package/dist/testing/conformance.d.ts +3 -2
- package/dist/testing/conformance.js +2 -1
- package/dist/testing/index.d.ts +3 -3
- package/dist/testing/index.js +1 -1
- package/dist/testing/probe-conformance.d.ts +90 -13
- package/dist/testing/probe-conformance.js +375 -160
- package/package.json +4 -2
- package/schemas/eval-contract.schema.json +571 -18
- package/schemas/probe.schema.json +152 -12
- package/schemas/rubric.schema.json +1 -1
- package/schemas/sealed-run-record.schema.json +24 -17
|
@@ -12,6 +12,17 @@ export const Identifier = z
|
|
|
12
12
|
.string()
|
|
13
13
|
.regex(IDENTIFIER_PATTERN)
|
|
14
14
|
.describe('A kebab-case slug. Excludes "/" and "~" so an identifier can be embedded in an interaction-rooted pointer without escaping.');
|
|
15
|
+
// A tool name is the ecosystem's own spelling rather than this repository's.
|
|
16
|
+
// MCP servers publish `search_notes` and `searchNotes`, and `Identifier`'s
|
|
17
|
+
// kebab charset would reject both, which would make a contract unable to name
|
|
18
|
+
// the tool it evaluates. The charset admits letters, digits, underscore, and
|
|
19
|
+
// hyphen and nothing else, so an address in any of its three spellings is a
|
|
20
|
+
// parse error.
|
|
21
|
+
export const TOOL_NAME_PATTERN = /^[A-Za-z0-9_-]+$/;
|
|
22
|
+
export const ToolName = z
|
|
23
|
+
.string()
|
|
24
|
+
.regex(TOOL_NAME_PATTERN)
|
|
25
|
+
.describe('The name the MCP server publishes for one tool, in the server\'s own spelling. AD-35: a logical identifier, never a URL, host, or port. The charset admits no "/", ":", or ".", so an address is unrepresentable rather than refused by a later check, and it excludes "~" so a tool name can be embedded in a pointer without escaping.');
|
|
15
26
|
// All eight Consistency-Conventions prefixes are defined here even though this
|
|
16
27
|
// artifact uses five: the Probe, Evidence Artifact, and finding shapes import
|
|
17
28
|
// P-, D-, and F- rather than re-spelling the quantifier, and a second spelling
|
|
@@ -81,6 +92,24 @@ export const JsonTypeName = z.enum([
|
|
|
81
92
|
// A declared key name: used in a request shape, a response descriptor, or a
|
|
82
93
|
// reference-set key list. Plain caller-supplied text.
|
|
83
94
|
export const KeyName = z.string().min(1);
|
|
95
|
+
/**
|
|
96
|
+
* An environment variable name, at the port boundary where one becomes a real
|
|
97
|
+
* variable on a real process. `KeyName` is plain text, and a plain-text key is
|
|
98
|
+
* a smuggling channel here: `A=B` reaches the child as a variable `A` whose
|
|
99
|
+
* value carries `B=` in front of the declared one, and a key holding a NUL
|
|
100
|
+
* fails the spawn itself. The charset is the portable one, which is what a
|
|
101
|
+
* caller reading a mapping's allowlist assumes it is reading.
|
|
102
|
+
*
|
|
103
|
+
* This keys the port message, which every request crosses, and the
|
|
104
|
+
* authorization allowlist, which only a caller who parses their own mapping
|
|
105
|
+
* crosses: nothing in this package parses `CommandTargetPolicy`. The contract
|
|
106
|
+
* side keeps `KeyName`, since AD-19's channel shapes are one grammar across
|
|
107
|
+
* every channel, so a contract declaring a malformed environment key compiles
|
|
108
|
+
* and fails at this boundary with `schema-parse-failure`.
|
|
109
|
+
*/
|
|
110
|
+
export const EnvironmentKeyName = z
|
|
111
|
+
.string()
|
|
112
|
+
.regex(/^[A-Za-z_][A-Za-z0-9_]*$/, 'an environment key is a letter or underscore followed by letters, digits or underscores');
|
|
84
113
|
// The one shape whose keys belong to the caller, so `additionalProperties` is
|
|
85
114
|
// schema-valued rather than false here alone. Hand-rolled with z.lazy rather
|
|
86
115
|
// than z.json(): the latter exports a generated `$defs` ref name that the
|
|
@@ -91,6 +91,7 @@ export declare const CommandTargetAuthorization: z.ZodObject<{
|
|
|
91
91
|
executable: z.ZodString;
|
|
92
92
|
target: z.ZodString;
|
|
93
93
|
permittedSubcommandPaths: z.ZodArray<z.ZodArray<z.ZodString>>;
|
|
94
|
+
permittedEnvironmentKeys: z.ZodArray<z.ZodString>;
|
|
94
95
|
cwd: z.ZodString;
|
|
95
96
|
artifacts: z.ZodRecord<z.ZodString, z.ZodString>;
|
|
96
97
|
maxElapsedMs: z.ZodInt;
|
|
@@ -102,6 +103,7 @@ export declare const CommandTargetPolicy: z.ZodObject<{
|
|
|
102
103
|
executable: z.ZodString;
|
|
103
104
|
target: z.ZodString;
|
|
104
105
|
permittedSubcommandPaths: z.ZodArray<z.ZodArray<z.ZodString>>;
|
|
106
|
+
permittedEnvironmentKeys: z.ZodArray<z.ZodString>;
|
|
105
107
|
cwd: z.ZodString;
|
|
106
108
|
artifacts: z.ZodRecord<z.ZodString, z.ZodString>;
|
|
107
109
|
maxElapsedMs: z.ZodInt;
|
|
@@ -110,3 +112,42 @@ export declare const CommandTargetPolicy: z.ZodObject<{
|
|
|
110
112
|
}, z.core.$strict>;
|
|
111
113
|
export type CommandTargetAuthorization = z.infer<typeof CommandTargetAuthorization>;
|
|
112
114
|
export type CommandTargetPolicy = z.infer<typeof CommandTargetPolicy>;
|
|
115
|
+
/**
|
|
116
|
+
* One authorized tool server, AD-35's mapping for the `mcp` mechanism.
|
|
117
|
+
*
|
|
118
|
+
* Keyed by `interfaceId` alone. A `cli` authorization is keyed by
|
|
119
|
+
* `(interfaceId, executable)` because `CommandInvocation` is declared per
|
|
120
|
+
* operation and one interface may name two executables. A tool session has no
|
|
121
|
+
* such split: it is opened against one server and every tool it offers belongs
|
|
122
|
+
* to that server, so the interface identifier is the server identity, which is
|
|
123
|
+
* how the HTTP authorization is keyed too.
|
|
124
|
+
*
|
|
125
|
+
* Two of the eight fields are authorization-scoped, `interfaceId` and `tools`.
|
|
126
|
+
* The other six are what an authorized call runs with, the way `cwd`,
|
|
127
|
+
* `artifacts`, and the two caps sit on a command authorization without adding
|
|
128
|
+
* a denial of their own.
|
|
129
|
+
*/
|
|
130
|
+
export declare const McpTargetAuthorization: z.ZodObject<{
|
|
131
|
+
interfaceId: z.ZodString;
|
|
132
|
+
target: z.ZodString;
|
|
133
|
+
targetArgs: z.ZodArray<z.ZodString>;
|
|
134
|
+
tools: z.ZodArray<z.ZodString>;
|
|
135
|
+
cwd: z.ZodString;
|
|
136
|
+
serverEnvironment: z.ZodRecord<z.ZodString, z.ZodString>;
|
|
137
|
+
maxElapsedMs: z.ZodInt;
|
|
138
|
+
maxOutputBytes: z.ZodInt;
|
|
139
|
+
}, z.core.$strict>;
|
|
140
|
+
export declare const McpTargetPolicy: z.ZodObject<{
|
|
141
|
+
authorizations: z.ZodArray<z.ZodObject<{
|
|
142
|
+
interfaceId: z.ZodString;
|
|
143
|
+
target: z.ZodString;
|
|
144
|
+
targetArgs: z.ZodArray<z.ZodString>;
|
|
145
|
+
tools: z.ZodArray<z.ZodString>;
|
|
146
|
+
cwd: z.ZodString;
|
|
147
|
+
serverEnvironment: z.ZodRecord<z.ZodString, z.ZodString>;
|
|
148
|
+
maxElapsedMs: z.ZodInt;
|
|
149
|
+
maxOutputBytes: z.ZodInt;
|
|
150
|
+
}, z.core.$strict>>;
|
|
151
|
+
}, z.core.$strict>;
|
|
152
|
+
export type McpTargetAuthorization = z.infer<typeof McpTargetAuthorization>;
|
|
153
|
+
export type McpTargetPolicy = z.infer<typeof McpTargetPolicy>;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/** AD-35's default-deny target authorization, as a declared mapping. */
|
|
2
2
|
import { z } from 'zod';
|
|
3
3
|
import { HttpMethod } from './interface.js';
|
|
4
|
-
import { Identifier } from './primitives.js';
|
|
4
|
+
import { EnvironmentKeyName, Identifier, KeyName, ToolName, } from './primitives.js';
|
|
5
5
|
/**
|
|
6
6
|
* One authorized target. AD-35: "An adapter denies by default and permits only
|
|
7
7
|
* what that mapping names." Every field is required and none has a default:
|
|
@@ -54,6 +54,12 @@ export const CommandTargetAuthorization = z.strictObject({
|
|
|
54
54
|
.array(z.array(Identifier))
|
|
55
55
|
.min(1)
|
|
56
56
|
.describe('The exact subcommand paths this authorization allows, compared literally the way AD-40 compares them. An empty inner array authorizes invoking target with no subcommand. A path the request declares that matches none of these is denied before target is ever spawned, the same role methods plays on the HTTP side.'),
|
|
57
|
+
permittedEnvironmentKeys: z
|
|
58
|
+
.array(EnvironmentKeyName)
|
|
59
|
+
.refine((keys) => !keys.some((key) => key.toUpperCase() === 'PATH'), {
|
|
60
|
+
message: 'PATH cannot be permitted: target may name a bare command, and a declared PATH would then choose which binary runs',
|
|
61
|
+
})
|
|
62
|
+
.describe('The environment keys a request may carry into the process. `CommandProbeRequest.channels.environment` is declared by the contract author, and this is where the operator bounds it: a key absent from this list is denied before target is ever spawned, the same role permittedSubcommandPaths plays for a subcommand. An empty array is legal and permits no declared key, which is the default-deny base case. PATH is refused outright, because target may be "a name the adapter resolves through its own PATH" and the child environment is what resolves it: permitting PATH would hand executable selection to the contract author, which is the direction AD-35 exists to prevent. That refusal is narrow and covers executable selection alone. A key such as LD_PRELOAD or NODE_OPTIONS injects into the binary the mapping already chose, and this list is what keeps such a key out: naming one here is a deliberate act. The adapter refuses PATH again at its own boundary, since nothing in this package parses this policy and a refinement leaves no trace in the TypeScript type. The adapter\'s own PATH reaches the child from the process the mapping launched, under AD-18.'),
|
|
57
63
|
cwd: z
|
|
58
64
|
.string()
|
|
59
65
|
.min(1)
|
|
@@ -75,3 +81,57 @@ export const CommandTargetPolicy = z.strictObject({
|
|
|
75
81
|
.array(CommandTargetAuthorization)
|
|
76
82
|
.describe('An empty array is legal and authorizes nothing, the same default-deny base case as ProbeTargetPolicy.'),
|
|
77
83
|
});
|
|
84
|
+
/**
|
|
85
|
+
* One authorized tool server, AD-35's mapping for the `mcp` mechanism.
|
|
86
|
+
*
|
|
87
|
+
* Keyed by `interfaceId` alone. A `cli` authorization is keyed by
|
|
88
|
+
* `(interfaceId, executable)` because `CommandInvocation` is declared per
|
|
89
|
+
* operation and one interface may name two executables. A tool session has no
|
|
90
|
+
* such split: it is opened against one server and every tool it offers belongs
|
|
91
|
+
* to that server, so the interface identifier is the server identity, which is
|
|
92
|
+
* how the HTTP authorization is keyed too.
|
|
93
|
+
*
|
|
94
|
+
* Two of the eight fields are authorization-scoped, `interfaceId` and `tools`.
|
|
95
|
+
* The other six are what an authorized call runs with, the way `cwd`,
|
|
96
|
+
* `artifacts`, and the two caps sit on a command authorization without adding
|
|
97
|
+
* a denial of their own.
|
|
98
|
+
*/
|
|
99
|
+
export const McpTargetAuthorization = z.strictObject({
|
|
100
|
+
interfaceId: Identifier.describe('The logical interface identifier the contract names, which for this mechanism is the server identity. This mapping is where it becomes a launchable server, outside the contract.'),
|
|
101
|
+
target: z
|
|
102
|
+
.string()
|
|
103
|
+
.min(1)
|
|
104
|
+
.describe('The real, spawnable command that starts the server: an absolute path, or a name the adapter resolves through its own PATH. Never taken from the contract, which never carries one (AD-35).'),
|
|
105
|
+
targetArgs: z
|
|
106
|
+
.array(z.string())
|
|
107
|
+
.describe('The argument vector target is launched with, passed as an array so no value is ever concatenated into a shell string. Empty is legal and launches target with no arguments.'),
|
|
108
|
+
tools: z
|
|
109
|
+
.array(ToolName)
|
|
110
|
+
.min(1)
|
|
111
|
+
.describe("The tools this authorization permits, in the server's own spelling, compared literally. A toolName the request names that is absent from this list is denied before the server process starts, the same role permittedSubcommandPaths plays on the command side. That draws AD-35's disclosure boundary around the tool as well as the server: a tool the list omits is unreachable through this adapter even when the server publishes it."),
|
|
112
|
+
cwd: z
|
|
113
|
+
.string()
|
|
114
|
+
.min(1)
|
|
115
|
+
.describe('The working directory the server runs from. Declared rather than inherited from the adapter process, so a server resolving a relative fixture path resolves it against a directory the mapping names.'),
|
|
116
|
+
serverEnvironment: z
|
|
117
|
+
.record(KeyName, z.string())
|
|
118
|
+
.describe("The environment the server process is launched with, over a base of the host's own PATH so a target naming a bare command still resolves; a declared PATH key wins over that default. A tool call's only channel is its arguments, so a server needing a credential has nowhere else to receive one, and port-messages.ts places that material here: authorization material is the adapter's, supplied by the same mapping that authorizes the target (AD-18)."),
|
|
119
|
+
maxElapsedMs: z
|
|
120
|
+
.int()
|
|
121
|
+
.min(1)
|
|
122
|
+
.max(2_147_483_647)
|
|
123
|
+
.describe('Wall-clock budget for the whole port invocation: server launch, the initialize handshake, the tool call, and teardown. A handshake that never completes and a tool call that never answers are the same event to the caller, and both throw budget-exhausted. Bounded above by the largest delay a timer accepts: a larger value is silently clamped to one millisecond, which turns a generous budget into an immediate cap.'),
|
|
124
|
+
maxOutputBytes: z
|
|
125
|
+
.int()
|
|
126
|
+
.min(1)
|
|
127
|
+
.describe("Applies independently to the bytes the server writes on stdout, which carry the tool result, and to the bytes it writes on its own stderr, which MCP's stdio transport reserves for logging. The first stream to cross it tears the session down with budget-exhausted."),
|
|
128
|
+
});
|
|
129
|
+
export const McpTargetPolicy = z.strictObject({
|
|
130
|
+
authorizations: z
|
|
131
|
+
.array(McpTargetAuthorization)
|
|
132
|
+
.refine((authorizations) => new Set(authorizations.map((each) => each.interfaceId)).size ===
|
|
133
|
+
authorizations.length, {
|
|
134
|
+
message: 'two authorizations name one interfaceId, so which server the interface resolves to would depend on which tool was asked for',
|
|
135
|
+
})
|
|
136
|
+
.describe('An empty array is legal and authorizes nothing, the same default-deny base case as ProbeTargetPolicy. One entry per interfaceId: the interface identifier is the server identity for this mechanism, so a second entry naming it could point one logical interface at a second binary. That is the difference from CommandTargetPolicy, whose entries are keyed by (interfaceId, executable) and so cannot disagree about what runs.'),
|
|
137
|
+
});
|
|
@@ -13,7 +13,7 @@ export declare const ProbeClass: z.ZodEnum<{
|
|
|
13
13
|
gameability: "gameability";
|
|
14
14
|
"zero-action": "zero-action";
|
|
15
15
|
}>;
|
|
16
|
-
/** the prior art's six-field
|
|
16
|
+
/** AD-9's seeded defect: the prior art's six fields, plus the nullable `manifestationWitness` a six-field defect round-trips through. */
|
|
17
17
|
export declare const Defect: z.ZodObject<{
|
|
18
18
|
defectId: z.ZodString;
|
|
19
19
|
behaviorId: z.ZodString;
|
|
@@ -65,6 +65,8 @@ export declare const Defect: z.ZodObject<{
|
|
|
65
65
|
}, z.core.$strict>, z.ZodObject<{
|
|
66
66
|
kind: z.ZodLiteral<"absent">;
|
|
67
67
|
}, z.core.$strict>], "kind">;
|
|
68
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
69
|
+
arguments: z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>;
|
|
68
70
|
}, z.core.$strict>]>;
|
|
69
71
|
relation: z.ZodType<import("./expression.ts").Expression, unknown, z.core.$ZodTypeInternals<import("./expression.ts").Expression, unknown>>;
|
|
70
72
|
}, z.core.$strict>>;
|
|
@@ -270,6 +272,8 @@ export declare const Probe: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
270
272
|
}, z.core.$strict>, z.ZodObject<{
|
|
271
273
|
kind: z.ZodLiteral<"absent">;
|
|
272
274
|
}, z.core.$strict>], "kind">;
|
|
275
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
276
|
+
arguments: z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>;
|
|
273
277
|
}, z.core.$strict>]>;
|
|
274
278
|
relation: z.ZodType<import("./expression.ts").Expression, unknown, z.core.$ZodTypeInternals<import("./expression.ts").Expression, unknown>>;
|
|
275
279
|
}, z.core.$strict>>;
|
|
@@ -466,6 +470,8 @@ export declare const Probe: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
466
470
|
}, z.core.$strict>, z.ZodObject<{
|
|
467
471
|
kind: z.ZodLiteral<"absent">;
|
|
468
472
|
}, z.core.$strict>], "kind">;
|
|
473
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
474
|
+
arguments: z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>;
|
|
469
475
|
}, z.core.$strict>]>;
|
|
470
476
|
relation: z.ZodType<import("./expression.ts").Expression, unknown, z.core.$ZodTypeInternals<import("./expression.ts").Expression, unknown>>;
|
|
471
477
|
}, z.core.$strict>>;
|
|
@@ -548,13 +554,20 @@ export declare const Probe: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
548
554
|
"type-violating": "type-violating";
|
|
549
555
|
}>;
|
|
550
556
|
}, z.core.$strict>]>>>;
|
|
557
|
+
arguments: z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodObject<{
|
|
558
|
+
literal: z.ZodType<import("./primitives.ts").JsonValue, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonValue, unknown>>;
|
|
559
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
560
|
+
matcher: z.ZodEnum<{
|
|
561
|
+
any: "any";
|
|
562
|
+
"type-violating": "type-violating";
|
|
563
|
+
}>;
|
|
564
|
+
}, z.core.$strict>]>>>;
|
|
551
565
|
}, z.core.$strict>;
|
|
552
566
|
}, z.core.$strict>;
|
|
553
567
|
predicate: z.ZodType<import("./expression.ts").Expression, unknown, z.core.$ZodTypeInternals<import("./expression.ts").Expression, unknown>>;
|
|
554
568
|
}, z.core.$strict>;
|
|
555
569
|
interfaceKind: z.ZodEnum<{
|
|
556
570
|
api: "api";
|
|
557
|
-
mcp: "mcp";
|
|
558
571
|
web: "web";
|
|
559
572
|
}>;
|
|
560
573
|
method: z.ZodEnum<{
|
|
@@ -645,6 +658,14 @@ export declare const Probe: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
645
658
|
"type-violating": "type-violating";
|
|
646
659
|
}>;
|
|
647
660
|
}, z.core.$strict>]>>>;
|
|
661
|
+
arguments: z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodObject<{
|
|
662
|
+
literal: z.ZodType<import("./primitives.ts").JsonValue, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonValue, unknown>>;
|
|
663
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
664
|
+
matcher: z.ZodEnum<{
|
|
665
|
+
any: "any";
|
|
666
|
+
"type-violating": "type-violating";
|
|
667
|
+
}>;
|
|
668
|
+
}, z.core.$strict>]>>>;
|
|
648
669
|
}, z.core.$strict>;
|
|
649
670
|
}, z.core.$strict>;
|
|
650
671
|
predicate: z.ZodType<import("./expression.ts").Expression, unknown, z.core.$ZodTypeInternals<import("./expression.ts").Expression, unknown>>;
|
|
@@ -654,6 +675,98 @@ export declare const Probe: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
654
675
|
executable: z.ZodString;
|
|
655
676
|
subcommandPath: z.ZodArray<z.ZodString>;
|
|
656
677
|
}, z.core.$strict>;
|
|
678
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
679
|
+
observableChannel: z.ZodEnum<{
|
|
680
|
+
artifact: "artifact";
|
|
681
|
+
"call-inputs": "call-inputs";
|
|
682
|
+
"exit-code": "exit-code";
|
|
683
|
+
"response-body": "response-body";
|
|
684
|
+
"response-headers": "response-headers";
|
|
685
|
+
"response-status": "response-status";
|
|
686
|
+
stderr: "stderr";
|
|
687
|
+
stdout: "stdout";
|
|
688
|
+
}>;
|
|
689
|
+
condition: z.ZodObject<{
|
|
690
|
+
selector: z.ZodObject<{
|
|
691
|
+
inputBinding: z.ZodObject<{
|
|
692
|
+
path: z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodObject<{
|
|
693
|
+
literal: z.ZodType<import("./primitives.ts").JsonValue, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonValue, unknown>>;
|
|
694
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
695
|
+
matcher: z.ZodEnum<{
|
|
696
|
+
any: "any";
|
|
697
|
+
"type-violating": "type-violating";
|
|
698
|
+
}>;
|
|
699
|
+
}, z.core.$strict>]>>>;
|
|
700
|
+
query: z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodObject<{
|
|
701
|
+
literal: z.ZodType<import("./primitives.ts").JsonValue, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonValue, unknown>>;
|
|
702
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
703
|
+
matcher: z.ZodEnum<{
|
|
704
|
+
any: "any";
|
|
705
|
+
"type-violating": "type-violating";
|
|
706
|
+
}>;
|
|
707
|
+
}, z.core.$strict>]>>>;
|
|
708
|
+
header: z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodObject<{
|
|
709
|
+
literal: z.ZodType<import("./primitives.ts").JsonValue, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonValue, unknown>>;
|
|
710
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
711
|
+
matcher: z.ZodEnum<{
|
|
712
|
+
any: "any";
|
|
713
|
+
"type-violating": "type-violating";
|
|
714
|
+
}>;
|
|
715
|
+
}, z.core.$strict>]>>>;
|
|
716
|
+
body: z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodObject<{
|
|
717
|
+
literal: z.ZodType<import("./primitives.ts").JsonValue, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonValue, unknown>>;
|
|
718
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
719
|
+
matcher: z.ZodEnum<{
|
|
720
|
+
any: "any";
|
|
721
|
+
"type-violating": "type-violating";
|
|
722
|
+
}>;
|
|
723
|
+
}, z.core.$strict>]>>>;
|
|
724
|
+
argument: z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodObject<{
|
|
725
|
+
literal: z.ZodType<import("./primitives.ts").JsonValue, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonValue, unknown>>;
|
|
726
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
727
|
+
matcher: z.ZodEnum<{
|
|
728
|
+
any: "any";
|
|
729
|
+
"type-violating": "type-violating";
|
|
730
|
+
}>;
|
|
731
|
+
}, z.core.$strict>]>>>;
|
|
732
|
+
option: z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodObject<{
|
|
733
|
+
literal: z.ZodType<import("./primitives.ts").JsonValue, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonValue, unknown>>;
|
|
734
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
735
|
+
matcher: z.ZodEnum<{
|
|
736
|
+
any: "any";
|
|
737
|
+
"type-violating": "type-violating";
|
|
738
|
+
}>;
|
|
739
|
+
}, z.core.$strict>]>>>;
|
|
740
|
+
environment: z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodObject<{
|
|
741
|
+
literal: z.ZodType<import("./primitives.ts").JsonValue, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonValue, unknown>>;
|
|
742
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
743
|
+
matcher: z.ZodEnum<{
|
|
744
|
+
any: "any";
|
|
745
|
+
"type-violating": "type-violating";
|
|
746
|
+
}>;
|
|
747
|
+
}, z.core.$strict>]>>>;
|
|
748
|
+
stdin: z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodObject<{
|
|
749
|
+
literal: z.ZodType<import("./primitives.ts").JsonValue, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonValue, unknown>>;
|
|
750
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
751
|
+
matcher: z.ZodEnum<{
|
|
752
|
+
any: "any";
|
|
753
|
+
"type-violating": "type-violating";
|
|
754
|
+
}>;
|
|
755
|
+
}, z.core.$strict>]>>>;
|
|
756
|
+
arguments: z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodObject<{
|
|
757
|
+
literal: z.ZodType<import("./primitives.ts").JsonValue, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonValue, unknown>>;
|
|
758
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
759
|
+
matcher: z.ZodEnum<{
|
|
760
|
+
any: "any";
|
|
761
|
+
"type-violating": "type-violating";
|
|
762
|
+
}>;
|
|
763
|
+
}, z.core.$strict>]>>>;
|
|
764
|
+
}, z.core.$strict>;
|
|
765
|
+
}, z.core.$strict>;
|
|
766
|
+
predicate: z.ZodType<import("./expression.ts").Expression, unknown, z.core.$ZodTypeInternals<import("./expression.ts").Expression, unknown>>;
|
|
767
|
+
}, z.core.$strict>;
|
|
768
|
+
interfaceKind: z.ZodLiteral<"mcp">;
|
|
769
|
+
toolName: z.ZodString;
|
|
657
770
|
}, z.core.$strict>]>>;
|
|
658
771
|
}, z.core.$strict>], "expectedClean">;
|
|
659
772
|
export type Probe = z.infer<typeof Probe>;
|
|
@@ -20,7 +20,7 @@ export const PROBE_CLASSES = [
|
|
|
20
20
|
'canary',
|
|
21
21
|
];
|
|
22
22
|
export const ProbeClass = z.enum(PROBE_CLASSES);
|
|
23
|
-
/** the prior art's six-field
|
|
23
|
+
/** AD-9's seeded defect: the prior art's six fields, plus the nullable `manifestationWitness` a six-field defect round-trips through. */
|
|
24
24
|
export const Defect = z.strictObject({
|
|
25
25
|
defectId: DefectId,
|
|
26
26
|
behaviorId: BehaviorId.describe('The behaviour this defect breaks. AD-9 also puts a behaviour on the probe itself and this schema carries both; that the two may disagree is a cross-field rule with no AD-5 code and is left unenforced in v0 rather than silently refined here.'),
|
|
@@ -82,5 +82,5 @@ export const Probe = z
|
|
|
82
82
|
])
|
|
83
83
|
.meta({
|
|
84
84
|
id: 'Probe',
|
|
85
|
-
description: "One corpus probe. Succeeds the prior-art `h0-ground-truth` schema per AD-24, carrying its system identifier, implementation digest, `expectedClean` flag, seeded defects, and rationale, and adding AD-9's probe class and AD-9's per-probe artifact and commit digests. Divergences: `implementationSha` becomes `implementationDigest`, `taskId` does not survive because the probe pins what it describes by digest, and `expectedGate` does not survive because AD-40 makes detection a signature match rather than a verdict comparison and AD-7 keeps comparisons inside the dominance vector, so carrying an expected gate would invite a comparison the architecture forbids. Two constructions landed here together under one BREAKING `schemaVersion` bump: AD-9's per-class QUALIFICATION record, as a five-route tagged union required on every branch, and AD-40's machine-readable DEFECT SIGNATURE on the `expectedClean: false` branch, carrying the interface kind, the home operation
|
|
85
|
+
description: "One corpus probe. Succeeds the prior-art `h0-ground-truth` schema per AD-24, carrying its system identifier, implementation digest, `expectedClean` flag, seeded defects, and rationale, and adding AD-9's probe class and AD-9's per-probe artifact and commit digests. Divergences: `implementationSha` becomes `implementationDigest`, `taskId` does not survive because the probe pins what it describes by digest, and `expectedGate` does not survive because AD-40 makes detection a signature match rather than a verdict comparison and AD-7 keeps comparisons inside the dominance vector, so carrying an expected gate would invite a comparison the architecture forbids. Two constructions landed here together under one BREAKING `schemaVersion` bump: AD-9's per-class QUALIFICATION record, as a five-route tagged union required on every branch, and AD-40's machine-readable DEFECT SIGNATURE on the `expectedClean: false` branch, carrying the interface kind, the home operation by its transport identity, the observable channel, and the discriminating condition. Both are required rather than optional, which is what makes the bump breaking under AD-11: every corpus written against version 1 fails to parse. Version 3 opened the signature to a second interface kind: it is a union on `interfaceKind`, where the `cli` branch declares a logical invocation in place of a method and a path template, and the selector's input binding carries the four command channels beside the four transport ones. Version 4 is a BREAKING bump on the witness side: a manifestation witness's `inputs` gained a third leg shape over a tool call's arguments, so a witness leg against a tool call is expressible; every version-3 probe's own bytes still parse against it. Version 5 opened the signature to a third kind and closed both gaps version 4 left. `DefectSignature` gains a tool-call branch declaring the published tool name, which is the transport identity AD-40 resolves an mcp signature against, and the api-shaped branch narrows to `api` and `web`, so a signature carrying `mcp` beside a method and a path template stops parsing. The selector's input binding gains a ninth `arguments` channel on the same required-and-nullable terms as the other eight, so a version-3 or version-4 probe declaring eight channels stops parsing too. The signature is therefore three branch shapes rather than one, each declaring its own kind's transport identity: a method and a path template, a logical invocation, or a published tool name. What the schema still does not decide is stated rather than hidden. AD-9's \"an unqualified probe cannot enter a sealed set\" is a corpus-construction invariant enforced by the qualification gate in `core/score/qualification.ts`, not by this schema: all eight class-and-`expectedClean` pairings parse, a route incompatible with the pair parses, and a signature-less non-canary parses, each so the gate can return a reason code carrying an artifact path instead of an anonymous parse failure. The constraint ledger carries both gaps.",
|
|
86
86
|
});
|
|
@@ -190,10 +190,14 @@ export type OracleDisposition = z.infer<typeof OracleDisposition>;
|
|
|
190
190
|
* A flat map would break pointer addressing: AD-26 keys `call-inputs` by
|
|
191
191
|
* transport channel, so a pointer like
|
|
192
192
|
* `/interactions/write/call-inputs/body/title` needs that segment to resolve
|
|
193
|
-
* against. A
|
|
194
|
-
* enum, for the same reason as `RequestShape` and `InputBinding`: a
|
|
195
|
-
* demands every enum member at parse time,
|
|
196
|
-
* a subset.
|
|
193
|
+
* against. A strict object keyed by channel rather than a record over the
|
|
194
|
+
* channel enum, for the same reason as `RequestShape` and `InputBinding`: a
|
|
195
|
+
* record demands every enum member at parse time, and a real observation binds
|
|
196
|
+
* only a subset.
|
|
197
|
+
*
|
|
198
|
+
* One key per member of `INPUT_CHANNELS`, in the order the vocabulary spells
|
|
199
|
+
* them. A loop over that vocabulary indexes this shape directly, so the two
|
|
200
|
+
* have to stay the same width.
|
|
197
201
|
*/
|
|
198
202
|
export declare const ObservedCallInputs: z.ZodObject<{
|
|
199
203
|
path: z.ZodNullable<z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>>;
|
|
@@ -204,6 +208,7 @@ export declare const ObservedCallInputs: z.ZodObject<{
|
|
|
204
208
|
option: z.ZodNullable<z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>>;
|
|
205
209
|
environment: z.ZodNullable<z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>>;
|
|
206
210
|
stdin: z.ZodNullable<z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>>;
|
|
211
|
+
arguments: z.ZodNullable<z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>>;
|
|
207
212
|
}, z.core.$strict>;
|
|
208
213
|
/** the constraint identifier the ledger carries for the check below. */
|
|
209
214
|
export declare const OBSERVATION_SEQUENCE_UNIQUE = "observation-sequence-unique";
|
|
@@ -233,6 +238,7 @@ export declare const Observation: z.ZodObject<{
|
|
|
233
238
|
option: z.ZodNullable<z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>>;
|
|
234
239
|
environment: z.ZodNullable<z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>>;
|
|
235
240
|
stdin: z.ZodNullable<z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>>;
|
|
241
|
+
arguments: z.ZodNullable<z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>>;
|
|
236
242
|
}, z.core.$strict>;
|
|
237
243
|
responseBody: z.ZodNullable<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
|
|
238
244
|
responseHeaders: z.ZodNullable<z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>>;
|
|
@@ -454,6 +460,7 @@ export declare const SealedRunRecord: z.ZodObject<{
|
|
|
454
460
|
option: z.ZodNullable<z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>>;
|
|
455
461
|
environment: z.ZodNullable<z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>>;
|
|
456
462
|
stdin: z.ZodNullable<z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>>;
|
|
463
|
+
arguments: z.ZodNullable<z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>>;
|
|
457
464
|
}, z.core.$strict>;
|
|
458
465
|
responseBody: z.ZodNullable<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
|
|
459
466
|
responseHeaders: z.ZodNullable<z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>>;
|
|
@@ -526,6 +533,5 @@ export declare const SealedRunRecord: z.ZodObject<{
|
|
|
526
533
|
truncationBound: z.ZodNullable<z.ZodInt>;
|
|
527
534
|
reportedIncomplete: z.ZodBoolean;
|
|
528
535
|
}, z.core.$strict>;
|
|
529
|
-
invalidReason: z.ZodNullable<z.ZodString>;
|
|
530
536
|
}, z.core.$strict>;
|
|
531
537
|
export type SealedRunRecord = z.infer<typeof SealedRunRecord>;
|
|
@@ -135,10 +135,14 @@ export const OracleDisposition = z.strictObject({
|
|
|
135
135
|
* A flat map would break pointer addressing: AD-26 keys `call-inputs` by
|
|
136
136
|
* transport channel, so a pointer like
|
|
137
137
|
* `/interactions/write/call-inputs/body/title` needs that segment to resolve
|
|
138
|
-
* against. A
|
|
139
|
-
* enum, for the same reason as `RequestShape` and `InputBinding`: a
|
|
140
|
-
* demands every enum member at parse time,
|
|
141
|
-
* a subset.
|
|
138
|
+
* against. A strict object keyed by channel rather than a record over the
|
|
139
|
+
* channel enum, for the same reason as `RequestShape` and `InputBinding`: a
|
|
140
|
+
* record demands every enum member at parse time, and a real observation binds
|
|
141
|
+
* only a subset.
|
|
142
|
+
*
|
|
143
|
+
* One key per member of `INPUT_CHANNELS`, in the order the vocabulary spells
|
|
144
|
+
* them. A loop over that vocabulary indexes this shape directly, so the two
|
|
145
|
+
* have to stay the same width.
|
|
142
146
|
*/
|
|
143
147
|
export const ObservedCallInputs = z.strictObject({
|
|
144
148
|
path: JsonObjectValue.nullable(),
|
|
@@ -149,6 +153,7 @@ export const ObservedCallInputs = z.strictObject({
|
|
|
149
153
|
option: JsonObjectValue.nullable(),
|
|
150
154
|
environment: JsonObjectValue.nullable(),
|
|
151
155
|
stdin: JsonObjectValue.nullable(),
|
|
156
|
+
arguments: JsonObjectValue.nullable().describe("AD-26's `arguments` channel, the one channel a tool call accepts input on. Its arrival retypes this shape from eight keys to nine, which AD-11 calls breaking and which moves the Sealed Run Record's `schemaVersion` from 4 to 5: a version-4 record declares no `arguments` key and fails to parse against version 5. A name-to-value map on `responseHeaders`' terms, since a pointer descends INTO the channel to address one argument. `null` on an observation that exercised no tool call, which is what every channel here already spells for a channel nothing was sent on."),
|
|
152
157
|
});
|
|
153
158
|
/** the constraint identifier the ledger carries for the check below. */
|
|
154
159
|
export const OBSERVATION_SEQUENCE_UNIQUE = 'observation-sequence-unique';
|
|
@@ -172,13 +177,13 @@ export const Observation = z.strictObject({
|
|
|
172
177
|
.describe("AD-23: `baseline` for a pre-canned or deterministic test, `evaluator-chosen` for an action the evaluator selected. The distinction is the one the product's central finding rests on: what a sealed evaluator detects beyond the pre-canned baseline. It lives on the observation, never on the finding."),
|
|
173
178
|
principal: Identifier.nullable().describe("The declared principal the harness acted as, or `null` where the run named none. Owed item 3's other half: a `{ principal }` input binding is presence-only by construction, since the contract declares a name and the harness provisions the value, so without this field two steps of one operation binding `owner` and `other-user` both match every observation and both resolve `several`. That is exactly the act-as-A-read-as-B shape the two critical-severity cross-user behaviours need, and it was unscoreable while the record said nothing about which account was used. An opaque label carrying no account identifier or credential, on `testData.principals`' own AD-18 terms."),
|
|
174
179
|
callInputs: ObservedCallInputs,
|
|
175
|
-
responseBody: JsonValue.nullable().describe('AD-26\'s `response-body` channel. The null branch is redundant against the value container, which already admits `null`; it is kept so
|
|
180
|
+
responseBody: JsonValue.nullable().describe('AD-26\'s `response-body` channel. The null branch is redundant against the value container, which already admits `null`; it is kept so every observation field reads the same way, and it means "no body observed" and "a body that was JSON null" are indistinguishable here, which is an accepted cost of one uniform spelling.'),
|
|
176
181
|
responseHeaders: JsonObjectValue.nullable().describe('A name-to-value map, not the open value container. AD-26 gives `response-headers` a tail, so a pointer resolves INTO this channel; a scalar here would leave `/interactions/x/response-headers/Content-Type` addressing nothing. That is the difference from `responseBody`, where a scalar or an array is a legitimate body and the open container is correct.'),
|
|
177
182
|
responseStatus: z
|
|
178
183
|
.int()
|
|
179
184
|
.min(0)
|
|
180
185
|
.nullable()
|
|
181
|
-
.describe('Deliberately not bounded to a protocol range. A negative status is meaningless and excluded, but the upper end is left open: AD-19 declares four interface kinds and
|
|
186
|
+
.describe('Deliberately not bounded to a protocol range. A negative status is meaningless and excluded, but the upper end is left open: AD-19 declares four interface kinds and not all of them speak HTTP, so bounding this to HTTP would encode a protocol assumption the artifact outlives. `null` where the channel does not apply.'),
|
|
182
187
|
stdout: ProbeObservedBody.describe('AD-26\'s `stdout` channel, tagged rather than a bare string. An operation may nominate standard output as the channel its response descriptor describes, in which case a pointer descends into it, and an untagged string could not tell output that was JSON from output that was only ever text. `{ "kind": "absent" }` is the channel a run did not observe.'),
|
|
183
188
|
stderr: ProbeObservedBody.describe("AD-26's `stderr` channel, tagged on the same terms as `stdout`."),
|
|
184
189
|
exitCode: z
|
|
@@ -243,6 +248,14 @@ export const RUN_MODES = ['production', 'contract-scoring'];
|
|
|
243
248
|
export const RunMode = z.enum(RUN_MODES);
|
|
244
249
|
export const SealedRunRecord = z
|
|
245
250
|
.strictObject({
|
|
251
|
+
// A record carries lineage fields and nothing here ever puts one in a
|
|
252
|
+
// chain. `validateLineageChain` has no caller in this package at all: it
|
|
253
|
+
// is exported for consumers, and `score.ts` carries the reasoning for
|
|
254
|
+
// why score deliberately does not call it. That is what lets a field be
|
|
255
|
+
// added or removed here without moving any caller's scoring version,
|
|
256
|
+
// since a chain digests each member whole. A consumer who chains records
|
|
257
|
+
// makes that false for themselves, and the version-6 changelog's
|
|
258
|
+
// comparability claim stops holding for them.
|
|
246
259
|
...lineageFields,
|
|
247
260
|
runId: z
|
|
248
261
|
.string()
|
|
@@ -256,7 +269,7 @@ export const SealedRunRecord = z
|
|
|
256
269
|
trialIndex: z
|
|
257
270
|
.int()
|
|
258
271
|
.min(1)
|
|
259
|
-
.describe("Which trial this record is. AD-24 excludes the trial index from the Evaluator Configuration \"so trials pool into one scoring version\", which requires it somewhere else, and a Sealed Run Record is the only artifact carrying exactly one trial. One-based, matching the only instance that exists. AD-6's aggregate of trial count, invalidated attempts, and each attempt's reason is the Evidence Artifact's
|
|
272
|
+
.describe("Which trial this record is. AD-24 excludes the trial index from the Evaluator Configuration \"so trials pool into one scoring version\", which requires it somewhere else, and a Sealed Run Record is the only artifact carrying exactly one trial. One-based, matching the only instance that exists. AD-6's aggregate of trial count, invalidated attempts, and each attempt's reason is the Evidence Artifact's, computed by `score` over the trial set a caller assembles. The reducer keys an attempt by its position in that set, so this field is the caller's own bookkeeping and a label for a reader of one record."),
|
|
260
273
|
contractDigest: Digest,
|
|
261
274
|
sealedBriefDigest: Digest,
|
|
262
275
|
evaluatorConfigurationDigest: Digest.describe('A bare digest rather than an `ArtifactReference`, and required on both this record and the isolation manifest. AD-32 requires the two to *agree*, and an `ArtifactReference` on one side against a bare digest on the other makes the comparison lopsided; AD-2 already has ingest receiving the Evaluator Configuration as its own input, so nothing needs resolving through a reference. The agreement itself is a cross-artifact rule no schema can see.'),
|
|
@@ -284,9 +297,8 @@ export const SealedRunRecord = z
|
|
|
284
297
|
isolationManifestArtifact: ArtifactReference,
|
|
285
298
|
resourceUse: ResourceUse,
|
|
286
299
|
evidenceDisclosure: EvidenceDisclosure,
|
|
287
|
-
invalidReason: z.string().nullable(),
|
|
288
300
|
})
|
|
289
301
|
.meta({
|
|
290
302
|
id: 'SealedRunRecord',
|
|
291
|
-
description: "One sealed evaluator trial, as the caller presents it. Succeeds the prior-art `h0-run-result` schema per AD-24, keeping its run identifier, condition arm, findings, action-log reference, resource use,
|
|
303
|
+
description: "One sealed evaluator trial, as the caller presents it. Succeeds the prior-art `h0-run-result` schema per AD-24, keeping its run identifier, condition arm, findings, action-log reference, resource use, evaluator recommendation as a closed enum, and per-finding confidence on a declared scale. Divergences: `condition` is demoted to the opaque `conditionArm`, `verdict` becomes `evaluatorRecommendation` without `NOT_APPLICABLE`, money is a decimal string, and `taskId`, `note`, per-finding `actionIds`, and the prior art's `invalidReason` do not survive: the contract is pinned by `contractDigest`, an unstructured orchestrator annotation is the free-prose channel the Conventions close everywhere else, and two citation vocabularies on one finding is the ambiguity ADR-009 removed. The run MODE landed here as a required field under a BREAKING `schemaVersion` bump, which is where AD-21's \"fixed before ingest\" puts it; owed item 4 is now closed: mode enters AD-11's identity inputs as `ScoringVersionInputs`'s sixth field, and `core/score/ladder.ts` carries `ProductionAssessment`/`ContractAssessment` as the two assessment input types with their own total ladders. Observation ORDERING landed here too, under its own BREAKING `schemaVersion` bump: `sequence` is required and unique per record, closing owed item 2's ADR-006 gap, since array position was never a legal ordering. Version 4 opened the record to a system under test that runs behind a command: `callInputs` carries the four command channels beside the four transport ones, `stdout` and `stderr` are tagged rather than bare strings so a nominated output channel can be descended into, and `artifacts` records the files the run wrote, keyed by the identifier the operation declares. Version 5 opened it to a tool call: `callInputs` carries a ninth `arguments` channel beside those eight, so what a tool call supplied has somewhere to live and a defect signature's selector filtering on that channel has something to read. A version-4 record declares eight call-input channels and fails to parse against version 5. Version 6 drops `invalidReason`, the prior art's run-level invalidation reason: nothing in this package ever read it, so a caller attesting that a run was invalid was ignored by every stage while the field looked like a supported channel. The attestation that works is `IsolationManifest.violation`, which `core/ingest` raises as an `isolation-manifest-violation` condition and which reaches the verdict basis, and AD-6's invalidating outcome states carry a failure the run itself produced. A version-5 record carrying `invalidReason` fails to parse against version 6.",
|
|
292
304
|
});
|
|
@@ -51,7 +51,17 @@ export declare const CommandWitnessInputs: z.ZodObject<{
|
|
|
51
51
|
}, z.core.$strict>;
|
|
52
52
|
export type CommandWitnessInputs = z.infer<typeof CommandWitnessInputs>;
|
|
53
53
|
/**
|
|
54
|
-
*
|
|
54
|
+
* The same, for a leg supplied to a tool call. One key, because a tool call
|
|
55
|
+
* carries an arguments object and nothing else, and a JSON object rather than a
|
|
56
|
+
* tagged value because an argument-less call supplies `{}` and has no absent
|
|
57
|
+
* spelling to tell apart from a JSON null.
|
|
58
|
+
*/
|
|
59
|
+
export declare const McpWitnessInputs: z.ZodObject<{
|
|
60
|
+
arguments: z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>;
|
|
61
|
+
}, z.core.$strict>;
|
|
62
|
+
export type McpWitnessInputs = z.infer<typeof McpWitnessInputs>;
|
|
63
|
+
/**
|
|
64
|
+
* Any spelling. A plain union rather than a discriminated one for the same
|
|
55
65
|
* reason `InputBinding` is: the leg names an operation and the kind of the
|
|
56
66
|
* interface declaring it lives in another subtree, so no discriminator is
|
|
57
67
|
* available to the schema and the agreement is a compile-time check.
|
|
@@ -61,11 +71,12 @@ export type CommandWitnessInputs = z.infer<typeof CommandWitnessInputs>;
|
|
|
61
71
|
* took it from 0.3.0 and the other two followed in 1.3.0, since the transport
|
|
62
72
|
* spelling left a seeded defect against a command-line system under test
|
|
63
73
|
* unrepresentable. The reach is the port's: all three legs are issued through
|
|
64
|
-
* the environment-probe port, whose `ProbeRequest`
|
|
65
|
-
* `
|
|
66
|
-
*
|
|
67
|
-
*
|
|
68
|
-
*
|
|
74
|
+
* the environment-probe port, whose `ProbeRequest` carries one member per kind
|
|
75
|
+
* `isSupportedInterfaceKind` admits, and `preflight/plan.ts` gates on that same
|
|
76
|
+
* predicate. The invariant is the count rather than the list: a leg shape
|
|
77
|
+
* narrower than the port it feeds leaves a kind the adapter can run with no way
|
|
78
|
+
* to declare a leg for it, so this union takes one member per port member and
|
|
79
|
+
* opening a kind adds one to each.
|
|
69
80
|
*/
|
|
70
81
|
export declare const WitnessInputs: z.ZodUnion<readonly [z.ZodObject<{
|
|
71
82
|
path: z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>;
|
|
@@ -90,6 +101,8 @@ export declare const WitnessInputs: z.ZodUnion<readonly [z.ZodObject<{
|
|
|
90
101
|
}, z.core.$strict>, z.ZodObject<{
|
|
91
102
|
kind: z.ZodLiteral<"absent">;
|
|
92
103
|
}, z.core.$strict>], "kind">;
|
|
104
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
105
|
+
arguments: z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>;
|
|
93
106
|
}, z.core.$strict>]>;
|
|
94
107
|
export type WitnessInputs = z.infer<typeof WitnessInputs>;
|
|
95
108
|
/**
|
|
@@ -122,14 +135,18 @@ export declare const SensitivityWitnessLeg: z.ZodObject<{
|
|
|
122
135
|
}, z.core.$strict>, z.ZodObject<{
|
|
123
136
|
kind: z.ZodLiteral<"absent">;
|
|
124
137
|
}, z.core.$strict>], "kind">;
|
|
138
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
139
|
+
arguments: z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>;
|
|
125
140
|
}, z.core.$strict>]>;
|
|
126
141
|
}, z.core.$strict>;
|
|
127
142
|
export type SensitivityWitnessLeg = z.infer<typeof SensitivityWitnessLeg>;
|
|
128
143
|
export declare const API_WITNESS_CHANNELS: readonly ['path', 'query', 'body'];
|
|
129
144
|
export declare const COMMAND_WITNESS_CHANNELS: readonly ['argument', 'option', 'environment', 'stdin'];
|
|
130
|
-
export declare const
|
|
145
|
+
export declare const MCP_WITNESS_CHANNELS: readonly ['arguments'];
|
|
146
|
+
export declare const WITNESS_CHANNELS: readonly ["path", "query", "body", "argument", "option", "environment", "stdin", "arguments"];
|
|
131
147
|
export declare const WitnessChannel: z.ZodEnum<{
|
|
132
148
|
argument: "argument";
|
|
149
|
+
arguments: "arguments";
|
|
133
150
|
body: "body";
|
|
134
151
|
environment: "environment";
|
|
135
152
|
option: "option";
|
|
@@ -156,6 +173,7 @@ export declare const SensitivityWitness: z.ZodObject<{
|
|
|
156
173
|
witnessId: z.ZodString;
|
|
157
174
|
channel: z.ZodEnum<{
|
|
158
175
|
argument: "argument";
|
|
176
|
+
arguments: "arguments";
|
|
159
177
|
body: "body";
|
|
160
178
|
environment: "environment";
|
|
161
179
|
option: "option";
|
|
@@ -188,6 +206,8 @@ export declare const SensitivityWitness: z.ZodObject<{
|
|
|
188
206
|
}, z.core.$strict>, z.ZodObject<{
|
|
189
207
|
kind: z.ZodLiteral<"absent">;
|
|
190
208
|
}, z.core.$strict>], "kind">;
|
|
209
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
210
|
+
arguments: z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>;
|
|
191
211
|
}, z.core.$strict>]>;
|
|
192
212
|
}, z.core.$strict>>;
|
|
193
213
|
relation: z.ZodType<Expression, unknown, z.core.$ZodTypeInternals<Expression, unknown>>;
|
|
@@ -239,6 +259,8 @@ export declare const ManifestationWitness: z.ZodObject<{
|
|
|
239
259
|
}, z.core.$strict>, z.ZodObject<{
|
|
240
260
|
kind: z.ZodLiteral<"absent">;
|
|
241
261
|
}, z.core.$strict>], "kind">;
|
|
262
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
263
|
+
arguments: z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>;
|
|
242
264
|
}, z.core.$strict>]>;
|
|
243
265
|
relation: z.ZodType<Expression, unknown, z.core.$ZodTypeInternals<Expression, unknown>>;
|
|
244
266
|
}, z.core.$strict>;
|
|
@@ -275,6 +297,8 @@ export declare const FixtureReset: z.ZodObject<{
|
|
|
275
297
|
}, z.core.$strict>, z.ZodObject<{
|
|
276
298
|
kind: z.ZodLiteral<"absent">;
|
|
277
299
|
}, z.core.$strict>], "kind">;
|
|
300
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
301
|
+
arguments: z.ZodType<import("./primitives.ts").JsonObject, unknown, z.core.$ZodTypeInternals<import("./primitives.ts").JsonObject, unknown>>;
|
|
278
302
|
}, z.core.$strict>]>;
|
|
279
303
|
}, z.core.$strict>;
|
|
280
304
|
export type FixtureReset = z.infer<typeof FixtureReset>;
|