eval-quality 1.4.2 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -5
- package/corpus/dev/README.md +24 -13
- package/corpus/dev/compile-seal-example/brief.json +1 -1
- package/corpus/dev/compile-seal-example/contract.json +1 -1
- package/corpus/dev/contracts/absent-collection-locations.json +1 -1
- package/corpus/dev/contracts/absent-sibling-groups.json +1 -1
- package/corpus/dev/contracts/absent-success-indicator.json +1 -1
- package/corpus/dev/contracts/captured-read-back.json +1 -0
- package/corpus/dev/contracts/checklist-selection.json +1 -0
- package/corpus/dev/contracts/empty-channel-roles.json +1 -1
- package/corpus/dev/contracts/empty-collection-locations.json +1 -1
- package/corpus/dev/contracts/empty-request-shapes.json +1 -1
- package/corpus/dev/contracts/empty-sibling-groups.json +1 -1
- package/corpus/dev/contracts/fragment-selection.json +1 -1
- package/corpus/dev/contracts/no-collection-quantifier.json +1 -1
- package/corpus/dev/contracts/no-operation-inventory.json +1 -1
- package/corpus/dev/contracts/no-read-back-relation.json +1 -1
- package/corpus/dev/contracts/no-state-change-marker.json +1 -1
- package/corpus/dev/contracts/no-type-violating-step.json +1 -1
- package/corpus/dev/contracts/notes-tool-server.json +1 -0
- package/corpus/dev/contracts/per-key-split-oracles.json +1 -1
- package/corpus/dev/contracts/review-corpus.json +1 -1
- package/corpus/dev/contracts/satisfied-declarations.json +1 -1
- package/corpus/dev/contracts/single-required-response-key.json +1 -1
- package/corpus/dev/contracts/split-indicator-oracle.json +1 -1
- package/corpus/dev/contracts/unaddressed-parameter-sibling.json +1 -1
- package/corpus/dev/contracts/unnamed-reference-set.json +1 -1
- package/corpus/dev/contracts/wrong-cardinality-form.json +1 -1
- package/corpus/dev/index.json +1 -1
- package/dist/adapters/command-line-adapter.js +44 -6
- package/dist/adapters/index.d.ts +2 -0
- package/dist/adapters/index.js +1 -0
- package/dist/adapters/mcp-adapter.d.ts +32 -0
- package/dist/adapters/mcp-adapter.js +357 -0
- package/dist/adapters/mcp-target-policy.d.ts +39 -0
- package/dist/adapters/mcp-target-policy.js +30 -0
- package/dist/cli/render.d.ts +8 -3
- package/dist/cli/render.js +13 -6
- package/dist/cli/run.js +2 -1
- package/dist/core/compile/bindings.d.ts +8 -18
- package/dist/core/compile/bindings.js +13 -10
- package/dist/core/compile/interface-inventory.d.ts +81 -3
- package/dist/core/compile/interface-inventory.js +134 -23
- package/dist/core/compile/reachability.d.ts +73 -1
- package/dist/core/compile/reachability.js +134 -19
- package/dist/core/compile/sensitivity-witness.d.ts +22 -10
- package/dist/core/compile/sensitivity-witness.js +101 -13
- package/dist/core/coverage/operations.d.ts +1 -1
- package/dist/core/coverage/operations.js +1 -1
- package/dist/core/coverage/relevance.d.ts +3 -3
- package/dist/core/coverage/relevance.js +3 -3
- package/dist/core/declared-inputs.d.ts +21 -9
- package/dist/core/declared-inputs.js +51 -16
- package/dist/core/evaluate/evidence-resolution.d.ts +8 -7
- package/dist/core/evaluate/evidence-resolution.js +25 -18
- package/dist/core/preflight/plan.js +28 -6
- package/dist/core/preflight/projection.d.ts +10 -1
- package/dist/core/preflight/projection.js +9 -5
- package/dist/core/preflight/reduce.js +6 -2
- package/dist/core/preflight/witness-evidence.js +31 -9
- package/dist/core/schemas/artifact.d.ts +145 -49
- package/dist/core/schemas/defect-signature.d.ts +280 -23
- package/dist/core/schemas/defect-signature.js +65 -37
- package/dist/core/schemas/eval-contract.d.ts +33 -48
- package/dist/core/schemas/eval-contract.js +3 -3
- package/dist/core/schemas/interface.d.ts +193 -55
- package/dist/core/schemas/interface.js +82 -15
- package/dist/core/schemas/plan.d.ts +45 -1
- package/dist/core/schemas/plan.js +13 -2
- package/dist/core/schemas/pointer.d.ts +23 -9
- package/dist/core/schemas/pointer.js +25 -11
- package/dist/core/schemas/port-messages.d.ts +81 -0
- package/dist/core/schemas/port-messages.js +50 -3
- package/dist/core/schemas/primitives.d.ts +18 -0
- package/dist/core/schemas/primitives.js +29 -0
- package/dist/core/schemas/probe-policy.d.ts +41 -0
- package/dist/core/schemas/probe-policy.js +61 -1
- package/dist/core/schemas/probe.d.ts +115 -2
- package/dist/core/schemas/probe.js +2 -2
- package/dist/core/schemas/sealed-run-record.d.ts +11 -5
- package/dist/core/schemas/sealed-run-record.js +21 -9
- package/dist/core/schemas/sensitivity-witness.d.ts +31 -7
- package/dist/core/schemas/sensitivity-witness.js +32 -9
- package/dist/core/score/bindings.d.ts +1 -1
- package/dist/core/score/bindings.js +4 -4
- package/dist/core/score/qualification.d.ts +7 -5
- package/dist/core/score/qualification.js +92 -23
- package/dist/core/score/score.js +1 -1
- package/dist/core/seal/derived-reference.js +20 -17
- package/dist/core/seal/plan-index.d.ts +18 -9
- package/dist/core/seal/plan-index.js +79 -37
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/ports/environment-probe-port.d.ts +50 -17
- package/dist/ports/environment-probe-port.js +26 -17
- package/dist/testing/conformance.d.ts +3 -2
- package/dist/testing/conformance.js +2 -1
- package/dist/testing/index.d.ts +3 -3
- package/dist/testing/index.js +1 -1
- package/dist/testing/probe-conformance.d.ts +90 -13
- package/dist/testing/probe-conformance.js +375 -160
- package/package.json +4 -2
- package/schemas/eval-contract.schema.json +571 -18
- package/schemas/probe.schema.json +152 -12
- package/schemas/rubric.schema.json +1 -1
- package/schemas/sealed-run-record.schema.json +24 -17
|
@@ -403,8 +403,27 @@
|
|
|
403
403
|
"stdin"
|
|
404
404
|
],
|
|
405
405
|
"additionalProperties": false
|
|
406
|
+
},
|
|
407
|
+
{
|
|
408
|
+
"type": "object",
|
|
409
|
+
"properties": {
|
|
410
|
+
"arguments": {
|
|
411
|
+
"type": "object",
|
|
412
|
+
"propertyNames": {
|
|
413
|
+
"type": "string"
|
|
414
|
+
},
|
|
415
|
+
"additionalProperties": {
|
|
416
|
+
"$ref": "#/$defs/JsonValue"
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
},
|
|
420
|
+
"required": [
|
|
421
|
+
"arguments"
|
|
422
|
+
],
|
|
423
|
+
"additionalProperties": false
|
|
406
424
|
}
|
|
407
|
-
]
|
|
425
|
+
],
|
|
426
|
+
"description": "The probe's `schemaVersion` 3 -> 4 BREAKING bump under AD-11, whose rule is that removing or retyping is breaking. The union gained a third leg shape, one key over a tool call's arguments, so a witness leg against a tool call is expressible. The matching defect signature is a separate shape and version 5 is where it landed, so a version-4 probe declares a witness leg against a tool call and no signature for the defect that leg exercises. Against version 4 every version-3 probe's own bytes still parse, since the widening adds a branch and narrows none; version 5 narrows two shapes and is where a version-3 or version-4 probe stops parsing."
|
|
408
427
|
},
|
|
409
428
|
"relation": {
|
|
410
429
|
"$ref": "#/$defs/Expression"
|
|
@@ -859,8 +878,27 @@
|
|
|
859
878
|
"stdin"
|
|
860
879
|
],
|
|
861
880
|
"additionalProperties": false
|
|
881
|
+
},
|
|
882
|
+
{
|
|
883
|
+
"type": "object",
|
|
884
|
+
"properties": {
|
|
885
|
+
"arguments": {
|
|
886
|
+
"type": "object",
|
|
887
|
+
"propertyNames": {
|
|
888
|
+
"type": "string"
|
|
889
|
+
},
|
|
890
|
+
"additionalProperties": {
|
|
891
|
+
"$ref": "#/$defs/JsonValue"
|
|
892
|
+
}
|
|
893
|
+
}
|
|
894
|
+
},
|
|
895
|
+
"required": [
|
|
896
|
+
"arguments"
|
|
897
|
+
],
|
|
898
|
+
"additionalProperties": false
|
|
862
899
|
}
|
|
863
|
-
]
|
|
900
|
+
],
|
|
901
|
+
"description": "The probe's `schemaVersion` 3 -> 4 BREAKING bump under AD-11, whose rule is that removing or retyping is breaking. The union gained a third leg shape, one key over a tool call's arguments, so a witness leg against a tool call is expressible. The matching defect signature is a separate shape and version 5 is where it landed, so a version-4 probe declares a witness leg against a tool call and no signature for the defect that leg exercises. Against version 4 every version-3 probe's own bytes still parse, since the widening adds a branch and narrows none; version 5 narrows two shapes and is where a version-3 or version-4 probe stops parsing."
|
|
864
902
|
},
|
|
865
903
|
"relation": {
|
|
866
904
|
"$ref": "#/$defs/Expression"
|
|
@@ -906,8 +944,7 @@
|
|
|
906
944
|
"type": "string",
|
|
907
945
|
"enum": [
|
|
908
946
|
"api",
|
|
909
|
-
"web"
|
|
910
|
-
"mcp"
|
|
947
|
+
"web"
|
|
911
948
|
]
|
|
912
949
|
},
|
|
913
950
|
"method": {
|
|
@@ -963,6 +1000,9 @@
|
|
|
963
1000
|
},
|
|
964
1001
|
"stdin": {
|
|
965
1002
|
"$ref": "#/$defs/ProbeInputBindingChannel"
|
|
1003
|
+
},
|
|
1004
|
+
"arguments": {
|
|
1005
|
+
"$ref": "#/$defs/ProbeInputBindingChannel"
|
|
966
1006
|
}
|
|
967
1007
|
},
|
|
968
1008
|
"required": [
|
|
@@ -973,9 +1013,11 @@
|
|
|
973
1013
|
"argument",
|
|
974
1014
|
"option",
|
|
975
1015
|
"environment",
|
|
976
|
-
"stdin"
|
|
1016
|
+
"stdin",
|
|
1017
|
+
"arguments"
|
|
977
1018
|
],
|
|
978
|
-
"additionalProperties": false
|
|
1019
|
+
"additionalProperties": false,
|
|
1020
|
+
"description": "The probe's `schemaVersion` 4 -> 5 BREAKING bump under AD-11, whose rule is that removing or retyping is breaking. Two retypings land under this one stamp and both are named here. This selector gained a ninth channel, `arguments`, so a signature against a tool call can filter on what the call supplied; the sealed run record's `ObservedCallInputs` gained the same key under its own breaking bump, since a selector naming a channel no observation records would filter against nothing. And `DefectSignature` gained an `McpDefectSignature` branch declaring a published tool name, while `ApiDefectSignature.interfaceKind` narrowed to `api` and `web`, so a version-4 probe carrying `interfaceKind: \"mcp\"` beside a method and a path template stops parsing. Version 4 took the witness leg shape and this stamp takes both changes above, so each retype stays independently releasable."
|
|
979
1021
|
}
|
|
980
1022
|
},
|
|
981
1023
|
"required": [
|
|
@@ -1071,6 +1113,9 @@
|
|
|
1071
1113
|
},
|
|
1072
1114
|
"stdin": {
|
|
1073
1115
|
"$ref": "#/$defs/ProbeInputBindingChannel"
|
|
1116
|
+
},
|
|
1117
|
+
"arguments": {
|
|
1118
|
+
"$ref": "#/$defs/ProbeInputBindingChannel"
|
|
1074
1119
|
}
|
|
1075
1120
|
},
|
|
1076
1121
|
"required": [
|
|
@@ -1081,9 +1126,11 @@
|
|
|
1081
1126
|
"argument",
|
|
1082
1127
|
"option",
|
|
1083
1128
|
"environment",
|
|
1084
|
-
"stdin"
|
|
1129
|
+
"stdin",
|
|
1130
|
+
"arguments"
|
|
1085
1131
|
],
|
|
1086
|
-
"additionalProperties": false
|
|
1132
|
+
"additionalProperties": false,
|
|
1133
|
+
"description": "The probe's `schemaVersion` 4 -> 5 BREAKING bump under AD-11, whose rule is that removing or retyping is breaking. Two retypings land under this one stamp and both are named here. This selector gained a ninth channel, `arguments`, so a signature against a tool call can filter on what the call supplied; the sealed run record's `ObservedCallInputs` gained the same key under its own breaking bump, since a selector naming a channel no observation records would filter against nothing. And `DefectSignature` gained an `McpDefectSignature` branch declaring a published tool name, while `ApiDefectSignature.interfaceKind` narrowed to `api` and `web`, so a version-4 probe carrying `interfaceKind: \"mcp\"` beside a method and a path template stops parsing. Version 4 took the witness leg shape and this stamp takes both changes above, so each retype stays independently releasable."
|
|
1087
1134
|
}
|
|
1088
1135
|
},
|
|
1089
1136
|
"required": [
|
|
@@ -1110,6 +1157,99 @@
|
|
|
1110
1157
|
"condition"
|
|
1111
1158
|
],
|
|
1112
1159
|
"additionalProperties": false
|
|
1160
|
+
},
|
|
1161
|
+
{
|
|
1162
|
+
"type": "object",
|
|
1163
|
+
"properties": {
|
|
1164
|
+
"interfaceKind": {
|
|
1165
|
+
"type": "string",
|
|
1166
|
+
"const": "mcp"
|
|
1167
|
+
},
|
|
1168
|
+
"toolName": {
|
|
1169
|
+
"type": "string",
|
|
1170
|
+
"pattern": "^[A-Za-z0-9_-]+$",
|
|
1171
|
+
"description": "The name the MCP server publishes for one tool, in the server's own spelling. AD-35: a logical identifier, never a URL, host, or port. The charset admits no \"/\", \":\", or \".\", so an address is unrepresentable rather than refused by a later check, and it excludes \"~\" so a tool name can be embedded in a pointer without escaping."
|
|
1172
|
+
},
|
|
1173
|
+
"observableChannel": {
|
|
1174
|
+
"description": "AD-26's channel the seeded defect manifests in. The qualification gate reads it: a condition passes only if its pointers name this channel, or name two channels with at least one response-side member. That rule exists to reject a condition collapsing to \"the evidence contains the string I sent\", which is satisfied by a finding that merely echoes its own input. It must also be a channel the declared kind produces: an api interface writes nothing to standard output and a command returns no HTTP status.",
|
|
1175
|
+
"$ref": "#/$defs/EvidenceChannel"
|
|
1176
|
+
},
|
|
1177
|
+
"condition": {
|
|
1178
|
+
"type": "object",
|
|
1179
|
+
"properties": {
|
|
1180
|
+
"selector": {
|
|
1181
|
+
"type": "object",
|
|
1182
|
+
"properties": {
|
|
1183
|
+
"inputBinding": {
|
|
1184
|
+
"type": "object",
|
|
1185
|
+
"properties": {
|
|
1186
|
+
"path": {
|
|
1187
|
+
"$ref": "#/$defs/ProbeInputBindingChannel"
|
|
1188
|
+
},
|
|
1189
|
+
"query": {
|
|
1190
|
+
"$ref": "#/$defs/ProbeInputBindingChannel"
|
|
1191
|
+
},
|
|
1192
|
+
"header": {
|
|
1193
|
+
"$ref": "#/$defs/ProbeInputBindingChannel"
|
|
1194
|
+
},
|
|
1195
|
+
"body": {
|
|
1196
|
+
"$ref": "#/$defs/ProbeInputBindingChannel"
|
|
1197
|
+
},
|
|
1198
|
+
"argument": {
|
|
1199
|
+
"$ref": "#/$defs/ProbeInputBindingChannel"
|
|
1200
|
+
},
|
|
1201
|
+
"option": {
|
|
1202
|
+
"$ref": "#/$defs/ProbeInputBindingChannel"
|
|
1203
|
+
},
|
|
1204
|
+
"environment": {
|
|
1205
|
+
"$ref": "#/$defs/ProbeInputBindingChannel"
|
|
1206
|
+
},
|
|
1207
|
+
"stdin": {
|
|
1208
|
+
"$ref": "#/$defs/ProbeInputBindingChannel"
|
|
1209
|
+
},
|
|
1210
|
+
"arguments": {
|
|
1211
|
+
"$ref": "#/$defs/ProbeInputBindingChannel"
|
|
1212
|
+
}
|
|
1213
|
+
},
|
|
1214
|
+
"required": [
|
|
1215
|
+
"path",
|
|
1216
|
+
"query",
|
|
1217
|
+
"header",
|
|
1218
|
+
"body",
|
|
1219
|
+
"argument",
|
|
1220
|
+
"option",
|
|
1221
|
+
"environment",
|
|
1222
|
+
"stdin",
|
|
1223
|
+
"arguments"
|
|
1224
|
+
],
|
|
1225
|
+
"additionalProperties": false,
|
|
1226
|
+
"description": "The probe's `schemaVersion` 4 -> 5 BREAKING bump under AD-11, whose rule is that removing or retyping is breaking. Two retypings land under this one stamp and both are named here. This selector gained a ninth channel, `arguments`, so a signature against a tool call can filter on what the call supplied; the sealed run record's `ObservedCallInputs` gained the same key under its own breaking bump, since a selector naming a channel no observation records would filter against nothing. And `DefectSignature` gained an `McpDefectSignature` branch declaring a published tool name, while `ApiDefectSignature.interfaceKind` narrowed to `api` and `web`, so a version-4 probe carrying `interfaceKind: \"mcp\"` beside a method and a path template stops parsing. Version 4 took the witness leg shape and this stamp takes both changes above, so each retype stays independently releasable."
|
|
1227
|
+
}
|
|
1228
|
+
},
|
|
1229
|
+
"required": [
|
|
1230
|
+
"inputBinding"
|
|
1231
|
+
],
|
|
1232
|
+
"additionalProperties": false
|
|
1233
|
+
},
|
|
1234
|
+
"predicate": {
|
|
1235
|
+
"description": "An AD-4 expression over AD-26 response channels, every pointer rooted at the reserved step identifier `observed`. Legality is the corpus qualification gate's: an unchecked predicate admits `existence` over a literal, which is true of every observation and makes the catch rate 1.00 by construction.",
|
|
1236
|
+
"$ref": "#/$defs/Expression"
|
|
1237
|
+
}
|
|
1238
|
+
},
|
|
1239
|
+
"required": [
|
|
1240
|
+
"selector",
|
|
1241
|
+
"predicate"
|
|
1242
|
+
],
|
|
1243
|
+
"additionalProperties": false
|
|
1244
|
+
}
|
|
1245
|
+
},
|
|
1246
|
+
"required": [
|
|
1247
|
+
"interfaceKind",
|
|
1248
|
+
"toolName",
|
|
1249
|
+
"observableChannel",
|
|
1250
|
+
"condition"
|
|
1251
|
+
],
|
|
1252
|
+
"additionalProperties": false
|
|
1113
1253
|
}
|
|
1114
1254
|
]
|
|
1115
1255
|
},
|
|
@@ -1141,7 +1281,7 @@
|
|
|
1141
1281
|
"description": "A probe that is not a known-clean control. The one branch AD-40 gives a defect signature."
|
|
1142
1282
|
}
|
|
1143
1283
|
],
|
|
1144
|
-
"description": "One corpus probe. Succeeds the prior-art `h0-ground-truth` schema per AD-24, carrying its system identifier, implementation digest, `expectedClean` flag, seeded defects, and rationale, and adding AD-9's probe class and AD-9's per-probe artifact and commit digests. Divergences: `implementationSha` becomes `implementationDigest`, `taskId` does not survive because the probe pins what it describes by digest, and `expectedGate` does not survive because AD-40 makes detection a signature match rather than a verdict comparison and AD-7 keeps comparisons inside the dominance vector, so carrying an expected gate would invite a comparison the architecture forbids. Two constructions landed here together under one BREAKING `schemaVersion` bump: AD-9's per-class QUALIFICATION record, as a five-route tagged union required on every branch, and AD-40's machine-readable DEFECT SIGNATURE on the `expectedClean: false` branch, carrying the interface kind, the home operation
|
|
1284
|
+
"description": "One corpus probe. Succeeds the prior-art `h0-ground-truth` schema per AD-24, carrying its system identifier, implementation digest, `expectedClean` flag, seeded defects, and rationale, and adding AD-9's probe class and AD-9's per-probe artifact and commit digests. Divergences: `implementationSha` becomes `implementationDigest`, `taskId` does not survive because the probe pins what it describes by digest, and `expectedGate` does not survive because AD-40 makes detection a signature match rather than a verdict comparison and AD-7 keeps comparisons inside the dominance vector, so carrying an expected gate would invite a comparison the architecture forbids. Two constructions landed here together under one BREAKING `schemaVersion` bump: AD-9's per-class QUALIFICATION record, as a five-route tagged union required on every branch, and AD-40's machine-readable DEFECT SIGNATURE on the `expectedClean: false` branch, carrying the interface kind, the home operation by its transport identity, the observable channel, and the discriminating condition. Both are required rather than optional, which is what makes the bump breaking under AD-11: every corpus written against version 1 fails to parse. Version 3 opened the signature to a second interface kind: it is a union on `interfaceKind`, where the `cli` branch declares a logical invocation in place of a method and a path template, and the selector's input binding carries the four command channels beside the four transport ones. Version 4 is a BREAKING bump on the witness side: a manifestation witness's `inputs` gained a third leg shape over a tool call's arguments, so a witness leg against a tool call is expressible; every version-3 probe's own bytes still parse against it. Version 5 opened the signature to a third kind and closed both gaps version 4 left. `DefectSignature` gains a tool-call branch declaring the published tool name, which is the transport identity AD-40 resolves an mcp signature against, and the api-shaped branch narrows to `api` and `web`, so a signature carrying `mcp` beside a method and a path template stops parsing. The selector's input binding gains a ninth `arguments` channel on the same required-and-nullable terms as the other eight, so a version-3 or version-4 probe declaring eight channels stops parsing too. The signature is therefore three branch shapes rather than one, each declaring its own kind's transport identity: a method and a path template, a logical invocation, or a published tool name. What the schema still does not decide is stated rather than hidden. AD-9's \"an unqualified probe cannot enter a sealed set\" is a corpus-construction invariant enforced by the qualification gate in `core/score/qualification.ts`, not by this schema: all eight class-and-`expectedClean` pairings parse, a route incompatible with the pair parses, and a signature-less non-canary parses, each so the gate can return a reason code carrying an artifact path instead of an anonymous parse failure. The constraint ledger carries both gaps.",
|
|
1145
1285
|
"$defs": {
|
|
1146
1286
|
"ArtifactReference": {
|
|
1147
1287
|
"oneOf": [
|
|
@@ -1278,7 +1418,7 @@
|
|
|
1278
1418
|
"body"
|
|
1279
1419
|
],
|
|
1280
1420
|
"additionalProperties": false,
|
|
1281
|
-
"description": "One probe leg's supplied inputs, keyed by AD-19 transport channel, in the spelling the environment-probe port accepts.
|
|
1421
|
+
"description": "One probe leg's supplied inputs, keyed by AD-19 transport channel, in the spelling the environment-probe port accepts. The transport branch of the union every leg shape takes: a sensitivity leg, a manifestation witness, and the fixture reset."
|
|
1282
1422
|
},
|
|
1283
1423
|
"JsonValue": {
|
|
1284
1424
|
"description": "AD-36 value domain. Every number is a finite IEEE 754 double-precision value, and every integer lies in the safe-integer range; a larger integer, and any value needing exact decimal semantics such as money, is carried as a string in its own declared format. JSON Schema cannot express finiteness, so this restriction is stated rather than encoded: producers are told the rule here instead of discovering it through a digest mismatch, and the canonical scanner rejects a violating value before any parse.",
|
|
@@ -1863,8 +2003,8 @@
|
|
|
1863
2003
|
"anyOf": [
|
|
1864
2004
|
{
|
|
1865
2005
|
"type": "string",
|
|
1866
|
-
"pattern": "^\\/interactions\\/[a-z0-9]+(?:-[a-z0-9]+)*\\/(?:(?:response-body|response-headers|stdout|stderr)(?:\\/(?:[^/~]|~[01])*)*|(?:response-status|exit-code)|call-inputs\\/(?:path|query|header|body|argument|option|environment|stdin)(?:\\/(?:[^/~]|~[01])*)*|artifact\\/[a-z0-9]+(?:-[a-z0-9]+)*(?:\\/(?:[^/~]|~[01])*)*)$",
|
|
1867
|
-
"description": "AD-26 interaction-rooted pointer: \"/interactions/{stepId}/\" followed by one channel of the closed vocabulary. `call-inputs` takes one input channel as its next segment, one of the four transport channels
|
|
2006
|
+
"pattern": "^\\/interactions\\/[a-z0-9]+(?:-[a-z0-9]+)*\\/(?:(?:response-body|response-headers|stdout|stderr)(?:\\/(?:[^/~]|~[01])*)*|(?:response-status|exit-code)|call-inputs\\/(?:path|query|header|body|argument|option|environment|stdin|arguments)(?:\\/(?:[^/~]|~[01])*)*|artifact\\/[a-z0-9]+(?:-[a-z0-9]+)*(?:\\/(?:[^/~]|~[01])*)*)$",
|
|
2007
|
+
"description": "AD-26 interaction-rooted pointer: \"/interactions/{stepId}/\" followed by one channel of the closed vocabulary. `call-inputs` takes one input channel as its next segment, one of the four transport channels, one of the four command channels, or the `arguments` channel a tool call accepts; `artifact` takes the identifier of a file the operation declares it writes; `response-status` and `exit-code` take no tail. Syntax only: whether the step exists and whether the evidence is reachable are compile-time checks, not schema checks."
|
|
1868
2008
|
},
|
|
1869
2009
|
{
|
|
1870
2010
|
"type": "string",
|
|
@@ -95,7 +95,7 @@
|
|
|
95
95
|
},
|
|
96
96
|
"evidence": {
|
|
97
97
|
"type": "string",
|
|
98
|
-
"pattern": "^\\/interactions\\/[a-z0-9]+(?:-[a-z0-9]+)*\\/(?:(?:response-body|response-headers|stdout|stderr)(?:\\/(?:[^/~]|~[01])*)*|(?:response-status|exit-code)|call-inputs\\/(?:path|query|header|body|argument|option|environment|stdin)(?:\\/(?:[^/~]|~[01])*)*|artifact\\/[a-z0-9]+(?:-[a-z0-9]+)*(?:\\/(?:[^/~]|~[01])*)*)$",
|
|
98
|
+
"pattern": "^\\/interactions\\/[a-z0-9]+(?:-[a-z0-9]+)*\\/(?:(?:response-body|response-headers|stdout|stderr)(?:\\/(?:[^/~]|~[01])*)*|(?:response-status|exit-code)|call-inputs\\/(?:path|query|header|body|argument|option|environment|stdin|arguments)(?:\\/(?:[^/~]|~[01])*)*|artifact\\/[a-z0-9]+(?:-[a-z0-9]+)*(?:\\/(?:[^/~]|~[01])*)*)$",
|
|
99
99
|
"description": "Where the criterion is answerable from. A pointer that resolves nowhere is `rubric-evidence-unreachable`, a compile-time code, and is deliberately not a schema rejection."
|
|
100
100
|
}
|
|
101
101
|
},
|
|
@@ -50,7 +50,7 @@
|
|
|
50
50
|
"type": "integer",
|
|
51
51
|
"minimum": 1,
|
|
52
52
|
"maximum": 9007199254740991,
|
|
53
|
-
"description": "Which trial this record is. AD-24 excludes the trial index from the Evaluator Configuration \"so trials pool into one scoring version\", which requires it somewhere else, and a Sealed Run Record is the only artifact carrying exactly one trial. One-based, matching the only instance that exists. AD-6's aggregate of trial count, invalidated attempts, and each attempt's reason is the Evidence Artifact's
|
|
53
|
+
"description": "Which trial this record is. AD-24 excludes the trial index from the Evaluator Configuration \"so trials pool into one scoring version\", which requires it somewhere else, and a Sealed Run Record is the only artifact carrying exactly one trial. One-based, matching the only instance that exists. AD-6's aggregate of trial count, invalidated attempts, and each attempt's reason is the Evidence Artifact's, computed by `score` over the trial set a caller assembles. The reducer keys an attempt by its position in that set, so this field is the caller's own bookkeeping and a label for a reader of one record."
|
|
54
54
|
},
|
|
55
55
|
"contractDigest": {
|
|
56
56
|
"type": "string",
|
|
@@ -294,6 +294,23 @@
|
|
|
294
294
|
"type": "null"
|
|
295
295
|
}
|
|
296
296
|
]
|
|
297
|
+
},
|
|
298
|
+
"arguments": {
|
|
299
|
+
"anyOf": [
|
|
300
|
+
{
|
|
301
|
+
"type": "object",
|
|
302
|
+
"propertyNames": {
|
|
303
|
+
"type": "string"
|
|
304
|
+
},
|
|
305
|
+
"additionalProperties": {
|
|
306
|
+
"$ref": "#/$defs/JsonValue"
|
|
307
|
+
}
|
|
308
|
+
},
|
|
309
|
+
{
|
|
310
|
+
"type": "null"
|
|
311
|
+
}
|
|
312
|
+
],
|
|
313
|
+
"description": "AD-26's `arguments` channel, the one channel a tool call accepts input on. Its arrival retypes this shape from eight keys to nine, which AD-11 calls breaking and which moves the Sealed Run Record's `schemaVersion` from 4 to 5: a version-4 record declares no `arguments` key and fails to parse against version 5. A name-to-value map on `responseHeaders`' terms, since a pointer descends INTO the channel to address one argument. `null` on an observation that exercised no tool call, which is what every channel here already spells for a channel nothing was sent on."
|
|
297
314
|
}
|
|
298
315
|
},
|
|
299
316
|
"required": [
|
|
@@ -304,7 +321,8 @@
|
|
|
304
321
|
"argument",
|
|
305
322
|
"option",
|
|
306
323
|
"environment",
|
|
307
|
-
"stdin"
|
|
324
|
+
"stdin",
|
|
325
|
+
"arguments"
|
|
308
326
|
],
|
|
309
327
|
"additionalProperties": false
|
|
310
328
|
},
|
|
@@ -317,7 +335,7 @@
|
|
|
317
335
|
"type": "null"
|
|
318
336
|
}
|
|
319
337
|
],
|
|
320
|
-
"description": "AD-26's `response-body` channel. The null branch is redundant against the value container, which already admits `null`; it is kept so
|
|
338
|
+
"description": "AD-26's `response-body` channel. The null branch is redundant against the value container, which already admits `null`; it is kept so every observation field reads the same way, and it means \"no body observed\" and \"a body that was JSON null\" are indistinguishable here, which is an accepted cost of one uniform spelling."
|
|
321
339
|
},
|
|
322
340
|
"responseHeaders": {
|
|
323
341
|
"anyOf": [
|
|
@@ -347,7 +365,7 @@
|
|
|
347
365
|
"type": "null"
|
|
348
366
|
}
|
|
349
367
|
],
|
|
350
|
-
"description": "Deliberately not bounded to a protocol range. A negative status is meaningless and excluded, but the upper end is left open: AD-19 declares four interface kinds and
|
|
368
|
+
"description": "Deliberately not bounded to a protocol range. A negative status is meaningless and excluded, but the upper end is left open: AD-19 declares four interface kinds and not all of them speak HTTP, so bounding this to HTTP would encode a protocol assumption the artifact outlives. `null` where the channel does not apply."
|
|
351
369
|
},
|
|
352
370
|
"stdout": {
|
|
353
371
|
"oneOf": [
|
|
@@ -663,16 +681,6 @@
|
|
|
663
681
|
"reportedIncomplete"
|
|
664
682
|
],
|
|
665
683
|
"additionalProperties": false
|
|
666
|
-
},
|
|
667
|
-
"invalidReason": {
|
|
668
|
-
"anyOf": [
|
|
669
|
-
{
|
|
670
|
-
"type": "string"
|
|
671
|
-
},
|
|
672
|
-
{
|
|
673
|
-
"type": "null"
|
|
674
|
-
}
|
|
675
|
-
]
|
|
676
684
|
}
|
|
677
685
|
},
|
|
678
686
|
"required": [
|
|
@@ -694,11 +702,10 @@
|
|
|
694
702
|
"actionsArtifact",
|
|
695
703
|
"isolationManifestArtifact",
|
|
696
704
|
"resourceUse",
|
|
697
|
-
"evidenceDisclosure"
|
|
698
|
-
"invalidReason"
|
|
705
|
+
"evidenceDisclosure"
|
|
699
706
|
],
|
|
700
707
|
"additionalProperties": false,
|
|
701
|
-
"description": "One sealed evaluator trial, as the caller presents it. Succeeds the prior-art `h0-run-result` schema per AD-24, keeping its run identifier, condition arm, findings, action-log reference, resource use,
|
|
708
|
+
"description": "One sealed evaluator trial, as the caller presents it. Succeeds the prior-art `h0-run-result` schema per AD-24, keeping its run identifier, condition arm, findings, action-log reference, resource use, evaluator recommendation as a closed enum, and per-finding confidence on a declared scale. Divergences: `condition` is demoted to the opaque `conditionArm`, `verdict` becomes `evaluatorRecommendation` without `NOT_APPLICABLE`, money is a decimal string, and `taskId`, `note`, per-finding `actionIds`, and the prior art's `invalidReason` do not survive: the contract is pinned by `contractDigest`, an unstructured orchestrator annotation is the free-prose channel the Conventions close everywhere else, and two citation vocabularies on one finding is the ambiguity ADR-009 removed. The run MODE landed here as a required field under a BREAKING `schemaVersion` bump, which is where AD-21's \"fixed before ingest\" puts it; owed item 4 is now closed: mode enters AD-11's identity inputs as `ScoringVersionInputs`'s sixth field, and `core/score/ladder.ts` carries `ProductionAssessment`/`ContractAssessment` as the two assessment input types with their own total ladders. Observation ORDERING landed here too, under its own BREAKING `schemaVersion` bump: `sequence` is required and unique per record, closing owed item 2's ADR-006 gap, since array position was never a legal ordering. Version 4 opened the record to a system under test that runs behind a command: `callInputs` carries the four command channels beside the four transport ones, `stdout` and `stderr` are tagged rather than bare strings so a nominated output channel can be descended into, and `artifacts` records the files the run wrote, keyed by the identifier the operation declares. Version 5 opened it to a tool call: `callInputs` carries a ninth `arguments` channel beside those eight, so what a tool call supplied has somewhere to live and a defect signature's selector filtering on that channel has something to read. A version-4 record declares eight call-input channels and fails to parse against version 5. Version 6 drops `invalidReason`, the prior art's run-level invalidation reason: nothing in this package ever read it, so a caller attesting that a run was invalid was ignored by every stage while the field looked like a supported channel. The attestation that works is `IsolationManifest.violation`, which `core/ingest` raises as an `isolation-manifest-violation` condition and which reaches the verdict basis, and AD-6's invalidating outcome states carry a failure the run itself produced. A version-5 record carrying `invalidReason` fails to parse against version 6.",
|
|
702
709
|
"$defs": {
|
|
703
710
|
"EvaluatorRecommendation": {
|
|
704
711
|
"type": "string",
|