@mappies/miniwat-eval 0.0.0-stage → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,74 @@
1
1
  {
2
- "name": "@mappies/miniwat-eval",
3
- "version": "0.0.0-stage",
4
- "stub": true,
5
- "description": "Temporary package placeholder for staged publishing"
6
- }
2
+ "name": "@mappies/miniwat-eval",
3
+ "version": "1.0.0",
4
+ "description": "Black-box, YAML-driven evaluation framework for agents built on miniwat-agent",
5
+ "homepage": "https://gitlab.com/mappies/miniwat-eval#readme",
6
+ "bugs": {
7
+ "url": "https://gitlab.com/mappies/miniwat-eval/issues"
8
+ },
9
+ "repository": {
10
+ "type": "git",
11
+ "url": "git+ssh://git@gitlab.com/mappies/miniwat-eval.git"
12
+ },
13
+ "license": "PolyForm-Noncommercial-1.0.0",
14
+ "author": "Nithiwat Kampanya",
15
+ "type": "module",
16
+ "main": "./dist/index.cjs",
17
+ "module": "./dist/index.js",
18
+ "types": "./dist/index.d.ts",
19
+ "exports": {
20
+ ".": {
21
+ "import": {
22
+ "types": "./dist/index.d.ts",
23
+ "default": "./dist/index.js"
24
+ },
25
+ "require": {
26
+ "types": "./dist/index.d.cts",
27
+ "default": "./dist/index.cjs"
28
+ }
29
+ }
30
+ },
31
+ "bin": {
32
+ "miniwat-eval": "./dist/main.js"
33
+ },
34
+ "files": [
35
+ "dist",
36
+ "schemas"
37
+ ],
38
+ "engines": {
39
+ "node": ">=22"
40
+ },
41
+ "scripts": {
42
+ "build": "tsdown",
43
+ "typecheck": "tsc --noEmit -p . && tsc --noEmit -p spec",
44
+ "test": "vitest run",
45
+ "test-watch": "vitest"
46
+ },
47
+ "dependencies": {
48
+ "@mappies/miniwat-agent": "^2.1.2",
49
+ "@mappies/miniwat-agent-core": "^0.26.1",
50
+ "ajv": "^8.20.0",
51
+ "commander": "^15.1.0",
52
+ "configurapi-handler-ws": "^1.16.1",
53
+ "yaml": "^2.9.1"
54
+ },
55
+ "peerDependencies": {
56
+ "@mappies/miniwat-provider-bedrock": "^0.17.1",
57
+ "@mappies/miniwat-provider-openai": "^0.17.1"
58
+ },
59
+ "peerDependenciesMeta": {
60
+ "@mappies/miniwat-provider-bedrock": {
61
+ "optional": true
62
+ },
63
+ "@mappies/miniwat-provider-openai": {
64
+ "optional": true
65
+ }
66
+ },
67
+ "devDependencies": {
68
+ "@types/node": "^26.6.6",
69
+ "configurapi-runner-ws": "^1.21.0",
70
+ "tsdown": "^0.23.7",
71
+ "typescript": "^5.9.3",
72
+ "vitest": "^5.1.0"
73
+ }
74
+ }
@@ -0,0 +1,102 @@
1
+ {
2
+ "$schema": "http://json-schema.org/draft-07/schema#",
3
+ "title": "Miniwat Eval conversation",
4
+ "type": "object",
5
+ "additionalProperties": false,
6
+ "required": ["version", "name", "turns"],
7
+ "properties": {
8
+ "version": { "const": 1 },
9
+ "name": { "$ref": "#/definitions/text" },
10
+ "description": { "type": "string" },
11
+ "turns": {
12
+ "type": "array",
13
+ "minItems": 1,
14
+ "items": { "$ref": "#/definitions/turn" }
15
+ },
16
+ "expect": {
17
+ "type": "object",
18
+ "additionalProperties": false,
19
+ "required": ["conversation"],
20
+ "properties": {
21
+ "conversation": {
22
+ "type": "object",
23
+ "additionalProperties": false,
24
+ "required": ["criteria"],
25
+ "properties": {
26
+ "criteria": { "$ref": "#/definitions/text" }
27
+ }
28
+ }
29
+ }
30
+ }
31
+ },
32
+ "definitions": {
33
+ "text": { "type": "string", "pattern": "\\S" },
34
+ "textList": {
35
+ "type": "array",
36
+ "minItems": 1,
37
+ "items": { "$ref": "#/definitions/text" }
38
+ },
39
+ "turn": {
40
+ "type": "object",
41
+ "additionalProperties": false,
42
+ "required": ["input"],
43
+ "properties": {
44
+ "input": { "$ref": "#/definitions/text" },
45
+ "expect": { "$ref": "#/definitions/turnExpectation" }
46
+ }
47
+ },
48
+ "turnExpectation": {
49
+ "type": "object",
50
+ "additionalProperties": false,
51
+ "properties": {
52
+ "response": { "$ref": "#/definitions/response" },
53
+ "toolUsed": {
54
+ "type": "array",
55
+ "items": { "$ref": "#/definitions/toolUsed" }
56
+ },
57
+ "toolNotUsed": {
58
+ "type": "array",
59
+ "items": { "$ref": "#/definitions/text" }
60
+ },
61
+ "toolsInOrder": {
62
+ "type": "array",
63
+ "items": { "$ref": "#/definitions/toolInOrder" }
64
+ }
65
+ }
66
+ },
67
+ "response": {
68
+ "description": "A string is shorthand for criteria.",
69
+ "type": ["string", "object"],
70
+ "pattern": "\\S",
71
+ "additionalProperties": false,
72
+ "properties": {
73
+ "criteria": { "$ref": "#/definitions/text" },
74
+ "contains": { "$ref": "#/definitions/textList" },
75
+ "notContains": { "$ref": "#/definitions/textList" }
76
+ }
77
+ },
78
+ "toolUsed": {
79
+ "description": "A string is shorthand for a name-only entry.",
80
+ "type": ["string", "object"],
81
+ "pattern": "\\S",
82
+ "additionalProperties": false,
83
+ "required": ["name"],
84
+ "properties": {
85
+ "name": { "$ref": "#/definitions/text" },
86
+ "arguments": { "type": "object" },
87
+ "status": { "type": "boolean" }
88
+ }
89
+ },
90
+ "toolInOrder": {
91
+ "description": "A string is shorthand for a name-only entry.",
92
+ "type": ["string", "object"],
93
+ "pattern": "\\S",
94
+ "additionalProperties": false,
95
+ "required": ["name"],
96
+ "properties": {
97
+ "name": { "$ref": "#/definitions/text" },
98
+ "arguments": { "type": "object" }
99
+ }
100
+ }
101
+ }
102
+ }
@@ -0,0 +1,141 @@
1
+ {
2
+ "$schema": "http://json-schema.org/draft-07/schema#",
3
+ "title": "Miniwat Eval configuration",
4
+ "type": "object",
5
+ "additionalProperties": false,
6
+ "required": ["version", "websocket", "evaluator"],
7
+ "properties": {
8
+ "version": { "const": 1 },
9
+ "websocket": {
10
+ "type": "object",
11
+ "additionalProperties": false,
12
+ "required": ["endpoint", "protocol"],
13
+ "properties": {
14
+ "endpoint": { "type": "string", "pattern": "^wss?://\\S" },
15
+ "connection": {
16
+ "type": "object",
17
+ "additionalProperties": false,
18
+ "properties": {
19
+ "subprotocols": {
20
+ "type": "array",
21
+ "items": { "$ref": "#/definitions/text" }
22
+ }
23
+ }
24
+ },
25
+ "protocol": {
26
+ "type": "object",
27
+ "additionalProperties": false,
28
+ "required": ["request"],
29
+ "properties": {
30
+ "request": { "$ref": "#/definitions/request" },
31
+ "capabilities": {
32
+ "type": "object",
33
+ "additionalProperties": false,
34
+ "properties": {
35
+ "toolUses": { "type": "boolean" },
36
+ "toolResults": { "type": "boolean" },
37
+ "tokenUsage": { "type": "boolean" }
38
+ }
39
+ }
40
+ }
41
+ }
42
+ }
43
+ },
44
+ "execution": {
45
+ "type": "object",
46
+ "additionalProperties": false,
47
+ "properties": {
48
+ "scenarioConcurrency": { "$ref": "#/definitions/positiveInteger" },
49
+ "repeat": { "$ref": "#/definitions/positiveInteger" },
50
+ "minPass": { "$ref": "#/definitions/positiveInteger" },
51
+ "continueOnFailure": { "type": "boolean" },
52
+ "flakyPolicy": { "enum": ["warn", "fail"] },
53
+ "connectionRetries": { "$ref": "#/definitions/retries" },
54
+ "timeouts": {
55
+ "type": "object",
56
+ "additionalProperties": false,
57
+ "properties": {
58
+ "connection": { "$ref": "#/definitions/positiveInteger" },
59
+ "turn": { "$ref": "#/definitions/positiveInteger" },
60
+ "conversation": { "$ref": "#/definitions/positiveInteger" }
61
+ }
62
+ }
63
+ }
64
+ },
65
+ "evaluator": { "$ref": "#/definitions/evaluator" },
66
+ "report": {
67
+ "type": "object",
68
+ "additionalProperties": false,
69
+ "properties": {
70
+ "directory": { "$ref": "#/definitions/text" },
71
+ "formats": {
72
+ "type": "array",
73
+ "minItems": 1,
74
+ "uniqueItems": true,
75
+ "items": { "enum": ["console", "json"] }
76
+ }
77
+ }
78
+ }
79
+ },
80
+ "definitions": {
81
+ "text": { "type": "string", "pattern": "\\S" },
82
+ "positiveInteger": { "type": "integer", "minimum": 1 },
83
+ "retries": { "type": "integer", "minimum": 0 },
84
+ "threshold": { "type": "number", "minimum": 0, "maximum": 1 },
85
+ "request": {
86
+ "type": "object",
87
+ "additionalProperties": false,
88
+ "required": ["value", "inputPath"],
89
+ "properties": {
90
+ "format": { "enum": ["json"] },
91
+ "value": { "type": ["object", "array"] },
92
+ "inputPath": { "type": "string", "pattern": "^(/([^~/]|~[01])*)+$" }
93
+ }
94
+ },
95
+ "evaluator": {
96
+ "type": "object",
97
+ "additionalProperties": false,
98
+ "required": ["provider"],
99
+ "properties": {
100
+ "provider": { "enum": ["openai", "bedrock"] },
101
+ "model": { "$ref": "#/definitions/text" },
102
+ "apiKey": { "$ref": "#/definitions/text" },
103
+ "region": { "$ref": "#/definitions/text" },
104
+ "concurrency": { "$ref": "#/definitions/positiveInteger" },
105
+ "timeout": { "$ref": "#/definitions/positiveInteger" },
106
+ "retries": { "$ref": "#/definitions/retries" },
107
+ "maxToolResultChars": { "type": "integer", "minimum": 0 },
108
+ "thresholds": {
109
+ "type": "object",
110
+ "additionalProperties": false,
111
+ "properties": {
112
+ "response": { "$ref": "#/definitions/threshold" },
113
+ "conversation": { "$ref": "#/definitions/threshold" }
114
+ }
115
+ }
116
+ },
117
+ "allOf": [
118
+ {
119
+ "if": {
120
+ "required": ["provider"],
121
+ "properties": { "provider": { "const": "openai" } }
122
+ },
123
+ "then": {
124
+ "required": ["model", "apiKey"],
125
+ "properties": { "model": true, "apiKey": true, "region": false }
126
+ }
127
+ },
128
+ {
129
+ "if": {
130
+ "required": ["provider"],
131
+ "properties": { "provider": { "const": "bedrock" } }
132
+ },
133
+ "then": {
134
+ "required": ["model", "region"],
135
+ "properties": { "model": true, "region": true, "apiKey": false }
136
+ }
137
+ }
138
+ ]
139
+ }
140
+ }
141
+ }
@@ -0,0 +1,326 @@
1
+ {
2
+ "$schema": "http://json-schema.org/draft-07/schema#",
3
+ "title": "Miniwat Eval JSON report",
4
+ "description": "Written by miniwat-eval run --report json. Absent values are left out, never null.",
5
+ "type": "object",
6
+ "additionalProperties": false,
7
+ "required": ["version", "status", "startedAt", "durationMs", "exitCode", "unsuccessfulCount", "settings", "configuration", "summary", "scenarios"],
8
+ "properties": {
9
+ "version": { "const": 1 },
10
+ "status": { "enum": ["COMPLETED", "INTERRUPTED"] },
11
+ "startedAt": { "type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$" },
12
+ "durationMs": { "$ref": "#/definitions/count" },
13
+ "exitCode": { "type": "integer", "minimum": 0, "maximum": 255 },
14
+ "unsuccessfulCount": { "$ref": "#/definitions/count" },
15
+ "settings": {
16
+ "type": "object",
17
+ "additionalProperties": false,
18
+ "required": ["repeat", "minPass", "quick", "scenarioConcurrency", "flakyPolicy", "continueOnFailure"],
19
+ "properties": {
20
+ "repeat": { "$ref": "#/definitions/positive" },
21
+ "minPass": { "$ref": "#/definitions/positive" },
22
+ "quick": { "type": "boolean" },
23
+ "scenarioConcurrency": { "$ref": "#/definitions/positive" },
24
+ "flakyPolicy": { "enum": ["warn", "fail"] },
25
+ "continueOnFailure": { "type": "boolean" }
26
+ }
27
+ },
28
+ "configuration": { "$ref": "#/definitions/configuration" },
29
+ "summary": {
30
+ "type": "object",
31
+ "additionalProperties": false,
32
+ "required": ["scenarios", "tests", "agentUsage", "evaluatorUsage"],
33
+ "properties": {
34
+ "scenarios": { "$ref": "#/definitions/tally" },
35
+ "tests": { "$ref": "#/definitions/tally" },
36
+ "agentUsage": { "$ref": "#/definitions/usageSummary" },
37
+ "evaluatorUsage": { "$ref": "#/definitions/usageSummary" }
38
+ }
39
+ },
40
+ "scenarios": { "type": "array", "items": { "$ref": "#/definitions/scenario" } }
41
+ },
42
+ "definitions": {
43
+ "count": { "type": "integer", "minimum": 0 },
44
+ "positive": { "type": "integer", "minimum": 1 },
45
+ "status": { "enum": ["PASS", "FAIL", "ERROR", "SKIPPED", "FLAKY", "INTERRUPTED"] },
46
+ "kind": { "enum": ["response", "toolUsed", "toolNotUsed", "toolsInOrder", "conversation"] },
47
+ "tally": {
48
+ "type": "object",
49
+ "additionalProperties": false,
50
+ "required": ["passed", "flaky", "failed", "errors", "interrupted"],
51
+ "properties": {
52
+ "passed": { "$ref": "#/definitions/count" },
53
+ "flaky": { "$ref": "#/definitions/count" },
54
+ "failed": { "$ref": "#/definitions/count" },
55
+ "errors": { "$ref": "#/definitions/count" },
56
+ "interrupted": { "$ref": "#/definitions/count" }
57
+ }
58
+ },
59
+ "tokenUsage": {
60
+ "type": "object",
61
+ "additionalProperties": false,
62
+ "required": ["inputTokens", "outputTokens", "totalTokens"],
63
+ "properties": {
64
+ "inputTokens": { "type": "number", "minimum": 0 },
65
+ "outputTokens": { "type": "number", "minimum": 0 },
66
+ "totalTokens": { "type": "number", "minimum": 0 }
67
+ }
68
+ },
69
+ "usageSummary": {
70
+ "description": "total is left out when no usage was observed.",
71
+ "type": "object",
72
+ "additionalProperties": false,
73
+ "required": ["reported", "expected"],
74
+ "properties": {
75
+ "total": { "$ref": "#/definitions/tokenUsage" },
76
+ "reported": { "$ref": "#/definitions/count" },
77
+ "expected": { "$ref": "#/definitions/count" }
78
+ }
79
+ },
80
+ "error": {
81
+ "type": "object",
82
+ "additionalProperties": false,
83
+ "required": ["category", "message"],
84
+ "properties": {
85
+ "category": { "enum": ["configuration", "specification", "connection", "protocol", "host", "timeout", "evaluator", "interrupted"] },
86
+ "message": { "type": "string" }
87
+ }
88
+ },
89
+ "location": {
90
+ "type": "object",
91
+ "additionalProperties": false,
92
+ "required": ["file", "line", "column", "path"],
93
+ "properties": {
94
+ "file": { "type": "string" },
95
+ "line": { "$ref": "#/definitions/count" },
96
+ "column": { "$ref": "#/definitions/count" },
97
+ "path": { "type": "string" }
98
+ }
99
+ },
100
+ "configuration": {
101
+ "description": "The effective configuration after defaults and CLI overrides. Registered secrets are [REDACTED].",
102
+ "type": "object",
103
+ "additionalProperties": false,
104
+ "required": ["path", "websocket", "execution", "evaluator", "report"],
105
+ "properties": {
106
+ "path": { "type": "string" },
107
+ "websocket": {
108
+ "type": "object",
109
+ "additionalProperties": false,
110
+ "required": ["endpoint", "subprotocols", "request", "capabilities"],
111
+ "properties": {
112
+ "endpoint": { "type": "string" },
113
+ "subprotocols": { "type": "array", "items": { "type": "string" } },
114
+ "request": {
115
+ "type": "object",
116
+ "additionalProperties": false,
117
+ "required": ["format", "value", "inputPath"],
118
+ "properties": {
119
+ "format": { "const": "json" },
120
+ "value": { "type": ["object", "array"] },
121
+ "inputPath": { "type": "string" }
122
+ }
123
+ },
124
+ "capabilities": {
125
+ "type": "object",
126
+ "additionalProperties": false,
127
+ "required": ["toolUses", "toolResults", "tokenUsage"],
128
+ "properties": {
129
+ "toolUses": { "type": "boolean" },
130
+ "toolResults": { "type": "boolean" },
131
+ "tokenUsage": { "type": "boolean" }
132
+ }
133
+ }
134
+ }
135
+ },
136
+ "execution": {
137
+ "type": "object",
138
+ "additionalProperties": false,
139
+ "required": ["connectionRetries", "timeouts"],
140
+ "properties": {
141
+ "connectionRetries": { "$ref": "#/definitions/count" },
142
+ "timeouts": {
143
+ "type": "object",
144
+ "additionalProperties": false,
145
+ "required": ["connection", "turn", "conversation"],
146
+ "properties": {
147
+ "connection": { "$ref": "#/definitions/positive" },
148
+ "turn": { "$ref": "#/definitions/positive" },
149
+ "conversation": { "$ref": "#/definitions/positive" }
150
+ }
151
+ }
152
+ }
153
+ },
154
+ "evaluator": {
155
+ "type": "object",
156
+ "additionalProperties": false,
157
+ "required": ["provider", "model", "concurrency", "timeout", "retries", "maxToolResultChars", "thresholds"],
158
+ "properties": {
159
+ "provider": { "enum": ["openai", "bedrock"] },
160
+ "model": { "type": "string" },
161
+ "apiKey": { "type": "string" },
162
+ "region": { "type": "string" },
163
+ "concurrency": { "$ref": "#/definitions/positive" },
164
+ "timeout": { "$ref": "#/definitions/positive" },
165
+ "retries": { "$ref": "#/definitions/count" },
166
+ "maxToolResultChars": { "$ref": "#/definitions/count" },
167
+ "thresholds": {
168
+ "type": "object",
169
+ "additionalProperties": false,
170
+ "required": ["response", "conversation"],
171
+ "properties": {
172
+ "response": { "type": "number", "minimum": 0, "maximum": 1 },
173
+ "conversation": { "type": "number", "minimum": 0, "maximum": 1 }
174
+ }
175
+ }
176
+ }
177
+ },
178
+ "report": {
179
+ "type": "object",
180
+ "additionalProperties": false,
181
+ "required": ["directory", "formats"],
182
+ "properties": {
183
+ "directory": { "type": "string" },
184
+ "formats": { "type": "array", "minItems": 1, "items": { "enum": ["console", "json"] } }
185
+ }
186
+ }
187
+ }
188
+ },
189
+ "scenario": {
190
+ "description": "A scenario that never started has no attempts and no tests.",
191
+ "type": "object",
192
+ "additionalProperties": false,
193
+ "required": ["name", "file", "status", "passed", "failed", "errors", "required", "agentUsage", "evaluatorUsage", "tests", "attempts"],
194
+ "properties": {
195
+ "name": { "type": "string" },
196
+ "file": { "type": "string" },
197
+ "status": { "$ref": "#/definitions/status" },
198
+ "passed": { "$ref": "#/definitions/count" },
199
+ "failed": { "$ref": "#/definitions/count" },
200
+ "errors": { "$ref": "#/definitions/count" },
201
+ "required": { "$ref": "#/definitions/positive" },
202
+ "agentUsage": { "$ref": "#/definitions/usageSummary" },
203
+ "evaluatorUsage": { "$ref": "#/definitions/usageSummary" },
204
+ "tests": { "type": "array", "items": { "$ref": "#/definitions/testAggregate" } },
205
+ "attempts": { "type": "array", "items": { "$ref": "#/definitions/attempt" } }
206
+ }
207
+ },
208
+ "testAggregate": {
209
+ "description": "One test across attempts. history lists each attempt's status; the full entries are under attempts.",
210
+ "type": "object",
211
+ "additionalProperties": false,
212
+ "required": ["kind", "turn", "title", "location", "status", "passed", "failed", "errors", "required", "history"],
213
+ "properties": {
214
+ "kind": { "$ref": "#/definitions/kind" },
215
+ "turn": { "$ref": "#/definitions/count" },
216
+ "title": { "type": "string" },
217
+ "location": { "$ref": "#/definitions/location" },
218
+ "status": { "$ref": "#/definitions/status" },
219
+ "passed": { "$ref": "#/definitions/count" },
220
+ "failed": { "$ref": "#/definitions/count" },
221
+ "errors": { "$ref": "#/definitions/count" },
222
+ "required": { "$ref": "#/definitions/positive" },
223
+ "history": {
224
+ "type": "array",
225
+ "items": {
226
+ "type": "object",
227
+ "additionalProperties": false,
228
+ "required": ["attempt", "status"],
229
+ "properties": {
230
+ "attempt": { "$ref": "#/definitions/positive" },
231
+ "status": { "$ref": "#/definitions/status" }
232
+ }
233
+ }
234
+ }
235
+ }
236
+ },
237
+ "attempt": {
238
+ "type": "object",
239
+ "additionalProperties": false,
240
+ "required": ["attempt", "status", "durationMs", "connectionAttempts", "turnsSent", "attemptErrors", "agentUsage", "evaluatorUsage", "turns", "tests"],
241
+ "properties": {
242
+ "attempt": { "$ref": "#/definitions/positive" },
243
+ "status": { "$ref": "#/definitions/status" },
244
+ "durationMs": { "$ref": "#/definitions/count" },
245
+ "connectionAttempts": { "$ref": "#/definitions/count" },
246
+ "turnsSent": { "$ref": "#/definitions/count" },
247
+ "failure": { "$ref": "#/definitions/error" },
248
+ "attemptErrors": { "type": "array", "items": { "$ref": "#/definitions/error" } },
249
+ "closeError": { "$ref": "#/definitions/error" },
250
+ "agentUsage": { "$ref": "#/definitions/usageSummary" },
251
+ "evaluatorUsage": { "$ref": "#/definitions/usageSummary" },
252
+ "turns": { "type": "array", "items": { "$ref": "#/definitions/turn" } },
253
+ "tests": { "type": "array", "items": { "$ref": "#/definitions/attemptTest" } }
254
+ }
255
+ },
256
+ "turn": {
257
+ "description": "toolUses, toolResults and tokenUsage are left out when not observed, which differs from an empty list.",
258
+ "type": "object",
259
+ "additionalProperties": false,
260
+ "required": ["turn", "input", "response", "durationMs", "statuses"],
261
+ "properties": {
262
+ "turn": { "$ref": "#/definitions/positive" },
263
+ "input": { "type": "string" },
264
+ "response": { "type": "string" },
265
+ "durationMs": { "$ref": "#/definitions/count" },
266
+ "statuses": { "type": "array", "items": { "type": "string" } },
267
+ "toolUses": {
268
+ "type": "array",
269
+ "items": {
270
+ "type": "object",
271
+ "additionalProperties": false,
272
+ "required": ["toolUseId", "name"],
273
+ "properties": {
274
+ "toolUseId": { "type": "string" },
275
+ "name": { "type": "string" },
276
+ "input": { "type": "object" },
277
+ "inputBuffer": { "type": "string" }
278
+ }
279
+ }
280
+ },
281
+ "toolResults": {
282
+ "type": "array",
283
+ "items": {
284
+ "type": "object",
285
+ "additionalProperties": false,
286
+ "required": ["toolUseId", "status"],
287
+ "properties": {
288
+ "toolUseId": { "type": "string" },
289
+ "status": { "type": "boolean" },
290
+ "outcome": { "enum": ["succeeded", "failed", "unknown"] },
291
+ "content": { "description": "The host's tool result content, in full." }
292
+ }
293
+ }
294
+ },
295
+ "tokenUsage": { "$ref": "#/definitions/tokenUsage" }
296
+ }
297
+ },
298
+ "attemptTest": {
299
+ "description": "One test in one attempt. evaluation is present only for a semantic test that got a valid verdict.",
300
+ "type": "object",
301
+ "additionalProperties": false,
302
+ "required": ["kind", "turn", "title", "location", "status", "details", "evaluatorUsage"],
303
+ "properties": {
304
+ "kind": { "$ref": "#/definitions/kind" },
305
+ "turn": { "$ref": "#/definitions/count" },
306
+ "title": { "type": "string" },
307
+ "location": { "$ref": "#/definitions/location" },
308
+ "status": { "$ref": "#/definitions/status" },
309
+ "details": { "type": "array", "items": { "type": "string" } },
310
+ "evaluation": {
311
+ "type": "object",
312
+ "additionalProperties": false,
313
+ "required": ["threshold", "score", "explanation", "evidence", "calls"],
314
+ "properties": {
315
+ "threshold": { "type": "number", "minimum": 0, "maximum": 1 },
316
+ "score": { "type": "number", "minimum": 0, "maximum": 1 },
317
+ "explanation": { "type": "string" },
318
+ "evidence": { "type": "array", "items": { "type": "string" } },
319
+ "calls": { "$ref": "#/definitions/positive" }
320
+ }
321
+ },
322
+ "evaluatorUsage": { "$ref": "#/definitions/usageSummary" }
323
+ }
324
+ }
325
+ }
326
+ }