@jmoak/vibrissa 0.1.0 → 2.0.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +268 -254
- package/bin/vib-test.js +1 -1
- package/dist/chunk-7GFB72TI.js +10706 -0
- package/dist/chunk-7GFB72TI.js.map +1 -0
- package/dist/chunk-TFXZJODE.js +2329 -0
- package/dist/chunk-TFXZJODE.js.map +1 -0
- package/dist/cli.d.ts +57 -0
- package/dist/cli.js +5896 -613
- package/dist/cli.js.map +1 -1
- package/dist/index.d.ts +837 -58
- package/dist/index.js +211 -10
- package/dist/index.js.map +1 -1
- package/dist/runner-J_V7pgax.d.ts +1194 -0
- package/dist/vitest.d.ts +44 -0
- package/dist/vitest.js +240 -0
- package/dist/vitest.js.map +1 -0
- package/package.json +27 -9
- package/schema/case.schema.json +405 -14
- package/schema/vibrissa.schema.json +528 -12
- package/dist/chunk-FTVETYUK.js +0 -656
- package/dist/chunk-FTVETYUK.js.map +0 -1
package/schema/case.schema.json
CHANGED
|
@@ -1,43 +1,434 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
-
"$id": "https://unpkg.com/vibrissa/schema/case.schema.json",
|
|
3
|
+
"$id": "https://unpkg.com/@jmoak/vibrissa/schema/case.schema.json",
|
|
4
4
|
"title": "Vibrissa Test Case",
|
|
5
|
+
"description": "One Vibrissa test case: a single operation (tool, resource, resourceTemplate or prompt) with expectations, or a multi-step flow under \"steps\".",
|
|
5
6
|
"type": "object",
|
|
6
7
|
"additionalProperties": false,
|
|
7
|
-
"
|
|
8
|
+
"anyOf": [
|
|
9
|
+
{
|
|
10
|
+
"required": ["tool"]
|
|
11
|
+
},
|
|
12
|
+
{
|
|
13
|
+
"required": ["resource"]
|
|
14
|
+
},
|
|
15
|
+
{
|
|
16
|
+
"required": ["resourceTemplate"]
|
|
17
|
+
},
|
|
18
|
+
{
|
|
19
|
+
"required": ["prompt"]
|
|
20
|
+
},
|
|
21
|
+
{
|
|
22
|
+
"required": ["steps"]
|
|
23
|
+
}
|
|
24
|
+
],
|
|
8
25
|
"properties": {
|
|
9
|
-
"$schema": {
|
|
10
|
-
|
|
11
|
-
|
|
26
|
+
"$schema": {
|
|
27
|
+
"type": "string",
|
|
28
|
+
"description": "Path or URL of this schema (for editor completion)."
|
|
29
|
+
},
|
|
30
|
+
"name": {
|
|
31
|
+
"type": "string",
|
|
32
|
+
"description": "Human-readable name shown in reports."
|
|
33
|
+
},
|
|
34
|
+
"tool": {
|
|
35
|
+
"type": "string",
|
|
36
|
+
"minLength": 1,
|
|
37
|
+
"description": "Tool to call (tools/call)."
|
|
38
|
+
},
|
|
39
|
+
"resource": {
|
|
40
|
+
"type": "string",
|
|
41
|
+
"minLength": 1,
|
|
42
|
+
"description": "Resource URI to read (resources/read)."
|
|
43
|
+
},
|
|
44
|
+
"resourceTemplate": {
|
|
45
|
+
"type": "string",
|
|
46
|
+
"minLength": 1,
|
|
47
|
+
"description": "RFC 6570 URI template, expanded with args, then read (resources/read)."
|
|
48
|
+
},
|
|
49
|
+
"prompt": {
|
|
50
|
+
"type": "string",
|
|
51
|
+
"minLength": 1,
|
|
52
|
+
"description": "Prompt to get (prompts/get)."
|
|
53
|
+
},
|
|
12
54
|
"args": {
|
|
13
55
|
"type": "object",
|
|
14
|
-
"additionalProperties": true
|
|
56
|
+
"additionalProperties": true,
|
|
57
|
+
"description": "Tool arguments, template variables, or prompt arguments. Strings may use {{capture}} and {{env.NAME}}."
|
|
58
|
+
},
|
|
59
|
+
"expect": {
|
|
60
|
+
"description": "Expected (normalized) result. Exact match by default; matchers start with $ ($partial, $regex, $wildcard, $unordered, $any, $type, ...)."
|
|
15
61
|
},
|
|
16
|
-
"expect": {},
|
|
17
62
|
"expectError": {
|
|
18
63
|
"type": "object",
|
|
19
64
|
"additionalProperties": false,
|
|
65
|
+
"description": "Expect the operation to fail (protocol error or tool isError result).",
|
|
20
66
|
"properties": {
|
|
21
|
-
"code": {
|
|
67
|
+
"code": {
|
|
68
|
+
"description": "Error code name (InvalidParams, MethodNotFound, ResourceNotFound, RequestTimeout, UnsupportedProtocolVersion, ...) or numeric JSON-RPC code.",
|
|
69
|
+
"oneOf": [
|
|
70
|
+
{
|
|
71
|
+
"type": "string"
|
|
72
|
+
},
|
|
73
|
+
{
|
|
74
|
+
"type": "integer"
|
|
75
|
+
}
|
|
76
|
+
]
|
|
77
|
+
},
|
|
22
78
|
"message": {
|
|
79
|
+
"description": "Exact error message, or { \"$regex\": \"...\", \"flags\": \"i\" }.",
|
|
23
80
|
"oneOf": [
|
|
24
|
-
{
|
|
81
|
+
{
|
|
82
|
+
"type": "string"
|
|
83
|
+
},
|
|
25
84
|
{
|
|
26
85
|
"type": "object",
|
|
27
86
|
"additionalProperties": false,
|
|
87
|
+
"required": ["$regex"],
|
|
28
88
|
"properties": {
|
|
29
|
-
"$regex": {
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
89
|
+
"$regex": {
|
|
90
|
+
"type": "string",
|
|
91
|
+
"description": "Regular expression the message must match."
|
|
92
|
+
},
|
|
93
|
+
"flags": {
|
|
94
|
+
"type": "string",
|
|
95
|
+
"description": "RegExp flags, e.g. \"i\"."
|
|
96
|
+
}
|
|
97
|
+
}
|
|
33
98
|
}
|
|
34
99
|
]
|
|
35
100
|
}
|
|
36
101
|
}
|
|
37
102
|
},
|
|
103
|
+
"respond": {
|
|
104
|
+
"type": "object",
|
|
105
|
+
"description": "Scripted answers for server input requests (MRTR input_required rounds or legacy server requests), keyed by method: \"elicitation/create\", \"sampling/createMessage\", \"roots/list\". Values are the bare result, e.g. { \"action\": \"accept\", \"content\": { ... } }.",
|
|
106
|
+
"additionalProperties": true
|
|
107
|
+
},
|
|
108
|
+
"expectInputRequired": {
|
|
109
|
+
"description": "Matcher over the InputRequests map(s) the server sent before completing."
|
|
110
|
+
},
|
|
111
|
+
"expectTask": {
|
|
112
|
+
"description": "Matcher over the CreateTaskResult when the server answers with a task (tasks extension)."
|
|
113
|
+
},
|
|
114
|
+
"await": {
|
|
115
|
+
"description": "Tasks extension: poll tasks/get until the task is terminal, then match the final result. true = defaults.",
|
|
116
|
+
"oneOf": [
|
|
117
|
+
{
|
|
118
|
+
"type": "boolean"
|
|
119
|
+
},
|
|
120
|
+
{
|
|
121
|
+
"type": "object",
|
|
122
|
+
"additionalProperties": false,
|
|
123
|
+
"properties": {
|
|
124
|
+
"pollMs": {
|
|
125
|
+
"type": "integer",
|
|
126
|
+
"minimum": 0,
|
|
127
|
+
"description": "Poll interval in ms. Default: the task pollIntervalMs, else 500."
|
|
128
|
+
},
|
|
129
|
+
"timeoutMs": {
|
|
130
|
+
"type": "integer",
|
|
131
|
+
"minimum": 0,
|
|
132
|
+
"description": "Give up after this many ms. Default 30000."
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
]
|
|
137
|
+
},
|
|
138
|
+
"capture": {
|
|
139
|
+
"type": "object",
|
|
140
|
+
"additionalProperties": {
|
|
141
|
+
"type": "string"
|
|
142
|
+
},
|
|
143
|
+
"description": "Save values for later steps: name -> dotted path into the normalized result (e.g. \"structuredContent.id\")."
|
|
144
|
+
},
|
|
145
|
+
"budgetMs": {
|
|
146
|
+
"type": "integer",
|
|
147
|
+
"minimum": 0,
|
|
148
|
+
"description": "Fail if this step takes longer than this many ms."
|
|
149
|
+
},
|
|
150
|
+
"timeoutMs": {
|
|
151
|
+
"type": "integer",
|
|
152
|
+
"minimum": 0,
|
|
153
|
+
"description": "Per-request timeout for this step (overrides config)."
|
|
154
|
+
},
|
|
155
|
+
"ignoreErrors": {
|
|
156
|
+
"type": "boolean",
|
|
157
|
+
"description": "Never fail the case because of this step (teardown)."
|
|
158
|
+
},
|
|
159
|
+
"validateOutput": {
|
|
160
|
+
"type": "boolean",
|
|
161
|
+
"description": "Validate structuredContent against the tool's outputSchema. Default true when one is declared."
|
|
162
|
+
},
|
|
163
|
+
"normalize": {
|
|
164
|
+
"description": "Override config normalization for this step, or false to match the raw wire result.",
|
|
165
|
+
"oneOf": [
|
|
166
|
+
{
|
|
167
|
+
"const": false
|
|
168
|
+
},
|
|
169
|
+
{
|
|
170
|
+
"type": "object",
|
|
171
|
+
"additionalProperties": false,
|
|
172
|
+
"properties": {
|
|
173
|
+
"stripMeta": {
|
|
174
|
+
"type": "array",
|
|
175
|
+
"items": {
|
|
176
|
+
"type": "string"
|
|
177
|
+
},
|
|
178
|
+
"description": "Glob-ish keys to strip from every _meta object."
|
|
179
|
+
},
|
|
180
|
+
"dropKeys": {
|
|
181
|
+
"type": "array",
|
|
182
|
+
"items": {
|
|
183
|
+
"type": "string"
|
|
184
|
+
},
|
|
185
|
+
"description": "Top-level result keys to drop."
|
|
186
|
+
},
|
|
187
|
+
"ttlMs": {
|
|
188
|
+
"type": "string",
|
|
189
|
+
"enum": ["keep", "ignore"],
|
|
190
|
+
"description": "Keep or ignore list cache hints."
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
]
|
|
195
|
+
},
|
|
38
196
|
"tags": {
|
|
39
197
|
"type": "array",
|
|
40
|
-
"items": {
|
|
198
|
+
"items": {
|
|
199
|
+
"type": "string"
|
|
200
|
+
},
|
|
201
|
+
"description": "Tags for --tag / --exclude-tag filtering."
|
|
202
|
+
},
|
|
203
|
+
"server": {
|
|
204
|
+
"type": "string",
|
|
205
|
+
"description": "Key into config servers; default is defaultServer."
|
|
206
|
+
},
|
|
207
|
+
"steps": {
|
|
208
|
+
"type": "array",
|
|
209
|
+
"minItems": 1,
|
|
210
|
+
"items": {
|
|
211
|
+
"$ref": "#/$defs/step"
|
|
212
|
+
},
|
|
213
|
+
"description": "Multi-step flow run on one session; top-level operation fields are ignored when present."
|
|
214
|
+
},
|
|
215
|
+
"setup": {
|
|
216
|
+
"type": "array",
|
|
217
|
+
"items": {
|
|
218
|
+
"$ref": "#/$defs/step"
|
|
219
|
+
},
|
|
220
|
+
"description": "Steps run before the case."
|
|
221
|
+
},
|
|
222
|
+
"teardown": {
|
|
223
|
+
"type": "array",
|
|
224
|
+
"items": {
|
|
225
|
+
"$ref": "#/$defs/step"
|
|
226
|
+
},
|
|
227
|
+
"description": "Steps run after the case, even when it failed."
|
|
228
|
+
},
|
|
229
|
+
"flaky": {
|
|
230
|
+
"type": "boolean",
|
|
231
|
+
"description": "Known-flaky: with --repeat, judged by pass rate instead of 100%."
|
|
232
|
+
},
|
|
233
|
+
"skip": {
|
|
234
|
+
"description": "Skip this case (true, or a reason string).",
|
|
235
|
+
"oneOf": [
|
|
236
|
+
{
|
|
237
|
+
"type": "boolean"
|
|
238
|
+
},
|
|
239
|
+
{
|
|
240
|
+
"type": "string"
|
|
241
|
+
}
|
|
242
|
+
]
|
|
243
|
+
}
|
|
244
|
+
},
|
|
245
|
+
"$defs": {
|
|
246
|
+
"step": {
|
|
247
|
+
"type": "object",
|
|
248
|
+
"description": "One operation in a flow, with its expectations.",
|
|
249
|
+
"additionalProperties": false,
|
|
250
|
+
"anyOf": [
|
|
251
|
+
{
|
|
252
|
+
"required": ["tool"]
|
|
253
|
+
},
|
|
254
|
+
{
|
|
255
|
+
"required": ["resource"]
|
|
256
|
+
},
|
|
257
|
+
{
|
|
258
|
+
"required": ["resourceTemplate"]
|
|
259
|
+
},
|
|
260
|
+
{
|
|
261
|
+
"required": ["prompt"]
|
|
262
|
+
}
|
|
263
|
+
],
|
|
264
|
+
"properties": {
|
|
265
|
+
"name": {
|
|
266
|
+
"type": "string",
|
|
267
|
+
"description": "Human-readable name shown in reports."
|
|
268
|
+
},
|
|
269
|
+
"tool": {
|
|
270
|
+
"type": "string",
|
|
271
|
+
"minLength": 1,
|
|
272
|
+
"description": "Tool to call (tools/call)."
|
|
273
|
+
},
|
|
274
|
+
"resource": {
|
|
275
|
+
"type": "string",
|
|
276
|
+
"minLength": 1,
|
|
277
|
+
"description": "Resource URI to read (resources/read)."
|
|
278
|
+
},
|
|
279
|
+
"resourceTemplate": {
|
|
280
|
+
"type": "string",
|
|
281
|
+
"minLength": 1,
|
|
282
|
+
"description": "RFC 6570 URI template, expanded with args, then read (resources/read)."
|
|
283
|
+
},
|
|
284
|
+
"prompt": {
|
|
285
|
+
"type": "string",
|
|
286
|
+
"minLength": 1,
|
|
287
|
+
"description": "Prompt to get (prompts/get)."
|
|
288
|
+
},
|
|
289
|
+
"args": {
|
|
290
|
+
"type": "object",
|
|
291
|
+
"additionalProperties": true,
|
|
292
|
+
"description": "Tool arguments, template variables, or prompt arguments. Strings may use {{capture}} and {{env.NAME}}."
|
|
293
|
+
},
|
|
294
|
+
"expect": {
|
|
295
|
+
"description": "Expected (normalized) result. Exact match by default; matchers start with $ ($partial, $regex, $wildcard, $unordered, $any, $type, ...)."
|
|
296
|
+
},
|
|
297
|
+
"expectError": {
|
|
298
|
+
"type": "object",
|
|
299
|
+
"additionalProperties": false,
|
|
300
|
+
"description": "Expect the operation to fail (protocol error or tool isError result).",
|
|
301
|
+
"properties": {
|
|
302
|
+
"code": {
|
|
303
|
+
"description": "Error code name (InvalidParams, MethodNotFound, ResourceNotFound, RequestTimeout, UnsupportedProtocolVersion, ...) or numeric JSON-RPC code.",
|
|
304
|
+
"oneOf": [
|
|
305
|
+
{
|
|
306
|
+
"type": "string"
|
|
307
|
+
},
|
|
308
|
+
{
|
|
309
|
+
"type": "integer"
|
|
310
|
+
}
|
|
311
|
+
]
|
|
312
|
+
},
|
|
313
|
+
"message": {
|
|
314
|
+
"description": "Exact error message, or { \"$regex\": \"...\", \"flags\": \"i\" }.",
|
|
315
|
+
"oneOf": [
|
|
316
|
+
{
|
|
317
|
+
"type": "string"
|
|
318
|
+
},
|
|
319
|
+
{
|
|
320
|
+
"type": "object",
|
|
321
|
+
"additionalProperties": false,
|
|
322
|
+
"required": ["$regex"],
|
|
323
|
+
"properties": {
|
|
324
|
+
"$regex": {
|
|
325
|
+
"type": "string",
|
|
326
|
+
"description": "Regular expression the message must match."
|
|
327
|
+
},
|
|
328
|
+
"flags": {
|
|
329
|
+
"type": "string",
|
|
330
|
+
"description": "RegExp flags, e.g. \"i\"."
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
]
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
},
|
|
338
|
+
"respond": {
|
|
339
|
+
"type": "object",
|
|
340
|
+
"description": "Scripted answers for server input requests (MRTR input_required rounds or legacy server requests), keyed by method: \"elicitation/create\", \"sampling/createMessage\", \"roots/list\". Values are the bare result, e.g. { \"action\": \"accept\", \"content\": { ... } }.",
|
|
341
|
+
"additionalProperties": true
|
|
342
|
+
},
|
|
343
|
+
"expectInputRequired": {
|
|
344
|
+
"description": "Matcher over the InputRequests map(s) the server sent before completing."
|
|
345
|
+
},
|
|
346
|
+
"expectTask": {
|
|
347
|
+
"description": "Matcher over the CreateTaskResult when the server answers with a task (tasks extension)."
|
|
348
|
+
},
|
|
349
|
+
"await": {
|
|
350
|
+
"description": "Tasks extension: poll tasks/get until the task is terminal, then match the final result. true = defaults.",
|
|
351
|
+
"oneOf": [
|
|
352
|
+
{
|
|
353
|
+
"type": "boolean"
|
|
354
|
+
},
|
|
355
|
+
{
|
|
356
|
+
"type": "object",
|
|
357
|
+
"additionalProperties": false,
|
|
358
|
+
"properties": {
|
|
359
|
+
"pollMs": {
|
|
360
|
+
"type": "integer",
|
|
361
|
+
"minimum": 0,
|
|
362
|
+
"description": "Poll interval in ms. Default: the task pollIntervalMs, else 500."
|
|
363
|
+
},
|
|
364
|
+
"timeoutMs": {
|
|
365
|
+
"type": "integer",
|
|
366
|
+
"minimum": 0,
|
|
367
|
+
"description": "Give up after this many ms. Default 30000."
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
]
|
|
372
|
+
},
|
|
373
|
+
"capture": {
|
|
374
|
+
"type": "object",
|
|
375
|
+
"additionalProperties": {
|
|
376
|
+
"type": "string"
|
|
377
|
+
},
|
|
378
|
+
"description": "Save values for later steps: name -> dotted path into the normalized result (e.g. \"structuredContent.id\")."
|
|
379
|
+
},
|
|
380
|
+
"budgetMs": {
|
|
381
|
+
"type": "integer",
|
|
382
|
+
"minimum": 0,
|
|
383
|
+
"description": "Fail if this step takes longer than this many ms."
|
|
384
|
+
},
|
|
385
|
+
"timeoutMs": {
|
|
386
|
+
"type": "integer",
|
|
387
|
+
"minimum": 0,
|
|
388
|
+
"description": "Per-request timeout for this step (overrides config)."
|
|
389
|
+
},
|
|
390
|
+
"ignoreErrors": {
|
|
391
|
+
"type": "boolean",
|
|
392
|
+
"description": "Never fail the case because of this step (teardown)."
|
|
393
|
+
},
|
|
394
|
+
"validateOutput": {
|
|
395
|
+
"type": "boolean",
|
|
396
|
+
"description": "Validate structuredContent against the tool's outputSchema. Default true when one is declared."
|
|
397
|
+
},
|
|
398
|
+
"normalize": {
|
|
399
|
+
"description": "Override config normalization for this step, or false to match the raw wire result.",
|
|
400
|
+
"oneOf": [
|
|
401
|
+
{
|
|
402
|
+
"const": false
|
|
403
|
+
},
|
|
404
|
+
{
|
|
405
|
+
"type": "object",
|
|
406
|
+
"additionalProperties": false,
|
|
407
|
+
"properties": {
|
|
408
|
+
"stripMeta": {
|
|
409
|
+
"type": "array",
|
|
410
|
+
"items": {
|
|
411
|
+
"type": "string"
|
|
412
|
+
},
|
|
413
|
+
"description": "Glob-ish keys to strip from every _meta object."
|
|
414
|
+
},
|
|
415
|
+
"dropKeys": {
|
|
416
|
+
"type": "array",
|
|
417
|
+
"items": {
|
|
418
|
+
"type": "string"
|
|
419
|
+
},
|
|
420
|
+
"description": "Top-level result keys to drop."
|
|
421
|
+
},
|
|
422
|
+
"ttlMs": {
|
|
423
|
+
"type": "string",
|
|
424
|
+
"enum": ["keep", "ignore"],
|
|
425
|
+
"description": "Keep or ignore list cache hints."
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
]
|
|
430
|
+
}
|
|
431
|
+
}
|
|
41
432
|
}
|
|
42
433
|
}
|
|
43
434
|
}
|