@danceiny/gotry 0.0.1-rc.19 → 0.0.1-rc.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -3
- package/README.zh-CN.md +15 -4
- package/bin/gotry-bootstrap.js +136 -45
- package/bin/gotry-inner.js +11 -6
- package/cordis.gotry-patch.yml +19 -8
- package/dist/capabilities/anything.js +62 -29
- package/dist/capabilities/doctor.js +113 -19
- package/dist/capabilities/hbcli.js +28 -2
- package/dist/capabilities/session/health-watch.js +10 -3
- package/dist/scripts/agent-planning-turn-deadline-e2e.js +2 -1
- package/dist/scripts/anything-tests.js +70 -21
- package/dist/scripts/benchmark-environment-bridge-e2e.js +178 -33
- package/dist/scripts/benchmark-environment-bridge-tests.js +1480 -271
- package/dist/scripts/booking-copilot-dsh-planner-proof-tests.js +211 -2
- package/dist/scripts/booking-copilot-runtime-proof-tests.js +132 -2
- package/dist/scripts/bootstrap-tests.js +30 -3
- package/dist/scripts/doctor-tests.js +55 -1
- package/dist/scripts/hbcli-e2e-tests.js +1 -1
- package/dist/scripts/hbcli-tests.js +38 -1
- package/dist/scripts/health-watch-cli.js +9 -1
- package/dist/scripts/map-tools-vendor-package-proof.js +276 -0
- package/dist/scripts/persona-surface-guard-tests.js +8 -3
- package/dist/src/benchmark-agent-conformance.js +187 -17
- package/dist/src/benchmark-environment-bridge.js +287 -102
- package/dist/src/booking-surface/dsh-planner.js +24 -8
- package/dist/src/index.js +5 -2
- package/extension/manifest.json +2 -2
- package/package.json +3 -1
- package/ts/capabilities/anything.ts +89 -38
- package/ts/capabilities/hbcli.ts +53 -4
- package/ts/capabilities/session/health-watch.ts +9 -2
- package/ts/dsh-runtime/vendor/README.md +60 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/LICENSE +21 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/README.en.md +200 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/README.md +202 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/client/client.js +216 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/cordis.patch.yml +5 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/clients/amap.js +371 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/clients/nominatim.js +63 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/clients/osrm.js +56 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/clients/photon.js +55 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/config-file.js +87 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/config-route.js +126 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/config.js +16 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/index.js +109 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/settings-ns.js +27 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/tools/geocode.js +127 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/tools/poi.js +84 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/tools/routes.js +170 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/clients/amap.d.ts +53 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/clients/nominatim.d.ts +17 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/clients/osrm.d.ts +15 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/clients/photon.d.ts +22 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/config-file.d.ts +33 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/config-route.d.ts +20 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/config.d.ts +23 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/index.d.ts +18 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/settings-ns.d.ts +14 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/tools/geocode.d.ts +11 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/tools/poi.d.ts +9 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/tools/routes.d.ts +16 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/types.d.ts +55 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types.js +16 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/package.json +91 -0
- package/ts/package.json +17 -1
- package/ts/scripts/map-tools-vendor-package-proof.ts +268 -0
- package/ts/src/benchmark-agent-conformance.ts +175 -13
- package/ts/src/benchmark-environment-bridge.ts +220 -66
- package/ts/src/booking-surface/dsh-planner.ts +25 -13
- package/ts/src/index.ts +4 -2
- package/ts/src/turn-deadline.ts +4 -0
|
@@ -4,9 +4,9 @@ import { tmpdir } from 'node:os';
|
|
|
4
4
|
import { join } from 'node:path';
|
|
5
5
|
import { Context } from '@deepseek-ai/cordis';
|
|
6
6
|
import { apply } from '../src/index.js';
|
|
7
|
-
import { registerBenchmarkEnvironmentBridge } from '../src/benchmark-environment-bridge.js';
|
|
7
|
+
import { BRIDGE_ERROR_CONTRACT, BENCHMARK_TOOL_RESULT_SCHEMA_VERSION, registerBenchmarkEnvironmentBridge } from '../src/benchmark-environment-bridge.js';
|
|
8
8
|
import { installBenchmarkToolIsolation } from '../src/benchmark-tool-isolation.js';
|
|
9
|
-
import { BENCHMARK_BRIDGE_CALL_FAILED, BENCHMARK_BRIDGE_CALL_REQUIRED, BENCHMARK_BRIDGE_RETRY_CALL_NOT_ALLOWED, BENCHMARK_BRIDGE_OUTPUT_TRUNCATED, BENCHMARK_BRIDGE_RUNNER_FAILED, BENCHMARK_BRIDGE_SPAWN_FAILED, BENCHMARK_BRIDGE_TIMED_OUT, BENCHMARK_CONFORMANCE_STATE_UNAVAILABLE, BENCHMARK_TERMINAL_INVALID, MAX_CONFORMANCE_RETRIES, benchmarkChildFailureForConformanceCode, createBenchmarkAgentConformance, installBenchmarkAgentConformance, parseBenchmarkTerminal, validateTerminalOutputConfig } from '../src/benchmark-agent-conformance.js';
|
|
9
|
+
import { BENCHMARK_BRIDGE_CALL_FAILED, BENCHMARK_BRIDGE_CALL_REQUIRED, BENCHMARK_BRIDGE_RETRY_CALL_NOT_ALLOWED, BENCHMARK_BRIDGE_OUTPUT_TRUNCATED, BENCHMARK_BRIDGE_RUNNER_FAILED, BENCHMARK_BRIDGE_SPAWN_FAILED, BENCHMARK_BRIDGE_TIMED_OUT, BENCHMARK_CONFORMANCE_STATE_UNAVAILABLE, BENCHMARK_TERMINAL_INVALID, MAX_CONFORMANCE_RETRIES, benchmarkChildFailureForConformanceCode, createBenchmarkAgentConformance, installBenchmarkAgentConformance, parseBenchmarkTerminal, terminalSchemaOutline, validateTerminalBodySchema, validateTerminalBodyValue, validateTerminalOutputConfig } from '../src/benchmark-agent-conformance.js';
|
|
10
10
|
import { BENCHMARK_CHILD_DIAGNOSTIC_MAX_BYTES, BENCHMARK_CHILD_DIAGNOSTIC_SCHEMA, appendBoundedChildDiagnostic, classifyBenchmarkChildFailure, classifyBenchmarkTurnEnd, createBenchmarkDiagnosticArbiter, parseBenchmarkChildDiagnostic } from '../src/benchmark-headless-child-diagnostics.js';
|
|
11
11
|
{
|
|
12
12
|
const exactFamilies = [
|
|
@@ -262,6 +262,22 @@ import { BENCHMARK_CHILD_DIAGNOSTIC_MAX_BYTES, BENCHMARK_CHILD_DIAGNOSTIC_SCHEMA
|
|
|
262
262
|
'child_model_server'
|
|
263
263
|
]);
|
|
264
264
|
}assert.equal(MAX_CONFORMANCE_RETRIES, 1);
|
|
265
|
+
const OK_BODY_SCHEMA = {
|
|
266
|
+
type: 'object',
|
|
267
|
+
properties: {
|
|
268
|
+
ok: {
|
|
269
|
+
type: 'boolean'
|
|
270
|
+
},
|
|
271
|
+
status: {
|
|
272
|
+
type: 'string'
|
|
273
|
+
},
|
|
274
|
+
x: {
|
|
275
|
+
type: 'string'
|
|
276
|
+
}
|
|
277
|
+
},
|
|
278
|
+
required: [],
|
|
279
|
+
additionalProperties: false
|
|
280
|
+
};
|
|
265
281
|
const projection = {
|
|
266
282
|
toolName: 'gotry_benchmark_environment',
|
|
267
283
|
allowedTools: [
|
|
@@ -269,7 +285,8 @@ const projection = {
|
|
|
269
285
|
],
|
|
270
286
|
terminal: {
|
|
271
287
|
tag: 'done',
|
|
272
|
-
max_bytes: 1024
|
|
288
|
+
max_bytes: 1024,
|
|
289
|
+
body_schema: OK_BODY_SCHEMA
|
|
273
290
|
}
|
|
274
291
|
};
|
|
275
292
|
assert.equal(validateTerminalOutputConfig(projection.terminal), true);
|
|
@@ -294,6 +311,20 @@ for (const invalid of [
|
|
|
294
311
|
tag: 'done',
|
|
295
312
|
max_bytes: 1024,
|
|
296
313
|
extra: true
|
|
314
|
+
},
|
|
315
|
+
{
|
|
316
|
+
tag: 'done',
|
|
317
|
+
max_bytes: 1024
|
|
318
|
+
},
|
|
319
|
+
{
|
|
320
|
+
tag: 'done',
|
|
321
|
+
max_bytes: 1024,
|
|
322
|
+
body_schema: {
|
|
323
|
+
type: 'object',
|
|
324
|
+
properties: {},
|
|
325
|
+
required: [],
|
|
326
|
+
additionalProperties: true
|
|
327
|
+
}
|
|
297
328
|
}
|
|
298
329
|
])assert.equal(validateTerminalOutputConfig(invalid), false);
|
|
299
330
|
assert.deepEqual(parseBenchmarkTerminal(' \n<done>{"ok":true}</done>\n', projection.terminal), {
|
|
@@ -338,7 +369,284 @@ for (const [raw, expectOk] of [
|
|
|
338
369
|
]){
|
|
339
370
|
assert.equal(parseBenchmarkTerminal(raw, projection.terminal).ok, expectOk, raw);
|
|
340
371
|
}
|
|
341
|
-
|
|
372
|
+
{
|
|
373
|
+
const travelSchema = {
|
|
374
|
+
type: 'object',
|
|
375
|
+
properties: {
|
|
376
|
+
people_number: {
|
|
377
|
+
type: 'integer'
|
|
378
|
+
},
|
|
379
|
+
start_city: {
|
|
380
|
+
type: 'string'
|
|
381
|
+
},
|
|
382
|
+
target_city: {
|
|
383
|
+
type: 'string'
|
|
384
|
+
},
|
|
385
|
+
itinerary: {
|
|
386
|
+
type: 'array',
|
|
387
|
+
items: {
|
|
388
|
+
type: 'object',
|
|
389
|
+
properties: {
|
|
390
|
+
day: {
|
|
391
|
+
type: 'integer'
|
|
392
|
+
},
|
|
393
|
+
activities: {
|
|
394
|
+
type: 'array',
|
|
395
|
+
items: {
|
|
396
|
+
type: 'object',
|
|
397
|
+
properties: {
|
|
398
|
+
type: {
|
|
399
|
+
type: 'string'
|
|
400
|
+
},
|
|
401
|
+
start_time: {
|
|
402
|
+
type: 'string'
|
|
403
|
+
},
|
|
404
|
+
cost: {
|
|
405
|
+
type: 'number'
|
|
406
|
+
},
|
|
407
|
+
transports: {
|
|
408
|
+
type: 'array',
|
|
409
|
+
items: {
|
|
410
|
+
type: 'object',
|
|
411
|
+
properties: {
|
|
412
|
+
start: {
|
|
413
|
+
type: 'string'
|
|
414
|
+
},
|
|
415
|
+
mode: {
|
|
416
|
+
type: 'string'
|
|
417
|
+
}
|
|
418
|
+
},
|
|
419
|
+
required: [
|
|
420
|
+
'start',
|
|
421
|
+
'mode'
|
|
422
|
+
],
|
|
423
|
+
additionalProperties: false
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
},
|
|
427
|
+
required: [
|
|
428
|
+
'type',
|
|
429
|
+
'start_time',
|
|
430
|
+
'cost',
|
|
431
|
+
'transports'
|
|
432
|
+
],
|
|
433
|
+
additionalProperties: false
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
},
|
|
437
|
+
required: [
|
|
438
|
+
'day',
|
|
439
|
+
'activities'
|
|
440
|
+
],
|
|
441
|
+
additionalProperties: false
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
},
|
|
445
|
+
required: [
|
|
446
|
+
'people_number',
|
|
447
|
+
'start_city',
|
|
448
|
+
'target_city',
|
|
449
|
+
'itinerary'
|
|
450
|
+
],
|
|
451
|
+
additionalProperties: false
|
|
452
|
+
};
|
|
453
|
+
const travelConfig = {
|
|
454
|
+
tag: 'done',
|
|
455
|
+
max_bytes: 65_536,
|
|
456
|
+
body_schema: travelSchema
|
|
457
|
+
};
|
|
458
|
+
assert.equal(validateTerminalOutputConfig(travelConfig), true, 'ChinaTravel-like closed schema is a valid terminal config');
|
|
459
|
+
const legalBody = {
|
|
460
|
+
people_number: 2,
|
|
461
|
+
start_city: 'Dubai',
|
|
462
|
+
target_city: 'Chengdu',
|
|
463
|
+
itinerary: [
|
|
464
|
+
{
|
|
465
|
+
day: 1,
|
|
466
|
+
activities: [
|
|
467
|
+
{
|
|
468
|
+
type: 'attraction',
|
|
469
|
+
start_time: '09:00',
|
|
470
|
+
cost: 0,
|
|
471
|
+
transports: [
|
|
472
|
+
{
|
|
473
|
+
start: 'hotel',
|
|
474
|
+
mode: 'walk'
|
|
475
|
+
}
|
|
476
|
+
]
|
|
477
|
+
}
|
|
478
|
+
]
|
|
479
|
+
}
|
|
480
|
+
]
|
|
481
|
+
};
|
|
482
|
+
assert.equal(validateTerminalBodySchema(travelSchema), true);
|
|
483
|
+
const mutate = (fn)=>{
|
|
484
|
+
const copy = JSON.parse(JSON.stringify(travelSchema));
|
|
485
|
+
fn(copy);
|
|
486
|
+
return copy;
|
|
487
|
+
};
|
|
488
|
+
for (const [label, schema] of [
|
|
489
|
+
[
|
|
490
|
+
'extra root key',
|
|
491
|
+
mutate((s)=>{
|
|
492
|
+
s.description = 'the plan';
|
|
493
|
+
})
|
|
494
|
+
],
|
|
495
|
+
[
|
|
496
|
+
'enum on scalar',
|
|
497
|
+
mutate((s)=>{
|
|
498
|
+
s.properties.people_number.enum = [
|
|
499
|
+
2
|
|
500
|
+
];
|
|
501
|
+
})
|
|
502
|
+
],
|
|
503
|
+
[
|
|
504
|
+
'const on scalar',
|
|
505
|
+
mutate((s)=>{
|
|
506
|
+
s.properties.start_city.const = 'Dubai';
|
|
507
|
+
})
|
|
508
|
+
],
|
|
509
|
+
[
|
|
510
|
+
'default on scalar',
|
|
511
|
+
mutate((s)=>{
|
|
512
|
+
s.properties.target_city.default = 'Chengdu';
|
|
513
|
+
})
|
|
514
|
+
],
|
|
515
|
+
[
|
|
516
|
+
'examples key',
|
|
517
|
+
mutate((s)=>{
|
|
518
|
+
s.properties.itinerary.examples = [];
|
|
519
|
+
})
|
|
520
|
+
],
|
|
521
|
+
[
|
|
522
|
+
'pattern on scalar',
|
|
523
|
+
mutate((s)=>{
|
|
524
|
+
s.properties.start_city.pattern = '^[A-Z]';
|
|
525
|
+
})
|
|
526
|
+
],
|
|
527
|
+
[
|
|
528
|
+
'open object',
|
|
529
|
+
mutate((s)=>{
|
|
530
|
+
s.additionalProperties = true;
|
|
531
|
+
})
|
|
532
|
+
],
|
|
533
|
+
[
|
|
534
|
+
'missing additionalProperties',
|
|
535
|
+
mutate((s)=>{
|
|
536
|
+
delete s.additionalProperties;
|
|
537
|
+
})
|
|
538
|
+
],
|
|
539
|
+
[
|
|
540
|
+
'array without items',
|
|
541
|
+
mutate((s)=>{
|
|
542
|
+
s.properties.itinerary = {
|
|
543
|
+
type: 'array'
|
|
544
|
+
};
|
|
545
|
+
})
|
|
546
|
+
],
|
|
547
|
+
[
|
|
548
|
+
'object without required',
|
|
549
|
+
mutate((s)=>{
|
|
550
|
+
delete s.required;
|
|
551
|
+
})
|
|
552
|
+
],
|
|
553
|
+
[
|
|
554
|
+
'unknown type',
|
|
555
|
+
mutate((s)=>{
|
|
556
|
+
s.properties.people_number.type = 'float';
|
|
557
|
+
})
|
|
558
|
+
],
|
|
559
|
+
[
|
|
560
|
+
'allOf composition',
|
|
561
|
+
mutate((s)=>{
|
|
562
|
+
s.allOf = [];
|
|
563
|
+
})
|
|
564
|
+
]
|
|
565
|
+
]){
|
|
566
|
+
assert.equal(validateTerminalBodySchema(schema), false, `schema dialect rejects: ${label}`);
|
|
567
|
+
}
|
|
568
|
+
assert.equal(validateTerminalBodyValue(legalBody, travelSchema), true, 'legal ChinaTravel-like hierarchy passes');
|
|
569
|
+
assert.equal(parseBenchmarkTerminal(`<done>${JSON.stringify(legalBody)}</done>`, travelConfig).ok, true, 'legal body parses through the terminal gate');
|
|
570
|
+
for (const [label, body] of [
|
|
571
|
+
[
|
|
572
|
+
'extra root key (Round 11 budget/total_cost case)',
|
|
573
|
+
{
|
|
574
|
+
...legalBody,
|
|
575
|
+
budget: {
|
|
576
|
+
total_cost: 1
|
|
577
|
+
}
|
|
578
|
+
}
|
|
579
|
+
],
|
|
580
|
+
[
|
|
581
|
+
'itinerary item written as direct activity (missing day)',
|
|
582
|
+
{
|
|
583
|
+
...legalBody,
|
|
584
|
+
itinerary: [
|
|
585
|
+
{
|
|
586
|
+
type: 'attraction',
|
|
587
|
+
start_time: '09:00',
|
|
588
|
+
cost: 0,
|
|
589
|
+
transports: []
|
|
590
|
+
}
|
|
591
|
+
]
|
|
592
|
+
}
|
|
593
|
+
],
|
|
594
|
+
[
|
|
595
|
+
'missing activities array',
|
|
596
|
+
{
|
|
597
|
+
...legalBody,
|
|
598
|
+
itinerary: [
|
|
599
|
+
{
|
|
600
|
+
day: 1
|
|
601
|
+
}
|
|
602
|
+
]
|
|
603
|
+
}
|
|
604
|
+
],
|
|
605
|
+
[
|
|
606
|
+
'wrong type (string day)',
|
|
607
|
+
{
|
|
608
|
+
...legalBody,
|
|
609
|
+
itinerary: [
|
|
610
|
+
{
|
|
611
|
+
day: '1',
|
|
612
|
+
activities: []
|
|
613
|
+
}
|
|
614
|
+
]
|
|
615
|
+
}
|
|
616
|
+
],
|
|
617
|
+
[
|
|
618
|
+
'nested extra field',
|
|
619
|
+
{
|
|
620
|
+
...legalBody,
|
|
621
|
+
itinerary: [
|
|
622
|
+
{
|
|
623
|
+
day: 1,
|
|
624
|
+
activities: [
|
|
625
|
+
{
|
|
626
|
+
type: 'attraction',
|
|
627
|
+
start_time: '09:00',
|
|
628
|
+
cost: 0,
|
|
629
|
+
transports: [],
|
|
630
|
+
unexpected: 1
|
|
631
|
+
}
|
|
632
|
+
]
|
|
633
|
+
}
|
|
634
|
+
]
|
|
635
|
+
}
|
|
636
|
+
]
|
|
637
|
+
]){
|
|
638
|
+
assert.equal(validateTerminalBodyValue(body, travelSchema), false, `value validation rejects: ${label}`);
|
|
639
|
+
assert.equal(parseBenchmarkTerminal(`<done>${JSON.stringify(body)}</done>`, travelConfig).ok, false, `terminal gate rejects: ${label}`);
|
|
640
|
+
}
|
|
641
|
+
assert.equal(parseBenchmarkTerminal(`<done>${' '.repeat(2000)}${JSON.stringify({
|
|
642
|
+
ok: true
|
|
643
|
+
})}</done>`, projection.terminal).ok, false, 'schema-valid body over max_bytes still rejects');
|
|
644
|
+
const outline = terminalSchemaOutline(travelSchema);
|
|
645
|
+
assert.equal(outline, terminalSchemaOutline(JSON.parse(JSON.stringify(travelSchema))), 'outline is deterministic');
|
|
646
|
+
assert.equal(outline, 'object{people_number:integer,start_city:string,target_city:string,itinerary:array<object{day:integer,activities:array<object{type:string,start_time:string,cost:number,transports:array<object{start:string,mode:string}>}>}>}', 'outline renders the exact structural contract');
|
|
647
|
+
assert.match(outline, /object\{/, 'outline renders object nodes');
|
|
648
|
+
assert.ok(!outline.includes('optional') && !outline.includes('?'), 'all keys required render without optional markers');
|
|
649
|
+
}function turnStart(turn = 1) {
|
|
342
650
|
return {
|
|
343
651
|
type: 'turn/start',
|
|
344
652
|
data: {
|
|
@@ -375,7 +683,7 @@ function toolCall(callId = 'call-1', options = {}) {
|
|
|
375
683
|
};
|
|
376
684
|
}
|
|
377
685
|
function toolResult(callId = 'call-1', options = {}) {
|
|
378
|
-
const { turn = 1, step = 1, ok = true, isError = false, error = 'runner_failed' } = options;
|
|
686
|
+
const { turn = 1, step = 1, ok = true, isError = false, error = 'runner_failed', outcome, payload } = options;
|
|
379
687
|
return {
|
|
380
688
|
type: 'tool/result',
|
|
381
689
|
data: {
|
|
@@ -394,13 +702,16 @@ function toolResult(callId = 'call-1', options = {}) {
|
|
|
394
702
|
content: [
|
|
395
703
|
{
|
|
396
704
|
type: 'text',
|
|
397
|
-
text: JSON.stringify(
|
|
705
|
+
text: JSON.stringify(payload ?? (outcome ? {
|
|
706
|
+
ok: true,
|
|
707
|
+
outcome
|
|
708
|
+
} : ok ? {
|
|
398
709
|
ok: true,
|
|
399
710
|
result: {}
|
|
400
711
|
} : {
|
|
401
712
|
ok: false,
|
|
402
713
|
error
|
|
403
|
-
})
|
|
714
|
+
}))
|
|
404
715
|
}
|
|
405
716
|
]
|
|
406
717
|
}
|
|
@@ -409,7 +720,107 @@ function toolResult(callId = 'call-1', options = {}) {
|
|
|
409
720
|
}
|
|
410
721
|
};
|
|
411
722
|
}
|
|
412
|
-
|
|
723
|
+
{
|
|
724
|
+
const state = createBenchmarkAgentConformance(projection);
|
|
725
|
+
state.observe(turnStart());
|
|
726
|
+
state.observe(toolCall());
|
|
727
|
+
state.observe(toolResult('call-1', {
|
|
728
|
+
outcome: {
|
|
729
|
+
schema_version: 'gotry_benchmark_tool_result_v1',
|
|
730
|
+
status: 'miss',
|
|
731
|
+
code: 'NOT_FOUND',
|
|
732
|
+
recovery: 'revise_arguments'
|
|
733
|
+
}
|
|
734
|
+
}));
|
|
735
|
+
assert.deepEqual(state.stopping(1), {
|
|
736
|
+
kind: 'steer',
|
|
737
|
+
mode: 'terminal'
|
|
738
|
+
}, 'domain outcome alone permits tagged terminal');
|
|
739
|
+
state.observe(assistant('<done>{"status":"miss"}</done>', {
|
|
740
|
+
step: 2
|
|
741
|
+
}));
|
|
742
|
+
assert.deepEqual(state.stopping(1), {
|
|
743
|
+
kind: 'accept'
|
|
744
|
+
}, 'domain-only tagged terminal is accepted');
|
|
745
|
+
}{
|
|
746
|
+
const state = createBenchmarkAgentConformance(projection);
|
|
747
|
+
state.observe(turnStart());
|
|
748
|
+
state.observe(toolCall());
|
|
749
|
+
state.observe(toolResult('call-1', {
|
|
750
|
+
outcome: {
|
|
751
|
+
schema_version: 'gotry_benchmark_tool_result_v1',
|
|
752
|
+
status: 'miss',
|
|
753
|
+
code: 'NOT_FOUND',
|
|
754
|
+
recovery: 'revise_arguments'
|
|
755
|
+
}
|
|
756
|
+
}));
|
|
757
|
+
state.observe(toolCall('call-2', {
|
|
758
|
+
step: 2
|
|
759
|
+
}));
|
|
760
|
+
state.observe(toolResult('call-2'));
|
|
761
|
+
state.observe(assistant('<done>{"status":"succeeded"}</done>', {
|
|
762
|
+
step: 3
|
|
763
|
+
}));
|
|
764
|
+
assert.deepEqual(state.stopping(1), {
|
|
765
|
+
kind: 'accept'
|
|
766
|
+
}, 'domain outcome followed by concrete result accepts');
|
|
767
|
+
}{
|
|
768
|
+
const state = createBenchmarkAgentConformance(projection);
|
|
769
|
+
state.observe(turnStart());
|
|
770
|
+
state.observe(toolCall());
|
|
771
|
+
state.observe(toolResult('call-1', {
|
|
772
|
+
outcome: {
|
|
773
|
+
schema_version: 'gotry_benchmark_tool_result_v1',
|
|
774
|
+
status: 'miss',
|
|
775
|
+
code: 'NOT_FOUND',
|
|
776
|
+
recovery: 'revise_arguments'
|
|
777
|
+
}
|
|
778
|
+
}));
|
|
779
|
+
state.observe(toolCall('call-2', {
|
|
780
|
+
step: 2
|
|
781
|
+
}));
|
|
782
|
+
state.observe(toolResult('call-2', {
|
|
783
|
+
ok: false,
|
|
784
|
+
error: 'runner_failed'
|
|
785
|
+
}));
|
|
786
|
+
assert.deepEqual(state.stopping(1), {
|
|
787
|
+
kind: 'reject',
|
|
788
|
+
code: BENCHMARK_BRIDGE_RUNNER_FAILED
|
|
789
|
+
}, 'infra failure overrides domain-only outcome');
|
|
790
|
+
}{
|
|
791
|
+
const state = createBenchmarkAgentConformance(projection);
|
|
792
|
+
state.observe(turnStart());
|
|
793
|
+
state.observe(toolCall());
|
|
794
|
+
state.observe(toolResult('call-1', {
|
|
795
|
+
payload: {
|
|
796
|
+
ok: true,
|
|
797
|
+
result: {},
|
|
798
|
+
outcome: {
|
|
799
|
+
status: 'miss'
|
|
800
|
+
}
|
|
801
|
+
}
|
|
802
|
+
}));
|
|
803
|
+
state.observe(assistant('<done>{"status":"succeeded"}</done>'));
|
|
804
|
+
assert.deepEqual(state.stopping(1), {
|
|
805
|
+
kind: 'reject',
|
|
806
|
+
code: BENCHMARK_BRIDGE_CALL_FAILED
|
|
807
|
+
}, 'ambiguous ok wrapper with result and outcome fails closed');
|
|
808
|
+
}{
|
|
809
|
+
const state = createBenchmarkAgentConformance(projection);
|
|
810
|
+
state.observe(turnStart());
|
|
811
|
+
state.observe(toolCall());
|
|
812
|
+
state.observe(toolResult('call-1', {
|
|
813
|
+
payload: {
|
|
814
|
+
ok: true,
|
|
815
|
+
outcome: {}
|
|
816
|
+
}
|
|
817
|
+
}));
|
|
818
|
+
state.observe(assistant('<done>{"status":"miss"}</done>'));
|
|
819
|
+
assert.deepEqual(state.stopping(1), {
|
|
820
|
+
kind: 'reject',
|
|
821
|
+
code: BENCHMARK_BRIDGE_CALL_FAILED
|
|
822
|
+
}, 'malformed inner domain outcome fails closed');
|
|
823
|
+
}function assistant(text, options = {}) {
|
|
413
824
|
const { turn = 1, step = 2, interrupted = false } = options;
|
|
414
825
|
return {
|
|
415
826
|
type: 'assistant/message',
|
|
@@ -588,6 +999,69 @@ function assistant(text, options = {}) {
|
|
|
588
999
|
assert.deepEqual(state.stopping(1), {
|
|
589
1000
|
kind: 'accept'
|
|
590
1001
|
}, 'a later failed optional call does not erase an already paired successful result');
|
|
1002
|
+
}{
|
|
1003
|
+
const state = createBenchmarkAgentConformance(projection);
|
|
1004
|
+
state.observe(turnStart());
|
|
1005
|
+
state.observe(toolCall('successful', {
|
|
1006
|
+
step: 1
|
|
1007
|
+
}));
|
|
1008
|
+
state.observe(toolResult('successful', {
|
|
1009
|
+
step: 1
|
|
1010
|
+
}));
|
|
1011
|
+
state.observe(assistant('<done>{"status":"succeeded"}</done>', {
|
|
1012
|
+
step: 2
|
|
1013
|
+
}));
|
|
1014
|
+
state.observe(toolCall('domain', {
|
|
1015
|
+
step: 3
|
|
1016
|
+
}));
|
|
1017
|
+
state.observe(toolResult('domain', {
|
|
1018
|
+
step: 3,
|
|
1019
|
+
outcome: {
|
|
1020
|
+
schema_version: 'gotry_benchmark_tool_result_v1',
|
|
1021
|
+
status: 'miss',
|
|
1022
|
+
code: 'NOT_FOUND',
|
|
1023
|
+
recovery: 'revise_arguments'
|
|
1024
|
+
}
|
|
1025
|
+
}));
|
|
1026
|
+
assert.deepEqual(state.stopping(1), {
|
|
1027
|
+
kind: 'steer',
|
|
1028
|
+
mode: 'terminal'
|
|
1029
|
+
}, 'a terminal before the latest domain response is stale');
|
|
1030
|
+
state.observe(assistant('<done>{"status":"succeeded"}</done>', {
|
|
1031
|
+
step: 4
|
|
1032
|
+
}));
|
|
1033
|
+
assert.deepEqual(state.stopping(1), {
|
|
1034
|
+
kind: 'accept'
|
|
1035
|
+
}, 'a fresh terminal after the latest domain response may reuse the earlier concrete result');
|
|
1036
|
+
}{
|
|
1037
|
+
const state = createBenchmarkAgentConformance(projection);
|
|
1038
|
+
state.observe(turnStart());
|
|
1039
|
+
state.observe(toolCall('successful', {
|
|
1040
|
+
step: 1
|
|
1041
|
+
}));
|
|
1042
|
+
state.observe(toolResult('successful', {
|
|
1043
|
+
step: 1
|
|
1044
|
+
}));
|
|
1045
|
+
state.observe(assistant('<done>{"status":"succeeded"}</done>', {
|
|
1046
|
+
step: 2
|
|
1047
|
+
}));
|
|
1048
|
+
state.observe(toolCall('failed', {
|
|
1049
|
+
step: 3
|
|
1050
|
+
}));
|
|
1051
|
+
state.observe(toolResult('failed', {
|
|
1052
|
+
step: 3,
|
|
1053
|
+
ok: false
|
|
1054
|
+
}));
|
|
1055
|
+
assert.deepEqual(state.stopping(1), {
|
|
1056
|
+
kind: 'steer',
|
|
1057
|
+
mode: 'terminal'
|
|
1058
|
+
}, 'a terminal before a later optional failure is stale');
|
|
1059
|
+
state.observe(assistant('<done>{"status":"succeeded"}</done>', {
|
|
1060
|
+
step: 4
|
|
1061
|
+
}));
|
|
1062
|
+
assert.deepEqual(state.stopping(1), {
|
|
1063
|
+
kind: 'accept'
|
|
1064
|
+
}, 'a fresh terminal may still converge after an optional failure when a concrete result exists');
|
|
591
1065
|
}{
|
|
592
1066
|
const state = createBenchmarkAgentConformance(projection);
|
|
593
1067
|
state.observe(turnStart());
|
|
@@ -708,6 +1182,8 @@ function assistant(text, options = {}) {
|
|
|
708
1182
|
assert.match(assembled.sections[0].text, /\"action\":\"call\"/);
|
|
709
1183
|
assert.match(assembled.sections[0].text, /<done>/);
|
|
710
1184
|
assert.equal(assembled.sections[0].text.includes('/tmp/'), false);
|
|
1185
|
+
const round12Outline = terminalSchemaOutline(OK_BODY_SCHEMA);
|
|
1186
|
+
assert.ok(assembled.sections[0].text.includes(`matching exactly ${round12Outline}`), 'system section projects the terminal body schema outline');
|
|
711
1187
|
rootListeners.get('session/event')[0](session, turnEnd());
|
|
712
1188
|
assert.deepEqual(runtimeWrites, []);
|
|
713
1189
|
rootListeners.get('session/event')[0](session, turnStart(2));
|
|
@@ -838,16 +1314,77 @@ function assistant(text, options = {}) {
|
|
|
838
1314
|
waitForExit: async ()=>true
|
|
839
1315
|
};
|
|
840
1316
|
}
|
|
1317
|
+
function okEnvelope(result) {
|
|
1318
|
+
return JSON.stringify({
|
|
1319
|
+
schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION,
|
|
1320
|
+
status: 'ok',
|
|
1321
|
+
result
|
|
1322
|
+
});
|
|
1323
|
+
}
|
|
1324
|
+
function lookupInputSchema() {
|
|
1325
|
+
return {
|
|
1326
|
+
type: 'object',
|
|
1327
|
+
properties: {
|
|
1328
|
+
city: {
|
|
1329
|
+
type: 'string',
|
|
1330
|
+
enum: [
|
|
1331
|
+
'Dubai',
|
|
1332
|
+
'Abu Dhabi'
|
|
1333
|
+
],
|
|
1334
|
+
description: 'Declared city name.'
|
|
1335
|
+
},
|
|
1336
|
+
payload: {
|
|
1337
|
+
type: 'string'
|
|
1338
|
+
},
|
|
1339
|
+
notes: {
|
|
1340
|
+
type: 'string'
|
|
1341
|
+
},
|
|
1342
|
+
executable: {
|
|
1343
|
+
type: 'string'
|
|
1344
|
+
},
|
|
1345
|
+
cwd: {
|
|
1346
|
+
type: 'string'
|
|
1347
|
+
},
|
|
1348
|
+
argv: {
|
|
1349
|
+
type: 'array',
|
|
1350
|
+
items: {
|
|
1351
|
+
type: 'string'
|
|
1352
|
+
}
|
|
1353
|
+
}
|
|
1354
|
+
},
|
|
1355
|
+
required: [
|
|
1356
|
+
'city'
|
|
1357
|
+
],
|
|
1358
|
+
additionalProperties: false
|
|
1359
|
+
};
|
|
1360
|
+
}
|
|
1361
|
+
function bridgeCall(tool, argumentsValue) {
|
|
1362
|
+
return {
|
|
1363
|
+
action: 'call',
|
|
1364
|
+
tool,
|
|
1365
|
+
arguments: argumentsValue
|
|
1366
|
+
};
|
|
1367
|
+
}
|
|
841
1368
|
async function assertRealCordisWaterfallOrdering() {
|
|
842
1369
|
const ctx = new Context();
|
|
843
1370
|
const bridge = {
|
|
844
1371
|
name: 'gotry_benchmark_environment',
|
|
845
1372
|
description: 'benchmark bridge',
|
|
846
1373
|
parameters: {
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
1374
|
+
oneOf: [
|
|
1375
|
+
{
|
|
1376
|
+
type: 'object',
|
|
1377
|
+
properties: {
|
|
1378
|
+
action: {
|
|
1379
|
+
const: 'tools'
|
|
1380
|
+
}
|
|
1381
|
+
},
|
|
1382
|
+
required: [
|
|
1383
|
+
'action'
|
|
1384
|
+
],
|
|
1385
|
+
additionalProperties: false
|
|
1386
|
+
}
|
|
1387
|
+
]
|
|
851
1388
|
}
|
|
852
1389
|
};
|
|
853
1390
|
const exactSchema = structuredClone(bridge);
|
|
@@ -979,7 +1516,7 @@ try {
|
|
|
979
1516
|
assert.equal(parseBenchmarkChildDiagnostic(boundedControl.toString('utf8')), 'child_bridge_timed_out', 'rolling tail preserves a final structured reason after bounded noise');
|
|
980
1517
|
const configPath = join(root, 'bridge.json');
|
|
981
1518
|
writeFileSync(configPath, JSON.stringify({
|
|
982
|
-
schema_version: '
|
|
1519
|
+
schema_version: 'gotry_benchmark_environment_bridge_v4',
|
|
983
1520
|
enabled: true,
|
|
984
1521
|
executable: process.execPath,
|
|
985
1522
|
cwd: root,
|
|
@@ -989,25 +1526,60 @@ try {
|
|
|
989
1526
|
'--lang',
|
|
990
1527
|
'en'
|
|
991
1528
|
],
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
1529
|
+
tools: [
|
|
1530
|
+
{
|
|
1531
|
+
name: 'lookup',
|
|
1532
|
+
description: 'Lookup one declared city.',
|
|
1533
|
+
input_schema: lookupInputSchema(),
|
|
1534
|
+
output_keys: [
|
|
1535
|
+
'city',
|
|
1536
|
+
'nested'
|
|
1537
|
+
],
|
|
1538
|
+
domain_outcomes: [
|
|
1539
|
+
{
|
|
1540
|
+
status: 'miss',
|
|
1541
|
+
code: 'NOT_FOUND',
|
|
1542
|
+
recovery: 'revise_arguments'
|
|
1543
|
+
},
|
|
1544
|
+
{
|
|
1545
|
+
status: 'error',
|
|
1546
|
+
code: 'AMBIGUOUS',
|
|
1547
|
+
recovery: 'choose_alternative'
|
|
1548
|
+
}
|
|
1549
|
+
]
|
|
1550
|
+
},
|
|
1551
|
+
{
|
|
1552
|
+
name: 'constructor',
|
|
1553
|
+
description: 'Construct one declared city.',
|
|
1554
|
+
input_schema: lookupInputSchema(),
|
|
1555
|
+
output_keys: [
|
|
1556
|
+
'legacy'
|
|
1557
|
+
],
|
|
1558
|
+
domain_outcomes: [
|
|
1559
|
+
{
|
|
1560
|
+
status: 'error',
|
|
1561
|
+
code: 'INVALID',
|
|
1562
|
+
recovery: 'retry_same'
|
|
1563
|
+
}
|
|
1564
|
+
]
|
|
1565
|
+
},
|
|
1566
|
+
{
|
|
1567
|
+
name: 'toString',
|
|
1568
|
+
description: 'Stringify one declared city.',
|
|
1569
|
+
input_schema: lookupInputSchema(),
|
|
1570
|
+
output_keys: [
|
|
1571
|
+
'city',
|
|
1572
|
+
'nested'
|
|
1573
|
+
],
|
|
1574
|
+
domain_outcomes: []
|
|
1575
|
+
}
|
|
996
1576
|
],
|
|
997
|
-
allowed_output_keys: {
|
|
998
|
-
lookup: [
|
|
999
|
-
'city',
|
|
1000
|
-
'nested'
|
|
1001
|
-
],
|
|
1002
|
-
constructor: [
|
|
1003
|
-
'legacy'
|
|
1004
|
-
]
|
|
1005
|
-
},
|
|
1006
1577
|
timeout_ms: 20,
|
|
1007
1578
|
max_output_bytes: 4_096,
|
|
1008
1579
|
terminal_output: {
|
|
1009
1580
|
tag: 'done',
|
|
1010
|
-
max_bytes: 4_096
|
|
1581
|
+
max_bytes: 4_096,
|
|
1582
|
+
body_schema: OK_BODY_SCHEMA
|
|
1011
1583
|
},
|
|
1012
1584
|
isolation: {
|
|
1013
1585
|
mode: 'host-enforced',
|
|
@@ -1048,7 +1620,11 @@ try {
|
|
|
1048
1620
|
let timeoutSignal;
|
|
1049
1621
|
const outcomes = [
|
|
1050
1622
|
{
|
|
1051
|
-
stdout:
|
|
1623
|
+
stdout: okEnvelope([
|
|
1624
|
+
{
|
|
1625
|
+
city: 'Dubai'
|
|
1626
|
+
}
|
|
1627
|
+
])
|
|
1052
1628
|
}
|
|
1053
1629
|
];
|
|
1054
1630
|
process.env.GOTRY_BENCHMARK_BRIDGE_PARENT_SENTINEL = 'must-not-cross-boundary';
|
|
@@ -1108,7 +1684,7 @@ try {
|
|
|
1108
1684
|
spawn (spec) {
|
|
1109
1685
|
spawnSpecs.push(spec);
|
|
1110
1686
|
const outcome = outcomes.shift() ?? {
|
|
1111
|
-
stdout:
|
|
1687
|
+
stdout: okEnvelope({})
|
|
1112
1688
|
};
|
|
1113
1689
|
if (outcome.spawnError) throw new Error('fake spawn failed');
|
|
1114
1690
|
if (outcome.waitForAbort) {
|
|
@@ -1625,15 +2201,95 @@ try {
|
|
|
1625
2201
|
}, 'agent disposal releases its quarantined isolation and conformance effects');
|
|
1626
2202
|
const bridge = registered.find((tool)=>tool.name === 'gotry_benchmark_environment');
|
|
1627
2203
|
assert.ok(bridge.execute, 'registered bridge exposes execute');
|
|
2204
|
+
const parameters = bridge.parameters;
|
|
2205
|
+
assert.equal(parameters.type, 'object', 'flat bridge uses an object root for model-facing parameters');
|
|
2206
|
+
assert.equal(parameters.oneOf, undefined, 'model-facing wire has no top-level oneOf');
|
|
2207
|
+
assert.deepEqual(parameters.required, [
|
|
2208
|
+
'action'
|
|
2209
|
+
], 'action is the only universally required wire field');
|
|
2210
|
+
assert.equal(parameters.additionalProperties, false);
|
|
2211
|
+
assert.deepEqual(parameters.properties?.action?.enum, [
|
|
2212
|
+
'tools',
|
|
2213
|
+
'call',
|
|
2214
|
+
'errors'
|
|
2215
|
+
]);
|
|
2216
|
+
assert.deepEqual(parameters.properties?.tool?.enum, [
|
|
2217
|
+
'lookup',
|
|
2218
|
+
'constructor',
|
|
2219
|
+
'toString'
|
|
2220
|
+
], 'tool enum exposes frozen descriptor names');
|
|
2221
|
+
assert.equal(parameters.properties?.arguments?.type, 'object', 'arguments remains a generic object on the open wire');
|
|
2222
|
+
assert.equal(parameters.properties?.arguments?.additionalProperties, true);
|
|
2223
|
+
assert.equal(Object.isFrozen(bridge.parameters), true, 'registered raw parameters are frozen');
|
|
2224
|
+
assert.equal(Object.isFrozen(parameters.properties?.tool?.enum), true, 'descriptor-derived tool enum is frozen');
|
|
2225
|
+
assert.throws(()=>{
|
|
2226
|
+
parameters.properties?.tool?.enum.push('escape');
|
|
2227
|
+
}, TypeError);
|
|
2228
|
+
const beforeProtocolRejected = spawnSpecs.length;
|
|
2229
|
+
await assert.rejects(()=>bridge.execute({
|
|
2230
|
+
query: bridgeCall('lookup', {
|
|
2231
|
+
city: 'Dubai'
|
|
2232
|
+
})
|
|
2233
|
+
}, null), /invalid arguments/, 'legacy nested query envelope is rejected by the flat protocol');
|
|
2234
|
+
await assert.rejects(()=>bridge.execute({}, null), /invalid arguments/, 'empty object is rejected by the flat protocol');
|
|
2235
|
+
await assert.rejects(()=>bridge.execute({
|
|
2236
|
+
action: 'tools',
|
|
2237
|
+
tool: 'lookup'
|
|
2238
|
+
}, null), /invalid arguments/, 'mixed action/tool fields are rejected by the flat protocol');
|
|
2239
|
+
await assert.rejects(()=>bridge.execute({
|
|
2240
|
+
action: 'tools',
|
|
2241
|
+
arguments: {}
|
|
2242
|
+
}, null), /invalid arguments/, 'tools rejects mixed arguments');
|
|
2243
|
+
await assert.rejects(()=>bridge.execute({
|
|
2244
|
+
action: 'errors',
|
|
2245
|
+
tool: 'lookup'
|
|
2246
|
+
}, null), /invalid arguments/, 'errors rejects mixed tool fields');
|
|
2247
|
+
await assert.rejects(()=>bridge.execute({
|
|
2248
|
+
action: 'errors',
|
|
2249
|
+
arguments: {}
|
|
2250
|
+
}, null), /invalid arguments/, 'errors rejects mixed arguments');
|
|
2251
|
+
await assert.rejects(()=>bridge.execute({
|
|
2252
|
+
...bridgeCall('lookup', {
|
|
2253
|
+
city: 'Dubai'
|
|
2254
|
+
}),
|
|
2255
|
+
extra: true
|
|
2256
|
+
}, null), /invalid arguments/, 'extra top-level fields are rejected by the flat protocol');
|
|
2257
|
+
await assert.rejects(()=>bridge.execute({
|
|
2258
|
+
action: 'call'
|
|
2259
|
+
}, null), /invalid arguments/, 'call without tool or arguments is rejected');
|
|
2260
|
+
await assert.rejects(()=>bridge.execute({
|
|
2261
|
+
action: 'call',
|
|
2262
|
+
tool: 'lookup'
|
|
2263
|
+
}, null), /invalid arguments/, 'call without arguments is rejected');
|
|
2264
|
+
await assert.rejects(()=>bridge.execute({
|
|
2265
|
+
action: 'call',
|
|
2266
|
+
arguments: {
|
|
2267
|
+
city: 'Dubai'
|
|
2268
|
+
}
|
|
2269
|
+
}, null), /invalid arguments/, 'call without tool is rejected');
|
|
2270
|
+
await assert.rejects(()=>bridge.execute(bridgeCall('lookup', {
|
|
2271
|
+
city: 'Sharjah'
|
|
2272
|
+
}), null), /invalid arguments/, 'invalid descriptor argument values are rejected by the flat protocol');
|
|
2273
|
+
await assert.rejects(()=>bridge.execute({
|
|
2274
|
+
action: 'call',
|
|
2275
|
+
tool: 'lookup',
|
|
2276
|
+
arguments: {}
|
|
2277
|
+
}, null), /invalid arguments/, 'missing required descriptor arguments are rejected by the flat protocol');
|
|
2278
|
+
for (const tool of [
|
|
2279
|
+
'lookup',
|
|
2280
|
+
'constructor',
|
|
2281
|
+
'toString'
|
|
2282
|
+
]){
|
|
2283
|
+
await assert.rejects(()=>bridge.execute(bridgeCall(tool, {
|
|
2284
|
+
city: 'Sharjah'
|
|
2285
|
+
}), null), /invalid arguments/, `${tool} rejects invalid descriptor arguments`);
|
|
2286
|
+
}
|
|
2287
|
+
assert.equal(spawnSpecs.length, beforeProtocolRejected, 'flat protocol rejections happen before spawn');
|
|
1628
2288
|
const args = {
|
|
1629
2289
|
city: 'Dubai',
|
|
1630
2290
|
payload: '$(touch /tmp/nope)'
|
|
1631
2291
|
};
|
|
1632
|
-
const result = await bridge.execute(
|
|
1633
|
-
action: 'call',
|
|
1634
|
-
tool: 'lookup',
|
|
1635
|
-
arguments: args
|
|
1636
|
-
}, null);
|
|
2292
|
+
const result = await bridge.execute(bridgeCall('lookup', args), null);
|
|
1637
2293
|
assert.deepEqual(spawnSpecs[0]?.argv, [
|
|
1638
2294
|
process.execPath,
|
|
1639
2295
|
'-m',
|
|
@@ -1670,15 +2326,13 @@ try {
|
|
|
1670
2326
|
]
|
|
1671
2327
|
}, 'one-line JSON stdout becomes structured result');
|
|
1672
2328
|
outcomes.push({
|
|
1673
|
-
stdout:
|
|
1674
|
-
});
|
|
1675
|
-
const unmappedResult = await bridge.execute({
|
|
1676
|
-
action: 'call',
|
|
1677
|
-
tool: 'toString',
|
|
1678
|
-
arguments: {
|
|
2329
|
+
stdout: okEnvelope({
|
|
1679
2330
|
city: 'Dubai'
|
|
1680
|
-
}
|
|
1681
|
-
}
|
|
2331
|
+
})
|
|
2332
|
+
});
|
|
2333
|
+
const unmappedResult = await bridge.execute(bridgeCall('toString', {
|
|
2334
|
+
city: 'Dubai'
|
|
2335
|
+
}), null);
|
|
1682
2336
|
assert.deepEqual(unmappedResult, {
|
|
1683
2337
|
ok: true,
|
|
1684
2338
|
result: {
|
|
@@ -1686,15 +2340,13 @@ try {
|
|
|
1686
2340
|
}
|
|
1687
2341
|
}, 'allowed tool without a positive mapping retains the recursive denylist only');
|
|
1688
2342
|
outcomes.push({
|
|
1689
|
-
stdout:
|
|
2343
|
+
stdout: okEnvelope({
|
|
2344
|
+
legacy: 'value'
|
|
2345
|
+
})
|
|
1690
2346
|
});
|
|
1691
|
-
const legacyResult = await bridge.execute({
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
arguments: {
|
|
1695
|
-
city: 'Dubai'
|
|
1696
|
-
}
|
|
1697
|
-
}, null);
|
|
2347
|
+
const legacyResult = await bridge.execute(bridgeCall('constructor', {
|
|
2348
|
+
city: 'Dubai'
|
|
2349
|
+
}), null);
|
|
1698
2350
|
assert.deepEqual(legacyResult, {
|
|
1699
2351
|
ok: true,
|
|
1700
2352
|
result: {
|
|
@@ -1702,29 +2354,27 @@ try {
|
|
|
1702
2354
|
}
|
|
1703
2355
|
}, 'mapped constructor accepts its declared positive key');
|
|
1704
2356
|
outcomes.push({
|
|
1705
|
-
stdout:
|
|
1706
|
-
});
|
|
1707
|
-
const constructorUnexpected = await bridge.execute({
|
|
1708
|
-
action: 'call',
|
|
1709
|
-
tool: 'constructor',
|
|
1710
|
-
arguments: {
|
|
2357
|
+
stdout: okEnvelope({
|
|
1711
2358
|
city: 'Dubai'
|
|
1712
|
-
}
|
|
1713
|
-
}
|
|
2359
|
+
})
|
|
2360
|
+
});
|
|
2361
|
+
const constructorUnexpected = await bridge.execute(bridgeCall('constructor', {
|
|
2362
|
+
city: 'Dubai'
|
|
2363
|
+
}), null);
|
|
1714
2364
|
assert.deepEqual(constructorUnexpected, {
|
|
1715
2365
|
ok: false,
|
|
1716
2366
|
error: 'forbidden_output'
|
|
1717
2367
|
}, 'mapped constructor rejects undeclared positive keys');
|
|
1718
2368
|
outcomes.push({
|
|
1719
|
-
stdout:
|
|
2369
|
+
stdout: okEnvelope({
|
|
2370
|
+
nested: {
|
|
2371
|
+
city: 'Dubai'
|
|
2372
|
+
}
|
|
2373
|
+
})
|
|
1720
2374
|
});
|
|
1721
|
-
const nestedAllowedResult = await bridge.execute({
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
arguments: {
|
|
1725
|
-
city: 'Dubai'
|
|
1726
|
-
}
|
|
1727
|
-
}, null);
|
|
2375
|
+
const nestedAllowedResult = await bridge.execute(bridgeCall('lookup', {
|
|
2376
|
+
city: 'Dubai'
|
|
2377
|
+
}), null);
|
|
1728
2378
|
assert.deepEqual(nestedAllowedResult, {
|
|
1729
2379
|
ok: true,
|
|
1730
2380
|
result: {
|
|
@@ -1734,29 +2384,31 @@ try {
|
|
|
1734
2384
|
}
|
|
1735
2385
|
}, 'configured positive output allowlist accepts declared nested keys');
|
|
1736
2386
|
outcomes.push({
|
|
1737
|
-
stdout:
|
|
2387
|
+
stdout: okEnvelope({
|
|
2388
|
+
city: 'Dubai',
|
|
2389
|
+
unexpected: 'secret'
|
|
2390
|
+
})
|
|
1738
2391
|
});
|
|
1739
|
-
const unexpectedResult = await bridge.execute({
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
arguments: {
|
|
1743
|
-
city: 'Dubai'
|
|
1744
|
-
}
|
|
1745
|
-
}, null);
|
|
2392
|
+
const unexpectedResult = await bridge.execute(bridgeCall('lookup', {
|
|
2393
|
+
city: 'Dubai'
|
|
2394
|
+
}), null);
|
|
1746
2395
|
assert.deepEqual(unexpectedResult, {
|
|
1747
2396
|
ok: false,
|
|
1748
2397
|
error: 'forbidden_output'
|
|
1749
2398
|
}, 'configured positive output allowlist rejects unexpected keys without reflecting them');
|
|
1750
2399
|
outcomes.push({
|
|
1751
|
-
stdout:
|
|
2400
|
+
stdout: okEnvelope([
|
|
2401
|
+
{
|
|
2402
|
+
city: 'Dubai',
|
|
2403
|
+
nested: {
|
|
2404
|
+
unexpected: 'secret'
|
|
2405
|
+
}
|
|
2406
|
+
}
|
|
2407
|
+
])
|
|
1752
2408
|
});
|
|
1753
|
-
const nestedUnexpectedResult = await bridge.execute({
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
arguments: {
|
|
1757
|
-
city: 'Dubai'
|
|
1758
|
-
}
|
|
1759
|
-
}, null);
|
|
2409
|
+
const nestedUnexpectedResult = await bridge.execute(bridgeCall('lookup', {
|
|
2410
|
+
city: 'Dubai'
|
|
2411
|
+
}), null);
|
|
1760
2412
|
assert.deepEqual(nestedUnexpectedResult, {
|
|
1761
2413
|
ok: false,
|
|
1762
2414
|
error: 'forbidden_output'
|
|
@@ -1781,48 +2433,125 @@ try {
|
|
|
1781
2433
|
'gоld'
|
|
1782
2434
|
]){
|
|
1783
2435
|
outcomes.push({
|
|
1784
|
-
stdout:
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
[forbidden]: 'secret'
|
|
1788
|
-
}
|
|
2436
|
+
stdout: okEnvelope({
|
|
2437
|
+
nested: {
|
|
2438
|
+
[forbidden]: 'secret'
|
|
1789
2439
|
}
|
|
1790
2440
|
})
|
|
1791
2441
|
});
|
|
1792
|
-
const forbiddenResult = await bridge.execute({
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
arguments: {
|
|
1796
|
-
city: 'Dubai'
|
|
1797
|
-
}
|
|
1798
|
-
}, null);
|
|
2442
|
+
const forbiddenResult = await bridge.execute(bridgeCall('lookup', {
|
|
2443
|
+
city: 'Dubai'
|
|
2444
|
+
}), null);
|
|
1799
2445
|
assert.deepEqual(forbiddenResult, {
|
|
1800
2446
|
ok: false,
|
|
1801
2447
|
error: 'forbidden_output'
|
|
1802
2448
|
}, `recursive no-oracle key ${forbidden} is rejected without reflecting its name`);
|
|
1803
2449
|
}
|
|
1804
2450
|
outcomes.push({
|
|
1805
|
-
stdout: '
|
|
2451
|
+
stdout: okEnvelope('the hidden answer')
|
|
1806
2452
|
});
|
|
1807
|
-
const primitiveOutput = await bridge.execute({
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
arguments: {
|
|
1811
|
-
city: 'Dubai'
|
|
1812
|
-
}
|
|
1813
|
-
}, null);
|
|
2453
|
+
const primitiveOutput = await bridge.execute(bridgeCall('lookup', {
|
|
2454
|
+
city: 'Dubai'
|
|
2455
|
+
}), null);
|
|
1814
2456
|
assert.deepEqual(primitiveOutput, {
|
|
1815
2457
|
ok: false,
|
|
1816
2458
|
error: 'invalid_output'
|
|
1817
2459
|
}, 'primitive result strings cannot bypass the structured visible-output boundary');
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
|
|
2460
|
+
outcomes.push({
|
|
2461
|
+
stdout: okEnvelope([
|
|
2462
|
+
'PRIVATE_ARRAY_VALUE'
|
|
2463
|
+
])
|
|
2464
|
+
});
|
|
2465
|
+
const primitiveArrayOutput = await bridge.execute(bridgeCall('lookup', {
|
|
2466
|
+
city: 'Dubai'
|
|
2467
|
+
}), null);
|
|
2468
|
+
assert.deepEqual(primitiveArrayOutput, {
|
|
2469
|
+
ok: false,
|
|
2470
|
+
error: 'forbidden_output'
|
|
2471
|
+
}, 'top-level primitive arrays cannot bypass the positive output-key boundary');
|
|
2472
|
+
assert.equal(JSON.stringify(primitiveArrayOutput).includes('PRIVATE_ARRAY_VALUE'), false, 'rejected primitive array values are not reflected');
|
|
2473
|
+
outcomes.push({
|
|
2474
|
+
stdout: okEnvelope([
|
|
2475
|
+
[
|
|
2476
|
+
"PRIVATE_NESTED_ARRAY_VALUE"
|
|
2477
|
+
]
|
|
2478
|
+
])
|
|
2479
|
+
});
|
|
2480
|
+
const nestedPrimitiveArrayOutput = await bridge.execute(bridgeCall('lookup', {
|
|
2481
|
+
city: 'Dubai'
|
|
2482
|
+
}), null);
|
|
2483
|
+
assert.deepEqual(nestedPrimitiveArrayOutput, {
|
|
2484
|
+
ok: false,
|
|
2485
|
+
error: 'forbidden_output'
|
|
2486
|
+
}, 'nested primitive arrays without a declared-key ancestor fail closed');
|
|
2487
|
+
assert.equal(JSON.stringify(nestedPrimitiveArrayOutput).includes('PRIVATE_NESTED_ARRAY_VALUE'), false, 'rejected nested primitive array values are not reflected');
|
|
2488
|
+
outcomes.push({
|
|
2489
|
+
stdout: okEnvelope([
|
|
2490
|
+
{
|
|
2491
|
+
city: 'Dubai'
|
|
2492
|
+
},
|
|
2493
|
+
'PRIVATE_MIXED_ARRAY_VALUE'
|
|
2494
|
+
])
|
|
2495
|
+
});
|
|
2496
|
+
const mixedArrayOutput = await bridge.execute(bridgeCall('lookup', {
|
|
2497
|
+
city: 'Dubai'
|
|
2498
|
+
}), null);
|
|
2499
|
+
assert.deepEqual(mixedArrayOutput, {
|
|
2500
|
+
ok: false,
|
|
2501
|
+
error: 'forbidden_output'
|
|
2502
|
+
}, 'a valid record cannot mask an unkeyed primitive sibling');
|
|
2503
|
+
assert.equal(JSON.stringify(mixedArrayOutput).includes('PRIVATE_MIXED_ARRAY_VALUE'), false, 'rejected mixed-array primitive values are not reflected');
|
|
2504
|
+
outcomes.push({
|
|
2505
|
+
stdout: okEnvelope([])
|
|
2506
|
+
});
|
|
2507
|
+
const emptyArrayOutput = await bridge.execute(bridgeCall('lookup', {
|
|
2508
|
+
city: 'Dubai'
|
|
2509
|
+
}), null);
|
|
2510
|
+
assert.deepEqual(emptyArrayOutput, {
|
|
2511
|
+
ok: true,
|
|
2512
|
+
result: []
|
|
2513
|
+
}, 'an empty top-level result array is an explicit non-reflecting collection');
|
|
2514
|
+
outcomes.push({
|
|
2515
|
+
stdout: okEnvelope([
|
|
2516
|
+
{
|
|
2517
|
+
city: 'Dubai'
|
|
2518
|
+
}
|
|
2519
|
+
])
|
|
2520
|
+
});
|
|
2521
|
+
const recordArrayOutput = await bridge.execute(bridgeCall('lookup', {
|
|
2522
|
+
city: 'Dubai'
|
|
2523
|
+
}), null);
|
|
2524
|
+
assert.deepEqual(recordArrayOutput, {
|
|
2525
|
+
ok: true,
|
|
2526
|
+
result: [
|
|
2527
|
+
{
|
|
2528
|
+
city: 'Dubai'
|
|
2529
|
+
}
|
|
2530
|
+
]
|
|
2531
|
+
}, 'top-level arrays of records remain valid when every leaf is covered by a declared key');
|
|
2532
|
+
outcomes.push({
|
|
2533
|
+
stdout: okEnvelope({
|
|
2534
|
+
city: [
|
|
2535
|
+
'Dubai'
|
|
2536
|
+
]
|
|
2537
|
+
})
|
|
2538
|
+
});
|
|
2539
|
+
const keyedPrimitiveArrayOutput = await bridge.execute(bridgeCall('lookup', {
|
|
2540
|
+
city: 'Dubai'
|
|
2541
|
+
}), null);
|
|
2542
|
+
assert.deepEqual(keyedPrimitiveArrayOutput, {
|
|
2543
|
+
ok: true,
|
|
2544
|
+
result: {
|
|
2545
|
+
city: [
|
|
2546
|
+
'Dubai'
|
|
2547
|
+
]
|
|
1824
2548
|
}
|
|
1825
|
-
},
|
|
2549
|
+
}, 'primitive arrays remain valid below a declared output key');
|
|
2550
|
+
const beforeOversized = spawnSpecs.length;
|
|
2551
|
+
const oversized = await bridge.execute(bridgeCall('lookup', {
|
|
2552
|
+
city: 'Dubai',
|
|
2553
|
+
notes: 'x'.repeat(65_537)
|
|
2554
|
+
}), null);
|
|
1826
2555
|
assert.deepEqual(oversized, {
|
|
1827
2556
|
ok: false,
|
|
1828
2557
|
error: 'invalid_arguments',
|
|
@@ -1834,30 +2563,20 @@ try {
|
|
|
1834
2563
|
for(let index = 0; index < 13; index++)deep = {
|
|
1835
2564
|
next: deep
|
|
1836
2565
|
};
|
|
1837
|
-
|
|
1838
|
-
action: 'call',
|
|
1839
|
-
tool: 'lookup',
|
|
1840
|
-
arguments: deep
|
|
1841
|
-
}, null);
|
|
1842
|
-
assert.deepEqual(deepResult, {
|
|
1843
|
-
ok: false,
|
|
1844
|
-
error: 'invalid_arguments',
|
|
1845
|
-
reason: 'serialization_limit'
|
|
1846
|
-
});
|
|
1847
|
-
assert.equal(spawnSpecs.length, beforeDeep, 'over-deep arguments are rejected before spawn');
|
|
1848
|
-
const beforeOverride = spawnSpecs.length;
|
|
1849
|
-
const overrideResult = await bridge.execute({
|
|
1850
|
-
action: 'call',
|
|
1851
|
-
tool: 'lookup',
|
|
1852
|
-
arguments: {
|
|
2566
|
+
await assert.rejects(()=>bridge.execute(bridgeCall('lookup', {
|
|
1853
2567
|
city: 'Dubai',
|
|
1854
|
-
|
|
1855
|
-
|
|
1856
|
-
|
|
1857
|
-
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
|
|
2568
|
+
...deep
|
|
2569
|
+
}), null), /invalid arguments/);
|
|
2570
|
+
assert.equal(spawnSpecs.length, beforeDeep, 'schema-invalid deep arguments are rejected before spawn');
|
|
2571
|
+
const beforeOverride = spawnSpecs.length;
|
|
2572
|
+
const overrideResult = await bridge.execute(bridgeCall('lookup', {
|
|
2573
|
+
city: 'Dubai',
|
|
2574
|
+
executable: '/tmp/evil',
|
|
2575
|
+
cwd: '/tmp/evil',
|
|
2576
|
+
argv: [
|
|
2577
|
+
'--unsafe'
|
|
2578
|
+
]
|
|
2579
|
+
}), null);
|
|
1861
2580
|
assert.deepEqual(spawnSpecs[beforeOverride]?.argv, [
|
|
1862
2581
|
process.execPath,
|
|
1863
2582
|
'-m',
|
|
@@ -1877,33 +2596,29 @@ try {
|
|
|
1877
2596
|
], 'model executable/cwd/argv fields remain data and cannot override config');
|
|
1878
2597
|
assert.equal(overrideResult.ok, true, 'override-shaped arguments still use the configured bridge');
|
|
1879
2598
|
outcomes.push({
|
|
1880
|
-
stdout:
|
|
2599
|
+
stdout: okEnvelope({
|
|
2600
|
+
city: 'Dubai'
|
|
2601
|
+
}),
|
|
1881
2602
|
lossy: true
|
|
1882
2603
|
});
|
|
1883
|
-
const truncated = await bridge.execute({
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
arguments: {
|
|
1887
|
-
city: 'Dubai'
|
|
1888
|
-
}
|
|
1889
|
-
}, null);
|
|
2604
|
+
const truncated = await bridge.execute(bridgeCall('lookup', {
|
|
2605
|
+
city: 'Dubai'
|
|
2606
|
+
}), null);
|
|
1890
2607
|
assert.deepEqual(truncated, {
|
|
1891
2608
|
ok: false,
|
|
1892
2609
|
error: 'output_truncated'
|
|
1893
2610
|
}, 'lossy stdout is rejected without parsing partial output');
|
|
1894
2611
|
outcomes.push({
|
|
1895
|
-
stdout:
|
|
2612
|
+
stdout: okEnvelope({
|
|
2613
|
+
city: 'Dubai'
|
|
2614
|
+
}),
|
|
1896
2615
|
stderr: 'private runner diagnostic',
|
|
1897
2616
|
exitCode: 17,
|
|
1898
2617
|
signal: 'SIGTERM'
|
|
1899
2618
|
});
|
|
1900
|
-
const failed = await bridge.execute({
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
arguments: {
|
|
1904
|
-
city: 'Dubai'
|
|
1905
|
-
}
|
|
1906
|
-
}, null);
|
|
2619
|
+
const failed = await bridge.execute(bridgeCall('lookup', {
|
|
2620
|
+
city: 'Dubai'
|
|
2621
|
+
}), null);
|
|
1907
2622
|
assert.deepEqual(failed, {
|
|
1908
2623
|
ok: false,
|
|
1909
2624
|
error: 'runner_failed',
|
|
@@ -1913,7 +2628,7 @@ try {
|
|
|
1913
2628
|
const errorsTable = await bridge.execute({
|
|
1914
2629
|
action: 'errors'
|
|
1915
2630
|
}, null);
|
|
1916
|
-
assert.ok(errorsTable.ok === true && errorsTable.errors, '
|
|
2631
|
+
assert.ok(errorsTable.ok === true && errorsTable.errors, 'bridge failure contract is discoverable');
|
|
1917
2632
|
for (const code of [
|
|
1918
2633
|
'invalid_action',
|
|
1919
2634
|
'disallowed_tool',
|
|
@@ -1927,10 +2642,68 @@ try {
|
|
|
1927
2642
|
'forbidden_output'
|
|
1928
2643
|
]){
|
|
1929
2644
|
const row = errorsTable.errors?.[code];
|
|
1930
|
-
assert.ok(row && typeof row.recoverable === 'boolean' && row.remedy.length > 0,
|
|
2645
|
+
assert.ok(row && typeof row.recoverable === 'boolean' && row.remedy.length > 0, `failure contract contains ${code} with recovery guidance`);
|
|
1931
2646
|
}
|
|
1932
|
-
assert.equal(errorsTable.errors?.invalid_arguments?.recoverable, true, '
|
|
1933
|
-
assert.equal(errorsTable.errors?.forbidden_output?.recoverable, false, '
|
|
2647
|
+
assert.equal(errorsTable.errors?.invalid_arguments?.recoverable, true, 'invalid caller arguments are recoverable');
|
|
2648
|
+
assert.equal(errorsTable.errors?.forbidden_output?.recoverable, false, 'policy-boundary output is not recoverable');
|
|
2649
|
+
assert.deepEqual(errorsTable, {
|
|
2650
|
+
ok: true,
|
|
2651
|
+
errors: BRIDGE_ERROR_CONTRACT
|
|
2652
|
+
}, 'errors action returns the exact closed failure contract');
|
|
2653
|
+
const toolsTable = await bridge.execute({
|
|
2654
|
+
action: 'tools'
|
|
2655
|
+
}, null);
|
|
2656
|
+
assert.deepEqual(toolsTable, {
|
|
2657
|
+
ok: true,
|
|
2658
|
+
tools: [
|
|
2659
|
+
{
|
|
2660
|
+
name: 'lookup',
|
|
2661
|
+
description: 'Lookup one declared city.',
|
|
2662
|
+
input_schema: lookupInputSchema(),
|
|
2663
|
+
output_keys: [
|
|
2664
|
+
'city',
|
|
2665
|
+
'nested'
|
|
2666
|
+
],
|
|
2667
|
+
domain_outcomes: [
|
|
2668
|
+
{
|
|
2669
|
+
status: 'miss',
|
|
2670
|
+
code: 'NOT_FOUND',
|
|
2671
|
+
recovery: 'revise_arguments'
|
|
2672
|
+
},
|
|
2673
|
+
{
|
|
2674
|
+
status: 'error',
|
|
2675
|
+
code: 'AMBIGUOUS',
|
|
2676
|
+
recovery: 'choose_alternative'
|
|
2677
|
+
}
|
|
2678
|
+
]
|
|
2679
|
+
},
|
|
2680
|
+
{
|
|
2681
|
+
name: 'constructor',
|
|
2682
|
+
description: 'Construct one declared city.',
|
|
2683
|
+
input_schema: lookupInputSchema(),
|
|
2684
|
+
output_keys: [
|
|
2685
|
+
'legacy'
|
|
2686
|
+
],
|
|
2687
|
+
domain_outcomes: [
|
|
2688
|
+
{
|
|
2689
|
+
status: 'error',
|
|
2690
|
+
code: 'INVALID',
|
|
2691
|
+
recovery: 'retry_same'
|
|
2692
|
+
}
|
|
2693
|
+
]
|
|
2694
|
+
},
|
|
2695
|
+
{
|
|
2696
|
+
name: 'toString',
|
|
2697
|
+
description: 'Stringify one declared city.',
|
|
2698
|
+
input_schema: lookupInputSchema(),
|
|
2699
|
+
output_keys: [
|
|
2700
|
+
'city',
|
|
2701
|
+
'nested'
|
|
2702
|
+
],
|
|
2703
|
+
domain_outcomes: []
|
|
2704
|
+
}
|
|
2705
|
+
]
|
|
2706
|
+
}, 'tools action returns every exact frozen descriptor, including its input schema');
|
|
1934
2707
|
for (const stdout of [
|
|
1935
2708
|
'not-json',
|
|
1936
2709
|
'{"result":1}{"result":2}'
|
|
@@ -1938,28 +2711,127 @@ try {
|
|
|
1938
2711
|
outcomes.push({
|
|
1939
2712
|
stdout
|
|
1940
2713
|
});
|
|
1941
|
-
const malformed = await bridge.execute({
|
|
1942
|
-
|
|
1943
|
-
|
|
1944
|
-
arguments: {
|
|
1945
|
-
city: 'Dubai'
|
|
1946
|
-
}
|
|
1947
|
-
}, null);
|
|
2714
|
+
const malformed = await bridge.execute(bridgeCall('lookup', {
|
|
2715
|
+
city: 'Dubai'
|
|
2716
|
+
}), null);
|
|
1948
2717
|
assert.deepEqual(malformed, {
|
|
1949
2718
|
ok: false,
|
|
1950
2719
|
error: 'invalid_json'
|
|
1951
2720
|
}, 'malformed or multi-value JSON is rejected');
|
|
1952
2721
|
}
|
|
1953
2722
|
outcomes.push({
|
|
1954
|
-
|
|
2723
|
+
stdout: JSON.stringify({
|
|
2724
|
+
schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION,
|
|
2725
|
+
status: 'miss',
|
|
2726
|
+
code: 'NOT_FOUND',
|
|
2727
|
+
recovery: 'revise_arguments'
|
|
2728
|
+
})
|
|
1955
2729
|
});
|
|
1956
|
-
const
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
2730
|
+
const miss = await bridge.execute(bridgeCall('lookup', {
|
|
2731
|
+
city: 'Dubai'
|
|
2732
|
+
}), null);
|
|
2733
|
+
assert.deepEqual(miss, {
|
|
2734
|
+
ok: true,
|
|
2735
|
+
outcome: {
|
|
2736
|
+
schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION,
|
|
2737
|
+
status: 'miss',
|
|
2738
|
+
code: 'NOT_FOUND',
|
|
2739
|
+
recovery: 'revise_arguments'
|
|
1961
2740
|
}
|
|
1962
|
-
},
|
|
2741
|
+
}, 'exit-zero domain miss is a successful typed outcome');
|
|
2742
|
+
outcomes.push({
|
|
2743
|
+
stdout: JSON.stringify({
|
|
2744
|
+
schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION,
|
|
2745
|
+
status: 'error',
|
|
2746
|
+
code: 'AMBIGUOUS',
|
|
2747
|
+
recovery: 'choose_alternative'
|
|
2748
|
+
})
|
|
2749
|
+
});
|
|
2750
|
+
const domainError = await bridge.execute(bridgeCall('lookup', {
|
|
2751
|
+
city: 'Dubai'
|
|
2752
|
+
}), null);
|
|
2753
|
+
assert.deepEqual(domainError, {
|
|
2754
|
+
ok: true,
|
|
2755
|
+
outcome: {
|
|
2756
|
+
schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION,
|
|
2757
|
+
status: 'error',
|
|
2758
|
+
code: 'AMBIGUOUS',
|
|
2759
|
+
recovery: 'choose_alternative'
|
|
2760
|
+
}
|
|
2761
|
+
}, 'declared exit-zero domain error is transport success');
|
|
2762
|
+
for (const invalidEnvelope of [
|
|
2763
|
+
{
|
|
2764
|
+
schema_version: 'legacy',
|
|
2765
|
+
status: 'miss',
|
|
2766
|
+
code: 'NOT_FOUND',
|
|
2767
|
+
recovery: 'revise_arguments'
|
|
2768
|
+
},
|
|
2769
|
+
{
|
|
2770
|
+
schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION,
|
|
2771
|
+
status: 'miss',
|
|
2772
|
+
code: 'UNKNOWN',
|
|
2773
|
+
recovery: 'revise_arguments'
|
|
2774
|
+
},
|
|
2775
|
+
{
|
|
2776
|
+
schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION,
|
|
2777
|
+
status: 'miss',
|
|
2778
|
+
code: 'NOT_FOUND',
|
|
2779
|
+
recovery: 'retry_same'
|
|
2780
|
+
},
|
|
2781
|
+
{
|
|
2782
|
+
schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION,
|
|
2783
|
+
status: 'error',
|
|
2784
|
+
code: 'AMBIGUOUS',
|
|
2785
|
+
recovery: 'choose_alternative',
|
|
2786
|
+
message: 'private free text'
|
|
2787
|
+
}
|
|
2788
|
+
]){
|
|
2789
|
+
outcomes.push({
|
|
2790
|
+
stdout: JSON.stringify(invalidEnvelope)
|
|
2791
|
+
});
|
|
2792
|
+
assert.deepEqual(await bridge.execute(bridgeCall('lookup', {
|
|
2793
|
+
city: 'Dubai'
|
|
2794
|
+
}), null), {
|
|
2795
|
+
ok: false,
|
|
2796
|
+
error: 'invalid_output'
|
|
2797
|
+
}, 'undeclared or non-exact domain envelope fails closed');
|
|
2798
|
+
}
|
|
2799
|
+
outcomes.push({
|
|
2800
|
+
stdout: JSON.stringify({
|
|
2801
|
+
result: {
|
|
2802
|
+
city: 'Dubai'
|
|
2803
|
+
}
|
|
2804
|
+
})
|
|
2805
|
+
});
|
|
2806
|
+
assert.deepEqual(await bridge.execute(bridgeCall('lookup', {
|
|
2807
|
+
city: 'Dubai'
|
|
2808
|
+
}), null), {
|
|
2809
|
+
ok: false,
|
|
2810
|
+
error: 'invalid_output'
|
|
2811
|
+
}, 'legacy versionless result envelope is rejected');
|
|
2812
|
+
outcomes.push({
|
|
2813
|
+
stdout: JSON.stringify({
|
|
2814
|
+
schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION,
|
|
2815
|
+
status: 'miss',
|
|
2816
|
+
code: 'NOT_FOUND',
|
|
2817
|
+
recovery: 'revise_arguments'
|
|
2818
|
+
}),
|
|
2819
|
+
exitCode: 23
|
|
2820
|
+
});
|
|
2821
|
+
assert.deepEqual(await bridge.execute(bridgeCall('lookup', {
|
|
2822
|
+
city: 'Dubai'
|
|
2823
|
+
}), null), {
|
|
2824
|
+
ok: false,
|
|
2825
|
+
error: 'runner_failed',
|
|
2826
|
+
exit_code: 23,
|
|
2827
|
+
signal: null
|
|
2828
|
+
}, 'nonzero exit remains infrastructure failure even when stdout resembles a domain envelope');
|
|
2829
|
+
outcomes.push({
|
|
2830
|
+
spawnError: true
|
|
2831
|
+
});
|
|
2832
|
+
const spawnFailed = await bridge.execute(bridgeCall('lookup', {
|
|
2833
|
+
city: 'Dubai'
|
|
2834
|
+
}), null);
|
|
1963
2835
|
assert.deepEqual(spawnFailed, {
|
|
1964
2836
|
ok: false,
|
|
1965
2837
|
error: 'spawn_failed'
|
|
@@ -1967,13 +2839,9 @@ try {
|
|
|
1967
2839
|
outcomes.push({
|
|
1968
2840
|
spawnReject: true
|
|
1969
2841
|
});
|
|
1970
|
-
const asyncSpawnFailed = await bridge.execute({
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
arguments: {
|
|
1974
|
-
city: 'Dubai'
|
|
1975
|
-
}
|
|
1976
|
-
}, null);
|
|
2842
|
+
const asyncSpawnFailed = await bridge.execute(bridgeCall('lookup', {
|
|
2843
|
+
city: 'Dubai'
|
|
2844
|
+
}), null);
|
|
1977
2845
|
assert.deepEqual(asyncSpawnFailed, {
|
|
1978
2846
|
ok: false,
|
|
1979
2847
|
error: 'spawn_failed'
|
|
@@ -1981,53 +2849,47 @@ try {
|
|
|
1981
2849
|
outcomes.push({
|
|
1982
2850
|
waitForAbort: true
|
|
1983
2851
|
});
|
|
1984
|
-
const timedOut = await bridge.execute({
|
|
1985
|
-
|
|
1986
|
-
|
|
1987
|
-
arguments: {
|
|
1988
|
-
city: 'Dubai'
|
|
1989
|
-
}
|
|
1990
|
-
}, null);
|
|
2852
|
+
const timedOut = await bridge.execute(bridgeCall('lookup', {
|
|
2853
|
+
city: 'Dubai'
|
|
2854
|
+
}), null);
|
|
1991
2855
|
assert.deepEqual(timedOut, {
|
|
1992
2856
|
ok: false,
|
|
1993
2857
|
error: 'timed_out'
|
|
1994
2858
|
}, 'deadline abort is surfaced as timed_out');
|
|
1995
2859
|
assert.equal(timeoutSignal?.aborted, true, 'timeout abort signal is fired');
|
|
1996
2860
|
const beforeInvalidArgs = spawnSpecs.length;
|
|
1997
|
-
|
|
1998
|
-
|
|
1999
|
-
|
|
2000
|
-
|
|
2001
|
-
tool: 'lookup',
|
|
2002
|
-
arguments: [
|
|
2003
|
-
'not',
|
|
2004
|
-
'plain'
|
|
2005
|
-
]
|
|
2006
|
-
}, null);
|
|
2007
|
-
} catch (e) {
|
|
2008
|
-
argsTypeRejected = e.name;
|
|
2009
|
-
}
|
|
2010
|
-
assert.equal(argsTypeRejected, 'ToolArgsError', 'arguments 非对象 → 宿主权 schema 结构化拒绝');
|
|
2861
|
+
await assert.rejects(()=>bridge.execute(bridgeCall('lookup', [
|
|
2862
|
+
'not',
|
|
2863
|
+
'plain'
|
|
2864
|
+
]), null), /invalid arguments/);
|
|
2011
2865
|
assert.equal(spawnSpecs.length, beforeInvalidArgs, 'non-object arguments are rejected before spawn');
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
|
|
2866
|
+
for (const [label, argumentsValue] of [
|
|
2867
|
+
[
|
|
2868
|
+
'missing required city',
|
|
2869
|
+
{}
|
|
2870
|
+
],
|
|
2871
|
+
[
|
|
2872
|
+
'wrong city type',
|
|
2873
|
+
{
|
|
2874
|
+
city: 7
|
|
2875
|
+
}
|
|
2876
|
+
],
|
|
2877
|
+
[
|
|
2878
|
+
'undeclared country',
|
|
2879
|
+
{
|
|
2880
|
+
city: 'Dubai',
|
|
2881
|
+
country: 'AE'
|
|
2882
|
+
}
|
|
2883
|
+
]
|
|
2884
|
+
]){
|
|
2885
|
+
await assert.rejects(()=>bridge.execute(bridgeCall('lookup', argumentsValue), null), /invalid arguments/, `${label} is rejected by the descriptor-derived runtime validator`);
|
|
2019
2886
|
}
|
|
2020
|
-
assert.equal(
|
|
2887
|
+
assert.equal(spawnSpecs.length, beforeInvalidArgs, 'required, type, and closed-object argument failures all happen before spawn');
|
|
2888
|
+
await assert.rejects(()=>bridge.execute({
|
|
2889
|
+
action: 'inspect'
|
|
2890
|
+
}, null), /invalid arguments/);
|
|
2021
2891
|
const beforeRejected = spawnSpecs.length;
|
|
2022
|
-
|
|
2023
|
-
action: 'call',
|
|
2024
|
-
tool: 'delete_all',
|
|
2025
|
-
arguments: {}
|
|
2026
|
-
}, null);
|
|
2027
|
-
assert.deepEqual(rejected, {
|
|
2028
|
-
ok: false,
|
|
2029
|
-
error: 'disallowed_tool'
|
|
2030
|
-
}, 'disallowed tool is rejected structurally');
|
|
2892
|
+
await assert.rejects(()=>bridge.execute(bridgeCall('delete_all', {}), null), /invalid arguments/);
|
|
2031
2893
|
assert.equal(spawnSpecs.length, beforeRejected, 'disallowed tool is rejected before spawn');
|
|
2032
2894
|
const spacedConfigPath = join(root, ' benchmark-environment-config.json ');
|
|
2033
2895
|
writeFileSync(spacedConfigPath, readFileSync(configPath));
|
|
@@ -2214,7 +3076,7 @@ try {
|
|
|
2214
3076
|
else process.env.GOTRY_LLM_MODEL = originalModelOverride;
|
|
2215
3077
|
}
|
|
2216
3078
|
const validConfig = {
|
|
2217
|
-
schema_version: '
|
|
3079
|
+
schema_version: 'gotry_benchmark_environment_bridge_v4',
|
|
2218
3080
|
enabled: true,
|
|
2219
3081
|
executable: process.execPath,
|
|
2220
3082
|
cwd: root,
|
|
@@ -2224,14 +3086,45 @@ try {
|
|
|
2224
3086
|
'--lang',
|
|
2225
3087
|
'en'
|
|
2226
3088
|
],
|
|
2227
|
-
|
|
2228
|
-
|
|
3089
|
+
tools: [
|
|
3090
|
+
{
|
|
3091
|
+
name: 'lookup',
|
|
3092
|
+
description: 'Lookup one declared city.',
|
|
3093
|
+
input_schema: {
|
|
3094
|
+
type: 'object',
|
|
3095
|
+
properties: {
|
|
3096
|
+
city: {
|
|
3097
|
+
type: 'string',
|
|
3098
|
+
enum: [
|
|
3099
|
+
'Dubai',
|
|
3100
|
+
'Abu Dhabi'
|
|
3101
|
+
],
|
|
3102
|
+
description: 'Declared city name.'
|
|
3103
|
+
}
|
|
3104
|
+
},
|
|
3105
|
+
required: [
|
|
3106
|
+
'city'
|
|
3107
|
+
],
|
|
3108
|
+
additionalProperties: false
|
|
3109
|
+
},
|
|
3110
|
+
output_keys: [
|
|
3111
|
+
'city'
|
|
3112
|
+
],
|
|
3113
|
+
domain_outcomes: [
|
|
3114
|
+
{
|
|
3115
|
+
status: 'miss',
|
|
3116
|
+
code: 'NOT_FOUND',
|
|
3117
|
+
recovery: 'revise_arguments'
|
|
3118
|
+
}
|
|
3119
|
+
]
|
|
3120
|
+
}
|
|
2229
3121
|
],
|
|
2230
3122
|
timeout_ms: 20,
|
|
2231
3123
|
max_output_bytes: 4_096,
|
|
2232
3124
|
terminal_output: {
|
|
2233
3125
|
tag: 'done',
|
|
2234
|
-
max_bytes: 4_096
|
|
3126
|
+
max_bytes: 4_096,
|
|
3127
|
+
body_schema: OK_BODY_SCHEMA
|
|
2235
3128
|
},
|
|
2236
3129
|
isolation: {
|
|
2237
3130
|
mode: 'host-enforced',
|
|
@@ -2239,6 +3132,86 @@ try {
|
|
|
2239
3132
|
network: 'denied'
|
|
2240
3133
|
}
|
|
2241
3134
|
};
|
|
3135
|
+
const clonedConfig = ()=>JSON.parse(JSON.stringify(validConfig));
|
|
3136
|
+
const configWithTool = (mutate)=>{
|
|
3137
|
+
const config = clonedConfig();
|
|
3138
|
+
mutate(config.tools[0]);
|
|
3139
|
+
return config;
|
|
3140
|
+
};
|
|
3141
|
+
const nestedInputSchema = (objectLevels)=>{
|
|
3142
|
+
let child = {
|
|
3143
|
+
type: 'string'
|
|
3144
|
+
};
|
|
3145
|
+
for(let index = 0; index < objectLevels; index += 1){
|
|
3146
|
+
child = {
|
|
3147
|
+
type: 'object',
|
|
3148
|
+
properties: {
|
|
3149
|
+
next: child
|
|
3150
|
+
},
|
|
3151
|
+
required: [
|
|
3152
|
+
'next'
|
|
3153
|
+
],
|
|
3154
|
+
additionalProperties: false
|
|
3155
|
+
};
|
|
3156
|
+
}
|
|
3157
|
+
return child;
|
|
3158
|
+
};
|
|
3159
|
+
const flatInputSchema = (count)=>({
|
|
3160
|
+
type: 'object',
|
|
3161
|
+
properties: Object.fromEntries(Array.from({
|
|
3162
|
+
length: count
|
|
3163
|
+
}, (_, index)=>[
|
|
3164
|
+
`p${index}`,
|
|
3165
|
+
{
|
|
3166
|
+
type: 'string'
|
|
3167
|
+
}
|
|
3168
|
+
])),
|
|
3169
|
+
required: [],
|
|
3170
|
+
additionalProperties: false
|
|
3171
|
+
});
|
|
3172
|
+
const totalEnumInputSchema = (groups)=>({
|
|
3173
|
+
type: 'object',
|
|
3174
|
+
properties: Object.fromEntries(Array.from({
|
|
3175
|
+
length: groups
|
|
3176
|
+
}, (_, group)=>[
|
|
3177
|
+
`e${group}`,
|
|
3178
|
+
{
|
|
3179
|
+
type: 'string',
|
|
3180
|
+
enum: Array.from({
|
|
3181
|
+
length: 64
|
|
3182
|
+
}, (_, index)=>`v${group}_${index}`)
|
|
3183
|
+
}
|
|
3184
|
+
])),
|
|
3185
|
+
required: [],
|
|
3186
|
+
additionalProperties: false
|
|
3187
|
+
});
|
|
3188
|
+
const byteSizedInputSchema = (targetBytes)=>{
|
|
3189
|
+
const properties = Object.fromEntries(Array.from({
|
|
3190
|
+
length: 64
|
|
3191
|
+
}, (_, index)=>[
|
|
3192
|
+
`b${index}`,
|
|
3193
|
+
{
|
|
3194
|
+
type: 'string',
|
|
3195
|
+
description: ''
|
|
3196
|
+
}
|
|
3197
|
+
]));
|
|
3198
|
+
const schema = {
|
|
3199
|
+
type: 'object',
|
|
3200
|
+
properties,
|
|
3201
|
+
required: [],
|
|
3202
|
+
additionalProperties: false
|
|
3203
|
+
};
|
|
3204
|
+
let remaining = targetBytes - Buffer.byteLength(JSON.stringify(schema), 'utf8');
|
|
3205
|
+
assert.ok(remaining >= 0 && remaining <= 64 * 512, 'byte-boundary fixture has enough bounded description capacity');
|
|
3206
|
+
for (const property of Object.values(properties)){
|
|
3207
|
+
const added = Math.min(512, remaining);
|
|
3208
|
+
property.description = 'x'.repeat(added);
|
|
3209
|
+
remaining -= added;
|
|
3210
|
+
}
|
|
3211
|
+
assert.equal(remaining, 0, 'byte-boundary fixture reaches the requested serialized size');
|
|
3212
|
+
assert.equal(Buffer.byteLength(JSON.stringify(schema), 'utf8'), targetBytes);
|
|
3213
|
+
return schema;
|
|
3214
|
+
};
|
|
2242
3215
|
const frozenProjection = registerBenchmarkEnvironmentBridge(configPath, ()=>{}, {
|
|
2243
3216
|
spawn: (_spec)=>fakeHandle({
|
|
2244
3217
|
stdout: '{"result":{}}'
|
|
@@ -2279,7 +3252,7 @@ try {
|
|
|
2279
3252
|
return ()=>{};
|
|
2280
3253
|
},
|
|
2281
3254
|
effect (action) {
|
|
2282
|
-
return action
|
|
3255
|
+
return runEffect(action);
|
|
2283
3256
|
},
|
|
2284
3257
|
agents: {
|
|
2285
3258
|
list () {
|
|
@@ -2313,62 +3286,300 @@ try {
|
|
|
2313
3286
|
}), /benchmark environment bridge configuration unavailable/, 'unknown top-level config key fails hard');
|
|
2314
3287
|
assert.throws(()=>bridgeRegistrationFor({
|
|
2315
3288
|
...validConfig,
|
|
2316
|
-
|
|
2317
|
-
|
|
3289
|
+
tools: [
|
|
3290
|
+
{
|
|
3291
|
+
...validConfig.tools[0],
|
|
3292
|
+
input_schema: {
|
|
3293
|
+
type: 'object'
|
|
3294
|
+
}
|
|
3295
|
+
}
|
|
3296
|
+
]
|
|
3297
|
+
}), /benchmark environment bridge configuration unavailable/, 'v3 rejects an open annotation-only input schema');
|
|
2318
3298
|
assert.throws(()=>bridgeRegistrationFor({
|
|
2319
3299
|
...validConfig,
|
|
2320
|
-
|
|
2321
|
-
|
|
2322
|
-
'lookup'
|
|
2323
|
-
]
|
|
2324
|
-
}), /benchmark environment bridge configuration unavailable/, 'duplicate allowed tool fails hard');
|
|
3300
|
+
schema_version: 'gotry_benchmark_environment_bridge_v1'
|
|
3301
|
+
}), /benchmark environment bridge configuration unavailable/, 'v1 config cannot silently omit the Round 3 terminal semantics');
|
|
2325
3302
|
assert.throws(()=>bridgeRegistrationFor({
|
|
2326
3303
|
...validConfig,
|
|
2327
|
-
|
|
2328
|
-
}), /benchmark environment bridge configuration unavailable/, '
|
|
3304
|
+
schema_version: 'gotry_benchmark_environment_bridge_v3'
|
|
3305
|
+
}), /benchmark environment bridge configuration unavailable/, 'v3 config cannot silently omit the Round 12 terminal body schema (#215)');
|
|
2329
3306
|
assert.throws(()=>bridgeRegistrationFor({
|
|
2330
3307
|
...validConfig,
|
|
2331
|
-
|
|
2332
|
-
|
|
3308
|
+
terminal_output: {
|
|
3309
|
+
...validConfig.terminal_output,
|
|
3310
|
+
body_schema: {
|
|
3311
|
+
...validConfig.terminal_output.body_schema,
|
|
3312
|
+
additionalProperties: true
|
|
3313
|
+
}
|
|
2333
3314
|
}
|
|
2334
|
-
}), /benchmark environment bridge configuration unavailable/, '
|
|
3315
|
+
}), /benchmark environment bridge configuration unavailable/, 'non-closed terminal body schema fails the config load closed');
|
|
2335
3316
|
assert.throws(()=>bridgeRegistrationFor({
|
|
2336
3317
|
...validConfig,
|
|
2337
|
-
|
|
2338
|
-
|
|
3318
|
+
tools: [
|
|
3319
|
+
validConfig.tools[0],
|
|
3320
|
+
validConfig.tools[0]
|
|
3321
|
+
]
|
|
3322
|
+
}), /benchmark environment bridge configuration unavailable/, 'duplicate tool descriptor fails hard');
|
|
3323
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3324
|
+
delete tool.output_keys;
|
|
3325
|
+
})), /benchmark environment bridge configuration unavailable/, 'missing output_keys fails hard');
|
|
3326
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3327
|
+
tool.output_keys = [];
|
|
3328
|
+
})), /benchmark environment bridge configuration unavailable/, 'empty output allowlist fails closed at registration');
|
|
3329
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3330
|
+
tool.output_keys = [
|
|
3331
|
+
'city',
|
|
3332
|
+
'city'
|
|
3333
|
+
];
|
|
3334
|
+
})), /benchmark environment bridge configuration unavailable/, 'duplicate output key fails hard');
|
|
3335
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3336
|
+
tool.output_keys = [
|
|
3337
|
+
'not a key'
|
|
3338
|
+
];
|
|
3339
|
+
})), /benchmark environment bridge configuration unavailable/, 'non-identifier output key fails hard');
|
|
3340
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3341
|
+
delete tool.domain_outcomes;
|
|
3342
|
+
})), /benchmark environment bridge configuration unavailable/, 'missing domain_outcomes fails hard');
|
|
3343
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3344
|
+
tool.domain_outcomes = [
|
|
3345
|
+
{
|
|
3346
|
+
status: 'unknown',
|
|
3347
|
+
code: 'NOT_FOUND',
|
|
3348
|
+
recovery: 'none'
|
|
3349
|
+
}
|
|
3350
|
+
];
|
|
3351
|
+
})), /benchmark environment bridge configuration unavailable/, 'unknown domain status fails hard');
|
|
3352
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3353
|
+
tool.domain_outcomes = [
|
|
3354
|
+
{
|
|
3355
|
+
status: 'miss',
|
|
3356
|
+
code: 'NOT_FOUND',
|
|
3357
|
+
recovery: 'free_text'
|
|
3358
|
+
}
|
|
3359
|
+
];
|
|
3360
|
+
})), /benchmark environment bridge configuration unavailable/, 'free-text domain recovery fails hard');
|
|
3361
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3362
|
+
tool.domain_outcomes = [
|
|
3363
|
+
{
|
|
3364
|
+
status: 'miss',
|
|
3365
|
+
code: 'NOT_FOUND',
|
|
3366
|
+
recovery: 'none'
|
|
3367
|
+
},
|
|
3368
|
+
{
|
|
3369
|
+
status: 'error',
|
|
3370
|
+
code: 'NOT_FOUND',
|
|
3371
|
+
recovery: 'none'
|
|
3372
|
+
}
|
|
3373
|
+
];
|
|
3374
|
+
})), /benchmark environment bridge configuration unavailable/, 'duplicate domain code fails hard');
|
|
3375
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3376
|
+
tool.name = 'lookup;rm';
|
|
3377
|
+
})), /benchmark environment bridge configuration unavailable/, 'shell metacharacter tool identifier fails hard');
|
|
3378
|
+
for (const [label, schema] of [
|
|
3379
|
+
[
|
|
3380
|
+
'reference keyword',
|
|
3381
|
+
{
|
|
3382
|
+
type: 'object',
|
|
3383
|
+
properties: {
|
|
3384
|
+
city: {
|
|
3385
|
+
$ref: '#/city'
|
|
3386
|
+
}
|
|
3387
|
+
},
|
|
3388
|
+
required: [
|
|
2339
3389
|
'city'
|
|
2340
|
-
]
|
|
3390
|
+
],
|
|
3391
|
+
additionalProperties: false
|
|
2341
3392
|
}
|
|
2342
|
-
|
|
2343
|
-
|
|
2344
|
-
|
|
2345
|
-
|
|
2346
|
-
|
|
3393
|
+
],
|
|
3394
|
+
[
|
|
3395
|
+
'default keyword',
|
|
3396
|
+
{
|
|
3397
|
+
type: 'object',
|
|
3398
|
+
properties: {
|
|
3399
|
+
city: {
|
|
3400
|
+
type: 'string',
|
|
3401
|
+
default: 'Dubai'
|
|
3402
|
+
}
|
|
3403
|
+
},
|
|
3404
|
+
required: [
|
|
3405
|
+
'city'
|
|
3406
|
+
],
|
|
3407
|
+
additionalProperties: false
|
|
3408
|
+
}
|
|
3409
|
+
],
|
|
3410
|
+
[
|
|
3411
|
+
'examples keyword',
|
|
3412
|
+
{
|
|
3413
|
+
type: 'object',
|
|
3414
|
+
properties: {
|
|
3415
|
+
city: {
|
|
3416
|
+
type: 'string',
|
|
3417
|
+
examples: [
|
|
3418
|
+
'Dubai'
|
|
3419
|
+
]
|
|
3420
|
+
}
|
|
3421
|
+
},
|
|
3422
|
+
required: [
|
|
3423
|
+
'city'
|
|
3424
|
+
],
|
|
3425
|
+
additionalProperties: false
|
|
3426
|
+
}
|
|
3427
|
+
],
|
|
3428
|
+
[
|
|
3429
|
+
'unknown keyword',
|
|
3430
|
+
{
|
|
3431
|
+
type: 'object',
|
|
3432
|
+
properties: {
|
|
3433
|
+
city: {
|
|
3434
|
+
type: 'string',
|
|
3435
|
+
title: 'City'
|
|
3436
|
+
}
|
|
3437
|
+
},
|
|
3438
|
+
required: [
|
|
3439
|
+
'city'
|
|
3440
|
+
],
|
|
3441
|
+
additionalProperties: false
|
|
3442
|
+
}
|
|
3443
|
+
],
|
|
3444
|
+
[
|
|
3445
|
+
'open nested object',
|
|
3446
|
+
{
|
|
3447
|
+
type: 'object',
|
|
3448
|
+
properties: {
|
|
3449
|
+
nested: {
|
|
3450
|
+
type: 'object',
|
|
3451
|
+
properties: {},
|
|
3452
|
+
required: []
|
|
3453
|
+
}
|
|
3454
|
+
},
|
|
3455
|
+
required: [
|
|
3456
|
+
'nested'
|
|
3457
|
+
],
|
|
3458
|
+
additionalProperties: false
|
|
3459
|
+
}
|
|
3460
|
+
],
|
|
3461
|
+
[
|
|
3462
|
+
'array without items',
|
|
3463
|
+
{
|
|
3464
|
+
type: 'object',
|
|
3465
|
+
properties: {
|
|
3466
|
+
values: {
|
|
3467
|
+
type: 'array'
|
|
3468
|
+
}
|
|
3469
|
+
},
|
|
3470
|
+
required: [
|
|
3471
|
+
'values'
|
|
3472
|
+
],
|
|
3473
|
+
additionalProperties: false
|
|
3474
|
+
}
|
|
3475
|
+
],
|
|
3476
|
+
[
|
|
3477
|
+
'duplicate required',
|
|
3478
|
+
{
|
|
3479
|
+
type: 'object',
|
|
3480
|
+
properties: {
|
|
3481
|
+
city: {
|
|
3482
|
+
type: 'string'
|
|
3483
|
+
}
|
|
3484
|
+
},
|
|
3485
|
+
required: [
|
|
2347
3486
|
'city',
|
|
2348
3487
|
'city'
|
|
2349
|
-
]
|
|
3488
|
+
],
|
|
3489
|
+
additionalProperties: false
|
|
2350
3490
|
}
|
|
2351
|
-
|
|
2352
|
-
|
|
2353
|
-
|
|
2354
|
-
|
|
2355
|
-
|
|
2356
|
-
|
|
2357
|
-
|
|
3491
|
+
],
|
|
3492
|
+
[
|
|
3493
|
+
'unknown required',
|
|
3494
|
+
{
|
|
3495
|
+
type: 'object',
|
|
3496
|
+
properties: {
|
|
3497
|
+
city: {
|
|
3498
|
+
type: 'string'
|
|
3499
|
+
}
|
|
3500
|
+
},
|
|
3501
|
+
required: [
|
|
3502
|
+
'country'
|
|
3503
|
+
],
|
|
3504
|
+
additionalProperties: false
|
|
2358
3505
|
}
|
|
2359
|
-
|
|
3506
|
+
],
|
|
3507
|
+
[
|
|
3508
|
+
'type-mismatched enum',
|
|
3509
|
+
{
|
|
3510
|
+
type: 'object',
|
|
3511
|
+
properties: {
|
|
3512
|
+
city: {
|
|
3513
|
+
type: 'string',
|
|
3514
|
+
enum: [
|
|
3515
|
+
'Dubai',
|
|
3516
|
+
7
|
|
3517
|
+
]
|
|
3518
|
+
}
|
|
3519
|
+
},
|
|
3520
|
+
required: [
|
|
3521
|
+
'city'
|
|
3522
|
+
],
|
|
3523
|
+
additionalProperties: false
|
|
3524
|
+
}
|
|
3525
|
+
],
|
|
3526
|
+
[
|
|
3527
|
+
'duplicate enum',
|
|
3528
|
+
{
|
|
3529
|
+
type: 'object',
|
|
3530
|
+
properties: {
|
|
3531
|
+
city: {
|
|
3532
|
+
type: 'string',
|
|
3533
|
+
enum: [
|
|
3534
|
+
'Dubai',
|
|
3535
|
+
'Dubai'
|
|
3536
|
+
]
|
|
3537
|
+
}
|
|
3538
|
+
},
|
|
3539
|
+
required: [
|
|
3540
|
+
'city'
|
|
3541
|
+
],
|
|
3542
|
+
additionalProperties: false
|
|
3543
|
+
}
|
|
3544
|
+
]
|
|
3545
|
+
]){
|
|
3546
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3547
|
+
tool.input_schema = schema;
|
|
3548
|
+
})), /benchmark environment bridge configuration unavailable/, `${label} fails closed`);
|
|
3549
|
+
}
|
|
3550
|
+
assert.equal(bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3551
|
+
tool.input_schema = nestedInputSchema(8);
|
|
3552
|
+
})), true, 'schema depth eight is accepted');
|
|
3553
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3554
|
+
tool.input_schema = nestedInputSchema(9);
|
|
3555
|
+
})), /benchmark environment bridge configuration unavailable/, 'schema depth nine is rejected');
|
|
3556
|
+
assert.equal(bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3557
|
+
tool.input_schema = flatInputSchema(255);
|
|
3558
|
+
})), true, 'the 256-node total boundary is accepted');
|
|
3559
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3560
|
+
tool.input_schema = flatInputSchema(256);
|
|
3561
|
+
})), /benchmark environment bridge configuration unavailable/, 'the 256-node total boundary plus one is rejected');
|
|
3562
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3563
|
+
tool.input_schema = flatInputSchema(257);
|
|
3564
|
+
})), /benchmark environment bridge configuration unavailable/, 'the property total bound rejects 257 properties');
|
|
3565
|
+
assert.equal(bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3566
|
+
tool.input_schema = totalEnumInputSchema(4);
|
|
3567
|
+
})), true, 'the 256-enum-value total boundary is accepted');
|
|
3568
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3569
|
+
tool.input_schema = totalEnumInputSchema(5);
|
|
3570
|
+
})), /benchmark environment bridge configuration unavailable/, 'the enum-value total bound rejects values above 256');
|
|
3571
|
+
assert.equal(bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3572
|
+
tool.input_schema = byteSizedInputSchema(16 * 1024);
|
|
3573
|
+
})), true, 'the 16 KiB schema boundary is accepted');
|
|
3574
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3575
|
+
tool.input_schema = byteSizedInputSchema(16 * 1024 + 1);
|
|
3576
|
+
})), /benchmark environment bridge configuration unavailable/, 'the schema byte boundary plus one is rejected');
|
|
2360
3577
|
assert.throws(()=>bridgeRegistrationFor({
|
|
2361
3578
|
...validConfig,
|
|
2362
3579
|
argv_prefix: [
|
|
2363
3580
|
'agent\n--unsafe'
|
|
2364
3581
|
]
|
|
2365
3582
|
}), /benchmark environment bridge configuration unavailable/, 'argv control separator fails hard');
|
|
2366
|
-
assert.throws(()=>bridgeRegistrationFor({
|
|
2367
|
-
...validConfig,
|
|
2368
|
-
allowed_tools: [
|
|
2369
|
-
'lookup;rm'
|
|
2370
|
-
]
|
|
2371
|
-
}), /benchmark environment bridge configuration unavailable/, 'shell metacharacter tool identifier fails hard');
|
|
2372
3583
|
assert.throws(()=>bridgeRegistrationFor({
|
|
2373
3584
|
...validConfig,
|
|
2374
3585
|
isolation: {
|
|
@@ -2418,20 +3629,20 @@ try {
|
|
|
2418
3629
|
writeFileSync(widePath, JSON.stringify(validConfig));
|
|
2419
3630
|
chmodSync(widePath, 0o666);
|
|
2420
3631
|
assert.throws(()=>loadConfigRegistration(widePath, root), /benchmark environment bridge configuration unavailable/, 'group/world writable config fails hard');
|
|
2421
|
-
const
|
|
2422
|
-
writeFileSync(
|
|
2423
|
-
const
|
|
2424
|
-
const
|
|
3632
|
+
const strictConfigPath = join(root, 'strict-bridge.json');
|
|
3633
|
+
writeFileSync(strictConfigPath, JSON.stringify(validConfig));
|
|
3634
|
+
const strictTools = [];
|
|
3635
|
+
const strictCtx = {
|
|
2425
3636
|
tools: {
|
|
2426
3637
|
register (tool) {
|
|
2427
|
-
|
|
3638
|
+
strictTools.push(tool);
|
|
2428
3639
|
return ()=>{};
|
|
2429
3640
|
},
|
|
2430
3641
|
get (name) {
|
|
2431
|
-
return
|
|
3642
|
+
return strictTools.find((tool)=>tool.name === name);
|
|
2432
3643
|
},
|
|
2433
3644
|
schemas () {
|
|
2434
|
-
return
|
|
3645
|
+
return strictTools.map((tool)=>({
|
|
2435
3646
|
name: tool.name
|
|
2436
3647
|
}));
|
|
2437
3648
|
}
|
|
@@ -2456,31 +3667,29 @@ try {
|
|
|
2456
3667
|
},
|
|
2457
3668
|
subprocess: {
|
|
2458
3669
|
spawn: (_spec)=>fakeHandle({
|
|
2459
|
-
stdout:
|
|
3670
|
+
stdout: okEnvelope({
|
|
3671
|
+
city: 'Dubai'
|
|
3672
|
+
})
|
|
2460
3673
|
})
|
|
2461
3674
|
}
|
|
2462
3675
|
};
|
|
2463
|
-
apply(
|
|
3676
|
+
apply(strictCtx, {
|
|
2464
3677
|
stateRoot: root,
|
|
2465
3678
|
timeoutMs: 20,
|
|
2466
3679
|
hbcliBin: '',
|
|
2467
3680
|
sessionAccess: 'off',
|
|
2468
|
-
benchmarkEnvironmentConfigPath:
|
|
3681
|
+
benchmarkEnvironmentConfigPath: strictConfigPath
|
|
2469
3682
|
});
|
|
2470
|
-
const
|
|
2471
|
-
assert.ok(
|
|
2472
|
-
assert.deepEqual(await
|
|
2473
|
-
|
|
2474
|
-
|
|
2475
|
-
arguments: {
|
|
2476
|
-
city: 'Dubai'
|
|
2477
|
-
}
|
|
2478
|
-
}, null), {
|
|
3683
|
+
const strictBridge = strictTools.find((tool)=>tool.name === 'gotry_benchmark_environment');
|
|
3684
|
+
assert.ok(strictBridge?.execute, 'strict v3 config registers the bridge');
|
|
3685
|
+
assert.deepEqual(await strictBridge.execute(bridgeCall('lookup', {
|
|
3686
|
+
city: 'Dubai'
|
|
3687
|
+
}), null), {
|
|
2479
3688
|
ok: true,
|
|
2480
3689
|
result: {
|
|
2481
3690
|
city: 'Dubai'
|
|
2482
3691
|
}
|
|
2483
|
-
}, '
|
|
3692
|
+
}, 'strict v3 result envelope executes safe structured output');
|
|
2484
3693
|
console.log('BENCHMARK ENVIRONMENT BRIDGE TESTS: registration + TDD bridge contract assertions');
|
|
2485
3694
|
} catch (error) {
|
|
2486
3695
|
console.error(error);
|