@danceiny/gotry 0.0.1-rc.19 → 0.0.1-rc.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/README.zh-CN.md +1 -1
- package/bin/gotry-bootstrap.js +83 -37
- package/bin/gotry-inner.js +11 -6
- package/cordis.gotry-patch.yml +7 -3
- package/dist/capabilities/anything.js +62 -29
- package/dist/capabilities/doctor.js +25 -3
- package/dist/capabilities/hbcli.js +28 -2
- package/dist/scripts/agent-planning-turn-deadline-e2e.js +2 -1
- package/dist/scripts/anything-tests.js +70 -21
- package/dist/scripts/benchmark-environment-bridge-e2e.js +136 -29
- package/dist/scripts/benchmark-environment-bridge-tests.js +1056 -269
- package/dist/scripts/bootstrap-tests.js +28 -2
- package/dist/scripts/doctor-tests.js +55 -1
- package/dist/scripts/hbcli-tests.js +38 -1
- package/dist/src/benchmark-agent-conformance.js +32 -12
- package/dist/src/benchmark-environment-bridge.js +317 -114
- package/dist/src/index.js +5 -2
- package/extension/manifest.json +2 -2
- package/package.json +2 -1
- package/ts/capabilities/anything.ts +89 -38
- package/ts/capabilities/hbcli.ts +53 -4
- package/ts/dsh-runtime/vendor/README.md +60 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/LICENSE +21 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/README.en.md +200 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/README.md +202 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/client/client.js +216 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/cordis.patch.yml +5 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/clients/amap.js +371 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/clients/nominatim.js +63 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/clients/osrm.js +56 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/clients/photon.js +55 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/config-file.js +87 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/config-route.js +126 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/config.js +16 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/index.js +109 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/settings-ns.js +27 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/tools/geocode.js +127 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/tools/poi.js +84 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/tools/routes.js +170 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/clients/amap.d.ts +53 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/clients/nominatim.d.ts +17 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/clients/osrm.d.ts +15 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/clients/photon.d.ts +22 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/config-file.d.ts +33 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/config-route.d.ts +20 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/config.d.ts +23 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/index.d.ts +18 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/settings-ns.d.ts +14 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/tools/geocode.d.ts +11 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/tools/poi.d.ts +9 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/tools/routes.d.ts +16 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/types.d.ts +55 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types.js +16 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/package.json +91 -0
- package/ts/package.json +9 -1
- package/ts/src/benchmark-agent-conformance.ts +40 -9
- package/ts/src/benchmark-environment-bridge.ts +235 -66
- package/ts/src/index.ts +4 -2
- package/ts/src/turn-deadline.ts +4 -0
|
@@ -4,7 +4,7 @@ import { tmpdir } from 'node:os';
|
|
|
4
4
|
import { join } from 'node:path';
|
|
5
5
|
import { Context } from '@deepseek-ai/cordis';
|
|
6
6
|
import { apply } from '../src/index.js';
|
|
7
|
-
import { registerBenchmarkEnvironmentBridge } from '../src/benchmark-environment-bridge.js';
|
|
7
|
+
import { BENCHMARK_TOOL_RESULT_SCHEMA_VERSION, registerBenchmarkEnvironmentBridge } from '../src/benchmark-environment-bridge.js';
|
|
8
8
|
import { installBenchmarkToolIsolation } from '../src/benchmark-tool-isolation.js';
|
|
9
9
|
import { BENCHMARK_BRIDGE_CALL_FAILED, BENCHMARK_BRIDGE_CALL_REQUIRED, BENCHMARK_BRIDGE_RETRY_CALL_NOT_ALLOWED, BENCHMARK_BRIDGE_OUTPUT_TRUNCATED, BENCHMARK_BRIDGE_RUNNER_FAILED, BENCHMARK_BRIDGE_SPAWN_FAILED, BENCHMARK_BRIDGE_TIMED_OUT, BENCHMARK_CONFORMANCE_STATE_UNAVAILABLE, BENCHMARK_TERMINAL_INVALID, MAX_CONFORMANCE_RETRIES, benchmarkChildFailureForConformanceCode, createBenchmarkAgentConformance, installBenchmarkAgentConformance, parseBenchmarkTerminal, validateTerminalOutputConfig } from '../src/benchmark-agent-conformance.js';
|
|
10
10
|
import { BENCHMARK_CHILD_DIAGNOSTIC_MAX_BYTES, BENCHMARK_CHILD_DIAGNOSTIC_SCHEMA, appendBoundedChildDiagnostic, classifyBenchmarkChildFailure, classifyBenchmarkTurnEnd, createBenchmarkDiagnosticArbiter, parseBenchmarkChildDiagnostic } from '../src/benchmark-headless-child-diagnostics.js';
|
|
@@ -375,7 +375,7 @@ function toolCall(callId = 'call-1', options = {}) {
|
|
|
375
375
|
};
|
|
376
376
|
}
|
|
377
377
|
function toolResult(callId = 'call-1', options = {}) {
|
|
378
|
-
const { turn = 1, step = 1, ok = true, isError = false, error = 'runner_failed' } = options;
|
|
378
|
+
const { turn = 1, step = 1, ok = true, isError = false, error = 'runner_failed', outcome, payload } = options;
|
|
379
379
|
return {
|
|
380
380
|
type: 'tool/result',
|
|
381
381
|
data: {
|
|
@@ -394,13 +394,16 @@ function toolResult(callId = 'call-1', options = {}) {
|
|
|
394
394
|
content: [
|
|
395
395
|
{
|
|
396
396
|
type: 'text',
|
|
397
|
-
text: JSON.stringify(
|
|
397
|
+
text: JSON.stringify(payload ?? (outcome ? {
|
|
398
|
+
ok: true,
|
|
399
|
+
outcome
|
|
400
|
+
} : ok ? {
|
|
398
401
|
ok: true,
|
|
399
402
|
result: {}
|
|
400
403
|
} : {
|
|
401
404
|
ok: false,
|
|
402
405
|
error
|
|
403
|
-
})
|
|
406
|
+
}))
|
|
404
407
|
}
|
|
405
408
|
]
|
|
406
409
|
}
|
|
@@ -409,7 +412,107 @@ function toolResult(callId = 'call-1', options = {}) {
|
|
|
409
412
|
}
|
|
410
413
|
};
|
|
411
414
|
}
|
|
412
|
-
|
|
415
|
+
{
|
|
416
|
+
const state = createBenchmarkAgentConformance(projection);
|
|
417
|
+
state.observe(turnStart());
|
|
418
|
+
state.observe(toolCall());
|
|
419
|
+
state.observe(toolResult('call-1', {
|
|
420
|
+
outcome: {
|
|
421
|
+
schema_version: 'gotry_benchmark_tool_result_v1',
|
|
422
|
+
status: 'miss',
|
|
423
|
+
code: 'NOT_FOUND',
|
|
424
|
+
recovery: 'revise_arguments'
|
|
425
|
+
}
|
|
426
|
+
}));
|
|
427
|
+
assert.deepEqual(state.stopping(1), {
|
|
428
|
+
kind: 'steer',
|
|
429
|
+
mode: 'terminal'
|
|
430
|
+
}, 'domain outcome alone permits tagged terminal');
|
|
431
|
+
state.observe(assistant('<done>{"status":"miss"}</done>', {
|
|
432
|
+
step: 2
|
|
433
|
+
}));
|
|
434
|
+
assert.deepEqual(state.stopping(1), {
|
|
435
|
+
kind: 'accept'
|
|
436
|
+
}, 'domain-only tagged terminal is accepted');
|
|
437
|
+
}{
|
|
438
|
+
const state = createBenchmarkAgentConformance(projection);
|
|
439
|
+
state.observe(turnStart());
|
|
440
|
+
state.observe(toolCall());
|
|
441
|
+
state.observe(toolResult('call-1', {
|
|
442
|
+
outcome: {
|
|
443
|
+
schema_version: 'gotry_benchmark_tool_result_v1',
|
|
444
|
+
status: 'miss',
|
|
445
|
+
code: 'NOT_FOUND',
|
|
446
|
+
recovery: 'revise_arguments'
|
|
447
|
+
}
|
|
448
|
+
}));
|
|
449
|
+
state.observe(toolCall('call-2', {
|
|
450
|
+
step: 2
|
|
451
|
+
}));
|
|
452
|
+
state.observe(toolResult('call-2'));
|
|
453
|
+
state.observe(assistant('<done>{"status":"succeeded"}</done>', {
|
|
454
|
+
step: 3
|
|
455
|
+
}));
|
|
456
|
+
assert.deepEqual(state.stopping(1), {
|
|
457
|
+
kind: 'accept'
|
|
458
|
+
}, 'domain outcome followed by concrete result accepts');
|
|
459
|
+
}{
|
|
460
|
+
const state = createBenchmarkAgentConformance(projection);
|
|
461
|
+
state.observe(turnStart());
|
|
462
|
+
state.observe(toolCall());
|
|
463
|
+
state.observe(toolResult('call-1', {
|
|
464
|
+
outcome: {
|
|
465
|
+
schema_version: 'gotry_benchmark_tool_result_v1',
|
|
466
|
+
status: 'miss',
|
|
467
|
+
code: 'NOT_FOUND',
|
|
468
|
+
recovery: 'revise_arguments'
|
|
469
|
+
}
|
|
470
|
+
}));
|
|
471
|
+
state.observe(toolCall('call-2', {
|
|
472
|
+
step: 2
|
|
473
|
+
}));
|
|
474
|
+
state.observe(toolResult('call-2', {
|
|
475
|
+
ok: false,
|
|
476
|
+
error: 'runner_failed'
|
|
477
|
+
}));
|
|
478
|
+
assert.deepEqual(state.stopping(1), {
|
|
479
|
+
kind: 'reject',
|
|
480
|
+
code: BENCHMARK_BRIDGE_RUNNER_FAILED
|
|
481
|
+
}, 'infra failure overrides domain-only outcome');
|
|
482
|
+
}{
|
|
483
|
+
const state = createBenchmarkAgentConformance(projection);
|
|
484
|
+
state.observe(turnStart());
|
|
485
|
+
state.observe(toolCall());
|
|
486
|
+
state.observe(toolResult('call-1', {
|
|
487
|
+
payload: {
|
|
488
|
+
ok: true,
|
|
489
|
+
result: {},
|
|
490
|
+
outcome: {
|
|
491
|
+
status: 'miss'
|
|
492
|
+
}
|
|
493
|
+
}
|
|
494
|
+
}));
|
|
495
|
+
state.observe(assistant('<done>{"status":"succeeded"}</done>'));
|
|
496
|
+
assert.deepEqual(state.stopping(1), {
|
|
497
|
+
kind: 'reject',
|
|
498
|
+
code: BENCHMARK_BRIDGE_CALL_FAILED
|
|
499
|
+
}, 'ambiguous ok wrapper with result and outcome fails closed');
|
|
500
|
+
}{
|
|
501
|
+
const state = createBenchmarkAgentConformance(projection);
|
|
502
|
+
state.observe(turnStart());
|
|
503
|
+
state.observe(toolCall());
|
|
504
|
+
state.observe(toolResult('call-1', {
|
|
505
|
+
payload: {
|
|
506
|
+
ok: true,
|
|
507
|
+
outcome: {}
|
|
508
|
+
}
|
|
509
|
+
}));
|
|
510
|
+
state.observe(assistant('<done>{"status":"miss"}</done>'));
|
|
511
|
+
assert.deepEqual(state.stopping(1), {
|
|
512
|
+
kind: 'reject',
|
|
513
|
+
code: BENCHMARK_BRIDGE_CALL_FAILED
|
|
514
|
+
}, 'malformed inner domain outcome fails closed');
|
|
515
|
+
}function assistant(text, options = {}) {
|
|
413
516
|
const { turn = 1, step = 2, interrupted = false } = options;
|
|
414
517
|
return {
|
|
415
518
|
type: 'assistant/message',
|
|
@@ -588,6 +691,69 @@ function assistant(text, options = {}) {
|
|
|
588
691
|
assert.deepEqual(state.stopping(1), {
|
|
589
692
|
kind: 'accept'
|
|
590
693
|
}, 'a later failed optional call does not erase an already paired successful result');
|
|
694
|
+
}{
|
|
695
|
+
const state = createBenchmarkAgentConformance(projection);
|
|
696
|
+
state.observe(turnStart());
|
|
697
|
+
state.observe(toolCall('successful', {
|
|
698
|
+
step: 1
|
|
699
|
+
}));
|
|
700
|
+
state.observe(toolResult('successful', {
|
|
701
|
+
step: 1
|
|
702
|
+
}));
|
|
703
|
+
state.observe(assistant('<done>{"status":"succeeded"}</done>', {
|
|
704
|
+
step: 2
|
|
705
|
+
}));
|
|
706
|
+
state.observe(toolCall('domain', {
|
|
707
|
+
step: 3
|
|
708
|
+
}));
|
|
709
|
+
state.observe(toolResult('domain', {
|
|
710
|
+
step: 3,
|
|
711
|
+
outcome: {
|
|
712
|
+
schema_version: 'gotry_benchmark_tool_result_v1',
|
|
713
|
+
status: 'miss',
|
|
714
|
+
code: 'NOT_FOUND',
|
|
715
|
+
recovery: 'revise_arguments'
|
|
716
|
+
}
|
|
717
|
+
}));
|
|
718
|
+
assert.deepEqual(state.stopping(1), {
|
|
719
|
+
kind: 'steer',
|
|
720
|
+
mode: 'terminal'
|
|
721
|
+
}, 'a terminal before the latest domain response is stale');
|
|
722
|
+
state.observe(assistant('<done>{"status":"succeeded"}</done>', {
|
|
723
|
+
step: 4
|
|
724
|
+
}));
|
|
725
|
+
assert.deepEqual(state.stopping(1), {
|
|
726
|
+
kind: 'accept'
|
|
727
|
+
}, 'a fresh terminal after the latest domain response may reuse the earlier concrete result');
|
|
728
|
+
}{
|
|
729
|
+
const state = createBenchmarkAgentConformance(projection);
|
|
730
|
+
state.observe(turnStart());
|
|
731
|
+
state.observe(toolCall('successful', {
|
|
732
|
+
step: 1
|
|
733
|
+
}));
|
|
734
|
+
state.observe(toolResult('successful', {
|
|
735
|
+
step: 1
|
|
736
|
+
}));
|
|
737
|
+
state.observe(assistant('<done>{"status":"succeeded"}</done>', {
|
|
738
|
+
step: 2
|
|
739
|
+
}));
|
|
740
|
+
state.observe(toolCall('failed', {
|
|
741
|
+
step: 3
|
|
742
|
+
}));
|
|
743
|
+
state.observe(toolResult('failed', {
|
|
744
|
+
step: 3,
|
|
745
|
+
ok: false
|
|
746
|
+
}));
|
|
747
|
+
assert.deepEqual(state.stopping(1), {
|
|
748
|
+
kind: 'steer',
|
|
749
|
+
mode: 'terminal'
|
|
750
|
+
}, 'a terminal before a later optional failure is stale');
|
|
751
|
+
state.observe(assistant('<done>{"status":"succeeded"}</done>', {
|
|
752
|
+
step: 4
|
|
753
|
+
}));
|
|
754
|
+
assert.deepEqual(state.stopping(1), {
|
|
755
|
+
kind: 'accept'
|
|
756
|
+
}, 'a fresh terminal may still converge after an optional failure when a concrete result exists');
|
|
591
757
|
}{
|
|
592
758
|
const state = createBenchmarkAgentConformance(projection);
|
|
593
759
|
state.observe(turnStart());
|
|
@@ -838,16 +1004,77 @@ function assistant(text, options = {}) {
|
|
|
838
1004
|
waitForExit: async ()=>true
|
|
839
1005
|
};
|
|
840
1006
|
}
|
|
1007
|
+
function okEnvelope(result) {
|
|
1008
|
+
return JSON.stringify({
|
|
1009
|
+
schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION,
|
|
1010
|
+
status: 'ok',
|
|
1011
|
+
result
|
|
1012
|
+
});
|
|
1013
|
+
}
|
|
1014
|
+
function lookupInputSchema() {
|
|
1015
|
+
return {
|
|
1016
|
+
type: 'object',
|
|
1017
|
+
properties: {
|
|
1018
|
+
city: {
|
|
1019
|
+
type: 'string',
|
|
1020
|
+
enum: [
|
|
1021
|
+
'Dubai',
|
|
1022
|
+
'Abu Dhabi'
|
|
1023
|
+
],
|
|
1024
|
+
description: 'Declared city name.'
|
|
1025
|
+
},
|
|
1026
|
+
payload: {
|
|
1027
|
+
type: 'string'
|
|
1028
|
+
},
|
|
1029
|
+
notes: {
|
|
1030
|
+
type: 'string'
|
|
1031
|
+
},
|
|
1032
|
+
executable: {
|
|
1033
|
+
type: 'string'
|
|
1034
|
+
},
|
|
1035
|
+
cwd: {
|
|
1036
|
+
type: 'string'
|
|
1037
|
+
},
|
|
1038
|
+
argv: {
|
|
1039
|
+
type: 'array',
|
|
1040
|
+
items: {
|
|
1041
|
+
type: 'string'
|
|
1042
|
+
}
|
|
1043
|
+
}
|
|
1044
|
+
},
|
|
1045
|
+
required: [
|
|
1046
|
+
'city'
|
|
1047
|
+
],
|
|
1048
|
+
additionalProperties: false
|
|
1049
|
+
};
|
|
1050
|
+
}
|
|
1051
|
+
function bridgeCall(tool, argumentsValue) {
|
|
1052
|
+
return {
|
|
1053
|
+
action: 'call',
|
|
1054
|
+
tool,
|
|
1055
|
+
arguments: argumentsValue
|
|
1056
|
+
};
|
|
1057
|
+
}
|
|
841
1058
|
async function assertRealCordisWaterfallOrdering() {
|
|
842
1059
|
const ctx = new Context();
|
|
843
1060
|
const bridge = {
|
|
844
1061
|
name: 'gotry_benchmark_environment',
|
|
845
1062
|
description: 'benchmark bridge',
|
|
846
1063
|
parameters: {
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
1064
|
+
oneOf: [
|
|
1065
|
+
{
|
|
1066
|
+
type: 'object',
|
|
1067
|
+
properties: {
|
|
1068
|
+
action: {
|
|
1069
|
+
const: 'tools'
|
|
1070
|
+
}
|
|
1071
|
+
},
|
|
1072
|
+
required: [
|
|
1073
|
+
'action'
|
|
1074
|
+
],
|
|
1075
|
+
additionalProperties: false
|
|
1076
|
+
}
|
|
1077
|
+
]
|
|
851
1078
|
}
|
|
852
1079
|
};
|
|
853
1080
|
const exactSchema = structuredClone(bridge);
|
|
@@ -979,7 +1206,7 @@ try {
|
|
|
979
1206
|
assert.equal(parseBenchmarkChildDiagnostic(boundedControl.toString('utf8')), 'child_bridge_timed_out', 'rolling tail preserves a final structured reason after bounded noise');
|
|
980
1207
|
const configPath = join(root, 'bridge.json');
|
|
981
1208
|
writeFileSync(configPath, JSON.stringify({
|
|
982
|
-
schema_version: '
|
|
1209
|
+
schema_version: 'gotry_benchmark_environment_bridge_v3',
|
|
983
1210
|
enabled: true,
|
|
984
1211
|
executable: process.execPath,
|
|
985
1212
|
cwd: root,
|
|
@@ -989,20 +1216,54 @@ try {
|
|
|
989
1216
|
'--lang',
|
|
990
1217
|
'en'
|
|
991
1218
|
],
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
1219
|
+
tools: [
|
|
1220
|
+
{
|
|
1221
|
+
name: 'lookup',
|
|
1222
|
+
description: 'Lookup one declared city.',
|
|
1223
|
+
input_schema: lookupInputSchema(),
|
|
1224
|
+
output_keys: [
|
|
1225
|
+
'city',
|
|
1226
|
+
'nested'
|
|
1227
|
+
],
|
|
1228
|
+
domain_outcomes: [
|
|
1229
|
+
{
|
|
1230
|
+
status: 'miss',
|
|
1231
|
+
code: 'NOT_FOUND',
|
|
1232
|
+
recovery: 'revise_arguments'
|
|
1233
|
+
},
|
|
1234
|
+
{
|
|
1235
|
+
status: 'error',
|
|
1236
|
+
code: 'AMBIGUOUS',
|
|
1237
|
+
recovery: 'choose_alternative'
|
|
1238
|
+
}
|
|
1239
|
+
]
|
|
1240
|
+
},
|
|
1241
|
+
{
|
|
1242
|
+
name: 'constructor',
|
|
1243
|
+
description: 'Construct one declared city.',
|
|
1244
|
+
input_schema: lookupInputSchema(),
|
|
1245
|
+
output_keys: [
|
|
1246
|
+
'legacy'
|
|
1247
|
+
],
|
|
1248
|
+
domain_outcomes: [
|
|
1249
|
+
{
|
|
1250
|
+
status: 'error',
|
|
1251
|
+
code: 'INVALID',
|
|
1252
|
+
recovery: 'retry_same'
|
|
1253
|
+
}
|
|
1254
|
+
]
|
|
1255
|
+
},
|
|
1256
|
+
{
|
|
1257
|
+
name: 'toString',
|
|
1258
|
+
description: 'Stringify one declared city.',
|
|
1259
|
+
input_schema: lookupInputSchema(),
|
|
1260
|
+
output_keys: [
|
|
1261
|
+
'city',
|
|
1262
|
+
'nested'
|
|
1263
|
+
],
|
|
1264
|
+
domain_outcomes: []
|
|
1265
|
+
}
|
|
996
1266
|
],
|
|
997
|
-
allowed_output_keys: {
|
|
998
|
-
lookup: [
|
|
999
|
-
'city',
|
|
1000
|
-
'nested'
|
|
1001
|
-
],
|
|
1002
|
-
constructor: [
|
|
1003
|
-
'legacy'
|
|
1004
|
-
]
|
|
1005
|
-
},
|
|
1006
1267
|
timeout_ms: 20,
|
|
1007
1268
|
max_output_bytes: 4_096,
|
|
1008
1269
|
terminal_output: {
|
|
@@ -1048,7 +1309,11 @@ try {
|
|
|
1048
1309
|
let timeoutSignal;
|
|
1049
1310
|
const outcomes = [
|
|
1050
1311
|
{
|
|
1051
|
-
stdout:
|
|
1312
|
+
stdout: okEnvelope([
|
|
1313
|
+
{
|
|
1314
|
+
city: 'Dubai'
|
|
1315
|
+
}
|
|
1316
|
+
])
|
|
1052
1317
|
}
|
|
1053
1318
|
];
|
|
1054
1319
|
process.env.GOTRY_BENCHMARK_BRIDGE_PARENT_SENTINEL = 'must-not-cross-boundary';
|
|
@@ -1108,7 +1373,7 @@ try {
|
|
|
1108
1373
|
spawn (spec) {
|
|
1109
1374
|
spawnSpecs.push(spec);
|
|
1110
1375
|
const outcome = outcomes.shift() ?? {
|
|
1111
|
-
stdout:
|
|
1376
|
+
stdout: okEnvelope({})
|
|
1112
1377
|
};
|
|
1113
1378
|
if (outcome.spawnError) throw new Error('fake spawn failed');
|
|
1114
1379
|
if (outcome.waitForAbort) {
|
|
@@ -1625,15 +1890,57 @@ try {
|
|
|
1625
1890
|
}, 'agent disposal releases its quarantined isolation and conformance effects');
|
|
1626
1891
|
const bridge = registered.find((tool)=>tool.name === 'gotry_benchmark_environment');
|
|
1627
1892
|
assert.ok(bridge.execute, 'registered bridge exposes execute');
|
|
1893
|
+
const parameters = bridge.parameters;
|
|
1894
|
+
assert.equal(Array.isArray(parameters.oneOf), true, 'v3 bridge exposes flat descriptor-derived model branches');
|
|
1895
|
+
const lookupBranch = parameters.oneOf.find((branch)=>branch?.properties?.tool?.const === 'lookup');
|
|
1896
|
+
assert.ok(lookupBranch, 'model schema contains the configured lookup tool branch');
|
|
1897
|
+
assert.equal(lookupBranch.description, 'Lookup one declared city.', 'descriptor description reaches the model-visible call branch');
|
|
1898
|
+
assert.deepEqual(lookupBranch.required, [
|
|
1899
|
+
'action',
|
|
1900
|
+
'tool',
|
|
1901
|
+
'arguments'
|
|
1902
|
+
]);
|
|
1903
|
+
assert.equal(lookupBranch.additionalProperties, false);
|
|
1904
|
+
assert.deepEqual(lookupBranch.properties.arguments.required, [
|
|
1905
|
+
'city'
|
|
1906
|
+
]);
|
|
1907
|
+
assert.equal(lookupBranch.properties.arguments.additionalProperties, false);
|
|
1908
|
+
assert.equal(lookupBranch.properties.arguments.properties.city.type, 'string');
|
|
1909
|
+
assert.deepEqual(lookupBranch.properties.arguments.properties.city.enum, [
|
|
1910
|
+
'Dubai',
|
|
1911
|
+
'Abu Dhabi'
|
|
1912
|
+
]);
|
|
1913
|
+
assert.equal(Object.isFrozen(bridge.parameters), true, 'registered raw parameters are frozen');
|
|
1914
|
+
assert.equal(Object.isFrozen(lookupBranch.properties.arguments.properties.city.enum), true, 'descriptor-derived nested schema is frozen');
|
|
1915
|
+
assert.throws(()=>{
|
|
1916
|
+
lookupBranch.properties.arguments.properties.city.enum.push('escape');
|
|
1917
|
+
}, TypeError);
|
|
1918
|
+
const beforeProtocolRejected = spawnSpecs.length;
|
|
1919
|
+
await assert.rejects(()=>bridge.execute({
|
|
1920
|
+
query: bridgeCall('lookup', {
|
|
1921
|
+
city: 'Dubai'
|
|
1922
|
+
})
|
|
1923
|
+
}, null), /invalid arguments/, 'legacy nested query envelope is rejected by the flat protocol');
|
|
1924
|
+
await assert.rejects(()=>bridge.execute({}, null), /invalid arguments/, 'empty object is rejected by the flat protocol');
|
|
1925
|
+
await assert.rejects(()=>bridge.execute({
|
|
1926
|
+
action: 'tools',
|
|
1927
|
+
tool: 'lookup'
|
|
1928
|
+
}, null), /invalid arguments/, 'mixed action/tool fields are rejected by the flat protocol');
|
|
1929
|
+
await assert.rejects(()=>bridge.execute({
|
|
1930
|
+
...bridgeCall('lookup', {
|
|
1931
|
+
city: 'Dubai'
|
|
1932
|
+
}),
|
|
1933
|
+
extra: true
|
|
1934
|
+
}, null), /invalid arguments/, 'extra top-level fields are rejected by the flat protocol');
|
|
1935
|
+
await assert.rejects(()=>bridge.execute(bridgeCall('lookup', {
|
|
1936
|
+
city: 'Sharjah'
|
|
1937
|
+
}), null), /invalid arguments/, 'invalid descriptor argument values are rejected by the flat protocol');
|
|
1938
|
+
assert.equal(spawnSpecs.length, beforeProtocolRejected, 'flat protocol rejections happen before spawn');
|
|
1628
1939
|
const args = {
|
|
1629
1940
|
city: 'Dubai',
|
|
1630
1941
|
payload: '$(touch /tmp/nope)'
|
|
1631
1942
|
};
|
|
1632
|
-
const result = await bridge.execute(
|
|
1633
|
-
action: 'call',
|
|
1634
|
-
tool: 'lookup',
|
|
1635
|
-
arguments: args
|
|
1636
|
-
}, null);
|
|
1943
|
+
const result = await bridge.execute(bridgeCall('lookup', args), null);
|
|
1637
1944
|
assert.deepEqual(spawnSpecs[0]?.argv, [
|
|
1638
1945
|
process.execPath,
|
|
1639
1946
|
'-m',
|
|
@@ -1670,15 +1977,13 @@ try {
|
|
|
1670
1977
|
]
|
|
1671
1978
|
}, 'one-line JSON stdout becomes structured result');
|
|
1672
1979
|
outcomes.push({
|
|
1673
|
-
stdout:
|
|
1674
|
-
});
|
|
1675
|
-
const unmappedResult = await bridge.execute({
|
|
1676
|
-
action: 'call',
|
|
1677
|
-
tool: 'toString',
|
|
1678
|
-
arguments: {
|
|
1980
|
+
stdout: okEnvelope({
|
|
1679
1981
|
city: 'Dubai'
|
|
1680
|
-
}
|
|
1681
|
-
}
|
|
1982
|
+
})
|
|
1983
|
+
});
|
|
1984
|
+
const unmappedResult = await bridge.execute(bridgeCall('toString', {
|
|
1985
|
+
city: 'Dubai'
|
|
1986
|
+
}), null);
|
|
1682
1987
|
assert.deepEqual(unmappedResult, {
|
|
1683
1988
|
ok: true,
|
|
1684
1989
|
result: {
|
|
@@ -1686,15 +1991,13 @@ try {
|
|
|
1686
1991
|
}
|
|
1687
1992
|
}, 'allowed tool without a positive mapping retains the recursive denylist only');
|
|
1688
1993
|
outcomes.push({
|
|
1689
|
-
stdout:
|
|
1994
|
+
stdout: okEnvelope({
|
|
1995
|
+
legacy: 'value'
|
|
1996
|
+
})
|
|
1690
1997
|
});
|
|
1691
|
-
const legacyResult = await bridge.execute({
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
arguments: {
|
|
1695
|
-
city: 'Dubai'
|
|
1696
|
-
}
|
|
1697
|
-
}, null);
|
|
1998
|
+
const legacyResult = await bridge.execute(bridgeCall('constructor', {
|
|
1999
|
+
city: 'Dubai'
|
|
2000
|
+
}), null);
|
|
1698
2001
|
assert.deepEqual(legacyResult, {
|
|
1699
2002
|
ok: true,
|
|
1700
2003
|
result: {
|
|
@@ -1702,29 +2005,27 @@ try {
|
|
|
1702
2005
|
}
|
|
1703
2006
|
}, 'mapped constructor accepts its declared positive key');
|
|
1704
2007
|
outcomes.push({
|
|
1705
|
-
stdout:
|
|
1706
|
-
});
|
|
1707
|
-
const constructorUnexpected = await bridge.execute({
|
|
1708
|
-
action: 'call',
|
|
1709
|
-
tool: 'constructor',
|
|
1710
|
-
arguments: {
|
|
2008
|
+
stdout: okEnvelope({
|
|
1711
2009
|
city: 'Dubai'
|
|
1712
|
-
}
|
|
1713
|
-
}
|
|
2010
|
+
})
|
|
2011
|
+
});
|
|
2012
|
+
const constructorUnexpected = await bridge.execute(bridgeCall('constructor', {
|
|
2013
|
+
city: 'Dubai'
|
|
2014
|
+
}), null);
|
|
1714
2015
|
assert.deepEqual(constructorUnexpected, {
|
|
1715
2016
|
ok: false,
|
|
1716
2017
|
error: 'forbidden_output'
|
|
1717
2018
|
}, 'mapped constructor rejects undeclared positive keys');
|
|
1718
2019
|
outcomes.push({
|
|
1719
|
-
stdout:
|
|
2020
|
+
stdout: okEnvelope({
|
|
2021
|
+
nested: {
|
|
2022
|
+
city: 'Dubai'
|
|
2023
|
+
}
|
|
2024
|
+
})
|
|
1720
2025
|
});
|
|
1721
|
-
const nestedAllowedResult = await bridge.execute({
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
arguments: {
|
|
1725
|
-
city: 'Dubai'
|
|
1726
|
-
}
|
|
1727
|
-
}, null);
|
|
2026
|
+
const nestedAllowedResult = await bridge.execute(bridgeCall('lookup', {
|
|
2027
|
+
city: 'Dubai'
|
|
2028
|
+
}), null);
|
|
1728
2029
|
assert.deepEqual(nestedAllowedResult, {
|
|
1729
2030
|
ok: true,
|
|
1730
2031
|
result: {
|
|
@@ -1734,29 +2035,31 @@ try {
|
|
|
1734
2035
|
}
|
|
1735
2036
|
}, 'configured positive output allowlist accepts declared nested keys');
|
|
1736
2037
|
outcomes.push({
|
|
1737
|
-
stdout:
|
|
2038
|
+
stdout: okEnvelope({
|
|
2039
|
+
city: 'Dubai',
|
|
2040
|
+
unexpected: 'secret'
|
|
2041
|
+
})
|
|
1738
2042
|
});
|
|
1739
|
-
const unexpectedResult = await bridge.execute({
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
arguments: {
|
|
1743
|
-
city: 'Dubai'
|
|
1744
|
-
}
|
|
1745
|
-
}, null);
|
|
2043
|
+
const unexpectedResult = await bridge.execute(bridgeCall('lookup', {
|
|
2044
|
+
city: 'Dubai'
|
|
2045
|
+
}), null);
|
|
1746
2046
|
assert.deepEqual(unexpectedResult, {
|
|
1747
2047
|
ok: false,
|
|
1748
2048
|
error: 'forbidden_output'
|
|
1749
2049
|
}, 'configured positive output allowlist rejects unexpected keys without reflecting them');
|
|
1750
2050
|
outcomes.push({
|
|
1751
|
-
stdout:
|
|
2051
|
+
stdout: okEnvelope([
|
|
2052
|
+
{
|
|
2053
|
+
city: 'Dubai',
|
|
2054
|
+
nested: {
|
|
2055
|
+
unexpected: 'secret'
|
|
2056
|
+
}
|
|
2057
|
+
}
|
|
2058
|
+
])
|
|
1752
2059
|
});
|
|
1753
|
-
const nestedUnexpectedResult = await bridge.execute({
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
arguments: {
|
|
1757
|
-
city: 'Dubai'
|
|
1758
|
-
}
|
|
1759
|
-
}, null);
|
|
2060
|
+
const nestedUnexpectedResult = await bridge.execute(bridgeCall('lookup', {
|
|
2061
|
+
city: 'Dubai'
|
|
2062
|
+
}), null);
|
|
1760
2063
|
assert.deepEqual(nestedUnexpectedResult, {
|
|
1761
2064
|
ok: false,
|
|
1762
2065
|
error: 'forbidden_output'
|
|
@@ -1781,48 +2084,125 @@ try {
|
|
|
1781
2084
|
'gоld'
|
|
1782
2085
|
]){
|
|
1783
2086
|
outcomes.push({
|
|
1784
|
-
stdout:
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
[forbidden]: 'secret'
|
|
1788
|
-
}
|
|
2087
|
+
stdout: okEnvelope({
|
|
2088
|
+
nested: {
|
|
2089
|
+
[forbidden]: 'secret'
|
|
1789
2090
|
}
|
|
1790
2091
|
})
|
|
1791
2092
|
});
|
|
1792
|
-
const forbiddenResult = await bridge.execute({
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
arguments: {
|
|
1796
|
-
city: 'Dubai'
|
|
1797
|
-
}
|
|
1798
|
-
}, null);
|
|
2093
|
+
const forbiddenResult = await bridge.execute(bridgeCall('lookup', {
|
|
2094
|
+
city: 'Dubai'
|
|
2095
|
+
}), null);
|
|
1799
2096
|
assert.deepEqual(forbiddenResult, {
|
|
1800
2097
|
ok: false,
|
|
1801
2098
|
error: 'forbidden_output'
|
|
1802
2099
|
}, `recursive no-oracle key ${forbidden} is rejected without reflecting its name`);
|
|
1803
2100
|
}
|
|
1804
2101
|
outcomes.push({
|
|
1805
|
-
stdout: '
|
|
2102
|
+
stdout: okEnvelope('the hidden answer')
|
|
1806
2103
|
});
|
|
1807
|
-
const primitiveOutput = await bridge.execute({
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
arguments: {
|
|
1811
|
-
city: 'Dubai'
|
|
1812
|
-
}
|
|
1813
|
-
}, null);
|
|
2104
|
+
const primitiveOutput = await bridge.execute(bridgeCall('lookup', {
|
|
2105
|
+
city: 'Dubai'
|
|
2106
|
+
}), null);
|
|
1814
2107
|
assert.deepEqual(primitiveOutput, {
|
|
1815
2108
|
ok: false,
|
|
1816
2109
|
error: 'invalid_output'
|
|
1817
2110
|
}, 'primitive result strings cannot bypass the structured visible-output boundary');
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
|
|
2111
|
+
outcomes.push({
|
|
2112
|
+
stdout: okEnvelope([
|
|
2113
|
+
'PRIVATE_ARRAY_VALUE'
|
|
2114
|
+
])
|
|
2115
|
+
});
|
|
2116
|
+
const primitiveArrayOutput = await bridge.execute(bridgeCall('lookup', {
|
|
2117
|
+
city: 'Dubai'
|
|
2118
|
+
}), null);
|
|
2119
|
+
assert.deepEqual(primitiveArrayOutput, {
|
|
2120
|
+
ok: false,
|
|
2121
|
+
error: 'forbidden_output'
|
|
2122
|
+
}, 'top-level primitive arrays cannot bypass the positive output-key boundary');
|
|
2123
|
+
assert.equal(JSON.stringify(primitiveArrayOutput).includes('PRIVATE_ARRAY_VALUE'), false, 'rejected primitive array values are not reflected');
|
|
2124
|
+
outcomes.push({
|
|
2125
|
+
stdout: okEnvelope([
|
|
2126
|
+
[
|
|
2127
|
+
"PRIVATE_NESTED_ARRAY_VALUE"
|
|
2128
|
+
]
|
|
2129
|
+
])
|
|
2130
|
+
});
|
|
2131
|
+
const nestedPrimitiveArrayOutput = await bridge.execute(bridgeCall('lookup', {
|
|
2132
|
+
city: 'Dubai'
|
|
2133
|
+
}), null);
|
|
2134
|
+
assert.deepEqual(nestedPrimitiveArrayOutput, {
|
|
2135
|
+
ok: false,
|
|
2136
|
+
error: 'forbidden_output'
|
|
2137
|
+
}, 'nested primitive arrays without a declared-key ancestor fail closed');
|
|
2138
|
+
assert.equal(JSON.stringify(nestedPrimitiveArrayOutput).includes('PRIVATE_NESTED_ARRAY_VALUE'), false, 'rejected nested primitive array values are not reflected');
|
|
2139
|
+
outcomes.push({
|
|
2140
|
+
stdout: okEnvelope([
|
|
2141
|
+
{
|
|
2142
|
+
city: 'Dubai'
|
|
2143
|
+
},
|
|
2144
|
+
'PRIVATE_MIXED_ARRAY_VALUE'
|
|
2145
|
+
])
|
|
2146
|
+
});
|
|
2147
|
+
const mixedArrayOutput = await bridge.execute(bridgeCall('lookup', {
|
|
2148
|
+
city: 'Dubai'
|
|
2149
|
+
}), null);
|
|
2150
|
+
assert.deepEqual(mixedArrayOutput, {
|
|
2151
|
+
ok: false,
|
|
2152
|
+
error: 'forbidden_output'
|
|
2153
|
+
}, 'a valid record cannot mask an unkeyed primitive sibling');
|
|
2154
|
+
assert.equal(JSON.stringify(mixedArrayOutput).includes('PRIVATE_MIXED_ARRAY_VALUE'), false, 'rejected mixed-array primitive values are not reflected');
|
|
2155
|
+
outcomes.push({
|
|
2156
|
+
stdout: okEnvelope([])
|
|
2157
|
+
});
|
|
2158
|
+
const emptyArrayOutput = await bridge.execute(bridgeCall('lookup', {
|
|
2159
|
+
city: 'Dubai'
|
|
2160
|
+
}), null);
|
|
2161
|
+
assert.deepEqual(emptyArrayOutput, {
|
|
2162
|
+
ok: true,
|
|
2163
|
+
result: []
|
|
2164
|
+
}, 'an empty top-level result array is an explicit non-reflecting collection');
|
|
2165
|
+
outcomes.push({
|
|
2166
|
+
stdout: okEnvelope([
|
|
2167
|
+
{
|
|
2168
|
+
city: 'Dubai'
|
|
2169
|
+
}
|
|
2170
|
+
])
|
|
2171
|
+
});
|
|
2172
|
+
const recordArrayOutput = await bridge.execute(bridgeCall('lookup', {
|
|
2173
|
+
city: 'Dubai'
|
|
2174
|
+
}), null);
|
|
2175
|
+
assert.deepEqual(recordArrayOutput, {
|
|
2176
|
+
ok: true,
|
|
2177
|
+
result: [
|
|
2178
|
+
{
|
|
2179
|
+
city: 'Dubai'
|
|
2180
|
+
}
|
|
2181
|
+
]
|
|
2182
|
+
}, 'top-level arrays of records remain valid when every leaf is covered by a declared key');
|
|
2183
|
+
outcomes.push({
|
|
2184
|
+
stdout: okEnvelope({
|
|
2185
|
+
city: [
|
|
2186
|
+
'Dubai'
|
|
2187
|
+
]
|
|
2188
|
+
})
|
|
2189
|
+
});
|
|
2190
|
+
const keyedPrimitiveArrayOutput = await bridge.execute(bridgeCall('lookup', {
|
|
2191
|
+
city: 'Dubai'
|
|
2192
|
+
}), null);
|
|
2193
|
+
assert.deepEqual(keyedPrimitiveArrayOutput, {
|
|
2194
|
+
ok: true,
|
|
2195
|
+
result: {
|
|
2196
|
+
city: [
|
|
2197
|
+
'Dubai'
|
|
2198
|
+
]
|
|
1824
2199
|
}
|
|
1825
|
-
},
|
|
2200
|
+
}, 'primitive arrays remain valid below a declared output key');
|
|
2201
|
+
const beforeOversized = spawnSpecs.length;
|
|
2202
|
+
const oversized = await bridge.execute(bridgeCall('lookup', {
|
|
2203
|
+
city: 'Dubai',
|
|
2204
|
+
notes: 'x'.repeat(65_537)
|
|
2205
|
+
}), null);
|
|
1826
2206
|
assert.deepEqual(oversized, {
|
|
1827
2207
|
ok: false,
|
|
1828
2208
|
error: 'invalid_arguments',
|
|
@@ -1834,30 +2214,20 @@ try {
|
|
|
1834
2214
|
for(let index = 0; index < 13; index++)deep = {
|
|
1835
2215
|
next: deep
|
|
1836
2216
|
};
|
|
1837
|
-
|
|
1838
|
-
action: 'call',
|
|
1839
|
-
tool: 'lookup',
|
|
1840
|
-
arguments: deep
|
|
1841
|
-
}, null);
|
|
1842
|
-
assert.deepEqual(deepResult, {
|
|
1843
|
-
ok: false,
|
|
1844
|
-
error: 'invalid_arguments',
|
|
1845
|
-
reason: 'serialization_limit'
|
|
1846
|
-
});
|
|
1847
|
-
assert.equal(spawnSpecs.length, beforeDeep, 'over-deep arguments are rejected before spawn');
|
|
1848
|
-
const beforeOverride = spawnSpecs.length;
|
|
1849
|
-
const overrideResult = await bridge.execute({
|
|
1850
|
-
action: 'call',
|
|
1851
|
-
tool: 'lookup',
|
|
1852
|
-
arguments: {
|
|
2217
|
+
await assert.rejects(()=>bridge.execute(bridgeCall('lookup', {
|
|
1853
2218
|
city: 'Dubai',
|
|
1854
|
-
|
|
1855
|
-
|
|
1856
|
-
|
|
1857
|
-
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
|
|
2219
|
+
...deep
|
|
2220
|
+
}), null), /invalid arguments/);
|
|
2221
|
+
assert.equal(spawnSpecs.length, beforeDeep, 'schema-invalid deep arguments are rejected before spawn');
|
|
2222
|
+
const beforeOverride = spawnSpecs.length;
|
|
2223
|
+
const overrideResult = await bridge.execute(bridgeCall('lookup', {
|
|
2224
|
+
city: 'Dubai',
|
|
2225
|
+
executable: '/tmp/evil',
|
|
2226
|
+
cwd: '/tmp/evil',
|
|
2227
|
+
argv: [
|
|
2228
|
+
'--unsafe'
|
|
2229
|
+
]
|
|
2230
|
+
}), null);
|
|
1861
2231
|
assert.deepEqual(spawnSpecs[beforeOverride]?.argv, [
|
|
1862
2232
|
process.execPath,
|
|
1863
2233
|
'-m',
|
|
@@ -1877,33 +2247,29 @@ try {
|
|
|
1877
2247
|
], 'model executable/cwd/argv fields remain data and cannot override config');
|
|
1878
2248
|
assert.equal(overrideResult.ok, true, 'override-shaped arguments still use the configured bridge');
|
|
1879
2249
|
outcomes.push({
|
|
1880
|
-
stdout:
|
|
2250
|
+
stdout: okEnvelope({
|
|
2251
|
+
city: 'Dubai'
|
|
2252
|
+
}),
|
|
1881
2253
|
lossy: true
|
|
1882
2254
|
});
|
|
1883
|
-
const truncated = await bridge.execute({
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
arguments: {
|
|
1887
|
-
city: 'Dubai'
|
|
1888
|
-
}
|
|
1889
|
-
}, null);
|
|
2255
|
+
const truncated = await bridge.execute(bridgeCall('lookup', {
|
|
2256
|
+
city: 'Dubai'
|
|
2257
|
+
}), null);
|
|
1890
2258
|
assert.deepEqual(truncated, {
|
|
1891
2259
|
ok: false,
|
|
1892
2260
|
error: 'output_truncated'
|
|
1893
2261
|
}, 'lossy stdout is rejected without parsing partial output');
|
|
1894
2262
|
outcomes.push({
|
|
1895
|
-
stdout:
|
|
2263
|
+
stdout: okEnvelope({
|
|
2264
|
+
city: 'Dubai'
|
|
2265
|
+
}),
|
|
1896
2266
|
stderr: 'private runner diagnostic',
|
|
1897
2267
|
exitCode: 17,
|
|
1898
2268
|
signal: 'SIGTERM'
|
|
1899
2269
|
});
|
|
1900
|
-
const failed = await bridge.execute({
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
arguments: {
|
|
1904
|
-
city: 'Dubai'
|
|
1905
|
-
}
|
|
1906
|
-
}, null);
|
|
2270
|
+
const failed = await bridge.execute(bridgeCall('lookup', {
|
|
2271
|
+
city: 'Dubai'
|
|
2272
|
+
}), null);
|
|
1907
2273
|
assert.deepEqual(failed, {
|
|
1908
2274
|
ok: false,
|
|
1909
2275
|
error: 'runner_failed',
|
|
@@ -1913,7 +2279,7 @@ try {
|
|
|
1913
2279
|
const errorsTable = await bridge.execute({
|
|
1914
2280
|
action: 'errors'
|
|
1915
2281
|
}, null);
|
|
1916
|
-
assert.ok(errorsTable.ok === true && errorsTable.errors, '
|
|
2282
|
+
assert.ok(errorsTable.ok === true && errorsTable.errors, 'bridge failure contract is discoverable');
|
|
1917
2283
|
for (const code of [
|
|
1918
2284
|
'invalid_action',
|
|
1919
2285
|
'disallowed_tool',
|
|
@@ -1927,10 +2293,10 @@ try {
|
|
|
1927
2293
|
'forbidden_output'
|
|
1928
2294
|
]){
|
|
1929
2295
|
const row = errorsTable.errors?.[code];
|
|
1930
|
-
assert.ok(row && typeof row.recoverable === 'boolean' && row.remedy.length > 0,
|
|
2296
|
+
assert.ok(row && typeof row.recoverable === 'boolean' && row.remedy.length > 0, `failure contract contains ${code} with recovery guidance`);
|
|
1931
2297
|
}
|
|
1932
|
-
assert.equal(errorsTable.errors?.invalid_arguments?.recoverable, true, '
|
|
1933
|
-
assert.equal(errorsTable.errors?.forbidden_output?.recoverable, false, '
|
|
2298
|
+
assert.equal(errorsTable.errors?.invalid_arguments?.recoverable, true, 'invalid caller arguments are recoverable');
|
|
2299
|
+
assert.equal(errorsTable.errors?.forbidden_output?.recoverable, false, 'policy-boundary output is not recoverable');
|
|
1934
2300
|
for (const stdout of [
|
|
1935
2301
|
'not-json',
|
|
1936
2302
|
'{"result":1}{"result":2}'
|
|
@@ -1938,28 +2304,127 @@ try {
|
|
|
1938
2304
|
outcomes.push({
|
|
1939
2305
|
stdout
|
|
1940
2306
|
});
|
|
1941
|
-
const malformed = await bridge.execute({
|
|
1942
|
-
|
|
1943
|
-
|
|
1944
|
-
arguments: {
|
|
1945
|
-
city: 'Dubai'
|
|
1946
|
-
}
|
|
1947
|
-
}, null);
|
|
2307
|
+
const malformed = await bridge.execute(bridgeCall('lookup', {
|
|
2308
|
+
city: 'Dubai'
|
|
2309
|
+
}), null);
|
|
1948
2310
|
assert.deepEqual(malformed, {
|
|
1949
2311
|
ok: false,
|
|
1950
2312
|
error: 'invalid_json'
|
|
1951
2313
|
}, 'malformed or multi-value JSON is rejected');
|
|
1952
2314
|
}
|
|
1953
2315
|
outcomes.push({
|
|
1954
|
-
|
|
2316
|
+
stdout: JSON.stringify({
|
|
2317
|
+
schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION,
|
|
2318
|
+
status: 'miss',
|
|
2319
|
+
code: 'NOT_FOUND',
|
|
2320
|
+
recovery: 'revise_arguments'
|
|
2321
|
+
})
|
|
1955
2322
|
});
|
|
1956
|
-
const
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
2323
|
+
const miss = await bridge.execute(bridgeCall('lookup', {
|
|
2324
|
+
city: 'Dubai'
|
|
2325
|
+
}), null);
|
|
2326
|
+
assert.deepEqual(miss, {
|
|
2327
|
+
ok: true,
|
|
2328
|
+
outcome: {
|
|
2329
|
+
schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION,
|
|
2330
|
+
status: 'miss',
|
|
2331
|
+
code: 'NOT_FOUND',
|
|
2332
|
+
recovery: 'revise_arguments'
|
|
1961
2333
|
}
|
|
1962
|
-
},
|
|
2334
|
+
}, 'exit-zero domain miss is a successful typed outcome');
|
|
2335
|
+
outcomes.push({
|
|
2336
|
+
stdout: JSON.stringify({
|
|
2337
|
+
schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION,
|
|
2338
|
+
status: 'error',
|
|
2339
|
+
code: 'AMBIGUOUS',
|
|
2340
|
+
recovery: 'choose_alternative'
|
|
2341
|
+
})
|
|
2342
|
+
});
|
|
2343
|
+
const domainError = await bridge.execute(bridgeCall('lookup', {
|
|
2344
|
+
city: 'Dubai'
|
|
2345
|
+
}), null);
|
|
2346
|
+
assert.deepEqual(domainError, {
|
|
2347
|
+
ok: true,
|
|
2348
|
+
outcome: {
|
|
2349
|
+
schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION,
|
|
2350
|
+
status: 'error',
|
|
2351
|
+
code: 'AMBIGUOUS',
|
|
2352
|
+
recovery: 'choose_alternative'
|
|
2353
|
+
}
|
|
2354
|
+
}, 'declared exit-zero domain error is transport success');
|
|
2355
|
+
for (const invalidEnvelope of [
|
|
2356
|
+
{
|
|
2357
|
+
schema_version: 'legacy',
|
|
2358
|
+
status: 'miss',
|
|
2359
|
+
code: 'NOT_FOUND',
|
|
2360
|
+
recovery: 'revise_arguments'
|
|
2361
|
+
},
|
|
2362
|
+
{
|
|
2363
|
+
schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION,
|
|
2364
|
+
status: 'miss',
|
|
2365
|
+
code: 'UNKNOWN',
|
|
2366
|
+
recovery: 'revise_arguments'
|
|
2367
|
+
},
|
|
2368
|
+
{
|
|
2369
|
+
schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION,
|
|
2370
|
+
status: 'miss',
|
|
2371
|
+
code: 'NOT_FOUND',
|
|
2372
|
+
recovery: 'retry_same'
|
|
2373
|
+
},
|
|
2374
|
+
{
|
|
2375
|
+
schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION,
|
|
2376
|
+
status: 'error',
|
|
2377
|
+
code: 'AMBIGUOUS',
|
|
2378
|
+
recovery: 'choose_alternative',
|
|
2379
|
+
message: 'private free text'
|
|
2380
|
+
}
|
|
2381
|
+
]){
|
|
2382
|
+
outcomes.push({
|
|
2383
|
+
stdout: JSON.stringify(invalidEnvelope)
|
|
2384
|
+
});
|
|
2385
|
+
assert.deepEqual(await bridge.execute(bridgeCall('lookup', {
|
|
2386
|
+
city: 'Dubai'
|
|
2387
|
+
}), null), {
|
|
2388
|
+
ok: false,
|
|
2389
|
+
error: 'invalid_output'
|
|
2390
|
+
}, 'undeclared or non-exact domain envelope fails closed');
|
|
2391
|
+
}
|
|
2392
|
+
outcomes.push({
|
|
2393
|
+
stdout: JSON.stringify({
|
|
2394
|
+
result: {
|
|
2395
|
+
city: 'Dubai'
|
|
2396
|
+
}
|
|
2397
|
+
})
|
|
2398
|
+
});
|
|
2399
|
+
assert.deepEqual(await bridge.execute(bridgeCall('lookup', {
|
|
2400
|
+
city: 'Dubai'
|
|
2401
|
+
}), null), {
|
|
2402
|
+
ok: false,
|
|
2403
|
+
error: 'invalid_output'
|
|
2404
|
+
}, 'legacy versionless result envelope is rejected');
|
|
2405
|
+
outcomes.push({
|
|
2406
|
+
stdout: JSON.stringify({
|
|
2407
|
+
schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION,
|
|
2408
|
+
status: 'miss',
|
|
2409
|
+
code: 'NOT_FOUND',
|
|
2410
|
+
recovery: 'revise_arguments'
|
|
2411
|
+
}),
|
|
2412
|
+
exitCode: 23
|
|
2413
|
+
});
|
|
2414
|
+
assert.deepEqual(await bridge.execute(bridgeCall('lookup', {
|
|
2415
|
+
city: 'Dubai'
|
|
2416
|
+
}), null), {
|
|
2417
|
+
ok: false,
|
|
2418
|
+
error: 'runner_failed',
|
|
2419
|
+
exit_code: 23,
|
|
2420
|
+
signal: null
|
|
2421
|
+
}, 'nonzero exit remains infrastructure failure even when stdout resembles a domain envelope');
|
|
2422
|
+
outcomes.push({
|
|
2423
|
+
spawnError: true
|
|
2424
|
+
});
|
|
2425
|
+
const spawnFailed = await bridge.execute(bridgeCall('lookup', {
|
|
2426
|
+
city: 'Dubai'
|
|
2427
|
+
}), null);
|
|
1963
2428
|
assert.deepEqual(spawnFailed, {
|
|
1964
2429
|
ok: false,
|
|
1965
2430
|
error: 'spawn_failed'
|
|
@@ -1967,13 +2432,9 @@ try {
|
|
|
1967
2432
|
outcomes.push({
|
|
1968
2433
|
spawnReject: true
|
|
1969
2434
|
});
|
|
1970
|
-
const asyncSpawnFailed = await bridge.execute({
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
arguments: {
|
|
1974
|
-
city: 'Dubai'
|
|
1975
|
-
}
|
|
1976
|
-
}, null);
|
|
2435
|
+
const asyncSpawnFailed = await bridge.execute(bridgeCall('lookup', {
|
|
2436
|
+
city: 'Dubai'
|
|
2437
|
+
}), null);
|
|
1977
2438
|
assert.deepEqual(asyncSpawnFailed, {
|
|
1978
2439
|
ok: false,
|
|
1979
2440
|
error: 'spawn_failed'
|
|
@@ -1981,53 +2442,47 @@ try {
|
|
|
1981
2442
|
outcomes.push({
|
|
1982
2443
|
waitForAbort: true
|
|
1983
2444
|
});
|
|
1984
|
-
const timedOut = await bridge.execute({
|
|
1985
|
-
|
|
1986
|
-
|
|
1987
|
-
arguments: {
|
|
1988
|
-
city: 'Dubai'
|
|
1989
|
-
}
|
|
1990
|
-
}, null);
|
|
2445
|
+
const timedOut = await bridge.execute(bridgeCall('lookup', {
|
|
2446
|
+
city: 'Dubai'
|
|
2447
|
+
}), null);
|
|
1991
2448
|
assert.deepEqual(timedOut, {
|
|
1992
2449
|
ok: false,
|
|
1993
2450
|
error: 'timed_out'
|
|
1994
2451
|
}, 'deadline abort is surfaced as timed_out');
|
|
1995
2452
|
assert.equal(timeoutSignal?.aborted, true, 'timeout abort signal is fired');
|
|
1996
2453
|
const beforeInvalidArgs = spawnSpecs.length;
|
|
1997
|
-
|
|
1998
|
-
|
|
1999
|
-
|
|
2000
|
-
|
|
2001
|
-
tool: 'lookup',
|
|
2002
|
-
arguments: [
|
|
2003
|
-
'not',
|
|
2004
|
-
'plain'
|
|
2005
|
-
]
|
|
2006
|
-
}, null);
|
|
2007
|
-
} catch (e) {
|
|
2008
|
-
argsTypeRejected = e.name;
|
|
2009
|
-
}
|
|
2010
|
-
assert.equal(argsTypeRejected, 'ToolArgsError', 'arguments 非对象 → 宿主权 schema 结构化拒绝');
|
|
2454
|
+
await assert.rejects(()=>bridge.execute(bridgeCall('lookup', [
|
|
2455
|
+
'not',
|
|
2456
|
+
'plain'
|
|
2457
|
+
]), null), /invalid arguments/);
|
|
2011
2458
|
assert.equal(spawnSpecs.length, beforeInvalidArgs, 'non-object arguments are rejected before spawn');
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
|
|
2459
|
+
for (const [label, argumentsValue] of [
|
|
2460
|
+
[
|
|
2461
|
+
'missing required city',
|
|
2462
|
+
{}
|
|
2463
|
+
],
|
|
2464
|
+
[
|
|
2465
|
+
'wrong city type',
|
|
2466
|
+
{
|
|
2467
|
+
city: 7
|
|
2468
|
+
}
|
|
2469
|
+
],
|
|
2470
|
+
[
|
|
2471
|
+
'undeclared country',
|
|
2472
|
+
{
|
|
2473
|
+
city: 'Dubai',
|
|
2474
|
+
country: 'AE'
|
|
2475
|
+
}
|
|
2476
|
+
]
|
|
2477
|
+
]){
|
|
2478
|
+
await assert.rejects(()=>bridge.execute(bridgeCall('lookup', argumentsValue), null), /invalid arguments/, `${label} is rejected by the descriptor-derived runtime validator`);
|
|
2019
2479
|
}
|
|
2020
|
-
assert.equal(
|
|
2480
|
+
assert.equal(spawnSpecs.length, beforeInvalidArgs, 'required, type, and closed-object argument failures all happen before spawn');
|
|
2481
|
+
await assert.rejects(()=>bridge.execute({
|
|
2482
|
+
action: 'inspect'
|
|
2483
|
+
}, null), /invalid arguments/);
|
|
2021
2484
|
const beforeRejected = spawnSpecs.length;
|
|
2022
|
-
|
|
2023
|
-
action: 'call',
|
|
2024
|
-
tool: 'delete_all',
|
|
2025
|
-
arguments: {}
|
|
2026
|
-
}, null);
|
|
2027
|
-
assert.deepEqual(rejected, {
|
|
2028
|
-
ok: false,
|
|
2029
|
-
error: 'disallowed_tool'
|
|
2030
|
-
}, 'disallowed tool is rejected structurally');
|
|
2485
|
+
await assert.rejects(()=>bridge.execute(bridgeCall('delete_all', {}), null), /invalid arguments/);
|
|
2031
2486
|
assert.equal(spawnSpecs.length, beforeRejected, 'disallowed tool is rejected before spawn');
|
|
2032
2487
|
const spacedConfigPath = join(root, ' benchmark-environment-config.json ');
|
|
2033
2488
|
writeFileSync(spacedConfigPath, readFileSync(configPath));
|
|
@@ -2214,7 +2669,7 @@ try {
|
|
|
2214
2669
|
else process.env.GOTRY_LLM_MODEL = originalModelOverride;
|
|
2215
2670
|
}
|
|
2216
2671
|
const validConfig = {
|
|
2217
|
-
schema_version: '
|
|
2672
|
+
schema_version: 'gotry_benchmark_environment_bridge_v3',
|
|
2218
2673
|
enabled: true,
|
|
2219
2674
|
executable: process.execPath,
|
|
2220
2675
|
cwd: root,
|
|
@@ -2224,8 +2679,38 @@ try {
|
|
|
2224
2679
|
'--lang',
|
|
2225
2680
|
'en'
|
|
2226
2681
|
],
|
|
2227
|
-
|
|
2228
|
-
|
|
2682
|
+
tools: [
|
|
2683
|
+
{
|
|
2684
|
+
name: 'lookup',
|
|
2685
|
+
description: 'Lookup one declared city.',
|
|
2686
|
+
input_schema: {
|
|
2687
|
+
type: 'object',
|
|
2688
|
+
properties: {
|
|
2689
|
+
city: {
|
|
2690
|
+
type: 'string',
|
|
2691
|
+
enum: [
|
|
2692
|
+
'Dubai',
|
|
2693
|
+
'Abu Dhabi'
|
|
2694
|
+
],
|
|
2695
|
+
description: 'Declared city name.'
|
|
2696
|
+
}
|
|
2697
|
+
},
|
|
2698
|
+
required: [
|
|
2699
|
+
'city'
|
|
2700
|
+
],
|
|
2701
|
+
additionalProperties: false
|
|
2702
|
+
},
|
|
2703
|
+
output_keys: [
|
|
2704
|
+
'city'
|
|
2705
|
+
],
|
|
2706
|
+
domain_outcomes: [
|
|
2707
|
+
{
|
|
2708
|
+
status: 'miss',
|
|
2709
|
+
code: 'NOT_FOUND',
|
|
2710
|
+
recovery: 'revise_arguments'
|
|
2711
|
+
}
|
|
2712
|
+
]
|
|
2713
|
+
}
|
|
2229
2714
|
],
|
|
2230
2715
|
timeout_ms: 20,
|
|
2231
2716
|
max_output_bytes: 4_096,
|
|
@@ -2239,6 +2724,86 @@ try {
|
|
|
2239
2724
|
network: 'denied'
|
|
2240
2725
|
}
|
|
2241
2726
|
};
|
|
2727
|
+
const clonedConfig = ()=>JSON.parse(JSON.stringify(validConfig));
|
|
2728
|
+
const configWithTool = (mutate)=>{
|
|
2729
|
+
const config = clonedConfig();
|
|
2730
|
+
mutate(config.tools[0]);
|
|
2731
|
+
return config;
|
|
2732
|
+
};
|
|
2733
|
+
const nestedInputSchema = (objectLevels)=>{
|
|
2734
|
+
let child = {
|
|
2735
|
+
type: 'string'
|
|
2736
|
+
};
|
|
2737
|
+
for(let index = 0; index < objectLevels; index += 1){
|
|
2738
|
+
child = {
|
|
2739
|
+
type: 'object',
|
|
2740
|
+
properties: {
|
|
2741
|
+
next: child
|
|
2742
|
+
},
|
|
2743
|
+
required: [
|
|
2744
|
+
'next'
|
|
2745
|
+
],
|
|
2746
|
+
additionalProperties: false
|
|
2747
|
+
};
|
|
2748
|
+
}
|
|
2749
|
+
return child;
|
|
2750
|
+
};
|
|
2751
|
+
const flatInputSchema = (count)=>({
|
|
2752
|
+
type: 'object',
|
|
2753
|
+
properties: Object.fromEntries(Array.from({
|
|
2754
|
+
length: count
|
|
2755
|
+
}, (_, index)=>[
|
|
2756
|
+
`p${index}`,
|
|
2757
|
+
{
|
|
2758
|
+
type: 'string'
|
|
2759
|
+
}
|
|
2760
|
+
])),
|
|
2761
|
+
required: [],
|
|
2762
|
+
additionalProperties: false
|
|
2763
|
+
});
|
|
2764
|
+
const totalEnumInputSchema = (groups)=>({
|
|
2765
|
+
type: 'object',
|
|
2766
|
+
properties: Object.fromEntries(Array.from({
|
|
2767
|
+
length: groups
|
|
2768
|
+
}, (_, group)=>[
|
|
2769
|
+
`e${group}`,
|
|
2770
|
+
{
|
|
2771
|
+
type: 'string',
|
|
2772
|
+
enum: Array.from({
|
|
2773
|
+
length: 64
|
|
2774
|
+
}, (_, index)=>`v${group}_${index}`)
|
|
2775
|
+
}
|
|
2776
|
+
])),
|
|
2777
|
+
required: [],
|
|
2778
|
+
additionalProperties: false
|
|
2779
|
+
});
|
|
2780
|
+
const byteSizedInputSchema = (targetBytes)=>{
|
|
2781
|
+
const properties = Object.fromEntries(Array.from({
|
|
2782
|
+
length: 64
|
|
2783
|
+
}, (_, index)=>[
|
|
2784
|
+
`b${index}`,
|
|
2785
|
+
{
|
|
2786
|
+
type: 'string',
|
|
2787
|
+
description: ''
|
|
2788
|
+
}
|
|
2789
|
+
]));
|
|
2790
|
+
const schema = {
|
|
2791
|
+
type: 'object',
|
|
2792
|
+
properties,
|
|
2793
|
+
required: [],
|
|
2794
|
+
additionalProperties: false
|
|
2795
|
+
};
|
|
2796
|
+
let remaining = targetBytes - Buffer.byteLength(JSON.stringify(schema), 'utf8');
|
|
2797
|
+
assert.ok(remaining >= 0 && remaining <= 64 * 512, 'byte-boundary fixture has enough bounded description capacity');
|
|
2798
|
+
for (const property of Object.values(properties)){
|
|
2799
|
+
const added = Math.min(512, remaining);
|
|
2800
|
+
property.description = 'x'.repeat(added);
|
|
2801
|
+
remaining -= added;
|
|
2802
|
+
}
|
|
2803
|
+
assert.equal(remaining, 0, 'byte-boundary fixture reaches the requested serialized size');
|
|
2804
|
+
assert.equal(Buffer.byteLength(JSON.stringify(schema), 'utf8'), targetBytes);
|
|
2805
|
+
return schema;
|
|
2806
|
+
};
|
|
2242
2807
|
const frozenProjection = registerBenchmarkEnvironmentBridge(configPath, ()=>{}, {
|
|
2243
2808
|
spawn: (_spec)=>fakeHandle({
|
|
2244
2809
|
stdout: '{"result":{}}'
|
|
@@ -2279,7 +2844,7 @@ try {
|
|
|
2279
2844
|
return ()=>{};
|
|
2280
2845
|
},
|
|
2281
2846
|
effect (action) {
|
|
2282
|
-
return action
|
|
2847
|
+
return runEffect(action);
|
|
2283
2848
|
},
|
|
2284
2849
|
agents: {
|
|
2285
2850
|
list () {
|
|
@@ -2313,62 +2878,286 @@ try {
|
|
|
2313
2878
|
}), /benchmark environment bridge configuration unavailable/, 'unknown top-level config key fails hard');
|
|
2314
2879
|
assert.throws(()=>bridgeRegistrationFor({
|
|
2315
2880
|
...validConfig,
|
|
2316
|
-
|
|
2317
|
-
|
|
2318
|
-
|
|
2319
|
-
|
|
2320
|
-
|
|
2321
|
-
|
|
2322
|
-
|
|
2881
|
+
tools: [
|
|
2882
|
+
{
|
|
2883
|
+
...validConfig.tools[0],
|
|
2884
|
+
input_schema: {
|
|
2885
|
+
type: 'object'
|
|
2886
|
+
}
|
|
2887
|
+
}
|
|
2323
2888
|
]
|
|
2324
|
-
}), /benchmark environment bridge configuration unavailable/, '
|
|
2889
|
+
}), /benchmark environment bridge configuration unavailable/, 'v3 rejects an open annotation-only input schema');
|
|
2325
2890
|
assert.throws(()=>bridgeRegistrationFor({
|
|
2326
2891
|
...validConfig,
|
|
2327
|
-
|
|
2328
|
-
}), /benchmark environment bridge configuration unavailable/, '
|
|
2892
|
+
schema_version: 'gotry_benchmark_environment_bridge_v1'
|
|
2893
|
+
}), /benchmark environment bridge configuration unavailable/, 'v1 config cannot silently omit the Round 3 terminal semantics');
|
|
2329
2894
|
assert.throws(()=>bridgeRegistrationFor({
|
|
2330
2895
|
...validConfig,
|
|
2331
|
-
|
|
2332
|
-
|
|
2896
|
+
tools: [
|
|
2897
|
+
validConfig.tools[0],
|
|
2898
|
+
validConfig.tools[0]
|
|
2899
|
+
]
|
|
2900
|
+
}), /benchmark environment bridge configuration unavailable/, 'duplicate tool descriptor fails hard');
|
|
2901
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
2902
|
+
delete tool.output_keys;
|
|
2903
|
+
})), /benchmark environment bridge configuration unavailable/, 'missing output_keys fails hard');
|
|
2904
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
2905
|
+
tool.output_keys = [];
|
|
2906
|
+
})), /benchmark environment bridge configuration unavailable/, 'empty output allowlist fails closed at registration');
|
|
2907
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
2908
|
+
tool.output_keys = [
|
|
2909
|
+
'city',
|
|
2910
|
+
'city'
|
|
2911
|
+
];
|
|
2912
|
+
})), /benchmark environment bridge configuration unavailable/, 'duplicate output key fails hard');
|
|
2913
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
2914
|
+
tool.output_keys = [
|
|
2915
|
+
'not a key'
|
|
2916
|
+
];
|
|
2917
|
+
})), /benchmark environment bridge configuration unavailable/, 'non-identifier output key fails hard');
|
|
2918
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
2919
|
+
delete tool.domain_outcomes;
|
|
2920
|
+
})), /benchmark environment bridge configuration unavailable/, 'missing domain_outcomes fails hard');
|
|
2921
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
2922
|
+
tool.domain_outcomes = [
|
|
2923
|
+
{
|
|
2924
|
+
status: 'unknown',
|
|
2925
|
+
code: 'NOT_FOUND',
|
|
2926
|
+
recovery: 'none'
|
|
2927
|
+
}
|
|
2928
|
+
];
|
|
2929
|
+
})), /benchmark environment bridge configuration unavailable/, 'unknown domain status fails hard');
|
|
2930
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
2931
|
+
tool.domain_outcomes = [
|
|
2932
|
+
{
|
|
2933
|
+
status: 'miss',
|
|
2934
|
+
code: 'NOT_FOUND',
|
|
2935
|
+
recovery: 'free_text'
|
|
2936
|
+
}
|
|
2937
|
+
];
|
|
2938
|
+
})), /benchmark environment bridge configuration unavailable/, 'free-text domain recovery fails hard');
|
|
2939
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
2940
|
+
tool.domain_outcomes = [
|
|
2941
|
+
{
|
|
2942
|
+
status: 'miss',
|
|
2943
|
+
code: 'NOT_FOUND',
|
|
2944
|
+
recovery: 'none'
|
|
2945
|
+
},
|
|
2946
|
+
{
|
|
2947
|
+
status: 'error',
|
|
2948
|
+
code: 'NOT_FOUND',
|
|
2949
|
+
recovery: 'none'
|
|
2950
|
+
}
|
|
2951
|
+
];
|
|
2952
|
+
})), /benchmark environment bridge configuration unavailable/, 'duplicate domain code fails hard');
|
|
2953
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
2954
|
+
tool.name = 'lookup;rm';
|
|
2955
|
+
})), /benchmark environment bridge configuration unavailable/, 'shell metacharacter tool identifier fails hard');
|
|
2956
|
+
for (const [label, schema] of [
|
|
2957
|
+
[
|
|
2958
|
+
'reference keyword',
|
|
2959
|
+
{
|
|
2960
|
+
type: 'object',
|
|
2961
|
+
properties: {
|
|
2962
|
+
city: {
|
|
2963
|
+
$ref: '#/city'
|
|
2964
|
+
}
|
|
2965
|
+
},
|
|
2966
|
+
required: [
|
|
2967
|
+
'city'
|
|
2968
|
+
],
|
|
2969
|
+
additionalProperties: false
|
|
2333
2970
|
}
|
|
2334
|
-
|
|
2335
|
-
|
|
2336
|
-
|
|
2337
|
-
|
|
2338
|
-
|
|
2971
|
+
],
|
|
2972
|
+
[
|
|
2973
|
+
'default keyword',
|
|
2974
|
+
{
|
|
2975
|
+
type: 'object',
|
|
2976
|
+
properties: {
|
|
2977
|
+
city: {
|
|
2978
|
+
type: 'string',
|
|
2979
|
+
default: 'Dubai'
|
|
2980
|
+
}
|
|
2981
|
+
},
|
|
2982
|
+
required: [
|
|
2339
2983
|
'city'
|
|
2340
|
-
]
|
|
2984
|
+
],
|
|
2985
|
+
additionalProperties: false
|
|
2341
2986
|
}
|
|
2342
|
-
|
|
2343
|
-
|
|
2344
|
-
|
|
2345
|
-
|
|
2346
|
-
|
|
2987
|
+
],
|
|
2988
|
+
[
|
|
2989
|
+
'examples keyword',
|
|
2990
|
+
{
|
|
2991
|
+
type: 'object',
|
|
2992
|
+
properties: {
|
|
2993
|
+
city: {
|
|
2994
|
+
type: 'string',
|
|
2995
|
+
examples: [
|
|
2996
|
+
'Dubai'
|
|
2997
|
+
]
|
|
2998
|
+
}
|
|
2999
|
+
},
|
|
3000
|
+
required: [
|
|
3001
|
+
'city'
|
|
3002
|
+
],
|
|
3003
|
+
additionalProperties: false
|
|
3004
|
+
}
|
|
3005
|
+
],
|
|
3006
|
+
[
|
|
3007
|
+
'unknown keyword',
|
|
3008
|
+
{
|
|
3009
|
+
type: 'object',
|
|
3010
|
+
properties: {
|
|
3011
|
+
city: {
|
|
3012
|
+
type: 'string',
|
|
3013
|
+
title: 'City'
|
|
3014
|
+
}
|
|
3015
|
+
},
|
|
3016
|
+
required: [
|
|
3017
|
+
'city'
|
|
3018
|
+
],
|
|
3019
|
+
additionalProperties: false
|
|
3020
|
+
}
|
|
3021
|
+
],
|
|
3022
|
+
[
|
|
3023
|
+
'open nested object',
|
|
3024
|
+
{
|
|
3025
|
+
type: 'object',
|
|
3026
|
+
properties: {
|
|
3027
|
+
nested: {
|
|
3028
|
+
type: 'object',
|
|
3029
|
+
properties: {},
|
|
3030
|
+
required: []
|
|
3031
|
+
}
|
|
3032
|
+
},
|
|
3033
|
+
required: [
|
|
3034
|
+
'nested'
|
|
3035
|
+
],
|
|
3036
|
+
additionalProperties: false
|
|
3037
|
+
}
|
|
3038
|
+
],
|
|
3039
|
+
[
|
|
3040
|
+
'array without items',
|
|
3041
|
+
{
|
|
3042
|
+
type: 'object',
|
|
3043
|
+
properties: {
|
|
3044
|
+
values: {
|
|
3045
|
+
type: 'array'
|
|
3046
|
+
}
|
|
3047
|
+
},
|
|
3048
|
+
required: [
|
|
3049
|
+
'values'
|
|
3050
|
+
],
|
|
3051
|
+
additionalProperties: false
|
|
3052
|
+
}
|
|
3053
|
+
],
|
|
3054
|
+
[
|
|
3055
|
+
'duplicate required',
|
|
3056
|
+
{
|
|
3057
|
+
type: 'object',
|
|
3058
|
+
properties: {
|
|
3059
|
+
city: {
|
|
3060
|
+
type: 'string'
|
|
3061
|
+
}
|
|
3062
|
+
},
|
|
3063
|
+
required: [
|
|
2347
3064
|
'city',
|
|
2348
3065
|
'city'
|
|
2349
|
-
]
|
|
3066
|
+
],
|
|
3067
|
+
additionalProperties: false
|
|
2350
3068
|
}
|
|
2351
|
-
|
|
2352
|
-
|
|
2353
|
-
|
|
2354
|
-
|
|
2355
|
-
|
|
2356
|
-
|
|
2357
|
-
|
|
3069
|
+
],
|
|
3070
|
+
[
|
|
3071
|
+
'unknown required',
|
|
3072
|
+
{
|
|
3073
|
+
type: 'object',
|
|
3074
|
+
properties: {
|
|
3075
|
+
city: {
|
|
3076
|
+
type: 'string'
|
|
3077
|
+
}
|
|
3078
|
+
},
|
|
3079
|
+
required: [
|
|
3080
|
+
'country'
|
|
3081
|
+
],
|
|
3082
|
+
additionalProperties: false
|
|
3083
|
+
}
|
|
3084
|
+
],
|
|
3085
|
+
[
|
|
3086
|
+
'type-mismatched enum',
|
|
3087
|
+
{
|
|
3088
|
+
type: 'object',
|
|
3089
|
+
properties: {
|
|
3090
|
+
city: {
|
|
3091
|
+
type: 'string',
|
|
3092
|
+
enum: [
|
|
3093
|
+
'Dubai',
|
|
3094
|
+
7
|
|
3095
|
+
]
|
|
3096
|
+
}
|
|
3097
|
+
},
|
|
3098
|
+
required: [
|
|
3099
|
+
'city'
|
|
3100
|
+
],
|
|
3101
|
+
additionalProperties: false
|
|
3102
|
+
}
|
|
3103
|
+
],
|
|
3104
|
+
[
|
|
3105
|
+
'duplicate enum',
|
|
3106
|
+
{
|
|
3107
|
+
type: 'object',
|
|
3108
|
+
properties: {
|
|
3109
|
+
city: {
|
|
3110
|
+
type: 'string',
|
|
3111
|
+
enum: [
|
|
3112
|
+
'Dubai',
|
|
3113
|
+
'Dubai'
|
|
3114
|
+
]
|
|
3115
|
+
}
|
|
3116
|
+
},
|
|
3117
|
+
required: [
|
|
3118
|
+
'city'
|
|
3119
|
+
],
|
|
3120
|
+
additionalProperties: false
|
|
2358
3121
|
}
|
|
2359
|
-
|
|
3122
|
+
]
|
|
3123
|
+
]){
|
|
3124
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3125
|
+
tool.input_schema = schema;
|
|
3126
|
+
})), /benchmark environment bridge configuration unavailable/, `${label} fails closed`);
|
|
3127
|
+
}
|
|
3128
|
+
assert.equal(bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3129
|
+
tool.input_schema = nestedInputSchema(8);
|
|
3130
|
+
})), true, 'schema depth eight is accepted');
|
|
3131
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3132
|
+
tool.input_schema = nestedInputSchema(9);
|
|
3133
|
+
})), /benchmark environment bridge configuration unavailable/, 'schema depth nine is rejected');
|
|
3134
|
+
assert.equal(bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3135
|
+
tool.input_schema = flatInputSchema(255);
|
|
3136
|
+
})), true, 'the 256-node total boundary is accepted');
|
|
3137
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3138
|
+
tool.input_schema = flatInputSchema(256);
|
|
3139
|
+
})), /benchmark environment bridge configuration unavailable/, 'the 256-node total boundary plus one is rejected');
|
|
3140
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3141
|
+
tool.input_schema = flatInputSchema(257);
|
|
3142
|
+
})), /benchmark environment bridge configuration unavailable/, 'the property total bound rejects 257 properties');
|
|
3143
|
+
assert.equal(bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3144
|
+
tool.input_schema = totalEnumInputSchema(4);
|
|
3145
|
+
})), true, 'the 256-enum-value total boundary is accepted');
|
|
3146
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3147
|
+
tool.input_schema = totalEnumInputSchema(5);
|
|
3148
|
+
})), /benchmark environment bridge configuration unavailable/, 'the enum-value total bound rejects values above 256');
|
|
3149
|
+
assert.equal(bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3150
|
+
tool.input_schema = byteSizedInputSchema(16 * 1024);
|
|
3151
|
+
})), true, 'the 16 KiB schema boundary is accepted');
|
|
3152
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3153
|
+
tool.input_schema = byteSizedInputSchema(16 * 1024 + 1);
|
|
3154
|
+
})), /benchmark environment bridge configuration unavailable/, 'the schema byte boundary plus one is rejected');
|
|
2360
3155
|
assert.throws(()=>bridgeRegistrationFor({
|
|
2361
3156
|
...validConfig,
|
|
2362
3157
|
argv_prefix: [
|
|
2363
3158
|
'agent\n--unsafe'
|
|
2364
3159
|
]
|
|
2365
3160
|
}), /benchmark environment bridge configuration unavailable/, 'argv control separator fails hard');
|
|
2366
|
-
assert.throws(()=>bridgeRegistrationFor({
|
|
2367
|
-
...validConfig,
|
|
2368
|
-
allowed_tools: [
|
|
2369
|
-
'lookup;rm'
|
|
2370
|
-
]
|
|
2371
|
-
}), /benchmark environment bridge configuration unavailable/, 'shell metacharacter tool identifier fails hard');
|
|
2372
3161
|
assert.throws(()=>bridgeRegistrationFor({
|
|
2373
3162
|
...validConfig,
|
|
2374
3163
|
isolation: {
|
|
@@ -2418,20 +3207,20 @@ try {
|
|
|
2418
3207
|
writeFileSync(widePath, JSON.stringify(validConfig));
|
|
2419
3208
|
chmodSync(widePath, 0o666);
|
|
2420
3209
|
assert.throws(()=>loadConfigRegistration(widePath, root), /benchmark environment bridge configuration unavailable/, 'group/world writable config fails hard');
|
|
2421
|
-
const
|
|
2422
|
-
writeFileSync(
|
|
2423
|
-
const
|
|
2424
|
-
const
|
|
3210
|
+
const strictConfigPath = join(root, 'strict-bridge.json');
|
|
3211
|
+
writeFileSync(strictConfigPath, JSON.stringify(validConfig));
|
|
3212
|
+
const strictTools = [];
|
|
3213
|
+
const strictCtx = {
|
|
2425
3214
|
tools: {
|
|
2426
3215
|
register (tool) {
|
|
2427
|
-
|
|
3216
|
+
strictTools.push(tool);
|
|
2428
3217
|
return ()=>{};
|
|
2429
3218
|
},
|
|
2430
3219
|
get (name) {
|
|
2431
|
-
return
|
|
3220
|
+
return strictTools.find((tool)=>tool.name === name);
|
|
2432
3221
|
},
|
|
2433
3222
|
schemas () {
|
|
2434
|
-
return
|
|
3223
|
+
return strictTools.map((tool)=>({
|
|
2435
3224
|
name: tool.name
|
|
2436
3225
|
}));
|
|
2437
3226
|
}
|
|
@@ -2456,31 +3245,29 @@ try {
|
|
|
2456
3245
|
},
|
|
2457
3246
|
subprocess: {
|
|
2458
3247
|
spawn: (_spec)=>fakeHandle({
|
|
2459
|
-
stdout:
|
|
3248
|
+
stdout: okEnvelope({
|
|
3249
|
+
city: 'Dubai'
|
|
3250
|
+
})
|
|
2460
3251
|
})
|
|
2461
3252
|
}
|
|
2462
3253
|
};
|
|
2463
|
-
apply(
|
|
3254
|
+
apply(strictCtx, {
|
|
2464
3255
|
stateRoot: root,
|
|
2465
3256
|
timeoutMs: 20,
|
|
2466
3257
|
hbcliBin: '',
|
|
2467
3258
|
sessionAccess: 'off',
|
|
2468
|
-
benchmarkEnvironmentConfigPath:
|
|
3259
|
+
benchmarkEnvironmentConfigPath: strictConfigPath
|
|
2469
3260
|
});
|
|
2470
|
-
const
|
|
2471
|
-
assert.ok(
|
|
2472
|
-
assert.deepEqual(await
|
|
2473
|
-
|
|
2474
|
-
|
|
2475
|
-
arguments: {
|
|
2476
|
-
city: 'Dubai'
|
|
2477
|
-
}
|
|
2478
|
-
}, null), {
|
|
3261
|
+
const strictBridge = strictTools.find((tool)=>tool.name === 'gotry_benchmark_environment');
|
|
3262
|
+
assert.ok(strictBridge?.execute, 'strict v3 config registers the bridge');
|
|
3263
|
+
assert.deepEqual(await strictBridge.execute(bridgeCall('lookup', {
|
|
3264
|
+
city: 'Dubai'
|
|
3265
|
+
}), null), {
|
|
2479
3266
|
ok: true,
|
|
2480
3267
|
result: {
|
|
2481
3268
|
city: 'Dubai'
|
|
2482
3269
|
}
|
|
2483
|
-
}, '
|
|
3270
|
+
}, 'strict v3 result envelope executes safe structured output');
|
|
2484
3271
|
console.log('BENCHMARK ENVIRONMENT BRIDGE TESTS: registration + TDD bridge contract assertions');
|
|
2485
3272
|
} catch (error) {
|
|
2486
3273
|
console.error(error);
|