@danceiny/gotry 0.0.1-rc.18 → 0.0.1-rc.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -16
- package/README.zh-CN.md +37 -15
- package/bin/gotry-bootstrap.js +309 -30
- package/bin/gotry-inner.js +74 -23
- package/bin/gotry-runtime-resolution.d.ts +1 -1
- package/bin/gotry-runtime-resolution.js +7 -8
- package/cordis.gotry-patch.yml +22 -8
- package/data/stations-12306-verify.json +524 -0
- package/dist/capabilities/agent-reach.js +4 -4
- package/dist/capabilities/anything.js +62 -29
- package/dist/capabilities/channel-health.js +103 -0
- package/dist/capabilities/channel-registry.js +213 -0
- package/dist/capabilities/doctor.js +307 -0
- package/dist/capabilities/effect.js +178 -5
- package/dist/capabilities/flyai.js +13 -0
- package/dist/capabilities/hbcli.js +28 -2
- package/dist/capabilities/session/adapters/ctrip-hotel.js +207 -0
- package/dist/capabilities/session/adapters/rail-12306.js +285 -0
- package/dist/capabilities/session/extension-bridge.js +14 -1
- package/dist/capabilities/session/extension-channel.js +4 -2
- package/dist/capabilities/session/extension-distribution.js +1 -1
- package/dist/capabilities/session-search.js +311 -0
- package/dist/capabilities/visa-policy.js +126 -0
- package/dist/scripts/agent-planning-turn-deadline-e2e.js +305 -0
- package/dist/scripts/agent-planning-turn-deadline-tests.js +281 -0
- package/dist/scripts/anything-tests.js +70 -21
- package/dist/scripts/benchmark-environment-bridge-e2e.js +139 -37
- package/dist/scripts/benchmark-environment-bridge-tests.js +1099 -306
- package/dist/scripts/booking-copilot-availability-ledger-binding-tests.js +26 -26
- package/dist/scripts/{booking-copilot-availability-policy-v2-tests.js → booking-copilot-availability-policy-tests.js} +115 -115
- package/dist/scripts/booking-copilot-crossrepo-fixture-server.js +41 -1
- package/dist/scripts/booking-copilot-dsh-core-proof-tests.js +75 -120
- package/dist/scripts/booking-copilot-dsh-planner-proof-tests.js +370 -61
- package/dist/scripts/booking-copilot-dsh-plugin-proof-tests.js +2 -2
- package/dist/scripts/booking-copilot-event-sequence-concurrency-proof-tests.js +7 -3
- package/dist/scripts/booking-copilot-gap-code-contract-proof-tests.js +15 -9
- package/dist/scripts/booking-copilot-operation-ledger-concurrency-proof-tests.js +19 -11
- package/dist/scripts/booking-copilot-receipt-ledger-concurrency-proof-tests.js +16 -52
- package/dist/scripts/booking-copilot-runtime-proof-tests.js +3953 -155
- package/dist/scripts/booking-copilot-server-proof-tests.js +58 -23
- package/dist/scripts/booking-copilot-startup-proof-tests.js +10 -119
- package/dist/scripts/booking-surface-contract-proof-tests.js +1154 -326
- package/dist/scripts/bootstrap-tests.js +113 -2
- package/dist/scripts/build-metrics-report.js +339 -0
- package/dist/scripts/channel-probe-tests.js +148 -0
- package/dist/scripts/channel-probe.js +252 -0
- package/dist/scripts/channel-registry-tests.js +262 -0
- package/dist/scripts/doctor-tests.js +175 -0
- package/dist/scripts/effect-tests.js +274 -1
- package/dist/scripts/extension-tests.js +129 -11
- package/dist/scripts/fact-gate-tests.js +91 -2
- package/dist/scripts/flyai-tests.js +17 -1
- package/dist/scripts/hbcli-e2e-tests.js +1 -3
- package/dist/scripts/hbcli-tests.js +38 -1
- package/dist/scripts/metrics-report-tests.js +288 -0
- package/dist/scripts/persona-surface-guard-tests.js +19 -0
- package/dist/scripts/session-tests.js +401 -1
- package/dist/scripts/smoke.js +100 -83
- package/dist/scripts/turn-handoff-collect-tests.js +216 -0
- package/dist/scripts/turn-handoff-collect.js +189 -0
- package/dist/scripts/turn-policy-tests.js +56 -0
- package/dist/scripts/typed-contract-canary.js +261 -0
- package/dist/scripts/visa-policy-tests.js +75 -0
- package/dist/scripts/wish-channel-gate-tests.js +111 -0
- package/dist/src/artifact-gate.js +91 -3
- package/dist/src/benchmark-agent-conformance.js +39 -16
- package/dist/src/benchmark-environment-bridge.js +316 -52
- package/dist/src/bookable-facts.js +62 -3
- package/dist/src/booking-surface/{availability-policy-v2.js → availability-policy.js} +89 -89
- package/dist/src/booking-surface/canonical-schema.js +8 -22
- package/dist/src/booking-surface/contracts.js +70 -19
- package/dist/src/booking-surface/dsh-planner.js +310 -177
- package/dist/src/booking-surface/dsh-plugin.js +3 -3
- package/dist/src/booking-surface/index.js +1 -5
- package/dist/src/booking-surface/profile.js +1 -1
- package/dist/src/booking-surface/runtime.js +1682 -247
- package/dist/src/booking-surface/server.js +371 -185
- package/dist/src/booking-surface/startup.js +17 -27
- package/dist/src/booking-surface/validation.js +270 -760
- package/dist/src/index.js +739 -150
- package/dist/src/turn-deadline.js +292 -0
- package/dist/src/turn-policy.js +146 -0
- package/dist/src/wish-pool.js +5 -0
- package/extension/background.js +23 -4
- package/extension/content-main.js +47 -9
- package/extension/manifest.json +27 -9
- package/package.json +7 -26
- package/schemas/booking.surface.schema.json +2593 -0
- package/ts/capabilities/agent-reach.ts +4 -4
- package/ts/capabilities/anything.ts +89 -38
- package/ts/capabilities/flyai.ts +21 -3
- package/ts/capabilities/hbcli.ts +53 -4
- package/ts/capabilities/session/action-cache.ts +1 -1
- package/ts/capabilities/session/adapters/ctrip-hotel.ts +238 -0
- package/ts/capabilities/session/adapters/rail-12306.ts +293 -0
- package/ts/capabilities/session/extension-bridge.ts +32 -2
- package/ts/capabilities/session/extension-channel.ts +9 -8
- package/ts/capabilities/session/extension-distribution.ts +3 -1
- package/ts/capabilities/session/health-watch.ts +1 -1
- package/ts/capabilities/session/read-guard.ts +1 -1
- package/ts/capabilities/session/wizard.ts +1 -1
- package/ts/capabilities/session-search.ts +323 -1
- package/ts/dsh-runtime/vendor/README.md +60 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/LICENSE +21 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/README.en.md +200 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/README.md +202 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/client/client.js +216 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/cordis.patch.yml +5 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/clients/amap.js +371 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/clients/nominatim.js +63 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/clients/osrm.js +56 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/clients/photon.js +55 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/config-file.js +87 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/config-route.js +126 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/config.js +16 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/index.js +109 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/settings-ns.js +27 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/tools/geocode.js +127 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/tools/poi.js +84 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/tools/routes.js +170 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/clients/amap.d.ts +53 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/clients/nominatim.d.ts +17 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/clients/osrm.d.ts +15 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/clients/photon.d.ts +22 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/config-file.d.ts +33 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/config-route.d.ts +20 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/config.d.ts +23 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/index.d.ts +18 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/settings-ns.d.ts +14 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/tools/geocode.d.ts +11 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/tools/poi.d.ts +9 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/tools/routes.d.ts +16 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/types.d.ts +55 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types.js +16 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/package.json +91 -0
- package/ts/package.json +9 -0
- package/ts/scripts/turn-handoff-collect.ts +178 -0
- package/ts/src/artifact-gate.ts +82 -2
- package/ts/src/benchmark-agent-conformance.ts +54 -15
- package/ts/src/benchmark-environment-bridge.ts +234 -46
- package/ts/src/bookable-facts.ts +109 -5
- package/ts/src/booking-saga.ts +1 -1
- package/ts/src/booking-surface/{availability-policy-v2.ts → availability-policy.ts} +124 -125
- package/ts/src/booking-surface/canonical-schema.js +8 -22
- package/ts/src/booking-surface/contracts.ts +278 -239
- package/ts/src/booking-surface/dsh-planner.ts +333 -194
- package/ts/src/booking-surface/dsh-plugin.js +3 -3
- package/ts/src/booking-surface/index.ts +1 -5
- package/ts/src/booking-surface/profile.ts +11 -11
- package/ts/src/booking-surface/runtime.ts +1351 -351
- package/ts/src/booking-surface/server.ts +282 -175
- package/ts/src/booking-surface/startup.ts +36 -48
- package/ts/src/booking-surface/validation.ts +194 -442
- package/ts/src/contracts.ts +1 -1
- package/ts/src/index.ts +492 -172
- package/ts/src/loop.ts +1 -1
- package/ts/src/state-ledger.ts +1 -1
- package/ts/src/turn-deadline.ts +365 -0
- package/ts/src/turn-policy.ts +154 -0
- package/ts/src/wish-pool.ts +17 -4
- package/dist/scripts/agent-planning-budget-e2e.js +0 -227
- package/dist/scripts/agent-planning-budget-tests.js +0 -173
- package/dist/scripts/booking-copilot-v2-runtime-proof-tests.js +0 -3935
- package/dist/scripts/booking-surface-v2-contract-proof-tests.js +0 -1174
- package/dist/src/booking-surface/contracts-v2.js +0 -89
- package/dist/src/booking-surface/runtime-v2.js +0 -1771
- package/dist/src/booking-surface/server-v2.js +0 -334
- package/dist/src/booking-surface/validation-v2.js +0 -319
- package/dist/src/tool-budget.js +0 -136
- package/schemas/booking.surface.v1.schema.json +0 -927
- package/schemas/booking.surface.v2.schema.json +0 -61
- package/ts/src/booking-surface/contracts-v2.ts +0 -118
- package/ts/src/booking-surface/runtime-v2.ts +0 -1466
- package/ts/src/booking-surface/server-v2.ts +0 -247
- package/ts/src/booking-surface/validation-v2.ts +0 -205
- package/ts/src/tool-budget.ts +0 -165
|
@@ -4,7 +4,7 @@ import { tmpdir } from 'node:os';
|
|
|
4
4
|
import { join } from 'node:path';
|
|
5
5
|
import { Context } from '@deepseek-ai/cordis';
|
|
6
6
|
import { apply } from '../src/index.js';
|
|
7
|
-
import { registerBenchmarkEnvironmentBridge } from '../src/benchmark-environment-bridge.js';
|
|
7
|
+
import { BENCHMARK_TOOL_RESULT_SCHEMA_VERSION, registerBenchmarkEnvironmentBridge } from '../src/benchmark-environment-bridge.js';
|
|
8
8
|
import { installBenchmarkToolIsolation } from '../src/benchmark-tool-isolation.js';
|
|
9
9
|
import { BENCHMARK_BRIDGE_CALL_FAILED, BENCHMARK_BRIDGE_CALL_REQUIRED, BENCHMARK_BRIDGE_RETRY_CALL_NOT_ALLOWED, BENCHMARK_BRIDGE_OUTPUT_TRUNCATED, BENCHMARK_BRIDGE_RUNNER_FAILED, BENCHMARK_BRIDGE_SPAWN_FAILED, BENCHMARK_BRIDGE_TIMED_OUT, BENCHMARK_CONFORMANCE_STATE_UNAVAILABLE, BENCHMARK_TERMINAL_INVALID, MAX_CONFORMANCE_RETRIES, benchmarkChildFailureForConformanceCode, createBenchmarkAgentConformance, installBenchmarkAgentConformance, parseBenchmarkTerminal, validateTerminalOutputConfig } from '../src/benchmark-agent-conformance.js';
|
|
10
10
|
import { BENCHMARK_CHILD_DIAGNOSTIC_MAX_BYTES, BENCHMARK_CHILD_DIAGNOSTIC_SCHEMA, appendBoundedChildDiagnostic, classifyBenchmarkChildFailure, classifyBenchmarkTurnEnd, createBenchmarkDiagnosticArbiter, parseBenchmarkChildDiagnostic } from '../src/benchmark-headless-child-diagnostics.js';
|
|
@@ -314,6 +314,30 @@ for (const invalid of [
|
|
|
314
314
|
'<done>{"ok":</done>'
|
|
315
315
|
])assert.equal(parseBenchmarkTerminal(invalid, projection.terminal).ok, false);
|
|
316
316
|
assert.equal(parseBenchmarkTerminal(`<done>{"x":"${'y'.repeat(1024)}"}</done>`, projection.terminal).ok, false);
|
|
317
|
+
for (const [raw, expectOk] of [
|
|
318
|
+
[
|
|
319
|
+
'<think>plan it</think><done>{"ok":true}</done>',
|
|
320
|
+
true
|
|
321
|
+
],
|
|
322
|
+
[
|
|
323
|
+
'<think>a</think>\n<THINK>b</THINK>\n<done>{"ok":true}</done>',
|
|
324
|
+
true
|
|
325
|
+
],
|
|
326
|
+
[
|
|
327
|
+
'<done>{"ok":true}</done><think>after</think>',
|
|
328
|
+
true
|
|
329
|
+
],
|
|
330
|
+
[
|
|
331
|
+
'<think>unclosed<done>{"ok":true}</done>',
|
|
332
|
+
false
|
|
333
|
+
],
|
|
334
|
+
[
|
|
335
|
+
'prose<think>x</think><done>{"ok":true}</done>',
|
|
336
|
+
false
|
|
337
|
+
]
|
|
338
|
+
]){
|
|
339
|
+
assert.equal(parseBenchmarkTerminal(raw, projection.terminal).ok, expectOk, raw);
|
|
340
|
+
}
|
|
317
341
|
function turnStart(turn = 1) {
|
|
318
342
|
return {
|
|
319
343
|
type: 'turn/start',
|
|
@@ -343,17 +367,15 @@ function toolCall(callId = 'call-1', options = {}) {
|
|
|
343
367
|
callId,
|
|
344
368
|
name: projection.toolName,
|
|
345
369
|
arguments: JSON.stringify({
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
arguments: {}
|
|
350
|
-
}
|
|
370
|
+
action,
|
|
371
|
+
tool,
|
|
372
|
+
arguments: {}
|
|
351
373
|
})
|
|
352
374
|
}
|
|
353
375
|
};
|
|
354
376
|
}
|
|
355
377
|
function toolResult(callId = 'call-1', options = {}) {
|
|
356
|
-
const { turn = 1, step = 1, ok = true, isError = false, error = 'runner_failed' } = options;
|
|
378
|
+
const { turn = 1, step = 1, ok = true, isError = false, error = 'runner_failed', outcome, payload } = options;
|
|
357
379
|
return {
|
|
358
380
|
type: 'tool/result',
|
|
359
381
|
data: {
|
|
@@ -372,13 +394,16 @@ function toolResult(callId = 'call-1', options = {}) {
|
|
|
372
394
|
content: [
|
|
373
395
|
{
|
|
374
396
|
type: 'text',
|
|
375
|
-
text: JSON.stringify(
|
|
397
|
+
text: JSON.stringify(payload ?? (outcome ? {
|
|
398
|
+
ok: true,
|
|
399
|
+
outcome
|
|
400
|
+
} : ok ? {
|
|
376
401
|
ok: true,
|
|
377
402
|
result: {}
|
|
378
403
|
} : {
|
|
379
404
|
ok: false,
|
|
380
405
|
error
|
|
381
|
-
})
|
|
406
|
+
}))
|
|
382
407
|
}
|
|
383
408
|
]
|
|
384
409
|
}
|
|
@@ -387,7 +412,107 @@ function toolResult(callId = 'call-1', options = {}) {
|
|
|
387
412
|
}
|
|
388
413
|
};
|
|
389
414
|
}
|
|
390
|
-
|
|
415
|
+
{
|
|
416
|
+
const state = createBenchmarkAgentConformance(projection);
|
|
417
|
+
state.observe(turnStart());
|
|
418
|
+
state.observe(toolCall());
|
|
419
|
+
state.observe(toolResult('call-1', {
|
|
420
|
+
outcome: {
|
|
421
|
+
schema_version: 'gotry_benchmark_tool_result_v1',
|
|
422
|
+
status: 'miss',
|
|
423
|
+
code: 'NOT_FOUND',
|
|
424
|
+
recovery: 'revise_arguments'
|
|
425
|
+
}
|
|
426
|
+
}));
|
|
427
|
+
assert.deepEqual(state.stopping(1), {
|
|
428
|
+
kind: 'steer',
|
|
429
|
+
mode: 'terminal'
|
|
430
|
+
}, 'domain outcome alone permits tagged terminal');
|
|
431
|
+
state.observe(assistant('<done>{"status":"miss"}</done>', {
|
|
432
|
+
step: 2
|
|
433
|
+
}));
|
|
434
|
+
assert.deepEqual(state.stopping(1), {
|
|
435
|
+
kind: 'accept'
|
|
436
|
+
}, 'domain-only tagged terminal is accepted');
|
|
437
|
+
}{
|
|
438
|
+
const state = createBenchmarkAgentConformance(projection);
|
|
439
|
+
state.observe(turnStart());
|
|
440
|
+
state.observe(toolCall());
|
|
441
|
+
state.observe(toolResult('call-1', {
|
|
442
|
+
outcome: {
|
|
443
|
+
schema_version: 'gotry_benchmark_tool_result_v1',
|
|
444
|
+
status: 'miss',
|
|
445
|
+
code: 'NOT_FOUND',
|
|
446
|
+
recovery: 'revise_arguments'
|
|
447
|
+
}
|
|
448
|
+
}));
|
|
449
|
+
state.observe(toolCall('call-2', {
|
|
450
|
+
step: 2
|
|
451
|
+
}));
|
|
452
|
+
state.observe(toolResult('call-2'));
|
|
453
|
+
state.observe(assistant('<done>{"status":"succeeded"}</done>', {
|
|
454
|
+
step: 3
|
|
455
|
+
}));
|
|
456
|
+
assert.deepEqual(state.stopping(1), {
|
|
457
|
+
kind: 'accept'
|
|
458
|
+
}, 'domain outcome followed by concrete result accepts');
|
|
459
|
+
}{
|
|
460
|
+
const state = createBenchmarkAgentConformance(projection);
|
|
461
|
+
state.observe(turnStart());
|
|
462
|
+
state.observe(toolCall());
|
|
463
|
+
state.observe(toolResult('call-1', {
|
|
464
|
+
outcome: {
|
|
465
|
+
schema_version: 'gotry_benchmark_tool_result_v1',
|
|
466
|
+
status: 'miss',
|
|
467
|
+
code: 'NOT_FOUND',
|
|
468
|
+
recovery: 'revise_arguments'
|
|
469
|
+
}
|
|
470
|
+
}));
|
|
471
|
+
state.observe(toolCall('call-2', {
|
|
472
|
+
step: 2
|
|
473
|
+
}));
|
|
474
|
+
state.observe(toolResult('call-2', {
|
|
475
|
+
ok: false,
|
|
476
|
+
error: 'runner_failed'
|
|
477
|
+
}));
|
|
478
|
+
assert.deepEqual(state.stopping(1), {
|
|
479
|
+
kind: 'reject',
|
|
480
|
+
code: BENCHMARK_BRIDGE_RUNNER_FAILED
|
|
481
|
+
}, 'infra failure overrides domain-only outcome');
|
|
482
|
+
}{
|
|
483
|
+
const state = createBenchmarkAgentConformance(projection);
|
|
484
|
+
state.observe(turnStart());
|
|
485
|
+
state.observe(toolCall());
|
|
486
|
+
state.observe(toolResult('call-1', {
|
|
487
|
+
payload: {
|
|
488
|
+
ok: true,
|
|
489
|
+
result: {},
|
|
490
|
+
outcome: {
|
|
491
|
+
status: 'miss'
|
|
492
|
+
}
|
|
493
|
+
}
|
|
494
|
+
}));
|
|
495
|
+
state.observe(assistant('<done>{"status":"succeeded"}</done>'));
|
|
496
|
+
assert.deepEqual(state.stopping(1), {
|
|
497
|
+
kind: 'reject',
|
|
498
|
+
code: BENCHMARK_BRIDGE_CALL_FAILED
|
|
499
|
+
}, 'ambiguous ok wrapper with result and outcome fails closed');
|
|
500
|
+
}{
|
|
501
|
+
const state = createBenchmarkAgentConformance(projection);
|
|
502
|
+
state.observe(turnStart());
|
|
503
|
+
state.observe(toolCall());
|
|
504
|
+
state.observe(toolResult('call-1', {
|
|
505
|
+
payload: {
|
|
506
|
+
ok: true,
|
|
507
|
+
outcome: {}
|
|
508
|
+
}
|
|
509
|
+
}));
|
|
510
|
+
state.observe(assistant('<done>{"status":"miss"}</done>'));
|
|
511
|
+
assert.deepEqual(state.stopping(1), {
|
|
512
|
+
kind: 'reject',
|
|
513
|
+
code: BENCHMARK_BRIDGE_CALL_FAILED
|
|
514
|
+
}, 'malformed inner domain outcome fails closed');
|
|
515
|
+
}function assistant(text, options = {}) {
|
|
391
516
|
const { turn = 1, step = 2, interrupted = false } = options;
|
|
392
517
|
return {
|
|
393
518
|
type: 'assistant/message',
|
|
@@ -566,6 +691,69 @@ function assistant(text, options = {}) {
|
|
|
566
691
|
assert.deepEqual(state.stopping(1), {
|
|
567
692
|
kind: 'accept'
|
|
568
693
|
}, 'a later failed optional call does not erase an already paired successful result');
|
|
694
|
+
}{
|
|
695
|
+
const state = createBenchmarkAgentConformance(projection);
|
|
696
|
+
state.observe(turnStart());
|
|
697
|
+
state.observe(toolCall('successful', {
|
|
698
|
+
step: 1
|
|
699
|
+
}));
|
|
700
|
+
state.observe(toolResult('successful', {
|
|
701
|
+
step: 1
|
|
702
|
+
}));
|
|
703
|
+
state.observe(assistant('<done>{"status":"succeeded"}</done>', {
|
|
704
|
+
step: 2
|
|
705
|
+
}));
|
|
706
|
+
state.observe(toolCall('domain', {
|
|
707
|
+
step: 3
|
|
708
|
+
}));
|
|
709
|
+
state.observe(toolResult('domain', {
|
|
710
|
+
step: 3,
|
|
711
|
+
outcome: {
|
|
712
|
+
schema_version: 'gotry_benchmark_tool_result_v1',
|
|
713
|
+
status: 'miss',
|
|
714
|
+
code: 'NOT_FOUND',
|
|
715
|
+
recovery: 'revise_arguments'
|
|
716
|
+
}
|
|
717
|
+
}));
|
|
718
|
+
assert.deepEqual(state.stopping(1), {
|
|
719
|
+
kind: 'steer',
|
|
720
|
+
mode: 'terminal'
|
|
721
|
+
}, 'a terminal before the latest domain response is stale');
|
|
722
|
+
state.observe(assistant('<done>{"status":"succeeded"}</done>', {
|
|
723
|
+
step: 4
|
|
724
|
+
}));
|
|
725
|
+
assert.deepEqual(state.stopping(1), {
|
|
726
|
+
kind: 'accept'
|
|
727
|
+
}, 'a fresh terminal after the latest domain response may reuse the earlier concrete result');
|
|
728
|
+
}{
|
|
729
|
+
const state = createBenchmarkAgentConformance(projection);
|
|
730
|
+
state.observe(turnStart());
|
|
731
|
+
state.observe(toolCall('successful', {
|
|
732
|
+
step: 1
|
|
733
|
+
}));
|
|
734
|
+
state.observe(toolResult('successful', {
|
|
735
|
+
step: 1
|
|
736
|
+
}));
|
|
737
|
+
state.observe(assistant('<done>{"status":"succeeded"}</done>', {
|
|
738
|
+
step: 2
|
|
739
|
+
}));
|
|
740
|
+
state.observe(toolCall('failed', {
|
|
741
|
+
step: 3
|
|
742
|
+
}));
|
|
743
|
+
state.observe(toolResult('failed', {
|
|
744
|
+
step: 3,
|
|
745
|
+
ok: false
|
|
746
|
+
}));
|
|
747
|
+
assert.deepEqual(state.stopping(1), {
|
|
748
|
+
kind: 'steer',
|
|
749
|
+
mode: 'terminal'
|
|
750
|
+
}, 'a terminal before a later optional failure is stale');
|
|
751
|
+
state.observe(assistant('<done>{"status":"succeeded"}</done>', {
|
|
752
|
+
step: 4
|
|
753
|
+
}));
|
|
754
|
+
assert.deepEqual(state.stopping(1), {
|
|
755
|
+
kind: 'accept'
|
|
756
|
+
}, 'a fresh terminal may still converge after an optional failure when a concrete result exists');
|
|
569
757
|
}{
|
|
570
758
|
const state = createBenchmarkAgentConformance(projection);
|
|
571
759
|
state.observe(turnStart());
|
|
@@ -816,16 +1004,77 @@ function assistant(text, options = {}) {
|
|
|
816
1004
|
waitForExit: async ()=>true
|
|
817
1005
|
};
|
|
818
1006
|
}
|
|
1007
|
+
function okEnvelope(result) {
|
|
1008
|
+
return JSON.stringify({
|
|
1009
|
+
schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION,
|
|
1010
|
+
status: 'ok',
|
|
1011
|
+
result
|
|
1012
|
+
});
|
|
1013
|
+
}
|
|
1014
|
+
function lookupInputSchema() {
|
|
1015
|
+
return {
|
|
1016
|
+
type: 'object',
|
|
1017
|
+
properties: {
|
|
1018
|
+
city: {
|
|
1019
|
+
type: 'string',
|
|
1020
|
+
enum: [
|
|
1021
|
+
'Dubai',
|
|
1022
|
+
'Abu Dhabi'
|
|
1023
|
+
],
|
|
1024
|
+
description: 'Declared city name.'
|
|
1025
|
+
},
|
|
1026
|
+
payload: {
|
|
1027
|
+
type: 'string'
|
|
1028
|
+
},
|
|
1029
|
+
notes: {
|
|
1030
|
+
type: 'string'
|
|
1031
|
+
},
|
|
1032
|
+
executable: {
|
|
1033
|
+
type: 'string'
|
|
1034
|
+
},
|
|
1035
|
+
cwd: {
|
|
1036
|
+
type: 'string'
|
|
1037
|
+
},
|
|
1038
|
+
argv: {
|
|
1039
|
+
type: 'array',
|
|
1040
|
+
items: {
|
|
1041
|
+
type: 'string'
|
|
1042
|
+
}
|
|
1043
|
+
}
|
|
1044
|
+
},
|
|
1045
|
+
required: [
|
|
1046
|
+
'city'
|
|
1047
|
+
],
|
|
1048
|
+
additionalProperties: false
|
|
1049
|
+
};
|
|
1050
|
+
}
|
|
1051
|
+
function bridgeCall(tool, argumentsValue) {
|
|
1052
|
+
return {
|
|
1053
|
+
action: 'call',
|
|
1054
|
+
tool,
|
|
1055
|
+
arguments: argumentsValue
|
|
1056
|
+
};
|
|
1057
|
+
}
|
|
819
1058
|
async function assertRealCordisWaterfallOrdering() {
|
|
820
1059
|
const ctx = new Context();
|
|
821
1060
|
const bridge = {
|
|
822
1061
|
name: 'gotry_benchmark_environment',
|
|
823
1062
|
description: 'benchmark bridge',
|
|
824
1063
|
parameters: {
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
1064
|
+
oneOf: [
|
|
1065
|
+
{
|
|
1066
|
+
type: 'object',
|
|
1067
|
+
properties: {
|
|
1068
|
+
action: {
|
|
1069
|
+
const: 'tools'
|
|
1070
|
+
}
|
|
1071
|
+
},
|
|
1072
|
+
required: [
|
|
1073
|
+
'action'
|
|
1074
|
+
],
|
|
1075
|
+
additionalProperties: false
|
|
1076
|
+
}
|
|
1077
|
+
]
|
|
829
1078
|
}
|
|
830
1079
|
};
|
|
831
1080
|
const exactSchema = structuredClone(bridge);
|
|
@@ -957,7 +1206,7 @@ try {
|
|
|
957
1206
|
assert.equal(parseBenchmarkChildDiagnostic(boundedControl.toString('utf8')), 'child_bridge_timed_out', 'rolling tail preserves a final structured reason after bounded noise');
|
|
958
1207
|
const configPath = join(root, 'bridge.json');
|
|
959
1208
|
writeFileSync(configPath, JSON.stringify({
|
|
960
|
-
schema_version: '
|
|
1209
|
+
schema_version: 'gotry_benchmark_environment_bridge_v3',
|
|
961
1210
|
enabled: true,
|
|
962
1211
|
executable: process.execPath,
|
|
963
1212
|
cwd: root,
|
|
@@ -967,20 +1216,54 @@ try {
|
|
|
967
1216
|
'--lang',
|
|
968
1217
|
'en'
|
|
969
1218
|
],
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
1219
|
+
tools: [
|
|
1220
|
+
{
|
|
1221
|
+
name: 'lookup',
|
|
1222
|
+
description: 'Lookup one declared city.',
|
|
1223
|
+
input_schema: lookupInputSchema(),
|
|
1224
|
+
output_keys: [
|
|
1225
|
+
'city',
|
|
1226
|
+
'nested'
|
|
1227
|
+
],
|
|
1228
|
+
domain_outcomes: [
|
|
1229
|
+
{
|
|
1230
|
+
status: 'miss',
|
|
1231
|
+
code: 'NOT_FOUND',
|
|
1232
|
+
recovery: 'revise_arguments'
|
|
1233
|
+
},
|
|
1234
|
+
{
|
|
1235
|
+
status: 'error',
|
|
1236
|
+
code: 'AMBIGUOUS',
|
|
1237
|
+
recovery: 'choose_alternative'
|
|
1238
|
+
}
|
|
1239
|
+
]
|
|
1240
|
+
},
|
|
1241
|
+
{
|
|
1242
|
+
name: 'constructor',
|
|
1243
|
+
description: 'Construct one declared city.',
|
|
1244
|
+
input_schema: lookupInputSchema(),
|
|
1245
|
+
output_keys: [
|
|
1246
|
+
'legacy'
|
|
1247
|
+
],
|
|
1248
|
+
domain_outcomes: [
|
|
1249
|
+
{
|
|
1250
|
+
status: 'error',
|
|
1251
|
+
code: 'INVALID',
|
|
1252
|
+
recovery: 'retry_same'
|
|
1253
|
+
}
|
|
1254
|
+
]
|
|
1255
|
+
},
|
|
1256
|
+
{
|
|
1257
|
+
name: 'toString',
|
|
1258
|
+
description: 'Stringify one declared city.',
|
|
1259
|
+
input_schema: lookupInputSchema(),
|
|
1260
|
+
output_keys: [
|
|
1261
|
+
'city',
|
|
1262
|
+
'nested'
|
|
1263
|
+
],
|
|
1264
|
+
domain_outcomes: []
|
|
1265
|
+
}
|
|
974
1266
|
],
|
|
975
|
-
allowed_output_keys: {
|
|
976
|
-
lookup: [
|
|
977
|
-
'city',
|
|
978
|
-
'nested'
|
|
979
|
-
],
|
|
980
|
-
constructor: [
|
|
981
|
-
'legacy'
|
|
982
|
-
]
|
|
983
|
-
},
|
|
984
1267
|
timeout_ms: 20,
|
|
985
1268
|
max_output_bytes: 4_096,
|
|
986
1269
|
terminal_output: {
|
|
@@ -1026,7 +1309,11 @@ try {
|
|
|
1026
1309
|
let timeoutSignal;
|
|
1027
1310
|
const outcomes = [
|
|
1028
1311
|
{
|
|
1029
|
-
stdout:
|
|
1312
|
+
stdout: okEnvelope([
|
|
1313
|
+
{
|
|
1314
|
+
city: 'Dubai'
|
|
1315
|
+
}
|
|
1316
|
+
])
|
|
1030
1317
|
}
|
|
1031
1318
|
];
|
|
1032
1319
|
process.env.GOTRY_BENCHMARK_BRIDGE_PARENT_SENTINEL = 'must-not-cross-boundary';
|
|
@@ -1086,7 +1373,7 @@ try {
|
|
|
1086
1373
|
spawn (spec) {
|
|
1087
1374
|
spawnSpecs.push(spec);
|
|
1088
1375
|
const outcome = outcomes.shift() ?? {
|
|
1089
|
-
stdout:
|
|
1376
|
+
stdout: okEnvelope({})
|
|
1090
1377
|
};
|
|
1091
1378
|
if (outcome.spawnError) throw new Error('fake spawn failed');
|
|
1092
1379
|
if (outcome.waitForAbort) {
|
|
@@ -1603,17 +1890,57 @@ try {
|
|
|
1603
1890
|
}, 'agent disposal releases its quarantined isolation and conformance effects');
|
|
1604
1891
|
const bridge = registered.find((tool)=>tool.name === 'gotry_benchmark_environment');
|
|
1605
1892
|
assert.ok(bridge.execute, 'registered bridge exposes execute');
|
|
1893
|
+
const parameters = bridge.parameters;
|
|
1894
|
+
assert.equal(Array.isArray(parameters.oneOf), true, 'v3 bridge exposes flat descriptor-derived model branches');
|
|
1895
|
+
const lookupBranch = parameters.oneOf.find((branch)=>branch?.properties?.tool?.const === 'lookup');
|
|
1896
|
+
assert.ok(lookupBranch, 'model schema contains the configured lookup tool branch');
|
|
1897
|
+
assert.equal(lookupBranch.description, 'Lookup one declared city.', 'descriptor description reaches the model-visible call branch');
|
|
1898
|
+
assert.deepEqual(lookupBranch.required, [
|
|
1899
|
+
'action',
|
|
1900
|
+
'tool',
|
|
1901
|
+
'arguments'
|
|
1902
|
+
]);
|
|
1903
|
+
assert.equal(lookupBranch.additionalProperties, false);
|
|
1904
|
+
assert.deepEqual(lookupBranch.properties.arguments.required, [
|
|
1905
|
+
'city'
|
|
1906
|
+
]);
|
|
1907
|
+
assert.equal(lookupBranch.properties.arguments.additionalProperties, false);
|
|
1908
|
+
assert.equal(lookupBranch.properties.arguments.properties.city.type, 'string');
|
|
1909
|
+
assert.deepEqual(lookupBranch.properties.arguments.properties.city.enum, [
|
|
1910
|
+
'Dubai',
|
|
1911
|
+
'Abu Dhabi'
|
|
1912
|
+
]);
|
|
1913
|
+
assert.equal(Object.isFrozen(bridge.parameters), true, 'registered raw parameters are frozen');
|
|
1914
|
+
assert.equal(Object.isFrozen(lookupBranch.properties.arguments.properties.city.enum), true, 'descriptor-derived nested schema is frozen');
|
|
1915
|
+
assert.throws(()=>{
|
|
1916
|
+
lookupBranch.properties.arguments.properties.city.enum.push('escape');
|
|
1917
|
+
}, TypeError);
|
|
1918
|
+
const beforeProtocolRejected = spawnSpecs.length;
|
|
1919
|
+
await assert.rejects(()=>bridge.execute({
|
|
1920
|
+
query: bridgeCall('lookup', {
|
|
1921
|
+
city: 'Dubai'
|
|
1922
|
+
})
|
|
1923
|
+
}, null), /invalid arguments/, 'legacy nested query envelope is rejected by the flat protocol');
|
|
1924
|
+
await assert.rejects(()=>bridge.execute({}, null), /invalid arguments/, 'empty object is rejected by the flat protocol');
|
|
1925
|
+
await assert.rejects(()=>bridge.execute({
|
|
1926
|
+
action: 'tools',
|
|
1927
|
+
tool: 'lookup'
|
|
1928
|
+
}, null), /invalid arguments/, 'mixed action/tool fields are rejected by the flat protocol');
|
|
1929
|
+
await assert.rejects(()=>bridge.execute({
|
|
1930
|
+
...bridgeCall('lookup', {
|
|
1931
|
+
city: 'Dubai'
|
|
1932
|
+
}),
|
|
1933
|
+
extra: true
|
|
1934
|
+
}, null), /invalid arguments/, 'extra top-level fields are rejected by the flat protocol');
|
|
1935
|
+
await assert.rejects(()=>bridge.execute(bridgeCall('lookup', {
|
|
1936
|
+
city: 'Sharjah'
|
|
1937
|
+
}), null), /invalid arguments/, 'invalid descriptor argument values are rejected by the flat protocol');
|
|
1938
|
+
assert.equal(spawnSpecs.length, beforeProtocolRejected, 'flat protocol rejections happen before spawn');
|
|
1606
1939
|
const args = {
|
|
1607
1940
|
city: 'Dubai',
|
|
1608
1941
|
payload: '$(touch /tmp/nope)'
|
|
1609
1942
|
};
|
|
1610
|
-
const result = await bridge.execute(
|
|
1611
|
-
query: {
|
|
1612
|
-
action: 'call',
|
|
1613
|
-
tool: 'lookup',
|
|
1614
|
-
arguments: args
|
|
1615
|
-
}
|
|
1616
|
-
}, null);
|
|
1943
|
+
const result = await bridge.execute(bridgeCall('lookup', args), null);
|
|
1617
1944
|
assert.deepEqual(spawnSpecs[0]?.argv, [
|
|
1618
1945
|
process.execPath,
|
|
1619
1946
|
'-m',
|
|
@@ -1650,17 +1977,13 @@ try {
|
|
|
1650
1977
|
]
|
|
1651
1978
|
}, 'one-line JSON stdout becomes structured result');
|
|
1652
1979
|
outcomes.push({
|
|
1653
|
-
stdout:
|
|
1980
|
+
stdout: okEnvelope({
|
|
1981
|
+
city: 'Dubai'
|
|
1982
|
+
})
|
|
1654
1983
|
});
|
|
1655
|
-
const unmappedResult = await bridge.execute({
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
tool: 'toString',
|
|
1659
|
-
arguments: {
|
|
1660
|
-
city: 'Dubai'
|
|
1661
|
-
}
|
|
1662
|
-
}
|
|
1663
|
-
}, null);
|
|
1984
|
+
const unmappedResult = await bridge.execute(bridgeCall('toString', {
|
|
1985
|
+
city: 'Dubai'
|
|
1986
|
+
}), null);
|
|
1664
1987
|
assert.deepEqual(unmappedResult, {
|
|
1665
1988
|
ok: true,
|
|
1666
1989
|
result: {
|
|
@@ -1668,17 +1991,13 @@ try {
|
|
|
1668
1991
|
}
|
|
1669
1992
|
}, 'allowed tool without a positive mapping retains the recursive denylist only');
|
|
1670
1993
|
outcomes.push({
|
|
1671
|
-
stdout:
|
|
1994
|
+
stdout: okEnvelope({
|
|
1995
|
+
legacy: 'value'
|
|
1996
|
+
})
|
|
1672
1997
|
});
|
|
1673
|
-
const legacyResult = await bridge.execute({
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
tool: 'constructor',
|
|
1677
|
-
arguments: {
|
|
1678
|
-
city: 'Dubai'
|
|
1679
|
-
}
|
|
1680
|
-
}
|
|
1681
|
-
}, null);
|
|
1998
|
+
const legacyResult = await bridge.execute(bridgeCall('constructor', {
|
|
1999
|
+
city: 'Dubai'
|
|
2000
|
+
}), null);
|
|
1682
2001
|
assert.deepEqual(legacyResult, {
|
|
1683
2002
|
ok: true,
|
|
1684
2003
|
result: {
|
|
@@ -1686,33 +2005,27 @@ try {
|
|
|
1686
2005
|
}
|
|
1687
2006
|
}, 'mapped constructor accepts its declared positive key');
|
|
1688
2007
|
outcomes.push({
|
|
1689
|
-
stdout:
|
|
2008
|
+
stdout: okEnvelope({
|
|
2009
|
+
city: 'Dubai'
|
|
2010
|
+
})
|
|
1690
2011
|
});
|
|
1691
|
-
const constructorUnexpected = await bridge.execute({
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
tool: 'constructor',
|
|
1695
|
-
arguments: {
|
|
1696
|
-
city: 'Dubai'
|
|
1697
|
-
}
|
|
1698
|
-
}
|
|
1699
|
-
}, null);
|
|
2012
|
+
const constructorUnexpected = await bridge.execute(bridgeCall('constructor', {
|
|
2013
|
+
city: 'Dubai'
|
|
2014
|
+
}), null);
|
|
1700
2015
|
assert.deepEqual(constructorUnexpected, {
|
|
1701
2016
|
ok: false,
|
|
1702
2017
|
error: 'forbidden_output'
|
|
1703
2018
|
}, 'mapped constructor rejects undeclared positive keys');
|
|
1704
2019
|
outcomes.push({
|
|
1705
|
-
stdout:
|
|
1706
|
-
|
|
1707
|
-
const nestedAllowedResult = await bridge.execute({
|
|
1708
|
-
query: {
|
|
1709
|
-
action: 'call',
|
|
1710
|
-
tool: 'lookup',
|
|
1711
|
-
arguments: {
|
|
2020
|
+
stdout: okEnvelope({
|
|
2021
|
+
nested: {
|
|
1712
2022
|
city: 'Dubai'
|
|
1713
2023
|
}
|
|
1714
|
-
}
|
|
1715
|
-
}
|
|
2024
|
+
})
|
|
2025
|
+
});
|
|
2026
|
+
const nestedAllowedResult = await bridge.execute(bridgeCall('lookup', {
|
|
2027
|
+
city: 'Dubai'
|
|
2028
|
+
}), null);
|
|
1716
2029
|
assert.deepEqual(nestedAllowedResult, {
|
|
1717
2030
|
ok: true,
|
|
1718
2031
|
result: {
|
|
@@ -1722,33 +2035,31 @@ try {
|
|
|
1722
2035
|
}
|
|
1723
2036
|
}, 'configured positive output allowlist accepts declared nested keys');
|
|
1724
2037
|
outcomes.push({
|
|
1725
|
-
stdout:
|
|
2038
|
+
stdout: okEnvelope({
|
|
2039
|
+
city: 'Dubai',
|
|
2040
|
+
unexpected: 'secret'
|
|
2041
|
+
})
|
|
1726
2042
|
});
|
|
1727
|
-
const unexpectedResult = await bridge.execute({
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
tool: 'lookup',
|
|
1731
|
-
arguments: {
|
|
1732
|
-
city: 'Dubai'
|
|
1733
|
-
}
|
|
1734
|
-
}
|
|
1735
|
-
}, null);
|
|
2043
|
+
const unexpectedResult = await bridge.execute(bridgeCall('lookup', {
|
|
2044
|
+
city: 'Dubai'
|
|
2045
|
+
}), null);
|
|
1736
2046
|
assert.deepEqual(unexpectedResult, {
|
|
1737
2047
|
ok: false,
|
|
1738
2048
|
error: 'forbidden_output'
|
|
1739
2049
|
}, 'configured positive output allowlist rejects unexpected keys without reflecting them');
|
|
1740
2050
|
outcomes.push({
|
|
1741
|
-
stdout:
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
arguments: {
|
|
1748
|
-
city: 'Dubai'
|
|
2051
|
+
stdout: okEnvelope([
|
|
2052
|
+
{
|
|
2053
|
+
city: 'Dubai',
|
|
2054
|
+
nested: {
|
|
2055
|
+
unexpected: 'secret'
|
|
2056
|
+
}
|
|
1749
2057
|
}
|
|
1750
|
-
|
|
1751
|
-
}
|
|
2058
|
+
])
|
|
2059
|
+
});
|
|
2060
|
+
const nestedUnexpectedResult = await bridge.execute(bridgeCall('lookup', {
|
|
2061
|
+
city: 'Dubai'
|
|
2062
|
+
}), null);
|
|
1752
2063
|
assert.deepEqual(nestedUnexpectedResult, {
|
|
1753
2064
|
ok: false,
|
|
1754
2065
|
error: 'forbidden_output'
|
|
@@ -1773,54 +2084,125 @@ try {
|
|
|
1773
2084
|
'gоld'
|
|
1774
2085
|
]){
|
|
1775
2086
|
outcomes.push({
|
|
1776
|
-
stdout:
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
[forbidden]: 'secret'
|
|
1780
|
-
}
|
|
2087
|
+
stdout: okEnvelope({
|
|
2088
|
+
nested: {
|
|
2089
|
+
[forbidden]: 'secret'
|
|
1781
2090
|
}
|
|
1782
2091
|
})
|
|
1783
2092
|
});
|
|
1784
|
-
const forbiddenResult = await bridge.execute({
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
tool: 'lookup',
|
|
1788
|
-
arguments: {
|
|
1789
|
-
city: 'Dubai'
|
|
1790
|
-
}
|
|
1791
|
-
}
|
|
1792
|
-
}, null);
|
|
2093
|
+
const forbiddenResult = await bridge.execute(bridgeCall('lookup', {
|
|
2094
|
+
city: 'Dubai'
|
|
2095
|
+
}), null);
|
|
1793
2096
|
assert.deepEqual(forbiddenResult, {
|
|
1794
2097
|
ok: false,
|
|
1795
2098
|
error: 'forbidden_output'
|
|
1796
2099
|
}, `recursive no-oracle key ${forbidden} is rejected without reflecting its name`);
|
|
1797
2100
|
}
|
|
1798
2101
|
outcomes.push({
|
|
1799
|
-
stdout: '
|
|
2102
|
+
stdout: okEnvelope('the hidden answer')
|
|
1800
2103
|
});
|
|
1801
|
-
const primitiveOutput = await bridge.execute({
|
|
1802
|
-
|
|
1803
|
-
|
|
1804
|
-
tool: 'lookup',
|
|
1805
|
-
arguments: {
|
|
1806
|
-
city: 'Dubai'
|
|
1807
|
-
}
|
|
1808
|
-
}
|
|
1809
|
-
}, null);
|
|
2104
|
+
const primitiveOutput = await bridge.execute(bridgeCall('lookup', {
|
|
2105
|
+
city: 'Dubai'
|
|
2106
|
+
}), null);
|
|
1810
2107
|
assert.deepEqual(primitiveOutput, {
|
|
1811
2108
|
ok: false,
|
|
1812
2109
|
error: 'invalid_output'
|
|
1813
2110
|
}, 'primitive result strings cannot bypass the structured visible-output boundary');
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
|
|
2111
|
+
outcomes.push({
|
|
2112
|
+
stdout: okEnvelope([
|
|
2113
|
+
'PRIVATE_ARRAY_VALUE'
|
|
2114
|
+
])
|
|
2115
|
+
});
|
|
2116
|
+
const primitiveArrayOutput = await bridge.execute(bridgeCall('lookup', {
|
|
2117
|
+
city: 'Dubai'
|
|
2118
|
+
}), null);
|
|
2119
|
+
assert.deepEqual(primitiveArrayOutput, {
|
|
2120
|
+
ok: false,
|
|
2121
|
+
error: 'forbidden_output'
|
|
2122
|
+
}, 'top-level primitive arrays cannot bypass the positive output-key boundary');
|
|
2123
|
+
assert.equal(JSON.stringify(primitiveArrayOutput).includes('PRIVATE_ARRAY_VALUE'), false, 'rejected primitive array values are not reflected');
|
|
2124
|
+
outcomes.push({
|
|
2125
|
+
stdout: okEnvelope([
|
|
2126
|
+
[
|
|
2127
|
+
"PRIVATE_NESTED_ARRAY_VALUE"
|
|
2128
|
+
]
|
|
2129
|
+
])
|
|
2130
|
+
});
|
|
2131
|
+
const nestedPrimitiveArrayOutput = await bridge.execute(bridgeCall('lookup', {
|
|
2132
|
+
city: 'Dubai'
|
|
2133
|
+
}), null);
|
|
2134
|
+
assert.deepEqual(nestedPrimitiveArrayOutput, {
|
|
2135
|
+
ok: false,
|
|
2136
|
+
error: 'forbidden_output'
|
|
2137
|
+
}, 'nested primitive arrays without a declared-key ancestor fail closed');
|
|
2138
|
+
assert.equal(JSON.stringify(nestedPrimitiveArrayOutput).includes('PRIVATE_NESTED_ARRAY_VALUE'), false, 'rejected nested primitive array values are not reflected');
|
|
2139
|
+
outcomes.push({
|
|
2140
|
+
stdout: okEnvelope([
|
|
2141
|
+
{
|
|
2142
|
+
city: 'Dubai'
|
|
2143
|
+
},
|
|
2144
|
+
'PRIVATE_MIXED_ARRAY_VALUE'
|
|
2145
|
+
])
|
|
2146
|
+
});
|
|
2147
|
+
const mixedArrayOutput = await bridge.execute(bridgeCall('lookup', {
|
|
2148
|
+
city: 'Dubai'
|
|
2149
|
+
}), null);
|
|
2150
|
+
assert.deepEqual(mixedArrayOutput, {
|
|
2151
|
+
ok: false,
|
|
2152
|
+
error: 'forbidden_output'
|
|
2153
|
+
}, 'a valid record cannot mask an unkeyed primitive sibling');
|
|
2154
|
+
assert.equal(JSON.stringify(mixedArrayOutput).includes('PRIVATE_MIXED_ARRAY_VALUE'), false, 'rejected mixed-array primitive values are not reflected');
|
|
2155
|
+
outcomes.push({
|
|
2156
|
+
stdout: okEnvelope([])
|
|
2157
|
+
});
|
|
2158
|
+
const emptyArrayOutput = await bridge.execute(bridgeCall('lookup', {
|
|
2159
|
+
city: 'Dubai'
|
|
2160
|
+
}), null);
|
|
2161
|
+
assert.deepEqual(emptyArrayOutput, {
|
|
2162
|
+
ok: true,
|
|
2163
|
+
result: []
|
|
2164
|
+
}, 'an empty top-level result array is an explicit non-reflecting collection');
|
|
2165
|
+
outcomes.push({
|
|
2166
|
+
stdout: okEnvelope([
|
|
2167
|
+
{
|
|
2168
|
+
city: 'Dubai'
|
|
2169
|
+
}
|
|
2170
|
+
])
|
|
2171
|
+
});
|
|
2172
|
+
const recordArrayOutput = await bridge.execute(bridgeCall('lookup', {
|
|
2173
|
+
city: 'Dubai'
|
|
2174
|
+
}), null);
|
|
2175
|
+
assert.deepEqual(recordArrayOutput, {
|
|
2176
|
+
ok: true,
|
|
2177
|
+
result: [
|
|
2178
|
+
{
|
|
2179
|
+
city: 'Dubai'
|
|
1821
2180
|
}
|
|
2181
|
+
]
|
|
2182
|
+
}, 'top-level arrays of records remain valid when every leaf is covered by a declared key');
|
|
2183
|
+
outcomes.push({
|
|
2184
|
+
stdout: okEnvelope({
|
|
2185
|
+
city: [
|
|
2186
|
+
'Dubai'
|
|
2187
|
+
]
|
|
2188
|
+
})
|
|
2189
|
+
});
|
|
2190
|
+
const keyedPrimitiveArrayOutput = await bridge.execute(bridgeCall('lookup', {
|
|
2191
|
+
city: 'Dubai'
|
|
2192
|
+
}), null);
|
|
2193
|
+
assert.deepEqual(keyedPrimitiveArrayOutput, {
|
|
2194
|
+
ok: true,
|
|
2195
|
+
result: {
|
|
2196
|
+
city: [
|
|
2197
|
+
'Dubai'
|
|
2198
|
+
]
|
|
1822
2199
|
}
|
|
1823
|
-
},
|
|
2200
|
+
}, 'primitive arrays remain valid below a declared output key');
|
|
2201
|
+
const beforeOversized = spawnSpecs.length;
|
|
2202
|
+
const oversized = await bridge.execute(bridgeCall('lookup', {
|
|
2203
|
+
city: 'Dubai',
|
|
2204
|
+
notes: 'x'.repeat(65_537)
|
|
2205
|
+
}), null);
|
|
1824
2206
|
assert.deepEqual(oversized, {
|
|
1825
2207
|
ok: false,
|
|
1826
2208
|
error: 'invalid_arguments',
|
|
@@ -1832,34 +2214,20 @@ try {
|
|
|
1832
2214
|
for(let index = 0; index < 13; index++)deep = {
|
|
1833
2215
|
next: deep
|
|
1834
2216
|
};
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
|
|
1838
|
-
|
|
1839
|
-
|
|
1840
|
-
}
|
|
1841
|
-
}, null);
|
|
1842
|
-
assert.deepEqual(deepResult, {
|
|
1843
|
-
ok: false,
|
|
1844
|
-
error: 'invalid_arguments',
|
|
1845
|
-
reason: 'serialization_limit'
|
|
1846
|
-
});
|
|
1847
|
-
assert.equal(spawnSpecs.length, beforeDeep, 'over-deep arguments are rejected before spawn');
|
|
2217
|
+
await assert.rejects(()=>bridge.execute(bridgeCall('lookup', {
|
|
2218
|
+
city: 'Dubai',
|
|
2219
|
+
...deep
|
|
2220
|
+
}), null), /invalid arguments/);
|
|
2221
|
+
assert.equal(spawnSpecs.length, beforeDeep, 'schema-invalid deep arguments are rejected before spawn');
|
|
1848
2222
|
const beforeOverride = spawnSpecs.length;
|
|
1849
|
-
const overrideResult = await bridge.execute({
|
|
1850
|
-
|
|
1851
|
-
|
|
1852
|
-
|
|
1853
|
-
|
|
1854
|
-
|
|
1855
|
-
|
|
1856
|
-
|
|
1857
|
-
argv: [
|
|
1858
|
-
'--unsafe'
|
|
1859
|
-
]
|
|
1860
|
-
}
|
|
1861
|
-
}
|
|
1862
|
-
}, null);
|
|
2223
|
+
const overrideResult = await bridge.execute(bridgeCall('lookup', {
|
|
2224
|
+
city: 'Dubai',
|
|
2225
|
+
executable: '/tmp/evil',
|
|
2226
|
+
cwd: '/tmp/evil',
|
|
2227
|
+
argv: [
|
|
2228
|
+
'--unsafe'
|
|
2229
|
+
]
|
|
2230
|
+
}), null);
|
|
1863
2231
|
assert.deepEqual(spawnSpecs[beforeOverride]?.argv, [
|
|
1864
2232
|
process.execPath,
|
|
1865
2233
|
'-m',
|
|
@@ -1879,43 +2247,56 @@ try {
|
|
|
1879
2247
|
], 'model executable/cwd/argv fields remain data and cannot override config');
|
|
1880
2248
|
assert.equal(overrideResult.ok, true, 'override-shaped arguments still use the configured bridge');
|
|
1881
2249
|
outcomes.push({
|
|
1882
|
-
stdout:
|
|
2250
|
+
stdout: okEnvelope({
|
|
2251
|
+
city: 'Dubai'
|
|
2252
|
+
}),
|
|
1883
2253
|
lossy: true
|
|
1884
2254
|
});
|
|
1885
|
-
const truncated = await bridge.execute({
|
|
1886
|
-
|
|
1887
|
-
|
|
1888
|
-
tool: 'lookup',
|
|
1889
|
-
arguments: {
|
|
1890
|
-
city: 'Dubai'
|
|
1891
|
-
}
|
|
1892
|
-
}
|
|
1893
|
-
}, null);
|
|
2255
|
+
const truncated = await bridge.execute(bridgeCall('lookup', {
|
|
2256
|
+
city: 'Dubai'
|
|
2257
|
+
}), null);
|
|
1894
2258
|
assert.deepEqual(truncated, {
|
|
1895
2259
|
ok: false,
|
|
1896
2260
|
error: 'output_truncated'
|
|
1897
2261
|
}, 'lossy stdout is rejected without parsing partial output');
|
|
1898
2262
|
outcomes.push({
|
|
1899
|
-
stdout:
|
|
2263
|
+
stdout: okEnvelope({
|
|
2264
|
+
city: 'Dubai'
|
|
2265
|
+
}),
|
|
1900
2266
|
stderr: 'private runner diagnostic',
|
|
1901
2267
|
exitCode: 17,
|
|
1902
2268
|
signal: 'SIGTERM'
|
|
1903
2269
|
});
|
|
1904
|
-
const failed = await bridge.execute({
|
|
1905
|
-
|
|
1906
|
-
|
|
1907
|
-
tool: 'lookup',
|
|
1908
|
-
arguments: {
|
|
1909
|
-
city: 'Dubai'
|
|
1910
|
-
}
|
|
1911
|
-
}
|
|
1912
|
-
}, null);
|
|
2270
|
+
const failed = await bridge.execute(bridgeCall('lookup', {
|
|
2271
|
+
city: 'Dubai'
|
|
2272
|
+
}), null);
|
|
1913
2273
|
assert.deepEqual(failed, {
|
|
1914
2274
|
ok: false,
|
|
1915
2275
|
error: 'runner_failed',
|
|
1916
2276
|
exit_code: 17,
|
|
1917
2277
|
signal: 'SIGTERM'
|
|
1918
2278
|
}, 'nonzero runner result is structured without stderr echo');
|
|
2279
|
+
const errorsTable = await bridge.execute({
|
|
2280
|
+
action: 'errors'
|
|
2281
|
+
}, null);
|
|
2282
|
+
assert.ok(errorsTable.ok === true && errorsTable.errors, 'bridge failure contract is discoverable');
|
|
2283
|
+
for (const code of [
|
|
2284
|
+
'invalid_action',
|
|
2285
|
+
'disallowed_tool',
|
|
2286
|
+
'invalid_arguments',
|
|
2287
|
+
'timed_out',
|
|
2288
|
+
'output_truncated',
|
|
2289
|
+
'invalid_json',
|
|
2290
|
+
'invalid_output',
|
|
2291
|
+
'runner_failed',
|
|
2292
|
+
'spawn_failed',
|
|
2293
|
+
'forbidden_output'
|
|
2294
|
+
]){
|
|
2295
|
+
const row = errorsTable.errors?.[code];
|
|
2296
|
+
assert.ok(row && typeof row.recoverable === 'boolean' && row.remedy.length > 0, `failure contract contains ${code} with recovery guidance`);
|
|
2297
|
+
}
|
|
2298
|
+
assert.equal(errorsTable.errors?.invalid_arguments?.recoverable, true, 'invalid caller arguments are recoverable');
|
|
2299
|
+
assert.equal(errorsTable.errors?.forbidden_output?.recoverable, false, 'policy-boundary output is not recoverable');
|
|
1919
2300
|
for (const stdout of [
|
|
1920
2301
|
'not-json',
|
|
1921
2302
|
'{"result":1}{"result":2}'
|
|
@@ -1923,32 +2304,127 @@ try {
|
|
|
1923
2304
|
outcomes.push({
|
|
1924
2305
|
stdout
|
|
1925
2306
|
});
|
|
1926
|
-
const malformed = await bridge.execute({
|
|
1927
|
-
|
|
1928
|
-
|
|
1929
|
-
tool: 'lookup',
|
|
1930
|
-
arguments: {
|
|
1931
|
-
city: 'Dubai'
|
|
1932
|
-
}
|
|
1933
|
-
}
|
|
1934
|
-
}, null);
|
|
2307
|
+
const malformed = await bridge.execute(bridgeCall('lookup', {
|
|
2308
|
+
city: 'Dubai'
|
|
2309
|
+
}), null);
|
|
1935
2310
|
assert.deepEqual(malformed, {
|
|
1936
2311
|
ok: false,
|
|
1937
2312
|
error: 'invalid_json'
|
|
1938
2313
|
}, 'malformed or multi-value JSON is rejected');
|
|
1939
2314
|
}
|
|
1940
2315
|
outcomes.push({
|
|
1941
|
-
|
|
2316
|
+
stdout: JSON.stringify({
|
|
2317
|
+
schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION,
|
|
2318
|
+
status: 'miss',
|
|
2319
|
+
code: 'NOT_FOUND',
|
|
2320
|
+
recovery: 'revise_arguments'
|
|
2321
|
+
})
|
|
2322
|
+
});
|
|
2323
|
+
const miss = await bridge.execute(bridgeCall('lookup', {
|
|
2324
|
+
city: 'Dubai'
|
|
2325
|
+
}), null);
|
|
2326
|
+
assert.deepEqual(miss, {
|
|
2327
|
+
ok: true,
|
|
2328
|
+
outcome: {
|
|
2329
|
+
schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION,
|
|
2330
|
+
status: 'miss',
|
|
2331
|
+
code: 'NOT_FOUND',
|
|
2332
|
+
recovery: 'revise_arguments'
|
|
2333
|
+
}
|
|
2334
|
+
}, 'exit-zero domain miss is a successful typed outcome');
|
|
2335
|
+
outcomes.push({
|
|
2336
|
+
stdout: JSON.stringify({
|
|
2337
|
+
schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION,
|
|
2338
|
+
status: 'error',
|
|
2339
|
+
code: 'AMBIGUOUS',
|
|
2340
|
+
recovery: 'choose_alternative'
|
|
2341
|
+
})
|
|
1942
2342
|
});
|
|
1943
|
-
const
|
|
1944
|
-
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
2343
|
+
const domainError = await bridge.execute(bridgeCall('lookup', {
|
|
2344
|
+
city: 'Dubai'
|
|
2345
|
+
}), null);
|
|
2346
|
+
assert.deepEqual(domainError, {
|
|
2347
|
+
ok: true,
|
|
2348
|
+
outcome: {
|
|
2349
|
+
schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION,
|
|
2350
|
+
status: 'error',
|
|
2351
|
+
code: 'AMBIGUOUS',
|
|
2352
|
+
recovery: 'choose_alternative'
|
|
2353
|
+
}
|
|
2354
|
+
}, 'declared exit-zero domain error is transport success');
|
|
2355
|
+
for (const invalidEnvelope of [
|
|
2356
|
+
{
|
|
2357
|
+
schema_version: 'legacy',
|
|
2358
|
+
status: 'miss',
|
|
2359
|
+
code: 'NOT_FOUND',
|
|
2360
|
+
recovery: 'revise_arguments'
|
|
2361
|
+
},
|
|
2362
|
+
{
|
|
2363
|
+
schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION,
|
|
2364
|
+
status: 'miss',
|
|
2365
|
+
code: 'UNKNOWN',
|
|
2366
|
+
recovery: 'revise_arguments'
|
|
2367
|
+
},
|
|
2368
|
+
{
|
|
2369
|
+
schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION,
|
|
2370
|
+
status: 'miss',
|
|
2371
|
+
code: 'NOT_FOUND',
|
|
2372
|
+
recovery: 'retry_same'
|
|
2373
|
+
},
|
|
2374
|
+
{
|
|
2375
|
+
schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION,
|
|
2376
|
+
status: 'error',
|
|
2377
|
+
code: 'AMBIGUOUS',
|
|
2378
|
+
recovery: 'choose_alternative',
|
|
2379
|
+
message: 'private free text'
|
|
2380
|
+
}
|
|
2381
|
+
]){
|
|
2382
|
+
outcomes.push({
|
|
2383
|
+
stdout: JSON.stringify(invalidEnvelope)
|
|
2384
|
+
});
|
|
2385
|
+
assert.deepEqual(await bridge.execute(bridgeCall('lookup', {
|
|
2386
|
+
city: 'Dubai'
|
|
2387
|
+
}), null), {
|
|
2388
|
+
ok: false,
|
|
2389
|
+
error: 'invalid_output'
|
|
2390
|
+
}, 'undeclared or non-exact domain envelope fails closed');
|
|
2391
|
+
}
|
|
2392
|
+
outcomes.push({
|
|
2393
|
+
stdout: JSON.stringify({
|
|
2394
|
+
result: {
|
|
1948
2395
|
city: 'Dubai'
|
|
1949
2396
|
}
|
|
1950
|
-
}
|
|
1951
|
-
}
|
|
2397
|
+
})
|
|
2398
|
+
});
|
|
2399
|
+
assert.deepEqual(await bridge.execute(bridgeCall('lookup', {
|
|
2400
|
+
city: 'Dubai'
|
|
2401
|
+
}), null), {
|
|
2402
|
+
ok: false,
|
|
2403
|
+
error: 'invalid_output'
|
|
2404
|
+
}, 'legacy versionless result envelope is rejected');
|
|
2405
|
+
outcomes.push({
|
|
2406
|
+
stdout: JSON.stringify({
|
|
2407
|
+
schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION,
|
|
2408
|
+
status: 'miss',
|
|
2409
|
+
code: 'NOT_FOUND',
|
|
2410
|
+
recovery: 'revise_arguments'
|
|
2411
|
+
}),
|
|
2412
|
+
exitCode: 23
|
|
2413
|
+
});
|
|
2414
|
+
assert.deepEqual(await bridge.execute(bridgeCall('lookup', {
|
|
2415
|
+
city: 'Dubai'
|
|
2416
|
+
}), null), {
|
|
2417
|
+
ok: false,
|
|
2418
|
+
error: 'runner_failed',
|
|
2419
|
+
exit_code: 23,
|
|
2420
|
+
signal: null
|
|
2421
|
+
}, 'nonzero exit remains infrastructure failure even when stdout resembles a domain envelope');
|
|
2422
|
+
outcomes.push({
|
|
2423
|
+
spawnError: true
|
|
2424
|
+
});
|
|
2425
|
+
const spawnFailed = await bridge.execute(bridgeCall('lookup', {
|
|
2426
|
+
city: 'Dubai'
|
|
2427
|
+
}), null);
|
|
1952
2428
|
assert.deepEqual(spawnFailed, {
|
|
1953
2429
|
ok: false,
|
|
1954
2430
|
error: 'spawn_failed'
|
|
@@ -1956,15 +2432,9 @@ try {
|
|
|
1956
2432
|
outcomes.push({
|
|
1957
2433
|
spawnReject: true
|
|
1958
2434
|
});
|
|
1959
|
-
const asyncSpawnFailed = await bridge.execute({
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
tool: 'lookup',
|
|
1963
|
-
arguments: {
|
|
1964
|
-
city: 'Dubai'
|
|
1965
|
-
}
|
|
1966
|
-
}
|
|
1967
|
-
}, null);
|
|
2435
|
+
const asyncSpawnFailed = await bridge.execute(bridgeCall('lookup', {
|
|
2436
|
+
city: 'Dubai'
|
|
2437
|
+
}), null);
|
|
1968
2438
|
assert.deepEqual(asyncSpawnFailed, {
|
|
1969
2439
|
ok: false,
|
|
1970
2440
|
error: 'spawn_failed'
|
|
@@ -1972,55 +2442,47 @@ try {
|
|
|
1972
2442
|
outcomes.push({
|
|
1973
2443
|
waitForAbort: true
|
|
1974
2444
|
});
|
|
1975
|
-
const timedOut = await bridge.execute({
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
tool: 'lookup',
|
|
1979
|
-
arguments: {
|
|
1980
|
-
city: 'Dubai'
|
|
1981
|
-
}
|
|
1982
|
-
}
|
|
1983
|
-
}, null);
|
|
2445
|
+
const timedOut = await bridge.execute(bridgeCall('lookup', {
|
|
2446
|
+
city: 'Dubai'
|
|
2447
|
+
}), null);
|
|
1984
2448
|
assert.deepEqual(timedOut, {
|
|
1985
2449
|
ok: false,
|
|
1986
2450
|
error: 'timed_out'
|
|
1987
2451
|
}, 'deadline abort is surfaced as timed_out');
|
|
1988
2452
|
assert.equal(timeoutSignal?.aborted, true, 'timeout abort signal is fired');
|
|
1989
2453
|
const beforeInvalidArgs = spawnSpecs.length;
|
|
1990
|
-
assert.
|
|
1991
|
-
|
|
1992
|
-
|
|
1993
|
-
|
|
1994
|
-
arguments: [
|
|
1995
|
-
'not',
|
|
1996
|
-
'plain'
|
|
1997
|
-
]
|
|
1998
|
-
}
|
|
1999
|
-
}, null), {
|
|
2000
|
-
ok: false,
|
|
2001
|
-
error: 'invalid_arguments'
|
|
2002
|
-
});
|
|
2454
|
+
await assert.rejects(()=>bridge.execute(bridgeCall('lookup', [
|
|
2455
|
+
'not',
|
|
2456
|
+
'plain'
|
|
2457
|
+
]), null), /invalid arguments/);
|
|
2003
2458
|
assert.equal(spawnSpecs.length, beforeInvalidArgs, 'non-object arguments are rejected before spawn');
|
|
2004
|
-
|
|
2005
|
-
|
|
2459
|
+
for (const [label, argumentsValue] of [
|
|
2460
|
+
[
|
|
2461
|
+
'missing required city',
|
|
2462
|
+
{}
|
|
2463
|
+
],
|
|
2464
|
+
[
|
|
2465
|
+
'wrong city type',
|
|
2466
|
+
{
|
|
2467
|
+
city: 7
|
|
2468
|
+
}
|
|
2469
|
+
],
|
|
2470
|
+
[
|
|
2471
|
+
'undeclared country',
|
|
2472
|
+
{
|
|
2473
|
+
city: 'Dubai',
|
|
2474
|
+
country: 'AE'
|
|
2475
|
+
}
|
|
2476
|
+
]
|
|
2477
|
+
]){
|
|
2478
|
+
await assert.rejects(()=>bridge.execute(bridgeCall('lookup', argumentsValue), null), /invalid arguments/, `${label} is rejected by the descriptor-derived runtime validator`);
|
|
2479
|
+
}
|
|
2480
|
+
assert.equal(spawnSpecs.length, beforeInvalidArgs, 'required, type, and closed-object argument failures all happen before spawn');
|
|
2481
|
+
await assert.rejects(()=>bridge.execute({
|
|
2006
2482
|
action: 'inspect'
|
|
2007
|
-
}
|
|
2008
|
-
}, null), {
|
|
2009
|
-
ok: false,
|
|
2010
|
-
error: 'invalid_action'
|
|
2011
|
-
}, 'unknown action is rejected structurally');
|
|
2483
|
+
}, null), /invalid arguments/);
|
|
2012
2484
|
const beforeRejected = spawnSpecs.length;
|
|
2013
|
-
|
|
2014
|
-
query: {
|
|
2015
|
-
action: 'call',
|
|
2016
|
-
tool: 'delete_all',
|
|
2017
|
-
arguments: {}
|
|
2018
|
-
}
|
|
2019
|
-
}, null);
|
|
2020
|
-
assert.deepEqual(rejected, {
|
|
2021
|
-
ok: false,
|
|
2022
|
-
error: 'disallowed_tool'
|
|
2023
|
-
}, 'disallowed tool is rejected structurally');
|
|
2485
|
+
await assert.rejects(()=>bridge.execute(bridgeCall('delete_all', {}), null), /invalid arguments/);
|
|
2024
2486
|
assert.equal(spawnSpecs.length, beforeRejected, 'disallowed tool is rejected before spawn');
|
|
2025
2487
|
const spacedConfigPath = join(root, ' benchmark-environment-config.json ');
|
|
2026
2488
|
writeFileSync(spacedConfigPath, readFileSync(configPath));
|
|
@@ -2105,7 +2567,8 @@ try {
|
|
|
2105
2567
|
assert.deepEqual(disabledVariables, [
|
|
2106
2568
|
'current_date',
|
|
2107
2569
|
'time_anchor_card',
|
|
2108
|
-
'motivation_brief'
|
|
2570
|
+
'motivation_brief',
|
|
2571
|
+
'channel_routing_card'
|
|
2109
2572
|
], 'normal product mode keeps its prompt variables');
|
|
2110
2573
|
const whitespace = [];
|
|
2111
2574
|
const whitespaceCtx = {
|
|
@@ -2206,7 +2669,7 @@ try {
|
|
|
2206
2669
|
else process.env.GOTRY_LLM_MODEL = originalModelOverride;
|
|
2207
2670
|
}
|
|
2208
2671
|
const validConfig = {
|
|
2209
|
-
schema_version: '
|
|
2672
|
+
schema_version: 'gotry_benchmark_environment_bridge_v3',
|
|
2210
2673
|
enabled: true,
|
|
2211
2674
|
executable: process.execPath,
|
|
2212
2675
|
cwd: root,
|
|
@@ -2216,8 +2679,38 @@ try {
|
|
|
2216
2679
|
'--lang',
|
|
2217
2680
|
'en'
|
|
2218
2681
|
],
|
|
2219
|
-
|
|
2220
|
-
|
|
2682
|
+
tools: [
|
|
2683
|
+
{
|
|
2684
|
+
name: 'lookup',
|
|
2685
|
+
description: 'Lookup one declared city.',
|
|
2686
|
+
input_schema: {
|
|
2687
|
+
type: 'object',
|
|
2688
|
+
properties: {
|
|
2689
|
+
city: {
|
|
2690
|
+
type: 'string',
|
|
2691
|
+
enum: [
|
|
2692
|
+
'Dubai',
|
|
2693
|
+
'Abu Dhabi'
|
|
2694
|
+
],
|
|
2695
|
+
description: 'Declared city name.'
|
|
2696
|
+
}
|
|
2697
|
+
},
|
|
2698
|
+
required: [
|
|
2699
|
+
'city'
|
|
2700
|
+
],
|
|
2701
|
+
additionalProperties: false
|
|
2702
|
+
},
|
|
2703
|
+
output_keys: [
|
|
2704
|
+
'city'
|
|
2705
|
+
],
|
|
2706
|
+
domain_outcomes: [
|
|
2707
|
+
{
|
|
2708
|
+
status: 'miss',
|
|
2709
|
+
code: 'NOT_FOUND',
|
|
2710
|
+
recovery: 'revise_arguments'
|
|
2711
|
+
}
|
|
2712
|
+
]
|
|
2713
|
+
}
|
|
2221
2714
|
],
|
|
2222
2715
|
timeout_ms: 20,
|
|
2223
2716
|
max_output_bytes: 4_096,
|
|
@@ -2231,6 +2724,86 @@ try {
|
|
|
2231
2724
|
network: 'denied'
|
|
2232
2725
|
}
|
|
2233
2726
|
};
|
|
2727
|
+
const clonedConfig = ()=>JSON.parse(JSON.stringify(validConfig));
|
|
2728
|
+
const configWithTool = (mutate)=>{
|
|
2729
|
+
const config = clonedConfig();
|
|
2730
|
+
mutate(config.tools[0]);
|
|
2731
|
+
return config;
|
|
2732
|
+
};
|
|
2733
|
+
const nestedInputSchema = (objectLevels)=>{
|
|
2734
|
+
let child = {
|
|
2735
|
+
type: 'string'
|
|
2736
|
+
};
|
|
2737
|
+
for(let index = 0; index < objectLevels; index += 1){
|
|
2738
|
+
child = {
|
|
2739
|
+
type: 'object',
|
|
2740
|
+
properties: {
|
|
2741
|
+
next: child
|
|
2742
|
+
},
|
|
2743
|
+
required: [
|
|
2744
|
+
'next'
|
|
2745
|
+
],
|
|
2746
|
+
additionalProperties: false
|
|
2747
|
+
};
|
|
2748
|
+
}
|
|
2749
|
+
return child;
|
|
2750
|
+
};
|
|
2751
|
+
const flatInputSchema = (count)=>({
|
|
2752
|
+
type: 'object',
|
|
2753
|
+
properties: Object.fromEntries(Array.from({
|
|
2754
|
+
length: count
|
|
2755
|
+
}, (_, index)=>[
|
|
2756
|
+
`p${index}`,
|
|
2757
|
+
{
|
|
2758
|
+
type: 'string'
|
|
2759
|
+
}
|
|
2760
|
+
])),
|
|
2761
|
+
required: [],
|
|
2762
|
+
additionalProperties: false
|
|
2763
|
+
});
|
|
2764
|
+
const totalEnumInputSchema = (groups)=>({
|
|
2765
|
+
type: 'object',
|
|
2766
|
+
properties: Object.fromEntries(Array.from({
|
|
2767
|
+
length: groups
|
|
2768
|
+
}, (_, group)=>[
|
|
2769
|
+
`e${group}`,
|
|
2770
|
+
{
|
|
2771
|
+
type: 'string',
|
|
2772
|
+
enum: Array.from({
|
|
2773
|
+
length: 64
|
|
2774
|
+
}, (_, index)=>`v${group}_${index}`)
|
|
2775
|
+
}
|
|
2776
|
+
])),
|
|
2777
|
+
required: [],
|
|
2778
|
+
additionalProperties: false
|
|
2779
|
+
});
|
|
2780
|
+
const byteSizedInputSchema = (targetBytes)=>{
|
|
2781
|
+
const properties = Object.fromEntries(Array.from({
|
|
2782
|
+
length: 64
|
|
2783
|
+
}, (_, index)=>[
|
|
2784
|
+
`b${index}`,
|
|
2785
|
+
{
|
|
2786
|
+
type: 'string',
|
|
2787
|
+
description: ''
|
|
2788
|
+
}
|
|
2789
|
+
]));
|
|
2790
|
+
const schema = {
|
|
2791
|
+
type: 'object',
|
|
2792
|
+
properties,
|
|
2793
|
+
required: [],
|
|
2794
|
+
additionalProperties: false
|
|
2795
|
+
};
|
|
2796
|
+
let remaining = targetBytes - Buffer.byteLength(JSON.stringify(schema), 'utf8');
|
|
2797
|
+
assert.ok(remaining >= 0 && remaining <= 64 * 512, 'byte-boundary fixture has enough bounded description capacity');
|
|
2798
|
+
for (const property of Object.values(properties)){
|
|
2799
|
+
const added = Math.min(512, remaining);
|
|
2800
|
+
property.description = 'x'.repeat(added);
|
|
2801
|
+
remaining -= added;
|
|
2802
|
+
}
|
|
2803
|
+
assert.equal(remaining, 0, 'byte-boundary fixture reaches the requested serialized size');
|
|
2804
|
+
assert.equal(Buffer.byteLength(JSON.stringify(schema), 'utf8'), targetBytes);
|
|
2805
|
+
return schema;
|
|
2806
|
+
};
|
|
2234
2807
|
const frozenProjection = registerBenchmarkEnvironmentBridge(configPath, ()=>{}, {
|
|
2235
2808
|
spawn: (_spec)=>fakeHandle({
|
|
2236
2809
|
stdout: '{"result":{}}'
|
|
@@ -2271,7 +2844,7 @@ try {
|
|
|
2271
2844
|
return ()=>{};
|
|
2272
2845
|
},
|
|
2273
2846
|
effect (action) {
|
|
2274
|
-
return action
|
|
2847
|
+
return runEffect(action);
|
|
2275
2848
|
},
|
|
2276
2849
|
agents: {
|
|
2277
2850
|
list () {
|
|
@@ -2305,62 +2878,286 @@ try {
|
|
|
2305
2878
|
}), /benchmark environment bridge configuration unavailable/, 'unknown top-level config key fails hard');
|
|
2306
2879
|
assert.throws(()=>bridgeRegistrationFor({
|
|
2307
2880
|
...validConfig,
|
|
2308
|
-
|
|
2309
|
-
|
|
2310
|
-
|
|
2311
|
-
|
|
2312
|
-
|
|
2313
|
-
|
|
2314
|
-
|
|
2881
|
+
tools: [
|
|
2882
|
+
{
|
|
2883
|
+
...validConfig.tools[0],
|
|
2884
|
+
input_schema: {
|
|
2885
|
+
type: 'object'
|
|
2886
|
+
}
|
|
2887
|
+
}
|
|
2315
2888
|
]
|
|
2316
|
-
}), /benchmark environment bridge configuration unavailable/, '
|
|
2889
|
+
}), /benchmark environment bridge configuration unavailable/, 'v3 rejects an open annotation-only input schema');
|
|
2317
2890
|
assert.throws(()=>bridgeRegistrationFor({
|
|
2318
2891
|
...validConfig,
|
|
2319
|
-
|
|
2320
|
-
}), /benchmark environment bridge configuration unavailable/, '
|
|
2892
|
+
schema_version: 'gotry_benchmark_environment_bridge_v1'
|
|
2893
|
+
}), /benchmark environment bridge configuration unavailable/, 'v1 config cannot silently omit the Round 3 terminal semantics');
|
|
2321
2894
|
assert.throws(()=>bridgeRegistrationFor({
|
|
2322
2895
|
...validConfig,
|
|
2323
|
-
|
|
2324
|
-
|
|
2896
|
+
tools: [
|
|
2897
|
+
validConfig.tools[0],
|
|
2898
|
+
validConfig.tools[0]
|
|
2899
|
+
]
|
|
2900
|
+
}), /benchmark environment bridge configuration unavailable/, 'duplicate tool descriptor fails hard');
|
|
2901
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
2902
|
+
delete tool.output_keys;
|
|
2903
|
+
})), /benchmark environment bridge configuration unavailable/, 'missing output_keys fails hard');
|
|
2904
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
2905
|
+
tool.output_keys = [];
|
|
2906
|
+
})), /benchmark environment bridge configuration unavailable/, 'empty output allowlist fails closed at registration');
|
|
2907
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
2908
|
+
tool.output_keys = [
|
|
2909
|
+
'city',
|
|
2910
|
+
'city'
|
|
2911
|
+
];
|
|
2912
|
+
})), /benchmark environment bridge configuration unavailable/, 'duplicate output key fails hard');
|
|
2913
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
2914
|
+
tool.output_keys = [
|
|
2915
|
+
'not a key'
|
|
2916
|
+
];
|
|
2917
|
+
})), /benchmark environment bridge configuration unavailable/, 'non-identifier output key fails hard');
|
|
2918
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
2919
|
+
delete tool.domain_outcomes;
|
|
2920
|
+
})), /benchmark environment bridge configuration unavailable/, 'missing domain_outcomes fails hard');
|
|
2921
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
2922
|
+
tool.domain_outcomes = [
|
|
2923
|
+
{
|
|
2924
|
+
status: 'unknown',
|
|
2925
|
+
code: 'NOT_FOUND',
|
|
2926
|
+
recovery: 'none'
|
|
2927
|
+
}
|
|
2928
|
+
];
|
|
2929
|
+
})), /benchmark environment bridge configuration unavailable/, 'unknown domain status fails hard');
|
|
2930
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
2931
|
+
tool.domain_outcomes = [
|
|
2932
|
+
{
|
|
2933
|
+
status: 'miss',
|
|
2934
|
+
code: 'NOT_FOUND',
|
|
2935
|
+
recovery: 'free_text'
|
|
2936
|
+
}
|
|
2937
|
+
];
|
|
2938
|
+
})), /benchmark environment bridge configuration unavailable/, 'free-text domain recovery fails hard');
|
|
2939
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
2940
|
+
tool.domain_outcomes = [
|
|
2941
|
+
{
|
|
2942
|
+
status: 'miss',
|
|
2943
|
+
code: 'NOT_FOUND',
|
|
2944
|
+
recovery: 'none'
|
|
2945
|
+
},
|
|
2946
|
+
{
|
|
2947
|
+
status: 'error',
|
|
2948
|
+
code: 'NOT_FOUND',
|
|
2949
|
+
recovery: 'none'
|
|
2950
|
+
}
|
|
2951
|
+
];
|
|
2952
|
+
})), /benchmark environment bridge configuration unavailable/, 'duplicate domain code fails hard');
|
|
2953
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
2954
|
+
tool.name = 'lookup;rm';
|
|
2955
|
+
})), /benchmark environment bridge configuration unavailable/, 'shell metacharacter tool identifier fails hard');
|
|
2956
|
+
for (const [label, schema] of [
|
|
2957
|
+
[
|
|
2958
|
+
'reference keyword',
|
|
2959
|
+
{
|
|
2960
|
+
type: 'object',
|
|
2961
|
+
properties: {
|
|
2962
|
+
city: {
|
|
2963
|
+
$ref: '#/city'
|
|
2964
|
+
}
|
|
2965
|
+
},
|
|
2966
|
+
required: [
|
|
2967
|
+
'city'
|
|
2968
|
+
],
|
|
2969
|
+
additionalProperties: false
|
|
2325
2970
|
}
|
|
2326
|
-
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2971
|
+
],
|
|
2972
|
+
[
|
|
2973
|
+
'default keyword',
|
|
2974
|
+
{
|
|
2975
|
+
type: 'object',
|
|
2976
|
+
properties: {
|
|
2977
|
+
city: {
|
|
2978
|
+
type: 'string',
|
|
2979
|
+
default: 'Dubai'
|
|
2980
|
+
}
|
|
2981
|
+
},
|
|
2982
|
+
required: [
|
|
2331
2983
|
'city'
|
|
2332
|
-
]
|
|
2984
|
+
],
|
|
2985
|
+
additionalProperties: false
|
|
2333
2986
|
}
|
|
2334
|
-
|
|
2335
|
-
|
|
2336
|
-
|
|
2337
|
-
|
|
2338
|
-
|
|
2987
|
+
],
|
|
2988
|
+
[
|
|
2989
|
+
'examples keyword',
|
|
2990
|
+
{
|
|
2991
|
+
type: 'object',
|
|
2992
|
+
properties: {
|
|
2993
|
+
city: {
|
|
2994
|
+
type: 'string',
|
|
2995
|
+
examples: [
|
|
2996
|
+
'Dubai'
|
|
2997
|
+
]
|
|
2998
|
+
}
|
|
2999
|
+
},
|
|
3000
|
+
required: [
|
|
3001
|
+
'city'
|
|
3002
|
+
],
|
|
3003
|
+
additionalProperties: false
|
|
3004
|
+
}
|
|
3005
|
+
],
|
|
3006
|
+
[
|
|
3007
|
+
'unknown keyword',
|
|
3008
|
+
{
|
|
3009
|
+
type: 'object',
|
|
3010
|
+
properties: {
|
|
3011
|
+
city: {
|
|
3012
|
+
type: 'string',
|
|
3013
|
+
title: 'City'
|
|
3014
|
+
}
|
|
3015
|
+
},
|
|
3016
|
+
required: [
|
|
3017
|
+
'city'
|
|
3018
|
+
],
|
|
3019
|
+
additionalProperties: false
|
|
3020
|
+
}
|
|
3021
|
+
],
|
|
3022
|
+
[
|
|
3023
|
+
'open nested object',
|
|
3024
|
+
{
|
|
3025
|
+
type: 'object',
|
|
3026
|
+
properties: {
|
|
3027
|
+
nested: {
|
|
3028
|
+
type: 'object',
|
|
3029
|
+
properties: {},
|
|
3030
|
+
required: []
|
|
3031
|
+
}
|
|
3032
|
+
},
|
|
3033
|
+
required: [
|
|
3034
|
+
'nested'
|
|
3035
|
+
],
|
|
3036
|
+
additionalProperties: false
|
|
3037
|
+
}
|
|
3038
|
+
],
|
|
3039
|
+
[
|
|
3040
|
+
'array without items',
|
|
3041
|
+
{
|
|
3042
|
+
type: 'object',
|
|
3043
|
+
properties: {
|
|
3044
|
+
values: {
|
|
3045
|
+
type: 'array'
|
|
3046
|
+
}
|
|
3047
|
+
},
|
|
3048
|
+
required: [
|
|
3049
|
+
'values'
|
|
3050
|
+
],
|
|
3051
|
+
additionalProperties: false
|
|
3052
|
+
}
|
|
3053
|
+
],
|
|
3054
|
+
[
|
|
3055
|
+
'duplicate required',
|
|
3056
|
+
{
|
|
3057
|
+
type: 'object',
|
|
3058
|
+
properties: {
|
|
3059
|
+
city: {
|
|
3060
|
+
type: 'string'
|
|
3061
|
+
}
|
|
3062
|
+
},
|
|
3063
|
+
required: [
|
|
2339
3064
|
'city',
|
|
2340
3065
|
'city'
|
|
2341
|
-
]
|
|
3066
|
+
],
|
|
3067
|
+
additionalProperties: false
|
|
2342
3068
|
}
|
|
2343
|
-
|
|
2344
|
-
|
|
2345
|
-
|
|
2346
|
-
|
|
2347
|
-
|
|
2348
|
-
|
|
2349
|
-
|
|
3069
|
+
],
|
|
3070
|
+
[
|
|
3071
|
+
'unknown required',
|
|
3072
|
+
{
|
|
3073
|
+
type: 'object',
|
|
3074
|
+
properties: {
|
|
3075
|
+
city: {
|
|
3076
|
+
type: 'string'
|
|
3077
|
+
}
|
|
3078
|
+
},
|
|
3079
|
+
required: [
|
|
3080
|
+
'country'
|
|
3081
|
+
],
|
|
3082
|
+
additionalProperties: false
|
|
3083
|
+
}
|
|
3084
|
+
],
|
|
3085
|
+
[
|
|
3086
|
+
'type-mismatched enum',
|
|
3087
|
+
{
|
|
3088
|
+
type: 'object',
|
|
3089
|
+
properties: {
|
|
3090
|
+
city: {
|
|
3091
|
+
type: 'string',
|
|
3092
|
+
enum: [
|
|
3093
|
+
'Dubai',
|
|
3094
|
+
7
|
|
3095
|
+
]
|
|
3096
|
+
}
|
|
3097
|
+
},
|
|
3098
|
+
required: [
|
|
3099
|
+
'city'
|
|
3100
|
+
],
|
|
3101
|
+
additionalProperties: false
|
|
2350
3102
|
}
|
|
2351
|
-
|
|
3103
|
+
],
|
|
3104
|
+
[
|
|
3105
|
+
'duplicate enum',
|
|
3106
|
+
{
|
|
3107
|
+
type: 'object',
|
|
3108
|
+
properties: {
|
|
3109
|
+
city: {
|
|
3110
|
+
type: 'string',
|
|
3111
|
+
enum: [
|
|
3112
|
+
'Dubai',
|
|
3113
|
+
'Dubai'
|
|
3114
|
+
]
|
|
3115
|
+
}
|
|
3116
|
+
},
|
|
3117
|
+
required: [
|
|
3118
|
+
'city'
|
|
3119
|
+
],
|
|
3120
|
+
additionalProperties: false
|
|
3121
|
+
}
|
|
3122
|
+
]
|
|
3123
|
+
]){
|
|
3124
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3125
|
+
tool.input_schema = schema;
|
|
3126
|
+
})), /benchmark environment bridge configuration unavailable/, `${label} fails closed`);
|
|
3127
|
+
}
|
|
3128
|
+
assert.equal(bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3129
|
+
tool.input_schema = nestedInputSchema(8);
|
|
3130
|
+
})), true, 'schema depth eight is accepted');
|
|
3131
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3132
|
+
tool.input_schema = nestedInputSchema(9);
|
|
3133
|
+
})), /benchmark environment bridge configuration unavailable/, 'schema depth nine is rejected');
|
|
3134
|
+
assert.equal(bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3135
|
+
tool.input_schema = flatInputSchema(255);
|
|
3136
|
+
})), true, 'the 256-node total boundary is accepted');
|
|
3137
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3138
|
+
tool.input_schema = flatInputSchema(256);
|
|
3139
|
+
})), /benchmark environment bridge configuration unavailable/, 'the 256-node total boundary plus one is rejected');
|
|
3140
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3141
|
+
tool.input_schema = flatInputSchema(257);
|
|
3142
|
+
})), /benchmark environment bridge configuration unavailable/, 'the property total bound rejects 257 properties');
|
|
3143
|
+
assert.equal(bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3144
|
+
tool.input_schema = totalEnumInputSchema(4);
|
|
3145
|
+
})), true, 'the 256-enum-value total boundary is accepted');
|
|
3146
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3147
|
+
tool.input_schema = totalEnumInputSchema(5);
|
|
3148
|
+
})), /benchmark environment bridge configuration unavailable/, 'the enum-value total bound rejects values above 256');
|
|
3149
|
+
assert.equal(bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3150
|
+
tool.input_schema = byteSizedInputSchema(16 * 1024);
|
|
3151
|
+
})), true, 'the 16 KiB schema boundary is accepted');
|
|
3152
|
+
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3153
|
+
tool.input_schema = byteSizedInputSchema(16 * 1024 + 1);
|
|
3154
|
+
})), /benchmark environment bridge configuration unavailable/, 'the schema byte boundary plus one is rejected');
|
|
2352
3155
|
assert.throws(()=>bridgeRegistrationFor({
|
|
2353
3156
|
...validConfig,
|
|
2354
3157
|
argv_prefix: [
|
|
2355
3158
|
'agent\n--unsafe'
|
|
2356
3159
|
]
|
|
2357
3160
|
}), /benchmark environment bridge configuration unavailable/, 'argv control separator fails hard');
|
|
2358
|
-
assert.throws(()=>bridgeRegistrationFor({
|
|
2359
|
-
...validConfig,
|
|
2360
|
-
allowed_tools: [
|
|
2361
|
-
'lookup;rm'
|
|
2362
|
-
]
|
|
2363
|
-
}), /benchmark environment bridge configuration unavailable/, 'shell metacharacter tool identifier fails hard');
|
|
2364
3161
|
assert.throws(()=>bridgeRegistrationFor({
|
|
2365
3162
|
...validConfig,
|
|
2366
3163
|
isolation: {
|
|
@@ -2410,20 +3207,20 @@ try {
|
|
|
2410
3207
|
writeFileSync(widePath, JSON.stringify(validConfig));
|
|
2411
3208
|
chmodSync(widePath, 0o666);
|
|
2412
3209
|
assert.throws(()=>loadConfigRegistration(widePath, root), /benchmark environment bridge configuration unavailable/, 'group/world writable config fails hard');
|
|
2413
|
-
const
|
|
2414
|
-
writeFileSync(
|
|
2415
|
-
const
|
|
2416
|
-
const
|
|
3210
|
+
const strictConfigPath = join(root, 'strict-bridge.json');
|
|
3211
|
+
writeFileSync(strictConfigPath, JSON.stringify(validConfig));
|
|
3212
|
+
const strictTools = [];
|
|
3213
|
+
const strictCtx = {
|
|
2417
3214
|
tools: {
|
|
2418
3215
|
register (tool) {
|
|
2419
|
-
|
|
3216
|
+
strictTools.push(tool);
|
|
2420
3217
|
return ()=>{};
|
|
2421
3218
|
},
|
|
2422
3219
|
get (name) {
|
|
2423
|
-
return
|
|
3220
|
+
return strictTools.find((tool)=>tool.name === name);
|
|
2424
3221
|
},
|
|
2425
3222
|
schemas () {
|
|
2426
|
-
return
|
|
3223
|
+
return strictTools.map((tool)=>({
|
|
2427
3224
|
name: tool.name
|
|
2428
3225
|
}));
|
|
2429
3226
|
}
|
|
@@ -2448,33 +3245,29 @@ try {
|
|
|
2448
3245
|
},
|
|
2449
3246
|
subprocess: {
|
|
2450
3247
|
spawn: (_spec)=>fakeHandle({
|
|
2451
|
-
stdout:
|
|
3248
|
+
stdout: okEnvelope({
|
|
3249
|
+
city: 'Dubai'
|
|
3250
|
+
})
|
|
2452
3251
|
})
|
|
2453
3252
|
}
|
|
2454
3253
|
};
|
|
2455
|
-
apply(
|
|
3254
|
+
apply(strictCtx, {
|
|
2456
3255
|
stateRoot: root,
|
|
2457
3256
|
timeoutMs: 20,
|
|
2458
3257
|
hbcliBin: '',
|
|
2459
3258
|
sessionAccess: 'off',
|
|
2460
|
-
benchmarkEnvironmentConfigPath:
|
|
3259
|
+
benchmarkEnvironmentConfigPath: strictConfigPath
|
|
2461
3260
|
});
|
|
2462
|
-
const
|
|
2463
|
-
assert.ok(
|
|
2464
|
-
assert.deepEqual(await
|
|
2465
|
-
|
|
2466
|
-
|
|
2467
|
-
tool: 'lookup',
|
|
2468
|
-
arguments: {
|
|
2469
|
-
city: 'Dubai'
|
|
2470
|
-
}
|
|
2471
|
-
}
|
|
2472
|
-
}, null), {
|
|
3261
|
+
const strictBridge = strictTools.find((tool)=>tool.name === 'gotry_benchmark_environment');
|
|
3262
|
+
assert.ok(strictBridge?.execute, 'strict v3 config registers the bridge');
|
|
3263
|
+
assert.deepEqual(await strictBridge.execute(bridgeCall('lookup', {
|
|
3264
|
+
city: 'Dubai'
|
|
3265
|
+
}), null), {
|
|
2473
3266
|
ok: true,
|
|
2474
3267
|
result: {
|
|
2475
3268
|
city: 'Dubai'
|
|
2476
3269
|
}
|
|
2477
|
-
}, '
|
|
3270
|
+
}, 'strict v3 result envelope executes safe structured output');
|
|
2478
3271
|
console.log('BENCHMARK ENVIRONMENT BRIDGE TESTS: registration + TDD bridge contract assertions');
|
|
2479
3272
|
} catch (error) {
|
|
2480
3273
|
console.error(error);
|