@danceiny/gotry 0.0.1-rc.21 → 0.0.1-rc.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +102 -194
- package/README.zh-CN.md +115 -210
- package/bin/gotry-backend.js +53 -0
- package/bin/gotry-bootstrap.js +466 -35
- package/bin/gotry-inner.js +255 -36
- package/bin/gotry-process-liveness.d.ts +27 -0
- package/bin/gotry-process-liveness.js +97 -0
- package/bin/gotry-runtime-resolution.d.ts +1 -1
- package/bin/gotry-runtime-resolution.js +2 -2
- package/client/client.js +174 -0
- package/cordis.gotry-patch.yml +70 -93
- package/dist/capabilities/agent-reach-deep.js +45 -122
- package/dist/capabilities/agent-reach.js +63 -148
- package/dist/capabilities/anything.js +89 -75
- package/dist/capabilities/artifacts.js +123 -104
- package/dist/capabilities/channel-health.js +65 -45
- package/dist/capabilities/channel-registry.js +108 -87
- package/dist/capabilities/doctor.js +357 -235
- package/dist/capabilities/effect.js +223 -309
- package/dist/capabilities/fact-log.js +28 -14
- package/dist/capabilities/flyai.js +245 -156
- package/dist/capabilities/ground-transfer.js +509 -0
- package/dist/capabilities/hbcli.js +144 -148
- package/dist/capabilities/hotelbyte-transaction.js +194 -0
- package/dist/capabilities/incident-log.js +131 -61
- package/dist/capabilities/model-override.js +30 -10
- package/dist/capabilities/opensky.js +54 -43
- package/dist/capabilities/resilience.js +50 -32
- package/dist/capabilities/session/action-cache.js +50 -44
- package/dist/capabilities/session/adapters/ctrip-flight.js +119 -62
- package/dist/capabilities/session/adapters/ctrip-hotel.js +125 -110
- package/dist/capabilities/session/adapters/dida-portal.js +206 -0
- package/dist/capabilities/session/adapters/meituan-local.js +35 -54
- package/dist/capabilities/session/adapters/rail-12306.js +169 -112
- package/dist/capabilities/session/benchmark.js +85 -149
- package/dist/capabilities/session/extension-bridge.js +320 -259
- package/dist/capabilities/session/extension-channel.js +57 -76
- package/dist/capabilities/session/extension-distribution.js +115 -119
- package/dist/capabilities/session/extract.js +31 -21
- package/dist/capabilities/session/golden-score.js +55 -33
- package/dist/capabilities/session/health-watch.js +146 -154
- package/dist/capabilities/session/read-guard.js +55 -47
- package/dist/capabilities/session/static-flight-golden.js +49 -36
- package/dist/capabilities/session/transport.js +54 -61
- package/dist/capabilities/session/wizard.js +63 -93
- package/dist/capabilities/session-consent.js +143 -47
- package/dist/capabilities/session-login.js +95 -126
- package/dist/capabilities/session-search.js +391 -253
- package/dist/capabilities/sponsor-plugin.js +341 -0
- package/dist/capabilities/visa-policy.js +87 -73
- package/dist/capabilities/weather.js +163 -235
- package/dist/scripts/action-cache-tests.js +24 -59
- package/dist/scripts/agent-planning-turn-deadline-e2e.js +91 -142
- package/dist/scripts/agent-planning-turn-deadline-tests.js +80 -176
- package/dist/scripts/agent-reach-deep-tests.js +30 -40
- package/dist/scripts/agent-reach-tests.js +26 -18
- package/dist/scripts/agent-reach-wrapper-tests.js +46 -50
- package/dist/scripts/anything-tests.js +58 -62
- package/dist/scripts/artifact-client-contract-tests.js +69 -0
- package/dist/scripts/artifacts-capability-tests.js +235 -0
- package/dist/scripts/async-collect.js +26 -15
- package/dist/scripts/benchmark-environment-bridge-e2e.js +287 -684
- package/dist/scripts/benchmark-environment-bridge-tests.js +818 -3123
- package/dist/scripts/booking-copilot-availability-ledger-binding-tests.js +66 -279
- package/dist/scripts/booking-copilot-availability-policy-tests.js +153 -1257
- package/dist/scripts/booking-copilot-bin-proof-tests.js +22 -45
- package/dist/scripts/booking-copilot-crossrepo-fixture-server.js +64 -76
- package/dist/scripts/booking-copilot-dsh-core-proof-tests.js +69 -199
- package/dist/scripts/booking-copilot-dsh-planner-proof-tests.js +393 -759
- package/dist/scripts/booking-copilot-dsh-plugin-proof-tests.js +33 -75
- package/dist/scripts/booking-copilot-event-sequence-concurrency-proof-tests.js +60 -110
- package/dist/scripts/booking-copilot-gap-code-contract-proof-tests.js +28 -66
- package/dist/scripts/booking-copilot-operation-ledger-concurrency-proof-tests.js +72 -271
- package/dist/scripts/booking-copilot-receipt-ledger-concurrency-proof-tests.js +73 -275
- package/dist/scripts/booking-copilot-runtime-proof-tests.js +784 -3398
- package/dist/scripts/booking-copilot-server-proof-tests.js +63 -168
- package/dist/scripts/booking-copilot-startup-proof-tests.js +38 -124
- package/dist/scripts/booking-executor-tests.js +80 -0
- package/dist/scripts/booking-recovery-chain-tests.js +182 -0
- package/dist/scripts/booking-saga-tests.js +61 -123
- package/dist/scripts/booking-surface-contract-proof-tests.js +152 -1167
- package/dist/scripts/bootstrap-tests.js +1383 -128
- package/dist/scripts/build-changelog.js +89 -130
- package/dist/scripts/build-metrics-report.js +135 -143
- package/dist/scripts/changelog-tests.js +35 -82
- package/dist/scripts/channel-probe-tests.js +44 -116
- package/dist/scripts/channel-probe.js +84 -167
- package/dist/scripts/channel-registry-tests.js +97 -185
- package/dist/scripts/companion-tests.js +25 -70
- package/dist/scripts/dida-cookie-probe.js +87 -0
- package/dist/scripts/dida-sw-probe.js +114 -0
- package/dist/scripts/diff-test.js +17 -12
- package/dist/scripts/doctor-tests.js +428 -120
- package/dist/scripts/dsh-artifact-e2e.js +150 -0
- package/dist/scripts/dsh-artifact-web-e2e.js +443 -0
- package/dist/scripts/dsh-http-proxy-sse-proof.js +90 -0
- package/dist/scripts/dsh-runtime-closure-tests.js +110 -153
- package/dist/scripts/dsh-runtime-closure.js +75 -63
- package/dist/scripts/dsh-session-v3-migration-proof.js +106 -0
- package/dist/scripts/dsh-subprocess-local-proof.js +149 -0
- package/dist/scripts/dsh-target-closure-proof.js +178 -0
- package/dist/scripts/effect-tests.js +165 -552
- package/dist/scripts/engine-run.js +1 -3
- package/dist/scripts/engine-tests.js +16 -10
- package/dist/scripts/evaluation-cadence-tests.js +69 -242
- package/dist/scripts/evaluation-contract-tests.js +193 -499
- package/dist/scripts/extension-distribution-cli.js +19 -15
- package/dist/scripts/extension-distribution-tests.js +98 -211
- package/dist/scripts/extension-tests.js +366 -389
- package/dist/scripts/fact-gate-tests.js +725 -263
- package/dist/scripts/fixtures/extension-bridge-unref-child.mjs +21 -0
- package/dist/scripts/flight-hotel-anchor-363-registered-e2e.js +169 -0
- package/dist/scripts/flight-malformed-tests.js +159 -0
- package/dist/scripts/flyai-tests.js +325 -68
- package/dist/scripts/g5-guard-tests.js +131 -0
- package/dist/scripts/g5-guard.js +189 -0
- package/dist/scripts/gen-lunar-anchors.js +106 -0
- package/dist/scripts/gotry-backend-kernel-tests.js +79 -0
- package/dist/scripts/ground-transfer-tests.js +956 -0
- package/dist/scripts/hbcli-e2e-tests.js +71 -90
- package/dist/scripts/hbcli-live-optin-tests.js +96 -0
- package/dist/scripts/hbcli-release-contract-tests.js +122 -0
- package/dist/scripts/hbcli-tests.js +41 -115
- package/dist/scripts/health-watch-cli.js +42 -28
- package/dist/scripts/hotel-date-gate-tests.js +496 -0
- package/dist/scripts/hotelbyte-reconcile-tests.js +141 -0
- package/dist/scripts/hotelbyte-spawn-e2e-tests.js +373 -0
- package/dist/scripts/i18n-tests.js +16 -9
- package/dist/scripts/incident-tests.js +281 -87
- package/dist/scripts/issue-194-job-id-guard-tests.js +106 -0
- package/dist/scripts/issue-233-cancel-refund-commission-tests.js +392 -0
- package/dist/scripts/issue-271-liveness-tests.js +547 -0
- package/dist/scripts/issue-289-model-retry-real-tests.js +296 -0
- package/dist/scripts/issue-289-model-retry-tests.js +441 -0
- package/dist/scripts/issue-289-web-retry-e2e.js +630 -0
- package/dist/scripts/issue-338-home-city-e2e.js +577 -0
- package/dist/scripts/issue343-real-entry-e2e.js +160 -0
- package/dist/scripts/journey-tests.js +22 -27
- package/dist/scripts/kernel-manifest-evidence.js +109 -0
- package/dist/scripts/kernel-manifest-gate.js +126 -0
- package/dist/scripts/kernel-manifest-tests.js +212 -0
- package/dist/scripts/kernel-manifest-trace-entry.js +7 -0
- package/dist/scripts/kernel-manifest-trace-hook.mjs +28 -0
- package/dist/scripts/kernel-manifest.js +369 -0
- package/dist/scripts/ledger-repair-apply-tests.js +316 -0
- package/dist/scripts/ledger-repair-plan-tests.js +375 -0
- package/dist/scripts/ledger-tests.js +325 -340
- package/dist/scripts/ledger-workflow-crash.js +10 -20
- package/dist/scripts/map-tools-vendor-package-proof.js +206 -134
- package/dist/scripts/memory-capture-tests.js +242 -61
- package/dist/scripts/memory-decay-tests.js +18 -52
- package/dist/scripts/memory-lifecycle-tests.js +657 -0
- package/dist/scripts/memory-lifecycle.js +196 -0
- package/dist/scripts/memory-metrics.js +24 -12
- package/dist/scripts/memory-value-report.js +474 -222
- package/dist/scripts/memory-value-tests.js +328 -0
- package/dist/scripts/metrics-report-tests.js +62 -199
- package/dist/scripts/model-override-e2e.js +66 -105
- package/dist/scripts/nightly-evidence-tests.js +64 -89
- package/dist/scripts/nightly-evidence.js +108 -111
- package/dist/scripts/nudge-digest.js +39 -28
- package/dist/scripts/onboarding-tests.js +63 -88
- package/dist/scripts/opensky-check.js +18 -13
- package/dist/scripts/opensky-tests.js +28 -28
- package/dist/scripts/persona-surface-guard-tests.js +37 -12
- package/dist/scripts/pnpm-dsh-closure-proof.js +2 -5
- package/dist/scripts/policy-anchor-359-registered-e2e.js +214 -0
- package/dist/scripts/price-drift-tests.js +126 -227
- package/dist/scripts/price-drift-watch.js +209 -253
- package/dist/scripts/probe-poi-tests.js +52 -45
- package/dist/scripts/product-metrics.js +187 -273
- package/dist/scripts/publish-preverify.js +47 -41
- package/dist/scripts/realtime-pricing-tests.js +37 -76
- package/dist/scripts/replay-async.js +20 -18
- package/dist/scripts/replay-real.js +16 -18
- package/dist/scripts/replay.js +52 -82
- package/dist/scripts/session-attach-diagnose.js +16 -12
- package/dist/scripts/session-attach-poc.js +34 -37
- package/dist/scripts/session-benchmark.js +66 -147
- package/dist/scripts/session-dida-live-e2e.js +151 -0
- package/dist/scripts/session-extract-tests.js +14 -35
- package/dist/scripts/session-login.js +14 -13
- package/dist/scripts/session-tests.js +545 -583
- package/dist/scripts/sf-live-benchmark.js +92 -110
- package/dist/scripts/sf-live-cli-tests.js +2 -9
- package/dist/scripts/sf-soft-score-tests.js +19 -60
- package/dist/scripts/sf-summary-tests.js +149 -0
- package/dist/scripts/sf-summary.js +466 -84
- package/dist/scripts/skeleton-check.js +16 -14
- package/dist/scripts/skeleton-integration-test.js +11 -10
- package/dist/scripts/skills-contract-tests.js +51 -46
- package/dist/scripts/smoke-session-gate-tests.js +12 -20
- package/dist/scripts/smoke.js +541 -422
- package/dist/scripts/sponsor-reuse-tests.js +262 -0
- package/dist/scripts/state-cli-tests.js +226 -101
- package/dist/scripts/state-cli.js +549 -212
- package/dist/scripts/static-golden-tests.js +46 -180
- package/dist/scripts/time-eval-tests.js +210 -192
- package/dist/scripts/travel-timeline-tests.js +29 -93
- package/dist/scripts/turn-handoff-collect-tests.js +46 -93
- package/dist/scripts/turn-handoff-collect.js +71 -99
- package/dist/scripts/turn-policy-tests.js +28 -29
- package/dist/scripts/typed-contract-canary.js +77 -162
- package/dist/scripts/tz-resolver-sanity.js +50 -0
- package/dist/scripts/tz-resolver-tests.js +584 -0
- package/dist/scripts/unified-tests.js +23 -19
- package/dist/scripts/visa-policy-tests.js +31 -44
- package/dist/scripts/weather-tests.js +121 -543
- package/dist/scripts/wish-channel-gate-tests.js +29 -86
- package/dist/scripts/write-gate-tests.js +522 -0
- package/dist/scripts/z3-lifecycle-fault-tests.js +188 -0
- package/dist/scripts/z3-lifecycle-tests.js +170 -0
- package/dist/scripts/z3-race-repeat-tests.js +39 -0
- package/dist/scripts/z3-race-tests.js +18 -12
- package/dist/src/artifact-gate.js +652 -221
- package/dist/src/backend/kernel.js +91 -0
- package/dist/src/backend/modules/booking-copilot.js +55 -0
- package/dist/src/backend/modules/booking-executor.js +124 -0
- package/dist/src/backend/modules/session-search.js +194 -0
- package/dist/src/benchmark-agent-conformance.js +296 -262
- package/dist/src/benchmark-environment-bridge.js +298 -413
- package/dist/src/benchmark-headless-child-diagnostics.js +93 -100
- package/dist/src/benchmark-tool-isolation.js +94 -67
- package/dist/src/bookable-facts.js +299 -174
- package/dist/src/booking-saga.js +64 -120
- package/dist/src/booking-surface/availability-policy.js +443 -576
- package/dist/src/booking-surface/cancel-refund-commission.js +379 -0
- package/dist/src/booking-surface/contracts.js +24 -80
- package/dist/src/booking-surface/dsh-planner.js +510 -281
- package/dist/src/booking-surface/dsh-plugin.js +17 -1
- package/dist/src/booking-surface/error-codes.js +82 -10
- package/dist/src/booking-surface/index.js +0 -3
- package/dist/src/booking-surface/profile.js +10 -54
- package/dist/src/booking-surface/recovery-chain.js +367 -0
- package/dist/src/booking-surface/runtime.js +765 -891
- package/dist/src/booking-surface/server.js +200 -251
- package/dist/src/booking-surface/startup.js +58 -54
- package/dist/src/booking-surface/validation.js +150 -218
- package/dist/src/bridge.js +20 -13
- package/dist/src/companions.js +48 -74
- package/dist/src/contracts.js +21 -43
- package/dist/src/dsh-llm.js +82 -120
- package/dist/src/engine.js +253 -239
- package/dist/src/evaluation-cadence.js +72 -209
- package/dist/src/evaluation-contracts.js +220 -843
- package/dist/src/flight-pack-adapter.js +54 -0
- package/dist/src/flight-pack-contract.js +2 -0
- package/dist/src/gotry-backend.js +70 -0
- package/dist/src/hotel-date-gate.js +118 -0
- package/dist/src/i18n.js +18 -7
- package/dist/src/index.js +1425 -1629
- package/dist/src/journey.js +132 -81
- package/dist/src/ledger-repair-apply.js +538 -0
- package/dist/src/ledger-repair-plan.js +463 -0
- package/dist/src/loop.js +257 -166
- package/dist/src/memory-capture.js +180 -32
- package/dist/src/memory-decay.js +30 -14
- package/dist/src/memory-lifecycle.js +1310 -0
- package/dist/src/memory-utility.js +27 -40
- package/dist/src/mock-llm.js +36 -61
- package/dist/src/model.js +81 -36
- package/dist/src/realtime-pricing.js +63 -72
- package/dist/src/slot-spec.js +84 -106
- package/dist/src/state-ledger.js +310 -471
- package/dist/src/subagent-job-id-guard.js +58 -0
- package/dist/src/time-anchor.js +231 -33
- package/dist/src/tool-packet.js +20 -8
- package/dist/src/travel-slots.js +58 -35
- package/dist/src/travel-timeline.js +44 -63
- package/dist/src/turn-deadline.js +121 -127
- package/dist/src/turn-policy.js +61 -89
- package/dist/src/tz-resolver.js +143 -0
- package/dist/src/unified.js +504 -381
- package/dist/src/wish-pool.js +39 -14
- package/dist/src/write-gate.js +495 -0
- package/dist/src/z3-shared.js +273 -7
- package/extension/README.md +4 -3
- package/extension/background.js +125 -14
- package/extension/content-bridge.js +23 -1
- package/extension/content-main.js +15 -4
- package/extension/manifest.json +16 -8
- package/package.json +251 -220
- package/ts/capabilities/agent-reach.ts +4 -4
- package/ts/capabilities/anything.ts +1 -1
- package/ts/capabilities/artifacts.ts +46 -9
- package/ts/capabilities/flyai.ts +156 -37
- package/ts/capabilities/hbcli.ts +1 -1
- package/ts/capabilities/incident-log.ts +88 -59
- package/ts/capabilities/session/adapters/ctrip-flight.ts +99 -36
- package/ts/capabilities/session/adapters/dida-portal.ts +250 -0
- package/ts/capabilities/session/adapters/rail-12306.ts +123 -24
- package/ts/capabilities/session/extension-bridge.ts +226 -101
- package/ts/capabilities/session/extension-channel.ts +22 -9
- package/ts/capabilities/session/wizard.ts +2 -2
- package/ts/capabilities/session-consent.ts +98 -14
- package/ts/capabilities/session-login.ts +1 -0
- package/ts/capabilities/session-search.ts +315 -31
- package/ts/dsh-runtime/vendor/README.md +8 -4
- package/ts/dsh-runtime/vendor/dsh-map-tools/README.en.md +7 -1
- package/ts/dsh-runtime/vendor/dsh-map-tools/README.md +5 -3
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/settings-ns.js +1 -1
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/settings-ns.d.ts +1 -1
- package/ts/dsh-runtime/vendor/dsh-map-tools/package.json +6 -6
- package/ts/package.json +236 -16
- package/ts/scripts/map-tools-vendor-package-proof.ts +152 -31
- package/ts/scripts/state-cli.ts +418 -51
- package/ts/src/artifact-gate.ts +561 -15
- package/ts/src/benchmark-environment-bridge.ts +37 -20
- package/ts/src/bookable-facts.ts +183 -7
- package/ts/src/booking-surface/cancel-refund-commission.ts +488 -0
- package/ts/src/booking-surface/dsh-planner.ts +240 -32
- package/ts/src/booking-surface/dsh-plugin.js +17 -1
- package/ts/src/booking-surface/error-codes.ts +75 -0
- package/ts/src/booking-surface/recovery-chain.ts +409 -0
- package/ts/src/booking-surface/server.ts +84 -51
- package/ts/src/booking-surface/validation.ts +2 -0
- package/ts/src/contracts.ts +12 -2
- package/ts/src/dsh-llm.ts +4 -8
- package/ts/src/index.ts +609 -90
- package/ts/src/loop.ts +116 -7
- package/ts/src/memory-capture.ts +227 -3
- package/ts/src/mock-llm.ts +4 -8
- package/ts/src/model.ts +88 -2
- package/ts/src/state-ledger.ts +47 -29
- package/ts/src/subagent-job-id-guard.ts +72 -0
- package/ts/src/time-anchor.ts +217 -13
- package/ts/src/unified.ts +316 -54
|
@@ -4,251 +4,54 @@ import { tmpdir } from 'node:os';
|
|
|
4
4
|
import { join } from 'node:path';
|
|
5
5
|
import { Context } from '@deepseek-ai/cordis';
|
|
6
6
|
import { apply } from '../src/index.js';
|
|
7
|
-
import { BRIDGE_ERROR_CONTRACT, BENCHMARK_TOOL_RESULT_SCHEMA_VERSION, registerBenchmarkEnvironmentBridge } from '../src/benchmark-environment-bridge.js';
|
|
7
|
+
import { BRIDGE_ERROR_CONTRACT, BENCHMARK_TOOL_RESULT_SCHEMA_VERSION, registerBenchmarkEnvironmentBridge, } from '../src/benchmark-environment-bridge.js';
|
|
8
8
|
import { installBenchmarkToolIsolation } from '../src/benchmark-tool-isolation.js';
|
|
9
|
-
import { BENCHMARK_BRIDGE_CALL_FAILED, BENCHMARK_BRIDGE_CALL_REQUIRED, BENCHMARK_BRIDGE_RETRY_CALL_NOT_ALLOWED, BENCHMARK_BRIDGE_OUTPUT_TRUNCATED, BENCHMARK_BRIDGE_RUNNER_FAILED, BENCHMARK_BRIDGE_SPAWN_FAILED, BENCHMARK_BRIDGE_TIMED_OUT, BENCHMARK_CONFORMANCE_STATE_UNAVAILABLE, BENCHMARK_TERMINAL_INVALID, MAX_CONFORMANCE_RETRIES, benchmarkChildFailureForConformanceCode, createBenchmarkAgentConformance, installBenchmarkAgentConformance, parseBenchmarkTerminal, terminalSchemaOutline, validateTerminalBodySchema, validateTerminalBodyValue, validateTerminalOutputConfig } from '../src/benchmark-agent-conformance.js';
|
|
10
|
-
import { BENCHMARK_CHILD_DIAGNOSTIC_MAX_BYTES, BENCHMARK_CHILD_DIAGNOSTIC_SCHEMA, appendBoundedChildDiagnostic, classifyBenchmarkChildFailure, classifyBenchmarkTurnEnd, createBenchmarkDiagnosticArbiter, parseBenchmarkChildDiagnostic } from '../src/benchmark-headless-child-diagnostics.js';
|
|
9
|
+
import { BENCHMARK_BRIDGE_CALL_FAILED, BENCHMARK_BRIDGE_CALL_REQUIRED, BENCHMARK_BRIDGE_RETRY_CALL_NOT_ALLOWED, BENCHMARK_BRIDGE_OUTPUT_TRUNCATED, BENCHMARK_BRIDGE_RUNNER_FAILED, BENCHMARK_BRIDGE_SPAWN_FAILED, BENCHMARK_BRIDGE_TIMED_OUT, BENCHMARK_CONFORMANCE_STATE_UNAVAILABLE, BENCHMARK_TERMINAL_INVALID, MAX_CONFORMANCE_RETRIES, benchmarkChildFailureForConformanceCode, createBenchmarkAgentConformance, installBenchmarkAgentConformance, parseBenchmarkTerminal, terminalSchemaOutline, validateTerminalBodySchema, validateTerminalBodyValue, validateTerminalOutputConfig, } from '../src/benchmark-agent-conformance.js';
|
|
10
|
+
import { BENCHMARK_CHILD_DIAGNOSTIC_MAX_BYTES, BENCHMARK_CHILD_DIAGNOSTIC_SCHEMA, appendBoundedChildDiagnostic, classifyBenchmarkChildFailure, classifyBenchmarkTurnEnd, createBenchmarkDiagnosticArbiter, parseBenchmarkChildDiagnostic, } from '../src/benchmark-headless-child-diagnostics.js';
|
|
11
|
+
// Round 6 RED tests: terminal facts must be classified from the structured
|
|
12
|
+
// turn/end envelope, without consulting stderr or reflecting its body.
|
|
11
13
|
{
|
|
12
14
|
const exactFamilies = [
|
|
13
|
-
[
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
],
|
|
21
|
-
[
|
|
22
|
-
[
|
|
23
|
-
'QUOTA',
|
|
24
|
-
'RATE_LIMIT'
|
|
25
|
-
],
|
|
26
|
-
'child_model_capacity'
|
|
27
|
-
],
|
|
28
|
-
[
|
|
29
|
-
[
|
|
30
|
-
'SERVER'
|
|
31
|
-
],
|
|
32
|
-
'child_model_server'
|
|
33
|
-
],
|
|
34
|
-
[
|
|
35
|
-
[
|
|
36
|
-
'TRANSPORT',
|
|
37
|
-
'TIMEOUT'
|
|
38
|
-
],
|
|
39
|
-
'child_model_transport'
|
|
40
|
-
],
|
|
41
|
-
[
|
|
42
|
-
[
|
|
43
|
-
'EMPTY_RESPONSE',
|
|
44
|
-
'STREAM_CLOSED',
|
|
45
|
-
'MALFORMED_RESPONSE',
|
|
46
|
-
'INVALID_RESPONSE'
|
|
47
|
-
],
|
|
48
|
-
'child_model_stream'
|
|
49
|
-
],
|
|
50
|
-
[
|
|
51
|
-
[
|
|
52
|
-
'INVALID_REQUEST',
|
|
53
|
-
'CONTEXT_WINDOW_EXCEEDED',
|
|
54
|
-
'NO_ADAPTER',
|
|
55
|
-
'UNKNOWN_MODEL',
|
|
56
|
-
'UNSUPPORTED_OPTION'
|
|
57
|
-
],
|
|
58
|
-
'child_model_request'
|
|
59
|
-
],
|
|
60
|
-
[
|
|
61
|
-
[
|
|
62
|
-
'ABORTED'
|
|
63
|
-
],
|
|
64
|
-
'child_aborted'
|
|
65
|
-
]
|
|
15
|
+
[['AUTH', 'INVALID_CREDENTIAL', 'MISSING_CREDENTIAL'], 'child_model_auth'],
|
|
16
|
+
[['QUOTA', 'RATE_LIMIT'], 'child_model_capacity'],
|
|
17
|
+
[['SERVER'], 'child_model_server'],
|
|
18
|
+
[['TRANSPORT', 'TIMEOUT'], 'child_model_transport'],
|
|
19
|
+
[['EMPTY_RESPONSE', 'STREAM_CLOSED', 'MALFORMED_RESPONSE', 'INVALID_RESPONSE'], 'child_model_stream'],
|
|
20
|
+
[['INVALID_REQUEST', 'CONTEXT_WINDOW_EXCEEDED', 'NO_ADAPTER', 'UNKNOWN_MODEL', 'UNSUPPORTED_OPTION'], 'child_model_request'],
|
|
21
|
+
[['ABORTED'], 'child_aborted'],
|
|
66
22
|
];
|
|
67
|
-
for (const [codes, expected] of exactFamilies){
|
|
68
|
-
for (const code of codes)
|
|
69
|
-
kind: 'error',
|
|
70
|
-
error: {
|
|
71
|
-
code
|
|
72
|
-
}
|
|
73
|
-
}), expected);
|
|
23
|
+
for (const [codes, expected] of exactFamilies) {
|
|
24
|
+
for (const code of codes)
|
|
25
|
+
assert.equal(classifyBenchmarkTurnEnd({ kind: 'error', error: { code } }), expected);
|
|
74
26
|
}
|
|
75
|
-
assert.equal(classifyBenchmarkTurnEnd({
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
}), '
|
|
82
|
-
assert.equal(classifyBenchmarkTurnEnd({
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
}), '
|
|
89
|
-
assert.equal(classifyBenchmarkTurnEnd({
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
}), '
|
|
96
|
-
assert.equal(classifyBenchmarkTurnEnd({
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
}
|
|
102
|
-
}), 'child_model_capacity');
|
|
103
|
-
assert.equal(classifyBenchmarkTurnEnd({
|
|
104
|
-
kind: 'error',
|
|
105
|
-
error: {
|
|
106
|
-
code: 'SERVER',
|
|
107
|
-
message: 'sentinel'
|
|
108
|
-
}
|
|
109
|
-
}), 'child_model_server');
|
|
110
|
-
assert.equal(classifyBenchmarkTurnEnd({
|
|
111
|
-
kind: 'error',
|
|
112
|
-
error: {
|
|
113
|
-
code: 'TRANSPORT',
|
|
114
|
-
message: 'api-key sentinel'
|
|
115
|
-
}
|
|
116
|
-
}), 'child_model_transport');
|
|
117
|
-
assert.equal(classifyBenchmarkTurnEnd({
|
|
118
|
-
kind: 'error',
|
|
119
|
-
error: {
|
|
120
|
-
code: 'OTHER',
|
|
121
|
-
status: 401,
|
|
122
|
-
message: 'key sentinel'
|
|
123
|
-
}
|
|
124
|
-
}), 'child_model_auth');
|
|
125
|
-
assert.equal(classifyBenchmarkTurnEnd({
|
|
126
|
-
kind: 'error',
|
|
127
|
-
error: {
|
|
128
|
-
code: 'OTHER',
|
|
129
|
-
status: 403,
|
|
130
|
-
message: 'key sentinel'
|
|
131
|
-
}
|
|
132
|
-
}), 'child_model_auth');
|
|
133
|
-
assert.equal(classifyBenchmarkTurnEnd({
|
|
134
|
-
kind: 'error',
|
|
135
|
-
error: {
|
|
136
|
-
code: 'OTHER',
|
|
137
|
-
status: 429,
|
|
138
|
-
message: 'quota sentinel'
|
|
139
|
-
}
|
|
140
|
-
}), 'child_model_capacity');
|
|
141
|
-
assert.equal(classifyBenchmarkTurnEnd({
|
|
142
|
-
kind: 'error',
|
|
143
|
-
error: {
|
|
144
|
-
code: 'OTHER',
|
|
145
|
-
status: 503,
|
|
146
|
-
message: 'server sentinel'
|
|
147
|
-
}
|
|
148
|
-
}), 'child_model_server');
|
|
149
|
-
assert.equal(classifyBenchmarkTurnEnd({
|
|
150
|
-
kind: 'error',
|
|
151
|
-
error: {
|
|
152
|
-
code: 'OTHER',
|
|
153
|
-
status: 500,
|
|
154
|
-
message: 'server sentinel'
|
|
155
|
-
}
|
|
156
|
-
}), 'child_model_server');
|
|
157
|
-
assert.equal(classifyBenchmarkTurnEnd({
|
|
158
|
-
kind: 'error',
|
|
159
|
-
error: {
|
|
160
|
-
code: 'OTHER',
|
|
161
|
-
status: 599,
|
|
162
|
-
message: 'server sentinel'
|
|
163
|
-
}
|
|
164
|
-
}), 'child_model_server');
|
|
165
|
-
assert.equal(classifyBenchmarkTurnEnd({
|
|
166
|
-
kind: 'error',
|
|
167
|
-
error: {
|
|
168
|
-
code: 'PI_AI_ERROR',
|
|
169
|
-
message: 'opaque'
|
|
170
|
-
}
|
|
171
|
-
}), 'child_runtime_error');
|
|
172
|
-
assert.equal(classifyBenchmarkTurnEnd({
|
|
173
|
-
kind: 'error',
|
|
174
|
-
error: {
|
|
175
|
-
code: 'INVALID_RESPONSE',
|
|
176
|
-
message: 'opaque'
|
|
177
|
-
}
|
|
178
|
-
}), 'child_model_stream');
|
|
179
|
-
assert.equal(classifyBenchmarkTurnEnd({
|
|
180
|
-
kind: 'error',
|
|
181
|
-
error: {
|
|
182
|
-
code: 'UNSUPPORTED_OPTION',
|
|
183
|
-
message: 'opaque'
|
|
184
|
-
}
|
|
185
|
-
}), 'child_model_request');
|
|
186
|
-
assert.equal(classifyBenchmarkTurnEnd({
|
|
187
|
-
kind: 'error',
|
|
188
|
-
error: {
|
|
189
|
-
code: 'OTHER',
|
|
190
|
-
status: 503.5,
|
|
191
|
-
message: 'opaque'
|
|
192
|
-
}
|
|
193
|
-
}), 'child_runtime_error');
|
|
194
|
-
assert.equal(classifyBenchmarkTurnEnd({
|
|
195
|
-
kind: 'error',
|
|
196
|
-
error: {
|
|
197
|
-
code: 'OTHER',
|
|
198
|
-
status: 99,
|
|
199
|
-
message: 'opaque'
|
|
200
|
-
}
|
|
201
|
-
}), 'child_runtime_error');
|
|
202
|
-
assert.equal(classifyBenchmarkTurnEnd({
|
|
203
|
-
kind: 'error',
|
|
204
|
-
error: {
|
|
205
|
-
code: 'OTHER',
|
|
206
|
-
status: 600,
|
|
207
|
-
message: 'opaque'
|
|
208
|
-
}
|
|
209
|
-
}), 'child_runtime_error');
|
|
210
|
-
assert.equal(classifyBenchmarkTurnEnd({
|
|
211
|
-
kind: 'error',
|
|
212
|
-
error: {
|
|
213
|
-
code: 'rate_limit',
|
|
214
|
-
message: 'opaque'
|
|
215
|
-
}
|
|
216
|
-
}), 'child_runtime_error');
|
|
217
|
-
assert.equal(classifyBenchmarkTurnEnd({
|
|
218
|
-
kind: 'error',
|
|
219
|
-
error: {
|
|
220
|
-
code: 'UNKNOWN',
|
|
221
|
-
message: 'opaque'
|
|
222
|
-
}
|
|
223
|
-
}), 'child_runtime_error');
|
|
224
|
-
assert.equal(classifyBenchmarkTurnEnd({
|
|
225
|
-
kind: 'error',
|
|
226
|
-
error: {
|
|
227
|
-
code: 'UNKNOWN',
|
|
228
|
-
message: 'message sentinel',
|
|
229
|
-
requestId: 'request sentinel',
|
|
230
|
-
path: 'path sentinel',
|
|
231
|
-
prompt: 'prompt sentinel',
|
|
232
|
-
key: 'key sentinel'
|
|
233
|
-
}
|
|
234
|
-
}), 'child_runtime_error');
|
|
235
|
-
assert.equal(classifyBenchmarkTurnEnd({
|
|
236
|
-
kind: 'blocked'
|
|
237
|
-
}), 'child_blocked');
|
|
238
|
-
assert.equal(classifyBenchmarkTurnEnd({
|
|
239
|
-
kind: 'max-tokens'
|
|
240
|
-
}), 'child_max_tokens');
|
|
241
|
-
assert.equal(classifyBenchmarkTurnEnd({
|
|
242
|
-
kind: 'aborted'
|
|
243
|
-
}), 'child_aborted');
|
|
244
|
-
assert.equal(classifyBenchmarkTurnEnd({
|
|
245
|
-
kind: 'interrupted'
|
|
246
|
-
}), 'child_interrupted');
|
|
247
|
-
assert.equal(classifyBenchmarkTurnEnd({
|
|
248
|
-
kind: 'completed'
|
|
249
|
-
}), undefined);
|
|
27
|
+
assert.equal(classifyBenchmarkTurnEnd({ kind: 'error', error: { code: 'RATE_LIMIT', message: 'sentinel' } }), 'child_model_capacity');
|
|
28
|
+
assert.equal(classifyBenchmarkTurnEnd({ kind: 'error', error: { code: 'AUTH', message: 'sentinel' } }), 'child_model_auth');
|
|
29
|
+
assert.equal(classifyBenchmarkTurnEnd({ kind: 'error', error: { code: 'MISSING_CREDENTIAL', message: 'sentinel' } }), 'child_model_auth');
|
|
30
|
+
assert.equal(classifyBenchmarkTurnEnd({ kind: 'error', error: { code: 'QUOTA', message: 'sentinel' } }), 'child_model_capacity');
|
|
31
|
+
assert.equal(classifyBenchmarkTurnEnd({ kind: 'error', error: { code: 'SERVER', message: 'sentinel' } }), 'child_model_server');
|
|
32
|
+
assert.equal(classifyBenchmarkTurnEnd({ kind: 'error', error: { code: 'TRANSPORT', message: 'api-key sentinel' } }), 'child_model_transport');
|
|
33
|
+
assert.equal(classifyBenchmarkTurnEnd({ kind: 'error', error: { code: 'OTHER', status: 401, message: 'key sentinel' } }), 'child_model_auth');
|
|
34
|
+
assert.equal(classifyBenchmarkTurnEnd({ kind: 'error', error: { code: 'OTHER', status: 403, message: 'key sentinel' } }), 'child_model_auth');
|
|
35
|
+
assert.equal(classifyBenchmarkTurnEnd({ kind: 'error', error: { code: 'OTHER', status: 429, message: 'quota sentinel' } }), 'child_model_capacity');
|
|
36
|
+
assert.equal(classifyBenchmarkTurnEnd({ kind: 'error', error: { code: 'OTHER', status: 503, message: 'server sentinel' } }), 'child_model_server');
|
|
37
|
+
assert.equal(classifyBenchmarkTurnEnd({ kind: 'error', error: { code: 'OTHER', status: 500, message: 'server sentinel' } }), 'child_model_server');
|
|
38
|
+
assert.equal(classifyBenchmarkTurnEnd({ kind: 'error', error: { code: 'OTHER', status: 599, message: 'server sentinel' } }), 'child_model_server');
|
|
39
|
+
assert.equal(classifyBenchmarkTurnEnd({ kind: 'error', error: { code: 'PI_AI_ERROR', message: 'opaque' } }), 'child_runtime_error');
|
|
40
|
+
assert.equal(classifyBenchmarkTurnEnd({ kind: 'error', error: { code: 'INVALID_RESPONSE', message: 'opaque' } }), 'child_model_stream');
|
|
41
|
+
assert.equal(classifyBenchmarkTurnEnd({ kind: 'error', error: { code: 'UNSUPPORTED_OPTION', message: 'opaque' } }), 'child_model_request');
|
|
42
|
+
assert.equal(classifyBenchmarkTurnEnd({ kind: 'error', error: { code: 'OTHER', status: 503.5, message: 'opaque' } }), 'child_runtime_error');
|
|
43
|
+
assert.equal(classifyBenchmarkTurnEnd({ kind: 'error', error: { code: 'OTHER', status: 99, message: 'opaque' } }), 'child_runtime_error');
|
|
44
|
+
assert.equal(classifyBenchmarkTurnEnd({ kind: 'error', error: { code: 'OTHER', status: 600, message: 'opaque' } }), 'child_runtime_error');
|
|
45
|
+
assert.equal(classifyBenchmarkTurnEnd({ kind: 'error', error: { code: 'rate_limit', message: 'opaque' } }), 'child_runtime_error');
|
|
46
|
+
assert.equal(classifyBenchmarkTurnEnd({ kind: 'error', error: { code: 'UNKNOWN', message: 'opaque' } }), 'child_runtime_error');
|
|
47
|
+
assert.equal(classifyBenchmarkTurnEnd({ kind: 'error', error: { code: 'UNKNOWN', message: 'message sentinel', requestId: 'request sentinel', path: 'path sentinel', prompt: 'prompt sentinel', key: 'key sentinel' } }), 'child_runtime_error');
|
|
48
|
+
assert.equal(classifyBenchmarkTurnEnd({ kind: 'blocked' }), 'child_blocked');
|
|
49
|
+
assert.equal(classifyBenchmarkTurnEnd({ kind: 'max-tokens' }), 'child_max_tokens');
|
|
50
|
+
assert.equal(classifyBenchmarkTurnEnd({ kind: 'aborted' }), 'child_aborted');
|
|
51
|
+
assert.equal(classifyBenchmarkTurnEnd({ kind: 'interrupted' }), 'child_interrupted');
|
|
52
|
+
assert.equal(classifyBenchmarkTurnEnd({ kind: 'completed' }), undefined);
|
|
250
53
|
const writes = [];
|
|
251
|
-
const arbiter = createBenchmarkDiagnosticArbiter(
|
|
54
|
+
const arbiter = createBenchmarkDiagnosticArbiter(code => writes.push(code));
|
|
252
55
|
arbiter.offer('session-a', 'child_conformance_failure');
|
|
253
56
|
arbiter.offer('session-a', 'child_runtime_error');
|
|
254
57
|
arbiter.offer('session-a', 'child_bridge_failure');
|
|
@@ -257,82 +60,40 @@ import { BENCHMARK_CHILD_DIAGNOSTIC_MAX_BYTES, BENCHMARK_CHILD_DIAGNOSTIC_SCHEMA
|
|
|
257
60
|
arbiter.flush('session-a');
|
|
258
61
|
arbiter.flush('session-b');
|
|
259
62
|
arbiter.flush('session-b');
|
|
260
|
-
assert.deepEqual(writes, [
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
63
|
+
assert.deepEqual(writes, ['child_bridge_failure', 'child_model_server']);
|
|
64
|
+
}
|
|
65
|
+
assert.equal(MAX_CONFORMANCE_RETRIES, 1);
|
|
66
|
+
// Round 12(#215):terminal body schema 是 closed 结构合同。既有信封/协议用例沿用
|
|
67
|
+
// 一个宽松但 closed 的 fixture(全部键可选),schema 严格性由 Round 12 专段的
|
|
68
|
+
// ChinaTravel-like closed schema 覆盖。
|
|
265
69
|
const OK_BODY_SCHEMA = {
|
|
266
70
|
type: 'object',
|
|
267
71
|
properties: {
|
|
268
|
-
ok: {
|
|
269
|
-
|
|
270
|
-
},
|
|
271
|
-
status: {
|
|
272
|
-
type: 'string'
|
|
273
|
-
},
|
|
274
|
-
x: {
|
|
275
|
-
type: 'string'
|
|
276
|
-
}
|
|
72
|
+
ok: { type: 'boolean' },
|
|
73
|
+
status: { type: 'string' },
|
|
74
|
+
x: { type: 'string' },
|
|
277
75
|
},
|
|
278
76
|
required: [],
|
|
279
|
-
additionalProperties: false
|
|
77
|
+
additionalProperties: false,
|
|
280
78
|
};
|
|
281
79
|
const projection = {
|
|
282
80
|
toolName: 'gotry_benchmark_environment',
|
|
283
|
-
allowedTools: [
|
|
284
|
-
|
|
285
|
-
],
|
|
286
|
-
terminal: {
|
|
287
|
-
tag: 'done',
|
|
288
|
-
max_bytes: 1024,
|
|
289
|
-
body_schema: OK_BODY_SCHEMA
|
|
290
|
-
}
|
|
81
|
+
allowedTools: ['lookup'],
|
|
82
|
+
terminal: { tag: 'done', max_bytes: 1024, body_schema: OK_BODY_SCHEMA },
|
|
291
83
|
};
|
|
292
84
|
assert.equal(validateTerminalOutputConfig(projection.terminal), true);
|
|
293
85
|
for (const invalid of [
|
|
294
86
|
null,
|
|
295
|
-
{
|
|
296
|
-
|
|
297
|
-
},
|
|
298
|
-
{
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
},
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
},
|
|
306
|
-
{
|
|
307
|
-
tag: 'done',
|
|
308
|
-
max_bytes: 1024 * 1024 + 1
|
|
309
|
-
},
|
|
310
|
-
{
|
|
311
|
-
tag: 'done',
|
|
312
|
-
max_bytes: 1024,
|
|
313
|
-
extra: true
|
|
314
|
-
},
|
|
315
|
-
{
|
|
316
|
-
tag: 'done',
|
|
317
|
-
max_bytes: 1024
|
|
318
|
-
},
|
|
319
|
-
{
|
|
320
|
-
tag: 'done',
|
|
321
|
-
max_bytes: 1024,
|
|
322
|
-
body_schema: {
|
|
323
|
-
type: 'object',
|
|
324
|
-
properties: {},
|
|
325
|
-
required: [],
|
|
326
|
-
additionalProperties: true
|
|
327
|
-
}
|
|
328
|
-
}
|
|
329
|
-
])assert.equal(validateTerminalOutputConfig(invalid), false);
|
|
330
|
-
assert.deepEqual(parseBenchmarkTerminal(' \n<done>{"ok":true}</done>\n', projection.terminal), {
|
|
331
|
-
ok: true,
|
|
332
|
-
value: {
|
|
333
|
-
ok: true
|
|
334
|
-
}
|
|
335
|
-
});
|
|
87
|
+
{ tag: 'done' },
|
|
88
|
+
{ tag: '1bad', max_bytes: 1024 },
|
|
89
|
+
{ tag: 'done', max_bytes: 0 },
|
|
90
|
+
{ tag: 'done', max_bytes: 1024 * 1024 + 1 },
|
|
91
|
+
{ tag: 'done', max_bytes: 1024, extra: true },
|
|
92
|
+
{ tag: 'done', max_bytes: 1024 },
|
|
93
|
+
{ tag: 'done', max_bytes: 1024, body_schema: { type: 'object', properties: {}, required: [], additionalProperties: true } },
|
|
94
|
+
])
|
|
95
|
+
assert.equal(validateTerminalOutputConfig(invalid), false);
|
|
96
|
+
assert.deepEqual(parseBenchmarkTerminal(' \n<done>{"ok":true}</done>\n', projection.terminal), { ok: true, value: { ok: true } });
|
|
336
97
|
for (const invalid of [
|
|
337
98
|
'prose <done>{"ok":true}</done>',
|
|
338
99
|
'<done>{"ok":true}</done> trailing',
|
|
@@ -342,328 +103,130 @@ for (const invalid of [
|
|
|
342
103
|
'<wrong>{"ok":true}</wrong>',
|
|
343
104
|
'<done>[{"ok":true}]</done>',
|
|
344
105
|
'<done>true</done>',
|
|
345
|
-
'<done>{"ok":</done>'
|
|
346
|
-
])
|
|
106
|
+
'<done>{"ok":</done>',
|
|
107
|
+
])
|
|
108
|
+
assert.equal(parseBenchmarkTerminal(invalid, projection.terminal).ok, false);
|
|
347
109
|
assert.equal(parseBenchmarkTerminal(`<done>{"x":"${'y'.repeat(1024)}"}</done>`, projection.terminal).ok, false);
|
|
110
|
+
// Reasoning-model compatibility (Round 9): remove complete <think> blocks before
|
|
111
|
+
// applying the unchanged strict terminal-envelope validation. Ordinary prose and
|
|
112
|
+
// unclosed thinking blocks remain fail-closed.
|
|
348
113
|
for (const [raw, expectOk] of [
|
|
349
|
-
[
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
],
|
|
353
|
-
[
|
|
354
|
-
|
|
355
|
-
true
|
|
356
|
-
],
|
|
357
|
-
[
|
|
358
|
-
'<done>{"ok":true}</done><think>after</think>',
|
|
359
|
-
true
|
|
360
|
-
],
|
|
361
|
-
[
|
|
362
|
-
'<think>unclosed<done>{"ok":true}</done>',
|
|
363
|
-
false
|
|
364
|
-
],
|
|
365
|
-
[
|
|
366
|
-
'prose<think>x</think><done>{"ok":true}</done>',
|
|
367
|
-
false
|
|
368
|
-
]
|
|
369
|
-
]){
|
|
114
|
+
['<think>plan it</think><done>{"ok":true}</done>', true],
|
|
115
|
+
['<think>a</think>\n<THINK>b</THINK>\n<done>{"ok":true}</done>', true],
|
|
116
|
+
['<done>{"ok":true}</done><think>after</think>', true],
|
|
117
|
+
['<think>unclosed<done>{"ok":true}</done>', false],
|
|
118
|
+
['prose<think>x</think><done>{"ok":true}</done>', false],
|
|
119
|
+
]) {
|
|
370
120
|
assert.equal(parseBenchmarkTerminal(raw, projection.terminal).ok, expectOk, raw);
|
|
371
121
|
}
|
|
122
|
+
// Round 12 RED tests (issue #215): exact terminal schema projection. The bridge
|
|
123
|
+
// config carries a data-value-free closed body schema; the same structure contract
|
|
124
|
+
// is projected into the system prompt and the single terminal correction; the
|
|
125
|
+
// terminal body is fail-closed validated against it — no autofix, no coercion.
|
|
372
126
|
{
|
|
373
127
|
const travelSchema = {
|
|
374
128
|
type: 'object',
|
|
375
129
|
properties: {
|
|
376
|
-
people_number: {
|
|
377
|
-
|
|
378
|
-
},
|
|
379
|
-
start_city: {
|
|
380
|
-
type: 'string'
|
|
381
|
-
},
|
|
382
|
-
target_city: {
|
|
383
|
-
type: 'string'
|
|
384
|
-
},
|
|
130
|
+
people_number: { type: 'integer' },
|
|
131
|
+
start_city: { type: 'string' },
|
|
132
|
+
target_city: { type: 'string' },
|
|
385
133
|
itinerary: {
|
|
386
134
|
type: 'array',
|
|
387
135
|
items: {
|
|
388
136
|
type: 'object',
|
|
389
137
|
properties: {
|
|
390
|
-
day: {
|
|
391
|
-
type: 'integer'
|
|
392
|
-
},
|
|
138
|
+
day: { type: 'integer' },
|
|
393
139
|
activities: {
|
|
394
140
|
type: 'array',
|
|
395
141
|
items: {
|
|
396
142
|
type: 'object',
|
|
397
143
|
properties: {
|
|
398
|
-
type: {
|
|
399
|
-
|
|
400
|
-
},
|
|
401
|
-
start_time: {
|
|
402
|
-
type: 'string'
|
|
403
|
-
},
|
|
404
|
-
cost: {
|
|
405
|
-
type: 'number'
|
|
406
|
-
},
|
|
144
|
+
type: { type: 'string' },
|
|
145
|
+
start_time: { type: 'string' },
|
|
146
|
+
cost: { type: 'number' },
|
|
407
147
|
transports: {
|
|
408
148
|
type: 'array',
|
|
409
149
|
items: {
|
|
410
150
|
type: 'object',
|
|
411
|
-
properties: {
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
type: 'string'
|
|
417
|
-
}
|
|
418
|
-
},
|
|
419
|
-
required: [
|
|
420
|
-
'start',
|
|
421
|
-
'mode'
|
|
422
|
-
],
|
|
423
|
-
additionalProperties: false
|
|
424
|
-
}
|
|
425
|
-
}
|
|
151
|
+
properties: { start: { type: 'string' }, mode: { type: 'string' } },
|
|
152
|
+
required: ['start', 'mode'],
|
|
153
|
+
additionalProperties: false,
|
|
154
|
+
},
|
|
155
|
+
},
|
|
426
156
|
},
|
|
427
|
-
required: [
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
'transports'
|
|
432
|
-
],
|
|
433
|
-
additionalProperties: false
|
|
434
|
-
}
|
|
435
|
-
}
|
|
157
|
+
required: ['type', 'start_time', 'cost', 'transports'],
|
|
158
|
+
additionalProperties: false,
|
|
159
|
+
},
|
|
160
|
+
},
|
|
436
161
|
},
|
|
437
|
-
required: [
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
additionalProperties: false
|
|
442
|
-
}
|
|
443
|
-
}
|
|
162
|
+
required: ['day', 'activities'],
|
|
163
|
+
additionalProperties: false,
|
|
164
|
+
},
|
|
165
|
+
},
|
|
444
166
|
},
|
|
445
|
-
required: [
|
|
446
|
-
|
|
447
|
-
'start_city',
|
|
448
|
-
'target_city',
|
|
449
|
-
'itinerary'
|
|
450
|
-
],
|
|
451
|
-
additionalProperties: false
|
|
452
|
-
};
|
|
453
|
-
const travelConfig = {
|
|
454
|
-
tag: 'done',
|
|
455
|
-
max_bytes: 65_536,
|
|
456
|
-
body_schema: travelSchema
|
|
167
|
+
required: ['people_number', 'start_city', 'target_city', 'itinerary'],
|
|
168
|
+
additionalProperties: false,
|
|
457
169
|
};
|
|
170
|
+
const travelConfig = { tag: 'done', max_bytes: 65_536, body_schema: travelSchema };
|
|
458
171
|
assert.equal(validateTerminalOutputConfig(travelConfig), true, 'ChinaTravel-like closed schema is a valid terminal config');
|
|
459
172
|
const legalBody = {
|
|
460
173
|
people_number: 2,
|
|
461
174
|
start_city: 'Dubai',
|
|
462
175
|
target_city: 'Chengdu',
|
|
463
|
-
itinerary: [
|
|
464
|
-
{
|
|
465
|
-
day: 1,
|
|
466
|
-
activities: [
|
|
467
|
-
{
|
|
468
|
-
type: 'attraction',
|
|
469
|
-
start_time: '09:00',
|
|
470
|
-
cost: 0,
|
|
471
|
-
transports: [
|
|
472
|
-
{
|
|
473
|
-
start: 'hotel',
|
|
474
|
-
mode: 'walk'
|
|
475
|
-
}
|
|
476
|
-
]
|
|
477
|
-
}
|
|
478
|
-
]
|
|
479
|
-
}
|
|
480
|
-
]
|
|
176
|
+
itinerary: [{ day: 1, activities: [{ type: 'attraction', start_time: '09:00', cost: 0, transports: [{ start: 'hotel', mode: 'walk' }] }] }],
|
|
481
177
|
};
|
|
178
|
+
// Schema dialect: structural keywords only; data-bearing annotation faces are rejected.
|
|
482
179
|
assert.equal(validateTerminalBodySchema(travelSchema), true);
|
|
483
|
-
const mutate = (fn)=>{
|
|
180
|
+
const mutate = (fn) => {
|
|
484
181
|
const copy = JSON.parse(JSON.stringify(travelSchema));
|
|
485
182
|
fn(copy);
|
|
486
183
|
return copy;
|
|
487
184
|
};
|
|
488
185
|
for (const [label, schema] of [
|
|
489
|
-
[
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
],
|
|
495
|
-
[
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
],
|
|
503
|
-
[
|
|
504
|
-
'const on scalar',
|
|
505
|
-
mutate((s)=>{
|
|
506
|
-
s.properties.start_city.const = 'Dubai';
|
|
507
|
-
})
|
|
508
|
-
],
|
|
509
|
-
[
|
|
510
|
-
'default on scalar',
|
|
511
|
-
mutate((s)=>{
|
|
512
|
-
s.properties.target_city.default = 'Chengdu';
|
|
513
|
-
})
|
|
514
|
-
],
|
|
515
|
-
[
|
|
516
|
-
'examples key',
|
|
517
|
-
mutate((s)=>{
|
|
518
|
-
s.properties.itinerary.examples = [];
|
|
519
|
-
})
|
|
520
|
-
],
|
|
521
|
-
[
|
|
522
|
-
'pattern on scalar',
|
|
523
|
-
mutate((s)=>{
|
|
524
|
-
s.properties.start_city.pattern = '^[A-Z]';
|
|
525
|
-
})
|
|
526
|
-
],
|
|
527
|
-
[
|
|
528
|
-
'open object',
|
|
529
|
-
mutate((s)=>{
|
|
530
|
-
s.additionalProperties = true;
|
|
531
|
-
})
|
|
532
|
-
],
|
|
533
|
-
[
|
|
534
|
-
'missing additionalProperties',
|
|
535
|
-
mutate((s)=>{
|
|
536
|
-
delete s.additionalProperties;
|
|
537
|
-
})
|
|
538
|
-
],
|
|
539
|
-
[
|
|
540
|
-
'array without items',
|
|
541
|
-
mutate((s)=>{
|
|
542
|
-
s.properties.itinerary = {
|
|
543
|
-
type: 'array'
|
|
544
|
-
};
|
|
545
|
-
})
|
|
546
|
-
],
|
|
547
|
-
[
|
|
548
|
-
'object without required',
|
|
549
|
-
mutate((s)=>{
|
|
550
|
-
delete s.required;
|
|
551
|
-
})
|
|
552
|
-
],
|
|
553
|
-
[
|
|
554
|
-
'unknown type',
|
|
555
|
-
mutate((s)=>{
|
|
556
|
-
s.properties.people_number.type = 'float';
|
|
557
|
-
})
|
|
558
|
-
],
|
|
559
|
-
[
|
|
560
|
-
'allOf composition',
|
|
561
|
-
mutate((s)=>{
|
|
562
|
-
s.allOf = [];
|
|
563
|
-
})
|
|
564
|
-
]
|
|
565
|
-
]){
|
|
186
|
+
['extra root key', mutate(s => { s.description = 'the plan'; })],
|
|
187
|
+
['enum on scalar', mutate(s => { s.properties.people_number.enum = [2]; })],
|
|
188
|
+
['const on scalar', mutate(s => { s.properties.start_city.const = 'Dubai'; })],
|
|
189
|
+
['default on scalar', mutate(s => { s.properties.target_city.default = 'Chengdu'; })],
|
|
190
|
+
['examples key', mutate(s => { s.properties.itinerary.examples = []; })],
|
|
191
|
+
['pattern on scalar', mutate(s => { s.properties.start_city.pattern = '^[A-Z]'; })],
|
|
192
|
+
['open object', mutate(s => { s.additionalProperties = true; })],
|
|
193
|
+
['missing additionalProperties', mutate(s => { delete s.additionalProperties; })],
|
|
194
|
+
['array without items', mutate(s => { s.properties.itinerary = { type: 'array' }; })],
|
|
195
|
+
['object without required', mutate(s => { delete s.required; })],
|
|
196
|
+
['unknown type', mutate(s => { s.properties.people_number.type = 'float'; })],
|
|
197
|
+
['allOf composition', mutate(s => { s.allOf = []; })],
|
|
198
|
+
]) {
|
|
566
199
|
assert.equal(validateTerminalBodySchema(schema), false, `schema dialect rejects: ${label}`);
|
|
567
200
|
}
|
|
201
|
+
// Value validation: legal ChinaTravel-like hierarchy passes; the five Round 11
|
|
202
|
+
// failure classes stay rejected without any autofix.
|
|
568
203
|
assert.equal(validateTerminalBodyValue(legalBody, travelSchema), true, 'legal ChinaTravel-like hierarchy passes');
|
|
569
204
|
assert.equal(parseBenchmarkTerminal(`<done>${JSON.stringify(legalBody)}</done>`, travelConfig).ok, true, 'legal body parses through the terminal gate');
|
|
570
205
|
for (const [label, body] of [
|
|
571
|
-
[
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
}
|
|
578
|
-
}
|
|
579
|
-
],
|
|
580
|
-
[
|
|
581
|
-
'itinerary item written as direct activity (missing day)',
|
|
582
|
-
{
|
|
583
|
-
...legalBody,
|
|
584
|
-
itinerary: [
|
|
585
|
-
{
|
|
586
|
-
type: 'attraction',
|
|
587
|
-
start_time: '09:00',
|
|
588
|
-
cost: 0,
|
|
589
|
-
transports: []
|
|
590
|
-
}
|
|
591
|
-
]
|
|
592
|
-
}
|
|
593
|
-
],
|
|
594
|
-
[
|
|
595
|
-
'missing activities array',
|
|
596
|
-
{
|
|
597
|
-
...legalBody,
|
|
598
|
-
itinerary: [
|
|
599
|
-
{
|
|
600
|
-
day: 1
|
|
601
|
-
}
|
|
602
|
-
]
|
|
603
|
-
}
|
|
604
|
-
],
|
|
605
|
-
[
|
|
606
|
-
'wrong type (string day)',
|
|
607
|
-
{
|
|
608
|
-
...legalBody,
|
|
609
|
-
itinerary: [
|
|
610
|
-
{
|
|
611
|
-
day: '1',
|
|
612
|
-
activities: []
|
|
613
|
-
}
|
|
614
|
-
]
|
|
615
|
-
}
|
|
616
|
-
],
|
|
617
|
-
[
|
|
618
|
-
'nested extra field',
|
|
619
|
-
{
|
|
620
|
-
...legalBody,
|
|
621
|
-
itinerary: [
|
|
622
|
-
{
|
|
623
|
-
day: 1,
|
|
624
|
-
activities: [
|
|
625
|
-
{
|
|
626
|
-
type: 'attraction',
|
|
627
|
-
start_time: '09:00',
|
|
628
|
-
cost: 0,
|
|
629
|
-
transports: [],
|
|
630
|
-
unexpected: 1
|
|
631
|
-
}
|
|
632
|
-
]
|
|
633
|
-
}
|
|
634
|
-
]
|
|
635
|
-
}
|
|
636
|
-
]
|
|
637
|
-
]){
|
|
206
|
+
['extra root key (Round 11 budget/total_cost case)', { ...legalBody, budget: { total_cost: 1 } }],
|
|
207
|
+
['itinerary item written as direct activity (missing day)', { ...legalBody, itinerary: [{ type: 'attraction', start_time: '09:00', cost: 0, transports: [] }] }],
|
|
208
|
+
['missing activities array', { ...legalBody, itinerary: [{ day: 1 }] }],
|
|
209
|
+
['wrong type (string day)', { ...legalBody, itinerary: [{ day: '1', activities: [] }] }],
|
|
210
|
+
['nested extra field', { ...legalBody, itinerary: [{ day: 1, activities: [{ type: 'attraction', start_time: '09:00', cost: 0, transports: [], unexpected: 1 }] }] }],
|
|
211
|
+
]) {
|
|
638
212
|
assert.equal(validateTerminalBodyValue(body, travelSchema), false, `value validation rejects: ${label}`);
|
|
639
213
|
assert.equal(parseBenchmarkTerminal(`<done>${JSON.stringify(body)}</done>`, travelConfig).ok, false, `terminal gate rejects: ${label}`);
|
|
640
214
|
}
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
215
|
+
// Envelope byte limit still fails closed for a schema-valid but oversized body.
|
|
216
|
+
assert.equal(parseBenchmarkTerminal(`<done>${' '.repeat(2000)}${JSON.stringify({ ok: true })}</done>`, projection.terminal).ok, false, 'schema-valid body over max_bytes still rejects');
|
|
217
|
+
// Single-source projection: the outline is deterministic and appears verbatim in
|
|
218
|
+
// both the system section and the one terminal correction.
|
|
644
219
|
const outline = terminalSchemaOutline(travelSchema);
|
|
645
220
|
assert.equal(outline, terminalSchemaOutline(JSON.parse(JSON.stringify(travelSchema))), 'outline is deterministic');
|
|
646
221
|
assert.equal(outline, 'object{people_number:integer,start_city:string,target_city:string,itinerary:array<object{day:integer,activities:array<object{type:string,start_time:string,cost:number,transports:array<object{start:string,mode:string}>}>}>}', 'outline renders the exact structural contract');
|
|
647
222
|
assert.match(outline, /object\{/, 'outline renders object nodes');
|
|
648
223
|
assert.ok(!outline.includes('optional') && !outline.includes('?'), 'all keys required render without optional markers');
|
|
649
|
-
}
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
data: {
|
|
653
|
-
turn
|
|
654
|
-
}
|
|
655
|
-
};
|
|
224
|
+
}
|
|
225
|
+
function turnStart(turn = 1) {
|
|
226
|
+
return { type: 'turn/start', data: { turn } };
|
|
656
227
|
}
|
|
657
228
|
function turnEnd(turn = 1) {
|
|
658
|
-
return {
|
|
659
|
-
type: 'turn/end',
|
|
660
|
-
data: {
|
|
661
|
-
turn,
|
|
662
|
-
reason: {
|
|
663
|
-
kind: 'completed'
|
|
664
|
-
}
|
|
665
|
-
}
|
|
666
|
-
};
|
|
229
|
+
return { type: 'turn/end', data: { turn, reason: { kind: 'completed' } } };
|
|
667
230
|
}
|
|
668
231
|
function toolCall(callId = 'call-1', options = {}) {
|
|
669
232
|
const { turn = 1, step = 1, action = 'call', tool = 'lookup' } = options;
|
|
@@ -674,12 +237,8 @@ function toolCall(callId = 'call-1', options = {}) {
|
|
|
674
237
|
step,
|
|
675
238
|
callId,
|
|
676
239
|
name: projection.toolName,
|
|
677
|
-
arguments: JSON.stringify({
|
|
678
|
-
|
|
679
|
-
tool,
|
|
680
|
-
arguments: {}
|
|
681
|
-
})
|
|
682
|
-
}
|
|
240
|
+
arguments: JSON.stringify({ action, tool, arguments: {} }),
|
|
241
|
+
},
|
|
683
242
|
};
|
|
684
243
|
}
|
|
685
244
|
function toolResult(callId = 'call-1', options = {}) {
|
|
@@ -690,774 +249,371 @@ function toolResult(callId = 'call-1', options = {}) {
|
|
|
690
249
|
turn,
|
|
691
250
|
step,
|
|
692
251
|
message: {
|
|
693
|
-
source: {
|
|
694
|
-
|
|
695
|
-
callId
|
|
696
|
-
},
|
|
697
|
-
content: [
|
|
698
|
-
{
|
|
252
|
+
source: { kind: 'tool', callId },
|
|
253
|
+
content: [{
|
|
699
254
|
type: 'tool-result',
|
|
700
255
|
toolCallId: callId,
|
|
701
256
|
isError,
|
|
702
|
-
content: [
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
ok: true,
|
|
707
|
-
outcome
|
|
708
|
-
} : ok ? {
|
|
709
|
-
ok: true,
|
|
710
|
-
result: {}
|
|
711
|
-
} : {
|
|
712
|
-
ok: false,
|
|
713
|
-
error
|
|
714
|
-
}))
|
|
715
|
-
}
|
|
716
|
-
]
|
|
717
|
-
}
|
|
718
|
-
]
|
|
719
|
-
}
|
|
720
|
-
}
|
|
257
|
+
content: [{ type: 'text', text: JSON.stringify(payload ?? (outcome ? { ok: true, outcome } : ok ? { ok: true, result: {} } : { ok: false, error })) }],
|
|
258
|
+
}],
|
|
259
|
+
},
|
|
260
|
+
},
|
|
721
261
|
};
|
|
722
262
|
}
|
|
723
263
|
{
|
|
724
264
|
const state = createBenchmarkAgentConformance(projection);
|
|
725
265
|
state.observe(turnStart());
|
|
726
266
|
state.observe(toolCall());
|
|
727
|
-
state.observe(toolResult('call-1', {
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
}
|
|
734
|
-
}));
|
|
735
|
-
assert.deepEqual(state.stopping(1), {
|
|
736
|
-
kind: 'steer',
|
|
737
|
-
mode: 'terminal'
|
|
738
|
-
}, 'domain outcome alone permits tagged terminal');
|
|
739
|
-
state.observe(assistant('<done>{"status":"miss"}</done>', {
|
|
740
|
-
step: 2
|
|
741
|
-
}));
|
|
742
|
-
assert.deepEqual(state.stopping(1), {
|
|
743
|
-
kind: 'accept'
|
|
744
|
-
}, 'domain-only tagged terminal is accepted');
|
|
745
|
-
}{
|
|
267
|
+
state.observe(toolResult('call-1', { outcome: { schema_version: 'gotry_benchmark_tool_result_v1', status: 'miss', code: 'NOT_FOUND', recovery: 'revise_arguments' } }));
|
|
268
|
+
assert.deepEqual(state.stopping(1), { kind: 'steer', mode: 'terminal' }, 'domain outcome alone permits tagged terminal');
|
|
269
|
+
state.observe(assistant('<done>{"status":"miss"}</done>', { step: 2 }));
|
|
270
|
+
assert.deepEqual(state.stopping(1), { kind: 'accept' }, 'domain-only tagged terminal is accepted');
|
|
271
|
+
}
|
|
272
|
+
{
|
|
746
273
|
const state = createBenchmarkAgentConformance(projection);
|
|
747
274
|
state.observe(turnStart());
|
|
748
275
|
state.observe(toolCall());
|
|
749
|
-
state.observe(toolResult('call-1', {
|
|
750
|
-
|
|
751
|
-
schema_version: 'gotry_benchmark_tool_result_v1',
|
|
752
|
-
status: 'miss',
|
|
753
|
-
code: 'NOT_FOUND',
|
|
754
|
-
recovery: 'revise_arguments'
|
|
755
|
-
}
|
|
756
|
-
}));
|
|
757
|
-
state.observe(toolCall('call-2', {
|
|
758
|
-
step: 2
|
|
759
|
-
}));
|
|
276
|
+
state.observe(toolResult('call-1', { outcome: { schema_version: 'gotry_benchmark_tool_result_v1', status: 'miss', code: 'NOT_FOUND', recovery: 'revise_arguments' } }));
|
|
277
|
+
state.observe(toolCall('call-2', { step: 2 }));
|
|
760
278
|
state.observe(toolResult('call-2'));
|
|
761
|
-
state.observe(assistant('<done>{"status":"succeeded"}</done>', {
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
kind: 'accept'
|
|
766
|
-
}, 'domain outcome followed by concrete result accepts');
|
|
767
|
-
}{
|
|
279
|
+
state.observe(assistant('<done>{"status":"succeeded"}</done>', { step: 3 }));
|
|
280
|
+
assert.deepEqual(state.stopping(1), { kind: 'accept' }, 'domain outcome followed by concrete result accepts');
|
|
281
|
+
}
|
|
282
|
+
{
|
|
768
283
|
const state = createBenchmarkAgentConformance(projection);
|
|
769
284
|
state.observe(turnStart());
|
|
770
285
|
state.observe(toolCall());
|
|
771
|
-
state.observe(toolResult('call-1', {
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
}
|
|
778
|
-
}));
|
|
779
|
-
state.observe(toolCall('call-2', {
|
|
780
|
-
step: 2
|
|
781
|
-
}));
|
|
782
|
-
state.observe(toolResult('call-2', {
|
|
783
|
-
ok: false,
|
|
784
|
-
error: 'runner_failed'
|
|
785
|
-
}));
|
|
786
|
-
assert.deepEqual(state.stopping(1), {
|
|
787
|
-
kind: 'reject',
|
|
788
|
-
code: BENCHMARK_BRIDGE_RUNNER_FAILED
|
|
789
|
-
}, 'infra failure overrides domain-only outcome');
|
|
790
|
-
}{
|
|
286
|
+
state.observe(toolResult('call-1', { outcome: { schema_version: 'gotry_benchmark_tool_result_v1', status: 'miss', code: 'NOT_FOUND', recovery: 'revise_arguments' } }));
|
|
287
|
+
state.observe(toolCall('call-2', { step: 2 }));
|
|
288
|
+
state.observe(toolResult('call-2', { ok: false, error: 'runner_failed' }));
|
|
289
|
+
assert.deepEqual(state.stopping(1), { kind: 'reject', code: BENCHMARK_BRIDGE_RUNNER_FAILED }, 'infra failure overrides domain-only outcome');
|
|
290
|
+
}
|
|
291
|
+
{
|
|
791
292
|
const state = createBenchmarkAgentConformance(projection);
|
|
792
293
|
state.observe(turnStart());
|
|
793
294
|
state.observe(toolCall());
|
|
794
|
-
state.observe(toolResult('call-1', {
|
|
795
|
-
payload: {
|
|
796
|
-
ok: true,
|
|
797
|
-
result: {},
|
|
798
|
-
outcome: {
|
|
799
|
-
status: 'miss'
|
|
800
|
-
}
|
|
801
|
-
}
|
|
802
|
-
}));
|
|
295
|
+
state.observe(toolResult('call-1', { payload: { ok: true, result: {}, outcome: { status: 'miss' } } }));
|
|
803
296
|
state.observe(assistant('<done>{"status":"succeeded"}</done>'));
|
|
804
|
-
assert.deepEqual(state.stopping(1), {
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
}, 'ambiguous ok wrapper with result and outcome fails closed');
|
|
808
|
-
}{
|
|
297
|
+
assert.deepEqual(state.stopping(1), { kind: 'reject', code: BENCHMARK_BRIDGE_CALL_FAILED }, 'ambiguous ok wrapper with result and outcome fails closed');
|
|
298
|
+
}
|
|
299
|
+
{
|
|
809
300
|
const state = createBenchmarkAgentConformance(projection);
|
|
810
301
|
state.observe(turnStart());
|
|
811
302
|
state.observe(toolCall());
|
|
812
|
-
state.observe(toolResult('call-1', {
|
|
813
|
-
payload: {
|
|
814
|
-
ok: true,
|
|
815
|
-
outcome: {}
|
|
816
|
-
}
|
|
817
|
-
}));
|
|
303
|
+
state.observe(toolResult('call-1', { payload: { ok: true, outcome: {} } }));
|
|
818
304
|
state.observe(assistant('<done>{"status":"miss"}</done>'));
|
|
819
|
-
assert.deepEqual(state.stopping(1), {
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
}, 'malformed inner domain outcome fails closed');
|
|
823
|
-
}function assistant(text, options = {}) {
|
|
305
|
+
assert.deepEqual(state.stopping(1), { kind: 'reject', code: BENCHMARK_BRIDGE_CALL_FAILED }, 'malformed inner domain outcome fails closed');
|
|
306
|
+
}
|
|
307
|
+
function assistant(text, options = {}) {
|
|
824
308
|
const { turn = 1, step = 2, interrupted = false } = options;
|
|
825
309
|
return {
|
|
826
310
|
type: 'assistant/message',
|
|
827
311
|
data: {
|
|
828
312
|
turn,
|
|
829
313
|
step,
|
|
830
|
-
message: {
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
type: 'text',
|
|
834
|
-
text
|
|
835
|
-
}
|
|
836
|
-
]
|
|
837
|
-
},
|
|
838
|
-
...interrupted ? {
|
|
839
|
-
interrupted: true
|
|
840
|
-
} : {}
|
|
841
|
-
}
|
|
314
|
+
message: { content: [{ type: 'text', text }] },
|
|
315
|
+
...(interrupted ? { interrupted: true } : {}),
|
|
316
|
+
},
|
|
842
317
|
};
|
|
843
318
|
}
|
|
844
319
|
{
|
|
845
320
|
const state = createBenchmarkAgentConformance(projection);
|
|
846
321
|
state.observe(turnStart());
|
|
847
|
-
state.observe(assistant('I would run the CLI.', {
|
|
848
|
-
|
|
849
|
-
}));
|
|
850
|
-
assert.deepEqual(state.stopping(1), {
|
|
851
|
-
kind: 'steer',
|
|
852
|
-
mode: 'call'
|
|
853
|
-
}, 'no-call first stop gets one correction');
|
|
322
|
+
state.observe(assistant('I would run the CLI.', { step: 1 }));
|
|
323
|
+
assert.deepEqual(state.stopping(1), { kind: 'steer', mode: 'call' }, 'no-call first stop gets one correction');
|
|
854
324
|
assert.equal(state.guardBridgeExecution(), undefined, 'call correction still permits the first real bridge dispatch');
|
|
855
|
-
state.observe(toolCall('call-a', {
|
|
856
|
-
|
|
857
|
-
}));
|
|
858
|
-
state.
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
state.observe(assistant('<done>{"status":"succeeded"}</done>', {
|
|
862
|
-
step: 3
|
|
863
|
-
}));
|
|
864
|
-
assert.deepEqual(state.stopping(1), {
|
|
865
|
-
kind: 'accept'
|
|
866
|
-
}, 'call correction may converge to one successful terminal');
|
|
867
|
-
}{
|
|
325
|
+
state.observe(toolCall('call-a', { step: 2 }));
|
|
326
|
+
state.observe(toolResult('call-a', { step: 2 }));
|
|
327
|
+
state.observe(assistant('<done>{"status":"succeeded"}</done>', { step: 3 }));
|
|
328
|
+
assert.deepEqual(state.stopping(1), { kind: 'accept' }, 'call correction may converge to one successful terminal');
|
|
329
|
+
}
|
|
330
|
+
{
|
|
868
331
|
const state = createBenchmarkAgentConformance(projection);
|
|
869
332
|
state.observe(turnStart());
|
|
870
|
-
state.observe(assistant('<done>{"status":"succeeded"}</done>', {
|
|
871
|
-
|
|
872
|
-
}));
|
|
873
|
-
assert.deepEqual(state.stopping(1), {
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
}, 'valid terminal without a call still needs a call');
|
|
877
|
-
state.observe(assistant('<done>{"status":"succeeded"}</done>', {
|
|
878
|
-
step: 2
|
|
879
|
-
}));
|
|
880
|
-
assert.deepEqual(state.stopping(1), {
|
|
881
|
-
kind: 'reject',
|
|
882
|
-
code: BENCHMARK_BRIDGE_CALL_REQUIRED
|
|
883
|
-
});
|
|
884
|
-
}{
|
|
333
|
+
state.observe(assistant('<done>{"status":"succeeded"}</done>', { step: 1 }));
|
|
334
|
+
assert.deepEqual(state.stopping(1), { kind: 'steer', mode: 'call' }, 'valid terminal without a call still needs a call');
|
|
335
|
+
state.observe(assistant('<done>{"status":"succeeded"}</done>', { step: 2 }));
|
|
336
|
+
assert.deepEqual(state.stopping(1), { kind: 'reject', code: BENCHMARK_BRIDGE_CALL_REQUIRED });
|
|
337
|
+
}
|
|
338
|
+
{
|
|
885
339
|
const state = createBenchmarkAgentConformance(projection);
|
|
886
340
|
state.observe(turnStart());
|
|
887
341
|
state.observe(toolCall());
|
|
888
342
|
state.observe(toolResult());
|
|
889
343
|
state.observe(assistant('bad terminal'));
|
|
890
|
-
assert.deepEqual(state.stopping(1), {
|
|
891
|
-
kind: 'steer',
|
|
892
|
-
mode: 'terminal'
|
|
893
|
-
}, 'bad terminal gets one format-only correction');
|
|
344
|
+
assert.deepEqual(state.stopping(1), { kind: 'steer', mode: 'terminal' }, 'bad terminal gets one format-only correction');
|
|
894
345
|
assert.equal(state.guardBridgeExecution(), BENCHMARK_BRIDGE_RETRY_CALL_NOT_ALLOWED);
|
|
895
|
-
state.observe(assistant('<done>{"status":"succeeded"}</done>', {
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
kind: 'accept'
|
|
900
|
-
}, 'format-only correction can reuse the successful result');
|
|
901
|
-
}{
|
|
346
|
+
state.observe(assistant('<done>{"status":"succeeded"}</done>', { step: 3 }));
|
|
347
|
+
assert.deepEqual(state.stopping(1), { kind: 'accept' }, 'format-only correction can reuse the successful result');
|
|
348
|
+
}
|
|
349
|
+
{
|
|
902
350
|
const state = createBenchmarkAgentConformance(projection);
|
|
903
351
|
state.observe(turnStart());
|
|
904
352
|
state.observe(toolCall());
|
|
905
353
|
state.observe(toolResult());
|
|
906
354
|
state.observe(assistant('bad terminal'));
|
|
907
|
-
assert.deepEqual(state.stopping(1), {
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
step: 3
|
|
913
|
-
}));
|
|
914
|
-
assert.deepEqual(state.stopping(1), {
|
|
915
|
-
kind: 'reject',
|
|
916
|
-
code: BENCHMARK_BRIDGE_RETRY_CALL_NOT_ALLOWED
|
|
917
|
-
});
|
|
918
|
-
}{
|
|
355
|
+
assert.deepEqual(state.stopping(1), { kind: 'steer', mode: 'terminal' });
|
|
356
|
+
state.observe(toolCall('call-2', { step: 3 }));
|
|
357
|
+
assert.deepEqual(state.stopping(1), { kind: 'reject', code: BENCHMARK_BRIDGE_RETRY_CALL_NOT_ALLOWED });
|
|
358
|
+
}
|
|
359
|
+
{
|
|
919
360
|
const state = createBenchmarkAgentConformance(projection);
|
|
920
361
|
state.observe(turnStart());
|
|
921
362
|
state.observe(toolCall());
|
|
922
363
|
state.observe(toolResult());
|
|
923
364
|
state.observe(assistant('bad terminal'));
|
|
924
|
-
assert.deepEqual(state.stopping(1), {
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
step: 3
|
|
930
|
-
}));
|
|
931
|
-
assert.deepEqual(state.stopping(1), {
|
|
932
|
-
kind: 'reject',
|
|
933
|
-
code: BENCHMARK_TERMINAL_INVALID
|
|
934
|
-
});
|
|
935
|
-
}{
|
|
365
|
+
assert.deepEqual(state.stopping(1), { kind: 'steer', mode: 'terminal' });
|
|
366
|
+
state.observe(assistant('still bad', { step: 3 }));
|
|
367
|
+
assert.deepEqual(state.stopping(1), { kind: 'reject', code: BENCHMARK_TERMINAL_INVALID });
|
|
368
|
+
}
|
|
369
|
+
{
|
|
936
370
|
const state = createBenchmarkAgentConformance(projection);
|
|
937
371
|
state.observe(turnStart());
|
|
938
372
|
state.observe(toolCall());
|
|
939
|
-
state.observe(toolResult('call-1', {
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
kind: 'reject',
|
|
944
|
-
code: BENCHMARK_BRIDGE_RUNNER_FAILED
|
|
945
|
-
}, 'structured runner failure is not retried');
|
|
946
|
-
}{
|
|
373
|
+
state.observe(toolResult('call-1', { ok: false }));
|
|
374
|
+
assert.deepEqual(state.stopping(1), { kind: 'reject', code: BENCHMARK_BRIDGE_RUNNER_FAILED }, 'structured runner failure is not retried');
|
|
375
|
+
}
|
|
376
|
+
{
|
|
947
377
|
const state = createBenchmarkAgentConformance(projection);
|
|
948
378
|
state.observe(turnStart());
|
|
949
379
|
state.observe(toolCall());
|
|
950
|
-
state.observe(toolResult('call-1', {
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
assert.deepEqual(state.stopping(1), {
|
|
955
|
-
kind: 'reject',
|
|
956
|
-
code: BENCHMARK_BRIDGE_OUTPUT_TRUNCATED
|
|
957
|
-
}, 'structured runner truncation has a distinct reason');
|
|
958
|
-
}{
|
|
380
|
+
state.observe(toolResult('call-1', { ok: false, error: 'output_truncated' }));
|
|
381
|
+
assert.deepEqual(state.stopping(1), { kind: 'reject', code: BENCHMARK_BRIDGE_OUTPUT_TRUNCATED }, 'structured runner truncation has a distinct reason');
|
|
382
|
+
}
|
|
383
|
+
{
|
|
959
384
|
const state = createBenchmarkAgentConformance(projection);
|
|
960
385
|
state.observe(turnStart());
|
|
961
|
-
state.observe(toolCall('failed', {
|
|
962
|
-
|
|
963
|
-
}));
|
|
964
|
-
state.observe(toolResult('
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
step: 2
|
|
970
|
-
}));
|
|
971
|
-
state.observe(toolResult('successful', {
|
|
972
|
-
step: 2
|
|
973
|
-
}));
|
|
974
|
-
state.observe(assistant('<done>{"status":"succeeded"}</done>', {
|
|
975
|
-
step: 3
|
|
976
|
-
}));
|
|
977
|
-
assert.deepEqual(state.stopping(1), {
|
|
978
|
-
kind: 'accept'
|
|
979
|
-
}, 'a model-owned later success can recover from an earlier failed call without a conformance retry');
|
|
980
|
-
}{
|
|
386
|
+
state.observe(toolCall('failed', { step: 1 }));
|
|
387
|
+
state.observe(toolResult('failed', { step: 1, ok: false }));
|
|
388
|
+
state.observe(toolCall('successful', { step: 2 }));
|
|
389
|
+
state.observe(toolResult('successful', { step: 2 }));
|
|
390
|
+
state.observe(assistant('<done>{"status":"succeeded"}</done>', { step: 3 }));
|
|
391
|
+
assert.deepEqual(state.stopping(1), { kind: 'accept' }, 'a model-owned later success can recover from an earlier failed call without a conformance retry');
|
|
392
|
+
}
|
|
393
|
+
{
|
|
981
394
|
const state = createBenchmarkAgentConformance(projection);
|
|
982
395
|
state.observe(turnStart());
|
|
983
|
-
state.observe(toolCall('successful', {
|
|
984
|
-
|
|
985
|
-
}));
|
|
986
|
-
state.observe(toolResult('
|
|
987
|
-
|
|
988
|
-
})
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
}));
|
|
992
|
-
state.observe(toolResult('failed', {
|
|
993
|
-
step: 2,
|
|
994
|
-
ok: false
|
|
995
|
-
}));
|
|
996
|
-
state.observe(assistant('<done>{"status":"succeeded"}</done>', {
|
|
997
|
-
step: 3
|
|
998
|
-
}));
|
|
999
|
-
assert.deepEqual(state.stopping(1), {
|
|
1000
|
-
kind: 'accept'
|
|
1001
|
-
}, 'a later failed optional call does not erase an already paired successful result');
|
|
1002
|
-
}{
|
|
396
|
+
state.observe(toolCall('successful', { step: 1 }));
|
|
397
|
+
state.observe(toolResult('successful', { step: 1 }));
|
|
398
|
+
state.observe(toolCall('failed', { step: 2 }));
|
|
399
|
+
state.observe(toolResult('failed', { step: 2, ok: false }));
|
|
400
|
+
state.observe(assistant('<done>{"status":"succeeded"}</done>', { step: 3 }));
|
|
401
|
+
assert.deepEqual(state.stopping(1), { kind: 'accept' }, 'a later failed optional call does not erase an already paired successful result');
|
|
402
|
+
}
|
|
403
|
+
{
|
|
1003
404
|
const state = createBenchmarkAgentConformance(projection);
|
|
1004
405
|
state.observe(turnStart());
|
|
1005
|
-
state.observe(toolCall('successful', {
|
|
1006
|
-
|
|
1007
|
-
}));
|
|
1008
|
-
state.observe(
|
|
1009
|
-
|
|
1010
|
-
})
|
|
1011
|
-
state.observe(assistant('<done>{"status":"succeeded"}</done>', {
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
step: 3
|
|
1016
|
-
}));
|
|
1017
|
-
state.observe(toolResult('domain', {
|
|
1018
|
-
step: 3,
|
|
1019
|
-
outcome: {
|
|
1020
|
-
schema_version: 'gotry_benchmark_tool_result_v1',
|
|
1021
|
-
status: 'miss',
|
|
1022
|
-
code: 'NOT_FOUND',
|
|
1023
|
-
recovery: 'revise_arguments'
|
|
1024
|
-
}
|
|
1025
|
-
}));
|
|
1026
|
-
assert.deepEqual(state.stopping(1), {
|
|
1027
|
-
kind: 'steer',
|
|
1028
|
-
mode: 'terminal'
|
|
1029
|
-
}, 'a terminal before the latest domain response is stale');
|
|
1030
|
-
state.observe(assistant('<done>{"status":"succeeded"}</done>', {
|
|
1031
|
-
step: 4
|
|
1032
|
-
}));
|
|
1033
|
-
assert.deepEqual(state.stopping(1), {
|
|
1034
|
-
kind: 'accept'
|
|
1035
|
-
}, 'a fresh terminal after the latest domain response may reuse the earlier concrete result');
|
|
1036
|
-
}{
|
|
406
|
+
state.observe(toolCall('successful', { step: 1 }));
|
|
407
|
+
state.observe(toolResult('successful', { step: 1 }));
|
|
408
|
+
state.observe(assistant('<done>{"status":"succeeded"}</done>', { step: 2 }));
|
|
409
|
+
state.observe(toolCall('domain', { step: 3 }));
|
|
410
|
+
state.observe(toolResult('domain', { step: 3, outcome: { schema_version: 'gotry_benchmark_tool_result_v1', status: 'miss', code: 'NOT_FOUND', recovery: 'revise_arguments' } }));
|
|
411
|
+
assert.deepEqual(state.stopping(1), { kind: 'steer', mode: 'terminal' }, 'a terminal before the latest domain response is stale');
|
|
412
|
+
state.observe(assistant('<done>{"status":"succeeded"}</done>', { step: 4 }));
|
|
413
|
+
assert.deepEqual(state.stopping(1), { kind: 'accept' }, 'a fresh terminal after the latest domain response may reuse the earlier concrete result');
|
|
414
|
+
}
|
|
415
|
+
{
|
|
1037
416
|
const state = createBenchmarkAgentConformance(projection);
|
|
1038
417
|
state.observe(turnStart());
|
|
1039
|
-
state.observe(toolCall('successful', {
|
|
1040
|
-
|
|
1041
|
-
}));
|
|
1042
|
-
state.observe(
|
|
1043
|
-
|
|
1044
|
-
})
|
|
1045
|
-
state.observe(assistant('<done>{"status":"succeeded"}</done>', {
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
step: 3
|
|
1050
|
-
}));
|
|
1051
|
-
state.observe(toolResult('failed', {
|
|
1052
|
-
step: 3,
|
|
1053
|
-
ok: false
|
|
1054
|
-
}));
|
|
1055
|
-
assert.deepEqual(state.stopping(1), {
|
|
1056
|
-
kind: 'steer',
|
|
1057
|
-
mode: 'terminal'
|
|
1058
|
-
}, 'a terminal before a later optional failure is stale');
|
|
1059
|
-
state.observe(assistant('<done>{"status":"succeeded"}</done>', {
|
|
1060
|
-
step: 4
|
|
1061
|
-
}));
|
|
1062
|
-
assert.deepEqual(state.stopping(1), {
|
|
1063
|
-
kind: 'accept'
|
|
1064
|
-
}, 'a fresh terminal may still converge after an optional failure when a concrete result exists');
|
|
1065
|
-
}{
|
|
418
|
+
state.observe(toolCall('successful', { step: 1 }));
|
|
419
|
+
state.observe(toolResult('successful', { step: 1 }));
|
|
420
|
+
state.observe(assistant('<done>{"status":"succeeded"}</done>', { step: 2 }));
|
|
421
|
+
state.observe(toolCall('failed', { step: 3 }));
|
|
422
|
+
state.observe(toolResult('failed', { step: 3, ok: false }));
|
|
423
|
+
assert.deepEqual(state.stopping(1), { kind: 'steer', mode: 'terminal' }, 'a terminal before a later optional failure is stale');
|
|
424
|
+
state.observe(assistant('<done>{"status":"succeeded"}</done>', { step: 4 }));
|
|
425
|
+
assert.deepEqual(state.stopping(1), { kind: 'accept' }, 'a fresh terminal may still converge after an optional failure when a concrete result exists');
|
|
426
|
+
}
|
|
427
|
+
{
|
|
1066
428
|
const state = createBenchmarkAgentConformance(projection);
|
|
1067
429
|
state.observe(turnStart());
|
|
1068
|
-
state.observe(toolCall('discovery', {
|
|
1069
|
-
action: 'tools'
|
|
1070
|
-
}));
|
|
430
|
+
state.observe(toolCall('discovery', { action: 'tools' }));
|
|
1071
431
|
state.observe(toolResult('discovery'));
|
|
1072
432
|
state.observe(assistant('<done>{"status":"succeeded"}</done>'));
|
|
1073
|
-
assert.deepEqual(state.stopping(1), {
|
|
1074
|
-
kind: 'steer',
|
|
1075
|
-
mode: 'call'
|
|
1076
|
-
}, 'action tools does not satisfy the call gate');
|
|
433
|
+
assert.deepEqual(state.stopping(1), { kind: 'steer', mode: 'call' }, 'action tools does not satisfy the call gate');
|
|
1077
434
|
state.observe(turnEnd());
|
|
1078
|
-
assert.deepEqual(state.stopping(1), {
|
|
1079
|
-
kind: 'reject',
|
|
1080
|
-
code: BENCHMARK_CONFORMANCE_STATE_UNAVAILABLE
|
|
1081
|
-
});
|
|
435
|
+
assert.deepEqual(state.stopping(1), { kind: 'reject', code: BENCHMARK_CONFORMANCE_STATE_UNAVAILABLE });
|
|
1082
436
|
state.observe(turnStart(2));
|
|
1083
|
-
assert.deepEqual(state.stopping(1), {
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
}, 'turn state cannot leak across turns');
|
|
1087
|
-
}{
|
|
437
|
+
assert.deepEqual(state.stopping(1), { kind: 'reject', code: BENCHMARK_CONFORMANCE_STATE_UNAVAILABLE }, 'turn state cannot leak across turns');
|
|
438
|
+
}
|
|
439
|
+
{
|
|
1088
440
|
const rootListeners = new Map();
|
|
1089
441
|
const scopedListeners = new Map();
|
|
1090
442
|
const guards = [];
|
|
1091
443
|
const steers = [];
|
|
1092
444
|
const runtimeWrites = [];
|
|
1093
|
-
const runEffect = (action)=>{
|
|
445
|
+
const runEffect = (action) => {
|
|
1094
446
|
const disposers = [];
|
|
1095
447
|
const value = action();
|
|
1096
448
|
if (value && typeof value.next === 'function') {
|
|
1097
449
|
let item = value.next();
|
|
1098
|
-
while(!item.done){
|
|
1099
|
-
if (typeof item.value === 'function')
|
|
450
|
+
while (!item.done) {
|
|
451
|
+
if (typeof item.value === 'function')
|
|
452
|
+
disposers.push(item.value);
|
|
1100
453
|
item = value.next();
|
|
1101
454
|
}
|
|
1102
455
|
}
|
|
1103
|
-
return ()=>{
|
|
1104
|
-
|
|
1105
|
-
};
|
|
456
|
+
return () => { for (const dispose of disposers.reverse())
|
|
457
|
+
dispose(); };
|
|
1106
458
|
};
|
|
1107
|
-
const add = (target, name, listener)=>{
|
|
459
|
+
const add = (target, name, listener) => {
|
|
1108
460
|
const list = target.get(name) ?? [];
|
|
1109
461
|
list.push(listener);
|
|
1110
462
|
target.set(name, list);
|
|
1111
|
-
return ()=>target.set(name, list.filter(
|
|
463
|
+
return () => target.set(name, list.filter(candidate => candidate !== listener));
|
|
1112
464
|
};
|
|
1113
465
|
const session = {};
|
|
1114
466
|
const agent = {
|
|
1115
467
|
session,
|
|
1116
|
-
steer (message)
|
|
1117
|
-
steers.push(message);
|
|
1118
|
-
},
|
|
468
|
+
steer(message) { steers.push(message); },
|
|
1119
469
|
ctx: {
|
|
1120
|
-
tools: {
|
|
1121
|
-
guard (check) {
|
|
1122
|
-
guards.push(check);
|
|
1123
|
-
return ()=>{};
|
|
1124
|
-
}
|
|
1125
|
-
},
|
|
470
|
+
tools: { guard(check) { guards.push(check); return () => { }; } },
|
|
1126
471
|
effect: runEffect,
|
|
1127
|
-
on (name, listener)
|
|
1128
|
-
|
|
1129
|
-
}
|
|
1130
|
-
}
|
|
472
|
+
on(name, listener) { return add(scopedListeners, name, listener); },
|
|
473
|
+
},
|
|
1131
474
|
};
|
|
1132
475
|
const ctx = {
|
|
1133
|
-
on (name, listener)
|
|
1134
|
-
return add(rootListeners, name, listener);
|
|
1135
|
-
}
|
|
476
|
+
on(name, listener) { return add(rootListeners, name, listener); },
|
|
1136
477
|
};
|
|
1137
|
-
installBenchmarkAgentConformance(ctx, projection,
|
|
1138
|
-
rootListeners.get('agent/created')[0]({
|
|
1139
|
-
agent
|
|
1140
|
-
});
|
|
478
|
+
installBenchmarkAgentConformance(ctx, projection, code => runtimeWrites.push(code));
|
|
479
|
+
rootListeners.get('agent/created')[0]({ agent });
|
|
1141
480
|
rootListeners.get('session/event')[0](session, turnStart());
|
|
1142
|
-
rootListeners.get('session/event')[0](session, {
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
reason: {
|
|
1147
|
-
kind: 'error',
|
|
1148
|
-
error: {
|
|
1149
|
-
code: 'RATE_LIMIT'
|
|
1150
|
-
}
|
|
1151
|
-
}
|
|
1152
|
-
}
|
|
1153
|
-
});
|
|
1154
|
-
rootListeners.get('session/event')[0](session, {
|
|
1155
|
-
type: 'agent/request-error',
|
|
1156
|
-
data: {
|
|
1157
|
-
turn: 1,
|
|
1158
|
-
error: {
|
|
1159
|
-
code: 'SERVER'
|
|
1160
|
-
}
|
|
1161
|
-
}
|
|
1162
|
-
});
|
|
1163
|
-
rootListeners.get('session/event')[0](session, assistant('prose only', {
|
|
1164
|
-
step: 1
|
|
1165
|
-
}));
|
|
1166
|
-
rootListeners.get('agent/turn-stopping')[0]({
|
|
1167
|
-
agent,
|
|
1168
|
-
turn: 1
|
|
1169
|
-
});
|
|
481
|
+
rootListeners.get('session/event')[0](session, { type: 'llm/retry', data: { turn: 1, reason: { kind: 'error', error: { code: 'RATE_LIMIT' } } } });
|
|
482
|
+
rootListeners.get('session/event')[0](session, { type: 'agent/request-error', data: { turn: 1, error: { code: 'SERVER' } } });
|
|
483
|
+
rootListeners.get('session/event')[0](session, assistant('prose only', { step: 1 }));
|
|
484
|
+
rootListeners.get('agent/turn-stopping')[0]({ agent, turn: 1 });
|
|
1170
485
|
assert.equal(steers.length, 1, 'runtime wiring steers exactly once at the stop boundary');
|
|
1171
486
|
assert.equal(steers[0].role, 'user');
|
|
1172
487
|
assert.equal(Object.isFrozen(steers[0]), true, 'correction uses the official immutable DSH user message');
|
|
1173
488
|
assert.equal(Object.isFrozen(steers[0].content), true, 'correction content is deeply frozen');
|
|
1174
|
-
assert.equal(guards[0]({
|
|
1175
|
-
|
|
1176
|
-
}), undefined, 'call correction leaves bridge execution available');
|
|
1177
|
-
const assembled = await scopedListeners.get('system-prompt/assemble')[0]({}, {}, async ()=>({
|
|
1178
|
-
sections: [],
|
|
1179
|
-
tools: []
|
|
1180
|
-
}));
|
|
489
|
+
assert.equal(guards[0]({ name: projection.toolName }), undefined, 'call correction leaves bridge execution available');
|
|
490
|
+
const assembled = await scopedListeners.get('system-prompt/assemble')[0]({}, {}, async () => ({ sections: [], tools: [] }));
|
|
1181
491
|
assert.match(assembled.sections[0].text, /agent_env\.cli/);
|
|
1182
492
|
assert.match(assembled.sections[0].text, /\"action\":\"call\"/);
|
|
1183
493
|
assert.match(assembled.sections[0].text, /<done>/);
|
|
1184
494
|
assert.equal(assembled.sections[0].text.includes('/tmp/'), false);
|
|
495
|
+
// Round 12(#215):system section 与 terminal 纠正投影同一份结构合同(单一来源)。
|
|
1185
496
|
const round12Outline = terminalSchemaOutline(OK_BODY_SCHEMA);
|
|
1186
497
|
assert.ok(assembled.sections[0].text.includes(`matching exactly ${round12Outline}`), 'system section projects the terminal body schema outline');
|
|
498
|
+
// Intermediate retry/request errors never write; only final completed observes recovery.
|
|
1187
499
|
rootListeners.get('session/event')[0](session, turnEnd());
|
|
1188
500
|
assert.deepEqual(runtimeWrites, []);
|
|
501
|
+
// A final structured model error writes exactly once, despite duplicate end/dispose.
|
|
1189
502
|
rootListeners.get('session/event')[0](session, turnStart(2));
|
|
1190
|
-
rootListeners.get('session/event')[0](session, {
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
turn: 2,
|
|
1194
|
-
reason: {
|
|
1195
|
-
kind: 'error',
|
|
1196
|
-
error: {
|
|
1197
|
-
code: 'SERVER',
|
|
1198
|
-
message: 'sentinel'
|
|
1199
|
-
}
|
|
1200
|
-
}
|
|
1201
|
-
}
|
|
1202
|
-
});
|
|
1203
|
-
rootListeners.get('session/event')[0](session, {
|
|
1204
|
-
type: 'turn/end',
|
|
1205
|
-
data: {
|
|
1206
|
-
turn: 2,
|
|
1207
|
-
reason: {
|
|
1208
|
-
kind: 'error',
|
|
1209
|
-
error: {
|
|
1210
|
-
code: 'UNKNOWN_MODEL'
|
|
1211
|
-
}
|
|
1212
|
-
}
|
|
1213
|
-
}
|
|
1214
|
-
});
|
|
1215
|
-
assert.deepEqual(runtimeWrites, [
|
|
1216
|
-
'child_model_server'
|
|
1217
|
-
]);
|
|
503
|
+
rootListeners.get('session/event')[0](session, { type: 'turn/end', data: { turn: 2, reason: { kind: 'error', error: { code: 'SERVER', message: 'sentinel' } } } });
|
|
504
|
+
rootListeners.get('session/event')[0](session, { type: 'turn/end', data: { turn: 2, reason: { kind: 'error', error: { code: 'UNKNOWN_MODEL' } } } });
|
|
505
|
+
assert.deepEqual(runtimeWrites, ['child_model_server']);
|
|
1218
506
|
rootListeners.get('session/disposed')[0](session);
|
|
507
|
+
// A malformed stopping payload with a valid session is arbited above a later generic error.
|
|
1219
508
|
const session2 = {};
|
|
1220
|
-
const agent2 = {
|
|
1221
|
-
|
|
1222
|
-
steer () {},
|
|
1223
|
-
ctx: agent.ctx
|
|
1224
|
-
};
|
|
1225
|
-
rootListeners.get('agent/created')[0]({
|
|
1226
|
-
agent: agent2
|
|
1227
|
-
});
|
|
509
|
+
const agent2 = { session: session2, steer() { }, ctx: agent.ctx };
|
|
510
|
+
rootListeners.get('agent/created')[0]({ agent: agent2 });
|
|
1228
511
|
rootListeners.get('session/event')[0](session2, turnStart());
|
|
1229
|
-
assert.throws(()=>rootListeners.get('agent/turn-stopping')[0]({
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
rootListeners.get('session/event')[0](session2, {
|
|
1233
|
-
type: 'turn/end',
|
|
1234
|
-
data: {
|
|
1235
|
-
turn: 1,
|
|
1236
|
-
reason: {
|
|
1237
|
-
kind: 'error',
|
|
1238
|
-
error: {
|
|
1239
|
-
code: 'UNKNOWN'
|
|
1240
|
-
}
|
|
1241
|
-
}
|
|
1242
|
-
}
|
|
1243
|
-
});
|
|
1244
|
-
assert.deepEqual(runtimeWrites, [
|
|
1245
|
-
'child_model_server',
|
|
1246
|
-
'child_conformance_failure'
|
|
1247
|
-
]);
|
|
512
|
+
assert.throws(() => rootListeners.get('agent/turn-stopping')[0]({ agent: agent2 }), new RegExp(BENCHMARK_CONFORMANCE_STATE_UNAVAILABLE));
|
|
513
|
+
rootListeners.get('session/event')[0](session2, { type: 'turn/end', data: { turn: 1, reason: { kind: 'error', error: { code: 'UNKNOWN' } } } });
|
|
514
|
+
assert.deepEqual(runtimeWrites, ['child_model_server', 'child_conformance_failure']);
|
|
1248
515
|
rootListeners.get('session/disposed')[0](session2);
|
|
1249
516
|
const session3 = {};
|
|
1250
|
-
const agent3 = {
|
|
1251
|
-
|
|
1252
|
-
steer () {},
|
|
1253
|
-
ctx: agent.ctx
|
|
1254
|
-
};
|
|
1255
|
-
rootListeners.get('agent/created')[0]({
|
|
1256
|
-
agent: agent3
|
|
1257
|
-
});
|
|
517
|
+
const agent3 = { session: session3, steer() { }, ctx: agent.ctx };
|
|
518
|
+
rootListeners.get('agent/created')[0]({ agent: agent3 });
|
|
1258
519
|
rootListeners.get('session/event')[0](session3, turnStart());
|
|
1259
|
-
rootListeners.get('session/event')[0](session3, {
|
|
1260
|
-
|
|
1261
|
-
data: {
|
|
1262
|
-
turn: 1,
|
|
1263
|
-
reason: {
|
|
1264
|
-
kind: 'error',
|
|
1265
|
-
error: {
|
|
1266
|
-
code: 'TIMEOUT'
|
|
1267
|
-
}
|
|
1268
|
-
}
|
|
1269
|
-
}
|
|
1270
|
-
});
|
|
1271
|
-
assert.deepEqual(runtimeWrites, [
|
|
1272
|
-
'child_model_server',
|
|
1273
|
-
'child_conformance_failure',
|
|
1274
|
-
'child_model_transport'
|
|
1275
|
-
], 'session diagnostics remain isolated');
|
|
520
|
+
rootListeners.get('session/event')[0](session3, { type: 'turn/end', data: { turn: 1, reason: { kind: 'error', error: { code: 'TIMEOUT' } } } });
|
|
521
|
+
assert.deepEqual(runtimeWrites, ['child_model_server', 'child_conformance_failure', 'child_model_transport'], 'session diagnostics remain isolated');
|
|
1276
522
|
rootListeners.get('session/disposed')[0](session3);
|
|
1277
|
-
}
|
|
523
|
+
}
|
|
524
|
+
function fakeHandle(outcome) {
|
|
1278
525
|
const stdout = outcome.stdout ?? '';
|
|
1279
526
|
const stderr = outcome.stderr ?? '';
|
|
1280
527
|
const reader = {
|
|
1281
|
-
readFrom: (_offset)=>
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
528
|
+
readFrom: (_offset) => {
|
|
529
|
+
if (outcome.readFailure)
|
|
530
|
+
throw new Error('collected output reading failed');
|
|
531
|
+
return { text: stdout, nextOffset: Buffer.byteLength(stdout), lossy: outcome.lossy ?? false };
|
|
532
|
+
},
|
|
1286
533
|
};
|
|
1287
534
|
const errorReader = {
|
|
1288
|
-
readFrom: (_offset)=>
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
535
|
+
readFrom: (_offset) => {
|
|
536
|
+
if (outcome.readFailure)
|
|
537
|
+
throw new Error('collected output reading failed');
|
|
538
|
+
return { text: stderr, nextOffset: Buffer.byteLength(stderr), lossy: false };
|
|
539
|
+
},
|
|
1293
540
|
};
|
|
1294
541
|
let rejectDone;
|
|
1295
|
-
const done = outcome.spawnReject
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
});
|
|
542
|
+
const done = outcome.spawnReject
|
|
543
|
+
? Promise.reject(new Error('spawn rejected'))
|
|
544
|
+
: outcome.waitForAbort
|
|
545
|
+
? new Promise((_resolve, reject) => { rejectDone = reject; })
|
|
546
|
+
: Promise.resolve({ exitCode: outcome.exitCode ?? 0, signal: outcome.signal ?? null });
|
|
1301
547
|
return {
|
|
1302
|
-
pid: outcome.spawnReject ? -1 : 4242,
|
|
1303
548
|
stdin: undefined,
|
|
1304
549
|
stdout: undefined,
|
|
1305
550
|
stderr: undefined,
|
|
1306
|
-
collected: {
|
|
1307
|
-
stdout: reader,
|
|
1308
|
-
stderr: errorReader
|
|
1309
|
-
},
|
|
551
|
+
collected: { stdout: reader, stderr: errorReader },
|
|
1310
552
|
done,
|
|
1311
|
-
terminate
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
waitForExit: async ()=>true
|
|
553
|
+
terminate() { if (outcome.waitForAbort)
|
|
554
|
+
rejectDone?.(new Error('timed out')); },
|
|
555
|
+
waitForExit: async () => true,
|
|
1315
556
|
};
|
|
1316
557
|
}
|
|
1317
558
|
function okEnvelope(result) {
|
|
1318
|
-
return JSON.stringify({
|
|
1319
|
-
schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION,
|
|
1320
|
-
status: 'ok',
|
|
1321
|
-
result
|
|
1322
|
-
});
|
|
559
|
+
return JSON.stringify({ schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION, status: 'ok', result });
|
|
1323
560
|
}
|
|
1324
561
|
function lookupInputSchema() {
|
|
1325
562
|
return {
|
|
1326
563
|
type: 'object',
|
|
1327
564
|
properties: {
|
|
1328
|
-
city: {
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
description: 'Declared city name.'
|
|
1335
|
-
},
|
|
1336
|
-
payload: {
|
|
1337
|
-
type: 'string'
|
|
1338
|
-
},
|
|
1339
|
-
notes: {
|
|
1340
|
-
type: 'string'
|
|
1341
|
-
},
|
|
1342
|
-
executable: {
|
|
1343
|
-
type: 'string'
|
|
1344
|
-
},
|
|
1345
|
-
cwd: {
|
|
1346
|
-
type: 'string'
|
|
1347
|
-
},
|
|
1348
|
-
argv: {
|
|
1349
|
-
type: 'array',
|
|
1350
|
-
items: {
|
|
1351
|
-
type: 'string'
|
|
1352
|
-
}
|
|
1353
|
-
}
|
|
565
|
+
city: { type: 'string', enum: ['Dubai', 'Abu Dhabi'], description: 'Declared city name.' },
|
|
566
|
+
payload: { type: 'string' },
|
|
567
|
+
notes: { type: 'string' },
|
|
568
|
+
executable: { type: 'string' },
|
|
569
|
+
cwd: { type: 'string' },
|
|
570
|
+
argv: { type: 'array', items: { type: 'string' } },
|
|
1354
571
|
},
|
|
1355
|
-
required: [
|
|
1356
|
-
|
|
1357
|
-
],
|
|
1358
|
-
additionalProperties: false
|
|
572
|
+
required: ['city'],
|
|
573
|
+
additionalProperties: false,
|
|
1359
574
|
};
|
|
1360
575
|
}
|
|
1361
576
|
function bridgeCall(tool, argumentsValue) {
|
|
1362
|
-
return {
|
|
1363
|
-
action: 'call',
|
|
1364
|
-
tool,
|
|
1365
|
-
arguments: argumentsValue
|
|
1366
|
-
};
|
|
577
|
+
return { action: 'call', tool, arguments: argumentsValue };
|
|
1367
578
|
}
|
|
1368
579
|
async function assertRealCordisWaterfallOrdering() {
|
|
1369
580
|
const ctx = new Context();
|
|
1370
581
|
const bridge = {
|
|
1371
582
|
name: 'gotry_benchmark_environment',
|
|
1372
583
|
description: 'benchmark bridge',
|
|
1373
|
-
parameters: {
|
|
1374
|
-
oneOf: [
|
|
1375
|
-
{
|
|
1376
|
-
type: 'object',
|
|
1377
|
-
properties: {
|
|
1378
|
-
action: {
|
|
1379
|
-
const: 'tools'
|
|
1380
|
-
}
|
|
1381
|
-
},
|
|
1382
|
-
required: [
|
|
1383
|
-
'action'
|
|
1384
|
-
],
|
|
1385
|
-
additionalProperties: false
|
|
1386
|
-
}
|
|
1387
|
-
]
|
|
1388
|
-
}
|
|
584
|
+
parameters: { oneOf: [{ type: 'object', properties: { action: { const: 'tools' } }, required: ['action'], additionalProperties: false }] },
|
|
1389
585
|
};
|
|
1390
586
|
const exactSchema = structuredClone(bridge);
|
|
1391
587
|
let addPreStepTool = false;
|
|
1392
588
|
const rootTools = {
|
|
1393
|
-
get
|
|
1394
|
-
|
|
589
|
+
get(name) { return name === bridge.name ? bridge : undefined; },
|
|
590
|
+
schemas(agent) {
|
|
591
|
+
return agent && addPreStepTool ? [exactSchema, { name: 'non_bridge' }] : [exactSchema];
|
|
1395
592
|
},
|
|
1396
|
-
schemas (agent) {
|
|
1397
|
-
return agent && addPreStepTool ? [
|
|
1398
|
-
exactSchema,
|
|
1399
|
-
{
|
|
1400
|
-
name: 'non_bridge'
|
|
1401
|
-
}
|
|
1402
|
-
] : [
|
|
1403
|
-
exactSchema
|
|
1404
|
-
];
|
|
1405
|
-
}
|
|
1406
593
|
};
|
|
1407
594
|
ctx.provide('tools', rootTools);
|
|
1408
|
-
ctx.provide('agents', {
|
|
1409
|
-
list: ()=>[]
|
|
1410
|
-
});
|
|
595
|
+
ctx.provide('agents', { list: () => [] });
|
|
1411
596
|
const bus = ctx;
|
|
1412
|
-
bus.on('system-prompt/assemble', async (_assembly, _context, next)=>{
|
|
597
|
+
bus.on('system-prompt/assemble', async (_assembly, _context, next) => {
|
|
1413
598
|
const result = await next();
|
|
1414
|
-
return {
|
|
1415
|
-
...result,
|
|
1416
|
-
tools: [
|
|
1417
|
-
...result.tools,
|
|
1418
|
-
{
|
|
1419
|
-
name: 'non_bridge'
|
|
1420
|
-
}
|
|
1421
|
-
]
|
|
1422
|
-
};
|
|
599
|
+
return { ...result, tools: [...result.tools, { name: 'non_bridge' }] };
|
|
1423
600
|
});
|
|
1424
|
-
bus.on('agent/pre-step', async (_payload, next)=>{
|
|
601
|
+
bus.on('agent/pre-step', async (_payload, next) => {
|
|
1425
602
|
const result = await next();
|
|
1426
603
|
addPreStepTool = true;
|
|
1427
604
|
return result;
|
|
1428
605
|
});
|
|
1429
606
|
installBenchmarkToolIsolation(ctx);
|
|
1430
|
-
const scopedEffect = (action, label)=>ctx.effect(action, label);
|
|
607
|
+
const scopedEffect = (action, label) => ctx.effect(action, label);
|
|
1431
608
|
const scopedTools = {
|
|
1432
|
-
guard: ()=>ctx.effect(()=>()=>undefined),
|
|
1433
|
-
presentAs: ()=>ctx.effect(()=>()=>undefined),
|
|
1434
|
-
restrict: ()=>ctx.effect(()=>()=>undefined)
|
|
1435
|
-
};
|
|
1436
|
-
const agent = {
|
|
1437
|
-
ctx: {
|
|
1438
|
-
tools: scopedTools,
|
|
1439
|
-
effect: scopedEffect,
|
|
1440
|
-
on: bus.on
|
|
1441
|
-
}
|
|
609
|
+
guard: () => ctx.effect(() => () => undefined),
|
|
610
|
+
presentAs: () => ctx.effect(() => () => undefined),
|
|
611
|
+
restrict: () => ctx.effect(() => () => undefined),
|
|
1442
612
|
};
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
});
|
|
1446
|
-
await assert.rejects(bus.waterfall('
|
|
1447
|
-
tools: []
|
|
1448
|
-
}, {
|
|
1449
|
-
agent,
|
|
1450
|
-
scope: agent
|
|
1451
|
-
}, async ()=>({
|
|
1452
|
-
tools: [
|
|
1453
|
-
exactSchema
|
|
1454
|
-
]
|
|
1455
|
-
})), /BENCHMARK_TOOL_SURFACE_VIOLATION/, 'prepend assembly guard observes an earlier listener post-next mutation');
|
|
1456
|
-
await assert.rejects(bus.waterfall('agent/pre-step', {
|
|
1457
|
-
agent
|
|
1458
|
-
}, async ()=>({
|
|
1459
|
-
kind: 'enter'
|
|
1460
|
-
})), /BENCHMARK_TOOL_SURFACE_VIOLATION/, 'prepend pre-step guard observes an earlier listener post-next scope mutation');
|
|
613
|
+
const agent = { ctx: { tools: scopedTools, effect: scopedEffect, on: bus.on } };
|
|
614
|
+
bus.emit('agent/created', { agent });
|
|
615
|
+
await assert.rejects(bus.waterfall('system-prompt/assemble', { tools: [] }, { agent, scope: agent }, async () => ({ tools: [exactSchema] })), /BENCHMARK_TOOL_SURFACE_VIOLATION/, 'prepend assembly guard observes an earlier listener post-next mutation');
|
|
616
|
+
await assert.rejects(bus.waterfall('agent/pre-step', { agent }, async () => ({ kind: 'enter' })), /BENCHMARK_TOOL_SURFACE_VIOLATION/, 'prepend pre-step guard observes an earlier listener post-next scope mutation');
|
|
1461
617
|
await ctx.fiber.dispose();
|
|
1462
618
|
}
|
|
1463
619
|
await assertRealCordisWaterfallOrdering();
|
|
@@ -1468,48 +624,25 @@ const ambientSentinelNames = [
|
|
|
1468
624
|
'DATABASE_URL',
|
|
1469
625
|
'SSH_AUTH_SOCK',
|
|
1470
626
|
'AWS_PROFILE',
|
|
1471
|
-
'HTTPS_PROXY'
|
|
627
|
+
'HTTPS_PROXY',
|
|
1472
628
|
];
|
|
1473
|
-
const ambientSentinels = new Map(ambientSentinelNames.map(
|
|
1474
|
-
name,
|
|
1475
|
-
process.env[name]
|
|
1476
|
-
]));
|
|
629
|
+
const ambientSentinels = new Map(ambientSentinelNames.map(name => [name, process.env[name]]));
|
|
1477
630
|
try {
|
|
1478
631
|
delete process.env.GOTRY_LLM_MODEL;
|
|
1479
632
|
const timedOutDiagnostic = '\n' + JSON.stringify({
|
|
1480
633
|
schema_version: BENCHMARK_CHILD_DIAGNOSTIC_SCHEMA,
|
|
1481
|
-
code: 'child_bridge_timed_out'
|
|
634
|
+
code: 'child_bridge_timed_out',
|
|
1482
635
|
}) + '\n';
|
|
1483
636
|
assert.equal(parseBenchmarkChildDiagnostic(timedOutDiagnostic), 'child_bridge_timed_out', 'strict control record parses to its allowlisted reason code');
|
|
1484
|
-
assert.equal(classifyBenchmarkChildFailure({
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
}), 'child_bridge_timed_out', 'structured control data maps to a stable bridge timeout code');
|
|
1488
|
-
assert.equal(classifyBenchmarkChildFailure({
|
|
1489
|
-
code: 1,
|
|
1490
|
-
diagnostic: `provider text contains child_bridge_timed_out and ${BENCHMARK_BRIDGE_TIMED_OUT}`
|
|
1491
|
-
}), 'child_nonzero_exit', 'free text cannot impersonate a structured bridge reason');
|
|
1492
|
-
assert.equal(parseBenchmarkChildDiagnostic(JSON.stringify({
|
|
1493
|
-
schema_version: BENCHMARK_CHILD_DIAGNOSTIC_SCHEMA,
|
|
1494
|
-
code: 'child_bridge_timed_out',
|
|
1495
|
-
extra: 'rejected'
|
|
1496
|
-
})), undefined, 'control records with extra keys fail closed');
|
|
637
|
+
assert.equal(classifyBenchmarkChildFailure({ code: 1, diagnostic: timedOutDiagnostic }), 'child_bridge_timed_out', 'structured control data maps to a stable bridge timeout code');
|
|
638
|
+
assert.equal(classifyBenchmarkChildFailure({ code: 1, diagnostic: `provider text contains child_bridge_timed_out and ${BENCHMARK_BRIDGE_TIMED_OUT}` }), 'child_nonzero_exit', 'free text cannot impersonate a structured bridge reason');
|
|
639
|
+
assert.equal(parseBenchmarkChildDiagnostic(JSON.stringify({ schema_version: BENCHMARK_CHILD_DIAGNOSTIC_SCHEMA, code: 'child_bridge_timed_out', extra: 'rejected' })), undefined, 'control records with extra keys fail closed');
|
|
1497
640
|
assert.equal(benchmarkChildFailureForConformanceCode(BENCHMARK_BRIDGE_RUNNER_FAILED), 'child_bridge_runner_failed', 'runner failure maps to a stable structured child reason');
|
|
1498
641
|
assert.equal(benchmarkChildFailureForConformanceCode(BENCHMARK_BRIDGE_SPAWN_FAILED), 'child_bridge_spawn_failed', 'spawn failure maps to a stable structured child reason');
|
|
1499
642
|
assert.equal(benchmarkChildFailureForConformanceCode(BENCHMARK_BRIDGE_OUTPUT_TRUNCATED), 'child_bridge_output_truncated', 'runner output truncation maps to a stable structured child reason');
|
|
1500
|
-
assert.equal(classifyBenchmarkChildFailure({
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
}), 'child_output_truncated', 'truncated terminal output takes precedence over control data');
|
|
1504
|
-
assert.equal(classifyBenchmarkChildFailure({
|
|
1505
|
-
code: null,
|
|
1506
|
-
signal: 'SIGTERM',
|
|
1507
|
-
diagnostic: timedOutDiagnostic
|
|
1508
|
-
}), 'child_signaled', 'outer child signal takes precedence over an inner bridge diagnostic');
|
|
1509
|
-
assert.equal(classifyBenchmarkChildFailure({
|
|
1510
|
-
code: 0,
|
|
1511
|
-
diagnostic: ''
|
|
1512
|
-
}), 'child_lifecycle_failure', 'unexpected zero-exit diagnostic path remains a stable lifecycle failure');
|
|
643
|
+
assert.equal(classifyBenchmarkChildFailure({ code: 0, outputTruncated: true }), 'child_output_truncated', 'truncated terminal output takes precedence over control data');
|
|
644
|
+
assert.equal(classifyBenchmarkChildFailure({ code: null, signal: 'SIGTERM', diagnostic: timedOutDiagnostic }), 'child_signaled', 'outer child signal takes precedence over an inner bridge diagnostic');
|
|
645
|
+
assert.equal(classifyBenchmarkChildFailure({ code: 0, diagnostic: '' }), 'child_lifecycle_failure', 'unexpected zero-exit diagnostic path remains a stable lifecycle failure');
|
|
1513
646
|
const noisyPrefix = Buffer.alloc(BENCHMARK_CHILD_DIAGNOSTIC_MAX_BYTES + 10, 'x');
|
|
1514
647
|
const boundedControl = appendBoundedChildDiagnostic(appendBoundedChildDiagnostic(Buffer.alloc(0), noisyPrefix), timedOutDiagnostic);
|
|
1515
648
|
assert.equal(boundedControl.length, BENCHMARK_CHILD_DIAGNOSTIC_MAX_BYTES, 'control capture is bounded');
|
|
@@ -1520,72 +653,20 @@ try {
|
|
|
1520
653
|
enabled: true,
|
|
1521
654
|
executable: process.execPath,
|
|
1522
655
|
cwd: root,
|
|
1523
|
-
argv_prefix: [
|
|
1524
|
-
'-m',
|
|
1525
|
-
'agent_env.cli',
|
|
1526
|
-
'--lang',
|
|
1527
|
-
'en'
|
|
1528
|
-
],
|
|
656
|
+
argv_prefix: ['-m', 'agent_env.cli', '--lang', 'en'],
|
|
1529
657
|
tools: [
|
|
1530
|
-
{
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
input_schema: lookupInputSchema(),
|
|
1534
|
-
output_keys: [
|
|
1535
|
-
'city',
|
|
1536
|
-
'nested'
|
|
1537
|
-
],
|
|
1538
|
-
domain_outcomes: [
|
|
1539
|
-
{
|
|
1540
|
-
status: 'miss',
|
|
1541
|
-
code: 'NOT_FOUND',
|
|
1542
|
-
recovery: 'revise_arguments'
|
|
1543
|
-
},
|
|
1544
|
-
{
|
|
1545
|
-
status: 'error',
|
|
1546
|
-
code: 'AMBIGUOUS',
|
|
1547
|
-
recovery: 'choose_alternative'
|
|
1548
|
-
}
|
|
1549
|
-
]
|
|
1550
|
-
},
|
|
1551
|
-
{
|
|
1552
|
-
name: 'constructor',
|
|
1553
|
-
description: 'Construct one declared city.',
|
|
1554
|
-
input_schema: lookupInputSchema(),
|
|
1555
|
-
output_keys: [
|
|
1556
|
-
'legacy'
|
|
1557
|
-
],
|
|
1558
|
-
domain_outcomes: [
|
|
1559
|
-
{
|
|
1560
|
-
status: 'error',
|
|
1561
|
-
code: 'INVALID',
|
|
1562
|
-
recovery: 'retry_same'
|
|
1563
|
-
}
|
|
1564
|
-
]
|
|
1565
|
-
},
|
|
1566
|
-
{
|
|
1567
|
-
name: 'toString',
|
|
1568
|
-
description: 'Stringify one declared city.',
|
|
1569
|
-
input_schema: lookupInputSchema(),
|
|
1570
|
-
output_keys: [
|
|
1571
|
-
'city',
|
|
1572
|
-
'nested'
|
|
1573
|
-
],
|
|
1574
|
-
domain_outcomes: []
|
|
1575
|
-
}
|
|
658
|
+
{ name: 'lookup', description: 'Lookup one declared city.', input_schema: lookupInputSchema(), output_keys: ['city', 'nested'], domain_outcomes: [{ status: 'miss', code: 'NOT_FOUND', recovery: 'revise_arguments' }, { status: 'error', code: 'AMBIGUOUS', recovery: 'choose_alternative' }] },
|
|
659
|
+
{ name: 'constructor', description: 'Construct one declared city.', input_schema: lookupInputSchema(), output_keys: ['legacy'], domain_outcomes: [{ status: 'error', code: 'INVALID', recovery: 'retry_same' }] },
|
|
660
|
+
{ name: 'toString', description: 'Stringify one declared city.', input_schema: lookupInputSchema(), output_keys: ['city', 'nested'], domain_outcomes: [] },
|
|
1576
661
|
],
|
|
1577
662
|
timeout_ms: 20,
|
|
1578
663
|
max_output_bytes: 4_096,
|
|
1579
|
-
terminal_output: {
|
|
1580
|
-
tag: 'done',
|
|
1581
|
-
max_bytes: 4_096,
|
|
1582
|
-
body_schema: OK_BODY_SCHEMA
|
|
1583
|
-
},
|
|
664
|
+
terminal_output: { tag: 'done', max_bytes: 4_096, body_schema: OK_BODY_SCHEMA },
|
|
1584
665
|
isolation: {
|
|
1585
666
|
mode: 'host-enforced',
|
|
1586
667
|
writes: 'forbidden',
|
|
1587
|
-
network: 'denied'
|
|
1588
|
-
}
|
|
668
|
+
network: 'denied',
|
|
669
|
+
},
|
|
1589
670
|
}));
|
|
1590
671
|
const registered = [];
|
|
1591
672
|
let visibleBridge;
|
|
@@ -1599,34 +680,29 @@ try {
|
|
|
1599
680
|
const preStepListeners = [];
|
|
1600
681
|
const disposedListeners = [];
|
|
1601
682
|
const eventOptions = new Map();
|
|
1602
|
-
const runEffect = (action)=>{
|
|
683
|
+
const runEffect = (action) => {
|
|
1603
684
|
const yielded = [];
|
|
1604
685
|
const value = action();
|
|
1605
686
|
if (value && typeof value.next === 'function') {
|
|
1606
687
|
let step = value.next();
|
|
1607
|
-
while(!step.done){
|
|
1608
|
-
if (typeof step.value === 'function')
|
|
688
|
+
while (!step.done) {
|
|
689
|
+
if (typeof step.value === 'function')
|
|
690
|
+
yielded.push(step.value);
|
|
1609
691
|
step = value.next();
|
|
1610
692
|
}
|
|
1611
693
|
}
|
|
1612
694
|
let active = true;
|
|
1613
|
-
return async ()=>{
|
|
1614
|
-
if (!active)
|
|
695
|
+
return async () => {
|
|
696
|
+
if (!active)
|
|
697
|
+
return;
|
|
1615
698
|
active = false;
|
|
1616
|
-
for (const dispose of yielded.reverse())
|
|
699
|
+
for (const dispose of yielded.reverse())
|
|
700
|
+
await dispose();
|
|
1617
701
|
};
|
|
1618
702
|
};
|
|
1619
703
|
let disposeRootIsolation;
|
|
1620
704
|
let timeoutSignal;
|
|
1621
|
-
const outcomes = [
|
|
1622
|
-
{
|
|
1623
|
-
stdout: okEnvelope([
|
|
1624
|
-
{
|
|
1625
|
-
city: 'Dubai'
|
|
1626
|
-
}
|
|
1627
|
-
])
|
|
1628
|
-
}
|
|
1629
|
-
];
|
|
705
|
+
const outcomes = [{ stdout: okEnvelope([{ city: 'Dubai' }]) }];
|
|
1630
706
|
process.env.GOTRY_BENCHMARK_BRIDGE_PARENT_SENTINEL = 'must-not-cross-boundary';
|
|
1631
707
|
process.env.DATABASE_URL = 'postgres://secret';
|
|
1632
708
|
process.env.SSH_AUTH_SOCK = '/tmp/secret.sock';
|
|
@@ -1634,132 +710,90 @@ try {
|
|
|
1634
710
|
process.env.HTTPS_PROXY = 'https://secret-proxy';
|
|
1635
711
|
const ctx = {
|
|
1636
712
|
tools: {
|
|
1637
|
-
register
|
|
713
|
+
register(tool) {
|
|
1638
714
|
registered.push(tool);
|
|
1639
|
-
return ()=>{};
|
|
1640
|
-
},
|
|
1641
|
-
get (name, agent) {
|
|
1642
|
-
return name === 'gotry_benchmark_environment' ? agent !== undefined && agent === shadowedAgent ? {
|
|
1643
|
-
name
|
|
1644
|
-
} : registered.find((tool)=>tool.name === name) : undefined;
|
|
715
|
+
return () => { };
|
|
1645
716
|
},
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
description: tool.description,
|
|
1650
|
-
parameters: structuredClone(tool.parameters)
|
|
1651
|
-
});
|
|
717
|
+
get(name, agent) { return name === 'gotry_benchmark_environment' ? (agent !== undefined && agent === shadowedAgent ? { name } : registered.find(tool => tool.name === name)) : undefined; },
|
|
718
|
+
schemas(agent) {
|
|
719
|
+
const project = (tool) => ({ name: tool.name, description: tool.description, parameters: structuredClone(tool.parameters) });
|
|
1652
720
|
const schemas = registered.map(project);
|
|
1653
|
-
if (agent === undefined)
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
];
|
|
1658
|
-
}
|
|
1659
|
-
},
|
|
1660
|
-
systemPrompt: {
|
|
1661
|
-
variable (name) {
|
|
1662
|
-
promptVariables.push(name);
|
|
1663
|
-
}
|
|
721
|
+
if (agent === undefined)
|
|
722
|
+
return schemas;
|
|
723
|
+
return [...schemas.filter(schema => schema.name === 'gotry_benchmark_environment'), ...(scopedExtraSchemas.get(agent) ?? [])];
|
|
724
|
+
},
|
|
1664
725
|
},
|
|
1665
|
-
|
|
726
|
+
systemPrompt: { variable(name) { promptVariables.push(name); } },
|
|
727
|
+
on(event, listener, options) {
|
|
1666
728
|
eventNames.push(event);
|
|
1667
729
|
eventOptions.set(event, options);
|
|
1668
|
-
if (event === 'agent/created')
|
|
1669
|
-
|
|
1670
|
-
if (event === '
|
|
1671
|
-
|
|
1672
|
-
|
|
730
|
+
if (event === 'agent/created')
|
|
731
|
+
agentCreatedListeners.push(listener);
|
|
732
|
+
if (event === 'agent/disposed')
|
|
733
|
+
disposedListeners.push(listener);
|
|
734
|
+
if (event === 'system-prompt/assemble')
|
|
735
|
+
assemblyListeners.push(listener);
|
|
736
|
+
if (event === 'agent/pre-step')
|
|
737
|
+
preStepListeners.push(listener);
|
|
738
|
+
return () => { };
|
|
1673
739
|
},
|
|
1674
|
-
effect
|
|
740
|
+
effect(action, label) {
|
|
1675
741
|
const dispose = runEffect(action);
|
|
1676
|
-
if (label === 'benchmark-environment-tool-isolation')
|
|
742
|
+
if (label === 'benchmark-environment-tool-isolation')
|
|
743
|
+
disposeRootIsolation = dispose;
|
|
1677
744
|
return dispose;
|
|
1678
745
|
},
|
|
1679
|
-
get
|
|
1680
|
-
if (name === 'subprocess')
|
|
1681
|
-
|
|
746
|
+
get(name) {
|
|
747
|
+
if (name === 'subprocess')
|
|
748
|
+
return this.subprocess;
|
|
749
|
+
if (name === 'agents')
|
|
750
|
+
return this.agents;
|
|
1682
751
|
},
|
|
1683
752
|
subprocess: {
|
|
1684
|
-
spawn
|
|
753
|
+
spawn(spec) {
|
|
1685
754
|
spawnSpecs.push(spec);
|
|
1686
|
-
const outcome = outcomes.shift() ?? {
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
if (outcome.spawnError) throw new Error('fake spawn failed');
|
|
755
|
+
const outcome = outcomes.shift() ?? { stdout: okEnvelope({}) };
|
|
756
|
+
if (outcome.spawnError)
|
|
757
|
+
throw new Error('fake spawn failed');
|
|
1690
758
|
if (outcome.waitForAbort) {
|
|
1691
759
|
timeoutSignal = spec.signal;
|
|
1692
760
|
const handle = fakeHandle(outcome);
|
|
1693
|
-
spec.signal?.addEventListener('abort', ()=>handle.terminate?.(), {
|
|
1694
|
-
once: true
|
|
1695
|
-
});
|
|
761
|
+
spec.signal?.addEventListener('abort', () => handle.terminate?.(), { once: true });
|
|
1696
762
|
return handle;
|
|
1697
763
|
}
|
|
1698
764
|
return fakeHandle(outcome);
|
|
1699
|
-
}
|
|
765
|
+
},
|
|
1700
766
|
},
|
|
1701
|
-
agents: {
|
|
1702
|
-
list () {
|
|
1703
|
-
return [];
|
|
1704
|
-
}
|
|
1705
|
-
}
|
|
767
|
+
agents: { list() { return []; } },
|
|
1706
768
|
};
|
|
1707
769
|
const coldStartListeners = [];
|
|
1708
|
-
const coldStartRoot = {
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
return name === 'gotry_benchmark_environment' ? coldStartRoot : undefined;
|
|
1715
|
-
},
|
|
1716
|
-
schemas () {
|
|
1717
|
-
return [
|
|
1718
|
-
{
|
|
1719
|
-
name: 'gotry_benchmark_environment'
|
|
1720
|
-
}
|
|
1721
|
-
];
|
|
1722
|
-
}
|
|
1723
|
-
},
|
|
1724
|
-
agents: {
|
|
1725
|
-
list () {
|
|
1726
|
-
return [
|
|
1727
|
-
{
|
|
1728
|
-
id: 'already-live'
|
|
1729
|
-
}
|
|
1730
|
-
];
|
|
1731
|
-
}
|
|
1732
|
-
},
|
|
1733
|
-
on (event) {
|
|
1734
|
-
coldStartListeners.push(event);
|
|
1735
|
-
return ()=>{};
|
|
1736
|
-
}
|
|
1737
|
-
}), /cold-start|live-agent/i, 'installing benchmark isolation with an existing agent fails hard');
|
|
770
|
+
const coldStartRoot = { name: 'gotry_benchmark_environment' };
|
|
771
|
+
assert.throws(() => installBenchmarkToolIsolation({
|
|
772
|
+
tools: { get(name) { return name === 'gotry_benchmark_environment' ? coldStartRoot : undefined; }, schemas() { return [{ name: 'gotry_benchmark_environment' }]; } },
|
|
773
|
+
agents: { list() { return [{ id: 'already-live' }]; } },
|
|
774
|
+
on(event) { coldStartListeners.push(event); return () => { }; },
|
|
775
|
+
}), /cold-start|live-agent/i, 'installing benchmark isolation with an existing agent fails hard');
|
|
1738
776
|
assert.deepEqual(coldStartListeners, [], 'cold-start rejection does not register an isolation listener');
|
|
1739
777
|
const processListenersBeforeBenchmark = {
|
|
1740
778
|
uncaughtException: process.listenerCount('uncaughtException'),
|
|
1741
|
-
unhandledRejection: process.listenerCount('unhandledRejection')
|
|
779
|
+
unhandledRejection: process.listenerCount('unhandledRejection'),
|
|
1742
780
|
};
|
|
1743
781
|
apply(ctx, {
|
|
1744
782
|
stateRoot: root,
|
|
1745
783
|
timeoutMs: 20,
|
|
1746
784
|
hbcliBin: '',
|
|
1747
785
|
sessionAccess: 'off',
|
|
1748
|
-
benchmarkEnvironmentConfigPath: configPath
|
|
786
|
+
benchmarkEnvironmentConfigPath: configPath,
|
|
1749
787
|
});
|
|
1750
|
-
assert.ok(registered.some(
|
|
1751
|
-
assert.deepEqual(registered.map(
|
|
1752
|
-
'gotry_benchmark_environment'
|
|
1753
|
-
], 'benchmark mode boots only the single bridge tool instead of the product tool catalog');
|
|
788
|
+
assert.ok(registered.some(tool => tool.name === 'gotry_benchmark_environment'), 'an explicit valid owner-local config registers the benchmark environment bridge');
|
|
789
|
+
assert.deepEqual(registered.map(tool => tool.name), ['gotry_benchmark_environment'], 'benchmark mode boots only the single bridge tool instead of the product tool catalog');
|
|
1754
790
|
assert.deepEqual(promptVariables, [], 'benchmark mode does not install product prompt variables');
|
|
1755
791
|
assert.equal(eventNames.includes('tools/pre-execute'), false, 'benchmark mode does not install the product session-consent hook');
|
|
1756
792
|
assert.deepEqual({
|
|
1757
793
|
uncaughtException: process.listenerCount('uncaughtException'),
|
|
1758
|
-
unhandledRejection: process.listenerCount('unhandledRejection')
|
|
794
|
+
unhandledRejection: process.listenerCount('unhandledRejection'),
|
|
1759
795
|
}, processListenersBeforeBenchmark, 'benchmark mode does not install product process incident guards');
|
|
1760
|
-
assert.deepEqual([
|
|
1761
|
-
...eventNames
|
|
1762
|
-
].sort(), [
|
|
796
|
+
assert.deepEqual([...eventNames].sort(), [
|
|
1763
797
|
'agent/created',
|
|
1764
798
|
'agent/created',
|
|
1765
799
|
'agent/disposed',
|
|
@@ -1772,533 +806,175 @@ try {
|
|
|
1772
806
|
'session/event',
|
|
1773
807
|
'system-prompt/assemble',
|
|
1774
808
|
'tools/execute',
|
|
1775
|
-
'tools/post-execute'
|
|
809
|
+
'tools/post-execute',
|
|
1776
810
|
], 'benchmark root listeners come only from budget, isolation, and conformance when model override is unset');
|
|
1777
|
-
visibleBridge = registered.find(
|
|
811
|
+
visibleBridge = registered.find(tool => tool.name === 'gotry_benchmark_environment');
|
|
1778
812
|
const restrictions = [];
|
|
1779
813
|
const guards = [];
|
|
1780
814
|
const presentations = [];
|
|
1781
|
-
const cleanupCounts = {
|
|
1782
|
-
restrict: 0,
|
|
1783
|
-
guard: 0,
|
|
1784
|
-
presentAs: 0,
|
|
1785
|
-
assembly: 0
|
|
1786
|
-
};
|
|
815
|
+
const cleanupCounts = { restrict: 0, guard: 0, presentAs: 0, assembly: 0 };
|
|
1787
816
|
const firstScopedAssemblyListeners = [];
|
|
1788
817
|
const scopedTools = {
|
|
1789
|
-
restrict (filter) {
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
cleanupCounts.restrict += 1;
|
|
1793
|
-
};
|
|
1794
|
-
},
|
|
1795
|
-
guard (check) {
|
|
1796
|
-
guards.push(check);
|
|
1797
|
-
return ()=>{
|
|
1798
|
-
cleanupCounts.guard += 1;
|
|
1799
|
-
};
|
|
1800
|
-
},
|
|
1801
|
-
presentAs (mode) {
|
|
1802
|
-
presentations.push(mode);
|
|
1803
|
-
return ()=>{
|
|
1804
|
-
cleanupCounts.presentAs += 1;
|
|
1805
|
-
};
|
|
1806
|
-
}
|
|
818
|
+
restrict(filter) { restrictions.push(filter); return () => { cleanupCounts.restrict += 1; }; },
|
|
819
|
+
guard(check) { guards.push(check); return () => { cleanupCounts.guard += 1; }; },
|
|
820
|
+
presentAs(mode) { presentations.push(mode); return () => { cleanupCounts.presentAs += 1; }; },
|
|
1807
821
|
};
|
|
1808
822
|
assert.equal(agentCreatedListeners.length, 2, 'opt-in bridge installs exactly isolation and conformance agent listeners');
|
|
1809
823
|
const isolatedAgentEffects = [];
|
|
1810
|
-
const isolatedAgent = {
|
|
1811
|
-
session: {},
|
|
1812
|
-
steer (_message) {},
|
|
1813
|
-
ctx: {
|
|
824
|
+
const isolatedAgent = { session: {}, steer(_message) { }, ctx: {
|
|
1814
825
|
tools: scopedTools,
|
|
1815
|
-
effect: (action)=>{
|
|
826
|
+
effect: (action) => {
|
|
1816
827
|
const dispose = runEffect(action);
|
|
1817
828
|
isolatedAgentEffects.push(dispose);
|
|
1818
829
|
return dispose;
|
|
1819
830
|
},
|
|
1820
|
-
on: (event, listener, options)=>{
|
|
831
|
+
on: (event, listener, options) => {
|
|
1821
832
|
assert.equal(event, 'system-prompt/assemble');
|
|
1822
|
-
assert.deepEqual(options, {
|
|
1823
|
-
prepend: true
|
|
1824
|
-
});
|
|
833
|
+
assert.deepEqual(options, { prepend: true });
|
|
1825
834
|
firstScopedAssemblyListeners.push(listener);
|
|
1826
|
-
return ()=>{
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
}
|
|
1831
|
-
};
|
|
1832
|
-
|
|
1833
|
-
agent: isolatedAgent
|
|
1834
|
-
});
|
|
1835
|
-
assert.deepEqual(restrictions, [
|
|
1836
|
-
{
|
|
1837
|
-
allow: [
|
|
1838
|
-
'gotry_benchmark_environment'
|
|
1839
|
-
]
|
|
1840
|
-
}
|
|
1841
|
-
], 'agent scope allows only the bridge tool');
|
|
1842
|
-
assert.deepEqual(presentations, [
|
|
1843
|
-
'native'
|
|
1844
|
-
], 'agent scope forces native tool presentation');
|
|
835
|
+
return () => { cleanupCounts.assembly += 1; };
|
|
836
|
+
},
|
|
837
|
+
} };
|
|
838
|
+
for (const listener of agentCreatedListeners)
|
|
839
|
+
listener({ agent: isolatedAgent });
|
|
840
|
+
assert.deepEqual(restrictions, [{ allow: ['gotry_benchmark_environment'] }], 'agent scope allows only the bridge tool');
|
|
841
|
+
assert.deepEqual(presentations, ['native'], 'agent scope forces native tool presentation');
|
|
1845
842
|
assert.ok(eventNames.includes('agent/pre-step'), 'isolation observes pre-step before each request');
|
|
1846
|
-
assert.deepEqual(eventOptions.get('agent/pre-step'), {
|
|
1847
|
-
prepend: true
|
|
1848
|
-
}, 'pre-step isolation wraps every previously registered listener');
|
|
843
|
+
assert.deepEqual(eventOptions.get('agent/pre-step'), { prepend: true }, 'pre-step isolation wraps every previously registered listener');
|
|
1849
844
|
assert.ok(eventNames.includes('agent/disposed'), 'isolation cleans up on agent disposal');
|
|
1850
845
|
assert.ok(assemblyListeners.length > 0, 'isolation validates final assembled tool surface');
|
|
1851
846
|
assert.equal(firstScopedAssemblyListeners.length, 2, 'agent owns one isolation assembly listener and one conformance section listener');
|
|
1852
|
-
assert.deepEqual(eventOptions.get('system-prompt/assemble'), {
|
|
1853
|
-
prepend: true
|
|
1854
|
-
}, 'assembly isolation wraps every previously registered listener');
|
|
847
|
+
assert.deepEqual(eventOptions.get('system-prompt/assemble'), { prepend: true }, 'assembly isolation wraps every previously registered listener');
|
|
1855
848
|
assert.equal(disposedListeners.length, 2, 'isolation and conformance each own one agent/disposed listener');
|
|
1856
849
|
assert.ok(disposeRootIsolation, 'root isolation effect exposes plugin-lifecycle cleanup');
|
|
1857
850
|
const exactSchema = {
|
|
1858
851
|
name: visibleBridge.name,
|
|
1859
852
|
description: visibleBridge.description,
|
|
1860
|
-
parameters: structuredClone(visibleBridge.parameters)
|
|
853
|
+
parameters: structuredClone(visibleBridge.parameters),
|
|
1861
854
|
};
|
|
1862
855
|
const assemble = assemblyListeners[0];
|
|
1863
|
-
const nextExact = async ()=>({
|
|
1864
|
-
|
|
1865
|
-
|
|
1866
|
-
|
|
1867
|
-
|
|
1868
|
-
assert.
|
|
1869
|
-
|
|
1870
|
-
}, {
|
|
1871
|
-
agent: isolatedAgent,
|
|
1872
|
-
scope: isolatedAgent
|
|
1873
|
-
}, nextExact), {
|
|
1874
|
-
tools: [
|
|
1875
|
-
exactSchema
|
|
1876
|
-
]
|
|
1877
|
-
}, 'legal final assembly passes unchanged');
|
|
1878
|
-
await assert.rejects(async ()=>await assemble({
|
|
1879
|
-
tools: []
|
|
1880
|
-
}, {
|
|
1881
|
-
agent: isolatedAgent,
|
|
1882
|
-
scope: isolatedAgent
|
|
1883
|
-
}, async ()=>({
|
|
1884
|
-
tools: [
|
|
1885
|
-
exactSchema,
|
|
1886
|
-
{
|
|
1887
|
-
name: 'own_side_effect'
|
|
1888
|
-
}
|
|
1889
|
-
]
|
|
1890
|
-
})), /BENCHMARK_TOOL_SURFACE_VIOLATION/);
|
|
1891
|
-
await assert.rejects(async ()=>await assemble({
|
|
1892
|
-
tools: []
|
|
1893
|
-
}, {
|
|
1894
|
-
agent: isolatedAgent,
|
|
1895
|
-
scope: isolatedAgent
|
|
1896
|
-
}, async ()=>({
|
|
1897
|
-
tools: [
|
|
1898
|
-
{
|
|
1899
|
-
...exactSchema,
|
|
1900
|
-
description: `${exactSchema.description ?? ''} tampered`
|
|
1901
|
-
}
|
|
1902
|
-
]
|
|
1903
|
-
})), /BENCHMARK_TOOL_SURFACE_VIOLATION/, 'final assembly rejects same-name schema mutation');
|
|
1904
|
-
const diagnosticAssembly = {
|
|
1905
|
-
tools: [
|
|
1906
|
-
{
|
|
1907
|
-
name: 'diagnostic_tool'
|
|
1908
|
-
}
|
|
1909
|
-
]
|
|
1910
|
-
};
|
|
1911
|
-
assert.equal(await assemble({
|
|
1912
|
-
tools: []
|
|
1913
|
-
}, {
|
|
1914
|
-
scope: {
|
|
1915
|
-
id: 'diagnostic-scope'
|
|
1916
|
-
}
|
|
1917
|
-
}, async ()=>diagnosticAssembly), diagnosticAssembly, 'non-agent diagnostic assembly passes through');
|
|
1918
|
-
await assert.rejects(async ()=>await assemble({
|
|
1919
|
-
tools: []
|
|
1920
|
-
}, {
|
|
1921
|
-
agent: isolatedAgent,
|
|
1922
|
-
scope: {
|
|
1923
|
-
id: 'wrong-scope'
|
|
1924
|
-
}
|
|
1925
|
-
}, nextExact), /BENCHMARK_TOOL_SURFACE_VIOLATION/, 'agent assembly requires the same agent and scope');
|
|
856
|
+
const nextExact = async () => ({ tools: [exactSchema] });
|
|
857
|
+
assert.deepEqual(await assemble({ tools: [] }, { agent: isolatedAgent, scope: isolatedAgent }, nextExact), { tools: [exactSchema] }, 'legal final assembly passes unchanged');
|
|
858
|
+
await assert.rejects(async () => await assemble({ tools: [] }, { agent: isolatedAgent, scope: isolatedAgent }, async () => ({ tools: [exactSchema, { name: 'own_side_effect' }] })), /BENCHMARK_TOOL_SURFACE_VIOLATION/);
|
|
859
|
+
await assert.rejects(async () => await assemble({ tools: [] }, { agent: isolatedAgent, scope: isolatedAgent }, async () => ({ tools: [{ ...exactSchema, description: `${exactSchema.description ?? ''} tampered` }] })), /BENCHMARK_TOOL_SURFACE_VIOLATION/, 'final assembly rejects same-name schema mutation');
|
|
860
|
+
const diagnosticAssembly = { tools: [{ name: 'diagnostic_tool' }] };
|
|
861
|
+
assert.equal(await assemble({ tools: [] }, { scope: { id: 'diagnostic-scope' } }, async () => diagnosticAssembly), diagnosticAssembly, 'non-agent diagnostic assembly passes through');
|
|
862
|
+
await assert.rejects(async () => await assemble({ tools: [] }, { agent: isolatedAgent, scope: { id: 'wrong-scope' } }, nextExact), /BENCHMARK_TOOL_SURFACE_VIOLATION/, 'agent assembly requires the same agent and scope');
|
|
1926
863
|
const preStep = preStepListeners[0];
|
|
1927
|
-
assert.deepEqual(await preStep({
|
|
1928
|
-
|
|
1929
|
-
}, async ()=>({
|
|
1930
|
-
decision: 'continue'
|
|
1931
|
-
})), {
|
|
1932
|
-
decision: 'continue'
|
|
1933
|
-
}, 'legal pre-step decision passes unchanged');
|
|
1934
|
-
scopedExtraSchemas.set(isolatedAgent, [
|
|
1935
|
-
{
|
|
1936
|
-
name: 'own_side_effect'
|
|
1937
|
-
}
|
|
1938
|
-
]);
|
|
1939
|
-
await assert.rejects(async ()=>await preStep({
|
|
1940
|
-
agent: isolatedAgent
|
|
1941
|
-
}, async ()=>({
|
|
1942
|
-
decision: 'continue'
|
|
1943
|
-
})), /BENCHMARK_TOOL_SURFACE_VIOLATION/, 'downstream pre-step scope expansion fails closed');
|
|
864
|
+
assert.deepEqual(await preStep({ agent: isolatedAgent }, async () => ({ decision: 'continue' })), { decision: 'continue' }, 'legal pre-step decision passes unchanged');
|
|
865
|
+
scopedExtraSchemas.set(isolatedAgent, [{ name: 'own_side_effect' }]);
|
|
866
|
+
await assert.rejects(async () => await preStep({ agent: isolatedAgent }, async () => ({ decision: 'continue' })), /BENCHMARK_TOOL_SURFACE_VIOLATION/, 'downstream pre-step scope expansion fails closed');
|
|
1944
867
|
scopedExtraSchemas.delete(isolatedAgent);
|
|
1945
868
|
shadowedAgent = isolatedAgent;
|
|
1946
|
-
await assert.rejects(async ()=>await assemble({
|
|
1947
|
-
tools: []
|
|
1948
|
-
}, {
|
|
1949
|
-
agent: isolatedAgent,
|
|
1950
|
-
scope: isolatedAgent
|
|
1951
|
-
}, nextExact), /BENCHMARK_TOOL_SURFACE_VIOLATION/, 'same-name scoped shadow fails final assembly identity check');
|
|
869
|
+
await assert.rejects(async () => await assemble({ tools: [] }, { agent: isolatedAgent, scope: isolatedAgent }, nextExact), /BENCHMARK_TOOL_SURFACE_VIOLATION/, 'same-name scoped shadow fails final assembly identity check');
|
|
1952
870
|
shadowedAgent = undefined;
|
|
1953
871
|
assert.equal(guards.length, 2, 'agent scope installs exact isolation and conformance guards');
|
|
1954
872
|
const guard = guards[0];
|
|
1955
|
-
const originalAgent = {
|
|
1956
|
-
|
|
1957
|
-
tools: {
|
|
1958
|
-
get: (name)=>name === 'gotry_benchmark_environment' ? visibleBridge : undefined
|
|
1959
|
-
}
|
|
1960
|
-
}
|
|
1961
|
-
};
|
|
1962
|
-
const shadowAgent = {
|
|
1963
|
-
ctx: {
|
|
1964
|
-
tools: {
|
|
1965
|
-
get: (name)=>name === 'gotry_benchmark_environment' ? {
|
|
1966
|
-
name
|
|
1967
|
-
} : undefined
|
|
1968
|
-
}
|
|
1969
|
-
}
|
|
1970
|
-
};
|
|
873
|
+
const originalAgent = { ctx: { tools: { get: (name) => name === 'gotry_benchmark_environment' ? visibleBridge : undefined } } };
|
|
874
|
+
const shadowAgent = { ctx: { tools: { get: (name) => name === 'gotry_benchmark_environment' ? { name } : undefined } } };
|
|
1971
875
|
shadowedAgent = shadowAgent;
|
|
1972
|
-
assert.equal(guard({
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
|
|
1976
|
-
}), undefined, 'original bridge definition is allowed');
|
|
1977
|
-
assert.equal(guard({
|
|
1978
|
-
name: 'gotry_benchmark_environment',
|
|
1979
|
-
args: {},
|
|
1980
|
-
agent: shadowAgent
|
|
1981
|
-
}), 'BENCHMARK_TOOL_NOT_ALLOWED', 'same-name shadow definition is denied');
|
|
1982
|
-
assert.equal(guard({
|
|
1983
|
-
name: 'gotry_benchmark_environment',
|
|
1984
|
-
args: {
|
|
1985
|
-
path: '/private/secret'
|
|
1986
|
-
},
|
|
1987
|
-
agent: isolatedAgent
|
|
1988
|
-
}), undefined, 'bridge tool is allowed');
|
|
1989
|
-
const denied = guard({
|
|
1990
|
-
name: 'other_tool',
|
|
1991
|
-
args: {
|
|
1992
|
-
path: '/private/secret',
|
|
1993
|
-
token: 'secret'
|
|
1994
|
-
}
|
|
1995
|
-
});
|
|
876
|
+
assert.equal(guard({ name: 'gotry_benchmark_environment', args: {}, agent: originalAgent }), undefined, 'original bridge definition is allowed');
|
|
877
|
+
assert.equal(guard({ name: 'gotry_benchmark_environment', args: {}, agent: shadowAgent }), 'BENCHMARK_TOOL_NOT_ALLOWED', 'same-name shadow definition is denied');
|
|
878
|
+
assert.equal(guard({ name: 'gotry_benchmark_environment', args: { path: '/private/secret' }, agent: isolatedAgent }), undefined, 'bridge tool is allowed');
|
|
879
|
+
const denied = guard({ name: 'other_tool', args: { path: '/private/secret', token: 'secret' } });
|
|
1996
880
|
assert.equal(denied, 'BENCHMARK_TOOL_NOT_ALLOWED', 'non-bridge tools are denied without argument/path echo');
|
|
1997
881
|
assert.equal(denied?.includes('/private/secret'), false);
|
|
1998
882
|
assert.equal(denied?.includes('secret'), false);
|
|
1999
|
-
assert.equal(guard({
|
|
2000
|
-
|
|
2001
|
-
|
|
2002
|
-
|
|
2003
|
-
|
|
2004
|
-
}),
|
|
2005
|
-
assert.throws(()=>
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
void check;
|
|
2014
|
-
}
|
|
2015
|
-
}
|
|
2016
|
-
}
|
|
2017
|
-
}
|
|
2018
|
-
}), /presentAs/, 'missing scoped presentAs fails hard');
|
|
2019
|
-
assert.throws(()=>agentCreatedListeners[0]({
|
|
2020
|
-
agent: {
|
|
2021
|
-
ctx: {
|
|
2022
|
-
tools: scopedTools
|
|
2023
|
-
}
|
|
2024
|
-
}
|
|
2025
|
-
}), /effect/, 'missing scoped effect fails hard');
|
|
2026
|
-
assert.throws(()=>agentCreatedListeners[0]({
|
|
2027
|
-
agent: {
|
|
2028
|
-
ctx: {
|
|
2029
|
-
tools: scopedTools,
|
|
2030
|
-
effect: (action)=>runEffect(action)
|
|
2031
|
-
}
|
|
2032
|
-
}
|
|
2033
|
-
}), /event bus/, 'missing scoped event bus fails hard');
|
|
2034
|
-
assert.throws(()=>agentCreatedListeners[0]({
|
|
2035
|
-
agent: {
|
|
2036
|
-
ctx: {
|
|
2037
|
-
tools: {
|
|
2038
|
-
guard: guards[0]
|
|
2039
|
-
}
|
|
2040
|
-
}
|
|
2041
|
-
}
|
|
2042
|
-
}), /restrict/, 'missing scoped restrict fails hard');
|
|
2043
|
-
assert.throws(()=>agentCreatedListeners[0]({
|
|
2044
|
-
agent: {
|
|
2045
|
-
ctx: {
|
|
2046
|
-
tools: {
|
|
2047
|
-
restrict (filter) {
|
|
2048
|
-
void filter;
|
|
2049
|
-
}
|
|
2050
|
-
}
|
|
2051
|
-
}
|
|
2052
|
-
}
|
|
2053
|
-
}), /guard/, 'missing scoped guard fails hard');
|
|
2054
|
-
assert.throws(()=>installBenchmarkToolIsolation({
|
|
2055
|
-
tools: {
|
|
2056
|
-
get (name) {
|
|
2057
|
-
return name === 'gotry_benchmark_environment' ? {
|
|
2058
|
-
name: 'gotry_benchmark_environment'
|
|
2059
|
-
} : undefined;
|
|
2060
|
-
},
|
|
2061
|
-
schemas () {
|
|
2062
|
-
return [
|
|
2063
|
-
{
|
|
2064
|
-
name: 'gotry_benchmark_environment'
|
|
2065
|
-
}
|
|
2066
|
-
];
|
|
2067
|
-
}
|
|
2068
|
-
},
|
|
2069
|
-
agents: {
|
|
2070
|
-
list () {
|
|
2071
|
-
return [];
|
|
2072
|
-
}
|
|
2073
|
-
},
|
|
2074
|
-
on () {
|
|
2075
|
-
return ()=>{};
|
|
2076
|
-
}
|
|
2077
|
-
}), /effect/, 'missing ctx.effect fails hard');
|
|
2078
|
-
await disposedListeners[0]({
|
|
2079
|
-
agent: isolatedAgent
|
|
2080
|
-
});
|
|
2081
|
-
assert.deepEqual(cleanupCounts, {
|
|
2082
|
-
restrict: 1,
|
|
2083
|
-
guard: 1,
|
|
2084
|
-
presentAs: 1,
|
|
2085
|
-
assembly: 1
|
|
2086
|
-
}, 'agent disposal releases every scoped isolation effect exactly once');
|
|
2087
|
-
await disposedListeners[1]({
|
|
2088
|
-
agent: isolatedAgent
|
|
2089
|
-
});
|
|
883
|
+
assert.equal(guard({ name: 'other_tool', args: { different: true } }), denied, 'denial reason is stable');
|
|
884
|
+
assert.throws(() => agentCreatedListeners[0]({ agent: { ctx: { tools: { restrict(filter) { void filter; }, guard(check) { void check; } } } } }), /presentAs/, 'missing scoped presentAs fails hard');
|
|
885
|
+
assert.throws(() => agentCreatedListeners[0]({ agent: { ctx: { tools: scopedTools } } }), /effect/, 'missing scoped effect fails hard');
|
|
886
|
+
assert.throws(() => agentCreatedListeners[0]({ agent: { ctx: { tools: scopedTools, effect: (action) => runEffect(action) } } }), /event bus/, 'missing scoped event bus fails hard');
|
|
887
|
+
assert.throws(() => agentCreatedListeners[0]({ agent: { ctx: { tools: { guard: guards[0] } } } }), /restrict/, 'missing scoped restrict fails hard');
|
|
888
|
+
assert.throws(() => agentCreatedListeners[0]({ agent: { ctx: { tools: { restrict(filter) { void filter; } } } } }), /guard/, 'missing scoped guard fails hard');
|
|
889
|
+
assert.throws(() => installBenchmarkToolIsolation({
|
|
890
|
+
tools: { get(name) { return name === 'gotry_benchmark_environment' ? { name: 'gotry_benchmark_environment' } : undefined; }, schemas() { return [{ name: 'gotry_benchmark_environment' }]; } },
|
|
891
|
+
agents: { list() { return []; } },
|
|
892
|
+
on() { return () => { }; },
|
|
893
|
+
}), /effect/, 'missing ctx.effect fails hard');
|
|
894
|
+
await disposedListeners[0]({ agent: isolatedAgent });
|
|
895
|
+
assert.deepEqual(cleanupCounts, { restrict: 1, guard: 1, presentAs: 1, assembly: 1 }, 'agent disposal releases every scoped isolation effect exactly once');
|
|
896
|
+
await disposedListeners[1]({ agent: isolatedAgent });
|
|
2090
897
|
await isolatedAgentEffects[1]();
|
|
2091
|
-
assert.deepEqual(cleanupCounts, {
|
|
2092
|
-
|
|
2093
|
-
guard: 2,
|
|
2094
|
-
presentAs: 1,
|
|
2095
|
-
assembly: 2
|
|
2096
|
-
}, 'agent-scope disposal also releases conformance guard and prompt section');
|
|
2097
|
-
const secondCleanup = {
|
|
2098
|
-
restrict: 0,
|
|
2099
|
-
guard: 0,
|
|
2100
|
-
presentAs: 0,
|
|
2101
|
-
assembly: 0
|
|
2102
|
-
};
|
|
898
|
+
assert.deepEqual(cleanupCounts, { restrict: 1, guard: 2, presentAs: 1, assembly: 2 }, 'agent-scope disposal also releases conformance guard and prompt section');
|
|
899
|
+
const secondCleanup = { restrict: 0, guard: 0, presentAs: 0, assembly: 0 };
|
|
2103
900
|
const secondGuards = [];
|
|
2104
901
|
const secondScopedAssemblyListeners = [];
|
|
2105
902
|
const secondTools = {
|
|
2106
|
-
restrict () {
|
|
2107
|
-
|
|
2108
|
-
|
|
2109
|
-
};
|
|
2110
|
-
},
|
|
2111
|
-
guard (check) {
|
|
2112
|
-
secondGuards.push(check);
|
|
2113
|
-
return ()=>{
|
|
2114
|
-
secondCleanup.guard += 1;
|
|
2115
|
-
};
|
|
2116
|
-
},
|
|
2117
|
-
presentAs () {
|
|
2118
|
-
return ()=>{
|
|
2119
|
-
secondCleanup.presentAs += 1;
|
|
2120
|
-
};
|
|
2121
|
-
}
|
|
903
|
+
restrict() { return () => { secondCleanup.restrict += 1; }; },
|
|
904
|
+
guard(check) { secondGuards.push(check); return () => { secondCleanup.guard += 1; }; },
|
|
905
|
+
presentAs() { return () => { secondCleanup.presentAs += 1; }; },
|
|
2122
906
|
};
|
|
2123
907
|
const secondAgentEffects = [];
|
|
2124
|
-
const secondAgent = {
|
|
2125
|
-
session: {},
|
|
2126
|
-
steer (_message) {},
|
|
2127
|
-
ctx: {
|
|
908
|
+
const secondAgent = { session: {}, steer(_message) { }, ctx: {
|
|
2128
909
|
tools: secondTools,
|
|
2129
|
-
effect: (action)=>{
|
|
910
|
+
effect: (action) => {
|
|
2130
911
|
const dispose = runEffect(action);
|
|
2131
912
|
secondAgentEffects.push(dispose);
|
|
2132
913
|
return dispose;
|
|
2133
914
|
},
|
|
2134
|
-
on: (_event, listener)=>{
|
|
915
|
+
on: (_event, listener) => {
|
|
2135
916
|
secondScopedAssemblyListeners.push(listener);
|
|
2136
|
-
return ()=>{
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
}
|
|
2141
|
-
};
|
|
2142
|
-
for (const listener of agentCreatedListeners)listener({
|
|
2143
|
-
agent: secondAgent
|
|
2144
|
-
});
|
|
917
|
+
return () => { secondCleanup.assembly += 1; };
|
|
918
|
+
},
|
|
919
|
+
} };
|
|
920
|
+
for (const listener of agentCreatedListeners)
|
|
921
|
+
listener({ agent: secondAgent });
|
|
2145
922
|
let inFlightNextCalls = 0;
|
|
2146
|
-
await assert.rejects(async ()=>await secondScopedAssemblyListeners[0]({}, {
|
|
2147
|
-
|
|
2148
|
-
|
|
2149
|
-
|
|
2150
|
-
|
|
2151
|
-
await disposeRootIsolation();
|
|
2152
|
-
return {
|
|
2153
|
-
tools: [
|
|
2154
|
-
exactSchema
|
|
2155
|
-
]
|
|
2156
|
-
};
|
|
2157
|
-
}), /BENCHMARK_TOOL_SURFACE_VIOLATION/, 'plugin unload during assembly fails closed before returning a model-visible result');
|
|
923
|
+
await assert.rejects(async () => await secondScopedAssemblyListeners[0]({}, { agent: secondAgent, scope: secondAgent }, async () => {
|
|
924
|
+
inFlightNextCalls += 1;
|
|
925
|
+
await disposeRootIsolation();
|
|
926
|
+
return { tools: [exactSchema] };
|
|
927
|
+
}), /BENCHMARK_TOOL_SURFACE_VIOLATION/, 'plugin unload during assembly fails closed before returning a model-visible result');
|
|
2158
928
|
assert.equal(inFlightNextCalls, 1, 'in-flight quarantine test reaches the controlled unload point');
|
|
2159
|
-
assert.deepEqual(secondCleanup, {
|
|
2160
|
-
restrict: 0,
|
|
2161
|
-
guard: 0,
|
|
2162
|
-
presentAs: 0,
|
|
2163
|
-
assembly: 0
|
|
2164
|
-
}, 'plugin unload keeps a live agent quarantined');
|
|
929
|
+
assert.deepEqual(secondCleanup, { restrict: 0, guard: 0, presentAs: 0, assembly: 0 }, 'plugin unload keeps a live agent quarantined');
|
|
2165
930
|
let quarantineNextCalls = 0;
|
|
2166
|
-
await assert.rejects(async ()=>await secondScopedAssemblyListeners[0]({}, {
|
|
2167
|
-
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
quarantineNextCalls += 1;
|
|
2171
|
-
return {
|
|
2172
|
-
tools: [
|
|
2173
|
-
exactSchema
|
|
2174
|
-
]
|
|
2175
|
-
};
|
|
2176
|
-
}), /BENCHMARK_TOOL_SURFACE_VIOLATION/, 'plugin-unloaded live agent rejects assembly before model request');
|
|
931
|
+
await assert.rejects(async () => await secondScopedAssemblyListeners[0]({}, { agent: secondAgent, scope: secondAgent }, async () => {
|
|
932
|
+
quarantineNextCalls += 1;
|
|
933
|
+
return { tools: [exactSchema] };
|
|
934
|
+
}), /BENCHMARK_TOOL_SURFACE_VIOLATION/, 'plugin-unloaded live agent rejects assembly before model request');
|
|
2177
935
|
assert.equal(quarantineNextCalls, 0, 'quarantine does not enter the remaining assembly chain');
|
|
2178
|
-
assert.equal(secondGuards[0]({
|
|
2179
|
-
|
|
2180
|
-
|
|
2181
|
-
}),
|
|
2182
|
-
|
|
2183
|
-
|
|
2184
|
-
|
|
2185
|
-
|
|
2186
|
-
assert.deepEqual(cleanupCounts, {
|
|
2187
|
-
restrict: 1,
|
|
2188
|
-
guard: 2,
|
|
2189
|
-
presentAs: 1,
|
|
2190
|
-
assembly: 2
|
|
2191
|
-
}, 'plugin unload does not double-dispose an already removed agent');
|
|
2192
|
-
assert.throws(()=>agentCreatedListeners[0]({
|
|
2193
|
-
agent: secondAgent
|
|
2194
|
-
}), /BENCHMARK_TOOL_SURFACE_VIOLATION/, 'agent creation during plugin stop fails closed');
|
|
2195
|
-
for (const dispose of secondAgentEffects)await dispose();
|
|
2196
|
-
assert.deepEqual(secondCleanup, {
|
|
2197
|
-
restrict: 1,
|
|
2198
|
-
guard: 2,
|
|
2199
|
-
presentAs: 1,
|
|
2200
|
-
assembly: 2
|
|
2201
|
-
}, 'agent disposal releases its quarantined isolation and conformance effects');
|
|
2202
|
-
const bridge = registered.find((tool)=>tool.name === 'gotry_benchmark_environment');
|
|
936
|
+
assert.equal(secondGuards[0]({ name: 'other_tool', agent: secondAgent }), 'BENCHMARK_TOOL_NOT_ALLOWED', 'quarantined agent still denies non-bridge dispatch after plugin unload');
|
|
937
|
+
assert.equal(secondGuards[0]({ name: 'gotry_benchmark_environment', agent: secondAgent }), 'BENCHMARK_TOOL_NOT_ALLOWED', 'quarantined agent also denies bridge dispatch after plugin unload');
|
|
938
|
+
assert.deepEqual(cleanupCounts, { restrict: 1, guard: 2, presentAs: 1, assembly: 2 }, 'plugin unload does not double-dispose an already removed agent');
|
|
939
|
+
assert.throws(() => agentCreatedListeners[0]({ agent: secondAgent }), /BENCHMARK_TOOL_SURFACE_VIOLATION/, 'agent creation during plugin stop fails closed');
|
|
940
|
+
for (const dispose of secondAgentEffects)
|
|
941
|
+
await dispose();
|
|
942
|
+
assert.deepEqual(secondCleanup, { restrict: 1, guard: 2, presentAs: 1, assembly: 2 }, 'agent disposal releases its quarantined isolation and conformance effects');
|
|
943
|
+
const bridge = registered.find(tool => tool.name === 'gotry_benchmark_environment');
|
|
2203
944
|
assert.ok(bridge.execute, 'registered bridge exposes execute');
|
|
2204
945
|
const parameters = bridge.parameters;
|
|
2205
946
|
assert.equal(parameters.type, 'object', 'flat bridge uses an object root for model-facing parameters');
|
|
2206
947
|
assert.equal(parameters.oneOf, undefined, 'model-facing wire has no top-level oneOf');
|
|
2207
|
-
assert.deepEqual(parameters.required, [
|
|
2208
|
-
'action'
|
|
2209
|
-
], 'action is the only universally required wire field');
|
|
948
|
+
assert.deepEqual(parameters.required, ['action'], 'action is the only universally required wire field');
|
|
2210
949
|
assert.equal(parameters.additionalProperties, false);
|
|
2211
|
-
assert.deepEqual(parameters.properties?.action?.enum, [
|
|
2212
|
-
|
|
2213
|
-
'call',
|
|
2214
|
-
'errors'
|
|
2215
|
-
]);
|
|
2216
|
-
assert.deepEqual(parameters.properties?.tool?.enum, [
|
|
2217
|
-
'lookup',
|
|
2218
|
-
'constructor',
|
|
2219
|
-
'toString'
|
|
2220
|
-
], 'tool enum exposes frozen descriptor names');
|
|
950
|
+
assert.deepEqual(parameters.properties?.action?.enum, ['tools', 'call', 'errors']);
|
|
951
|
+
assert.deepEqual(parameters.properties?.tool?.enum, ['lookup', 'constructor', 'toString'], 'tool enum exposes frozen descriptor names');
|
|
2221
952
|
assert.equal(parameters.properties?.arguments?.type, 'object', 'arguments remains a generic object on the open wire');
|
|
2222
953
|
assert.equal(parameters.properties?.arguments?.additionalProperties, true);
|
|
2223
954
|
assert.equal(Object.isFrozen(bridge.parameters), true, 'registered raw parameters are frozen');
|
|
2224
955
|
assert.equal(Object.isFrozen(parameters.properties?.tool?.enum), true, 'descriptor-derived tool enum is frozen');
|
|
2225
|
-
assert.throws(()=>{
|
|
2226
|
-
parameters.properties?.tool?.enum.push('escape');
|
|
2227
|
-
}, TypeError);
|
|
956
|
+
assert.throws(() => { parameters.properties?.tool?.enum.push('escape'); }, TypeError);
|
|
2228
957
|
const beforeProtocolRejected = spawnSpecs.length;
|
|
2229
|
-
await assert.rejects(()=>bridge.execute({
|
|
2230
|
-
|
|
2231
|
-
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
await assert.rejects(()=>bridge.execute({}, null), /invalid arguments/, '
|
|
2235
|
-
await assert.rejects(()=>bridge.execute({
|
|
2236
|
-
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
await assert.rejects(()=>bridge.execute({
|
|
2240
|
-
|
|
2241
|
-
|
|
2242
|
-
}, null), /invalid arguments/,
|
|
2243
|
-
await assert.rejects(()=>bridge.execute({
|
|
2244
|
-
action: 'errors',
|
|
2245
|
-
tool: 'lookup'
|
|
2246
|
-
}, null), /invalid arguments/, 'errors rejects mixed tool fields');
|
|
2247
|
-
await assert.rejects(()=>bridge.execute({
|
|
2248
|
-
action: 'errors',
|
|
2249
|
-
arguments: {}
|
|
2250
|
-
}, null), /invalid arguments/, 'errors rejects mixed arguments');
|
|
2251
|
-
await assert.rejects(()=>bridge.execute({
|
|
2252
|
-
...bridgeCall('lookup', {
|
|
2253
|
-
city: 'Dubai'
|
|
2254
|
-
}),
|
|
2255
|
-
extra: true
|
|
2256
|
-
}, null), /invalid arguments/, 'extra top-level fields are rejected by the flat protocol');
|
|
2257
|
-
await assert.rejects(()=>bridge.execute({
|
|
2258
|
-
action: 'call'
|
|
2259
|
-
}, null), /invalid arguments/, 'call without tool or arguments is rejected');
|
|
2260
|
-
await assert.rejects(()=>bridge.execute({
|
|
2261
|
-
action: 'call',
|
|
2262
|
-
tool: 'lookup'
|
|
2263
|
-
}, null), /invalid arguments/, 'call without arguments is rejected');
|
|
2264
|
-
await assert.rejects(()=>bridge.execute({
|
|
2265
|
-
action: 'call',
|
|
2266
|
-
arguments: {
|
|
2267
|
-
city: 'Dubai'
|
|
2268
|
-
}
|
|
2269
|
-
}, null), /invalid arguments/, 'call without tool is rejected');
|
|
2270
|
-
await assert.rejects(()=>bridge.execute(bridgeCall('lookup', {
|
|
2271
|
-
city: 'Sharjah'
|
|
2272
|
-
}), null), /invalid arguments/, 'invalid descriptor argument values are rejected by the flat protocol');
|
|
2273
|
-
await assert.rejects(()=>bridge.execute({
|
|
2274
|
-
action: 'call',
|
|
2275
|
-
tool: 'lookup',
|
|
2276
|
-
arguments: {}
|
|
2277
|
-
}, null), /invalid arguments/, 'missing required descriptor arguments are rejected by the flat protocol');
|
|
2278
|
-
for (const tool of [
|
|
2279
|
-
'lookup',
|
|
2280
|
-
'constructor',
|
|
2281
|
-
'toString'
|
|
2282
|
-
]){
|
|
2283
|
-
await assert.rejects(()=>bridge.execute(bridgeCall(tool, {
|
|
2284
|
-
city: 'Sharjah'
|
|
2285
|
-
}), null), /invalid arguments/, `${tool} rejects invalid descriptor arguments`);
|
|
958
|
+
await assert.rejects(() => bridge.execute({ query: bridgeCall('lookup', { city: 'Dubai' }) }, null), /invalid arguments/, 'legacy nested query envelope is rejected by the flat protocol');
|
|
959
|
+
await assert.rejects(() => bridge.execute({}, null), /invalid arguments/, 'empty object is rejected by the flat protocol');
|
|
960
|
+
await assert.rejects(() => bridge.execute({ action: 'tools', tool: 'lookup' }, null), /invalid arguments/, 'mixed action/tool fields are rejected by the flat protocol');
|
|
961
|
+
await assert.rejects(() => bridge.execute({ action: 'tools', arguments: {} }, null), /invalid arguments/, 'tools rejects mixed arguments');
|
|
962
|
+
await assert.rejects(() => bridge.execute({ action: 'errors', tool: 'lookup' }, null), /invalid arguments/, 'errors rejects mixed tool fields');
|
|
963
|
+
await assert.rejects(() => bridge.execute({ action: 'errors', arguments: {} }, null), /invalid arguments/, 'errors rejects mixed arguments');
|
|
964
|
+
await assert.rejects(() => bridge.execute({ ...bridgeCall('lookup', { city: 'Dubai' }), extra: true }, null), /invalid arguments/, 'extra top-level fields are rejected by the flat protocol');
|
|
965
|
+
await assert.rejects(() => bridge.execute({ action: 'call' }, null), /invalid arguments/, 'call without tool or arguments is rejected');
|
|
966
|
+
await assert.rejects(() => bridge.execute({ action: 'call', tool: 'lookup' }, null), /invalid arguments/, 'call without arguments is rejected');
|
|
967
|
+
await assert.rejects(() => bridge.execute({ action: 'call', arguments: { city: 'Dubai' } }, null), /invalid arguments/, 'call without tool is rejected');
|
|
968
|
+
await assert.rejects(() => bridge.execute(bridgeCall('lookup', { city: 'Sharjah' }), null), /invalid arguments/, 'invalid descriptor argument values are rejected by the flat protocol');
|
|
969
|
+
await assert.rejects(() => bridge.execute({ action: 'call', tool: 'lookup', arguments: {} }, null), /invalid arguments/, 'missing required descriptor arguments are rejected by the flat protocol');
|
|
970
|
+
for (const tool of ['lookup', 'constructor', 'toString']) {
|
|
971
|
+
await assert.rejects(() => bridge.execute(bridgeCall(tool, { city: 'Sharjah' }), null), /invalid arguments/, `${tool} rejects invalid descriptor arguments`);
|
|
2286
972
|
}
|
|
2287
973
|
assert.equal(spawnSpecs.length, beforeProtocolRejected, 'flat protocol rejections happen before spawn');
|
|
2288
|
-
const args = {
|
|
2289
|
-
city: 'Dubai',
|
|
2290
|
-
payload: '$(touch /tmp/nope)'
|
|
2291
|
-
};
|
|
974
|
+
const args = { city: 'Dubai', payload: '$(touch /tmp/nope)' };
|
|
2292
975
|
const result = await bridge.execute(bridgeCall('lookup', args), null);
|
|
2293
976
|
assert.deepEqual(spawnSpecs[0]?.argv, [
|
|
2294
|
-
process.execPath,
|
|
2295
|
-
'-m',
|
|
2296
|
-
'agent_env.cli',
|
|
2297
|
-
'--lang',
|
|
2298
|
-
'en',
|
|
2299
|
-
'call',
|
|
2300
|
-
'lookup',
|
|
2301
|
-
JSON.stringify(args)
|
|
977
|
+
process.execPath, '-m', 'agent_env.cli', '--lang', 'en', 'call', 'lookup', JSON.stringify(args),
|
|
2302
978
|
], 'call uses only the configured executable/prefix and fixed lookup subcommand argv');
|
|
2303
979
|
assert.equal(spawnSpecs[0]?.cwd, root, 'call uses configured cwd');
|
|
2304
980
|
assert.equal(spawnSpecs[0]?.stdio.stdin, 'ignore', 'call never exposes stdin');
|
|
@@ -2306,697 +982,207 @@ try {
|
|
|
2306
982
|
assert.equal(spawnSpecs[0]?.stdio.stderr.maxBytes, 4_096, 'stderr cap comes from config');
|
|
2307
983
|
assert.ok((spawnSpecs[0]?.graceMs ?? 0) > 0 && (spawnSpecs[0]?.graceMs ?? Infinity) <= 1_000, 'graceMs is bounded');
|
|
2308
984
|
assert.equal(spawnSpecs[0]?.env?.GOTRY_BENCHMARK_BRIDGE_PARENT_SENTINEL, undefined, 'parent sentinel is not inherited');
|
|
2309
|
-
for (const key of [
|
|
2310
|
-
'DATABASE_URL',
|
|
2311
|
-
'SSH_AUTH_SOCK',
|
|
2312
|
-
'AWS_PROFILE',
|
|
2313
|
-
'HTTPS_PROXY',
|
|
2314
|
-
'GOTRY_BENCHMARK_BRIDGE_PARENT_SENTINEL'
|
|
2315
|
-
]){
|
|
985
|
+
for (const key of ['DATABASE_URL', 'SSH_AUTH_SOCK', 'AWS_PROFILE', 'HTTPS_PROXY', 'GOTRY_BENCHMARK_BRIDGE_PARENT_SENTINEL']) {
|
|
2316
986
|
assert.equal(spawnSpecs[0]?.env?.[key], undefined, `${key} is tombstoned in bridge subprocess env`);
|
|
2317
987
|
}
|
|
2318
988
|
assert.equal(spawnSpecs[0]?.env?.PYTHONDONTWRITEBYTECODE, '1');
|
|
2319
989
|
assert.equal(spawnSpecs[0]?.env?.PYTHONNOUSERSITE, '1');
|
|
2320
|
-
assert.deepEqual(result, {
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
});
|
|
2333
|
-
|
|
2334
|
-
|
|
2335
|
-
}
|
|
2336
|
-
|
|
2337
|
-
|
|
2338
|
-
|
|
2339
|
-
city: 'Dubai'
|
|
2340
|
-
}
|
|
2341
|
-
}, 'allowed tool without a positive mapping retains the recursive denylist only');
|
|
2342
|
-
outcomes.push({
|
|
2343
|
-
stdout: okEnvelope({
|
|
2344
|
-
legacy: 'value'
|
|
2345
|
-
})
|
|
2346
|
-
});
|
|
2347
|
-
const legacyResult = await bridge.execute(bridgeCall('constructor', {
|
|
2348
|
-
city: 'Dubai'
|
|
2349
|
-
}), null);
|
|
2350
|
-
assert.deepEqual(legacyResult, {
|
|
2351
|
-
ok: true,
|
|
2352
|
-
result: {
|
|
2353
|
-
legacy: 'value'
|
|
2354
|
-
}
|
|
2355
|
-
}, 'mapped constructor accepts its declared positive key');
|
|
2356
|
-
outcomes.push({
|
|
2357
|
-
stdout: okEnvelope({
|
|
2358
|
-
city: 'Dubai'
|
|
2359
|
-
})
|
|
2360
|
-
});
|
|
2361
|
-
const constructorUnexpected = await bridge.execute(bridgeCall('constructor', {
|
|
2362
|
-
city: 'Dubai'
|
|
2363
|
-
}), null);
|
|
2364
|
-
assert.deepEqual(constructorUnexpected, {
|
|
2365
|
-
ok: false,
|
|
2366
|
-
error: 'forbidden_output'
|
|
2367
|
-
}, 'mapped constructor rejects undeclared positive keys');
|
|
2368
|
-
outcomes.push({
|
|
2369
|
-
stdout: okEnvelope({
|
|
2370
|
-
nested: {
|
|
2371
|
-
city: 'Dubai'
|
|
2372
|
-
}
|
|
2373
|
-
})
|
|
2374
|
-
});
|
|
2375
|
-
const nestedAllowedResult = await bridge.execute(bridgeCall('lookup', {
|
|
2376
|
-
city: 'Dubai'
|
|
2377
|
-
}), null);
|
|
2378
|
-
assert.deepEqual(nestedAllowedResult, {
|
|
2379
|
-
ok: true,
|
|
2380
|
-
result: {
|
|
2381
|
-
nested: {
|
|
2382
|
-
city: 'Dubai'
|
|
2383
|
-
}
|
|
2384
|
-
}
|
|
2385
|
-
}, 'configured positive output allowlist accepts declared nested keys');
|
|
2386
|
-
outcomes.push({
|
|
2387
|
-
stdout: okEnvelope({
|
|
2388
|
-
city: 'Dubai',
|
|
2389
|
-
unexpected: 'secret'
|
|
2390
|
-
})
|
|
2391
|
-
});
|
|
2392
|
-
const unexpectedResult = await bridge.execute(bridgeCall('lookup', {
|
|
2393
|
-
city: 'Dubai'
|
|
2394
|
-
}), null);
|
|
2395
|
-
assert.deepEqual(unexpectedResult, {
|
|
2396
|
-
ok: false,
|
|
2397
|
-
error: 'forbidden_output'
|
|
2398
|
-
}, 'configured positive output allowlist rejects unexpected keys without reflecting them');
|
|
2399
|
-
outcomes.push({
|
|
2400
|
-
stdout: okEnvelope([
|
|
2401
|
-
{
|
|
2402
|
-
city: 'Dubai',
|
|
2403
|
-
nested: {
|
|
2404
|
-
unexpected: 'secret'
|
|
2405
|
-
}
|
|
2406
|
-
}
|
|
2407
|
-
])
|
|
2408
|
-
});
|
|
2409
|
-
const nestedUnexpectedResult = await bridge.execute(bridgeCall('lookup', {
|
|
2410
|
-
city: 'Dubai'
|
|
2411
|
-
}), null);
|
|
2412
|
-
assert.deepEqual(nestedUnexpectedResult, {
|
|
2413
|
-
ok: false,
|
|
2414
|
-
error: 'forbidden_output'
|
|
2415
|
-
}, 'configured positive output allowlist recurses through arrays and objects');
|
|
990
|
+
assert.deepEqual(result, { ok: true, result: [{ city: 'Dubai' }] }, 'one-line JSON stdout becomes structured result');
|
|
991
|
+
outcomes.push({ stdout: okEnvelope({ city: 'Dubai' }) });
|
|
992
|
+
const unmappedResult = await bridge.execute(bridgeCall('toString', { city: 'Dubai' }), null);
|
|
993
|
+
assert.deepEqual(unmappedResult, { ok: true, result: { city: 'Dubai' } }, 'allowed tool without a positive mapping retains the recursive denylist only');
|
|
994
|
+
outcomes.push({ stdout: okEnvelope({ legacy: 'value' }) });
|
|
995
|
+
const legacyResult = await bridge.execute(bridgeCall('constructor', { city: 'Dubai' }), null);
|
|
996
|
+
assert.deepEqual(legacyResult, { ok: true, result: { legacy: 'value' } }, 'mapped constructor accepts its declared positive key');
|
|
997
|
+
outcomes.push({ stdout: okEnvelope({ city: 'Dubai' }) });
|
|
998
|
+
const constructorUnexpected = await bridge.execute(bridgeCall('constructor', { city: 'Dubai' }), null);
|
|
999
|
+
assert.deepEqual(constructorUnexpected, { ok: false, error: 'forbidden_output' }, 'mapped constructor rejects undeclared positive keys');
|
|
1000
|
+
outcomes.push({ stdout: okEnvelope({ nested: { city: 'Dubai' } }) });
|
|
1001
|
+
const nestedAllowedResult = await bridge.execute(bridgeCall('lookup', { city: 'Dubai' }), null);
|
|
1002
|
+
assert.deepEqual(nestedAllowedResult, { ok: true, result: { nested: { city: 'Dubai' } } }, 'configured positive output allowlist accepts declared nested keys');
|
|
1003
|
+
outcomes.push({ stdout: okEnvelope({ city: 'Dubai', unexpected: 'secret' }) });
|
|
1004
|
+
const unexpectedResult = await bridge.execute(bridgeCall('lookup', { city: 'Dubai' }), null);
|
|
1005
|
+
assert.deepEqual(unexpectedResult, { ok: false, error: 'forbidden_output' }, 'configured positive output allowlist rejects unexpected keys without reflecting them');
|
|
1006
|
+
outcomes.push({ stdout: okEnvelope([{ city: 'Dubai', nested: { unexpected: 'secret' } }]) });
|
|
1007
|
+
const nestedUnexpectedResult = await bridge.execute(bridgeCall('lookup', { city: 'Dubai' }), null);
|
|
1008
|
+
assert.deepEqual(nestedUnexpectedResult, { ok: false, error: 'forbidden_output' }, 'configured positive output allowlist recurses through arrays and objects');
|
|
2416
1009
|
for (const forbidden of [
|
|
2417
|
-
'gold',
|
|
2418
|
-
'
|
|
2419
|
-
'
|
|
2420
|
-
|
|
2421
|
-
'
|
|
2422
|
-
'
|
|
2423
|
-
'
|
|
2424
|
-
'score',
|
|
2425
|
-
'reward',
|
|
2426
|
-
'ground_truth',
|
|
2427
|
-
'groundTruth',
|
|
2428
|
-
'hidden_query',
|
|
2429
|
-
'hidden-query',
|
|
2430
|
-
'loader_metadata',
|
|
2431
|
-
'loaderMetadata',
|
|
2432
|
-
'reference',
|
|
2433
|
-
'gоld'
|
|
2434
|
-
]){
|
|
2435
|
-
outcomes.push({
|
|
2436
|
-
stdout: okEnvelope({
|
|
2437
|
-
nested: {
|
|
2438
|
-
[forbidden]: 'secret'
|
|
2439
|
-
}
|
|
2440
|
-
})
|
|
2441
|
-
});
|
|
2442
|
-
const forbiddenResult = await bridge.execute(bridgeCall('lookup', {
|
|
2443
|
-
city: 'Dubai'
|
|
2444
|
-
}), null);
|
|
2445
|
-
assert.deepEqual(forbiddenResult, {
|
|
2446
|
-
ok: false,
|
|
2447
|
-
error: 'forbidden_output'
|
|
2448
|
-
}, `recursive no-oracle key ${forbidden} is rejected without reflecting its name`);
|
|
1010
|
+
'gold', 'goldAnswer', 'oracle', 'expected', 'expected_answer', 'answer', 'label',
|
|
1011
|
+
'score', 'reward', 'ground_truth', 'groundTruth', 'hidden_query', 'hidden-query',
|
|
1012
|
+
'loader_metadata', 'loaderMetadata', 'reference', 'gоld',
|
|
1013
|
+
]) {
|
|
1014
|
+
outcomes.push({ stdout: okEnvelope({ nested: { [forbidden]: 'secret' } }) });
|
|
1015
|
+
const forbiddenResult = await bridge.execute(bridgeCall('lookup', { city: 'Dubai' }), null);
|
|
1016
|
+
assert.deepEqual(forbiddenResult, { ok: false, error: 'forbidden_output' }, `recursive no-oracle key ${forbidden} is rejected without reflecting its name`);
|
|
2449
1017
|
}
|
|
2450
|
-
outcomes.push({
|
|
2451
|
-
|
|
2452
|
-
});
|
|
2453
|
-
|
|
2454
|
-
|
|
2455
|
-
}
|
|
2456
|
-
assert.deepEqual(primitiveOutput, {
|
|
2457
|
-
ok: false,
|
|
2458
|
-
error: 'invalid_output'
|
|
2459
|
-
}, 'primitive result strings cannot bypass the structured visible-output boundary');
|
|
2460
|
-
outcomes.push({
|
|
2461
|
-
stdout: okEnvelope([
|
|
2462
|
-
'PRIVATE_ARRAY_VALUE'
|
|
2463
|
-
])
|
|
2464
|
-
});
|
|
2465
|
-
const primitiveArrayOutput = await bridge.execute(bridgeCall('lookup', {
|
|
2466
|
-
city: 'Dubai'
|
|
2467
|
-
}), null);
|
|
2468
|
-
assert.deepEqual(primitiveArrayOutput, {
|
|
2469
|
-
ok: false,
|
|
2470
|
-
error: 'forbidden_output'
|
|
2471
|
-
}, 'top-level primitive arrays cannot bypass the positive output-key boundary');
|
|
1018
|
+
outcomes.push({ stdout: okEnvelope('the hidden answer') });
|
|
1019
|
+
const primitiveOutput = await bridge.execute(bridgeCall('lookup', { city: 'Dubai' }), null);
|
|
1020
|
+
assert.deepEqual(primitiveOutput, { ok: false, error: 'invalid_output' }, 'primitive result strings cannot bypass the structured visible-output boundary');
|
|
1021
|
+
outcomes.push({ stdout: okEnvelope(['PRIVATE_ARRAY_VALUE']) });
|
|
1022
|
+
const primitiveArrayOutput = await bridge.execute(bridgeCall('lookup', { city: 'Dubai' }), null);
|
|
1023
|
+
assert.deepEqual(primitiveArrayOutput, { ok: false, error: 'forbidden_output' }, 'top-level primitive arrays cannot bypass the positive output-key boundary');
|
|
2472
1024
|
assert.equal(JSON.stringify(primitiveArrayOutput).includes('PRIVATE_ARRAY_VALUE'), false, 'rejected primitive array values are not reflected');
|
|
2473
|
-
outcomes.push({
|
|
2474
|
-
|
|
2475
|
-
|
|
2476
|
-
"PRIVATE_NESTED_ARRAY_VALUE"
|
|
2477
|
-
]
|
|
2478
|
-
])
|
|
2479
|
-
});
|
|
2480
|
-
const nestedPrimitiveArrayOutput = await bridge.execute(bridgeCall('lookup', {
|
|
2481
|
-
city: 'Dubai'
|
|
2482
|
-
}), null);
|
|
2483
|
-
assert.deepEqual(nestedPrimitiveArrayOutput, {
|
|
2484
|
-
ok: false,
|
|
2485
|
-
error: 'forbidden_output'
|
|
2486
|
-
}, 'nested primitive arrays without a declared-key ancestor fail closed');
|
|
1025
|
+
outcomes.push({ stdout: okEnvelope([["PRIVATE_NESTED_ARRAY_VALUE"]]) });
|
|
1026
|
+
const nestedPrimitiveArrayOutput = await bridge.execute(bridgeCall('lookup', { city: 'Dubai' }), null);
|
|
1027
|
+
assert.deepEqual(nestedPrimitiveArrayOutput, { ok: false, error: 'forbidden_output' }, 'nested primitive arrays without a declared-key ancestor fail closed');
|
|
2487
1028
|
assert.equal(JSON.stringify(nestedPrimitiveArrayOutput).includes('PRIVATE_NESTED_ARRAY_VALUE'), false, 'rejected nested primitive array values are not reflected');
|
|
2488
|
-
outcomes.push({
|
|
2489
|
-
|
|
2490
|
-
|
|
2491
|
-
city: 'Dubai'
|
|
2492
|
-
},
|
|
2493
|
-
'PRIVATE_MIXED_ARRAY_VALUE'
|
|
2494
|
-
])
|
|
2495
|
-
});
|
|
2496
|
-
const mixedArrayOutput = await bridge.execute(bridgeCall('lookup', {
|
|
2497
|
-
city: 'Dubai'
|
|
2498
|
-
}), null);
|
|
2499
|
-
assert.deepEqual(mixedArrayOutput, {
|
|
2500
|
-
ok: false,
|
|
2501
|
-
error: 'forbidden_output'
|
|
2502
|
-
}, 'a valid record cannot mask an unkeyed primitive sibling');
|
|
1029
|
+
outcomes.push({ stdout: okEnvelope([{ city: 'Dubai' }, 'PRIVATE_MIXED_ARRAY_VALUE']) });
|
|
1030
|
+
const mixedArrayOutput = await bridge.execute(bridgeCall('lookup', { city: 'Dubai' }), null);
|
|
1031
|
+
assert.deepEqual(mixedArrayOutput, { ok: false, error: 'forbidden_output' }, 'a valid record cannot mask an unkeyed primitive sibling');
|
|
2503
1032
|
assert.equal(JSON.stringify(mixedArrayOutput).includes('PRIVATE_MIXED_ARRAY_VALUE'), false, 'rejected mixed-array primitive values are not reflected');
|
|
2504
|
-
outcomes.push({
|
|
2505
|
-
|
|
2506
|
-
});
|
|
2507
|
-
|
|
2508
|
-
|
|
2509
|
-
}
|
|
2510
|
-
|
|
2511
|
-
|
|
2512
|
-
|
|
2513
|
-
}, 'an empty top-level result array is an explicit non-reflecting collection');
|
|
2514
|
-
outcomes.push({
|
|
2515
|
-
stdout: okEnvelope([
|
|
2516
|
-
{
|
|
2517
|
-
city: 'Dubai'
|
|
2518
|
-
}
|
|
2519
|
-
])
|
|
2520
|
-
});
|
|
2521
|
-
const recordArrayOutput = await bridge.execute(bridgeCall('lookup', {
|
|
2522
|
-
city: 'Dubai'
|
|
2523
|
-
}), null);
|
|
2524
|
-
assert.deepEqual(recordArrayOutput, {
|
|
2525
|
-
ok: true,
|
|
2526
|
-
result: [
|
|
2527
|
-
{
|
|
2528
|
-
city: 'Dubai'
|
|
2529
|
-
}
|
|
2530
|
-
]
|
|
2531
|
-
}, 'top-level arrays of records remain valid when every leaf is covered by a declared key');
|
|
2532
|
-
outcomes.push({
|
|
2533
|
-
stdout: okEnvelope({
|
|
2534
|
-
city: [
|
|
2535
|
-
'Dubai'
|
|
2536
|
-
]
|
|
2537
|
-
})
|
|
2538
|
-
});
|
|
2539
|
-
const keyedPrimitiveArrayOutput = await bridge.execute(bridgeCall('lookup', {
|
|
2540
|
-
city: 'Dubai'
|
|
2541
|
-
}), null);
|
|
2542
|
-
assert.deepEqual(keyedPrimitiveArrayOutput, {
|
|
2543
|
-
ok: true,
|
|
2544
|
-
result: {
|
|
2545
|
-
city: [
|
|
2546
|
-
'Dubai'
|
|
2547
|
-
]
|
|
2548
|
-
}
|
|
2549
|
-
}, 'primitive arrays remain valid below a declared output key');
|
|
1033
|
+
outcomes.push({ stdout: okEnvelope([]) });
|
|
1034
|
+
const emptyArrayOutput = await bridge.execute(bridgeCall('lookup', { city: 'Dubai' }), null);
|
|
1035
|
+
assert.deepEqual(emptyArrayOutput, { ok: true, result: [] }, 'an empty top-level result array is an explicit non-reflecting collection');
|
|
1036
|
+
outcomes.push({ stdout: okEnvelope([{ city: 'Dubai' }]) });
|
|
1037
|
+
const recordArrayOutput = await bridge.execute(bridgeCall('lookup', { city: 'Dubai' }), null);
|
|
1038
|
+
assert.deepEqual(recordArrayOutput, { ok: true, result: [{ city: 'Dubai' }] }, 'top-level arrays of records remain valid when every leaf is covered by a declared key');
|
|
1039
|
+
outcomes.push({ stdout: okEnvelope({ city: ['Dubai'] }) });
|
|
1040
|
+
const keyedPrimitiveArrayOutput = await bridge.execute(bridgeCall('lookup', { city: 'Dubai' }), null);
|
|
1041
|
+
assert.deepEqual(keyedPrimitiveArrayOutput, { ok: true, result: { city: ['Dubai'] } }, 'primitive arrays remain valid below a declared output key');
|
|
2550
1042
|
const beforeOversized = spawnSpecs.length;
|
|
2551
|
-
const oversized = await bridge.execute(bridgeCall('lookup', {
|
|
2552
|
-
|
|
2553
|
-
notes: 'x'.repeat(65_537)
|
|
2554
|
-
}), null);
|
|
2555
|
-
assert.deepEqual(oversized, {
|
|
2556
|
-
ok: false,
|
|
2557
|
-
error: 'invalid_arguments',
|
|
2558
|
-
reason: 'serialization_limit'
|
|
2559
|
-
});
|
|
1043
|
+
const oversized = await bridge.execute(bridgeCall('lookup', { city: 'Dubai', notes: 'x'.repeat(65_537) }), null);
|
|
1044
|
+
assert.deepEqual(oversized, { ok: false, error: 'invalid_arguments', reason: 'serialization_limit' });
|
|
2560
1045
|
assert.equal(spawnSpecs.length, beforeOversized, 'oversized serialized arguments are rejected before spawn');
|
|
2561
1046
|
const beforeDeep = spawnSpecs.length;
|
|
2562
1047
|
let deep = {};
|
|
2563
|
-
for(let index = 0; index < 13; index++)
|
|
2564
|
-
next: deep
|
|
2565
|
-
};
|
|
2566
|
-
await assert.rejects(()=>bridge.execute(bridgeCall('lookup', {
|
|
2567
|
-
city: 'Dubai',
|
|
2568
|
-
...deep
|
|
2569
|
-
}), null), /invalid arguments/);
|
|
1048
|
+
for (let index = 0; index < 13; index++)
|
|
1049
|
+
deep = { next: deep };
|
|
1050
|
+
await assert.rejects(() => bridge.execute(bridgeCall('lookup', { city: 'Dubai', ...deep }), null), /invalid arguments/);
|
|
2570
1051
|
assert.equal(spawnSpecs.length, beforeDeep, 'schema-invalid deep arguments are rejected before spawn');
|
|
2571
1052
|
const beforeOverride = spawnSpecs.length;
|
|
2572
1053
|
const overrideResult = await bridge.execute(bridgeCall('lookup', {
|
|
2573
|
-
city: 'Dubai',
|
|
2574
|
-
executable: '/tmp/evil',
|
|
2575
|
-
cwd: '/tmp/evil',
|
|
2576
|
-
argv: [
|
|
2577
|
-
'--unsafe'
|
|
2578
|
-
]
|
|
1054
|
+
city: 'Dubai', executable: '/tmp/evil', cwd: '/tmp/evil', argv: ['--unsafe'],
|
|
2579
1055
|
}), null);
|
|
2580
1056
|
assert.deepEqual(spawnSpecs[beforeOverride]?.argv, [
|
|
2581
|
-
process.execPath,
|
|
2582
|
-
|
|
2583
|
-
|
|
2584
|
-
'--lang',
|
|
2585
|
-
'en',
|
|
2586
|
-
'call',
|
|
2587
|
-
'lookup',
|
|
2588
|
-
JSON.stringify({
|
|
2589
|
-
city: 'Dubai',
|
|
2590
|
-
executable: '/tmp/evil',
|
|
2591
|
-
cwd: '/tmp/evil',
|
|
2592
|
-
argv: [
|
|
2593
|
-
'--unsafe'
|
|
2594
|
-
]
|
|
2595
|
-
})
|
|
1057
|
+
process.execPath, '-m', 'agent_env.cli', '--lang', 'en', 'call', 'lookup', JSON.stringify({
|
|
1058
|
+
city: 'Dubai', executable: '/tmp/evil', cwd: '/tmp/evil', argv: ['--unsafe'],
|
|
1059
|
+
}),
|
|
2596
1060
|
], 'model executable/cwd/argv fields remain data and cannot override config');
|
|
2597
1061
|
assert.equal(overrideResult.ok, true, 'override-shaped arguments still use the configured bridge');
|
|
2598
|
-
outcomes.push({
|
|
2599
|
-
|
|
2600
|
-
|
|
2601
|
-
|
|
2602
|
-
|
|
2603
|
-
});
|
|
2604
|
-
|
|
2605
|
-
|
|
2606
|
-
}
|
|
2607
|
-
assert.deepEqual(truncated, {
|
|
2608
|
-
ok: false,
|
|
2609
|
-
error: 'output_truncated'
|
|
2610
|
-
}, 'lossy stdout is rejected without parsing partial output');
|
|
2611
|
-
outcomes.push({
|
|
2612
|
-
stdout: okEnvelope({
|
|
2613
|
-
city: 'Dubai'
|
|
2614
|
-
}),
|
|
2615
|
-
stderr: 'private runner diagnostic',
|
|
2616
|
-
exitCode: 17,
|
|
2617
|
-
signal: 'SIGTERM'
|
|
2618
|
-
});
|
|
2619
|
-
const failed = await bridge.execute(bridgeCall('lookup', {
|
|
2620
|
-
city: 'Dubai'
|
|
2621
|
-
}), null);
|
|
2622
|
-
assert.deepEqual(failed, {
|
|
2623
|
-
ok: false,
|
|
2624
|
-
error: 'runner_failed',
|
|
2625
|
-
exit_code: 17,
|
|
2626
|
-
signal: 'SIGTERM'
|
|
2627
|
-
}, 'nonzero runner result is structured without stderr echo');
|
|
2628
|
-
const errorsTable = await bridge.execute({
|
|
2629
|
-
action: 'errors'
|
|
2630
|
-
}, null);
|
|
1062
|
+
outcomes.push({ stdout: okEnvelope({ city: 'Dubai' }), lossy: true });
|
|
1063
|
+
const truncated = await bridge.execute(bridgeCall('lookup', { city: 'Dubai' }), null);
|
|
1064
|
+
assert.deepEqual(truncated, { ok: false, error: 'output_truncated' }, 'lossy stdout is rejected without parsing partial output');
|
|
1065
|
+
outcomes.push({ stdout: okEnvelope({ city: 'Dubai' }), stderr: 'private runner diagnostic', exitCode: 17, signal: 'SIGTERM' });
|
|
1066
|
+
const failed = await bridge.execute(bridgeCall('lookup', { city: 'Dubai' }), null);
|
|
1067
|
+
assert.deepEqual(failed, { ok: false, error: 'runner_failed', exit_code: 17, signal: 'SIGTERM' }, 'nonzero runner result is structured without stderr echo');
|
|
1068
|
+
// The bridge protocol/infrastructure failure vocabulary is distinct from
|
|
1069
|
+
// per-tool domain outcomes and remains discoverable through action=errors.
|
|
1070
|
+
const errorsTable = await bridge.execute({ action: 'errors' }, null);
|
|
2631
1071
|
assert.ok(errorsTable.ok === true && errorsTable.errors, 'bridge failure contract is discoverable');
|
|
2632
|
-
for (const code of [
|
|
2633
|
-
'invalid_action',
|
|
2634
|
-
'disallowed_tool',
|
|
2635
|
-
'invalid_arguments',
|
|
2636
|
-
'timed_out',
|
|
2637
|
-
'output_truncated',
|
|
2638
|
-
'invalid_json',
|
|
2639
|
-
'invalid_output',
|
|
2640
|
-
'runner_failed',
|
|
2641
|
-
'spawn_failed',
|
|
2642
|
-
'forbidden_output'
|
|
2643
|
-
]){
|
|
1072
|
+
for (const code of ['invalid_action', 'disallowed_tool', 'invalid_arguments', 'timed_out', 'output_truncated', 'invalid_json', 'invalid_output', 'runner_failed', 'spawn_failed', 'forbidden_output']) {
|
|
2644
1073
|
const row = errorsTable.errors?.[code];
|
|
2645
1074
|
assert.ok(row && typeof row.recoverable === 'boolean' && row.remedy.length > 0, `failure contract contains ${code} with recovery guidance`);
|
|
2646
1075
|
}
|
|
2647
1076
|
assert.equal(errorsTable.errors?.invalid_arguments?.recoverable, true, 'invalid caller arguments are recoverable');
|
|
2648
1077
|
assert.equal(errorsTable.errors?.forbidden_output?.recoverable, false, 'policy-boundary output is not recoverable');
|
|
2649
|
-
assert.deepEqual(errorsTable, {
|
|
2650
|
-
|
|
2651
|
-
errors: BRIDGE_ERROR_CONTRACT
|
|
2652
|
-
}, 'errors action returns the exact closed failure contract');
|
|
2653
|
-
const toolsTable = await bridge.execute({
|
|
2654
|
-
action: 'tools'
|
|
2655
|
-
}, null);
|
|
1078
|
+
assert.deepEqual(errorsTable, { ok: true, errors: BRIDGE_ERROR_CONTRACT }, 'errors action returns the exact closed failure contract');
|
|
1079
|
+
const toolsTable = await bridge.execute({ action: 'tools' }, null);
|
|
2656
1080
|
assert.deepEqual(toolsTable, {
|
|
2657
1081
|
ok: true,
|
|
2658
1082
|
tools: [
|
|
2659
|
-
{
|
|
2660
|
-
|
|
2661
|
-
|
|
2662
|
-
|
|
2663
|
-
output_keys: [
|
|
2664
|
-
'city',
|
|
2665
|
-
'nested'
|
|
2666
|
-
],
|
|
2667
|
-
domain_outcomes: [
|
|
2668
|
-
{
|
|
2669
|
-
status: 'miss',
|
|
2670
|
-
code: 'NOT_FOUND',
|
|
2671
|
-
recovery: 'revise_arguments'
|
|
2672
|
-
},
|
|
2673
|
-
{
|
|
2674
|
-
status: 'error',
|
|
2675
|
-
code: 'AMBIGUOUS',
|
|
2676
|
-
recovery: 'choose_alternative'
|
|
2677
|
-
}
|
|
2678
|
-
]
|
|
2679
|
-
},
|
|
2680
|
-
{
|
|
2681
|
-
name: 'constructor',
|
|
2682
|
-
description: 'Construct one declared city.',
|
|
2683
|
-
input_schema: lookupInputSchema(),
|
|
2684
|
-
output_keys: [
|
|
2685
|
-
'legacy'
|
|
2686
|
-
],
|
|
2687
|
-
domain_outcomes: [
|
|
2688
|
-
{
|
|
2689
|
-
status: 'error',
|
|
2690
|
-
code: 'INVALID',
|
|
2691
|
-
recovery: 'retry_same'
|
|
2692
|
-
}
|
|
2693
|
-
]
|
|
2694
|
-
},
|
|
2695
|
-
{
|
|
2696
|
-
name: 'toString',
|
|
2697
|
-
description: 'Stringify one declared city.',
|
|
2698
|
-
input_schema: lookupInputSchema(),
|
|
2699
|
-
output_keys: [
|
|
2700
|
-
'city',
|
|
2701
|
-
'nested'
|
|
2702
|
-
],
|
|
2703
|
-
domain_outcomes: []
|
|
2704
|
-
}
|
|
2705
|
-
]
|
|
1083
|
+
{ name: 'lookup', description: 'Lookup one declared city.', input_schema: lookupInputSchema(), output_keys: ['city', 'nested'], domain_outcomes: [{ status: 'miss', code: 'NOT_FOUND', recovery: 'revise_arguments' }, { status: 'error', code: 'AMBIGUOUS', recovery: 'choose_alternative' }] },
|
|
1084
|
+
{ name: 'constructor', description: 'Construct one declared city.', input_schema: lookupInputSchema(), output_keys: ['legacy'], domain_outcomes: [{ status: 'error', code: 'INVALID', recovery: 'retry_same' }] },
|
|
1085
|
+
{ name: 'toString', description: 'Stringify one declared city.', input_schema: lookupInputSchema(), output_keys: ['city', 'nested'], domain_outcomes: [] },
|
|
1086
|
+
],
|
|
2706
1087
|
}, 'tools action returns every exact frozen descriptor, including its input schema');
|
|
2707
|
-
for (const stdout of [
|
|
2708
|
-
|
|
2709
|
-
'{
|
|
2710
|
-
|
|
2711
|
-
outcomes.push({
|
|
2712
|
-
stdout
|
|
2713
|
-
});
|
|
2714
|
-
const malformed = await bridge.execute(bridgeCall('lookup', {
|
|
2715
|
-
city: 'Dubai'
|
|
2716
|
-
}), null);
|
|
2717
|
-
assert.deepEqual(malformed, {
|
|
2718
|
-
ok: false,
|
|
2719
|
-
error: 'invalid_json'
|
|
2720
|
-
}, 'malformed or multi-value JSON is rejected');
|
|
1088
|
+
for (const stdout of ['not-json', '{"result":1}{"result":2}']) {
|
|
1089
|
+
outcomes.push({ stdout });
|
|
1090
|
+
const malformed = await bridge.execute(bridgeCall('lookup', { city: 'Dubai' }), null);
|
|
1091
|
+
assert.deepEqual(malformed, { ok: false, error: 'invalid_json' }, 'malformed or multi-value JSON is rejected');
|
|
2721
1092
|
}
|
|
2722
|
-
outcomes.push({
|
|
2723
|
-
|
|
2724
|
-
|
|
2725
|
-
|
|
2726
|
-
|
|
2727
|
-
|
|
2728
|
-
})
|
|
2729
|
-
});
|
|
2730
|
-
const miss = await bridge.execute(bridgeCall('lookup', {
|
|
2731
|
-
city: 'Dubai'
|
|
2732
|
-
}), null);
|
|
2733
|
-
assert.deepEqual(miss, {
|
|
2734
|
-
ok: true,
|
|
2735
|
-
outcome: {
|
|
2736
|
-
schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION,
|
|
2737
|
-
status: 'miss',
|
|
2738
|
-
code: 'NOT_FOUND',
|
|
2739
|
-
recovery: 'revise_arguments'
|
|
2740
|
-
}
|
|
2741
|
-
}, 'exit-zero domain miss is a successful typed outcome');
|
|
2742
|
-
outcomes.push({
|
|
2743
|
-
stdout: JSON.stringify({
|
|
2744
|
-
schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION,
|
|
2745
|
-
status: 'error',
|
|
2746
|
-
code: 'AMBIGUOUS',
|
|
2747
|
-
recovery: 'choose_alternative'
|
|
2748
|
-
})
|
|
2749
|
-
});
|
|
2750
|
-
const domainError = await bridge.execute(bridgeCall('lookup', {
|
|
2751
|
-
city: 'Dubai'
|
|
2752
|
-
}), null);
|
|
2753
|
-
assert.deepEqual(domainError, {
|
|
2754
|
-
ok: true,
|
|
2755
|
-
outcome: {
|
|
2756
|
-
schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION,
|
|
2757
|
-
status: 'error',
|
|
2758
|
-
code: 'AMBIGUOUS',
|
|
2759
|
-
recovery: 'choose_alternative'
|
|
2760
|
-
}
|
|
2761
|
-
}, 'declared exit-zero domain error is transport success');
|
|
1093
|
+
outcomes.push({ stdout: JSON.stringify({ schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION, status: 'miss', code: 'NOT_FOUND', recovery: 'revise_arguments' }) });
|
|
1094
|
+
const miss = await bridge.execute(bridgeCall('lookup', { city: 'Dubai' }), null);
|
|
1095
|
+
assert.deepEqual(miss, { ok: true, outcome: { schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION, status: 'miss', code: 'NOT_FOUND', recovery: 'revise_arguments' } }, 'exit-zero domain miss is a successful typed outcome');
|
|
1096
|
+
outcomes.push({ stdout: JSON.stringify({ schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION, status: 'error', code: 'AMBIGUOUS', recovery: 'choose_alternative' }) });
|
|
1097
|
+
const domainError = await bridge.execute(bridgeCall('lookup', { city: 'Dubai' }), null);
|
|
1098
|
+
assert.deepEqual(domainError, { ok: true, outcome: { schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION, status: 'error', code: 'AMBIGUOUS', recovery: 'choose_alternative' } }, 'declared exit-zero domain error is transport success');
|
|
2762
1099
|
for (const invalidEnvelope of [
|
|
2763
|
-
{
|
|
2764
|
-
|
|
2765
|
-
|
|
2766
|
-
|
|
2767
|
-
|
|
2768
|
-
}
|
|
2769
|
-
{
|
|
2770
|
-
schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION,
|
|
2771
|
-
status: 'miss',
|
|
2772
|
-
code: 'UNKNOWN',
|
|
2773
|
-
recovery: 'revise_arguments'
|
|
2774
|
-
},
|
|
2775
|
-
{
|
|
2776
|
-
schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION,
|
|
2777
|
-
status: 'miss',
|
|
2778
|
-
code: 'NOT_FOUND',
|
|
2779
|
-
recovery: 'retry_same'
|
|
2780
|
-
},
|
|
2781
|
-
{
|
|
2782
|
-
schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION,
|
|
2783
|
-
status: 'error',
|
|
2784
|
-
code: 'AMBIGUOUS',
|
|
2785
|
-
recovery: 'choose_alternative',
|
|
2786
|
-
message: 'private free text'
|
|
2787
|
-
}
|
|
2788
|
-
]){
|
|
2789
|
-
outcomes.push({
|
|
2790
|
-
stdout: JSON.stringify(invalidEnvelope)
|
|
2791
|
-
});
|
|
2792
|
-
assert.deepEqual(await bridge.execute(bridgeCall('lookup', {
|
|
2793
|
-
city: 'Dubai'
|
|
2794
|
-
}), null), {
|
|
2795
|
-
ok: false,
|
|
2796
|
-
error: 'invalid_output'
|
|
2797
|
-
}, 'undeclared or non-exact domain envelope fails closed');
|
|
1100
|
+
{ schema_version: 'legacy', status: 'miss', code: 'NOT_FOUND', recovery: 'revise_arguments' },
|
|
1101
|
+
{ schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION, status: 'miss', code: 'UNKNOWN', recovery: 'revise_arguments' },
|
|
1102
|
+
{ schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION, status: 'miss', code: 'NOT_FOUND', recovery: 'retry_same' },
|
|
1103
|
+
{ schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION, status: 'error', code: 'AMBIGUOUS', recovery: 'choose_alternative', message: 'private free text' },
|
|
1104
|
+
]) {
|
|
1105
|
+
outcomes.push({ stdout: JSON.stringify(invalidEnvelope) });
|
|
1106
|
+
assert.deepEqual(await bridge.execute(bridgeCall('lookup', { city: 'Dubai' }), null), { ok: false, error: 'invalid_output' }, 'undeclared or non-exact domain envelope fails closed');
|
|
2798
1107
|
}
|
|
2799
|
-
outcomes.push({
|
|
2800
|
-
|
|
2801
|
-
|
|
2802
|
-
|
|
2803
|
-
|
|
2804
|
-
|
|
2805
|
-
});
|
|
2806
|
-
|
|
2807
|
-
|
|
2808
|
-
}
|
|
2809
|
-
|
|
2810
|
-
|
|
2811
|
-
}, '
|
|
2812
|
-
outcomes.push({
|
|
2813
|
-
|
|
2814
|
-
|
|
2815
|
-
status: 'miss',
|
|
2816
|
-
code: 'NOT_FOUND',
|
|
2817
|
-
recovery: 'revise_arguments'
|
|
2818
|
-
}),
|
|
2819
|
-
exitCode: 23
|
|
2820
|
-
});
|
|
2821
|
-
assert.deepEqual(await bridge.execute(bridgeCall('lookup', {
|
|
2822
|
-
city: 'Dubai'
|
|
2823
|
-
}), null), {
|
|
2824
|
-
ok: false,
|
|
2825
|
-
error: 'runner_failed',
|
|
2826
|
-
exit_code: 23,
|
|
2827
|
-
signal: null
|
|
2828
|
-
}, 'nonzero exit remains infrastructure failure even when stdout resembles a domain envelope');
|
|
2829
|
-
outcomes.push({
|
|
2830
|
-
spawnError: true
|
|
2831
|
-
});
|
|
2832
|
-
const spawnFailed = await bridge.execute(bridgeCall('lookup', {
|
|
2833
|
-
city: 'Dubai'
|
|
2834
|
-
}), null);
|
|
2835
|
-
assert.deepEqual(spawnFailed, {
|
|
2836
|
-
ok: false,
|
|
2837
|
-
error: 'spawn_failed'
|
|
2838
|
-
}, 'spawn infrastructure failure is structured');
|
|
2839
|
-
outcomes.push({
|
|
2840
|
-
spawnReject: true
|
|
2841
|
-
});
|
|
2842
|
-
const asyncSpawnFailed = await bridge.execute(bridgeCall('lookup', {
|
|
2843
|
-
city: 'Dubai'
|
|
2844
|
-
}), null);
|
|
2845
|
-
assert.deepEqual(asyncSpawnFailed, {
|
|
2846
|
-
ok: false,
|
|
2847
|
-
error: 'spawn_failed'
|
|
2848
|
-
}, 'DSH pid=-1 spawn rejection is distinct from a started runner failure');
|
|
2849
|
-
outcomes.push({
|
|
2850
|
-
waitForAbort: true
|
|
2851
|
-
});
|
|
2852
|
-
const timedOut = await bridge.execute(bridgeCall('lookup', {
|
|
2853
|
-
city: 'Dubai'
|
|
2854
|
-
}), null);
|
|
2855
|
-
assert.deepEqual(timedOut, {
|
|
2856
|
-
ok: false,
|
|
2857
|
-
error: 'timed_out'
|
|
2858
|
-
}, 'deadline abort is surfaced as timed_out');
|
|
1108
|
+
outcomes.push({ stdout: JSON.stringify({ result: { city: 'Dubai' } }) });
|
|
1109
|
+
assert.deepEqual(await bridge.execute(bridgeCall('lookup', { city: 'Dubai' }), null), { ok: false, error: 'invalid_output' }, 'legacy versionless result envelope is rejected');
|
|
1110
|
+
outcomes.push({ stdout: JSON.stringify({ schema_version: BENCHMARK_TOOL_RESULT_SCHEMA_VERSION, status: 'miss', code: 'NOT_FOUND', recovery: 'revise_arguments' }), exitCode: 23 });
|
|
1111
|
+
assert.deepEqual(await bridge.execute(bridgeCall('lookup', { city: 'Dubai' }), null), { ok: false, error: 'runner_failed', exit_code: 23, signal: null }, 'nonzero exit remains infrastructure failure even when stdout resembles a domain envelope');
|
|
1112
|
+
outcomes.push({ spawnError: true });
|
|
1113
|
+
const spawnFailed = await bridge.execute(bridgeCall('lookup', { city: 'Dubai' }), null);
|
|
1114
|
+
assert.deepEqual(spawnFailed, { ok: false, error: 'spawn_failed' }, 'spawn infrastructure failure is structured');
|
|
1115
|
+
outcomes.push({ spawnReject: true });
|
|
1116
|
+
const asyncSpawnFailed = await bridge.execute(bridgeCall('lookup', { city: 'Dubai' }), null);
|
|
1117
|
+
assert.deepEqual(asyncSpawnFailed, { ok: false, error: 'spawn_failed' }, 'public done rejection is classified as spawn/provider failure');
|
|
1118
|
+
outcomes.push({ stdout: okEnvelope({ city: 'Dubai' }), readFailure: true });
|
|
1119
|
+
const postSuccessCollectFailed = await bridge.execute(bridgeCall('lookup', { city: 'Dubai' }), null);
|
|
1120
|
+
assert.deepEqual(postSuccessCollectFailed, { ok: false, error: 'runner_failed', exit_code: 0, signal: null }, 'collected output reading failure after a resolved done is runner_failed, not spawn_failed');
|
|
1121
|
+
outcomes.push({ waitForAbort: true });
|
|
1122
|
+
const timedOut = await bridge.execute(bridgeCall('lookup', { city: 'Dubai' }), null);
|
|
1123
|
+
assert.deepEqual(timedOut, { ok: false, error: 'timed_out' }, 'deadline abort is surfaced as timed_out');
|
|
2859
1124
|
assert.equal(timeoutSignal?.aborted, true, 'timeout abort signal is fired');
|
|
2860
1125
|
const beforeInvalidArgs = spawnSpecs.length;
|
|
2861
|
-
await assert.rejects(()=>bridge.execute(bridgeCall('lookup', [
|
|
2862
|
-
'not',
|
|
2863
|
-
'plain'
|
|
2864
|
-
]), null), /invalid arguments/);
|
|
1126
|
+
await assert.rejects(() => bridge.execute(bridgeCall('lookup', ['not', 'plain']), null), /invalid arguments/);
|
|
2865
1127
|
assert.equal(spawnSpecs.length, beforeInvalidArgs, 'non-object arguments are rejected before spawn');
|
|
2866
1128
|
for (const [label, argumentsValue] of [
|
|
2867
|
-
[
|
|
2868
|
-
|
|
2869
|
-
|
|
2870
|
-
|
|
2871
|
-
|
|
2872
|
-
'wrong city type',
|
|
2873
|
-
{
|
|
2874
|
-
city: 7
|
|
2875
|
-
}
|
|
2876
|
-
],
|
|
2877
|
-
[
|
|
2878
|
-
'undeclared country',
|
|
2879
|
-
{
|
|
2880
|
-
city: 'Dubai',
|
|
2881
|
-
country: 'AE'
|
|
2882
|
-
}
|
|
2883
|
-
]
|
|
2884
|
-
]){
|
|
2885
|
-
await assert.rejects(()=>bridge.execute(bridgeCall('lookup', argumentsValue), null), /invalid arguments/, `${label} is rejected by the descriptor-derived runtime validator`);
|
|
1129
|
+
['missing required city', {}],
|
|
1130
|
+
['wrong city type', { city: 7 }],
|
|
1131
|
+
['undeclared country', { city: 'Dubai', country: 'AE' }],
|
|
1132
|
+
]) {
|
|
1133
|
+
await assert.rejects(() => bridge.execute(bridgeCall('lookup', argumentsValue), null), /invalid arguments/, `${label} is rejected by the descriptor-derived runtime validator`);
|
|
2886
1134
|
}
|
|
2887
1135
|
assert.equal(spawnSpecs.length, beforeInvalidArgs, 'required, type, and closed-object argument failures all happen before spawn');
|
|
2888
|
-
await assert.rejects(()=>bridge.execute({
|
|
2889
|
-
action: 'inspect'
|
|
2890
|
-
}, null), /invalid arguments/);
|
|
1136
|
+
await assert.rejects(() => bridge.execute({ action: 'inspect' }, null), /invalid arguments/);
|
|
2891
1137
|
const beforeRejected = spawnSpecs.length;
|
|
2892
|
-
await assert.rejects(()=>bridge.execute(bridgeCall('delete_all', {}), null), /invalid arguments/);
|
|
1138
|
+
await assert.rejects(() => bridge.execute(bridgeCall('delete_all', {}), null), /invalid arguments/);
|
|
2893
1139
|
assert.equal(spawnSpecs.length, beforeRejected, 'disallowed tool is rejected before spawn');
|
|
2894
1140
|
const spacedConfigPath = join(root, ' benchmark-environment-config.json ');
|
|
2895
1141
|
writeFileSync(spacedConfigPath, readFileSync(configPath));
|
|
2896
1142
|
const spacedTools = [];
|
|
2897
1143
|
const spacedCtx = {
|
|
2898
1144
|
tools: {
|
|
2899
|
-
register (tool) {
|
|
2900
|
-
|
|
2901
|
-
|
|
2902
|
-
},
|
|
2903
|
-
get (name) {
|
|
2904
|
-
return spacedTools.find((tool)=>tool.name === name);
|
|
2905
|
-
},
|
|
2906
|
-
schemas () {
|
|
2907
|
-
return spacedTools.map((tool)=>({
|
|
2908
|
-
name: tool.name
|
|
2909
|
-
}));
|
|
2910
|
-
}
|
|
2911
|
-
},
|
|
2912
|
-
agents: {
|
|
2913
|
-
list () {
|
|
2914
|
-
return [];
|
|
2915
|
-
}
|
|
2916
|
-
},
|
|
2917
|
-
systemPrompt: {
|
|
2918
|
-
variable () {}
|
|
2919
|
-
},
|
|
2920
|
-
on () {
|
|
2921
|
-
return ()=>{};
|
|
1145
|
+
register(tool) { spacedTools.push(tool); return () => { }; },
|
|
1146
|
+
get(name) { return spacedTools.find(tool => tool.name === name); },
|
|
1147
|
+
schemas() { return spacedTools.map(tool => ({ name: tool.name })); },
|
|
2922
1148
|
},
|
|
2923
|
-
|
|
2924
|
-
|
|
2925
|
-
},
|
|
2926
|
-
|
|
2927
|
-
|
|
2928
|
-
|
|
2929
|
-
|
|
2930
|
-
|
|
2931
|
-
|
|
2932
|
-
if (name === 'agents') return {
|
|
2933
|
-
list () {
|
|
2934
|
-
return [];
|
|
2935
|
-
}
|
|
2936
|
-
};
|
|
1149
|
+
agents: { list() { return []; } },
|
|
1150
|
+
systemPrompt: { variable() { } },
|
|
1151
|
+
on() { return () => { }; },
|
|
1152
|
+
effect() { return () => { }; },
|
|
1153
|
+
get(name) {
|
|
1154
|
+
if (name === 'subprocess')
|
|
1155
|
+
return { spawn: (_spec) => fakeHandle({ stdout: '{}' }) };
|
|
1156
|
+
if (name === 'agents')
|
|
1157
|
+
return { list() { return []; } };
|
|
2937
1158
|
return undefined;
|
|
2938
|
-
}
|
|
1159
|
+
},
|
|
2939
1160
|
};
|
|
2940
1161
|
apply(spacedCtx, {
|
|
2941
|
-
stateRoot: root,
|
|
2942
|
-
timeoutMs: 20,
|
|
2943
|
-
hbcliBin: '',
|
|
2944
|
-
sessionAccess: 'off',
|
|
2945
|
-
benchmarkEnvironmentConfigPath: spacedConfigPath
|
|
1162
|
+
stateRoot: root, timeoutMs: 20, hbcliBin: '', sessionAccess: 'off', benchmarkEnvironmentConfigPath: spacedConfigPath,
|
|
2946
1163
|
});
|
|
2947
|
-
assert.deepEqual(spacedTools.map(
|
|
2948
|
-
'gotry_benchmark_environment'
|
|
2949
|
-
], 'benchmark bridge loads a valid raw path with whitespace basename');
|
|
1164
|
+
assert.deepEqual(spacedTools.map(tool => tool.name), ['gotry_benchmark_environment'], 'benchmark bridge loads a valid raw path with whitespace basename');
|
|
2950
1165
|
const disabled = [];
|
|
2951
1166
|
const disabledVariables = [];
|
|
2952
1167
|
const disabledCtx = {
|
|
2953
|
-
tools: {
|
|
2954
|
-
|
|
2955
|
-
disabled.push(tool);
|
|
2956
|
-
return ()=>{};
|
|
2957
|
-
}
|
|
2958
|
-
},
|
|
2959
|
-
systemPrompt: {
|
|
2960
|
-
variable (name) {
|
|
2961
|
-
disabledVariables.push(name);
|
|
2962
|
-
}
|
|
2963
|
-
}
|
|
1168
|
+
tools: { register(tool) { disabled.push(tool); return () => { }; } },
|
|
1169
|
+
systemPrompt: { variable(name) { disabledVariables.push(name); } },
|
|
2964
1170
|
};
|
|
2965
1171
|
apply(disabledCtx, {
|
|
2966
|
-
stateRoot: root,
|
|
2967
|
-
timeoutMs: 1_000,
|
|
2968
|
-
hbcliBin: '',
|
|
2969
|
-
sessionAccess: 'off',
|
|
2970
|
-
benchmarkEnvironmentConfigPath: ''
|
|
1172
|
+
stateRoot: root, timeoutMs: 1_000, hbcliBin: '', sessionAccess: 'off', benchmarkEnvironmentConfigPath: '',
|
|
2971
1173
|
});
|
|
2972
|
-
assert.equal(disabled.some(
|
|
1174
|
+
assert.equal(disabled.some(tool => tool.name === 'gotry_benchmark_environment'), false, 'empty config path keeps bridge default-off');
|
|
2973
1175
|
assert.ok(disabled.length > 1, 'normal product mode keeps the full GoTry tool catalog');
|
|
2974
|
-
assert.deepEqual(disabledVariables, [
|
|
2975
|
-
'current_date',
|
|
2976
|
-
'time_anchor_card',
|
|
2977
|
-
'motivation_brief',
|
|
2978
|
-
'channel_routing_card'
|
|
2979
|
-
], 'normal product mode keeps its prompt variables');
|
|
1176
|
+
assert.deepEqual(disabledVariables, ['current_date', 'time_anchor_card', 'motivation_brief', 'channel_routing_card'], 'normal product mode keeps its prompt variables');
|
|
2980
1177
|
const whitespace = [];
|
|
2981
1178
|
const whitespaceCtx = {
|
|
2982
|
-
tools: {
|
|
2983
|
-
|
|
2984
|
-
whitespace.push(tool);
|
|
2985
|
-
return ()=>{};
|
|
2986
|
-
}
|
|
2987
|
-
},
|
|
2988
|
-
systemPrompt: {
|
|
2989
|
-
variable () {}
|
|
2990
|
-
}
|
|
1179
|
+
tools: { register(tool) { whitespace.push(tool); return () => { }; } },
|
|
1180
|
+
systemPrompt: { variable() { } },
|
|
2991
1181
|
};
|
|
2992
1182
|
apply(whitespaceCtx, {
|
|
2993
|
-
stateRoot: root,
|
|
2994
|
-
timeoutMs: 1_000,
|
|
2995
|
-
hbcliBin: '',
|
|
2996
|
-
sessionAccess: 'off',
|
|
2997
|
-
benchmarkEnvironmentConfigPath: ' \t '
|
|
1183
|
+
stateRoot: root, timeoutMs: 1_000, hbcliBin: '', sessionAccess: 'off', benchmarkEnvironmentConfigPath: ' \t ',
|
|
2998
1184
|
});
|
|
2999
|
-
assert.equal(whitespace.some(
|
|
1185
|
+
assert.equal(whitespace.some(tool => tool.name === 'gotry_benchmark_environment'), false, 'whitespace config path keeps benchmark mode default-off');
|
|
3000
1186
|
assert.ok(whitespace.length > 1, 'whitespace path preserves the ordinary product tool catalog');
|
|
3001
1187
|
const originalModelOverride = process.env.GOTRY_LLM_MODEL;
|
|
3002
1188
|
process.env.GOTRY_LLM_MODEL = 'round7-model-preserved';
|
|
@@ -3006,204 +1192,108 @@ try {
|
|
|
3006
1192
|
const modelEvents = [];
|
|
3007
1193
|
const modelCtx = {
|
|
3008
1194
|
tools: {
|
|
3009
|
-
register (tool) {
|
|
3010
|
-
|
|
3011
|
-
|
|
3012
|
-
},
|
|
3013
|
-
get (name) {
|
|
3014
|
-
return modelTools.find((tool)=>tool.name === name);
|
|
3015
|
-
},
|
|
3016
|
-
schemas () {
|
|
3017
|
-
return modelTools.map((tool)=>({
|
|
3018
|
-
name: tool.name
|
|
3019
|
-
}));
|
|
3020
|
-
}
|
|
3021
|
-
},
|
|
3022
|
-
agents: {
|
|
3023
|
-
list () {
|
|
3024
|
-
return [];
|
|
3025
|
-
}
|
|
3026
|
-
},
|
|
3027
|
-
systemPrompt: {
|
|
3028
|
-
variable () {}
|
|
1195
|
+
register(tool) { modelTools.push(tool); return () => { }; },
|
|
1196
|
+
get(name) { return modelTools.find(tool => tool.name === name); },
|
|
1197
|
+
schemas() { return modelTools.map(tool => ({ name: tool.name })); },
|
|
3029
1198
|
},
|
|
3030
|
-
|
|
1199
|
+
agents: { list() { return []; } },
|
|
1200
|
+
systemPrompt: { variable() { } },
|
|
1201
|
+
on(event, listener) {
|
|
3031
1202
|
modelEvents.push(event);
|
|
3032
|
-
if (event === 'agent/request')
|
|
3033
|
-
|
|
3034
|
-
|
|
3035
|
-
effect () {
|
|
3036
|
-
return ()=>{};
|
|
1203
|
+
if (event === 'agent/request')
|
|
1204
|
+
modelRequest = listener;
|
|
1205
|
+
return () => { };
|
|
3037
1206
|
},
|
|
3038
|
-
|
|
3039
|
-
|
|
3040
|
-
|
|
3041
|
-
|
|
3042
|
-
|
|
3043
|
-
|
|
3044
|
-
if (name === 'agents') return {
|
|
3045
|
-
list () {
|
|
3046
|
-
return [];
|
|
3047
|
-
}
|
|
3048
|
-
};
|
|
1207
|
+
effect() { return () => { }; },
|
|
1208
|
+
get(name) {
|
|
1209
|
+
if (name === 'subprocess')
|
|
1210
|
+
return { spawn: (_spec) => fakeHandle({ stdout: '{}' }) };
|
|
1211
|
+
if (name === 'agents')
|
|
1212
|
+
return { list() { return []; } };
|
|
3049
1213
|
return undefined;
|
|
3050
|
-
}
|
|
1214
|
+
},
|
|
3051
1215
|
};
|
|
3052
1216
|
apply(modelCtx, {
|
|
3053
|
-
stateRoot: root,
|
|
3054
|
-
timeoutMs: 20,
|
|
3055
|
-
hbcliBin: '',
|
|
3056
|
-
sessionAccess: 'off',
|
|
3057
|
-
benchmarkEnvironmentConfigPath: configPath
|
|
1217
|
+
stateRoot: root, timeoutMs: 20, hbcliBin: '', sessionAccess: 'off', benchmarkEnvironmentConfigPath: configPath,
|
|
3058
1218
|
});
|
|
3059
|
-
assert.deepEqual(modelTools.map(
|
|
3060
|
-
'gotry_benchmark_environment'
|
|
3061
|
-
], 'benchmark model override does not re-enable product tools');
|
|
1219
|
+
assert.deepEqual(modelTools.map(tool => tool.name), ['gotry_benchmark_environment'], 'benchmark model override does not re-enable product tools');
|
|
3062
1220
|
assert.ok(modelEvents.includes('agent/request'), 'benchmark mode preserves the model override hook');
|
|
3063
1221
|
assert.ok(modelRequest);
|
|
3064
|
-
assert.deepEqual(await modelRequest({}, async ()=>({
|
|
3065
|
-
|
|
3066
|
-
|
|
3067
|
-
|
|
3068
|
-
|
|
3069
|
-
|
|
3070
|
-
|
|
3071
|
-
model: 'round7-model-preserved',
|
|
3072
|
-
marker: 'kept'
|
|
3073
|
-
}, 'benchmark model override remains effective and replaces the persisted model only');
|
|
3074
|
-
} finally{
|
|
3075
|
-
if (originalModelOverride === undefined) delete process.env.GOTRY_LLM_MODEL;
|
|
3076
|
-
else process.env.GOTRY_LLM_MODEL = originalModelOverride;
|
|
1222
|
+
assert.deepEqual(await modelRequest({}, async () => ({ provider: 'persisted', model: 'old', reasoningEffort: 'high', marker: 'kept' })), { provider: 'deepseek-official', model: 'round7-model-preserved', marker: 'kept' }, 'benchmark model override remains effective and replaces the persisted model only');
|
|
1223
|
+
}
|
|
1224
|
+
finally {
|
|
1225
|
+
if (originalModelOverride === undefined)
|
|
1226
|
+
delete process.env.GOTRY_LLM_MODEL;
|
|
1227
|
+
else
|
|
1228
|
+
process.env.GOTRY_LLM_MODEL = originalModelOverride;
|
|
3077
1229
|
}
|
|
3078
1230
|
const validConfig = {
|
|
3079
1231
|
schema_version: 'gotry_benchmark_environment_bridge_v4',
|
|
3080
1232
|
enabled: true,
|
|
3081
1233
|
executable: process.execPath,
|
|
3082
1234
|
cwd: root,
|
|
3083
|
-
argv_prefix: [
|
|
3084
|
-
|
|
3085
|
-
'agent_env.cli',
|
|
3086
|
-
'--lang',
|
|
3087
|
-
'en'
|
|
3088
|
-
],
|
|
3089
|
-
tools: [
|
|
3090
|
-
{
|
|
1235
|
+
argv_prefix: ['-m', 'agent_env.cli', '--lang', 'en'],
|
|
1236
|
+
tools: [{
|
|
3091
1237
|
name: 'lookup',
|
|
3092
1238
|
description: 'Lookup one declared city.',
|
|
3093
1239
|
input_schema: {
|
|
3094
1240
|
type: 'object',
|
|
3095
1241
|
properties: {
|
|
3096
|
-
city: {
|
|
3097
|
-
type: 'string',
|
|
3098
|
-
enum: [
|
|
3099
|
-
'Dubai',
|
|
3100
|
-
'Abu Dhabi'
|
|
3101
|
-
],
|
|
3102
|
-
description: 'Declared city name.'
|
|
3103
|
-
}
|
|
1242
|
+
city: { type: 'string', enum: ['Dubai', 'Abu Dhabi'], description: 'Declared city name.' },
|
|
3104
1243
|
},
|
|
3105
|
-
required: [
|
|
3106
|
-
|
|
3107
|
-
],
|
|
3108
|
-
additionalProperties: false
|
|
1244
|
+
required: ['city'],
|
|
1245
|
+
additionalProperties: false,
|
|
3109
1246
|
},
|
|
3110
|
-
output_keys: [
|
|
3111
|
-
|
|
3112
|
-
|
|
3113
|
-
domain_outcomes: [
|
|
3114
|
-
{
|
|
3115
|
-
status: 'miss',
|
|
3116
|
-
code: 'NOT_FOUND',
|
|
3117
|
-
recovery: 'revise_arguments'
|
|
3118
|
-
}
|
|
3119
|
-
]
|
|
3120
|
-
}
|
|
3121
|
-
],
|
|
1247
|
+
output_keys: ['city'],
|
|
1248
|
+
domain_outcomes: [{ status: 'miss', code: 'NOT_FOUND', recovery: 'revise_arguments' }],
|
|
1249
|
+
}],
|
|
3122
1250
|
timeout_ms: 20,
|
|
3123
1251
|
max_output_bytes: 4_096,
|
|
3124
|
-
terminal_output: {
|
|
3125
|
-
|
|
3126
|
-
max_bytes: 4_096,
|
|
3127
|
-
body_schema: OK_BODY_SCHEMA
|
|
3128
|
-
},
|
|
3129
|
-
isolation: {
|
|
3130
|
-
mode: 'host-enforced',
|
|
3131
|
-
writes: 'forbidden',
|
|
3132
|
-
network: 'denied'
|
|
3133
|
-
}
|
|
1252
|
+
terminal_output: { tag: 'done', max_bytes: 4_096, body_schema: OK_BODY_SCHEMA },
|
|
1253
|
+
isolation: { mode: 'host-enforced', writes: 'forbidden', network: 'denied' },
|
|
3134
1254
|
};
|
|
3135
|
-
const clonedConfig = ()=>JSON.parse(JSON.stringify(validConfig));
|
|
3136
|
-
const configWithTool = (mutate)=>{
|
|
1255
|
+
const clonedConfig = () => JSON.parse(JSON.stringify(validConfig));
|
|
1256
|
+
const configWithTool = (mutate) => {
|
|
3137
1257
|
const config = clonedConfig();
|
|
3138
1258
|
mutate(config.tools[0]);
|
|
3139
1259
|
return config;
|
|
3140
1260
|
};
|
|
3141
|
-
const nestedInputSchema = (objectLevels)=>{
|
|
3142
|
-
let child = {
|
|
3143
|
-
|
|
3144
|
-
};
|
|
3145
|
-
for(let index = 0; index < objectLevels; index += 1){
|
|
1261
|
+
const nestedInputSchema = (objectLevels) => {
|
|
1262
|
+
let child = { type: 'string' };
|
|
1263
|
+
for (let index = 0; index < objectLevels; index += 1) {
|
|
3146
1264
|
child = {
|
|
3147
1265
|
type: 'object',
|
|
3148
|
-
properties: {
|
|
3149
|
-
|
|
3150
|
-
|
|
3151
|
-
required: [
|
|
3152
|
-
'next'
|
|
3153
|
-
],
|
|
3154
|
-
additionalProperties: false
|
|
1266
|
+
properties: { next: child },
|
|
1267
|
+
required: ['next'],
|
|
1268
|
+
additionalProperties: false,
|
|
3155
1269
|
};
|
|
3156
1270
|
}
|
|
3157
1271
|
return child;
|
|
3158
1272
|
};
|
|
3159
|
-
const flatInputSchema = (count)=>({
|
|
3160
|
-
|
|
3161
|
-
|
|
3162
|
-
|
|
3163
|
-
|
|
3164
|
-
|
|
3165
|
-
|
|
3166
|
-
|
|
3167
|
-
|
|
3168
|
-
|
|
3169
|
-
|
|
3170
|
-
|
|
3171
|
-
|
|
3172
|
-
|
|
3173
|
-
|
|
3174
|
-
|
|
3175
|
-
|
|
3176
|
-
}
|
|
3177
|
-
|
|
3178
|
-
|
|
3179
|
-
|
|
3180
|
-
enum: Array.from({
|
|
3181
|
-
length: 64
|
|
3182
|
-
}, (_, index)=>`v${group}_${index}`)
|
|
3183
|
-
}
|
|
3184
|
-
])),
|
|
3185
|
-
required: [],
|
|
3186
|
-
additionalProperties: false
|
|
3187
|
-
});
|
|
3188
|
-
const byteSizedInputSchema = (targetBytes)=>{
|
|
3189
|
-
const properties = Object.fromEntries(Array.from({
|
|
3190
|
-
length: 64
|
|
3191
|
-
}, (_, index)=>[
|
|
3192
|
-
`b${index}`,
|
|
3193
|
-
{
|
|
3194
|
-
type: 'string',
|
|
3195
|
-
description: ''
|
|
3196
|
-
}
|
|
3197
|
-
]));
|
|
3198
|
-
const schema = {
|
|
3199
|
-
type: 'object',
|
|
3200
|
-
properties,
|
|
3201
|
-
required: [],
|
|
3202
|
-
additionalProperties: false
|
|
3203
|
-
};
|
|
1273
|
+
const flatInputSchema = (count) => ({
|
|
1274
|
+
type: 'object',
|
|
1275
|
+
properties: Object.fromEntries(Array.from({ length: count }, (_, index) => [`p${index}`, { type: 'string' }])),
|
|
1276
|
+
required: [],
|
|
1277
|
+
additionalProperties: false,
|
|
1278
|
+
});
|
|
1279
|
+
const totalEnumInputSchema = (groups) => ({
|
|
1280
|
+
type: 'object',
|
|
1281
|
+
properties: Object.fromEntries(Array.from({ length: groups }, (_, group) => [
|
|
1282
|
+
`e${group}`,
|
|
1283
|
+
{ type: 'string', enum: Array.from({ length: 64 }, (_, index) => `v${group}_${index}`) },
|
|
1284
|
+
])),
|
|
1285
|
+
required: [],
|
|
1286
|
+
additionalProperties: false,
|
|
1287
|
+
});
|
|
1288
|
+
const byteSizedInputSchema = (targetBytes) => {
|
|
1289
|
+
const properties = Object.fromEntries(Array.from({ length: 64 }, (_, index) => [
|
|
1290
|
+
`b${index}`,
|
|
1291
|
+
{ type: 'string', description: '' },
|
|
1292
|
+
]));
|
|
1293
|
+
const schema = { type: 'object', properties, required: [], additionalProperties: false };
|
|
3204
1294
|
let remaining = targetBytes - Buffer.byteLength(JSON.stringify(schema), 'utf8');
|
|
3205
1295
|
assert.ok(remaining >= 0 && remaining <= 64 * 512, 'byte-boundary fixture has enough bounded description capacity');
|
|
3206
|
-
for (const property of Object.values(properties)){
|
|
1296
|
+
for (const property of Object.values(properties)) {
|
|
3207
1297
|
const added = Math.min(512, remaining);
|
|
3208
1298
|
property.description = 'x'.repeat(added);
|
|
3209
1299
|
remaining -= added;
|
|
@@ -3212,548 +1302,153 @@ try {
|
|
|
3212
1302
|
assert.equal(Buffer.byteLength(JSON.stringify(schema), 'utf8'), targetBytes);
|
|
3213
1303
|
return schema;
|
|
3214
1304
|
};
|
|
3215
|
-
const frozenProjection = registerBenchmarkEnvironmentBridge(configPath, ()=>{}, {
|
|
3216
|
-
spawn: (_spec)=>fakeHandle({
|
|
3217
|
-
stdout: '{"result":{}}'
|
|
3218
|
-
})
|
|
1305
|
+
const frozenProjection = registerBenchmarkEnvironmentBridge(configPath, () => { }, {
|
|
1306
|
+
spawn: (_spec) => fakeHandle({ stdout: '{"result":{}}' }),
|
|
3219
1307
|
});
|
|
3220
1308
|
assert.equal(Object.isFrozen(frozenProjection), true, 'bridge projection is frozen');
|
|
3221
1309
|
assert.equal(Object.isFrozen(frozenProjection.allowedTools), true, 'projected allowlist is frozen');
|
|
3222
1310
|
assert.equal(Object.isFrozen(frozenProjection.terminal), true, 'projected terminal contract is frozen');
|
|
3223
|
-
assert.throws(()=>{
|
|
3224
|
-
|
|
3225
|
-
|
|
3226
|
-
assert.throws(()=>{
|
|
3227
|
-
frozenProjection.terminal.tag = 'escape';
|
|
3228
|
-
}, TypeError);
|
|
3229
|
-
const bridgeRegistrationFor = (config, withSubprocess = true)=>{
|
|
1311
|
+
assert.throws(() => { frozenProjection.allowedTools.push('escape'); }, TypeError);
|
|
1312
|
+
assert.throws(() => { frozenProjection.terminal.tag = 'escape'; }, TypeError);
|
|
1313
|
+
const bridgeRegistrationFor = (config, withSubprocess = true) => {
|
|
3230
1314
|
const path = join(root, `config-${Math.random().toString(36).slice(2)}.json`);
|
|
3231
1315
|
writeFileSync(path, JSON.stringify(config));
|
|
3232
1316
|
const tools = [];
|
|
3233
1317
|
const freshCtx = {
|
|
3234
|
-
tools: {
|
|
3235
|
-
|
|
3236
|
-
|
|
3237
|
-
|
|
3238
|
-
|
|
3239
|
-
|
|
3240
|
-
|
|
3241
|
-
|
|
3242
|
-
|
|
3243
|
-
return
|
|
3244
|
-
name: tool.name
|
|
3245
|
-
}));
|
|
3246
|
-
}
|
|
1318
|
+
tools: { register(tool) { tools.push(tool); return () => { }; }, get(name) { return tools.find(tool => tool.name === name); }, schemas() { return tools.map(tool => ({ name: tool.name })); } },
|
|
1319
|
+
systemPrompt: { variable() { } },
|
|
1320
|
+
on() { return () => { }; },
|
|
1321
|
+
effect(action) { return runEffect(action); },
|
|
1322
|
+
agents: { list() { return []; } },
|
|
1323
|
+
get(name) {
|
|
1324
|
+
if (name === 'subprocess')
|
|
1325
|
+
return this.subprocess;
|
|
1326
|
+
if (name === 'agents')
|
|
1327
|
+
return this.agents;
|
|
3247
1328
|
},
|
|
3248
|
-
|
|
3249
|
-
variable () {}
|
|
3250
|
-
},
|
|
3251
|
-
on () {
|
|
3252
|
-
return ()=>{};
|
|
3253
|
-
},
|
|
3254
|
-
effect (action) {
|
|
3255
|
-
return runEffect(action);
|
|
3256
|
-
},
|
|
3257
|
-
agents: {
|
|
3258
|
-
list () {
|
|
3259
|
-
return [];
|
|
3260
|
-
}
|
|
3261
|
-
},
|
|
3262
|
-
get (name) {
|
|
3263
|
-
if (name === 'subprocess') return this.subprocess;
|
|
3264
|
-
if (name === 'agents') return this.agents;
|
|
3265
|
-
},
|
|
3266
|
-
...withSubprocess ? {
|
|
3267
|
-
subprocess: {
|
|
3268
|
-
spawn: (_spec)=>fakeHandle({
|
|
3269
|
-
stdout: '{"result":{}}'
|
|
3270
|
-
})
|
|
3271
|
-
}
|
|
3272
|
-
} : {}
|
|
1329
|
+
...(withSubprocess ? { subprocess: { spawn: (_spec) => fakeHandle({ stdout: '{"result":{}}' }) } } : {}),
|
|
3273
1330
|
};
|
|
3274
1331
|
apply(freshCtx, {
|
|
3275
|
-
stateRoot: root,
|
|
3276
|
-
timeoutMs: 20,
|
|
3277
|
-
hbcliBin: '',
|
|
3278
|
-
sessionAccess: 'off',
|
|
3279
|
-
benchmarkEnvironmentConfigPath: path
|
|
1332
|
+
stateRoot: root, timeoutMs: 20, hbcliBin: '', sessionAccess: 'off', benchmarkEnvironmentConfigPath: path,
|
|
3280
1333
|
});
|
|
3281
|
-
return tools.some(
|
|
1334
|
+
return tools.some(tool => tool.name === 'gotry_benchmark_environment');
|
|
3282
1335
|
};
|
|
3283
|
-
assert.throws(()=>bridgeRegistrationFor({
|
|
3284
|
-
|
|
3285
|
-
|
|
3286
|
-
|
|
3287
|
-
|
|
3288
|
-
|
|
3289
|
-
|
|
3290
|
-
|
|
3291
|
-
|
|
3292
|
-
|
|
3293
|
-
|
|
3294
|
-
|
|
3295
|
-
|
|
3296
|
-
|
|
3297
|
-
|
|
3298
|
-
assert.throws(()=>bridgeRegistrationFor({
|
|
3299
|
-
|
|
3300
|
-
|
|
3301
|
-
}), /benchmark environment bridge configuration unavailable/, 'v1 config cannot silently omit the Round 3 terminal semantics');
|
|
3302
|
-
assert.throws(()=>bridgeRegistrationFor({
|
|
3303
|
-
...validConfig,
|
|
3304
|
-
schema_version: 'gotry_benchmark_environment_bridge_v3'
|
|
3305
|
-
}), /benchmark environment bridge configuration unavailable/, 'v3 config cannot silently omit the Round 12 terminal body schema (#215)');
|
|
3306
|
-
assert.throws(()=>bridgeRegistrationFor({
|
|
3307
|
-
...validConfig,
|
|
3308
|
-
terminal_output: {
|
|
3309
|
-
...validConfig.terminal_output,
|
|
3310
|
-
body_schema: {
|
|
3311
|
-
...validConfig.terminal_output.body_schema,
|
|
3312
|
-
additionalProperties: true
|
|
3313
|
-
}
|
|
3314
|
-
}
|
|
3315
|
-
}), /benchmark environment bridge configuration unavailable/, 'non-closed terminal body schema fails the config load closed');
|
|
3316
|
-
assert.throws(()=>bridgeRegistrationFor({
|
|
3317
|
-
...validConfig,
|
|
3318
|
-
tools: [
|
|
3319
|
-
validConfig.tools[0],
|
|
3320
|
-
validConfig.tools[0]
|
|
3321
|
-
]
|
|
3322
|
-
}), /benchmark environment bridge configuration unavailable/, 'duplicate tool descriptor fails hard');
|
|
3323
|
-
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3324
|
-
delete tool.output_keys;
|
|
3325
|
-
})), /benchmark environment bridge configuration unavailable/, 'missing output_keys fails hard');
|
|
3326
|
-
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3327
|
-
tool.output_keys = [];
|
|
3328
|
-
})), /benchmark environment bridge configuration unavailable/, 'empty output allowlist fails closed at registration');
|
|
3329
|
-
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3330
|
-
tool.output_keys = [
|
|
3331
|
-
'city',
|
|
3332
|
-
'city'
|
|
3333
|
-
];
|
|
3334
|
-
})), /benchmark environment bridge configuration unavailable/, 'duplicate output key fails hard');
|
|
3335
|
-
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3336
|
-
tool.output_keys = [
|
|
3337
|
-
'not a key'
|
|
3338
|
-
];
|
|
3339
|
-
})), /benchmark environment bridge configuration unavailable/, 'non-identifier output key fails hard');
|
|
3340
|
-
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3341
|
-
delete tool.domain_outcomes;
|
|
3342
|
-
})), /benchmark environment bridge configuration unavailable/, 'missing domain_outcomes fails hard');
|
|
3343
|
-
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3344
|
-
tool.domain_outcomes = [
|
|
3345
|
-
{
|
|
3346
|
-
status: 'unknown',
|
|
3347
|
-
code: 'NOT_FOUND',
|
|
3348
|
-
recovery: 'none'
|
|
3349
|
-
}
|
|
3350
|
-
];
|
|
3351
|
-
})), /benchmark environment bridge configuration unavailable/, 'unknown domain status fails hard');
|
|
3352
|
-
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3353
|
-
tool.domain_outcomes = [
|
|
3354
|
-
{
|
|
3355
|
-
status: 'miss',
|
|
3356
|
-
code: 'NOT_FOUND',
|
|
3357
|
-
recovery: 'free_text'
|
|
3358
|
-
}
|
|
3359
|
-
];
|
|
3360
|
-
})), /benchmark environment bridge configuration unavailable/, 'free-text domain recovery fails hard');
|
|
3361
|
-
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3362
|
-
tool.domain_outcomes = [
|
|
3363
|
-
{
|
|
3364
|
-
status: 'miss',
|
|
3365
|
-
code: 'NOT_FOUND',
|
|
3366
|
-
recovery: 'none'
|
|
3367
|
-
},
|
|
3368
|
-
{
|
|
3369
|
-
status: 'error',
|
|
3370
|
-
code: 'NOT_FOUND',
|
|
3371
|
-
recovery: 'none'
|
|
3372
|
-
}
|
|
3373
|
-
];
|
|
3374
|
-
})), /benchmark environment bridge configuration unavailable/, 'duplicate domain code fails hard');
|
|
3375
|
-
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3376
|
-
tool.name = 'lookup;rm';
|
|
3377
|
-
})), /benchmark environment bridge configuration unavailable/, 'shell metacharacter tool identifier fails hard');
|
|
1336
|
+
assert.throws(() => bridgeRegistrationFor({ ...validConfig, unknown: true }), /benchmark environment bridge configuration unavailable/, 'unknown top-level config key fails hard');
|
|
1337
|
+
assert.throws(() => bridgeRegistrationFor({
|
|
1338
|
+
...validConfig,
|
|
1339
|
+
tools: [{ ...validConfig.tools[0], input_schema: { type: 'object' } }],
|
|
1340
|
+
}), /benchmark environment bridge configuration unavailable/, 'v3 rejects an open annotation-only input schema');
|
|
1341
|
+
assert.throws(() => bridgeRegistrationFor({ ...validConfig, schema_version: 'gotry_benchmark_environment_bridge_v1' }), /benchmark environment bridge configuration unavailable/, 'v1 config cannot silently omit the Round 3 terminal semantics');
|
|
1342
|
+
assert.throws(() => bridgeRegistrationFor({ ...validConfig, schema_version: 'gotry_benchmark_environment_bridge_v3' }), /benchmark environment bridge configuration unavailable/, 'v3 config cannot silently omit the Round 12 terminal body schema (#215)');
|
|
1343
|
+
assert.throws(() => bridgeRegistrationFor({ ...validConfig, terminal_output: { ...validConfig.terminal_output, body_schema: { ...validConfig.terminal_output.body_schema, additionalProperties: true } } }), /benchmark environment bridge configuration unavailable/, 'non-closed terminal body schema fails the config load closed');
|
|
1344
|
+
assert.throws(() => bridgeRegistrationFor({ ...validConfig, tools: [validConfig.tools[0], validConfig.tools[0]] }), /benchmark environment bridge configuration unavailable/, 'duplicate tool descriptor fails hard');
|
|
1345
|
+
assert.throws(() => bridgeRegistrationFor(configWithTool(tool => { delete tool.output_keys; })), /benchmark environment bridge configuration unavailable/, 'missing output_keys fails hard');
|
|
1346
|
+
assert.throws(() => bridgeRegistrationFor(configWithTool(tool => { tool.output_keys = []; })), /benchmark environment bridge configuration unavailable/, 'empty output allowlist fails closed at registration');
|
|
1347
|
+
assert.throws(() => bridgeRegistrationFor(configWithTool(tool => { tool.output_keys = ['city', 'city']; })), /benchmark environment bridge configuration unavailable/, 'duplicate output key fails hard');
|
|
1348
|
+
assert.throws(() => bridgeRegistrationFor(configWithTool(tool => { tool.output_keys = ['not a key']; })), /benchmark environment bridge configuration unavailable/, 'non-identifier output key fails hard');
|
|
1349
|
+
assert.throws(() => bridgeRegistrationFor(configWithTool(tool => { delete tool.domain_outcomes; })), /benchmark environment bridge configuration unavailable/, 'missing domain_outcomes fails hard');
|
|
1350
|
+
assert.throws(() => bridgeRegistrationFor(configWithTool(tool => { tool.domain_outcomes = [{ status: 'unknown', code: 'NOT_FOUND', recovery: 'none' }]; })), /benchmark environment bridge configuration unavailable/, 'unknown domain status fails hard');
|
|
1351
|
+
assert.throws(() => bridgeRegistrationFor(configWithTool(tool => { tool.domain_outcomes = [{ status: 'miss', code: 'NOT_FOUND', recovery: 'free_text' }]; })), /benchmark environment bridge configuration unavailable/, 'free-text domain recovery fails hard');
|
|
1352
|
+
assert.throws(() => bridgeRegistrationFor(configWithTool(tool => { tool.domain_outcomes = [{ status: 'miss', code: 'NOT_FOUND', recovery: 'none' }, { status: 'error', code: 'NOT_FOUND', recovery: 'none' }]; })), /benchmark environment bridge configuration unavailable/, 'duplicate domain code fails hard');
|
|
1353
|
+
assert.throws(() => bridgeRegistrationFor(configWithTool(tool => { tool.name = 'lookup;rm'; })), /benchmark environment bridge configuration unavailable/, 'shell metacharacter tool identifier fails hard');
|
|
3378
1354
|
for (const [label, schema] of [
|
|
3379
|
-
[
|
|
3380
|
-
|
|
3381
|
-
|
|
3382
|
-
|
|
3383
|
-
|
|
3384
|
-
|
|
3385
|
-
|
|
3386
|
-
|
|
3387
|
-
|
|
3388
|
-
|
|
3389
|
-
|
|
3390
|
-
|
|
3391
|
-
additionalProperties: false
|
|
3392
|
-
}
|
|
3393
|
-
],
|
|
3394
|
-
[
|
|
3395
|
-
'default keyword',
|
|
3396
|
-
{
|
|
3397
|
-
type: 'object',
|
|
3398
|
-
properties: {
|
|
3399
|
-
city: {
|
|
3400
|
-
type: 'string',
|
|
3401
|
-
default: 'Dubai'
|
|
3402
|
-
}
|
|
3403
|
-
},
|
|
3404
|
-
required: [
|
|
3405
|
-
'city'
|
|
3406
|
-
],
|
|
3407
|
-
additionalProperties: false
|
|
3408
|
-
}
|
|
3409
|
-
],
|
|
3410
|
-
[
|
|
3411
|
-
'examples keyword',
|
|
3412
|
-
{
|
|
3413
|
-
type: 'object',
|
|
3414
|
-
properties: {
|
|
3415
|
-
city: {
|
|
3416
|
-
type: 'string',
|
|
3417
|
-
examples: [
|
|
3418
|
-
'Dubai'
|
|
3419
|
-
]
|
|
3420
|
-
}
|
|
3421
|
-
},
|
|
3422
|
-
required: [
|
|
3423
|
-
'city'
|
|
3424
|
-
],
|
|
3425
|
-
additionalProperties: false
|
|
3426
|
-
}
|
|
3427
|
-
],
|
|
3428
|
-
[
|
|
3429
|
-
'unknown keyword',
|
|
3430
|
-
{
|
|
3431
|
-
type: 'object',
|
|
3432
|
-
properties: {
|
|
3433
|
-
city: {
|
|
3434
|
-
type: 'string',
|
|
3435
|
-
title: 'City'
|
|
3436
|
-
}
|
|
3437
|
-
},
|
|
3438
|
-
required: [
|
|
3439
|
-
'city'
|
|
3440
|
-
],
|
|
3441
|
-
additionalProperties: false
|
|
3442
|
-
}
|
|
3443
|
-
],
|
|
3444
|
-
[
|
|
3445
|
-
'open nested object',
|
|
3446
|
-
{
|
|
3447
|
-
type: 'object',
|
|
3448
|
-
properties: {
|
|
3449
|
-
nested: {
|
|
3450
|
-
type: 'object',
|
|
3451
|
-
properties: {},
|
|
3452
|
-
required: []
|
|
3453
|
-
}
|
|
3454
|
-
},
|
|
3455
|
-
required: [
|
|
3456
|
-
'nested'
|
|
3457
|
-
],
|
|
3458
|
-
additionalProperties: false
|
|
3459
|
-
}
|
|
3460
|
-
],
|
|
3461
|
-
[
|
|
3462
|
-
'array without items',
|
|
3463
|
-
{
|
|
3464
|
-
type: 'object',
|
|
3465
|
-
properties: {
|
|
3466
|
-
values: {
|
|
3467
|
-
type: 'array'
|
|
3468
|
-
}
|
|
3469
|
-
},
|
|
3470
|
-
required: [
|
|
3471
|
-
'values'
|
|
3472
|
-
],
|
|
3473
|
-
additionalProperties: false
|
|
3474
|
-
}
|
|
3475
|
-
],
|
|
3476
|
-
[
|
|
3477
|
-
'duplicate required',
|
|
3478
|
-
{
|
|
3479
|
-
type: 'object',
|
|
3480
|
-
properties: {
|
|
3481
|
-
city: {
|
|
3482
|
-
type: 'string'
|
|
3483
|
-
}
|
|
3484
|
-
},
|
|
3485
|
-
required: [
|
|
3486
|
-
'city',
|
|
3487
|
-
'city'
|
|
3488
|
-
],
|
|
3489
|
-
additionalProperties: false
|
|
3490
|
-
}
|
|
3491
|
-
],
|
|
3492
|
-
[
|
|
3493
|
-
'unknown required',
|
|
3494
|
-
{
|
|
3495
|
-
type: 'object',
|
|
3496
|
-
properties: {
|
|
3497
|
-
city: {
|
|
3498
|
-
type: 'string'
|
|
3499
|
-
}
|
|
3500
|
-
},
|
|
3501
|
-
required: [
|
|
3502
|
-
'country'
|
|
3503
|
-
],
|
|
3504
|
-
additionalProperties: false
|
|
3505
|
-
}
|
|
3506
|
-
],
|
|
3507
|
-
[
|
|
3508
|
-
'type-mismatched enum',
|
|
3509
|
-
{
|
|
3510
|
-
type: 'object',
|
|
3511
|
-
properties: {
|
|
3512
|
-
city: {
|
|
3513
|
-
type: 'string',
|
|
3514
|
-
enum: [
|
|
3515
|
-
'Dubai',
|
|
3516
|
-
7
|
|
3517
|
-
]
|
|
3518
|
-
}
|
|
3519
|
-
},
|
|
3520
|
-
required: [
|
|
3521
|
-
'city'
|
|
3522
|
-
],
|
|
3523
|
-
additionalProperties: false
|
|
3524
|
-
}
|
|
3525
|
-
],
|
|
3526
|
-
[
|
|
3527
|
-
'duplicate enum',
|
|
3528
|
-
{
|
|
3529
|
-
type: 'object',
|
|
3530
|
-
properties: {
|
|
3531
|
-
city: {
|
|
3532
|
-
type: 'string',
|
|
3533
|
-
enum: [
|
|
3534
|
-
'Dubai',
|
|
3535
|
-
'Dubai'
|
|
3536
|
-
]
|
|
3537
|
-
}
|
|
3538
|
-
},
|
|
3539
|
-
required: [
|
|
3540
|
-
'city'
|
|
3541
|
-
],
|
|
3542
|
-
additionalProperties: false
|
|
3543
|
-
}
|
|
3544
|
-
]
|
|
3545
|
-
]){
|
|
3546
|
-
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3547
|
-
tool.input_schema = schema;
|
|
3548
|
-
})), /benchmark environment bridge configuration unavailable/, `${label} fails closed`);
|
|
1355
|
+
['reference keyword', { type: 'object', properties: { city: { $ref: '#/city' } }, required: ['city'], additionalProperties: false }],
|
|
1356
|
+
['default keyword', { type: 'object', properties: { city: { type: 'string', default: 'Dubai' } }, required: ['city'], additionalProperties: false }],
|
|
1357
|
+
['examples keyword', { type: 'object', properties: { city: { type: 'string', examples: ['Dubai'] } }, required: ['city'], additionalProperties: false }],
|
|
1358
|
+
['unknown keyword', { type: 'object', properties: { city: { type: 'string', title: 'City' } }, required: ['city'], additionalProperties: false }],
|
|
1359
|
+
['open nested object', { type: 'object', properties: { nested: { type: 'object', properties: {}, required: [] } }, required: ['nested'], additionalProperties: false }],
|
|
1360
|
+
['array without items', { type: 'object', properties: { values: { type: 'array' } }, required: ['values'], additionalProperties: false }],
|
|
1361
|
+
['duplicate required', { type: 'object', properties: { city: { type: 'string' } }, required: ['city', 'city'], additionalProperties: false }],
|
|
1362
|
+
['unknown required', { type: 'object', properties: { city: { type: 'string' } }, required: ['country'], additionalProperties: false }],
|
|
1363
|
+
['type-mismatched enum', { type: 'object', properties: { city: { type: 'string', enum: ['Dubai', 7] } }, required: ['city'], additionalProperties: false }],
|
|
1364
|
+
['duplicate enum', { type: 'object', properties: { city: { type: 'string', enum: ['Dubai', 'Dubai'] } }, required: ['city'], additionalProperties: false }],
|
|
1365
|
+
]) {
|
|
1366
|
+
assert.throws(() => bridgeRegistrationFor(configWithTool(tool => { tool.input_schema = schema; })), /benchmark environment bridge configuration unavailable/, `${label} fails closed`);
|
|
3549
1367
|
}
|
|
3550
|
-
assert.equal(bridgeRegistrationFor(configWithTool(
|
|
3551
|
-
|
|
3552
|
-
})), true, '
|
|
3553
|
-
assert.throws(()=>bridgeRegistrationFor(configWithTool(
|
|
3554
|
-
|
|
3555
|
-
|
|
3556
|
-
assert.
|
|
3557
|
-
|
|
3558
|
-
})),
|
|
3559
|
-
assert.throws(()=>bridgeRegistrationFor(
|
|
3560
|
-
|
|
3561
|
-
})), /benchmark environment bridge configuration unavailable/, 'the 256-node total boundary plus one is rejected');
|
|
3562
|
-
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3563
|
-
tool.input_schema = flatInputSchema(257);
|
|
3564
|
-
})), /benchmark environment bridge configuration unavailable/, 'the property total bound rejects 257 properties');
|
|
3565
|
-
assert.equal(bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3566
|
-
tool.input_schema = totalEnumInputSchema(4);
|
|
3567
|
-
})), true, 'the 256-enum-value total boundary is accepted');
|
|
3568
|
-
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3569
|
-
tool.input_schema = totalEnumInputSchema(5);
|
|
3570
|
-
})), /benchmark environment bridge configuration unavailable/, 'the enum-value total bound rejects values above 256');
|
|
3571
|
-
assert.equal(bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3572
|
-
tool.input_schema = byteSizedInputSchema(16 * 1024);
|
|
3573
|
-
})), true, 'the 16 KiB schema boundary is accepted');
|
|
3574
|
-
assert.throws(()=>bridgeRegistrationFor(configWithTool((tool)=>{
|
|
3575
|
-
tool.input_schema = byteSizedInputSchema(16 * 1024 + 1);
|
|
3576
|
-
})), /benchmark environment bridge configuration unavailable/, 'the schema byte boundary plus one is rejected');
|
|
3577
|
-
assert.throws(()=>bridgeRegistrationFor({
|
|
3578
|
-
...validConfig,
|
|
3579
|
-
argv_prefix: [
|
|
3580
|
-
'agent\n--unsafe'
|
|
3581
|
-
]
|
|
3582
|
-
}), /benchmark environment bridge configuration unavailable/, 'argv control separator fails hard');
|
|
3583
|
-
assert.throws(()=>bridgeRegistrationFor({
|
|
3584
|
-
...validConfig,
|
|
3585
|
-
isolation: {
|
|
3586
|
-
mode: 'host-enforced',
|
|
3587
|
-
writes: 'forbidden'
|
|
3588
|
-
}
|
|
3589
|
-
}), /benchmark environment bridge configuration unavailable/, 'incomplete isolation policy fails hard');
|
|
1368
|
+
assert.equal(bridgeRegistrationFor(configWithTool(tool => { tool.input_schema = nestedInputSchema(8); })), true, 'schema depth eight is accepted');
|
|
1369
|
+
assert.throws(() => bridgeRegistrationFor(configWithTool(tool => { tool.input_schema = nestedInputSchema(9); })), /benchmark environment bridge configuration unavailable/, 'schema depth nine is rejected');
|
|
1370
|
+
assert.equal(bridgeRegistrationFor(configWithTool(tool => { tool.input_schema = flatInputSchema(255); })), true, 'the 256-node total boundary is accepted');
|
|
1371
|
+
assert.throws(() => bridgeRegistrationFor(configWithTool(tool => { tool.input_schema = flatInputSchema(256); })), /benchmark environment bridge configuration unavailable/, 'the 256-node total boundary plus one is rejected');
|
|
1372
|
+
assert.throws(() => bridgeRegistrationFor(configWithTool(tool => { tool.input_schema = flatInputSchema(257); })), /benchmark environment bridge configuration unavailable/, 'the property total bound rejects 257 properties');
|
|
1373
|
+
assert.equal(bridgeRegistrationFor(configWithTool(tool => { tool.input_schema = totalEnumInputSchema(4); })), true, 'the 256-enum-value total boundary is accepted');
|
|
1374
|
+
assert.throws(() => bridgeRegistrationFor(configWithTool(tool => { tool.input_schema = totalEnumInputSchema(5); })), /benchmark environment bridge configuration unavailable/, 'the enum-value total bound rejects values above 256');
|
|
1375
|
+
assert.equal(bridgeRegistrationFor(configWithTool(tool => { tool.input_schema = byteSizedInputSchema(16 * 1024); })), true, 'the 16 KiB schema boundary is accepted');
|
|
1376
|
+
assert.throws(() => bridgeRegistrationFor(configWithTool(tool => { tool.input_schema = byteSizedInputSchema(16 * 1024 + 1); })), /benchmark environment bridge configuration unavailable/, 'the schema byte boundary plus one is rejected');
|
|
1377
|
+
assert.throws(() => bridgeRegistrationFor({ ...validConfig, argv_prefix: ['agent\n--unsafe'] }), /benchmark environment bridge configuration unavailable/, 'argv control separator fails hard');
|
|
1378
|
+
assert.throws(() => bridgeRegistrationFor({ ...validConfig, isolation: { mode: 'host-enforced', writes: 'forbidden' } }), /benchmark environment bridge configuration unavailable/, 'incomplete isolation policy fails hard');
|
|
3590
1379
|
const { terminal_output: _terminalOutput, ...missingTerminalConfig } = validConfig;
|
|
3591
|
-
assert.throws(()=>bridgeRegistrationFor(missingTerminalConfig), /benchmark environment bridge configuration unavailable/, 'terminal output contract is required');
|
|
3592
|
-
assert.throws(()=>bridgeRegistrationFor({
|
|
3593
|
-
|
|
3594
|
-
|
|
3595
|
-
|
|
3596
|
-
|
|
3597
|
-
}
|
|
3598
|
-
}), /benchmark environment bridge configuration unavailable/, 'terminal tag must be an identifier');
|
|
3599
|
-
assert.throws(()=>bridgeRegistrationFor({
|
|
3600
|
-
...validConfig,
|
|
3601
|
-
terminal_output: {
|
|
3602
|
-
tag: 'done',
|
|
3603
|
-
max_bytes: 0
|
|
3604
|
-
}
|
|
3605
|
-
}), /benchmark environment bridge configuration unavailable/, 'terminal output lower bound fails hard');
|
|
3606
|
-
assert.throws(()=>bridgeRegistrationFor({
|
|
3607
|
-
...validConfig,
|
|
3608
|
-
terminal_output: {
|
|
3609
|
-
tag: 'done',
|
|
3610
|
-
max_bytes: 1024 * 1024 + 1
|
|
3611
|
-
}
|
|
3612
|
-
}), /benchmark environment bridge configuration unavailable/, 'terminal output upper bound fails hard');
|
|
3613
|
-
assert.throws(()=>bridgeRegistrationFor({
|
|
3614
|
-
...validConfig,
|
|
3615
|
-
terminal_output: {
|
|
3616
|
-
tag: 'done',
|
|
3617
|
-
max_bytes: 4_096,
|
|
3618
|
-
extra: true
|
|
3619
|
-
}
|
|
3620
|
-
}), /benchmark environment bridge configuration unavailable/, 'terminal output rejects unknown keys');
|
|
3621
|
-
assert.throws(()=>bridgeRegistrationFor(validConfig, false), /benchmark environment bridge subprocess unavailable/, 'explicit opt-in without an active subprocess provider fails hard');
|
|
1380
|
+
assert.throws(() => bridgeRegistrationFor(missingTerminalConfig), /benchmark environment bridge configuration unavailable/, 'terminal output contract is required');
|
|
1381
|
+
assert.throws(() => bridgeRegistrationFor({ ...validConfig, terminal_output: { tag: '1invalid', max_bytes: 4_096 } }), /benchmark environment bridge configuration unavailable/, 'terminal tag must be an identifier');
|
|
1382
|
+
assert.throws(() => bridgeRegistrationFor({ ...validConfig, terminal_output: { tag: 'done', max_bytes: 0 } }), /benchmark environment bridge configuration unavailable/, 'terminal output lower bound fails hard');
|
|
1383
|
+
assert.throws(() => bridgeRegistrationFor({ ...validConfig, terminal_output: { tag: 'done', max_bytes: 1024 * 1024 + 1 } }), /benchmark environment bridge configuration unavailable/, 'terminal output upper bound fails hard');
|
|
1384
|
+
assert.throws(() => bridgeRegistrationFor({ ...validConfig, terminal_output: { tag: 'done', max_bytes: 4_096, extra: true } }), /benchmark environment bridge configuration unavailable/, 'terminal output rejects unknown keys');
|
|
1385
|
+
assert.throws(() => bridgeRegistrationFor(validConfig, false), /benchmark environment bridge subprocess unavailable/, 'explicit opt-in without an active subprocess provider fails hard');
|
|
3622
1386
|
assert.equal(loadConfigRegistration(configPath, root), true, 'owner-local regular 0644 config remains valid');
|
|
3623
|
-
assert.throws(()=>loadConfigRegistration('bridge.json', root), /benchmark environment bridge configuration unavailable/, 'relative config path fails hard');
|
|
1387
|
+
assert.throws(() => loadConfigRegistration('bridge.json', root), /benchmark environment bridge configuration unavailable/, 'relative config path fails hard');
|
|
3624
1388
|
const symlinkPath = join(root, 'bridge-symlink.json');
|
|
3625
1389
|
symlinkSync(configPath, symlinkPath);
|
|
3626
1390
|
assert.equal(lstatSync(symlinkPath).isSymbolicLink(), true);
|
|
3627
|
-
assert.throws(()=>loadConfigRegistration(symlinkPath, root), /benchmark environment bridge configuration unavailable/, 'symlink config path fails hard');
|
|
1391
|
+
assert.throws(() => loadConfigRegistration(symlinkPath, root), /benchmark environment bridge configuration unavailable/, 'symlink config path fails hard');
|
|
3628
1392
|
const widePath = join(root, 'bridge-wide.json');
|
|
3629
1393
|
writeFileSync(widePath, JSON.stringify(validConfig));
|
|
3630
1394
|
chmodSync(widePath, 0o666);
|
|
3631
|
-
assert.throws(()=>loadConfigRegistration(widePath, root), /benchmark environment bridge configuration unavailable/, 'group/world writable config fails hard');
|
|
1395
|
+
assert.throws(() => loadConfigRegistration(widePath, root), /benchmark environment bridge configuration unavailable/, 'group/world writable config fails hard');
|
|
3632
1396
|
const strictConfigPath = join(root, 'strict-bridge.json');
|
|
3633
1397
|
writeFileSync(strictConfigPath, JSON.stringify(validConfig));
|
|
3634
1398
|
const strictTools = [];
|
|
3635
1399
|
const strictCtx = {
|
|
3636
|
-
tools: {
|
|
3637
|
-
|
|
3638
|
-
|
|
3639
|
-
|
|
3640
|
-
|
|
3641
|
-
|
|
3642
|
-
|
|
3643
|
-
|
|
3644
|
-
|
|
3645
|
-
return
|
|
3646
|
-
name: tool.name
|
|
3647
|
-
}));
|
|
3648
|
-
}
|
|
3649
|
-
},
|
|
3650
|
-
systemPrompt: {
|
|
3651
|
-
variable () {}
|
|
3652
|
-
},
|
|
3653
|
-
on () {
|
|
3654
|
-
return ()=>{};
|
|
3655
|
-
},
|
|
3656
|
-
effect (_action) {
|
|
3657
|
-
return ()=>{};
|
|
1400
|
+
tools: { register(tool) { strictTools.push(tool); return () => { }; }, get(name) { return strictTools.find(tool => tool.name === name); }, schemas() { return strictTools.map(tool => ({ name: tool.name })); } },
|
|
1401
|
+
systemPrompt: { variable() { } },
|
|
1402
|
+
on() { return () => { }; },
|
|
1403
|
+
effect(_action) { return () => { }; },
|
|
1404
|
+
agents: { list() { return []; } },
|
|
1405
|
+
get(name) {
|
|
1406
|
+
if (name === 'subprocess')
|
|
1407
|
+
return this.subprocess;
|
|
1408
|
+
if (name === 'agents')
|
|
1409
|
+
return this.agents;
|
|
3658
1410
|
},
|
|
3659
|
-
|
|
3660
|
-
list () {
|
|
3661
|
-
return [];
|
|
3662
|
-
}
|
|
3663
|
-
},
|
|
3664
|
-
get (name) {
|
|
3665
|
-
if (name === 'subprocess') return this.subprocess;
|
|
3666
|
-
if (name === 'agents') return this.agents;
|
|
3667
|
-
},
|
|
3668
|
-
subprocess: {
|
|
3669
|
-
spawn: (_spec)=>fakeHandle({
|
|
3670
|
-
stdout: okEnvelope({
|
|
3671
|
-
city: 'Dubai'
|
|
3672
|
-
})
|
|
3673
|
-
})
|
|
3674
|
-
}
|
|
1411
|
+
subprocess: { spawn: (_spec) => fakeHandle({ stdout: okEnvelope({ city: 'Dubai' }) }) },
|
|
3675
1412
|
};
|
|
3676
1413
|
apply(strictCtx, {
|
|
3677
|
-
stateRoot: root,
|
|
3678
|
-
timeoutMs: 20,
|
|
3679
|
-
hbcliBin: '',
|
|
3680
|
-
sessionAccess: 'off',
|
|
3681
|
-
benchmarkEnvironmentConfigPath: strictConfigPath
|
|
1414
|
+
stateRoot: root, timeoutMs: 20, hbcliBin: '', sessionAccess: 'off', benchmarkEnvironmentConfigPath: strictConfigPath,
|
|
3682
1415
|
});
|
|
3683
|
-
const strictBridge = strictTools.find(
|
|
1416
|
+
const strictBridge = strictTools.find(tool => tool.name === 'gotry_benchmark_environment');
|
|
3684
1417
|
assert.ok(strictBridge?.execute, 'strict v3 config registers the bridge');
|
|
3685
|
-
assert.deepEqual(await strictBridge.execute(bridgeCall('lookup', {
|
|
3686
|
-
city: 'Dubai'
|
|
3687
|
-
}), null), {
|
|
3688
|
-
ok: true,
|
|
3689
|
-
result: {
|
|
3690
|
-
city: 'Dubai'
|
|
3691
|
-
}
|
|
3692
|
-
}, 'strict v3 result envelope executes safe structured output');
|
|
1418
|
+
assert.deepEqual(await strictBridge.execute(bridgeCall('lookup', { city: 'Dubai' }), null), { ok: true, result: { city: 'Dubai' } }, 'strict v3 result envelope executes safe structured output');
|
|
3693
1419
|
console.log('BENCHMARK ENVIRONMENT BRIDGE TESTS: registration + TDD bridge contract assertions');
|
|
3694
|
-
}
|
|
1420
|
+
}
|
|
1421
|
+
catch (error) {
|
|
3695
1422
|
console.error(error);
|
|
3696
1423
|
process.exitCode = 1;
|
|
3697
|
-
}
|
|
3698
|
-
|
|
3699
|
-
|
|
3700
|
-
|
|
1424
|
+
}
|
|
1425
|
+
finally {
|
|
1426
|
+
for (const [name, value] of ambientSentinels) {
|
|
1427
|
+
if (value === undefined)
|
|
1428
|
+
delete process.env[name];
|
|
1429
|
+
else
|
|
1430
|
+
process.env[name] = value;
|
|
3701
1431
|
}
|
|
3702
|
-
rmSync(root, {
|
|
3703
|
-
recursive: true,
|
|
3704
|
-
force: true
|
|
3705
|
-
});
|
|
1432
|
+
rmSync(root, { recursive: true, force: true });
|
|
3706
1433
|
}
|
|
3707
1434
|
function loadConfigRegistration(path, stateRoot) {
|
|
3708
1435
|
const tools = [];
|
|
3709
1436
|
const freshCtx = {
|
|
3710
|
-
tools: {
|
|
3711
|
-
|
|
3712
|
-
|
|
3713
|
-
|
|
3714
|
-
|
|
3715
|
-
|
|
3716
|
-
|
|
3717
|
-
|
|
3718
|
-
|
|
3719
|
-
return
|
|
3720
|
-
name: tool.name
|
|
3721
|
-
}));
|
|
3722
|
-
}
|
|
1437
|
+
tools: { register(tool) { tools.push(tool); return () => { }; }, get(name) { return tools.find(tool => tool.name === name); }, schemas() { return tools.map(tool => ({ name: tool.name })); } },
|
|
1438
|
+
systemPrompt: { variable() { } },
|
|
1439
|
+
agents: { list() { return []; } },
|
|
1440
|
+
on() { return () => { }; },
|
|
1441
|
+
effect(_action) { return () => { }; },
|
|
1442
|
+
get(name) {
|
|
1443
|
+
if (name === 'subprocess')
|
|
1444
|
+
return this.subprocess;
|
|
1445
|
+
if (name === 'agents')
|
|
1446
|
+
return this.agents;
|
|
3723
1447
|
},
|
|
3724
|
-
|
|
3725
|
-
variable () {}
|
|
3726
|
-
},
|
|
3727
|
-
agents: {
|
|
3728
|
-
list () {
|
|
3729
|
-
return [];
|
|
3730
|
-
}
|
|
3731
|
-
},
|
|
3732
|
-
on () {
|
|
3733
|
-
return ()=>{};
|
|
3734
|
-
},
|
|
3735
|
-
effect (_action) {
|
|
3736
|
-
return ()=>{};
|
|
3737
|
-
},
|
|
3738
|
-
get (name) {
|
|
3739
|
-
if (name === 'subprocess') return this.subprocess;
|
|
3740
|
-
if (name === 'agents') return this.agents;
|
|
3741
|
-
},
|
|
3742
|
-
subprocess: {
|
|
3743
|
-
spawn: (_spec)=>fakeHandle({
|
|
3744
|
-
stdout: '{"result":{}}'
|
|
3745
|
-
})
|
|
3746
|
-
}
|
|
1448
|
+
subprocess: { spawn: (_spec) => fakeHandle({ stdout: '{"result":{}}' }) },
|
|
3747
1449
|
};
|
|
3748
1450
|
apply(freshCtx, {
|
|
3749
|
-
stateRoot,
|
|
3750
|
-
timeoutMs: 20,
|
|
3751
|
-
hbcliBin: '',
|
|
3752
|
-
sessionAccess: 'off',
|
|
3753
|
-
benchmarkEnvironmentConfigPath: path
|
|
1451
|
+
stateRoot, timeoutMs: 20, hbcliBin: '', sessionAccess: 'off', benchmarkEnvironmentConfigPath: path,
|
|
3754
1452
|
});
|
|
3755
|
-
return tools.some(
|
|
1453
|
+
return tools.some(tool => tool.name === 'gotry_benchmark_environment');
|
|
3756
1454
|
}
|
|
3757
|
-
|
|
3758
|
-
|
|
3759
|
-
//# sourceURL=ts/scripts/benchmark-environment-bridge-tests.ts
|