@danceiny/gotry 0.0.1-rc.21 → 0.0.1-rc.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +102 -194
- package/README.zh-CN.md +115 -210
- package/bin/gotry-backend.js +53 -0
- package/bin/gotry-bootstrap.js +466 -35
- package/bin/gotry-inner.js +255 -36
- package/bin/gotry-process-liveness.d.ts +27 -0
- package/bin/gotry-process-liveness.js +97 -0
- package/bin/gotry-runtime-resolution.d.ts +1 -1
- package/bin/gotry-runtime-resolution.js +2 -2
- package/client/client.js +174 -0
- package/cordis.gotry-patch.yml +70 -93
- package/dist/capabilities/agent-reach-deep.js +45 -122
- package/dist/capabilities/agent-reach.js +63 -148
- package/dist/capabilities/anything.js +89 -75
- package/dist/capabilities/artifacts.js +123 -104
- package/dist/capabilities/channel-health.js +65 -45
- package/dist/capabilities/channel-registry.js +108 -87
- package/dist/capabilities/doctor.js +357 -235
- package/dist/capabilities/effect.js +223 -309
- package/dist/capabilities/fact-log.js +28 -14
- package/dist/capabilities/flyai.js +245 -156
- package/dist/capabilities/ground-transfer.js +509 -0
- package/dist/capabilities/hbcli.js +144 -148
- package/dist/capabilities/hotelbyte-transaction.js +194 -0
- package/dist/capabilities/incident-log.js +131 -61
- package/dist/capabilities/model-override.js +30 -10
- package/dist/capabilities/opensky.js +54 -43
- package/dist/capabilities/resilience.js +50 -32
- package/dist/capabilities/session/action-cache.js +50 -44
- package/dist/capabilities/session/adapters/ctrip-flight.js +119 -62
- package/dist/capabilities/session/adapters/ctrip-hotel.js +125 -110
- package/dist/capabilities/session/adapters/dida-portal.js +206 -0
- package/dist/capabilities/session/adapters/meituan-local.js +35 -54
- package/dist/capabilities/session/adapters/rail-12306.js +169 -112
- package/dist/capabilities/session/benchmark.js +85 -149
- package/dist/capabilities/session/extension-bridge.js +320 -259
- package/dist/capabilities/session/extension-channel.js +57 -76
- package/dist/capabilities/session/extension-distribution.js +115 -119
- package/dist/capabilities/session/extract.js +31 -21
- package/dist/capabilities/session/golden-score.js +55 -33
- package/dist/capabilities/session/health-watch.js +146 -154
- package/dist/capabilities/session/read-guard.js +55 -47
- package/dist/capabilities/session/static-flight-golden.js +49 -36
- package/dist/capabilities/session/transport.js +54 -61
- package/dist/capabilities/session/wizard.js +63 -93
- package/dist/capabilities/session-consent.js +143 -47
- package/dist/capabilities/session-login.js +95 -126
- package/dist/capabilities/session-search.js +391 -253
- package/dist/capabilities/sponsor-plugin.js +341 -0
- package/dist/capabilities/visa-policy.js +87 -73
- package/dist/capabilities/weather.js +163 -235
- package/dist/scripts/action-cache-tests.js +24 -59
- package/dist/scripts/agent-planning-turn-deadline-e2e.js +91 -142
- package/dist/scripts/agent-planning-turn-deadline-tests.js +80 -176
- package/dist/scripts/agent-reach-deep-tests.js +30 -40
- package/dist/scripts/agent-reach-tests.js +26 -18
- package/dist/scripts/agent-reach-wrapper-tests.js +46 -50
- package/dist/scripts/anything-tests.js +58 -62
- package/dist/scripts/artifact-client-contract-tests.js +69 -0
- package/dist/scripts/artifacts-capability-tests.js +235 -0
- package/dist/scripts/async-collect.js +26 -15
- package/dist/scripts/benchmark-environment-bridge-e2e.js +287 -684
- package/dist/scripts/benchmark-environment-bridge-tests.js +818 -3123
- package/dist/scripts/booking-copilot-availability-ledger-binding-tests.js +66 -279
- package/dist/scripts/booking-copilot-availability-policy-tests.js +153 -1257
- package/dist/scripts/booking-copilot-bin-proof-tests.js +22 -45
- package/dist/scripts/booking-copilot-crossrepo-fixture-server.js +64 -76
- package/dist/scripts/booking-copilot-dsh-core-proof-tests.js +69 -199
- package/dist/scripts/booking-copilot-dsh-planner-proof-tests.js +393 -759
- package/dist/scripts/booking-copilot-dsh-plugin-proof-tests.js +33 -75
- package/dist/scripts/booking-copilot-event-sequence-concurrency-proof-tests.js +60 -110
- package/dist/scripts/booking-copilot-gap-code-contract-proof-tests.js +28 -66
- package/dist/scripts/booking-copilot-operation-ledger-concurrency-proof-tests.js +72 -271
- package/dist/scripts/booking-copilot-receipt-ledger-concurrency-proof-tests.js +73 -275
- package/dist/scripts/booking-copilot-runtime-proof-tests.js +784 -3398
- package/dist/scripts/booking-copilot-server-proof-tests.js +63 -168
- package/dist/scripts/booking-copilot-startup-proof-tests.js +38 -124
- package/dist/scripts/booking-executor-tests.js +80 -0
- package/dist/scripts/booking-recovery-chain-tests.js +182 -0
- package/dist/scripts/booking-saga-tests.js +61 -123
- package/dist/scripts/booking-surface-contract-proof-tests.js +152 -1167
- package/dist/scripts/bootstrap-tests.js +1383 -128
- package/dist/scripts/build-changelog.js +89 -130
- package/dist/scripts/build-metrics-report.js +135 -143
- package/dist/scripts/changelog-tests.js +35 -82
- package/dist/scripts/channel-probe-tests.js +44 -116
- package/dist/scripts/channel-probe.js +84 -167
- package/dist/scripts/channel-registry-tests.js +97 -185
- package/dist/scripts/companion-tests.js +25 -70
- package/dist/scripts/dida-cookie-probe.js +87 -0
- package/dist/scripts/dida-sw-probe.js +114 -0
- package/dist/scripts/diff-test.js +17 -12
- package/dist/scripts/doctor-tests.js +428 -120
- package/dist/scripts/dsh-artifact-e2e.js +150 -0
- package/dist/scripts/dsh-artifact-web-e2e.js +443 -0
- package/dist/scripts/dsh-http-proxy-sse-proof.js +90 -0
- package/dist/scripts/dsh-runtime-closure-tests.js +110 -153
- package/dist/scripts/dsh-runtime-closure.js +75 -63
- package/dist/scripts/dsh-session-v3-migration-proof.js +106 -0
- package/dist/scripts/dsh-subprocess-local-proof.js +149 -0
- package/dist/scripts/dsh-target-closure-proof.js +178 -0
- package/dist/scripts/effect-tests.js +165 -552
- package/dist/scripts/engine-run.js +1 -3
- package/dist/scripts/engine-tests.js +16 -10
- package/dist/scripts/evaluation-cadence-tests.js +69 -242
- package/dist/scripts/evaluation-contract-tests.js +193 -499
- package/dist/scripts/extension-distribution-cli.js +19 -15
- package/dist/scripts/extension-distribution-tests.js +98 -211
- package/dist/scripts/extension-tests.js +366 -389
- package/dist/scripts/fact-gate-tests.js +725 -263
- package/dist/scripts/fixtures/extension-bridge-unref-child.mjs +21 -0
- package/dist/scripts/flight-hotel-anchor-363-registered-e2e.js +169 -0
- package/dist/scripts/flight-malformed-tests.js +159 -0
- package/dist/scripts/flyai-tests.js +325 -68
- package/dist/scripts/g5-guard-tests.js +131 -0
- package/dist/scripts/g5-guard.js +189 -0
- package/dist/scripts/gen-lunar-anchors.js +106 -0
- package/dist/scripts/gotry-backend-kernel-tests.js +79 -0
- package/dist/scripts/ground-transfer-tests.js +956 -0
- package/dist/scripts/hbcli-e2e-tests.js +71 -90
- package/dist/scripts/hbcli-live-optin-tests.js +96 -0
- package/dist/scripts/hbcli-release-contract-tests.js +122 -0
- package/dist/scripts/hbcli-tests.js +41 -115
- package/dist/scripts/health-watch-cli.js +42 -28
- package/dist/scripts/hotel-date-gate-tests.js +496 -0
- package/dist/scripts/hotelbyte-reconcile-tests.js +141 -0
- package/dist/scripts/hotelbyte-spawn-e2e-tests.js +373 -0
- package/dist/scripts/i18n-tests.js +16 -9
- package/dist/scripts/incident-tests.js +281 -87
- package/dist/scripts/issue-194-job-id-guard-tests.js +106 -0
- package/dist/scripts/issue-233-cancel-refund-commission-tests.js +392 -0
- package/dist/scripts/issue-271-liveness-tests.js +547 -0
- package/dist/scripts/issue-289-model-retry-real-tests.js +296 -0
- package/dist/scripts/issue-289-model-retry-tests.js +441 -0
- package/dist/scripts/issue-289-web-retry-e2e.js +630 -0
- package/dist/scripts/issue-338-home-city-e2e.js +577 -0
- package/dist/scripts/issue343-real-entry-e2e.js +160 -0
- package/dist/scripts/journey-tests.js +22 -27
- package/dist/scripts/kernel-manifest-evidence.js +109 -0
- package/dist/scripts/kernel-manifest-gate.js +126 -0
- package/dist/scripts/kernel-manifest-tests.js +212 -0
- package/dist/scripts/kernel-manifest-trace-entry.js +7 -0
- package/dist/scripts/kernel-manifest-trace-hook.mjs +28 -0
- package/dist/scripts/kernel-manifest.js +369 -0
- package/dist/scripts/ledger-repair-apply-tests.js +316 -0
- package/dist/scripts/ledger-repair-plan-tests.js +375 -0
- package/dist/scripts/ledger-tests.js +325 -340
- package/dist/scripts/ledger-workflow-crash.js +10 -20
- package/dist/scripts/map-tools-vendor-package-proof.js +206 -134
- package/dist/scripts/memory-capture-tests.js +242 -61
- package/dist/scripts/memory-decay-tests.js +18 -52
- package/dist/scripts/memory-lifecycle-tests.js +657 -0
- package/dist/scripts/memory-lifecycle.js +196 -0
- package/dist/scripts/memory-metrics.js +24 -12
- package/dist/scripts/memory-value-report.js +474 -222
- package/dist/scripts/memory-value-tests.js +328 -0
- package/dist/scripts/metrics-report-tests.js +62 -199
- package/dist/scripts/model-override-e2e.js +66 -105
- package/dist/scripts/nightly-evidence-tests.js +64 -89
- package/dist/scripts/nightly-evidence.js +108 -111
- package/dist/scripts/nudge-digest.js +39 -28
- package/dist/scripts/onboarding-tests.js +63 -88
- package/dist/scripts/opensky-check.js +18 -13
- package/dist/scripts/opensky-tests.js +28 -28
- package/dist/scripts/persona-surface-guard-tests.js +37 -12
- package/dist/scripts/pnpm-dsh-closure-proof.js +2 -5
- package/dist/scripts/policy-anchor-359-registered-e2e.js +214 -0
- package/dist/scripts/price-drift-tests.js +126 -227
- package/dist/scripts/price-drift-watch.js +209 -253
- package/dist/scripts/probe-poi-tests.js +52 -45
- package/dist/scripts/product-metrics.js +187 -273
- package/dist/scripts/publish-preverify.js +47 -41
- package/dist/scripts/realtime-pricing-tests.js +37 -76
- package/dist/scripts/replay-async.js +20 -18
- package/dist/scripts/replay-real.js +16 -18
- package/dist/scripts/replay.js +52 -82
- package/dist/scripts/session-attach-diagnose.js +16 -12
- package/dist/scripts/session-attach-poc.js +34 -37
- package/dist/scripts/session-benchmark.js +66 -147
- package/dist/scripts/session-dida-live-e2e.js +151 -0
- package/dist/scripts/session-extract-tests.js +14 -35
- package/dist/scripts/session-login.js +14 -13
- package/dist/scripts/session-tests.js +545 -583
- package/dist/scripts/sf-live-benchmark.js +92 -110
- package/dist/scripts/sf-live-cli-tests.js +2 -9
- package/dist/scripts/sf-soft-score-tests.js +19 -60
- package/dist/scripts/sf-summary-tests.js +149 -0
- package/dist/scripts/sf-summary.js +466 -84
- package/dist/scripts/skeleton-check.js +16 -14
- package/dist/scripts/skeleton-integration-test.js +11 -10
- package/dist/scripts/skills-contract-tests.js +51 -46
- package/dist/scripts/smoke-session-gate-tests.js +12 -20
- package/dist/scripts/smoke.js +541 -422
- package/dist/scripts/sponsor-reuse-tests.js +262 -0
- package/dist/scripts/state-cli-tests.js +226 -101
- package/dist/scripts/state-cli.js +549 -212
- package/dist/scripts/static-golden-tests.js +46 -180
- package/dist/scripts/time-eval-tests.js +210 -192
- package/dist/scripts/travel-timeline-tests.js +29 -93
- package/dist/scripts/turn-handoff-collect-tests.js +46 -93
- package/dist/scripts/turn-handoff-collect.js +71 -99
- package/dist/scripts/turn-policy-tests.js +28 -29
- package/dist/scripts/typed-contract-canary.js +77 -162
- package/dist/scripts/tz-resolver-sanity.js +50 -0
- package/dist/scripts/tz-resolver-tests.js +584 -0
- package/dist/scripts/unified-tests.js +23 -19
- package/dist/scripts/visa-policy-tests.js +31 -44
- package/dist/scripts/weather-tests.js +121 -543
- package/dist/scripts/wish-channel-gate-tests.js +29 -86
- package/dist/scripts/write-gate-tests.js +522 -0
- package/dist/scripts/z3-lifecycle-fault-tests.js +188 -0
- package/dist/scripts/z3-lifecycle-tests.js +170 -0
- package/dist/scripts/z3-race-repeat-tests.js +39 -0
- package/dist/scripts/z3-race-tests.js +18 -12
- package/dist/src/artifact-gate.js +652 -221
- package/dist/src/backend/kernel.js +91 -0
- package/dist/src/backend/modules/booking-copilot.js +55 -0
- package/dist/src/backend/modules/booking-executor.js +124 -0
- package/dist/src/backend/modules/session-search.js +194 -0
- package/dist/src/benchmark-agent-conformance.js +296 -262
- package/dist/src/benchmark-environment-bridge.js +298 -413
- package/dist/src/benchmark-headless-child-diagnostics.js +93 -100
- package/dist/src/benchmark-tool-isolation.js +94 -67
- package/dist/src/bookable-facts.js +299 -174
- package/dist/src/booking-saga.js +64 -120
- package/dist/src/booking-surface/availability-policy.js +443 -576
- package/dist/src/booking-surface/cancel-refund-commission.js +379 -0
- package/dist/src/booking-surface/contracts.js +24 -80
- package/dist/src/booking-surface/dsh-planner.js +510 -281
- package/dist/src/booking-surface/dsh-plugin.js +17 -1
- package/dist/src/booking-surface/error-codes.js +82 -10
- package/dist/src/booking-surface/index.js +0 -3
- package/dist/src/booking-surface/profile.js +10 -54
- package/dist/src/booking-surface/recovery-chain.js +367 -0
- package/dist/src/booking-surface/runtime.js +765 -891
- package/dist/src/booking-surface/server.js +200 -251
- package/dist/src/booking-surface/startup.js +58 -54
- package/dist/src/booking-surface/validation.js +150 -218
- package/dist/src/bridge.js +20 -13
- package/dist/src/companions.js +48 -74
- package/dist/src/contracts.js +21 -43
- package/dist/src/dsh-llm.js +82 -120
- package/dist/src/engine.js +253 -239
- package/dist/src/evaluation-cadence.js +72 -209
- package/dist/src/evaluation-contracts.js +220 -843
- package/dist/src/flight-pack-adapter.js +54 -0
- package/dist/src/flight-pack-contract.js +2 -0
- package/dist/src/gotry-backend.js +70 -0
- package/dist/src/hotel-date-gate.js +118 -0
- package/dist/src/i18n.js +18 -7
- package/dist/src/index.js +1425 -1629
- package/dist/src/journey.js +132 -81
- package/dist/src/ledger-repair-apply.js +538 -0
- package/dist/src/ledger-repair-plan.js +463 -0
- package/dist/src/loop.js +257 -166
- package/dist/src/memory-capture.js +180 -32
- package/dist/src/memory-decay.js +30 -14
- package/dist/src/memory-lifecycle.js +1310 -0
- package/dist/src/memory-utility.js +27 -40
- package/dist/src/mock-llm.js +36 -61
- package/dist/src/model.js +81 -36
- package/dist/src/realtime-pricing.js +63 -72
- package/dist/src/slot-spec.js +84 -106
- package/dist/src/state-ledger.js +310 -471
- package/dist/src/subagent-job-id-guard.js +58 -0
- package/dist/src/time-anchor.js +231 -33
- package/dist/src/tool-packet.js +20 -8
- package/dist/src/travel-slots.js +58 -35
- package/dist/src/travel-timeline.js +44 -63
- package/dist/src/turn-deadline.js +121 -127
- package/dist/src/turn-policy.js +61 -89
- package/dist/src/tz-resolver.js +143 -0
- package/dist/src/unified.js +504 -381
- package/dist/src/wish-pool.js +39 -14
- package/dist/src/write-gate.js +495 -0
- package/dist/src/z3-shared.js +273 -7
- package/extension/README.md +4 -3
- package/extension/background.js +125 -14
- package/extension/content-bridge.js +23 -1
- package/extension/content-main.js +15 -4
- package/extension/manifest.json +16 -8
- package/package.json +251 -220
- package/ts/capabilities/agent-reach.ts +4 -4
- package/ts/capabilities/anything.ts +1 -1
- package/ts/capabilities/artifacts.ts +46 -9
- package/ts/capabilities/flyai.ts +156 -37
- package/ts/capabilities/hbcli.ts +1 -1
- package/ts/capabilities/incident-log.ts +88 -59
- package/ts/capabilities/session/adapters/ctrip-flight.ts +99 -36
- package/ts/capabilities/session/adapters/dida-portal.ts +250 -0
- package/ts/capabilities/session/adapters/rail-12306.ts +123 -24
- package/ts/capabilities/session/extension-bridge.ts +226 -101
- package/ts/capabilities/session/extension-channel.ts +22 -9
- package/ts/capabilities/session/wizard.ts +2 -2
- package/ts/capabilities/session-consent.ts +98 -14
- package/ts/capabilities/session-login.ts +1 -0
- package/ts/capabilities/session-search.ts +315 -31
- package/ts/dsh-runtime/vendor/README.md +8 -4
- package/ts/dsh-runtime/vendor/dsh-map-tools/README.en.md +7 -1
- package/ts/dsh-runtime/vendor/dsh-map-tools/README.md +5 -3
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/settings-ns.js +1 -1
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/settings-ns.d.ts +1 -1
- package/ts/dsh-runtime/vendor/dsh-map-tools/package.json +6 -6
- package/ts/package.json +236 -16
- package/ts/scripts/map-tools-vendor-package-proof.ts +152 -31
- package/ts/scripts/state-cli.ts +418 -51
- package/ts/src/artifact-gate.ts +561 -15
- package/ts/src/benchmark-environment-bridge.ts +37 -20
- package/ts/src/bookable-facts.ts +183 -7
- package/ts/src/booking-surface/cancel-refund-commission.ts +488 -0
- package/ts/src/booking-surface/dsh-planner.ts +240 -32
- package/ts/src/booking-surface/dsh-plugin.js +17 -1
- package/ts/src/booking-surface/error-codes.ts +75 -0
- package/ts/src/booking-surface/recovery-chain.ts +409 -0
- package/ts/src/booking-surface/server.ts +84 -51
- package/ts/src/booking-surface/validation.ts +2 -0
- package/ts/src/contracts.ts +12 -2
- package/ts/src/dsh-llm.ts +4 -8
- package/ts/src/index.ts +609 -90
- package/ts/src/loop.ts +116 -7
- package/ts/src/memory-capture.ts +227 -3
- package/ts/src/mock-llm.ts +4 -8
- package/ts/src/model.ts +88 -2
- package/ts/src/state-ledger.ts +47 -29
- package/ts/src/subagent-job-id-guard.ts +72 -0
- package/ts/src/time-anchor.ts +217 -13
- package/ts/src/unified.ts +316 -54
|
@@ -1,5 +1,6 @@
|
|
|
1
|
+
/** Provider-neutral execution and terminal contract for benchmark agents. */
|
|
1
2
|
import { createUserMessage } from '@deepseek-ai/dsh-llm';
|
|
2
|
-
import { classifyBenchmarkTurnEnd, createBenchmarkDiagnosticArbiter, emitBenchmarkChildDiagnostic } from './benchmark-headless-child-diagnostics.js';
|
|
3
|
+
import { classifyBenchmarkTurnEnd, createBenchmarkDiagnosticArbiter, emitBenchmarkChildDiagnostic, } from './benchmark-headless-child-diagnostics.js';
|
|
3
4
|
export const MAX_CONFORMANCE_RETRIES = 1;
|
|
4
5
|
export const BENCHMARK_BRIDGE_CALL_REQUIRED = 'BENCHMARK_BRIDGE_CALL_REQUIRED';
|
|
5
6
|
export const BENCHMARK_BRIDGE_CALL_FAILED = 'BENCHMARK_BRIDGE_CALL_FAILED';
|
|
@@ -13,270 +14,306 @@ export const BENCHMARK_CONFORMANCE_STATE_UNAVAILABLE = 'BENCHMARK_CONFORMANCE_ST
|
|
|
13
14
|
const IDENTIFIER = /^[A-Za-z][A-Za-z0-9_.-]*$/;
|
|
14
15
|
const MAX_TERMINAL_BYTES = 1024 * 1024;
|
|
15
16
|
const BENCHMARK_TOOL_RESULT_SCHEMA_VERSION = 'gotry_benchmark_tool_result_v1';
|
|
16
|
-
const BENCHMARK_DOMAIN_RECOVERIES = new Set([
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
]);
|
|
17
|
+
const BENCHMARK_DOMAIN_RECOVERIES = new Set(['none', 'retry_same', 'revise_arguments', 'choose_alternative']);
|
|
18
|
+
// Round 12(issue #215):terminal body schema 是「无数据值的 closed 结构合同」——
|
|
19
|
+
// 只允许结构关键字(type/properties/required/additionalProperties/items),enum/
|
|
20
|
+
// const/example/default 等可夹带数据值的注解面一律禁止,对象必须 additionalProperties:
|
|
21
|
+
// false,大小/深度/节点数有界。校验语义 fail-closed:不补键、不删多余键、不转类型。
|
|
22
22
|
const TERMINAL_BODY_SCHEMA_MAX_BYTES = 16 * 1024;
|
|
23
23
|
const TERMINAL_BODY_SCHEMA_MAX_DEPTH = 8;
|
|
24
24
|
const TERMINAL_BODY_SCHEMA_MAX_NODES = 256;
|
|
25
25
|
const TERMINAL_BODY_SCHEMA_MAX_PROPERTIES = 256;
|
|
26
|
-
const TERMINAL_BODY_SCALAR_TYPES = new Set([
|
|
27
|
-
'string',
|
|
28
|
-
'number',
|
|
29
|
-
'integer',
|
|
30
|
-
'boolean',
|
|
31
|
-
'null'
|
|
32
|
-
]);
|
|
26
|
+
const TERMINAL_BODY_SCALAR_TYPES = new Set(['string', 'number', 'integer', 'boolean', 'null']);
|
|
33
27
|
const TERMINAL_BODY_VALUE_MAX_NODES = 10_000;
|
|
34
28
|
const TERMINAL_BODY_VALUE_MAX_DEPTH = 24;
|
|
35
29
|
function plainObject(value) {
|
|
36
30
|
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
37
31
|
}
|
|
38
32
|
export function validateTerminalOutputConfig(value) {
|
|
39
|
-
if (!plainObject(value))
|
|
33
|
+
if (!plainObject(value))
|
|
34
|
+
return false;
|
|
40
35
|
const keys = Object.keys(value).sort();
|
|
41
|
-
return JSON.stringify(keys) === JSON.stringify([
|
|
42
|
-
'
|
|
43
|
-
|
|
44
|
-
'
|
|
45
|
-
|
|
36
|
+
return JSON.stringify(keys) === JSON.stringify(['body_schema', 'max_bytes', 'tag'])
|
|
37
|
+
&& typeof value.tag === 'string'
|
|
38
|
+
&& IDENTIFIER.test(value.tag)
|
|
39
|
+
&& typeof value.max_bytes === 'number'
|
|
40
|
+
&& Number.isInteger(value.max_bytes)
|
|
41
|
+
&& value.max_bytes >= 1
|
|
42
|
+
&& value.max_bytes <= MAX_TERMINAL_BYTES
|
|
43
|
+
&& validateTerminalBodySchema(value.body_schema);
|
|
46
44
|
}
|
|
45
|
+
/** Accept only the closed structural dialect: exact keyword sets per node, bounded size. */
|
|
47
46
|
export function validateTerminalBodySchema(value) {
|
|
48
47
|
let serialized;
|
|
49
48
|
try {
|
|
50
49
|
serialized = JSON.stringify(value);
|
|
51
|
-
}
|
|
50
|
+
}
|
|
51
|
+
catch {
|
|
52
52
|
return false;
|
|
53
53
|
}
|
|
54
|
-
if (typeof serialized !== 'string' || Buffer.byteLength(serialized, 'utf8') > TERMINAL_BODY_SCHEMA_MAX_BYTES)
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
node: value,
|
|
58
|
-
depth: 0
|
|
59
|
-
}
|
|
60
|
-
];
|
|
54
|
+
if (typeof serialized !== 'string' || Buffer.byteLength(serialized, 'utf8') > TERMINAL_BODY_SCHEMA_MAX_BYTES)
|
|
55
|
+
return false;
|
|
56
|
+
const pending = [{ node: value, depth: 0 }];
|
|
61
57
|
const seen = new Set();
|
|
62
58
|
let nodes = 0;
|
|
63
59
|
let properties = 0;
|
|
64
|
-
while(pending.length > 0){
|
|
60
|
+
while (pending.length > 0) {
|
|
65
61
|
const { node, depth } = pending.pop();
|
|
66
|
-
if (!plainObject(node) || seen.has(node) || ++nodes > TERMINAL_BODY_SCHEMA_MAX_NODES || depth > TERMINAL_BODY_SCHEMA_MAX_DEPTH)
|
|
62
|
+
if (!plainObject(node) || seen.has(node) || ++nodes > TERMINAL_BODY_SCHEMA_MAX_NODES || depth > TERMINAL_BODY_SCHEMA_MAX_DEPTH)
|
|
63
|
+
return false;
|
|
67
64
|
seen.add(node);
|
|
68
65
|
const type = node.type;
|
|
69
66
|
if (type === 'object') {
|
|
70
|
-
if (!exactKeys(node, [
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
]) || !plainObject(node.properties) || !Array.isArray(node.required) || node.additionalProperties !== false) return false;
|
|
67
|
+
if (!exactKeys(node, ['type', 'properties', 'required', 'additionalProperties'])
|
|
68
|
+
|| !plainObject(node.properties)
|
|
69
|
+
|| !Array.isArray(node.required)
|
|
70
|
+
|| node.additionalProperties !== false)
|
|
71
|
+
return false;
|
|
76
72
|
const entries = Object.entries(node.properties);
|
|
77
73
|
properties += entries.length;
|
|
78
|
-
if (properties > TERMINAL_BODY_SCHEMA_MAX_PROPERTIES
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
74
|
+
if (properties > TERMINAL_BODY_SCHEMA_MAX_PROPERTIES
|
|
75
|
+
|| entries.some(([key, child]) => key.length === 0 || key.length > 64 || !plainObject(child)))
|
|
76
|
+
return false;
|
|
77
|
+
if (node.required.length > entries.length
|
|
78
|
+
|| new Set(node.required).size !== node.required.length
|
|
79
|
+
|| node.required.some(key => typeof key !== 'string' || !Object.hasOwn(node.properties, key)))
|
|
80
|
+
return false;
|
|
81
|
+
for (const [, child] of entries)
|
|
82
|
+
pending.push({ node: child, depth: depth + 1 });
|
|
84
83
|
continue;
|
|
85
84
|
}
|
|
86
85
|
if (type === 'array') {
|
|
87
|
-
if (!exactKeys(node, [
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
]) || !plainObject(node.items)) return false;
|
|
91
|
-
pending.push({
|
|
92
|
-
node: node.items,
|
|
93
|
-
depth: depth + 1
|
|
94
|
-
});
|
|
86
|
+
if (!exactKeys(node, ['type', 'items']) || !plainObject(node.items))
|
|
87
|
+
return false;
|
|
88
|
+
pending.push({ node: node.items, depth: depth + 1 });
|
|
95
89
|
continue;
|
|
96
90
|
}
|
|
97
|
-
if (typeof type !== 'string' || !TERMINAL_BODY_SCALAR_TYPES.has(type) || !exactKeys(node, [
|
|
98
|
-
|
|
99
|
-
])) return false;
|
|
91
|
+
if (typeof type !== 'string' || !TERMINAL_BODY_SCALAR_TYPES.has(type) || !exactKeys(node, ['type']))
|
|
92
|
+
return false;
|
|
100
93
|
}
|
|
101
94
|
return plainObject(value) && value.type === 'object';
|
|
102
95
|
}
|
|
103
96
|
function exactKeys(value, allowed) {
|
|
104
|
-
return Object.keys(value).every(
|
|
97
|
+
return Object.keys(value).every(key => allowed.includes(key));
|
|
105
98
|
}
|
|
99
|
+
/** Strict structural validation: exact keys, exact types, no coercion, no autofix. */
|
|
106
100
|
export function validateTerminalBodyValue(value, schema) {
|
|
107
|
-
const pending = [
|
|
108
|
-
{
|
|
109
|
-
value,
|
|
110
|
-
schema,
|
|
111
|
-
depth: 0
|
|
112
|
-
}
|
|
113
|
-
];
|
|
101
|
+
const pending = [{ value, schema, depth: 0 }];
|
|
114
102
|
let nodes = 0;
|
|
115
|
-
while(pending.length > 0){
|
|
103
|
+
while (pending.length > 0) {
|
|
116
104
|
const current = pending.pop();
|
|
117
|
-
if (++nodes > TERMINAL_BODY_VALUE_MAX_NODES || current.depth > TERMINAL_BODY_VALUE_MAX_DEPTH)
|
|
105
|
+
if (++nodes > TERMINAL_BODY_VALUE_MAX_NODES || current.depth > TERMINAL_BODY_VALUE_MAX_DEPTH)
|
|
106
|
+
return false;
|
|
118
107
|
const { schema: node } = current;
|
|
119
108
|
if (node.type === 'object') {
|
|
120
109
|
const record = current.value;
|
|
121
|
-
if (!plainObject(record))
|
|
110
|
+
if (!plainObject(record))
|
|
111
|
+
return false;
|
|
122
112
|
const properties = node.properties;
|
|
123
|
-
for (const key of node.required){
|
|
124
|
-
if (!Object.hasOwn(record, key))
|
|
113
|
+
for (const key of node.required) {
|
|
114
|
+
if (!Object.hasOwn(record, key))
|
|
115
|
+
return false;
|
|
125
116
|
}
|
|
126
|
-
for (const [key, child] of Object.entries(record)){
|
|
117
|
+
for (const [key, child] of Object.entries(record)) {
|
|
127
118
|
const childSchema = properties[key];
|
|
128
|
-
if (!childSchema)
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
schema: childSchema,
|
|
132
|
-
depth: current.depth + 1
|
|
133
|
-
});
|
|
119
|
+
if (!childSchema)
|
|
120
|
+
return false;
|
|
121
|
+
pending.push({ value: child, schema: childSchema, depth: current.depth + 1 });
|
|
134
122
|
}
|
|
135
123
|
continue;
|
|
136
124
|
}
|
|
137
125
|
if (node.type === 'array') {
|
|
138
|
-
if (!Array.isArray(current.value))
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
schema: node.items,
|
|
142
|
-
depth: current.depth + 1
|
|
143
|
-
});
|
|
126
|
+
if (!Array.isArray(current.value))
|
|
127
|
+
return false;
|
|
128
|
+
for (const item of current.value)
|
|
129
|
+
pending.push({ value: item, schema: node.items, depth: current.depth + 1 });
|
|
144
130
|
continue;
|
|
145
131
|
}
|
|
146
132
|
if (node.type === 'string') {
|
|
147
|
-
if (typeof current.value !== 'string')
|
|
133
|
+
if (typeof current.value !== 'string')
|
|
134
|
+
return false;
|
|
148
135
|
continue;
|
|
149
136
|
}
|
|
150
137
|
if (node.type === 'boolean') {
|
|
151
|
-
if (typeof current.value !== 'boolean')
|
|
138
|
+
if (typeof current.value !== 'boolean')
|
|
139
|
+
return false;
|
|
152
140
|
continue;
|
|
153
141
|
}
|
|
154
142
|
if (node.type === 'null') {
|
|
155
|
-
if (current.value !== null)
|
|
143
|
+
if (current.value !== null)
|
|
144
|
+
return false;
|
|
156
145
|
continue;
|
|
157
146
|
}
|
|
158
147
|
if (node.type === 'integer') {
|
|
159
|
-
if (typeof current.value !== 'number' || !Number.isSafeInteger(current.value))
|
|
148
|
+
if (typeof current.value !== 'number' || !Number.isSafeInteger(current.value))
|
|
149
|
+
return false;
|
|
160
150
|
continue;
|
|
161
151
|
}
|
|
162
|
-
if (typeof current.value !== 'number' || !Number.isFinite(current.value))
|
|
152
|
+
if (typeof current.value !== 'number' || !Number.isFinite(current.value))
|
|
153
|
+
return false;
|
|
163
154
|
}
|
|
164
155
|
return true;
|
|
165
156
|
}
|
|
157
|
+
/** Deterministic single-source structural outline projected into system prompt and correction alike. */
|
|
166
158
|
export function terminalSchemaOutline(schema) {
|
|
167
|
-
const render = (node, depth)=>{
|
|
168
|
-
if (depth > TERMINAL_BODY_SCHEMA_MAX_DEPTH)
|
|
159
|
+
const render = (node, depth) => {
|
|
160
|
+
if (depth > TERMINAL_BODY_SCHEMA_MAX_DEPTH)
|
|
161
|
+
return '…';
|
|
169
162
|
if (node.type === 'object') {
|
|
170
163
|
const properties = node.properties;
|
|
171
164
|
const required = new Set(node.required);
|
|
172
|
-
const parts = Object.entries(properties).map(([key, child])
|
|
165
|
+
const parts = Object.entries(properties).map(([key, child]) => `${required.has(key) ? '' : '?'}${key}:${render(child, depth + 1)}`);
|
|
173
166
|
return `object{${parts.join(',')}}`;
|
|
174
167
|
}
|
|
175
|
-
if (node.type === 'array')
|
|
168
|
+
if (node.type === 'array')
|
|
169
|
+
return `array<${render(node.items, depth + 1)}>`;
|
|
176
170
|
return node.type;
|
|
177
171
|
};
|
|
178
172
|
return render(schema, 0);
|
|
179
173
|
}
|
|
180
174
|
function invalidTerminal() {
|
|
181
|
-
return {
|
|
182
|
-
ok: false,
|
|
183
|
-
error: 'invalid_terminal_output'
|
|
184
|
-
};
|
|
175
|
+
return { ok: false, error: 'invalid_terminal_output' };
|
|
185
176
|
}
|
|
177
|
+
/** 剥离推理模型的思考标签对(如 MiniMax/DeepSeek 的 <think>…</think>):当代模型
|
|
178
|
+
* 即便被明令「只回终态包」也会以思考块前缀输出——不剥则严格终态门对推理模型
|
|
179
|
+
* 结构性失效(Round 9 治理面实测)。只剥配对的推理标签本体,其余前后缀仍 fail-closed。 */
|
|
186
180
|
function stripReasoningBlocks(raw) {
|
|
187
181
|
return raw.replace(/<think>[\s\S]*?<\/think>\s*/gi, '');
|
|
188
182
|
}
|
|
183
|
+
/** Parse exactly one configured tag pair containing one JSON object. */
|
|
189
184
|
export function parseBenchmarkTerminal(raw, config) {
|
|
190
|
-
if (!validateTerminalOutputConfig(config))
|
|
191
|
-
|
|
185
|
+
if (!validateTerminalOutputConfig(config))
|
|
186
|
+
return invalidTerminal();
|
|
187
|
+
if (Buffer.byteLength(raw, 'utf8') > config.max_bytes)
|
|
188
|
+
return invalidTerminal();
|
|
192
189
|
const trimmed = stripReasoningBlocks(raw).trim();
|
|
193
190
|
const opening = `<${config.tag}>`;
|
|
194
191
|
const closing = `</${config.tag}>`;
|
|
195
|
-
if (!trimmed.startsWith(opening) || !trimmed.endsWith(closing))
|
|
196
|
-
|
|
192
|
+
if (!trimmed.startsWith(opening) || !trimmed.endsWith(closing))
|
|
193
|
+
return invalidTerminal();
|
|
194
|
+
if (trimmed.indexOf(opening) !== trimmed.lastIndexOf(opening)
|
|
195
|
+
|| trimmed.indexOf(closing) !== trimmed.lastIndexOf(closing))
|
|
196
|
+
return invalidTerminal();
|
|
197
197
|
const body = trimmed.slice(opening.length, -closing.length);
|
|
198
|
-
if (body.length === 0 || body.includes('```'))
|
|
198
|
+
if (body.length === 0 || body.includes('```'))
|
|
199
|
+
return invalidTerminal();
|
|
199
200
|
try {
|
|
200
201
|
const value = JSON.parse(body);
|
|
201
|
-
if (!plainObject(value))
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
202
|
+
if (!plainObject(value))
|
|
203
|
+
return invalidTerminal();
|
|
204
|
+
// Round 12(#215):同一份 closed body schema 在接受终态前 fail-closed 校验——
|
|
205
|
+
// 结构不合法的终态永不进入正式计分链,不做任何 autofix。
|
|
206
|
+
if (!validateTerminalBodyValue(value, config.body_schema))
|
|
207
|
+
return invalidTerminal();
|
|
208
|
+
return { ok: true, value };
|
|
209
|
+
}
|
|
210
|
+
catch {
|
|
208
211
|
return invalidTerminal();
|
|
209
212
|
}
|
|
210
213
|
}
|
|
214
|
+
/** Backward-compatible name used by the bridge contract tests. */
|
|
211
215
|
export const validateTerminalOutput = parseBenchmarkTerminal;
|
|
212
216
|
function eventData(event) {
|
|
213
|
-
if (!plainObject(event) || typeof event.type !== 'string' || !plainObject(event.data))
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
data: event.data
|
|
217
|
-
};
|
|
217
|
+
if (!plainObject(event) || typeof event.type !== 'string' || !plainObject(event.data))
|
|
218
|
+
return undefined;
|
|
219
|
+
return { type: event.type, data: event.data };
|
|
218
220
|
}
|
|
219
221
|
function eventTurn(data) {
|
|
220
222
|
return typeof data.turn === 'number' && Number.isInteger(data.turn) ? data.turn : undefined;
|
|
221
223
|
}
|
|
222
224
|
function parseToolArguments(value) {
|
|
223
|
-
if (plainObject(value))
|
|
224
|
-
|
|
225
|
+
if (plainObject(value))
|
|
226
|
+
return value;
|
|
227
|
+
if (typeof value !== 'string')
|
|
228
|
+
return undefined;
|
|
225
229
|
try {
|
|
226
230
|
const parsed = JSON.parse(value);
|
|
227
231
|
return plainObject(parsed) ? parsed : undefined;
|
|
228
|
-
}
|
|
232
|
+
}
|
|
233
|
+
catch {
|
|
229
234
|
return undefined;
|
|
230
235
|
}
|
|
231
236
|
}
|
|
232
237
|
function assistantText(data) {
|
|
233
|
-
if (!plainObject(data.message) || !Array.isArray(data.message.content))
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
text
|
|
237
|
-
|
|
238
|
-
|
|
238
|
+
if (!plainObject(data.message) || !Array.isArray(data.message.content))
|
|
239
|
+
return undefined;
|
|
240
|
+
const text = data.message.content
|
|
241
|
+
.filter(block => plainObject(block) && block.type === 'text' && typeof block.text === 'string')
|
|
242
|
+
.map(block => String(block.text))
|
|
243
|
+
.join('');
|
|
244
|
+
return { text, interrupted: data.interrupted === true };
|
|
239
245
|
}
|
|
240
246
|
function bridgeResultStatus(data, callId) {
|
|
241
|
-
if (!plainObject(data.message) || !plainObject(data.message.source))
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
const
|
|
247
|
+
if (!plainObject(data.message) || !plainObject(data.message.source))
|
|
248
|
+
return undefined;
|
|
249
|
+
if (data.message.source.callId !== callId || !Array.isArray(data.message.content))
|
|
250
|
+
return undefined;
|
|
251
|
+
const block = data.message.content.find(candidate => plainObject(candidate)
|
|
252
|
+
&& candidate.type === 'tool-result'
|
|
253
|
+
&& candidate.toolCallId === callId);
|
|
254
|
+
if (!plainObject(block) || block.isError === true || !Array.isArray(block.content))
|
|
255
|
+
return undefined;
|
|
256
|
+
const text = block.content
|
|
257
|
+
.filter(candidate => plainObject(candidate) && candidate.type === 'text' && typeof candidate.text === 'string')
|
|
258
|
+
.map(candidate => String(candidate.text))
|
|
259
|
+
.join('');
|
|
246
260
|
try {
|
|
247
261
|
const parsed = JSON.parse(text);
|
|
248
|
-
if (!plainObject(parsed))
|
|
262
|
+
if (!plainObject(parsed))
|
|
263
|
+
return undefined;
|
|
249
264
|
const keys = Object.keys(parsed).sort().join(',');
|
|
250
|
-
if (parsed.ok === true && keys === 'ok,result' && (plainObject(parsed.result) || Array.isArray(parsed.result)))
|
|
251
|
-
kind: 'result'
|
|
252
|
-
};
|
|
265
|
+
if (parsed.ok === true && keys === 'ok,result' && (plainObject(parsed.result) || Array.isArray(parsed.result)))
|
|
266
|
+
return { kind: 'result' };
|
|
253
267
|
if (parsed.ok === true && keys === 'ok,outcome' && plainObject(parsed.outcome)) {
|
|
254
268
|
const outcomeKeys = Object.keys(parsed.outcome).sort().join(',');
|
|
255
|
-
if (outcomeKeys === 'code,recovery,schema_version,status'
|
|
256
|
-
|
|
257
|
-
|
|
269
|
+
if (outcomeKeys === 'code,recovery,schema_version,status'
|
|
270
|
+
&& parsed.outcome.schema_version === BENCHMARK_TOOL_RESULT_SCHEMA_VERSION
|
|
271
|
+
&& (parsed.outcome.status === 'miss' || parsed.outcome.status === 'error')
|
|
272
|
+
&& typeof parsed.outcome.code === 'string'
|
|
273
|
+
&& IDENTIFIER.test(parsed.outcome.code)
|
|
274
|
+
&& typeof parsed.outcome.recovery === 'string'
|
|
275
|
+
&& BENCHMARK_DOMAIN_RECOVERIES.has(parsed.outcome.recovery))
|
|
276
|
+
return { kind: 'domain' };
|
|
258
277
|
return undefined;
|
|
259
278
|
}
|
|
260
279
|
if (parsed.ok === false && typeof parsed.error === 'string') {
|
|
261
|
-
const exactFailure = parsed.error === 'runner_failed'
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
280
|
+
const exactFailure = parsed.error === 'runner_failed'
|
|
281
|
+
? (keys === 'error,ok' || (keys === 'error,exit_code,ok,signal'
|
|
282
|
+
&& (parsed.exit_code === null || (typeof parsed.exit_code === 'number' && Number.isInteger(parsed.exit_code)))
|
|
283
|
+
&& (parsed.signal === null || typeof parsed.signal === 'string')))
|
|
284
|
+
: parsed.error === 'invalid_arguments'
|
|
285
|
+
? keys === 'error,ok,reason' && typeof parsed.reason === 'string'
|
|
286
|
+
: keys === 'error,ok';
|
|
287
|
+
if (!exactFailure)
|
|
288
|
+
return undefined;
|
|
289
|
+
const code = parsed.error === 'timed_out'
|
|
290
|
+
? BENCHMARK_BRIDGE_TIMED_OUT
|
|
291
|
+
: parsed.error === 'runner_failed'
|
|
292
|
+
? BENCHMARK_BRIDGE_RUNNER_FAILED
|
|
293
|
+
: parsed.error === 'spawn_failed'
|
|
294
|
+
? BENCHMARK_BRIDGE_SPAWN_FAILED
|
|
295
|
+
: parsed.error === 'output_truncated'
|
|
296
|
+
? BENCHMARK_BRIDGE_OUTPUT_TRUNCATED
|
|
297
|
+
: BENCHMARK_BRIDGE_CALL_FAILED;
|
|
298
|
+
return { kind: 'failure', code };
|
|
268
299
|
}
|
|
269
300
|
return undefined;
|
|
270
|
-
}
|
|
301
|
+
}
|
|
302
|
+
catch {
|
|
271
303
|
return undefined;
|
|
272
304
|
}
|
|
273
305
|
}
|
|
274
306
|
export function benchmarkChildFailureForConformanceCode(code) {
|
|
275
|
-
if (code === BENCHMARK_BRIDGE_TIMED_OUT)
|
|
276
|
-
|
|
277
|
-
if (code ===
|
|
278
|
-
|
|
279
|
-
if (code ===
|
|
307
|
+
if (code === BENCHMARK_BRIDGE_TIMED_OUT)
|
|
308
|
+
return 'child_bridge_timed_out';
|
|
309
|
+
if (code === BENCHMARK_BRIDGE_RUNNER_FAILED)
|
|
310
|
+
return 'child_bridge_runner_failed';
|
|
311
|
+
if (code === BENCHMARK_BRIDGE_SPAWN_FAILED)
|
|
312
|
+
return 'child_bridge_spawn_failed';
|
|
313
|
+
if (code === BENCHMARK_BRIDGE_OUTPUT_TRUNCATED)
|
|
314
|
+
return 'child_bridge_output_truncated';
|
|
315
|
+
if (code === BENCHMARK_BRIDGE_CALL_FAILED)
|
|
316
|
+
return 'child_bridge_failure';
|
|
280
317
|
return 'child_conformance_failure';
|
|
281
318
|
}
|
|
282
319
|
function newTurn(turn) {
|
|
@@ -286,56 +323,70 @@ function newTurn(turn) {
|
|
|
286
323
|
retryCount: 0,
|
|
287
324
|
retryMode: 'none',
|
|
288
325
|
bridgeCallFailed: false,
|
|
289
|
-
retryRedispatchAttempted: false
|
|
326
|
+
retryRedispatchAttempted: false,
|
|
290
327
|
};
|
|
291
328
|
}
|
|
329
|
+
/** Deterministic controller used by both runtime wiring and offline contracts. */
|
|
292
330
|
export function createBenchmarkAgentConformance(projection) {
|
|
293
331
|
let state;
|
|
294
332
|
const controller = {
|
|
295
|
-
get retryMode
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
get retryCount () {
|
|
299
|
-
return state?.retryCount ?? 0;
|
|
300
|
-
},
|
|
301
|
-
observe (rawEvent) {
|
|
333
|
+
get retryMode() { return state?.retryMode ?? 'none'; },
|
|
334
|
+
get retryCount() { return state?.retryCount ?? 0; },
|
|
335
|
+
observe(rawEvent) {
|
|
302
336
|
const event = eventData(rawEvent);
|
|
303
|
-
if (!event)
|
|
337
|
+
if (!event)
|
|
338
|
+
return;
|
|
304
339
|
const turn = eventTurn(event.data);
|
|
305
340
|
if (event.type === 'turn/start' && turn !== undefined) {
|
|
306
341
|
state = newTurn(turn);
|
|
307
342
|
return;
|
|
308
343
|
}
|
|
309
344
|
if (event.type === 'turn/end') {
|
|
310
|
-
if (state && turn === state.turn)
|
|
345
|
+
if (state && turn === state.turn)
|
|
346
|
+
state = undefined;
|
|
311
347
|
return;
|
|
312
348
|
}
|
|
313
|
-
if (!state || turn !== state.turn)
|
|
349
|
+
if (!state || turn !== state.turn)
|
|
350
|
+
return;
|
|
314
351
|
if (event.type === 'tool/call') {
|
|
315
|
-
if (event.data.name !== projection.toolName)
|
|
352
|
+
if (event.data.name !== projection.toolName)
|
|
353
|
+
return;
|
|
316
354
|
if (state.retryMode === 'terminal') {
|
|
317
355
|
state.retryRedispatchAttempted = true;
|
|
318
356
|
return;
|
|
319
357
|
}
|
|
320
|
-
if (typeof event.data.callId !== 'string')
|
|
358
|
+
if (typeof event.data.callId !== 'string')
|
|
359
|
+
return;
|
|
321
360
|
const args = parseToolArguments(event.data.arguments);
|
|
361
|
+
// Round 8(issue #100/#102):bridge 工具 typed 泛化后,模型出参为平铺形态
|
|
322
362
|
const flat = args && plainObject(args) ? args : undefined;
|
|
323
|
-
if (flat?.action === 'call'
|
|
363
|
+
if (flat?.action === 'call'
|
|
364
|
+
&& typeof flat.tool === 'string'
|
|
365
|
+
&& projection.allowedTools.includes(flat.tool)) {
|
|
324
366
|
state.validCallIds.add(event.data.callId);
|
|
325
367
|
}
|
|
326
368
|
return;
|
|
327
369
|
}
|
|
328
370
|
if (event.type === 'tool/result') {
|
|
329
|
-
if (!plainObject(event.data.message) || !plainObject(event.data.message.source))
|
|
371
|
+
if (!plainObject(event.data.message) || !plainObject(event.data.message.source))
|
|
372
|
+
return;
|
|
330
373
|
const callId = event.data.message.source.callId;
|
|
331
|
-
if (typeof callId !== 'string' || !state.validCallIds.has(callId))
|
|
374
|
+
if (typeof callId !== 'string' || !state.validCallIds.has(callId))
|
|
375
|
+
return;
|
|
332
376
|
const resultStatus = bridgeResultStatus(event.data, callId);
|
|
333
|
-
if (typeof event.data.step === 'number')
|
|
377
|
+
if (typeof event.data.step === 'number')
|
|
378
|
+
state.lastBridgeResponseStep = event.data.step;
|
|
334
379
|
if (resultStatus?.kind === 'result') {
|
|
335
|
-
state.successfulResultStep = typeof event.data.step === 'number'
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
}
|
|
380
|
+
state.successfulResultStep = typeof event.data.step === 'number'
|
|
381
|
+
? event.data.step
|
|
382
|
+
: state.successfulResultStep;
|
|
383
|
+
}
|
|
384
|
+
else if (resultStatus?.kind === 'domain') {
|
|
385
|
+
state.domainOutcomeStep = typeof event.data.step === 'number'
|
|
386
|
+
? event.data.step
|
|
387
|
+
: state.domainOutcomeStep;
|
|
388
|
+
}
|
|
389
|
+
else {
|
|
339
390
|
state.bridgeCallFailed = true;
|
|
340
391
|
state.bridgeFailureCode = resultStatus?.code ?? BENCHMARK_BRIDGE_CALL_FAILED;
|
|
341
392
|
}
|
|
@@ -343,87 +394,63 @@ export function createBenchmarkAgentConformance(projection) {
|
|
|
343
394
|
}
|
|
344
395
|
if (event.type === 'assistant/message') {
|
|
345
396
|
const assistant = assistantText(event.data);
|
|
346
|
-
if (!assistant || typeof event.data.step !== 'number')
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
...assistant
|
|
350
|
-
};
|
|
397
|
+
if (!assistant || typeof event.data.step !== 'number')
|
|
398
|
+
return;
|
|
399
|
+
state.lastAssistant = { step: event.data.step, ...assistant };
|
|
351
400
|
}
|
|
352
401
|
},
|
|
353
|
-
stopping
|
|
402
|
+
stopping(turn) {
|
|
354
403
|
if (!state || state.turn !== turn) {
|
|
355
|
-
return {
|
|
356
|
-
kind: 'reject',
|
|
357
|
-
code: BENCHMARK_CONFORMANCE_STATE_UNAVAILABLE
|
|
358
|
-
};
|
|
404
|
+
return { kind: 'reject', code: BENCHMARK_CONFORMANCE_STATE_UNAVAILABLE };
|
|
359
405
|
}
|
|
360
406
|
if (state.retryRedispatchAttempted) {
|
|
361
|
-
return {
|
|
362
|
-
kind: 'reject',
|
|
363
|
-
code: BENCHMARK_BRIDGE_RETRY_CALL_NOT_ALLOWED
|
|
364
|
-
};
|
|
407
|
+
return { kind: 'reject', code: BENCHMARK_BRIDGE_RETRY_CALL_NOT_ALLOWED };
|
|
365
408
|
}
|
|
366
409
|
if (state.bridgeCallFailed && state.successfulResultStep === undefined) {
|
|
367
|
-
return {
|
|
368
|
-
kind: 'reject',
|
|
369
|
-
code: state.bridgeFailureCode ?? BENCHMARK_BRIDGE_CALL_FAILED
|
|
370
|
-
};
|
|
410
|
+
return { kind: 'reject', code: state.bridgeFailureCode ?? BENCHMARK_BRIDGE_CALL_FAILED };
|
|
371
411
|
}
|
|
372
412
|
if (state.successfulResultStep === undefined && state.domainOutcomeStep === undefined) {
|
|
373
413
|
if (state.retryCount >= MAX_CONFORMANCE_RETRIES) {
|
|
374
|
-
return {
|
|
375
|
-
kind: 'reject',
|
|
376
|
-
code: BENCHMARK_BRIDGE_CALL_REQUIRED
|
|
377
|
-
};
|
|
414
|
+
return { kind: 'reject', code: BENCHMARK_BRIDGE_CALL_REQUIRED };
|
|
378
415
|
}
|
|
379
416
|
state.retryCount += 1;
|
|
380
417
|
state.retryMode = 'call';
|
|
381
|
-
return {
|
|
382
|
-
kind: 'steer',
|
|
383
|
-
mode: 'call'
|
|
384
|
-
};
|
|
418
|
+
return { kind: 'steer', mode: 'call' };
|
|
385
419
|
}
|
|
386
420
|
const terminal = state.lastAssistant;
|
|
387
|
-
const terminalIsValid = terminal !== undefined
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
421
|
+
const terminalIsValid = terminal !== undefined
|
|
422
|
+
&& !terminal.interrupted
|
|
423
|
+
&& terminal.step > state.lastBridgeResponseStep
|
|
424
|
+
&& parseBenchmarkTerminal(terminal.text, projection.terminal).ok;
|
|
425
|
+
if (terminalIsValid)
|
|
426
|
+
return { kind: 'accept' };
|
|
391
427
|
if (state.retryCount >= MAX_CONFORMANCE_RETRIES) {
|
|
392
|
-
return {
|
|
393
|
-
kind: 'reject',
|
|
394
|
-
code: BENCHMARK_TERMINAL_INVALID
|
|
395
|
-
};
|
|
428
|
+
return { kind: 'reject', code: BENCHMARK_TERMINAL_INVALID };
|
|
396
429
|
}
|
|
397
430
|
state.retryCount += 1;
|
|
398
431
|
state.retryMode = 'terminal';
|
|
399
|
-
return {
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
432
|
+
return { kind: 'steer', mode: 'terminal' };
|
|
433
|
+
},
|
|
434
|
+
guardBridgeExecution() {
|
|
435
|
+
return state?.retryMode === 'terminal'
|
|
436
|
+
? BENCHMARK_BRIDGE_RETRY_CALL_NOT_ALLOWED
|
|
437
|
+
: undefined;
|
|
403
438
|
},
|
|
404
|
-
guardBridgeExecution () {
|
|
405
|
-
return state?.retryMode === 'terminal' ? BENCHMARK_BRIDGE_RETRY_CALL_NOT_ALLOWED : undefined;
|
|
406
|
-
}
|
|
407
439
|
};
|
|
408
440
|
return controller;
|
|
409
441
|
}
|
|
410
442
|
function requireDisposer(value, capability) {
|
|
411
|
-
if (typeof value !== 'function')
|
|
443
|
+
if (typeof value !== 'function')
|
|
444
|
+
throw new Error(`benchmark conformance ${capability} disposer unavailable`);
|
|
412
445
|
return value;
|
|
413
446
|
}
|
|
414
447
|
function correctionMessage(mode, projection) {
|
|
415
|
-
const text = mode === 'call'
|
|
448
|
+
const text = mode === 'call'
|
|
449
|
+
? `BENCHMARK_CONFORMANCE_CALL: Execute exactly one native ${projection.toolName} action:"call" now. Describing an intended CLI, shell, or Python command does not execute it.`
|
|
450
|
+
: `BENCHMARK_CONFORMANCE_TERMINAL: Reuse the existing successful tool result. Do not call any tool. Reply only <${projection.terminal.tag}> with one JSON object matching exactly ${terminalSchemaOutline(projection.terminal.body_schema)} — no extra keys, no missing keys, exact types.`;
|
|
416
451
|
return createUserMessage({
|
|
417
|
-
content: [
|
|
418
|
-
|
|
419
|
-
type: 'text',
|
|
420
|
-
text
|
|
421
|
-
}
|
|
422
|
-
],
|
|
423
|
-
source: {
|
|
424
|
-
kind: 'plugin',
|
|
425
|
-
plugin: 'gotry-benchmark-agent-conformance'
|
|
426
|
-
}
|
|
452
|
+
content: [{ type: 'text', text }],
|
|
453
|
+
source: { kind: 'plugin', plugin: 'gotry-benchmark-agent-conformance' },
|
|
427
454
|
});
|
|
428
455
|
}
|
|
429
456
|
function systemSection(projection) {
|
|
@@ -437,60 +464,62 @@ function systemSection(projection) {
|
|
|
437
464
|
`- Call it with exactly {"action":"call","tool":"<one of: ${allowed}>","arguments":{...}}.`,
|
|
438
465
|
'- action:"tools" is discovery only and does not satisfy the required environment call.',
|
|
439
466
|
`- After a successful tool result, reply only <${projection.terminal.tag}> with one JSON object matching exactly ${outline} — no extra keys, no missing keys, exact types; no prose or code fence.`,
|
|
440
|
-
'- If a terminal-format correction arrives, reuse the existing result and do not call the tool again.'
|
|
441
|
-
].join('\n')
|
|
467
|
+
'- If a terminal-format correction arrives, reuse the existing result and do not call the tool again.',
|
|
468
|
+
].join('\n'),
|
|
442
469
|
};
|
|
443
470
|
}
|
|
471
|
+
/** Install conformance only after the exact benchmark bridge isolation layer. */
|
|
444
472
|
export function installBenchmarkAgentConformance(ctx, projection, diagnosticWriter = emitBenchmarkChildDiagnostic) {
|
|
445
473
|
const bus = ctx;
|
|
446
|
-
if (typeof bus.on !== 'function')
|
|
474
|
+
if (typeof bus.on !== 'function')
|
|
475
|
+
throw new Error('benchmark conformance event bus unavailable');
|
|
447
476
|
const byAgent = new WeakMap();
|
|
448
477
|
const bySession = new WeakMap();
|
|
449
478
|
const sessionKeys = new WeakMap();
|
|
450
479
|
let sessionOrdinal = 0;
|
|
451
480
|
const diagnostics = createBenchmarkDiagnosticArbiter(diagnosticWriter);
|
|
452
|
-
bus.on('agent/created', (payload)=>{
|
|
481
|
+
bus.on('agent/created', (payload) => {
|
|
453
482
|
const agent = payload.agent;
|
|
454
483
|
if (!agent || typeof agent !== 'object' || !agent.session) {
|
|
455
484
|
throw new Error('benchmark conformance created agent unavailable');
|
|
456
485
|
}
|
|
457
|
-
if (typeof agent.steer !== 'function')
|
|
458
|
-
|
|
459
|
-
if (typeof agent.ctx
|
|
460
|
-
|
|
486
|
+
if (typeof agent.steer !== 'function')
|
|
487
|
+
throw new Error('benchmark conformance steering unavailable');
|
|
488
|
+
if (typeof agent.ctx?.tools?.guard !== 'function')
|
|
489
|
+
throw new Error('benchmark conformance scoped guard unavailable');
|
|
490
|
+
if (typeof agent.ctx.effect !== 'function')
|
|
491
|
+
throw new Error('benchmark conformance scoped effect unavailable');
|
|
492
|
+
if (typeof agent.ctx.on !== 'function')
|
|
493
|
+
throw new Error('benchmark conformance scoped event bus unavailable');
|
|
461
494
|
const controller = createBenchmarkAgentConformance(projection);
|
|
462
495
|
byAgent.set(agent, controller);
|
|
463
496
|
bySession.set(agent.session, controller);
|
|
464
497
|
sessionKeys.set(agent.session, `session-${++sessionOrdinal}`);
|
|
465
498
|
try {
|
|
466
|
-
requireDisposer(agent.ctx.effect.call(agent.ctx, function*() {
|
|
467
|
-
yield requireDisposer(agent.ctx.tools.guard(
|
|
468
|
-
if (execution.name !== projection.toolName)
|
|
499
|
+
requireDisposer(agent.ctx.effect.call(agent.ctx, function* () {
|
|
500
|
+
yield requireDisposer(agent.ctx.tools.guard(execution => {
|
|
501
|
+
if (execution.name !== projection.toolName)
|
|
502
|
+
return undefined;
|
|
469
503
|
return controller.guardBridgeExecution();
|
|
470
504
|
}), 'scoped guard');
|
|
471
|
-
yield requireDisposer(agent.ctx.on.call(agent.ctx, 'system-prompt/assemble', async (_assembly, _context, next)=>{
|
|
472
|
-
if (typeof next !== 'function')
|
|
505
|
+
yield requireDisposer(agent.ctx.on.call(agent.ctx, 'system-prompt/assemble', async (_assembly, _context, next) => {
|
|
506
|
+
if (typeof next !== 'function')
|
|
507
|
+
throw new Error('benchmark conformance assembly continuation unavailable');
|
|
473
508
|
const result = await next();
|
|
474
|
-
if (!plainObject(result))
|
|
509
|
+
if (!plainObject(result))
|
|
510
|
+
throw new Error('benchmark conformance assembly result unavailable');
|
|
475
511
|
const sections = Array.isArray(result.sections) ? result.sections : [];
|
|
476
|
-
return {
|
|
477
|
-
|
|
478
|
-
sections: [
|
|
479
|
-
...sections,
|
|
480
|
-
systemSection(projection)
|
|
481
|
-
]
|
|
482
|
-
};
|
|
483
|
-
}, {
|
|
484
|
-
prepend: true
|
|
485
|
-
}), 'scoped assembly listener');
|
|
512
|
+
return { ...result, sections: [...sections, systemSection(projection)] };
|
|
513
|
+
}, { prepend: true }), 'scoped assembly listener');
|
|
486
514
|
}, 'benchmark-agent-conformance'), 'scoped effect');
|
|
487
|
-
}
|
|
515
|
+
}
|
|
516
|
+
catch (error) {
|
|
488
517
|
byAgent.delete(agent);
|
|
489
518
|
bySession.delete(agent.session);
|
|
490
519
|
throw error;
|
|
491
520
|
}
|
|
492
521
|
});
|
|
493
|
-
bus.on('session/event', (session, event)=>{
|
|
522
|
+
bus.on('session/event', (session, event) => {
|
|
494
523
|
bySession.get(session)?.observe(event);
|
|
495
524
|
if (typeof event === 'object' && event !== null && event.type === 'turn/end') {
|
|
496
525
|
const key = sessionKeys.get(session);
|
|
@@ -500,44 +529,49 @@ export function installBenchmarkAgentConformance(ctx, projection, diagnosticWrit
|
|
|
500
529
|
}
|
|
501
530
|
}
|
|
502
531
|
});
|
|
503
|
-
bus.on('session/disposed', (session)=>{
|
|
532
|
+
bus.on('session/disposed', (session) => {
|
|
504
533
|
const key = sessionKeys.get(session);
|
|
505
|
-
if (key !== undefined)
|
|
534
|
+
if (key !== undefined)
|
|
535
|
+
diagnostics.flush(key);
|
|
506
536
|
sessionKeys.delete(session);
|
|
507
537
|
bySession.delete(session);
|
|
508
538
|
});
|
|
509
|
-
bus.on('agent/disposed', (payload)=>{
|
|
539
|
+
bus.on('agent/disposed', (payload) => {
|
|
510
540
|
const agent = payload.agent;
|
|
511
|
-
if (!agent || typeof agent !== 'object')
|
|
541
|
+
if (!agent || typeof agent !== 'object')
|
|
542
|
+
return;
|
|
512
543
|
byAgent.delete(agent);
|
|
513
|
-
if (agent.session)
|
|
544
|
+
if (agent.session)
|
|
545
|
+
bySession.delete(agent.session);
|
|
514
546
|
});
|
|
515
|
-
bus.on('agent/turn-stopping', (payload)=>{
|
|
547
|
+
bus.on('agent/turn-stopping', (payload) => {
|
|
516
548
|
const agent = payload.agent;
|
|
517
549
|
const session = agent?.session ?? payload.session;
|
|
518
550
|
const key = session && typeof session === 'object' ? sessionKeys.get(session) : undefined;
|
|
519
551
|
if (!agent || typeof agent !== 'object' || typeof payload.turn !== 'number') {
|
|
520
|
-
if (key !== undefined)
|
|
521
|
-
|
|
552
|
+
if (key !== undefined)
|
|
553
|
+
diagnostics.offer(key, 'child_conformance_failure');
|
|
554
|
+
else
|
|
555
|
+
diagnosticWriter('child_conformance_failure');
|
|
522
556
|
throw new Error(BENCHMARK_CONFORMANCE_STATE_UNAVAILABLE);
|
|
523
557
|
}
|
|
524
558
|
const controller = byAgent.get(agent);
|
|
525
559
|
if (!controller) {
|
|
526
|
-
if (key !== undefined)
|
|
527
|
-
|
|
560
|
+
if (key !== undefined)
|
|
561
|
+
diagnostics.offer(key, 'child_conformance_failure');
|
|
562
|
+
else
|
|
563
|
+
diagnosticWriter('child_conformance_failure');
|
|
528
564
|
throw new Error(BENCHMARK_CONFORMANCE_STATE_UNAVAILABLE);
|
|
529
565
|
}
|
|
530
566
|
const decision = controller.stopping(payload.turn);
|
|
531
567
|
if (decision.kind === 'reject') {
|
|
532
|
-
if (key !== undefined)
|
|
533
|
-
|
|
568
|
+
if (key !== undefined)
|
|
569
|
+
diagnostics.offer(key, benchmarkChildFailureForConformanceCode(decision.code));
|
|
570
|
+
else
|
|
571
|
+
diagnosticWriter(benchmarkChildFailureForConformanceCode(decision.code));
|
|
534
572
|
throw new Error(decision.code);
|
|
535
573
|
}
|
|
536
|
-
if (decision.kind === 'steer')
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
});
|
|
574
|
+
if (decision.kind === 'steer')
|
|
575
|
+
agent.steer(correctionMessage(decision.mode, projection));
|
|
576
|
+
}, { prepend: true });
|
|
540
577
|
}
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
//# sourceURL=ts/src/benchmark-agent-conformance.ts
|