@danceiny/gotry 0.0.1-rc.21 → 0.0.1-rc.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +102 -194
- package/README.zh-CN.md +115 -210
- package/bin/gotry-backend.js +53 -0
- package/bin/gotry-bootstrap.js +466 -35
- package/bin/gotry-inner.js +255 -36
- package/bin/gotry-process-liveness.d.ts +27 -0
- package/bin/gotry-process-liveness.js +97 -0
- package/bin/gotry-runtime-resolution.d.ts +1 -1
- package/bin/gotry-runtime-resolution.js +2 -2
- package/client/client.js +174 -0
- package/cordis.gotry-patch.yml +70 -93
- package/dist/capabilities/agent-reach-deep.js +45 -122
- package/dist/capabilities/agent-reach.js +63 -148
- package/dist/capabilities/anything.js +89 -75
- package/dist/capabilities/artifacts.js +123 -104
- package/dist/capabilities/channel-health.js +65 -45
- package/dist/capabilities/channel-registry.js +108 -87
- package/dist/capabilities/doctor.js +357 -235
- package/dist/capabilities/effect.js +223 -309
- package/dist/capabilities/fact-log.js +28 -14
- package/dist/capabilities/flyai.js +245 -156
- package/dist/capabilities/ground-transfer.js +509 -0
- package/dist/capabilities/hbcli.js +144 -148
- package/dist/capabilities/hotelbyte-transaction.js +194 -0
- package/dist/capabilities/incident-log.js +131 -61
- package/dist/capabilities/model-override.js +30 -10
- package/dist/capabilities/opensky.js +54 -43
- package/dist/capabilities/resilience.js +50 -32
- package/dist/capabilities/session/action-cache.js +50 -44
- package/dist/capabilities/session/adapters/ctrip-flight.js +119 -62
- package/dist/capabilities/session/adapters/ctrip-hotel.js +125 -110
- package/dist/capabilities/session/adapters/dida-portal.js +206 -0
- package/dist/capabilities/session/adapters/meituan-local.js +35 -54
- package/dist/capabilities/session/adapters/rail-12306.js +169 -112
- package/dist/capabilities/session/benchmark.js +85 -149
- package/dist/capabilities/session/extension-bridge.js +320 -259
- package/dist/capabilities/session/extension-channel.js +57 -76
- package/dist/capabilities/session/extension-distribution.js +115 -119
- package/dist/capabilities/session/extract.js +31 -21
- package/dist/capabilities/session/golden-score.js +55 -33
- package/dist/capabilities/session/health-watch.js +146 -154
- package/dist/capabilities/session/read-guard.js +55 -47
- package/dist/capabilities/session/static-flight-golden.js +49 -36
- package/dist/capabilities/session/transport.js +54 -61
- package/dist/capabilities/session/wizard.js +63 -93
- package/dist/capabilities/session-consent.js +143 -47
- package/dist/capabilities/session-login.js +95 -126
- package/dist/capabilities/session-search.js +391 -253
- package/dist/capabilities/sponsor-plugin.js +341 -0
- package/dist/capabilities/visa-policy.js +87 -73
- package/dist/capabilities/weather.js +163 -235
- package/dist/scripts/action-cache-tests.js +24 -59
- package/dist/scripts/agent-planning-turn-deadline-e2e.js +91 -142
- package/dist/scripts/agent-planning-turn-deadline-tests.js +80 -176
- package/dist/scripts/agent-reach-deep-tests.js +30 -40
- package/dist/scripts/agent-reach-tests.js +26 -18
- package/dist/scripts/agent-reach-wrapper-tests.js +46 -50
- package/dist/scripts/anything-tests.js +58 -62
- package/dist/scripts/artifact-client-contract-tests.js +69 -0
- package/dist/scripts/artifacts-capability-tests.js +235 -0
- package/dist/scripts/async-collect.js +26 -15
- package/dist/scripts/benchmark-environment-bridge-e2e.js +287 -684
- package/dist/scripts/benchmark-environment-bridge-tests.js +818 -3123
- package/dist/scripts/booking-copilot-availability-ledger-binding-tests.js +66 -279
- package/dist/scripts/booking-copilot-availability-policy-tests.js +153 -1257
- package/dist/scripts/booking-copilot-bin-proof-tests.js +22 -45
- package/dist/scripts/booking-copilot-crossrepo-fixture-server.js +64 -76
- package/dist/scripts/booking-copilot-dsh-core-proof-tests.js +69 -199
- package/dist/scripts/booking-copilot-dsh-planner-proof-tests.js +393 -759
- package/dist/scripts/booking-copilot-dsh-plugin-proof-tests.js +33 -75
- package/dist/scripts/booking-copilot-event-sequence-concurrency-proof-tests.js +60 -110
- package/dist/scripts/booking-copilot-gap-code-contract-proof-tests.js +28 -66
- package/dist/scripts/booking-copilot-operation-ledger-concurrency-proof-tests.js +72 -271
- package/dist/scripts/booking-copilot-receipt-ledger-concurrency-proof-tests.js +73 -275
- package/dist/scripts/booking-copilot-runtime-proof-tests.js +784 -3398
- package/dist/scripts/booking-copilot-server-proof-tests.js +63 -168
- package/dist/scripts/booking-copilot-startup-proof-tests.js +38 -124
- package/dist/scripts/booking-executor-tests.js +80 -0
- package/dist/scripts/booking-recovery-chain-tests.js +182 -0
- package/dist/scripts/booking-saga-tests.js +61 -123
- package/dist/scripts/booking-surface-contract-proof-tests.js +152 -1167
- package/dist/scripts/bootstrap-tests.js +1383 -128
- package/dist/scripts/build-changelog.js +89 -130
- package/dist/scripts/build-metrics-report.js +135 -143
- package/dist/scripts/changelog-tests.js +35 -82
- package/dist/scripts/channel-probe-tests.js +44 -116
- package/dist/scripts/channel-probe.js +84 -167
- package/dist/scripts/channel-registry-tests.js +97 -185
- package/dist/scripts/companion-tests.js +25 -70
- package/dist/scripts/dida-cookie-probe.js +87 -0
- package/dist/scripts/dida-sw-probe.js +114 -0
- package/dist/scripts/diff-test.js +17 -12
- package/dist/scripts/doctor-tests.js +428 -120
- package/dist/scripts/dsh-artifact-e2e.js +150 -0
- package/dist/scripts/dsh-artifact-web-e2e.js +443 -0
- package/dist/scripts/dsh-http-proxy-sse-proof.js +90 -0
- package/dist/scripts/dsh-runtime-closure-tests.js +110 -153
- package/dist/scripts/dsh-runtime-closure.js +75 -63
- package/dist/scripts/dsh-session-v3-migration-proof.js +106 -0
- package/dist/scripts/dsh-subprocess-local-proof.js +149 -0
- package/dist/scripts/dsh-target-closure-proof.js +178 -0
- package/dist/scripts/effect-tests.js +165 -552
- package/dist/scripts/engine-run.js +1 -3
- package/dist/scripts/engine-tests.js +16 -10
- package/dist/scripts/evaluation-cadence-tests.js +69 -242
- package/dist/scripts/evaluation-contract-tests.js +193 -499
- package/dist/scripts/extension-distribution-cli.js +19 -15
- package/dist/scripts/extension-distribution-tests.js +98 -211
- package/dist/scripts/extension-tests.js +366 -389
- package/dist/scripts/fact-gate-tests.js +725 -263
- package/dist/scripts/fixtures/extension-bridge-unref-child.mjs +21 -0
- package/dist/scripts/flight-hotel-anchor-363-registered-e2e.js +169 -0
- package/dist/scripts/flight-malformed-tests.js +159 -0
- package/dist/scripts/flyai-tests.js +325 -68
- package/dist/scripts/g5-guard-tests.js +131 -0
- package/dist/scripts/g5-guard.js +189 -0
- package/dist/scripts/gen-lunar-anchors.js +106 -0
- package/dist/scripts/gotry-backend-kernel-tests.js +79 -0
- package/dist/scripts/ground-transfer-tests.js +956 -0
- package/dist/scripts/hbcli-e2e-tests.js +71 -90
- package/dist/scripts/hbcli-live-optin-tests.js +96 -0
- package/dist/scripts/hbcli-release-contract-tests.js +122 -0
- package/dist/scripts/hbcli-tests.js +41 -115
- package/dist/scripts/health-watch-cli.js +42 -28
- package/dist/scripts/hotel-date-gate-tests.js +496 -0
- package/dist/scripts/hotelbyte-reconcile-tests.js +141 -0
- package/dist/scripts/hotelbyte-spawn-e2e-tests.js +373 -0
- package/dist/scripts/i18n-tests.js +16 -9
- package/dist/scripts/incident-tests.js +281 -87
- package/dist/scripts/issue-194-job-id-guard-tests.js +106 -0
- package/dist/scripts/issue-233-cancel-refund-commission-tests.js +392 -0
- package/dist/scripts/issue-271-liveness-tests.js +547 -0
- package/dist/scripts/issue-289-model-retry-real-tests.js +296 -0
- package/dist/scripts/issue-289-model-retry-tests.js +441 -0
- package/dist/scripts/issue-289-web-retry-e2e.js +630 -0
- package/dist/scripts/issue-338-home-city-e2e.js +577 -0
- package/dist/scripts/issue343-real-entry-e2e.js +160 -0
- package/dist/scripts/journey-tests.js +22 -27
- package/dist/scripts/kernel-manifest-evidence.js +109 -0
- package/dist/scripts/kernel-manifest-gate.js +126 -0
- package/dist/scripts/kernel-manifest-tests.js +212 -0
- package/dist/scripts/kernel-manifest-trace-entry.js +7 -0
- package/dist/scripts/kernel-manifest-trace-hook.mjs +28 -0
- package/dist/scripts/kernel-manifest.js +369 -0
- package/dist/scripts/ledger-repair-apply-tests.js +316 -0
- package/dist/scripts/ledger-repair-plan-tests.js +375 -0
- package/dist/scripts/ledger-tests.js +325 -340
- package/dist/scripts/ledger-workflow-crash.js +10 -20
- package/dist/scripts/map-tools-vendor-package-proof.js +206 -134
- package/dist/scripts/memory-capture-tests.js +242 -61
- package/dist/scripts/memory-decay-tests.js +18 -52
- package/dist/scripts/memory-lifecycle-tests.js +657 -0
- package/dist/scripts/memory-lifecycle.js +196 -0
- package/dist/scripts/memory-metrics.js +24 -12
- package/dist/scripts/memory-value-report.js +474 -222
- package/dist/scripts/memory-value-tests.js +328 -0
- package/dist/scripts/metrics-report-tests.js +62 -199
- package/dist/scripts/model-override-e2e.js +66 -105
- package/dist/scripts/nightly-evidence-tests.js +64 -89
- package/dist/scripts/nightly-evidence.js +108 -111
- package/dist/scripts/nudge-digest.js +39 -28
- package/dist/scripts/onboarding-tests.js +63 -88
- package/dist/scripts/opensky-check.js +18 -13
- package/dist/scripts/opensky-tests.js +28 -28
- package/dist/scripts/persona-surface-guard-tests.js +37 -12
- package/dist/scripts/pnpm-dsh-closure-proof.js +2 -5
- package/dist/scripts/policy-anchor-359-registered-e2e.js +214 -0
- package/dist/scripts/price-drift-tests.js +126 -227
- package/dist/scripts/price-drift-watch.js +209 -253
- package/dist/scripts/probe-poi-tests.js +52 -45
- package/dist/scripts/product-metrics.js +187 -273
- package/dist/scripts/publish-preverify.js +47 -41
- package/dist/scripts/realtime-pricing-tests.js +37 -76
- package/dist/scripts/replay-async.js +20 -18
- package/dist/scripts/replay-real.js +16 -18
- package/dist/scripts/replay.js +52 -82
- package/dist/scripts/session-attach-diagnose.js +16 -12
- package/dist/scripts/session-attach-poc.js +34 -37
- package/dist/scripts/session-benchmark.js +66 -147
- package/dist/scripts/session-dida-live-e2e.js +151 -0
- package/dist/scripts/session-extract-tests.js +14 -35
- package/dist/scripts/session-login.js +14 -13
- package/dist/scripts/session-tests.js +545 -583
- package/dist/scripts/sf-live-benchmark.js +92 -110
- package/dist/scripts/sf-live-cli-tests.js +2 -9
- package/dist/scripts/sf-soft-score-tests.js +19 -60
- package/dist/scripts/sf-summary-tests.js +149 -0
- package/dist/scripts/sf-summary.js +466 -84
- package/dist/scripts/skeleton-check.js +16 -14
- package/dist/scripts/skeleton-integration-test.js +11 -10
- package/dist/scripts/skills-contract-tests.js +51 -46
- package/dist/scripts/smoke-session-gate-tests.js +12 -20
- package/dist/scripts/smoke.js +541 -422
- package/dist/scripts/sponsor-reuse-tests.js +262 -0
- package/dist/scripts/state-cli-tests.js +226 -101
- package/dist/scripts/state-cli.js +549 -212
- package/dist/scripts/static-golden-tests.js +46 -180
- package/dist/scripts/time-eval-tests.js +210 -192
- package/dist/scripts/travel-timeline-tests.js +29 -93
- package/dist/scripts/turn-handoff-collect-tests.js +46 -93
- package/dist/scripts/turn-handoff-collect.js +71 -99
- package/dist/scripts/turn-policy-tests.js +28 -29
- package/dist/scripts/typed-contract-canary.js +77 -162
- package/dist/scripts/tz-resolver-sanity.js +50 -0
- package/dist/scripts/tz-resolver-tests.js +584 -0
- package/dist/scripts/unified-tests.js +23 -19
- package/dist/scripts/visa-policy-tests.js +31 -44
- package/dist/scripts/weather-tests.js +121 -543
- package/dist/scripts/wish-channel-gate-tests.js +29 -86
- package/dist/scripts/write-gate-tests.js +522 -0
- package/dist/scripts/z3-lifecycle-fault-tests.js +188 -0
- package/dist/scripts/z3-lifecycle-tests.js +170 -0
- package/dist/scripts/z3-race-repeat-tests.js +39 -0
- package/dist/scripts/z3-race-tests.js +18 -12
- package/dist/src/artifact-gate.js +652 -221
- package/dist/src/backend/kernel.js +91 -0
- package/dist/src/backend/modules/booking-copilot.js +55 -0
- package/dist/src/backend/modules/booking-executor.js +124 -0
- package/dist/src/backend/modules/session-search.js +194 -0
- package/dist/src/benchmark-agent-conformance.js +296 -262
- package/dist/src/benchmark-environment-bridge.js +298 -413
- package/dist/src/benchmark-headless-child-diagnostics.js +93 -100
- package/dist/src/benchmark-tool-isolation.js +94 -67
- package/dist/src/bookable-facts.js +299 -174
- package/dist/src/booking-saga.js +64 -120
- package/dist/src/booking-surface/availability-policy.js +443 -576
- package/dist/src/booking-surface/cancel-refund-commission.js +379 -0
- package/dist/src/booking-surface/contracts.js +24 -80
- package/dist/src/booking-surface/dsh-planner.js +510 -281
- package/dist/src/booking-surface/dsh-plugin.js +17 -1
- package/dist/src/booking-surface/error-codes.js +82 -10
- package/dist/src/booking-surface/index.js +0 -3
- package/dist/src/booking-surface/profile.js +10 -54
- package/dist/src/booking-surface/recovery-chain.js +367 -0
- package/dist/src/booking-surface/runtime.js +765 -891
- package/dist/src/booking-surface/server.js +200 -251
- package/dist/src/booking-surface/startup.js +58 -54
- package/dist/src/booking-surface/validation.js +150 -218
- package/dist/src/bridge.js +20 -13
- package/dist/src/companions.js +48 -74
- package/dist/src/contracts.js +21 -43
- package/dist/src/dsh-llm.js +82 -120
- package/dist/src/engine.js +253 -239
- package/dist/src/evaluation-cadence.js +72 -209
- package/dist/src/evaluation-contracts.js +220 -843
- package/dist/src/flight-pack-adapter.js +54 -0
- package/dist/src/flight-pack-contract.js +2 -0
- package/dist/src/gotry-backend.js +70 -0
- package/dist/src/hotel-date-gate.js +118 -0
- package/dist/src/i18n.js +18 -7
- package/dist/src/index.js +1425 -1629
- package/dist/src/journey.js +132 -81
- package/dist/src/ledger-repair-apply.js +538 -0
- package/dist/src/ledger-repair-plan.js +463 -0
- package/dist/src/loop.js +257 -166
- package/dist/src/memory-capture.js +180 -32
- package/dist/src/memory-decay.js +30 -14
- package/dist/src/memory-lifecycle.js +1310 -0
- package/dist/src/memory-utility.js +27 -40
- package/dist/src/mock-llm.js +36 -61
- package/dist/src/model.js +81 -36
- package/dist/src/realtime-pricing.js +63 -72
- package/dist/src/slot-spec.js +84 -106
- package/dist/src/state-ledger.js +310 -471
- package/dist/src/subagent-job-id-guard.js +58 -0
- package/dist/src/time-anchor.js +231 -33
- package/dist/src/tool-packet.js +20 -8
- package/dist/src/travel-slots.js +58 -35
- package/dist/src/travel-timeline.js +44 -63
- package/dist/src/turn-deadline.js +121 -127
- package/dist/src/turn-policy.js +61 -89
- package/dist/src/tz-resolver.js +143 -0
- package/dist/src/unified.js +504 -381
- package/dist/src/wish-pool.js +39 -14
- package/dist/src/write-gate.js +495 -0
- package/dist/src/z3-shared.js +273 -7
- package/extension/README.md +4 -3
- package/extension/background.js +125 -14
- package/extension/content-bridge.js +23 -1
- package/extension/content-main.js +15 -4
- package/extension/manifest.json +16 -8
- package/package.json +251 -220
- package/ts/capabilities/agent-reach.ts +4 -4
- package/ts/capabilities/anything.ts +1 -1
- package/ts/capabilities/artifacts.ts +46 -9
- package/ts/capabilities/flyai.ts +156 -37
- package/ts/capabilities/hbcli.ts +1 -1
- package/ts/capabilities/incident-log.ts +88 -59
- package/ts/capabilities/session/adapters/ctrip-flight.ts +99 -36
- package/ts/capabilities/session/adapters/dida-portal.ts +250 -0
- package/ts/capabilities/session/adapters/rail-12306.ts +123 -24
- package/ts/capabilities/session/extension-bridge.ts +226 -101
- package/ts/capabilities/session/extension-channel.ts +22 -9
- package/ts/capabilities/session/wizard.ts +2 -2
- package/ts/capabilities/session-consent.ts +98 -14
- package/ts/capabilities/session-login.ts +1 -0
- package/ts/capabilities/session-search.ts +315 -31
- package/ts/dsh-runtime/vendor/README.md +8 -4
- package/ts/dsh-runtime/vendor/dsh-map-tools/README.en.md +7 -1
- package/ts/dsh-runtime/vendor/dsh-map-tools/README.md +5 -3
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/settings-ns.js +1 -1
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/settings-ns.d.ts +1 -1
- package/ts/dsh-runtime/vendor/dsh-map-tools/package.json +6 -6
- package/ts/package.json +236 -16
- package/ts/scripts/map-tools-vendor-package-proof.ts +152 -31
- package/ts/scripts/state-cli.ts +418 -51
- package/ts/src/artifact-gate.ts +561 -15
- package/ts/src/benchmark-environment-bridge.ts +37 -20
- package/ts/src/bookable-facts.ts +183 -7
- package/ts/src/booking-surface/cancel-refund-commission.ts +488 -0
- package/ts/src/booking-surface/dsh-planner.ts +240 -32
- package/ts/src/booking-surface/dsh-plugin.js +17 -1
- package/ts/src/booking-surface/error-codes.ts +75 -0
- package/ts/src/booking-surface/recovery-chain.ts +409 -0
- package/ts/src/booking-surface/server.ts +84 -51
- package/ts/src/booking-surface/validation.ts +2 -0
- package/ts/src/contracts.ts +12 -2
- package/ts/src/dsh-llm.ts +4 -8
- package/ts/src/index.ts +609 -90
- package/ts/src/loop.ts +116 -7
- package/ts/src/memory-capture.ts +227 -3
- package/ts/src/mock-llm.ts +4 -8
- package/ts/src/model.ts +88 -2
- package/ts/src/state-ledger.ts +47 -29
- package/ts/src/subagent-job-id-guard.ts +72 -0
- package/ts/src/time-anchor.ts +217 -13
- package/ts/src/unified.ts +316 -54
|
@@ -1,3 +1,10 @@
|
|
|
1
|
+
/** Offline contract for the opt-in benchmark environment bridge.
|
|
2
|
+
*
|
|
3
|
+
* Covers default-off, explicit opt-in, and fail-closed configuration paths.
|
|
4
|
+
* A local developer run exercises the source checkout. The packaged consumer
|
|
5
|
+
* path is built from the current root @deepseek-ai/dsh 0.1.5-rc.1 closure;
|
|
6
|
+
* version/source counterexamples use isolated synthetic fixtures.
|
|
7
|
+
*/
|
|
1
8
|
import assert from 'node:assert/strict';
|
|
2
9
|
import { createServer } from 'node:http';
|
|
3
10
|
import { chmodSync, cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
|
@@ -6,115 +13,55 @@ import { tmpdir } from 'node:os';
|
|
|
6
13
|
import { dirname, join } from 'node:path';
|
|
7
14
|
import { pathToFileURL } from 'node:url';
|
|
8
15
|
import { spawn } from 'node:child_process';
|
|
9
|
-
import { benchmarkRuntimeSupported, selectDshCwd, selectDshRuntime, supportsNodeVersion } from '../../bin/gotry-runtime-resolution.js';
|
|
16
|
+
import { benchmarkRuntimeSupported, selectDshCwd, selectDshRuntime, supportsNodeVersion, } from '../../bin/gotry-runtime-resolution.js';
|
|
10
17
|
const ROOT = join(import.meta.dirname, '..', '..');
|
|
11
18
|
const BIN = join(ROOT, 'bin', 'gotry-inner.js');
|
|
12
19
|
const TOOL = 'gotry_benchmark_environment';
|
|
13
20
|
const MARKER = 'BENCHMARK_BRIDGE_LOOKUP_OK';
|
|
14
21
|
const TIMEOUT_MS = 30_000;
|
|
15
|
-
const LOOKUP_INPUT_SCHEMA = {
|
|
16
|
-
type: 'object',
|
|
17
|
-
properties: {
|
|
18
|
-
city: {
|
|
19
|
-
type: 'string',
|
|
20
|
-
enum: [
|
|
21
|
-
'Dubai',
|
|
22
|
-
'Singapore'
|
|
23
|
-
]
|
|
24
|
-
}
|
|
25
|
-
},
|
|
26
|
-
required: [
|
|
27
|
-
'city'
|
|
28
|
-
],
|
|
29
|
-
additionalProperties: false
|
|
30
|
-
};
|
|
22
|
+
const LOOKUP_INPUT_SCHEMA = { type: 'object', properties: { city: { type: 'string', enum: ['Dubai', 'Singapore'] } }, required: ['city'], additionalProperties: false };
|
|
31
23
|
const TSX_LOADER = pathToFileURL(createRequire(import.meta.url).resolve('tsx')).href;
|
|
32
24
|
function runRuntimeProbe(options) {
|
|
33
25
|
const fixture = mkdtempSync(join(tmpdir(), 'gotry-runtime-probe-'));
|
|
34
26
|
try {
|
|
35
|
-
writeFileSync(join(fixture, 'package.json'), JSON.stringify({
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
const writeDsh = (root, version)=>{
|
|
40
|
-
mkdirSync(join(root, 'lib'), {
|
|
41
|
-
recursive: true
|
|
42
|
-
});
|
|
43
|
-
writeFileSync(join(root, 'package.json'), JSON.stringify({
|
|
44
|
-
name: '@deepseek-ai/dsh',
|
|
45
|
-
version,
|
|
46
|
-
type: 'module'
|
|
47
|
-
}));
|
|
27
|
+
writeFileSync(join(fixture, 'package.json'), JSON.stringify({ name: 'runtime-probe', type: 'module' }));
|
|
28
|
+
const writeDsh = (root, version) => {
|
|
29
|
+
mkdirSync(join(root, 'lib'), { recursive: true });
|
|
30
|
+
writeFileSync(join(root, 'package.json'), JSON.stringify({ name: '@deepseek-ai/dsh', version, type: 'module' }));
|
|
48
31
|
writeFileSync(join(root, 'lib', 'bin.js'), 'export {}\n');
|
|
49
32
|
};
|
|
50
|
-
if (options.rootVersion)
|
|
51
|
-
|
|
33
|
+
if (options.rootVersion)
|
|
34
|
+
writeDsh(join(fixture, 'node_modules', '@deepseek-ai', 'dsh'), options.rootVersion);
|
|
35
|
+
if (options.vendorVersion)
|
|
36
|
+
writeDsh(join(fixture, 'ts', 'dsh-runtime', 'node_modules', '@deepseek-ai', 'dsh'), options.vendorVersion);
|
|
52
37
|
const runtime = selectDshRuntime({
|
|
53
38
|
repoRoot: fixture,
|
|
54
39
|
rootResolver: createRequire(join(fixture, 'package.json')),
|
|
55
|
-
benchmark: options.benchmark === true
|
|
56
|
-
});
|
|
57
|
-
return runtime ? {
|
|
58
|
-
source: runtime.source,
|
|
59
|
-
version: runtime.version
|
|
60
|
-
} : null;
|
|
61
|
-
} finally{
|
|
62
|
-
rmSync(fixture, {
|
|
63
|
-
recursive: true,
|
|
64
|
-
force: true
|
|
40
|
+
benchmark: options.benchmark === true,
|
|
65
41
|
});
|
|
42
|
+
return runtime ? { source: runtime.source, version: runtime.version } : null;
|
|
43
|
+
}
|
|
44
|
+
finally {
|
|
45
|
+
rmSync(fixture, { recursive: true, force: true });
|
|
66
46
|
}
|
|
67
47
|
}
|
|
68
48
|
function assertRuntimeSelectionAndVersionGuards() {
|
|
69
|
-
const sourcePriority = runRuntimeProbe({
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
});
|
|
73
|
-
assert.deepEqual(sourcePriority, {
|
|
74
|
-
source: 'root',
|
|
75
|
-
version: '0.1.2-alpha.3'
|
|
76
|
-
}, 'source checkout uses the root dsh package even when legacy vendor is alpha.1');
|
|
77
|
-
const legacyFallback = runRuntimeProbe({
|
|
78
|
-
vendorVersion: '0.1.2-alpha.1'
|
|
79
|
-
});
|
|
49
|
+
const sourcePriority = runRuntimeProbe({ rootVersion: '0.1.5-rc.1', vendorVersion: '0.1.2-alpha.1' });
|
|
50
|
+
assert.deepEqual(sourcePriority, { source: 'root', version: '0.1.5-rc.1' }, 'source checkout uses the root dsh package even when legacy vendor is alpha.1');
|
|
51
|
+
const legacyFallback = runRuntimeProbe({ vendorVersion: '0.1.2-alpha.1' });
|
|
80
52
|
assert.deepEqual(legacyFallback, null, 'non-benchmark source checkout fail-closes instead of using the removed legacy vendored dsh fallback');
|
|
81
|
-
const wrongBenchmarkVersion = runRuntimeProbe({
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
});
|
|
86
|
-
assert.deepEqual(wrongBenchmarkVersion, {
|
|
87
|
-
source: 'root',
|
|
88
|
-
version: '0.1.2-alpha.1'
|
|
89
|
-
});
|
|
90
|
-
assert.equal(benchmarkRuntimeSupported(wrongBenchmarkVersion), false, 'benchmark mode rejects a non-alpha.3 dsh runtime before spawn');
|
|
91
|
-
assert.equal(runRuntimeProbe({
|
|
92
|
-
vendorVersion: '0.1.2-alpha.1',
|
|
93
|
-
benchmark: true
|
|
94
|
-
}), null, 'benchmark mode never falls back to legacy vendored dsh');
|
|
53
|
+
const wrongBenchmarkVersion = runRuntimeProbe({ rootVersion: '0.1.2-alpha.1', vendorVersion: '0.1.2-alpha.1', benchmark: true });
|
|
54
|
+
assert.deepEqual(wrongBenchmarkVersion, { source: 'root', version: '0.1.2-alpha.1' });
|
|
55
|
+
assert.equal(benchmarkRuntimeSupported(wrongBenchmarkVersion), false, 'benchmark mode rejects a non-target dsh runtime before spawn');
|
|
56
|
+
assert.equal(runRuntimeProbe({ vendorVersion: '0.1.2-alpha.1', benchmark: true }), null, 'benchmark mode never falls back to legacy vendored dsh');
|
|
95
57
|
assert.equal(supportsNodeVersion('22.14.0'), false, 'Node 22.14 is rejected before dsh resolution/spawn');
|
|
96
58
|
assert.equal(supportsNodeVersion('22.15.0'), true, 'Node 22.15 is the accepted minimum');
|
|
97
59
|
assert.equal(supportsNodeVersion('24.0.0'), true, 'newer Node majors remain accepted');
|
|
98
60
|
const invocationCwd = join(tmpdir(), 'gotry-runtime-invocation-cwd');
|
|
99
61
|
const sourceStateRoot = join(ROOT, 'ts/dsh-runtime');
|
|
100
|
-
assert.equal(selectDshCwd({
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
sourceCheckoutMode: true,
|
|
104
|
-
benchmark: false
|
|
105
|
-
}), sourceStateRoot, 'source checkout normal mode keeps dsh cwd at ts/dsh-runtime for gotry-state continuity');
|
|
106
|
-
assert.equal(selectDshCwd({
|
|
107
|
-
repoRoot: ROOT,
|
|
108
|
-
invocationCwd,
|
|
109
|
-
sourceCheckoutMode: true,
|
|
110
|
-
benchmark: true
|
|
111
|
-
}), invocationCwd, 'source checkout benchmark mode uses the isolated invocation cwd');
|
|
112
|
-
assert.equal(selectDshCwd({
|
|
113
|
-
repoRoot: ROOT,
|
|
114
|
-
invocationCwd,
|
|
115
|
-
sourceCheckoutMode: false,
|
|
116
|
-
benchmark: false
|
|
117
|
-
}), invocationCwd, 'installed package normal mode uses the user invocation cwd');
|
|
62
|
+
assert.equal(selectDshCwd({ repoRoot: ROOT, invocationCwd, sourceCheckoutMode: true, benchmark: false }), sourceStateRoot, 'source checkout normal mode keeps dsh cwd at ts/dsh-runtime for gotry-state continuity');
|
|
63
|
+
assert.equal(selectDshCwd({ repoRoot: ROOT, invocationCwd, sourceCheckoutMode: true, benchmark: true }), invocationCwd, 'source checkout benchmark mode uses the isolated invocation cwd');
|
|
64
|
+
assert.equal(selectDshCwd({ repoRoot: ROOT, invocationCwd, sourceCheckoutMode: false, benchmark: false }), invocationCwd, 'installed package normal mode uses the user invocation cwd');
|
|
118
65
|
}
|
|
119
66
|
function writeResolutionProbe(path, resultPath, block) {
|
|
120
67
|
writeFileSync(path, `
|
|
@@ -167,197 +114,93 @@ Module.createRequire = function (...args) {
|
|
|
167
114
|
return required
|
|
168
115
|
}
|
|
169
116
|
Module.syncBuiltinESMExports()
|
|
170
|
-
`, {
|
|
171
|
-
mode: 0o600,
|
|
172
|
-
flag: 'wx'
|
|
173
|
-
});
|
|
117
|
+
`, { mode: 0o600, flag: 'wx' });
|
|
174
118
|
}
|
|
175
119
|
function sse(payload) {
|
|
176
120
|
return `data: ${JSON.stringify(payload)}\n\n`;
|
|
177
121
|
}
|
|
178
122
|
function finalText(text) {
|
|
179
|
-
return sse({
|
|
180
|
-
id: 'bridge-final',
|
|
181
|
-
object: 'chat.completion.chunk',
|
|
182
|
-
choices: [
|
|
183
|
-
{
|
|
184
|
-
delta: {
|
|
185
|
-
role: 'assistant',
|
|
186
|
-
content: text
|
|
187
|
-
},
|
|
188
|
-
finish_reason: null
|
|
189
|
-
}
|
|
190
|
-
]
|
|
191
|
-
}) + sse({
|
|
192
|
-
id: 'bridge-final-stop',
|
|
193
|
-
object: 'chat.completion.chunk',
|
|
194
|
-
choices: [
|
|
195
|
-
{
|
|
196
|
-
delta: {},
|
|
197
|
-
finish_reason: 'stop'
|
|
198
|
-
}
|
|
199
|
-
]
|
|
200
|
-
}) + 'data: [DONE]\n\n';
|
|
123
|
+
return sse({ id: 'bridge-final', object: 'chat.completion.chunk', choices: [{ delta: { role: 'assistant', content: text }, finish_reason: null }] })
|
|
124
|
+
+ sse({ id: 'bridge-final-stop', object: 'chat.completion.chunk', choices: [{ delta: {}, finish_reason: 'stop' }] }) + 'data: [DONE]\n\n';
|
|
201
125
|
}
|
|
202
126
|
function toolCall(callId = 'bridge-call-1', city = 'Dubai') {
|
|
203
|
-
return sse({
|
|
204
|
-
id:
|
|
205
|
-
object: 'chat.completion.chunk',
|
|
206
|
-
choices: [
|
|
207
|
-
{
|
|
208
|
-
delta: {
|
|
209
|
-
role: 'assistant',
|
|
210
|
-
tool_calls: [
|
|
211
|
-
{
|
|
212
|
-
index: 0,
|
|
213
|
-
id: callId,
|
|
214
|
-
type: 'function',
|
|
215
|
-
function: {
|
|
216
|
-
name: TOOL,
|
|
217
|
-
arguments: JSON.stringify({
|
|
218
|
-
action: 'call',
|
|
219
|
-
tool: 'lookup',
|
|
220
|
-
arguments: {
|
|
221
|
-
city
|
|
222
|
-
}
|
|
223
|
-
})
|
|
224
|
-
}
|
|
225
|
-
}
|
|
226
|
-
]
|
|
227
|
-
},
|
|
228
|
-
finish_reason: null
|
|
229
|
-
}
|
|
230
|
-
]
|
|
231
|
-
}) + sse({
|
|
232
|
-
id: 'bridge-call-stop',
|
|
233
|
-
object: 'chat.completion.chunk',
|
|
234
|
-
choices: [
|
|
235
|
-
{
|
|
236
|
-
delta: {},
|
|
237
|
-
finish_reason: 'tool_calls'
|
|
238
|
-
}
|
|
239
|
-
]
|
|
240
|
-
}) + 'data: [DONE]\n\n';
|
|
127
|
+
return sse({ id: `bridge-${callId}`, object: 'chat.completion.chunk', choices: [{ delta: { role: 'assistant', tool_calls: [{ index: 0, id: callId, type: 'function', function: { name: TOOL, arguments: JSON.stringify({ action: 'call', tool: 'lookup', arguments: { city } }) } }] }, finish_reason: null }] })
|
|
128
|
+
+ sse({ id: 'bridge-call-stop', object: 'chat.completion.chunk', choices: [{ delta: {}, finish_reason: 'tool_calls' }] }) + 'data: [DONE]\n\n';
|
|
241
129
|
}
|
|
242
130
|
function names(body) {
|
|
243
|
-
return (body.tools ?? []).map((t)
|
|
244
|
-
const f = t.function;
|
|
245
|
-
return String(f?.name ?? t.name ?? '');
|
|
246
|
-
}).filter(Boolean);
|
|
131
|
+
return (body.tools ?? []).map(t => { const f = t.function; return String(f?.name ?? t.name ?? ''); }).filter(Boolean);
|
|
247
132
|
}
|
|
248
133
|
function toolResultPresent(body) {
|
|
249
|
-
return (body.messages ?? []).some(
|
|
134
|
+
return (body.messages ?? []).some(m => m.role === 'tool' && JSON.stringify(m).includes(MARKER));
|
|
250
135
|
}
|
|
251
136
|
function anyToolResultPresent(body) {
|
|
252
|
-
return (body.messages ?? []).some(
|
|
137
|
+
return (body.messages ?? []).some(m => m.role === 'tool');
|
|
253
138
|
}
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
status: {
|
|
258
|
-
type: 'string'
|
|
259
|
-
},
|
|
260
|
-
payload: {
|
|
261
|
-
type: 'string'
|
|
262
|
-
}
|
|
263
|
-
},
|
|
264
|
-
required: [],
|
|
265
|
-
additionalProperties: false
|
|
266
|
-
};
|
|
139
|
+
// Round 12(#215):bridge config 显式携带无数据值 closed terminal body schema;
|
|
140
|
+
// 覆盖两个 e2e 终态形态 {"status":"succeeded"} 与 {payload:"…"}。
|
|
141
|
+
const BRIDGE_E2E_BODY_SCHEMA = { type: 'object', properties: { status: { type: 'string' }, payload: { type: 'string' } }, required: [], additionalProperties: false };
|
|
267
142
|
const TERMINAL_OUTLINE = 'object{?status:string,?payload:string}';
|
|
268
143
|
async function runCase(mode, executableOverride, extraEnv = {}) {
|
|
269
144
|
const requests = [];
|
|
270
145
|
let servedToolCalls = 0;
|
|
271
146
|
let spawnTarget = '';
|
|
272
147
|
const domainRecoveryMode = mode === 'domain-recovery' || mode === 'domain-recovery-failed';
|
|
273
|
-
const server = createServer((req, res)=>{
|
|
148
|
+
const server = createServer((req, res) => {
|
|
274
149
|
const chunks = [];
|
|
275
|
-
req.on('data',
|
|
276
|
-
req.on('end', ()=>{
|
|
150
|
+
req.on('data', c => chunks.push(Buffer.from(c)));
|
|
151
|
+
req.on('end', () => {
|
|
277
152
|
let body = {};
|
|
278
153
|
try {
|
|
279
154
|
body = JSON.parse(Buffer.concat(chunks).toString());
|
|
280
|
-
}
|
|
155
|
+
}
|
|
156
|
+
catch { /* diagnostic remains structural */ }
|
|
281
157
|
requests.push(body);
|
|
282
|
-
res.writeHead(200, {
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
if (mode === 'spawn-failed' && names(body).includes(TOOL) && !anyToolResultPresent(body) && spawnTarget) rmSync(spawnTarget, {
|
|
286
|
-
force: true
|
|
287
|
-
});
|
|
158
|
+
res.writeHead(200, { 'content-type': 'text/event-stream' });
|
|
159
|
+
if (mode === 'spawn-failed' && names(body).includes(TOOL) && !anyToolResultPresent(body) && spawnTarget)
|
|
160
|
+
rmSync(spawnTarget, { force: true });
|
|
288
161
|
if (domainRecoveryMode && servedToolCalls === 1 && names(body).includes(TOOL) && anyToolResultPresent(body) && !toolResultPresent(body)) {
|
|
289
162
|
servedToolCalls += 1;
|
|
290
163
|
res.end(toolCall('bridge-call-2', 'Singapore'));
|
|
291
|
-
}
|
|
164
|
+
}
|
|
165
|
+
else if (mode !== 'disabled' && mode !== 'invalid-path' && mode !== 'invalid-schema' && mode !== 'unsafe-config' && names(body).includes(TOOL) && !anyToolResultPresent(body)) {
|
|
292
166
|
servedToolCalls += 1;
|
|
293
167
|
res.end(toolCall());
|
|
294
|
-
}
|
|
168
|
+
}
|
|
169
|
+
else {
|
|
295
170
|
res.end(finalText('<benchmark_terminal>{"status":"succeeded"}</benchmark_terminal>'));
|
|
296
171
|
}
|
|
297
172
|
});
|
|
298
173
|
});
|
|
299
|
-
await new Promise(
|
|
174
|
+
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
|
|
300
175
|
const port = server.address().port;
|
|
301
176
|
const cwd = mkdtempSync(join(tmpdir(), 'gotry-bridge-cwd-'));
|
|
302
177
|
const dsh = mkdtempSync(join(tmpdir(), 'gotry-bridge-dsh-'));
|
|
303
178
|
const probe = join(cwd, 'benchmark-resolution-probe.cjs');
|
|
304
179
|
const probeResult = join(cwd, 'benchmark-resolution-hits.json');
|
|
305
|
-
writeFileSync(probeResult, '', {
|
|
306
|
-
mode: 0o600,
|
|
307
|
-
flag: 'wx'
|
|
308
|
-
});
|
|
180
|
+
writeFileSync(probeResult, '', { mode: 0o600, flag: 'wx' });
|
|
309
181
|
writeResolutionProbe(probe, probeResult, mode === 'disabled');
|
|
310
182
|
const runner = join(cwd, 'synthetic-runner.js');
|
|
311
183
|
const runnerTrace = join(cwd, 'synthetic-runner-trace.jsonl');
|
|
312
184
|
spawnTarget = join(cwd, 'synthetic-spawn-target.js');
|
|
313
185
|
const configPath = join(cwd, mode === 'invalid-path' ? 'benchmark-env-config-\n.json' : 'benchmark-env-config.json');
|
|
314
|
-
const runnerBody = mode === 'domain-recovery-failed'
|
|
186
|
+
const runnerBody = mode === 'domain-recovery-failed'
|
|
187
|
+
? `if (args.city === 'Dubai') process.stdout.write(JSON.stringify({ schema_version: 'gotry_benchmark_tool_result_v1', status: 'miss', code: 'NOT_FOUND', recovery: 'revise_arguments' })); else { process.stderr.write('PRIVATE_RECOVERY_RUNNER_DIAGNOSTIC_DO_NOT_REFLECT'); process.exit(17) }`
|
|
188
|
+
: mode === 'domain-recovery'
|
|
189
|
+
? `if (args.city === 'Dubai') process.stdout.write(JSON.stringify({ schema_version: 'gotry_benchmark_tool_result_v1', status: 'miss', code: 'NOT_FOUND', recovery: 'revise_arguments' })); else process.stdout.write(JSON.stringify({ schema_version: 'gotry_benchmark_tool_result_v1', status: 'ok', result: { marker: '${MARKER}', leaked: [] } }))`
|
|
190
|
+
: mode === 'timeout'
|
|
191
|
+
? `setTimeout(() => {}, 60_000)`
|
|
192
|
+
: mode === 'runner-failed'
|
|
193
|
+
? `process.stderr.write('PRIVATE_RUNNER_DIAGNOSTIC_DO_NOT_REFLECT'); process.exit(17)`
|
|
194
|
+
: mode === 'output-truncated'
|
|
195
|
+
? `process.stdout.write('x'.repeat(20_000))`
|
|
196
|
+
: mode === 'unexpected-output'
|
|
197
|
+
? `process.stdout.write(JSON.stringify({ schema_version: 'gotry_benchmark_tool_result_v1', status: 'ok', result: { marker: '${MARKER}', leaked: [], unexpected: 'must-not-reflect' } }))`
|
|
198
|
+
: `const forbidden = ['GOTRY_BENCHMARK_ENV_CONFIG', 'GOTRY_BENCHMARK_BRIDGE_PARENT_SECRET', 'LLM_API_KEY', 'LLM_BASE_URL', 'LLM_MODEL', 'DEEPSEEK_BASE_URL', 'GOTRY_LLM_MODEL', 'DATABASE_URL', 'SSH_AUTH_SOCK', 'AWS_PROFILE', 'HTTPS_PROXY']; const leaked = forbidden.filter(name => process.env[name] !== undefined); process.stdout.write(JSON.stringify({ schema_version: 'gotry_benchmark_tool_result_v1', status: 'ok', result: { marker: '${MARKER}', leaked } }))`;
|
|
315
199
|
writeFileSync(runner, `if (process.argv.length !== 5 || process.argv[2] !== 'call' || process.argv[3] !== 'lookup' || (!${JSON.stringify(domainRecoveryMode)} && JSON.parse(process.argv[4]).city !== 'Dubai') || (${JSON.stringify(domainRecoveryMode)} && !['Dubai', 'Singapore'].includes(JSON.parse(process.argv[4]).city))) process.exit(2); const fs = require('node:fs'); const args = JSON.parse(process.argv[4]); fs.appendFileSync(${JSON.stringify(runnerTrace)}, JSON.stringify({ city: args.city }) + '\\n'); ${runnerBody}`);
|
|
316
|
-
writeFileSync(spawnTarget, '#!/usr/bin/env node\nprocess.exit(0)\n', {
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
schema_version: mode === 'invalid-schema' ? 'invalid' : 'gotry_benchmark_environment_bridge_v4',
|
|
321
|
-
enabled: true,
|
|
322
|
-
executable: mode === 'spawn-failed' ? spawnTarget : process.execPath,
|
|
323
|
-
cwd,
|
|
324
|
-
argv_prefix: mode === 'spawn-failed' ? [
|
|
325
|
-
'placeholder'
|
|
326
|
-
] : [
|
|
327
|
-
runner
|
|
328
|
-
],
|
|
329
|
-
tools: [
|
|
330
|
-
{
|
|
331
|
-
name: 'lookup',
|
|
332
|
-
description: 'Lookup.',
|
|
333
|
-
input_schema: LOOKUP_INPUT_SCHEMA,
|
|
334
|
-
output_keys: [
|
|
335
|
-
'marker',
|
|
336
|
-
'leaked'
|
|
337
|
-
],
|
|
338
|
-
domain_outcomes: [
|
|
339
|
-
{
|
|
340
|
-
status: 'miss',
|
|
341
|
-
code: 'NOT_FOUND',
|
|
342
|
-
recovery: domainRecoveryMode ? 'revise_arguments' : 'none'
|
|
343
|
-
}
|
|
344
|
-
]
|
|
345
|
-
}
|
|
346
|
-
],
|
|
347
|
-
timeout_ms: mode === 'timeout' ? 50 : 10_000,
|
|
348
|
-
max_output_bytes: mode === 'output-truncated' ? 1_024 : 4_096,
|
|
349
|
-
terminal_output: {
|
|
350
|
-
tag: 'benchmark_terminal',
|
|
351
|
-
max_bytes: 4_096,
|
|
352
|
-
body_schema: BRIDGE_E2E_BODY_SCHEMA
|
|
353
|
-
},
|
|
354
|
-
isolation: {
|
|
355
|
-
mode: 'host-enforced',
|
|
356
|
-
writes: 'forbidden',
|
|
357
|
-
network: 'denied'
|
|
358
|
-
}
|
|
359
|
-
}));
|
|
360
|
-
if (mode === 'unsafe-config') chmodSync(configPath, 0o666);
|
|
200
|
+
writeFileSync(spawnTarget, '#!/usr/bin/env node\nprocess.exit(0)\n', { mode: 0o700 });
|
|
201
|
+
writeFileSync(configPath, JSON.stringify({ schema_version: mode === 'invalid-schema' ? 'invalid' : 'gotry_benchmark_environment_bridge_v4', enabled: true, executable: mode === 'spawn-failed' ? spawnTarget : process.execPath, cwd, argv_prefix: mode === 'spawn-failed' ? ['placeholder'] : [runner], tools: [{ name: 'lookup', description: 'Lookup.', input_schema: LOOKUP_INPUT_SCHEMA, output_keys: ['marker', 'leaked'], domain_outcomes: [{ status: 'miss', code: 'NOT_FOUND', recovery: domainRecoveryMode ? 'revise_arguments' : 'none' }] }], timeout_ms: mode === 'timeout' ? 50 : 10_000, max_output_bytes: mode === 'output-truncated' ? 1_024 : 4_096, terminal_output: { tag: 'benchmark_terminal', max_bytes: 4_096, body_schema: BRIDGE_E2E_BODY_SCHEMA }, isolation: { mode: 'host-enforced', writes: 'forbidden', network: 'denied' } }));
|
|
202
|
+
if (mode === 'unsafe-config')
|
|
203
|
+
chmodSync(configPath, 0o666);
|
|
361
204
|
const env = {
|
|
362
205
|
...process.env,
|
|
363
206
|
DSH_TOOLS_MODE: 'both',
|
|
@@ -367,108 +210,53 @@ async function runCase(mode, executableOverride, extraEnv = {}) {
|
|
|
367
210
|
LLM_MODEL: 'synthetic-bridge-model',
|
|
368
211
|
DEEPSEEK_API_KEY: 'synthetic-bridge-key',
|
|
369
212
|
DEEPSEEK_BASE_URL: `http://127.0.0.1:${port}/v1`,
|
|
370
|
-
...mode !== 'disabled' ? {
|
|
213
|
+
...(mode !== 'disabled' ? {
|
|
371
214
|
DATABASE_URL: 'postgres://sentinel',
|
|
372
215
|
SSH_AUTH_SOCK: '/tmp/sentinel.sock',
|
|
373
|
-
AWS_PROFILE: 'sentinel-profile'
|
|
374
|
-
} : {},
|
|
216
|
+
AWS_PROFILE: 'sentinel-profile',
|
|
217
|
+
} : {}),
|
|
375
218
|
GOTRY_BENCHMARK_ENV_CONFIG: mode === 'disabled' ? '' : configPath,
|
|
376
|
-
...mode !== 'disabled' ? {
|
|
377
|
-
|
|
378
|
-
} : {},
|
|
379
|
-
...mode === 'debug-redaction' ? {
|
|
380
|
-
GOTRY_DEBUG: '1'
|
|
381
|
-
} : {},
|
|
219
|
+
...(mode !== 'disabled' ? { GOTRY_BENCHMARK_BRIDGE_PARENT_SECRET: 'do-not-leak' } : {}),
|
|
220
|
+
...(mode === 'debug-redaction' ? { GOTRY_DEBUG: '1' } : {}),
|
|
382
221
|
...extraEnv,
|
|
383
|
-
NODE_OPTIONS: [
|
|
384
|
-
process.env.NODE_OPTIONS,
|
|
385
|
-
...!executableOverride ? [
|
|
386
|
-
`--import=${TSX_LOADER}`
|
|
387
|
-
] : [],
|
|
388
|
-
`--require=${probe}`
|
|
389
|
-
].filter(Boolean).join(' ')
|
|
222
|
+
NODE_OPTIONS: [process.env.NODE_OPTIONS, ...(!executableOverride ? [`--import=${TSX_LOADER}`] : []), `--require=${probe}`].filter(Boolean).join(' '),
|
|
390
223
|
};
|
|
391
|
-
for (const key of [
|
|
392
|
-
|
|
393
|
-
'HTTP_PROXY',
|
|
394
|
-
'HTTPS_PROXY',
|
|
395
|
-
'ALL_PROXY',
|
|
396
|
-
'http_proxy',
|
|
397
|
-
'https_proxy',
|
|
398
|
-
'all_proxy'
|
|
399
|
-
])delete env[key];
|
|
224
|
+
for (const key of ['GOTRY_LLM_MODEL', 'HTTP_PROXY', 'HTTPS_PROXY', 'ALL_PROXY', 'http_proxy', 'https_proxy', 'all_proxy'])
|
|
225
|
+
delete env[key];
|
|
400
226
|
env.NO_PROXY = '127.0.0.1,localhost';
|
|
401
|
-
if (mode !== 'disabled')
|
|
402
|
-
|
|
227
|
+
if (mode !== 'disabled')
|
|
228
|
+
env.HTTPS_PROXY = 'https://sentinel-proxy';
|
|
229
|
+
if (mode === 'disabled')
|
|
230
|
+
delete env.GOTRY_BENCHMARK_ENV_CONFIG;
|
|
403
231
|
const executable = executableOverride || process.execPath;
|
|
404
|
-
const invocation = mode === 'web-mode'
|
|
405
|
-
'web',
|
|
406
|
-
'
|
|
407
|
-
|
|
408
|
-
mode === 'debug-redaction' ? 'PRIVATE_QUERY_SENTINEL_DO_NOT_REFLECT' : 'bridge smoke'
|
|
409
|
-
];
|
|
410
|
-
const argv = executableOverride ? invocation : [
|
|
411
|
-
BIN,
|
|
412
|
-
...invocation
|
|
413
|
-
];
|
|
232
|
+
const invocation = mode === 'web-mode'
|
|
233
|
+
? ['web', '--no-open']
|
|
234
|
+
: [mode === 'debug-redaction' ? 'PRIVATE_QUERY_SENTINEL_DO_NOT_REFLECT' : 'bridge smoke'];
|
|
235
|
+
const argv = executableOverride ? invocation : [BIN, ...invocation];
|
|
414
236
|
try {
|
|
415
|
-
const child = spawn(executable, argv, {
|
|
416
|
-
cwd,
|
|
417
|
-
env,
|
|
418
|
-
stdio: [
|
|
419
|
-
'ignore',
|
|
420
|
-
'pipe',
|
|
421
|
-
'pipe'
|
|
422
|
-
]
|
|
423
|
-
});
|
|
237
|
+
const child = spawn(executable, argv, { cwd, env, stdio: ['ignore', 'pipe', 'pipe'] });
|
|
424
238
|
let stdout = '';
|
|
425
239
|
let stderr = '';
|
|
426
|
-
child.stdout.on('data',
|
|
427
|
-
|
|
428
|
-
});
|
|
429
|
-
|
|
430
|
-
stderr += c;
|
|
431
|
-
});
|
|
432
|
-
const exit = await new Promise((resolve)=>{
|
|
433
|
-
const timer = setTimeout(()=>{
|
|
434
|
-
child.kill('SIGKILL');
|
|
435
|
-
resolve(null);
|
|
436
|
-
}, TIMEOUT_MS);
|
|
437
|
-
child.once('close', (code)=>{
|
|
438
|
-
clearTimeout(timer);
|
|
439
|
-
resolve(code);
|
|
440
|
-
});
|
|
441
|
-
});
|
|
442
|
-
const optionalResolutionHits = readFileSync(probeResult, 'utf8').split('\n').filter(Boolean).reduce((hits, line)=>{
|
|
240
|
+
child.stdout.on('data', c => { stdout += c; });
|
|
241
|
+
child.stderr.on('data', c => { stderr += c; });
|
|
242
|
+
const exit = await new Promise(resolve => { const timer = setTimeout(() => { child.kill('SIGKILL'); resolve(null); }, TIMEOUT_MS); child.once('close', code => { clearTimeout(timer); resolve(code); }); });
|
|
243
|
+
const optionalResolutionHits = readFileSync(probeResult, 'utf8').split('\n').filter(Boolean).reduce((hits, line) => {
|
|
443
244
|
const event = JSON.parse(line);
|
|
444
|
-
if (event.target?.includes('dsh-calendar'))
|
|
445
|
-
|
|
245
|
+
if (event.target?.includes('dsh-calendar'))
|
|
246
|
+
hits.calendar += 1;
|
|
247
|
+
if (event.target?.includes('dsh-map-tools'))
|
|
248
|
+
hits.map += 1;
|
|
446
249
|
return hits;
|
|
447
|
-
}, {
|
|
448
|
-
|
|
449
|
-
map
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
requests,
|
|
458
|
-
optionalResolutionHits,
|
|
459
|
-
servedToolCalls,
|
|
460
|
-
runnerArguments
|
|
461
|
-
};
|
|
462
|
-
} finally{
|
|
463
|
-
await new Promise((resolve)=>server.close(()=>resolve()));
|
|
464
|
-
rmSync(dsh, {
|
|
465
|
-
recursive: true,
|
|
466
|
-
force: true
|
|
467
|
-
});
|
|
468
|
-
rmSync(cwd, {
|
|
469
|
-
recursive: true,
|
|
470
|
-
force: true
|
|
471
|
-
});
|
|
250
|
+
}, { calendar: 0, map: 0 });
|
|
251
|
+
const runnerArguments = existsSync(runnerTrace)
|
|
252
|
+
? readFileSync(runnerTrace, 'utf8').split('\n').filter(Boolean).map(line => JSON.parse(line))
|
|
253
|
+
: [];
|
|
254
|
+
return { exit, stdout, stderr, output: stdout + stderr, requests, optionalResolutionHits, servedToolCalls, runnerArguments };
|
|
255
|
+
}
|
|
256
|
+
finally {
|
|
257
|
+
await new Promise(resolve => server.close(() => resolve()));
|
|
258
|
+
rmSync(dsh, { recursive: true, force: true });
|
|
259
|
+
rmSync(cwd, { recursive: true, force: true });
|
|
472
260
|
}
|
|
473
261
|
}
|
|
474
262
|
async function assertRuntimeContract(executableOverride) {
|
|
@@ -479,93 +267,80 @@ async function assertRuntimeContract(executableOverride) {
|
|
|
479
267
|
assert.equal(binSource.includes('headless-startup-hold'), false, `${target} bin must not inject a timer-based startup hold`);
|
|
480
268
|
const disabled = await runCase('disabled', executableOverride);
|
|
481
269
|
assert.equal(disabled.exit, 0, `${target} default-off child exit=${disabled.exit}; output tail=${disabled.output.slice(-2_000)}`);
|
|
482
|
-
assert.ok(disabled.requests.length > 0 && !disabled.requests.some(
|
|
270
|
+
assert.ok(disabled.requests.length > 0 && !disabled.requests.some(r => names(r).includes(TOOL)), `${target} default-off must reach the relay without exposing benchmark tool; exit=${disabled.exit}; requests=${disabled.requests.length}; output=${disabled.output.slice(-2_000)}`);
|
|
483
271
|
assert.ok(disabled.optionalResolutionHits.calendar > 0 || disabled.optionalResolutionHits.map > 0, `${target} default-off must retain optional plugin resolution as a counter-proof`);
|
|
272
|
+
const ordinaryPlannerRequests = disabled.requests.filter(request => names(request).some(name => name.startsWith('gotry_')));
|
|
273
|
+
assert.ok(ordinaryPlannerRequests.length > 0, `${target} default-off must surface at least one ordinary planner request exposing a gotry_* tool (auxiliary title requests are excluded); requests=${disabled.requests.length}; tool surfaces=${JSON.stringify(disabled.requests.map(names))}; output=${disabled.output.slice(-2_000)}`);
|
|
274
|
+
const ORDINARY_IDENTITY_SENTENCE = '你是 GoTry——从出发到下一次出发的 AI 旅行伙伴';
|
|
275
|
+
const CANONICAL_RAW_VARIABLES = ['{{current_date}}', '{{time_anchor_card}}', '{{motivation_brief}}', '{{channel_routing_card}}'];
|
|
276
|
+
const EXPANDED_DATE_PATTERN = /今天是\s*\d{4}-\d{2}-\d{2}/;
|
|
277
|
+
for (const request of ordinaryPlannerRequests) {
|
|
278
|
+
const prompt = JSON.stringify(request);
|
|
279
|
+
assert.ok(prompt.includes(ORDINARY_IDENTITY_SENTENCE), `${target} ordinary planner request must carry the canonical Chinese GoTry identity sentence; prompt head=${prompt.slice(0, 600)}`);
|
|
280
|
+
for (const variable of CANONICAL_RAW_VARIABLES) {
|
|
281
|
+
assert.equal(prompt.includes(variable), false, `${target} ordinary planner request must not retain raw ${variable}; prompt head=${prompt.slice(0, 600)}`);
|
|
282
|
+
}
|
|
283
|
+
assert.match(prompt, EXPANDED_DATE_PATTERN, `${target} ordinary planner request must show the expanded "今天是 YYYY-MM-DD" shape from the canonical {{current_date}} variable; prompt head=${prompt.slice(0, 600)}`);
|
|
284
|
+
}
|
|
484
285
|
const enabled = await runCase('enabled', executableOverride);
|
|
485
286
|
assert.equal(enabled.exit, 0, `${target} opt-in child exit=${enabled.exit}; requests=${enabled.requests.length}; tool surfaces=${JSON.stringify(enabled.requests.map(names))}; output tail=${enabled.output.slice(-10000)}`);
|
|
486
|
-
assert.ok(enabled.requests.some(
|
|
487
|
-
const enabledToolNames = [
|
|
488
|
-
|
|
489
|
-
].
|
|
490
|
-
assert.deepEqual(enabledToolNames, [
|
|
491
|
-
TOOL
|
|
492
|
-
], `${target} enabled runtime must expose exactly the benchmark tool; observed tool names=${JSON.stringify(enabledToolNames)}`);
|
|
493
|
-
const bridgeTool = enabled.requests.find((request)=>names(request).includes(TOOL))?.tools?.find((tool)=>names({
|
|
494
|
-
tools: [
|
|
495
|
-
tool
|
|
496
|
-
]
|
|
497
|
-
}).includes(TOOL));
|
|
287
|
+
assert.ok(enabled.requests.some(r => names(r).includes(TOOL)), `${target} opt-in planner request must expose benchmark tool; schemas=${JSON.stringify(enabled.requests.map(names))}; output=${enabled.output.slice(-4_000)}`);
|
|
288
|
+
const enabledToolNames = [...new Set(enabled.requests.flatMap(names))].sort();
|
|
289
|
+
assert.deepEqual(enabledToolNames, [TOOL], `${target} enabled runtime must expose exactly the benchmark tool; observed tool names=${JSON.stringify(enabledToolNames)}`);
|
|
290
|
+
const bridgeTool = enabled.requests.find(request => names(request).includes(TOOL))?.tools?.find(tool => names({ tools: [tool] }).includes(TOOL));
|
|
498
291
|
const flatSchema = bridgeTool?.function?.parameters;
|
|
499
292
|
assert.equal(flatSchema?.type, 'object', `${target} bridge exposes an object-root wire schema`);
|
|
500
293
|
assert.equal(flatSchema?.oneOf, undefined, `${target} bridge wire has no top-level oneOf`);
|
|
501
|
-
assert.deepEqual(flatSchema?.required, [
|
|
502
|
-
'action'
|
|
503
|
-
]);
|
|
294
|
+
assert.deepEqual(flatSchema?.required, ['action']);
|
|
504
295
|
assert.equal(flatSchema?.additionalProperties, false);
|
|
505
|
-
assert.deepEqual(flatSchema?.properties?.action?.enum, [
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
assert.deepEqual(flatSchema?.properties?.tool?.enum, [
|
|
511
|
-
'lookup'
|
|
512
|
-
]);
|
|
513
|
-
assert.deepEqual(flatSchema?.properties?.arguments, {
|
|
514
|
-
type: 'object',
|
|
515
|
-
additionalProperties: true,
|
|
516
|
-
description: 'action=call 时传给工具的参数对象'
|
|
517
|
-
});
|
|
518
|
-
assert.equal(enabledToolNames.some((name)=>name.startsWith('calendar_') || name.startsWith('map_')), false, `${target} benchmark projection must not expose calendar/map tools`);
|
|
519
|
-
assert.deepEqual(enabled.optionalResolutionHits, {
|
|
520
|
-
calendar: 0,
|
|
521
|
-
map: 0
|
|
522
|
-
}, `${target} benchmark mode must not resolve optional calendar/map plugins`);
|
|
296
|
+
assert.deepEqual(flatSchema?.properties?.action?.enum, ['tools', 'call', 'errors']);
|
|
297
|
+
assert.deepEqual(flatSchema?.properties?.tool?.enum, ['lookup']);
|
|
298
|
+
assert.deepEqual(flatSchema?.properties?.arguments, { type: 'object', additionalProperties: true, description: 'action=call 时传给工具的参数对象' });
|
|
299
|
+
assert.equal(enabledToolNames.some(name => name.startsWith('calendar_') || name.startsWith('map_')), false, `${target} benchmark projection must not expose calendar/map tools`);
|
|
300
|
+
assert.deepEqual(enabled.optionalResolutionHits, { calendar: 0, map: 0 }, `${target} benchmark mode must not resolve optional calendar/map plugins`);
|
|
523
301
|
assert.ok(enabled.requests.some(toolResultPresent), `${target} marker must enter model history as tool result`);
|
|
524
|
-
const leakedReport = enabled.requests.map(
|
|
525
|
-
assert.ok(enabled.requests.some(
|
|
526
|
-
assert.equal(enabled.requests.some(
|
|
527
|
-
const enabledPrompt = enabled.requests.map(
|
|
528
|
-
for (const variable of [
|
|
529
|
-
'{{current_date}}',
|
|
530
|
-
'{{time_anchor_card}}',
|
|
531
|
-
'{{motivation_brief}}'
|
|
532
|
-
]){
|
|
302
|
+
const leakedReport = enabled.requests.map(r => JSON.stringify(r).match(/\\?"leaked\\?":\[(.*?)\]/)?.[1]).filter(Boolean).join('|');
|
|
303
|
+
assert.ok(enabled.requests.some(r => /\\?"leaked\\?":\[\]/.test(JSON.stringify(r))), `${target} tool result must report no forbidden environment names; observed names=${leakedReport || '(none)'}`);
|
|
304
|
+
assert.equal(enabled.requests.some(r => JSON.stringify(r).includes('do-not-leak')), false, `${target} tool result must not expose the parent secret value`);
|
|
305
|
+
const enabledPrompt = enabled.requests.map(r => JSON.stringify(r)).join('\n');
|
|
306
|
+
for (const variable of ['{{current_date}}', '{{time_anchor_card}}', '{{motivation_brief}}']) {
|
|
533
307
|
assert.equal(enabledPrompt.includes(variable), false, `${target} benchmark persona must not retain ${variable}`);
|
|
534
308
|
}
|
|
535
309
|
assert.equal(enabledPrompt.includes('gotry_feasibility_check'), false, `${target} benchmark persona must not retain ordinary GoTry tool instructions`);
|
|
536
|
-
|
|
310
|
+
// Persona invariant: every request that exposes the benchmark tool (a
|
|
311
|
+
// planner request) must carry each stable sentence exactly once. The
|
|
312
|
+
// separate session-title request does NOT expose the benchmark tool and is
|
|
313
|
+
// intentionally not required to carry the persona — dsh-system-prompt
|
|
314
|
+
// 0.1.5-rc.1 emits a distinct system prompt for it (auto-title). This
|
|
315
|
+
// is the precise observed public protocol invariant under target closure,
|
|
316
|
+
// not a relaxed `some` over the full request stream.
|
|
317
|
+
const SENTENCE_A = 'You are GoTry, a task-agnostic travel planning assistant.';
|
|
318
|
+
const SENTENCE_B = 'Use only the current conversation and tools available in this benchmark session.';
|
|
319
|
+
const plannerRequests = enabled.requests.filter(request => names(request).includes(TOOL));
|
|
320
|
+
assert.ok(plannerRequests.length > 0, `${target} must emit at least one planner request exposing the benchmark tool; requests=${enabled.requests.length}; schemas=${JSON.stringify(enabled.requests.map(names))}`);
|
|
321
|
+
for (const request of plannerRequests) {
|
|
537
322
|
const prompt = JSON.stringify(request);
|
|
538
|
-
|
|
539
|
-
|
|
323
|
+
const aCount = (prompt.match(new RegExp(SENTENCE_A.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g')) ?? []).length;
|
|
324
|
+
const bCount = (prompt.match(new RegExp(SENTENCE_B.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g')) ?? []).length;
|
|
325
|
+
assert.equal(aCount, 1, `${target} planner request must carry stable sentence A exactly once (observed=${aCount}); prompt head=${prompt.slice(0, 400)}`);
|
|
326
|
+
assert.equal(bCount, 1, `${target} planner request must carry stable sentence B exactly once (observed=${bCount}); prompt head=${prompt.slice(0, 400)}`);
|
|
327
|
+
}
|
|
540
328
|
assert.match(enabled.output, /benchmark_terminal/);
|
|
541
329
|
const recovered = await runCase('domain-recovery', executableOverride);
|
|
542
330
|
assert.equal(recovered.exit, 0, `${target} model-driven domain miss recovery exits successfully; output=${recovered.output.slice(-2_000)}`);
|
|
543
331
|
assert.equal(recovered.servedToolCalls, 2, `${target} model emits exactly two tool calls around one declared miss`);
|
|
544
|
-
assert.deepEqual(recovered.runnerArguments, [
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
], `${target} model revises the declared city before the second adapter invocation`);
|
|
552
|
-
assert.ok(recovered.requests.some((request)=>(request.messages ?? []).some((message)=>{
|
|
553
|
-
if (message.role !== 'tool') return false;
|
|
554
|
-
const serialized = JSON.stringify(message);
|
|
555
|
-
return /status\\?":\\?"miss/.test(serialized) && /recovery\\?":\\?"revise_arguments/.test(serialized);
|
|
556
|
-
})), `${target} declared typed miss reaches model history`);
|
|
332
|
+
assert.deepEqual(recovered.runnerArguments, [{ city: 'Dubai' }, { city: 'Singapore' }], `${target} model revises the declared city before the second adapter invocation`);
|
|
333
|
+
assert.ok(recovered.requests.some(request => (request.messages ?? []).some(message => {
|
|
334
|
+
if (message.role !== 'tool')
|
|
335
|
+
return false;
|
|
336
|
+
const serialized = JSON.stringify(message);
|
|
337
|
+
return /status\\?":\\?"miss/.test(serialized) && /recovery\\?":\\?"revise_arguments/.test(serialized);
|
|
338
|
+
})), `${target} declared typed miss reaches model history`);
|
|
557
339
|
assert.match(recovered.stdout, /<benchmark_terminal>/, `${target} corrected second call reaches tagged terminal output`);
|
|
558
340
|
const failedRecovery = await runCase('domain-recovery-failed', executableOverride);
|
|
559
341
|
assert.equal(failedRecovery.exit, 1, `${target} infrastructure failure after a declared miss cannot be masked by the earlier domain outcome`);
|
|
560
342
|
assert.equal(failedRecovery.servedToolCalls, 2, `${target} failed recovery still exercises exactly two model-owned tool calls`);
|
|
561
|
-
assert.deepEqual(failedRecovery.runnerArguments, [
|
|
562
|
-
{
|
|
563
|
-
city: 'Dubai'
|
|
564
|
-
},
|
|
565
|
-
{
|
|
566
|
-
city: 'Singapore'
|
|
567
|
-
}
|
|
568
|
-
], `${target} failed recovery reaches the revised second adapter invocation`);
|
|
343
|
+
assert.deepEqual(failedRecovery.runnerArguments, [{ city: 'Dubai' }, { city: 'Singapore' }], `${target} failed recovery reaches the revised second adapter invocation`);
|
|
569
344
|
assert.equal(failedRecovery.stdout, '', `${target} failed recovery releases no terminal stdout`);
|
|
570
345
|
assert.match(failedRecovery.stderr, /benchmark terminal output unavailable \(child_bridge_runner_failed\)/, `${target} failed recovery preserves the second runner failure classification`);
|
|
571
346
|
assert.equal(failedRecovery.output.includes('PRIVATE_RECOVERY_RUNNER_DIAGNOSTIC_DO_NOT_REFLECT'), false, `${target} failed recovery never reflects private runner stderr`);
|
|
@@ -575,8 +350,8 @@ async function assertRuntimeContract(executableOverride) {
|
|
|
575
350
|
const unexpected = await runCase('unexpected-output', executableOverride);
|
|
576
351
|
assert.equal(unexpected.exit, 1, `${target} unexpected output must fail the successful-call conformance gate; exit=${unexpected.exit}; output tail=${unexpected.output.slice(-1000)}`);
|
|
577
352
|
assert.equal(unexpected.stdout, '', `${target} rejected runner output keeps stdout empty`);
|
|
578
|
-
assert.ok(unexpected.requests.some(
|
|
579
|
-
assert.equal(unexpected.requests.some(
|
|
353
|
+
assert.ok(unexpected.requests.some(r => JSON.stringify(r).includes('forbidden_output')), `${target} positive output allowlist must reject unexpected output`);
|
|
354
|
+
assert.equal(unexpected.requests.some(r => JSON.stringify(r).includes('must-not-reflect')), false, `${target} positive output rejection must not reflect unexpected key/value`);
|
|
580
355
|
assert.equal(unexpected.output.includes('<benchmark_terminal>'), false, `${target} failed bridge result cannot release a terminal body`);
|
|
581
356
|
const invalidPath = await runCase('invalid-path', executableOverride);
|
|
582
357
|
assert.equal(invalidPath.exit, 1, `${target} invalid config basename child exit=${invalidPath.exit}`);
|
|
@@ -598,23 +373,23 @@ async function assertRuntimeContract(executableOverride) {
|
|
|
598
373
|
assert.match(webMode.output, /benchmark environment requires headless mode/);
|
|
599
374
|
const truncated = await runCase('output-truncated', executableOverride);
|
|
600
375
|
assert.equal(truncated.exit, 1, `${target} output truncation fails the successful-call conformance gate`);
|
|
601
|
-
assert.ok(truncated.requests.some(
|
|
376
|
+
assert.ok(truncated.requests.some(r => JSON.stringify(r).includes('output_truncated')), `${target} real runner output over the configured cap is rejected`);
|
|
602
377
|
assert.equal(truncated.stdout, '', `${target} output truncation keeps stdout empty`);
|
|
603
378
|
assert.match(truncated.stderr, /benchmark terminal output unavailable \(child_bridge_output_truncated\)/, `${target} output truncation emits a stable bridge reason code`);
|
|
604
379
|
const timedOut = await runCase('timeout', executableOverride);
|
|
605
380
|
assert.equal(timedOut.exit, 1, `${target} timeout fails the successful-call conformance gate`);
|
|
606
|
-
assert.ok(timedOut.requests.some(
|
|
381
|
+
assert.ok(timedOut.requests.some(r => JSON.stringify(r).includes('timed_out')), `${target} real runner deadline is enforced`);
|
|
607
382
|
assert.equal(timedOut.stdout, '', `${target} timeout keeps stdout empty`);
|
|
608
383
|
assert.match(timedOut.stderr, /benchmark terminal output unavailable \(child_bridge_timed_out\)/, `${target} timeout emits a stable child bridge reason code`);
|
|
609
384
|
const runnerFailed = await runCase('runner-failed', executableOverride);
|
|
610
385
|
assert.equal(runnerFailed.exit, 1, `${target} runner failure fails the successful-call conformance gate`);
|
|
611
|
-
assert.ok(runnerFailed.requests.some(
|
|
386
|
+
assert.ok(runnerFailed.requests.some(r => JSON.stringify(r).includes('runner_failed')), `${target} nonzero runner exit is surfaced structurally`);
|
|
612
387
|
assert.equal(runnerFailed.stdout, '', `${target} runner failure keeps stdout empty`);
|
|
613
388
|
assert.match(runnerFailed.stderr, /benchmark terminal output unavailable \(child_bridge_runner_failed\)/, `${target} runner failure emits a stable child bridge reason code`);
|
|
614
389
|
assert.equal(runnerFailed.output.includes('PRIVATE_RUNNER_DIAGNOSTIC_DO_NOT_REFLECT'), false, `${target} runner stderr is never reflected`);
|
|
615
390
|
const spawnFailed = await runCase('spawn-failed', executableOverride);
|
|
616
391
|
assert.equal(spawnFailed.exit, 1, `${target} runner spawn failure fails the successful-call conformance gate`);
|
|
617
|
-
assert.ok(spawnFailed.requests.some(
|
|
392
|
+
assert.ok(spawnFailed.requests.some(r => JSON.stringify(r).includes('spawn_failed')), `${target} runner spawn failure is surfaced structurally`);
|
|
618
393
|
assert.equal(spawnFailed.stdout, '', `${target} runner spawn failure keeps stdout empty`);
|
|
619
394
|
assert.match(spawnFailed.stderr, /benchmark terminal output unavailable \(child_bridge_spawn_failed\)/, `${target} runner spawn failure emits a stable child bridge reason code`);
|
|
620
395
|
}
|
|
@@ -625,7 +400,8 @@ async function assertSourceRuntimeContractWhenAvailable() {
|
|
|
625
400
|
try {
|
|
626
401
|
createRequire(BIN).resolve('@deepseek-ai/dsh/lib/bin.js');
|
|
627
402
|
sourceRuntimeAvailable = true;
|
|
628
|
-
}
|
|
403
|
+
}
|
|
404
|
+
catch {
|
|
629
405
|
sourceRuntimeAvailable = false;
|
|
630
406
|
}
|
|
631
407
|
}
|
|
@@ -655,42 +431,34 @@ async function assertPackagedPatchProjection(executable) {
|
|
|
655
431
|
const reorderedPoisonModule = join(probePackageRoot, 'future-reordered-plugin.cjs');
|
|
656
432
|
const inlinePoisonProof = join(probeParent, 'future-inline-loaded.txt');
|
|
657
433
|
const reorderedPoisonProof = join(probeParent, 'future-reordered-loaded.txt');
|
|
658
|
-
cpSync(sourcePackageRoot, probePackageRoot, {
|
|
659
|
-
recursive: true
|
|
660
|
-
});
|
|
434
|
+
cpSync(sourcePackageRoot, probePackageRoot, { recursive: true });
|
|
661
435
|
chmodSync(probeExecutable, 0o755);
|
|
662
436
|
const basePatch = readFileSync(patchPath, 'utf8');
|
|
663
437
|
const stableError = /benchmark environment configuration unavailable/;
|
|
664
438
|
const poisonEnv = {
|
|
665
439
|
GOTRY_FUTURE_INLINE_PROOF: inlinePoisonProof,
|
|
666
|
-
GOTRY_FUTURE_REORDERED_PROOF: reorderedPoisonProof
|
|
440
|
+
GOTRY_FUTURE_REORDERED_PROOF: reorderedPoisonProof,
|
|
667
441
|
};
|
|
668
|
-
const runRejectedPatch = async (label, patch, forbiddenValues = [])=>{
|
|
669
|
-
rmSync(inlinePoisonProof, {
|
|
670
|
-
|
|
671
|
-
});
|
|
672
|
-
rmSync(reorderedPoisonProof, {
|
|
673
|
-
force: true
|
|
674
|
-
});
|
|
442
|
+
const runRejectedPatch = async (label, patch, forbiddenValues = []) => {
|
|
443
|
+
rmSync(inlinePoisonProof, { force: true });
|
|
444
|
+
rmSync(reorderedPoisonProof, { force: true });
|
|
675
445
|
writeFileSync(patchPath, patch);
|
|
676
446
|
const result = await runCase('enabled', probeExecutable, poisonEnv);
|
|
677
447
|
assert.equal(result.exit, 1, `${label} must reject the benchmark startup`);
|
|
678
448
|
assert.equal(result.requests.length, 0, `${label} must fail before relay activity`);
|
|
679
|
-
assert.deepEqual(result.optionalResolutionHits, {
|
|
680
|
-
calendar: 0,
|
|
681
|
-
map: 0
|
|
682
|
-
}, `${label} must fail before optional plugin resolution`);
|
|
449
|
+
assert.deepEqual(result.optionalResolutionHits, { calendar: 0, map: 0 }, `${label} must fail before optional plugin resolution`);
|
|
683
450
|
assert.match(result.output, stableError, `${label} emits a stable generic error`);
|
|
684
451
|
assert.equal(result.output.includes(probePackageRoot), false, `${label} must not reflect the package path`);
|
|
685
452
|
assert.equal(existsSync(inlinePoisonProof) || existsSync(reorderedPoisonProof), false, `${label} must not execute a poison plugin`);
|
|
686
|
-
for (const value of forbiddenValues)
|
|
453
|
+
for (const value of forbiddenValues)
|
|
454
|
+
assert.equal(result.output.includes(value), false, `${label} must not reflect rejected input`);
|
|
687
455
|
};
|
|
688
456
|
try {
|
|
689
457
|
writeFileSync(inlinePoisonModule, `const fs = require('node:fs'); fs.appendFileSync(process.env.GOTRY_FUTURE_INLINE_PROOF, 'loaded\\n'); exports.name = 'round4-future-inline'; exports.apply = () => {}`);
|
|
690
458
|
writeFileSync(reorderedPoisonModule, `const fs = require('node:fs'); fs.appendFileSync(process.env.GOTRY_FUTURE_REORDERED_PROOF, 'loaded\\n'); exports.name = 'round4-future-reordered'; exports.apply = () => {}`);
|
|
691
459
|
const futureEntries = [
|
|
692
460
|
` - { id: dsh-future-inline, name: '${inlinePoisonModule}' }`,
|
|
693
|
-
` - name: '${reorderedPoisonModule}'\n id: dsh-future-reordered
|
|
461
|
+
` - name: '${reorderedPoisonModule}'\n id: dsh-future-reordered`,
|
|
694
462
|
].join('\n');
|
|
695
463
|
const futurePatch = basePatch.replace(" - id: dsh-map-tools", `${futureEntries}\n - id: dsh-map-tools`);
|
|
696
464
|
assert.notEqual(futurePatch, basePatch, 'future-plugin fixture must enter the insert sequence');
|
|
@@ -698,65 +466,42 @@ async function assertPackagedPatchProjection(executable) {
|
|
|
698
466
|
const ordinary = await runCase('disabled', probeExecutable, poisonEnv);
|
|
699
467
|
assert.ok(existsSync(inlinePoisonProof), `default-off must execute the inline future-plugin top level; exit=${ordinary.exit}; output=${ordinary.output.slice(-2_000)}`);
|
|
700
468
|
assert.ok(existsSync(reorderedPoisonProof), `default-off must execute the reordered future-plugin top level; exit=${ordinary.exit}; output=${ordinary.output.slice(-2_000)}`);
|
|
701
|
-
rmSync(inlinePoisonProof, {
|
|
702
|
-
|
|
703
|
-
});
|
|
704
|
-
rmSync(reorderedPoisonProof, {
|
|
705
|
-
force: true
|
|
706
|
-
});
|
|
469
|
+
rmSync(inlinePoisonProof, { force: true });
|
|
470
|
+
rmSync(reorderedPoisonProof, { force: true });
|
|
707
471
|
const benchmark = await runCase('enabled', probeExecutable, poisonEnv);
|
|
708
472
|
assert.equal(benchmark.exit, 0, `benchmark future-plugin projection exits 0; output=${benchmark.output.slice(-2_000)}`);
|
|
709
|
-
assert.ok(benchmark.requests.some(
|
|
473
|
+
assert.ok(benchmark.requests.some(request => names(request).includes(TOOL)), 'benchmark future-plugin projection reaches the bridge relay');
|
|
710
474
|
assert.equal(existsSync(inlinePoisonProof) || existsSync(reorderedPoisonProof), false, 'benchmark projection must not execute inline or reordered future plugins');
|
|
711
|
-
assert.deepEqual(benchmark.optionalResolutionHits, {
|
|
712
|
-
calendar: 0,
|
|
713
|
-
map: 0
|
|
714
|
-
}, 'benchmark future-plugin projection does not resolve optional host plugins');
|
|
475
|
+
assert.deepEqual(benchmark.optionalResolutionHits, { calendar: 0, map: 0 }, 'benchmark future-plugin projection does not resolve optional host plugins');
|
|
715
476
|
await runRejectedPatch('missing gotry-tools', basePatch.replace(' - id: gotry-tools', ' - id: gotry-tools-missing'));
|
|
716
477
|
await runRejectedPatch('duplicate gotry-tools', basePatch.replace(' - id: dsh-map-tools', " - id: gotry-tools\n name: 'duplicate/gotry-tools'\n - id: dsh-map-tools"));
|
|
717
|
-
await runRejectedPatch('second insert block', `${basePatch}\n- insert:\n - id: dsh-second-insert\n name: '${inlinePoisonModule}'\n`, [
|
|
718
|
-
|
|
719
|
-
]);
|
|
720
|
-
await runRejectedPatch('
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
inlinePoisonModule
|
|
725
|
-
]);
|
|
726
|
-
await runRejectedPatch('spoofed gotry-tools name with decoy anchor', `${basePatch.replace("name: 'placeholder/ts/src/index.ts'", `name: '${inlinePoisonModule}'`)}\n- id: benchmark-name-decoy\n name: 'placeholder/ts/src/index.ts'\n`, [
|
|
727
|
-
inlinePoisonModule
|
|
728
|
-
]);
|
|
729
|
-
await runRejectedPatch('spoofed gotry-tools name with nested decoy anchor', basePatch.replace("name: 'placeholder/ts/src/index.ts'", `name: '${inlinePoisonModule}'`).replace(" stateRoot: '.'", " name: 'placeholder/ts/src/index.ts'\n stateRoot: '.'"), [
|
|
730
|
-
inlinePoisonModule
|
|
731
|
-
]);
|
|
478
|
+
await runRejectedPatch('second insert block', `${basePatch}\n- insert:\n - id: dsh-second-insert\n name: '${inlinePoisonModule}'\n`, [inlinePoisonModule]);
|
|
479
|
+
await runRejectedPatch('flow second insert block', `${basePatch}\n- insert: [{ id: dsh-flow-second-insert, name: '${inlinePoisonModule}' }]\n`, [inlinePoisonModule]);
|
|
480
|
+
await runRejectedPatch('spoofed gotry-tools name', basePatch.replace("name: 'placeholder/ts/src/index.ts'", `name: '${inlinePoisonModule}'`), [inlinePoisonModule]);
|
|
481
|
+
await runRejectedPatch('spoofed gotry-tools name with decoy anchor', `${basePatch.replace("name: 'placeholder/ts/src/index.ts'", `name: '${inlinePoisonModule}'`)}\n- id: benchmark-name-decoy\n name: 'placeholder/ts/src/index.ts'\n`, [inlinePoisonModule]);
|
|
482
|
+
await runRejectedPatch('spoofed gotry-tools name with nested decoy anchor', basePatch
|
|
483
|
+
.replace("name: 'placeholder/ts/src/index.ts'", `name: '${inlinePoisonModule}'`)
|
|
484
|
+
.replace(" stateRoot: '.'", " name: 'placeholder/ts/src/index.ts'\n stateRoot: '.'"), [inlinePoisonModule]);
|
|
732
485
|
await runRejectedPatch('missing benchmark config anchor', basePatch.replace(/^\s*hbcliBin:.*\n/m, ''));
|
|
733
486
|
await runRejectedPatch('missing benchmark config anchor with decoy', `${basePatch.replace(/^\s*hbcliBin:.*\n/m, '')}\n- id: benchmark-anchor-decoy\n hbcliBin: 'hbcli'\n`);
|
|
734
|
-
await runRejectedPatch('missing benchmark config anchor with nested decoy', basePatch
|
|
487
|
+
await runRejectedPatch('missing benchmark config anchor with nested decoy', basePatch
|
|
488
|
+
.replace(/^\s*hbcliBin:.*\n/m, '')
|
|
489
|
+
.replace(" stateRoot: '.'", " nestedAnchorDecoy:\n hbcliBin: 'hbcli'\n stateRoot: '.'"));
|
|
735
490
|
await runRejectedPatch('duplicate benchmark config anchor', basePatch.replace(" hbcliBin: 'hbcli'", " hbcliBin: 'hbcli'\n hbcliBin: 'hbcli'"));
|
|
736
|
-
await runRejectedPatch('pre-existing benchmark config path', basePatch.replace(" hbcliBin: 'hbcli'", " hbcliBin: 'hbcli'\n benchmarkEnvironmentConfigPath: '/not/used'"), [
|
|
737
|
-
'/not/used'
|
|
738
|
-
]);
|
|
491
|
+
await runRejectedPatch('pre-existing benchmark config path', basePatch.replace(" hbcliBin: 'hbcli'", " hbcliBin: 'hbcli'\n benchmarkEnvironmentConfigPath: '/not/used'"), ['/not/used']);
|
|
739
492
|
await runRejectedPatch('missing system-prompt anchor', basePatch.replace(/^- id: system-prompt[\s\S]*$/m, ''));
|
|
740
|
-
await runRejectedPatch('duplicate system-prompt anchor', `${basePatch}\n- id: system-prompt\n config:\n
|
|
741
|
-
await runRejectedPatch('quoted system-prompt duplicate', `${basePatch}\n- id: 'system-prompt'\n config:\n
|
|
493
|
+
await runRejectedPatch('duplicate system-prompt anchor', `${basePatch}\n- id: system-prompt\n config:\n personaPrefix: >-\n duplicate\n`);
|
|
494
|
+
await runRejectedPatch('quoted system-prompt duplicate', `${basePatch}\n- id: 'system-prompt'\n config:\n personaPrefix: >-\n quoted duplicate\n`);
|
|
742
495
|
const systemPromptMutationSentinel = 'ROUND7_SYSTEM_PROMPT_MUTATION_SENTINEL_DO_NOT_REFLECT';
|
|
743
|
-
await runRejectedPatch('quoted mapping-key system-prompt duplicate', `${basePatch}\n- "id": system-prompt\n config:\n
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
await runRejectedPatch('flow
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
systemPromptMutationSentinel
|
|
753
|
-
]);
|
|
754
|
-
await runRejectedPatch('malformed system-prompt persona', basePatch.replace(/^ persona: >-$/m, ' persona: plain'));
|
|
755
|
-
} finally{
|
|
756
|
-
rmSync(probeParent, {
|
|
757
|
-
recursive: true,
|
|
758
|
-
force: true
|
|
759
|
-
});
|
|
496
|
+
await runRejectedPatch('quoted mapping-key system-prompt duplicate', `${basePatch}\n- "id": system-prompt\n config:\n personaPrefix: >-\n ${systemPromptMutationSentinel}\n`, [systemPromptMutationSentinel]);
|
|
497
|
+
await runRejectedPatch('flow quoted-key system-prompt duplicate', `${basePatch}\n- { "id": system-prompt, config: { personaPrefix: ${systemPromptMutationSentinel} } }\n`, [systemPromptMutationSentinel]);
|
|
498
|
+
await runRejectedPatch('reordered system-prompt duplicate', `${basePatch}\n- name: reordered-system-prompt\n id: system-prompt\n config:\n personaPrefix: >-\n reordered duplicate\n`);
|
|
499
|
+
await runRejectedPatch('flow system-prompt duplicate', `${basePatch}\n- { id: system-prompt, config: { personaPrefix: flow duplicate } }\n`);
|
|
500
|
+
await runRejectedPatch('noncanonical insert id root item', `${basePatch}\n- id: insert\n config:\n personaPrefix: >-\n ${systemPromptMutationSentinel}\n`, [systemPromptMutationSentinel]);
|
|
501
|
+
await runRejectedPatch('malformed system-prompt persona', basePatch.replace(/^ personaPrefix: >-$/m, ' personaPrefix: plain'));
|
|
502
|
+
}
|
|
503
|
+
finally {
|
|
504
|
+
rmSync(probeParent, { recursive: true, force: true });
|
|
760
505
|
}
|
|
761
506
|
}
|
|
762
507
|
const LARGE_TERMINAL_PAYLOAD = 'x'.repeat(80 * 1024);
|
|
@@ -764,23 +509,21 @@ function taggedTerminal(valid) {
|
|
|
764
509
|
return `<benchmark_terminal>${valid ? '{"status":"succeeded"}' : '{"status":'}</benchmark_terminal>`;
|
|
765
510
|
}
|
|
766
511
|
function conformanceResponse(mode, request, plannerCount) {
|
|
767
|
-
const hasToolResult = (request.messages ?? []).some(
|
|
768
|
-
if (mode === 'a' && plannerCount === 1 && !hasToolResult)
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
'd',
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
if (call && !hasToolResult) return toolCall();
|
|
777
|
-
if (mode === 'f' && plannerCount === 3) return toolCall('bridge-call-retry');
|
|
512
|
+
const hasToolResult = (request.messages ?? []).some(message => message.role === 'tool');
|
|
513
|
+
if (mode === 'a' && plannerCount === 1 && !hasToolResult)
|
|
514
|
+
return finalText('assistant prose without a call');
|
|
515
|
+
const call = mode === 'a' && plannerCount === 2
|
|
516
|
+
|| ['b', 'd', 'f', 'large', 'schema'].includes(mode) && plannerCount === 1;
|
|
517
|
+
if (call && !hasToolResult)
|
|
518
|
+
return toolCall();
|
|
519
|
+
if (mode === 'f' && plannerCount === 3)
|
|
520
|
+
return toolCall('bridge-call-retry');
|
|
778
521
|
if (mode === 'large' && hasToolResult) {
|
|
779
|
-
return finalText(`<benchmark_terminal>${JSON.stringify({
|
|
780
|
-
payload: LARGE_TERMINAL_PAYLOAD
|
|
781
|
-
})}</benchmark_terminal>`);
|
|
522
|
+
return finalText(`<benchmark_terminal>${JSON.stringify({ payload: LARGE_TERMINAL_PAYLOAD })}</benchmark_terminal>`);
|
|
782
523
|
}
|
|
783
524
|
if (mode === 'schema') {
|
|
525
|
+
// Round 11 真实失败形态:结构合法信封内多出 root 键(extra budget)——
|
|
526
|
+
// 信封解析通过,exact body schema 必须拒绝,且纠正后仍不 autofix。
|
|
784
527
|
return finalText('<benchmark_terminal>{"status":"succeeded","budget":{"total_cost":1}}</benchmark_terminal>');
|
|
785
528
|
}
|
|
786
529
|
const valid = mode === 'a' ? hasToolResult : mode === 'b' ? plannerCount >= 3 : mode === 'e' ? true : false;
|
|
@@ -791,51 +534,40 @@ async function runConformanceCase(mode, executableOverride) {
|
|
|
791
534
|
let plannerCount = 0;
|
|
792
535
|
let recoveredAttempts = 0;
|
|
793
536
|
let servedToolCalls = 0;
|
|
794
|
-
const server = createServer((req, res)=>{
|
|
537
|
+
const server = createServer((req, res) => {
|
|
795
538
|
const chunks = [];
|
|
796
|
-
req.on('data',
|
|
797
|
-
req.on('end', ()=>{
|
|
539
|
+
req.on('data', chunk => chunks.push(Buffer.from(chunk)));
|
|
540
|
+
req.on('end', () => {
|
|
798
541
|
let body = {};
|
|
799
542
|
try {
|
|
800
543
|
body = JSON.parse(Buffer.concat(chunks).toString());
|
|
801
|
-
}
|
|
544
|
+
}
|
|
545
|
+
catch { /* structural failure */ }
|
|
802
546
|
requests.push(body);
|
|
803
547
|
const requestHasToolResult = anyToolResultPresent(body);
|
|
804
548
|
const plannerRequest = names(body).includes(TOOL);
|
|
805
|
-
if (mode === 'exhausted' || mode === 'unknown' || mode === 'post-failure' && requestHasToolResult) {
|
|
806
|
-
res.writeHead(mode === 'exhausted' ? 429 : mode === 'unknown' ? 418 : 500, {
|
|
807
|
-
|
|
808
|
-
});
|
|
809
|
-
res.end(JSON.stringify({
|
|
810
|
-
error: {
|
|
811
|
-
message: 'PRIVATE_SENTINEL_DO_NOT_REFLECT',
|
|
812
|
-
type: 'server_error'
|
|
813
|
-
}
|
|
814
|
-
}));
|
|
549
|
+
if (mode === 'exhausted' || mode === 'unknown' || (mode === 'post-failure' && requestHasToolResult)) {
|
|
550
|
+
res.writeHead(mode === 'exhausted' ? 429 : mode === 'unknown' ? 418 : 500, { 'content-type': 'application/json' });
|
|
551
|
+
res.end(JSON.stringify({ error: { message: 'PRIVATE_SENTINEL_DO_NOT_REFLECT', type: 'server_error' } }));
|
|
815
552
|
return;
|
|
816
553
|
}
|
|
817
554
|
if (mode === 'recovered' && plannerRequest && !requestHasToolResult && recoveredAttempts++ === 0) {
|
|
818
|
-
res.writeHead(503, {
|
|
819
|
-
|
|
820
|
-
});
|
|
821
|
-
res.end(JSON.stringify({
|
|
822
|
-
error: {
|
|
823
|
-
message: 'PRIVATE_SENTINEL_DO_NOT_REFLECT',
|
|
824
|
-
type: 'server_error'
|
|
825
|
-
}
|
|
826
|
-
}));
|
|
555
|
+
res.writeHead(503, { 'content-type': 'application/json' });
|
|
556
|
+
res.end(JSON.stringify({ error: { message: 'PRIVATE_SENTINEL_DO_NOT_REFLECT', type: 'server_error' } }));
|
|
827
557
|
return;
|
|
828
558
|
}
|
|
829
|
-
if (plannerRequest)
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
559
|
+
if (plannerRequest)
|
|
560
|
+
plannerCount += 1;
|
|
561
|
+
res.writeHead(200, { 'content-type': 'text/event-stream' });
|
|
562
|
+
const response = plannerRequest
|
|
563
|
+
? conformanceResponse(mode === 'recovered' ? 'b' : mode === 'post-failure' ? 'f' : mode, body, plannerCount)
|
|
564
|
+
: finalText('auxiliary request');
|
|
565
|
+
if (response.includes(`"name":"${TOOL}"`))
|
|
566
|
+
servedToolCalls += 1;
|
|
835
567
|
res.end(response);
|
|
836
568
|
});
|
|
837
569
|
});
|
|
838
|
-
await new Promise(
|
|
570
|
+
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
|
|
839
571
|
const port = server.address().port;
|
|
840
572
|
const cwd = mkdtempSync(join(tmpdir(), 'gotry-conformance-cwd-'));
|
|
841
573
|
const dsh = mkdtempSync(join(tmpdir(), 'gotry-conformance-dsh-'));
|
|
@@ -844,139 +576,56 @@ async function runConformanceCase(mode, executableOverride) {
|
|
|
844
576
|
const configPath = join(cwd, 'benchmark-env-config.json');
|
|
845
577
|
writeFileSync(runner, `const fs = require('node:fs'); const path = ${JSON.stringify(runnerCount)}; const count = fs.existsSync(path) ? Number(fs.readFileSync(path, 'utf8')) : 0; fs.writeFileSync(path, String(count + 1)); process.stdout.write(JSON.stringify({ schema_version: 'gotry_benchmark_tool_result_v1', status: 'ok', result: { marker: '${MARKER}' } }))`);
|
|
846
578
|
writeFileSync(configPath, JSON.stringify({
|
|
847
|
-
schema_version: 'gotry_benchmark_environment_bridge_v4',
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
runner
|
|
853
|
-
],
|
|
854
|
-
tools: [
|
|
855
|
-
{
|
|
856
|
-
name: 'lookup',
|
|
857
|
-
description: 'Lookup.',
|
|
858
|
-
input_schema: LOOKUP_INPUT_SCHEMA,
|
|
859
|
-
output_keys: [
|
|
860
|
-
'marker'
|
|
861
|
-
],
|
|
862
|
-
domain_outcomes: [
|
|
863
|
-
{
|
|
864
|
-
status: 'miss',
|
|
865
|
-
code: 'NOT_FOUND',
|
|
866
|
-
recovery: 'none'
|
|
867
|
-
}
|
|
868
|
-
]
|
|
869
|
-
}
|
|
870
|
-
],
|
|
871
|
-
timeout_ms: 2_000,
|
|
872
|
-
max_output_bytes: 4_096,
|
|
873
|
-
terminal_output: {
|
|
874
|
-
tag: 'benchmark_terminal',
|
|
875
|
-
max_bytes: mode === 'large' ? 128 * 1024 : 4_096,
|
|
876
|
-
body_schema: BRIDGE_E2E_BODY_SCHEMA
|
|
877
|
-
},
|
|
878
|
-
isolation: {
|
|
879
|
-
mode: 'host-enforced',
|
|
880
|
-
writes: 'forbidden',
|
|
881
|
-
network: 'denied'
|
|
882
|
-
}
|
|
579
|
+
schema_version: 'gotry_benchmark_environment_bridge_v4', enabled: true,
|
|
580
|
+
executable: process.execPath, cwd, argv_prefix: [runner],
|
|
581
|
+
tools: [{ name: 'lookup', description: 'Lookup.', input_schema: LOOKUP_INPUT_SCHEMA, output_keys: ['marker'], domain_outcomes: [{ status: 'miss', code: 'NOT_FOUND', recovery: 'none' }] }], timeout_ms: 2_000, max_output_bytes: 4_096,
|
|
582
|
+
terminal_output: { tag: 'benchmark_terminal', max_bytes: mode === 'large' ? 128 * 1024 : 4_096, body_schema: BRIDGE_E2E_BODY_SCHEMA },
|
|
583
|
+
isolation: { mode: 'host-enforced', writes: 'forbidden', network: 'denied' },
|
|
883
584
|
}));
|
|
884
585
|
const env = {
|
|
885
586
|
...process.env,
|
|
886
|
-
DSH_TOOLS_MODE: 'both',
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
LLM_MODEL: 'synthetic-conformance-model',
|
|
891
|
-
GOTRY_BENCHMARK_ENV_CONFIG: configPath,
|
|
892
|
-
DEEPSEEK_API_KEY: 'synthetic-conformance-key',
|
|
893
|
-
DEEPSEEK_BASE_URL: `http://127.0.0.1:${port}/v1`,
|
|
587
|
+
DSH_TOOLS_MODE: 'both', DSH_HOME: dsh,
|
|
588
|
+
LLM_API_KEY: 'synthetic-conformance-key', LLM_BASE_URL: `http://127.0.0.1:${port}/v1`,
|
|
589
|
+
LLM_MODEL: 'synthetic-conformance-model', GOTRY_BENCHMARK_ENV_CONFIG: configPath,
|
|
590
|
+
DEEPSEEK_API_KEY: 'synthetic-conformance-key', DEEPSEEK_BASE_URL: `http://127.0.0.1:${port}/v1`,
|
|
894
591
|
NODE_OPTIONS: [
|
|
895
592
|
process.env.NODE_OPTIONS,
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
] : []
|
|
899
|
-
].filter(Boolean).join(' ')
|
|
593
|
+
...(!executableOverride ? [`--import=${TSX_LOADER}`] : []),
|
|
594
|
+
].filter(Boolean).join(' '),
|
|
900
595
|
};
|
|
901
|
-
for (const key of [
|
|
902
|
-
|
|
903
|
-
'HTTP_PROXY',
|
|
904
|
-
'HTTPS_PROXY',
|
|
905
|
-
'ALL_PROXY',
|
|
906
|
-
'http_proxy',
|
|
907
|
-
'https_proxy',
|
|
908
|
-
'all_proxy'
|
|
909
|
-
])delete env[key];
|
|
596
|
+
for (const key of ['GOTRY_LLM_MODEL', 'HTTP_PROXY', 'HTTPS_PROXY', 'ALL_PROXY', 'http_proxy', 'https_proxy', 'all_proxy'])
|
|
597
|
+
delete env[key];
|
|
910
598
|
env.NO_PROXY = '127.0.0.1,localhost';
|
|
911
599
|
let stdout = '';
|
|
912
600
|
let stderr = '';
|
|
913
601
|
try {
|
|
914
602
|
const executable = executableOverride || process.execPath;
|
|
915
|
-
const argv = executableOverride ? [
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
cwd,
|
|
923
|
-
env,
|
|
924
|
-
stdio: [
|
|
925
|
-
'ignore',
|
|
926
|
-
'pipe',
|
|
927
|
-
'pipe'
|
|
928
|
-
]
|
|
929
|
-
});
|
|
930
|
-
child.stdout.on('data', (chunk)=>{
|
|
931
|
-
stdout += chunk.toString();
|
|
932
|
-
});
|
|
933
|
-
child.stderr.on('data', (chunk)=>{
|
|
934
|
-
stderr += chunk.toString();
|
|
935
|
-
});
|
|
936
|
-
const exit = await new Promise((resolve)=>{
|
|
937
|
-
const timer = setTimeout(()=>{
|
|
938
|
-
child.kill('SIGKILL');
|
|
939
|
-
resolve(null);
|
|
940
|
-
}, TIMEOUT_MS);
|
|
941
|
-
child.once('close', (code)=>{
|
|
942
|
-
clearTimeout(timer);
|
|
943
|
-
resolve(code);
|
|
944
|
-
});
|
|
603
|
+
const argv = executableOverride ? ['conformance smoke'] : [BIN, 'conformance smoke'];
|
|
604
|
+
const child = spawn(executable, argv, { cwd, env, stdio: ['ignore', 'pipe', 'pipe'] });
|
|
605
|
+
child.stdout.on('data', chunk => { stdout += chunk.toString(); });
|
|
606
|
+
child.stderr.on('data', chunk => { stderr += chunk.toString(); });
|
|
607
|
+
const exit = await new Promise(resolve => {
|
|
608
|
+
const timer = setTimeout(() => { child.kill('SIGKILL'); resolve(null); }, TIMEOUT_MS);
|
|
609
|
+
child.once('close', code => { clearTimeout(timer); resolve(code); });
|
|
945
610
|
});
|
|
946
611
|
const runnerInvocations = existsSync(runnerCount) ? Number(readFileSync(runnerCount, 'utf8')) : 0;
|
|
947
|
-
return {
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
runnerInvocations
|
|
954
|
-
};
|
|
955
|
-
} finally{
|
|
956
|
-
await new Promise((resolve)=>server.close(()=>resolve()));
|
|
957
|
-
rmSync(dsh, {
|
|
958
|
-
recursive: true,
|
|
959
|
-
force: true
|
|
960
|
-
});
|
|
961
|
-
rmSync(cwd, {
|
|
962
|
-
recursive: true,
|
|
963
|
-
force: true
|
|
964
|
-
});
|
|
612
|
+
return { exit, stdout, stderr, requests, servedToolCalls, runnerInvocations };
|
|
613
|
+
}
|
|
614
|
+
finally {
|
|
615
|
+
await new Promise(resolve => server.close(() => resolve()));
|
|
616
|
+
rmSync(dsh, { recursive: true, force: true });
|
|
617
|
+
rmSync(cwd, { recursive: true, force: true });
|
|
965
618
|
}
|
|
966
619
|
}
|
|
967
620
|
async function assertTerminalDiagnostics(executableOverride) {
|
|
968
|
-
const terminalReasons = (stderr)=>[
|
|
969
|
-
...stderr.matchAll(/benchmark terminal output unavailable \(([^)]+)\)/g)
|
|
970
|
-
].map((match)=>match[1]);
|
|
621
|
+
const terminalReasons = (stderr) => [...stderr.matchAll(/benchmark terminal output unavailable \(([^)]+)\)/g)].map(match => match[1]);
|
|
971
622
|
const exhausted = await runConformanceCase('exhausted', executableOverride);
|
|
972
623
|
assert.notEqual(exhausted.exit, 0, 'exhausted transient model failure exits non-zero');
|
|
973
624
|
assert.match(exhausted.stderr, /child_model_capacity/, 'exhausted transient model failure emits coarse capacity enum');
|
|
974
625
|
assert.ok(exhausted.requests.length > 1, 'exhausted case actually exercises retry attempts');
|
|
975
626
|
assert.equal(exhausted.stdout, '', 'exhausted transient model failure releases no terminal stdout');
|
|
976
627
|
assert.equal(exhausted.stderr.includes('PRIVATE_SENTINEL_DO_NOT_REFLECT'), false, 'exhausted error body is never reflected');
|
|
977
|
-
assert.deepEqual(terminalReasons(exhausted.stderr), [
|
|
978
|
-
'child_model_capacity'
|
|
979
|
-
], 'exhausted emits exactly one terminal reason');
|
|
628
|
+
assert.deepEqual(terminalReasons(exhausted.stderr), ['child_model_capacity'], 'exhausted emits exactly one terminal reason');
|
|
980
629
|
const recovered = await runConformanceCase('recovered', executableOverride);
|
|
981
630
|
assert.equal(recovered.exit, 0, 'transient model failure followed by valid terminal recovers');
|
|
982
631
|
assert.match(recovered.stdout, /<benchmark_terminal>/, 'recovered run releases terminal stdout');
|
|
@@ -987,23 +636,17 @@ async function assertTerminalDiagnostics(executableOverride) {
|
|
|
987
636
|
assert.equal(postFailure.runnerInvocations, 1, 'post-bridge failure follows exactly one successful bridge invocation');
|
|
988
637
|
assert.match(postFailure.stderr, /child_model_server/, 'post-bridge model failure emits server enum');
|
|
989
638
|
assert.equal(postFailure.stdout, '', 'post-bridge model failure releases no terminal stdout');
|
|
990
|
-
assert.deepEqual(terminalReasons(postFailure.stderr), [
|
|
991
|
-
'child_model_server'
|
|
992
|
-
], 'post-bridge emits exactly one terminal reason');
|
|
639
|
+
assert.deepEqual(terminalReasons(postFailure.stderr), ['child_model_server'], 'post-bridge emits exactly one terminal reason');
|
|
993
640
|
const unknown = await runConformanceCase('unknown', executableOverride);
|
|
994
641
|
assert.notEqual(unknown.exit, 0, 'unknown model failure exits non-zero');
|
|
995
642
|
assert.match(unknown.stderr, /child_runtime_error/, 'unknown model failure collapses to generic runtime enum');
|
|
996
643
|
assert.equal(unknown.stdout, '', 'unknown model failure releases no terminal stdout');
|
|
997
644
|
assert.equal(unknown.stderr.includes('PRIVATE_SENTINEL_DO_NOT_REFLECT'), false, 'unknown error body is never reflected');
|
|
998
|
-
assert.deepEqual(terminalReasons(unknown.stderr), [
|
|
999
|
-
'child_runtime_error'
|
|
1000
|
-
], 'unknown emits exactly one terminal reason');
|
|
645
|
+
assert.deepEqual(terminalReasons(unknown.stderr), ['child_runtime_error'], 'unknown emits exactly one terminal reason');
|
|
1001
646
|
const precedence = await runConformanceCase('f', executableOverride);
|
|
1002
647
|
assert.match(precedence.stderr, /child_conformance_failure/, 'conformance-specific failure remains higher precedence than final generic error');
|
|
1003
648
|
assert.equal(precedence.stderr.includes('child_runtime_error'), false, 'generic terminal classification does not double-write');
|
|
1004
|
-
assert.deepEqual(terminalReasons(precedence.stderr), [
|
|
1005
|
-
'child_conformance_failure'
|
|
1006
|
-
], 'precedence emits exactly one terminal reason');
|
|
649
|
+
assert.deepEqual(terminalReasons(precedence.stderr), ['child_conformance_failure'], 'precedence emits exactly one terminal reason');
|
|
1007
650
|
}
|
|
1008
651
|
async function assertOutputConformance(executableOverride) {
|
|
1009
652
|
const a = await runConformanceCase('a', executableOverride);
|
|
@@ -1011,24 +654,19 @@ async function assertOutputConformance(executableOverride) {
|
|
|
1011
654
|
assert.equal(a.servedToolCalls, 1, 'A exposes exactly one bridge call');
|
|
1012
655
|
assert.equal(a.runnerInvocations, 1, 'A executes the bridge subprocess exactly once');
|
|
1013
656
|
assert.ok(a.stdout.includes('<benchmark_terminal>'), 'A forwards only tagged terminal output');
|
|
1014
|
-
|
|
657
|
+
// Round 12(#215):system prompt 投影 exact body schema outline(结构合同单一来源)。
|
|
658
|
+
assert.ok(a.requests.some(request => JSON.stringify(request).includes(`matching exactly ${TERMINAL_OUTLINE}`)), 'A system prompt projects the exact terminal schema outline');
|
|
1015
659
|
const b = await runConformanceCase('b', executableOverride);
|
|
1016
660
|
assert.equal(b.exit, 0, 'B malformed terminal correction then valid terminal exits 0');
|
|
1017
|
-
assert.equal(b.servedToolCalls, 1, `B exposes exactly one bridge call; request shapes=${JSON.stringify(b.requests.map((request)=>
|
|
1018
|
-
tools: names(request),
|
|
1019
|
-
roles: (request.messages ?? []).map((message)=>message.role)
|
|
1020
|
-
})))}`);
|
|
661
|
+
assert.equal(b.servedToolCalls, 1, `B exposes exactly one bridge call; request shapes=${JSON.stringify(b.requests.map(request => ({ tools: names(request), roles: (request.messages ?? []).map(message => message.role) })))}`);
|
|
1021
662
|
assert.equal(b.runnerInvocations, 1, 'B format-only correction does not rerun the bridge subprocess');
|
|
1022
|
-
assert.ok(b.requests.some(
|
|
663
|
+
assert.ok(b.requests.some(request => JSON.stringify(request).includes('BENCHMARK_CONFORMANCE_TERMINAL') && JSON.stringify(request).includes(`matching exactly ${TERMINAL_OUTLINE}`)), 'B terminal correction projects the same schema outline as the system prompt');
|
|
1023
664
|
const schema = await runConformanceCase('schema', executableOverride);
|
|
1024
665
|
assert.notEqual(schema.exit, 0, 'schema-invalid terminal body (extra root key) is rejected after the single correction');
|
|
1025
666
|
assert.match(schema.stderr, /benchmark terminal output unavailable \(child_conformance_failure\)/, 'schema-invalid terminal emits the stable conformance reason code');
|
|
1026
667
|
assert.equal(schema.runnerInvocations, 1, 'schema-invalid terminal still executed the bridge exactly once');
|
|
1027
668
|
assert.equal(schema.stdout.includes('<benchmark_terminal>'), false, 'schema-invalid terminal body is never released to stdout');
|
|
1028
|
-
for (const mode of [
|
|
1029
|
-
'c',
|
|
1030
|
-
'd'
|
|
1031
|
-
]){
|
|
669
|
+
for (const mode of ['c', 'd']) {
|
|
1032
670
|
const result = await runConformanceCase(mode, executableOverride);
|
|
1033
671
|
assert.notEqual(result.exit, 0, `${mode.toUpperCase()} repeated invalid output is non-zero`);
|
|
1034
672
|
assert.match(result.stderr, /benchmark terminal output unavailable \(child_conformance_failure\)/, `${mode.toUpperCase()} emits a stable conformance reason code`);
|
|
@@ -1056,8 +694,10 @@ async function assertOutputConformance(executableOverride) {
|
|
|
1056
694
|
const packaged = process.env.GOTRY_BRIDGE_E2E_BIN;
|
|
1057
695
|
assertRuntimeSelectionAndVersionGuards();
|
|
1058
696
|
const sourceRuntimeChecked = await assertSourceRuntimeContractWhenAvailable();
|
|
1059
|
-
if (sourceRuntimeChecked)
|
|
1060
|
-
|
|
697
|
+
if (sourceRuntimeChecked)
|
|
698
|
+
await assertOutputConformance();
|
|
699
|
+
if (sourceRuntimeChecked)
|
|
700
|
+
await assertTerminalDiagnostics();
|
|
1061
701
|
if (packaged) {
|
|
1062
702
|
await assertRuntimeContract(packaged);
|
|
1063
703
|
await assertOutputConformance(packaged);
|
|
@@ -1069,53 +709,16 @@ if (packaged) {
|
|
|
1069
709
|
try {
|
|
1070
710
|
const configPath = join(missingServiceRoot, 'bridge.json');
|
|
1071
711
|
writeFileSync(configPath, JSON.stringify({
|
|
1072
|
-
schema_version: 'gotry_benchmark_environment_bridge_v4',
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
'-e',
|
|
1078
|
-
'process.exit(0)'
|
|
1079
|
-
],
|
|
1080
|
-
tools: [
|
|
1081
|
-
{
|
|
1082
|
-
name: 'lookup',
|
|
1083
|
-
description: 'Lookup.',
|
|
1084
|
-
input_schema: LOOKUP_INPUT_SCHEMA,
|
|
1085
|
-
output_keys: [
|
|
1086
|
-
'marker'
|
|
1087
|
-
],
|
|
1088
|
-
domain_outcomes: [
|
|
1089
|
-
{
|
|
1090
|
-
status: 'miss',
|
|
1091
|
-
code: 'NOT_FOUND',
|
|
1092
|
-
recovery: 'none'
|
|
1093
|
-
}
|
|
1094
|
-
]
|
|
1095
|
-
}
|
|
1096
|
-
],
|
|
1097
|
-
timeout_ms: 100,
|
|
1098
|
-
max_output_bytes: 4_096,
|
|
1099
|
-
terminal_output: {
|
|
1100
|
-
tag: 'benchmark_terminal',
|
|
1101
|
-
max_bytes: 4_096,
|
|
1102
|
-
body_schema: BRIDGE_E2E_BODY_SCHEMA
|
|
1103
|
-
},
|
|
1104
|
-
isolation: {
|
|
1105
|
-
mode: 'host-enforced',
|
|
1106
|
-
writes: 'forbidden',
|
|
1107
|
-
network: 'denied'
|
|
1108
|
-
}
|
|
712
|
+
schema_version: 'gotry_benchmark_environment_bridge_v4', enabled: true,
|
|
713
|
+
executable: process.execPath, cwd: missingServiceRoot, argv_prefix: ['-e', 'process.exit(0)'],
|
|
714
|
+
tools: [{ name: 'lookup', description: 'Lookup.', input_schema: LOOKUP_INPUT_SCHEMA, output_keys: ['marker'], domain_outcomes: [{ status: 'miss', code: 'NOT_FOUND', recovery: 'none' }] }], timeout_ms: 100, max_output_bytes: 4_096,
|
|
715
|
+
terminal_output: { tag: 'benchmark_terminal', max_bytes: 4_096, body_schema: BRIDGE_E2E_BODY_SCHEMA },
|
|
716
|
+
isolation: { mode: 'host-enforced', writes: 'forbidden', network: 'denied' },
|
|
1109
717
|
}));
|
|
1110
|
-
assert.throws(()=>packagedBridge.registerBenchmarkEnvironmentBridge(configPath, ()=>{}, undefined), /benchmark environment bridge subprocess unavailable/, 'packaged explicit opt-in fails hard when no active subprocess provider exists');
|
|
1111
|
-
}
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
force: true
|
|
1115
|
-
});
|
|
718
|
+
assert.throws(() => packagedBridge.registerBenchmarkEnvironmentBridge(configPath, () => { }, undefined), /benchmark environment bridge subprocess unavailable/, 'packaged explicit opt-in fails hard when no active subprocess provider exists');
|
|
719
|
+
}
|
|
720
|
+
finally {
|
|
721
|
+
rmSync(missingServiceRoot, { recursive: true, force: true });
|
|
1116
722
|
}
|
|
1117
723
|
}
|
|
1118
|
-
console.log(`benchmark environment bridge E2E: OK (${packaged ? sourceRuntimeChecked ? 'source + packaged' : 'source-static + packaged' : sourceRuntimeChecked ? 'source' : 'source-static'})`);
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
//# sourceURL=ts/scripts/benchmark-environment-bridge-e2e.ts
|
|
724
|
+
console.log(`benchmark environment bridge E2E: OK (${packaged ? (sourceRuntimeChecked ? 'source + packaged' : 'source-static + packaged') : (sourceRuntimeChecked ? 'source' : 'source-static')})`);
|