@danceiny/gotry 0.0.1-rc.18 → 0.0.1-rc.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -16
- package/README.zh-CN.md +37 -15
- package/bin/gotry-bootstrap.js +309 -30
- package/bin/gotry-inner.js +74 -23
- package/bin/gotry-runtime-resolution.d.ts +1 -1
- package/bin/gotry-runtime-resolution.js +7 -8
- package/cordis.gotry-patch.yml +22 -8
- package/data/stations-12306-verify.json +524 -0
- package/dist/capabilities/agent-reach.js +4 -4
- package/dist/capabilities/anything.js +62 -29
- package/dist/capabilities/channel-health.js +103 -0
- package/dist/capabilities/channel-registry.js +213 -0
- package/dist/capabilities/doctor.js +307 -0
- package/dist/capabilities/effect.js +178 -5
- package/dist/capabilities/flyai.js +13 -0
- package/dist/capabilities/hbcli.js +28 -2
- package/dist/capabilities/session/adapters/ctrip-hotel.js +207 -0
- package/dist/capabilities/session/adapters/rail-12306.js +285 -0
- package/dist/capabilities/session/extension-bridge.js +14 -1
- package/dist/capabilities/session/extension-channel.js +4 -2
- package/dist/capabilities/session/extension-distribution.js +1 -1
- package/dist/capabilities/session-search.js +311 -0
- package/dist/capabilities/visa-policy.js +126 -0
- package/dist/scripts/agent-planning-turn-deadline-e2e.js +305 -0
- package/dist/scripts/agent-planning-turn-deadline-tests.js +281 -0
- package/dist/scripts/anything-tests.js +70 -21
- package/dist/scripts/benchmark-environment-bridge-e2e.js +139 -37
- package/dist/scripts/benchmark-environment-bridge-tests.js +1099 -306
- package/dist/scripts/booking-copilot-availability-ledger-binding-tests.js +26 -26
- package/dist/scripts/{booking-copilot-availability-policy-v2-tests.js → booking-copilot-availability-policy-tests.js} +115 -115
- package/dist/scripts/booking-copilot-crossrepo-fixture-server.js +41 -1
- package/dist/scripts/booking-copilot-dsh-core-proof-tests.js +75 -120
- package/dist/scripts/booking-copilot-dsh-planner-proof-tests.js +370 -61
- package/dist/scripts/booking-copilot-dsh-plugin-proof-tests.js +2 -2
- package/dist/scripts/booking-copilot-event-sequence-concurrency-proof-tests.js +7 -3
- package/dist/scripts/booking-copilot-gap-code-contract-proof-tests.js +15 -9
- package/dist/scripts/booking-copilot-operation-ledger-concurrency-proof-tests.js +19 -11
- package/dist/scripts/booking-copilot-receipt-ledger-concurrency-proof-tests.js +16 -52
- package/dist/scripts/booking-copilot-runtime-proof-tests.js +3953 -155
- package/dist/scripts/booking-copilot-server-proof-tests.js +58 -23
- package/dist/scripts/booking-copilot-startup-proof-tests.js +10 -119
- package/dist/scripts/booking-surface-contract-proof-tests.js +1154 -326
- package/dist/scripts/bootstrap-tests.js +113 -2
- package/dist/scripts/build-metrics-report.js +339 -0
- package/dist/scripts/channel-probe-tests.js +148 -0
- package/dist/scripts/channel-probe.js +252 -0
- package/dist/scripts/channel-registry-tests.js +262 -0
- package/dist/scripts/doctor-tests.js +175 -0
- package/dist/scripts/effect-tests.js +274 -1
- package/dist/scripts/extension-tests.js +129 -11
- package/dist/scripts/fact-gate-tests.js +91 -2
- package/dist/scripts/flyai-tests.js +17 -1
- package/dist/scripts/hbcli-e2e-tests.js +1 -3
- package/dist/scripts/hbcli-tests.js +38 -1
- package/dist/scripts/metrics-report-tests.js +288 -0
- package/dist/scripts/persona-surface-guard-tests.js +19 -0
- package/dist/scripts/session-tests.js +401 -1
- package/dist/scripts/smoke.js +100 -83
- package/dist/scripts/turn-handoff-collect-tests.js +216 -0
- package/dist/scripts/turn-handoff-collect.js +189 -0
- package/dist/scripts/turn-policy-tests.js +56 -0
- package/dist/scripts/typed-contract-canary.js +261 -0
- package/dist/scripts/visa-policy-tests.js +75 -0
- package/dist/scripts/wish-channel-gate-tests.js +111 -0
- package/dist/src/artifact-gate.js +91 -3
- package/dist/src/benchmark-agent-conformance.js +39 -16
- package/dist/src/benchmark-environment-bridge.js +316 -52
- package/dist/src/bookable-facts.js +62 -3
- package/dist/src/booking-surface/{availability-policy-v2.js → availability-policy.js} +89 -89
- package/dist/src/booking-surface/canonical-schema.js +8 -22
- package/dist/src/booking-surface/contracts.js +70 -19
- package/dist/src/booking-surface/dsh-planner.js +310 -177
- package/dist/src/booking-surface/dsh-plugin.js +3 -3
- package/dist/src/booking-surface/index.js +1 -5
- package/dist/src/booking-surface/profile.js +1 -1
- package/dist/src/booking-surface/runtime.js +1682 -247
- package/dist/src/booking-surface/server.js +371 -185
- package/dist/src/booking-surface/startup.js +17 -27
- package/dist/src/booking-surface/validation.js +270 -760
- package/dist/src/index.js +739 -150
- package/dist/src/turn-deadline.js +292 -0
- package/dist/src/turn-policy.js +146 -0
- package/dist/src/wish-pool.js +5 -0
- package/extension/background.js +23 -4
- package/extension/content-main.js +47 -9
- package/extension/manifest.json +27 -9
- package/package.json +7 -26
- package/schemas/booking.surface.schema.json +2593 -0
- package/ts/capabilities/agent-reach.ts +4 -4
- package/ts/capabilities/anything.ts +89 -38
- package/ts/capabilities/flyai.ts +21 -3
- package/ts/capabilities/hbcli.ts +53 -4
- package/ts/capabilities/session/action-cache.ts +1 -1
- package/ts/capabilities/session/adapters/ctrip-hotel.ts +238 -0
- package/ts/capabilities/session/adapters/rail-12306.ts +293 -0
- package/ts/capabilities/session/extension-bridge.ts +32 -2
- package/ts/capabilities/session/extension-channel.ts +9 -8
- package/ts/capabilities/session/extension-distribution.ts +3 -1
- package/ts/capabilities/session/health-watch.ts +1 -1
- package/ts/capabilities/session/read-guard.ts +1 -1
- package/ts/capabilities/session/wizard.ts +1 -1
- package/ts/capabilities/session-search.ts +323 -1
- package/ts/dsh-runtime/vendor/README.md +60 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/LICENSE +21 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/README.en.md +200 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/README.md +202 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/client/client.js +216 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/cordis.patch.yml +5 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/clients/amap.js +371 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/clients/nominatim.js +63 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/clients/osrm.js +56 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/clients/photon.js +55 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/config-file.js +87 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/config-route.js +126 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/config.js +16 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/index.js +109 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/settings-ns.js +27 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/tools/geocode.js +127 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/tools/poi.js +84 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/tools/routes.js +170 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/clients/amap.d.ts +53 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/clients/nominatim.d.ts +17 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/clients/osrm.d.ts +15 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/clients/photon.d.ts +22 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/config-file.d.ts +33 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/config-route.d.ts +20 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/config.d.ts +23 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/index.d.ts +18 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/settings-ns.d.ts +14 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/tools/geocode.d.ts +11 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/tools/poi.d.ts +9 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/tools/routes.d.ts +16 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/types.d.ts +55 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types.js +16 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/package.json +91 -0
- package/ts/package.json +9 -0
- package/ts/scripts/turn-handoff-collect.ts +178 -0
- package/ts/src/artifact-gate.ts +82 -2
- package/ts/src/benchmark-agent-conformance.ts +54 -15
- package/ts/src/benchmark-environment-bridge.ts +234 -46
- package/ts/src/bookable-facts.ts +109 -5
- package/ts/src/booking-saga.ts +1 -1
- package/ts/src/booking-surface/{availability-policy-v2.ts → availability-policy.ts} +124 -125
- package/ts/src/booking-surface/canonical-schema.js +8 -22
- package/ts/src/booking-surface/contracts.ts +278 -239
- package/ts/src/booking-surface/dsh-planner.ts +333 -194
- package/ts/src/booking-surface/dsh-plugin.js +3 -3
- package/ts/src/booking-surface/index.ts +1 -5
- package/ts/src/booking-surface/profile.ts +11 -11
- package/ts/src/booking-surface/runtime.ts +1351 -351
- package/ts/src/booking-surface/server.ts +282 -175
- package/ts/src/booking-surface/startup.ts +36 -48
- package/ts/src/booking-surface/validation.ts +194 -442
- package/ts/src/contracts.ts +1 -1
- package/ts/src/index.ts +492 -172
- package/ts/src/loop.ts +1 -1
- package/ts/src/state-ledger.ts +1 -1
- package/ts/src/turn-deadline.ts +365 -0
- package/ts/src/turn-policy.ts +154 -0
- package/ts/src/wish-pool.ts +17 -4
- package/dist/scripts/agent-planning-budget-e2e.js +0 -227
- package/dist/scripts/agent-planning-budget-tests.js +0 -173
- package/dist/scripts/booking-copilot-v2-runtime-proof-tests.js +0 -3935
- package/dist/scripts/booking-surface-v2-contract-proof-tests.js +0 -1174
- package/dist/src/booking-surface/contracts-v2.js +0 -89
- package/dist/src/booking-surface/runtime-v2.js +0 -1771
- package/dist/src/booking-surface/server-v2.js +0 -334
- package/dist/src/booking-surface/validation-v2.js +0 -319
- package/dist/src/tool-budget.js +0 -136
- package/schemas/booking.surface.v1.schema.json +0 -927
- package/schemas/booking.surface.v2.schema.json +0 -61
- package/ts/src/booking-surface/contracts-v2.ts +0 -118
- package/ts/src/booking-surface/runtime-v2.ts +0 -1466
- package/ts/src/booking-surface/server-v2.ts +0 -247
- package/ts/src/booking-surface/validation-v2.ts +0 -205
- package/ts/src/tool-budget.ts +0 -165
|
@@ -11,11 +11,17 @@ try {
|
|
|
11
11
|
await writeFile(p, `#!/bin/sh
|
|
12
12
|
cat <<'JSON'
|
|
13
13
|
{"candidates":[
|
|
14
|
-
{"type":"hotel","
|
|
15
|
-
{"type":"city","
|
|
16
|
-
{"type":"place","
|
|
14
|
+
{"type":"hotel","matchScore":180,"hotel":{"id":"h1","name":{"en":"Park Hyatt","zh":"柏悦酒店"},"star":5,"destinationId":"d1","latlngCoordinator":{"google":{"lat":22.31,"lng":114.16}}}},
|
|
15
|
+
{"type":"city","matchScore":250,"region":{"id":"d1","name":{"en":"Dali","zh":"大理市"},"countryCode":"CN","coordinates":{"centerLat":25.58,"centerLng":100.21}}},
|
|
16
|
+
{"type":"place","matchScore":90,"region":{"id":"r1","name":{"zh":"洱海"}},"place":{"latlngCoordinator":{"google":{"lat":25.74,"lng":100.25}}}}
|
|
17
17
|
]}
|
|
18
18
|
JSON
|
|
19
|
+
`, {
|
|
20
|
+
mode: 0o755
|
|
21
|
+
});
|
|
22
|
+
} else if (behaviour === 'echo-args') {
|
|
23
|
+
await writeFile(p, `#!/bin/sh
|
|
24
|
+
printf '{"candidates":[{"type":"city","region":{"id":"%s"}}]}' "$*"
|
|
19
25
|
`, {
|
|
20
26
|
mode: 0o755
|
|
21
27
|
});
|
|
@@ -31,6 +37,13 @@ JSON
|
|
|
31
37
|
await writeFile(p, `#!/bin/sh
|
|
32
38
|
echo 'hotelbe down' >&2
|
|
33
39
|
exit 1
|
|
40
|
+
`, {
|
|
41
|
+
mode: 0o755
|
|
42
|
+
});
|
|
43
|
+
} else if (behaviour === 'unknown-command') {
|
|
44
|
+
await writeFile(p, `#!/bin/sh
|
|
45
|
+
echo "error: unknown command 'anything'" >&2
|
|
46
|
+
exit 1
|
|
34
47
|
`, {
|
|
35
48
|
mode: 0o755
|
|
36
49
|
});
|
|
@@ -46,44 +59,80 @@ while :; do sleep 5; done
|
|
|
46
59
|
}
|
|
47
60
|
const ok = await fakeBin('hbcli-ok', 'ok');
|
|
48
61
|
const r1 = await anythingSearch({
|
|
49
|
-
keyword: '
|
|
62
|
+
keyword: '大理',
|
|
50
63
|
hbcliBin: ok
|
|
51
64
|
});
|
|
52
65
|
assert.equal(r1.verdict, 'hit');
|
|
53
66
|
assert.equal(r1.hits.length, 3);
|
|
54
67
|
assert.ok(r1.evidence.includes('hbcli-anything@'));
|
|
55
|
-
|
|
56
|
-
|
|
68
|
+
const [hotelHit, cityHit, placeHit] = r1.hits;
|
|
69
|
+
assert.equal(hotelHit.type, 'hotel');
|
|
70
|
+
assert.equal(hotelHit.name, '柏悦酒店', 'hotel i18n name 应 zh 优先');
|
|
71
|
+
assert.equal(hotelHit.hotelId, 'h1');
|
|
72
|
+
assert.equal(hotelHit.destinationId, 'd1', 'hotel 候选 destinationId 取所属目的地');
|
|
73
|
+
assert.equal(hotelHit.star, 5);
|
|
74
|
+
assert.equal(hotelHit.latitude, 22.31);
|
|
75
|
+
assert.equal(hotelHit.longitude, 114.16);
|
|
76
|
+
assert.equal(hotelHit.score, 180);
|
|
77
|
+
assert.equal(cityHit.type, 'city');
|
|
78
|
+
assert.equal(cityHit.name, '大理市', 'region i18n name 应 zh 优先');
|
|
79
|
+
assert.equal(cityHit.destinationId, 'd1');
|
|
80
|
+
assert.equal(cityHit.latitude, 25.58, 'city 坐标取 region.coordinates.centerLat');
|
|
81
|
+
assert.equal(cityHit.longitude, 100.21);
|
|
82
|
+
assert.equal(placeHit.type, 'place');
|
|
83
|
+
assert.equal(placeHit.name, '洱海');
|
|
84
|
+
assert.equal(placeHit.latitude, 25.74, 'place 坐标回退 place.latlngCoordinator');
|
|
85
|
+
console.log('1. 真实候选 wire 形状(hotel/city/place 逐字段)OK');
|
|
86
|
+
const echo = await fakeBin('hbcli-echo-args', 'echo-args');
|
|
57
87
|
const r2 = await anythingSearch({
|
|
88
|
+
keyword: '大理',
|
|
89
|
+
contentType: 'city',
|
|
90
|
+
parentDestinationId: 'd1',
|
|
91
|
+
hbcliBin: echo
|
|
92
|
+
});
|
|
93
|
+
assert.equal(r2.verdict, 'hit');
|
|
94
|
+
assert.equal(r2.hits[0].destinationId, '--json search anything 大理 --content-type city --destination-id d1', 'spawn 旗标形态(--json 前置/--destination-id 映射)应稳定,实际 ' + r2.hits[0].destinationId);
|
|
95
|
+
console.log('2. 旗标回归(--json 前置 + --destination-id 映射)OK');
|
|
96
|
+
const empty = await fakeBin('hbcli-empty', 'empty');
|
|
97
|
+
const r3 = await anythingSearch({
|
|
58
98
|
keyword: 'nothing',
|
|
59
99
|
hbcliBin: empty
|
|
60
100
|
});
|
|
61
|
-
assert.equal(
|
|
62
|
-
assert.equal(
|
|
63
|
-
console.log('
|
|
101
|
+
assert.equal(r3.verdict, 'miss');
|
|
102
|
+
assert.equal(r3.hits.length, 0);
|
|
103
|
+
console.log('3. nothing → miss OK');
|
|
64
104
|
const fail = await fakeBin('hbcli-fail', 'fail');
|
|
65
|
-
const
|
|
105
|
+
const r4 = await anythingSearch({
|
|
66
106
|
keyword: 'x',
|
|
67
107
|
hbcliBin: fail
|
|
68
108
|
});
|
|
69
|
-
assert.equal(
|
|
70
|
-
assert.equal(
|
|
71
|
-
assert.match(
|
|
72
|
-
console.log('
|
|
109
|
+
assert.equal(r4.verdict, 'error');
|
|
110
|
+
assert.equal(r4.ok, false);
|
|
111
|
+
assert.match(r4.evidence, /error/);
|
|
112
|
+
console.log('4. fail → error (降级) OK');
|
|
113
|
+
const old = await fakeBin('hbcli-old', 'unknown-command');
|
|
114
|
+
const r5 = await anythingSearch({
|
|
115
|
+
keyword: 'x',
|
|
116
|
+
hbcliBin: old
|
|
117
|
+
});
|
|
118
|
+
assert.equal(r5.verdict, 'error');
|
|
119
|
+
assert.match(r5.error ?? '', /hbcli update|gotry setup/, 'error 应带升级指引,实际 ' + r5.error);
|
|
120
|
+
assert.match(r5.evidence, /unknown command/, 'evidence 保留上游原话(溯源)');
|
|
121
|
+
console.log(`5. 旧 CLI unknown command → 升级指引 OK(${r5.error})`);
|
|
73
122
|
const hang = await fakeBin('hbcli-hang', 'hang');
|
|
74
|
-
const
|
|
123
|
+
const r6 = await anythingSearch({
|
|
75
124
|
keyword: 'x',
|
|
76
125
|
hbcliBin: hang,
|
|
77
126
|
timeoutMs: 800
|
|
78
127
|
});
|
|
79
|
-
assert.equal(
|
|
80
|
-
console.log(`
|
|
81
|
-
const
|
|
128
|
+
assert.equal(r6.verdict, 'error', 'timeout 应判 error');
|
|
129
|
+
console.log(`6. hang/timeout → error OK (latency=${r6.latencyMs}ms)`);
|
|
130
|
+
const r7 = await anythingSearch({
|
|
82
131
|
keyword: ' '
|
|
83
132
|
});
|
|
84
|
-
assert.equal(
|
|
85
|
-
console.log('
|
|
86
|
-
console.log('\nANYTHING TESTS:
|
|
133
|
+
assert.equal(r7.verdict, 'error');
|
|
134
|
+
console.log('7. empty keyword → error OK');
|
|
135
|
+
console.log('\nANYTHING TESTS: 7/7 OK(hbcli fake + wire 对齐 + 降级诚实)');
|
|
87
136
|
} finally{
|
|
88
137
|
await rm(tmp, {
|
|
89
138
|
recursive: true,
|
|
@@ -12,6 +12,22 @@ const BIN = join(ROOT, 'bin', 'gotry-inner.js');
|
|
|
12
12
|
const TOOL = 'gotry_benchmark_environment';
|
|
13
13
|
const MARKER = 'BENCHMARK_BRIDGE_LOOKUP_OK';
|
|
14
14
|
const TIMEOUT_MS = 30_000;
|
|
15
|
+
const LOOKUP_INPUT_SCHEMA = {
|
|
16
|
+
type: 'object',
|
|
17
|
+
properties: {
|
|
18
|
+
city: {
|
|
19
|
+
type: 'string',
|
|
20
|
+
enum: [
|
|
21
|
+
'Dubai',
|
|
22
|
+
'Singapore'
|
|
23
|
+
]
|
|
24
|
+
}
|
|
25
|
+
},
|
|
26
|
+
required: [
|
|
27
|
+
'city'
|
|
28
|
+
],
|
|
29
|
+
additionalProperties: false
|
|
30
|
+
};
|
|
15
31
|
const TSX_LOADER = pathToFileURL(createRequire(import.meta.url).resolve('tsx')).href;
|
|
16
32
|
function runRuntimeProbe(options) {
|
|
17
33
|
const fixture = mkdtempSync(join(tmpdir(), 'gotry-runtime-probe-'));
|
|
@@ -61,10 +77,7 @@ function assertRuntimeSelectionAndVersionGuards() {
|
|
|
61
77
|
const legacyFallback = runRuntimeProbe({
|
|
62
78
|
vendorVersion: '0.1.2-alpha.1'
|
|
63
79
|
});
|
|
64
|
-
assert.deepEqual(legacyFallback,
|
|
65
|
-
source: 'legacy-vendored',
|
|
66
|
-
version: '0.1.2-alpha.1'
|
|
67
|
-
}, 'non-benchmark source checkout may use the legacy vendored dsh fallback');
|
|
80
|
+
assert.deepEqual(legacyFallback, null, 'non-benchmark source checkout fail-closes instead of using the removed legacy vendored dsh fallback');
|
|
68
81
|
const wrongBenchmarkVersion = runRuntimeProbe({
|
|
69
82
|
rootVersion: '0.1.2-alpha.1',
|
|
70
83
|
vendorVersion: '0.1.2-alpha.1',
|
|
@@ -186,7 +199,7 @@ function finalText(text) {
|
|
|
186
199
|
]
|
|
187
200
|
}) + 'data: [DONE]\n\n';
|
|
188
201
|
}
|
|
189
|
-
function toolCall(callId = 'bridge-call-1') {
|
|
202
|
+
function toolCall(callId = 'bridge-call-1', city = 'Dubai') {
|
|
190
203
|
return sse({
|
|
191
204
|
id: `bridge-${callId}`,
|
|
192
205
|
object: 'chat.completion.chunk',
|
|
@@ -202,12 +215,10 @@ function toolCall(callId = 'bridge-call-1') {
|
|
|
202
215
|
function: {
|
|
203
216
|
name: TOOL,
|
|
204
217
|
arguments: JSON.stringify({
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
city: 'Dubai'
|
|
210
|
-
}
|
|
218
|
+
action: 'call',
|
|
219
|
+
tool: 'lookup',
|
|
220
|
+
arguments: {
|
|
221
|
+
city
|
|
211
222
|
}
|
|
212
223
|
})
|
|
213
224
|
}
|
|
@@ -242,7 +253,9 @@ function anyToolResultPresent(body) {
|
|
|
242
253
|
}
|
|
243
254
|
async function runCase(mode, executableOverride, extraEnv = {}) {
|
|
244
255
|
const requests = [];
|
|
256
|
+
let servedToolCalls = 0;
|
|
245
257
|
let spawnTarget = '';
|
|
258
|
+
const domainRecoveryMode = mode === 'domain-recovery' || mode === 'domain-recovery-failed';
|
|
246
259
|
const server = createServer((req, res)=>{
|
|
247
260
|
const chunks = [];
|
|
248
261
|
req.on('data', (c)=>chunks.push(Buffer.from(c)));
|
|
@@ -258,8 +271,15 @@ async function runCase(mode, executableOverride, extraEnv = {}) {
|
|
|
258
271
|
if (mode === 'spawn-failed' && names(body).includes(TOOL) && !anyToolResultPresent(body) && spawnTarget) rmSync(spawnTarget, {
|
|
259
272
|
force: true
|
|
260
273
|
});
|
|
261
|
-
if (
|
|
262
|
-
|
|
274
|
+
if (domainRecoveryMode && servedToolCalls === 1 && names(body).includes(TOOL) && anyToolResultPresent(body) && !toolResultPresent(body)) {
|
|
275
|
+
servedToolCalls += 1;
|
|
276
|
+
res.end(toolCall('bridge-call-2', 'Singapore'));
|
|
277
|
+
} else if (mode !== 'disabled' && mode !== 'invalid-path' && mode !== 'invalid-schema' && mode !== 'unsafe-config' && names(body).includes(TOOL) && !anyToolResultPresent(body)) {
|
|
278
|
+
servedToolCalls += 1;
|
|
279
|
+
res.end(toolCall());
|
|
280
|
+
} else {
|
|
281
|
+
res.end(finalText('<benchmark_terminal>{"status":"succeeded"}</benchmark_terminal>'));
|
|
282
|
+
}
|
|
263
283
|
});
|
|
264
284
|
});
|
|
265
285
|
await new Promise((resolve)=>server.listen(0, '127.0.0.1', resolve));
|
|
@@ -274,15 +294,16 @@ async function runCase(mode, executableOverride, extraEnv = {}) {
|
|
|
274
294
|
});
|
|
275
295
|
writeResolutionProbe(probe, probeResult, mode === 'disabled');
|
|
276
296
|
const runner = join(cwd, 'synthetic-runner.js');
|
|
297
|
+
const runnerTrace = join(cwd, 'synthetic-runner-trace.jsonl');
|
|
277
298
|
spawnTarget = join(cwd, 'synthetic-spawn-target.js');
|
|
278
299
|
const configPath = join(cwd, mode === 'invalid-path' ? 'benchmark-env-config-\n.json' : 'benchmark-env-config.json');
|
|
279
|
-
const runnerBody = mode === 'timeout' ? `setTimeout(() => {}, 60_000)` : mode === 'runner-failed' ? `process.stderr.write('PRIVATE_RUNNER_DIAGNOSTIC_DO_NOT_REFLECT'); process.exit(17)` : mode === 'output-truncated' ? `process.stdout.write('x'.repeat(20_000))` : mode === 'unexpected-output' ? `process.stdout.write(JSON.stringify({ result: { marker: '${MARKER}', leaked: [], unexpected: 'must-not-reflect' } }))` : `const forbidden = ['GOTRY_BENCHMARK_ENV_CONFIG', 'GOTRY_BENCHMARK_BRIDGE_PARENT_SECRET', 'LLM_API_KEY', 'LLM_BASE_URL', 'LLM_MODEL', 'DEEPSEEK_BASE_URL', 'GOTRY_LLM_MODEL', 'DATABASE_URL', 'SSH_AUTH_SOCK', 'AWS_PROFILE', 'HTTPS_PROXY']; const leaked = forbidden.filter(name => process.env[name] !== undefined); process.stdout.write(JSON.stringify({ result: { marker: '${MARKER}', leaked } }))`;
|
|
280
|
-
writeFileSync(runner, `if (process.argv.length !== 5 || process.argv[2] !== 'call' || process.argv[3] !== 'lookup' || JSON.parse(process.argv[4]).city !== 'Dubai') process.exit(2); ${runnerBody}`);
|
|
300
|
+
const runnerBody = mode === 'domain-recovery-failed' ? `if (args.city === 'Dubai') process.stdout.write(JSON.stringify({ schema_version: 'gotry_benchmark_tool_result_v1', status: 'miss', code: 'NOT_FOUND', recovery: 'revise_arguments' })); else { process.stderr.write('PRIVATE_RECOVERY_RUNNER_DIAGNOSTIC_DO_NOT_REFLECT'); process.exit(17) }` : mode === 'domain-recovery' ? `if (args.city === 'Dubai') process.stdout.write(JSON.stringify({ schema_version: 'gotry_benchmark_tool_result_v1', status: 'miss', code: 'NOT_FOUND', recovery: 'revise_arguments' })); else process.stdout.write(JSON.stringify({ schema_version: 'gotry_benchmark_tool_result_v1', status: 'ok', result: { marker: '${MARKER}', leaked: [] } }))` : mode === 'timeout' ? `setTimeout(() => {}, 60_000)` : mode === 'runner-failed' ? `process.stderr.write('PRIVATE_RUNNER_DIAGNOSTIC_DO_NOT_REFLECT'); process.exit(17)` : mode === 'output-truncated' ? `process.stdout.write('x'.repeat(20_000))` : mode === 'unexpected-output' ? `process.stdout.write(JSON.stringify({ schema_version: 'gotry_benchmark_tool_result_v1', status: 'ok', result: { marker: '${MARKER}', leaked: [], unexpected: 'must-not-reflect' } }))` : `const forbidden = ['GOTRY_BENCHMARK_ENV_CONFIG', 'GOTRY_BENCHMARK_BRIDGE_PARENT_SECRET', 'LLM_API_KEY', 'LLM_BASE_URL', 'LLM_MODEL', 'DEEPSEEK_BASE_URL', 'GOTRY_LLM_MODEL', 'DATABASE_URL', 'SSH_AUTH_SOCK', 'AWS_PROFILE', 'HTTPS_PROXY']; const leaked = forbidden.filter(name => process.env[name] !== undefined); process.stdout.write(JSON.stringify({ schema_version: 'gotry_benchmark_tool_result_v1', status: 'ok', result: { marker: '${MARKER}', leaked } }))`;
|
|
301
|
+
writeFileSync(runner, `if (process.argv.length !== 5 || process.argv[2] !== 'call' || process.argv[3] !== 'lookup' || (!${JSON.stringify(domainRecoveryMode)} && JSON.parse(process.argv[4]).city !== 'Dubai') || (${JSON.stringify(domainRecoveryMode)} && !['Dubai', 'Singapore'].includes(JSON.parse(process.argv[4]).city))) process.exit(2); const fs = require('node:fs'); const args = JSON.parse(process.argv[4]); fs.appendFileSync(${JSON.stringify(runnerTrace)}, JSON.stringify({ city: args.city }) + '\\n'); ${runnerBody}`);
|
|
281
302
|
writeFileSync(spawnTarget, '#!/usr/bin/env node\nprocess.exit(0)\n', {
|
|
282
303
|
mode: 0o700
|
|
283
304
|
});
|
|
284
305
|
writeFileSync(configPath, JSON.stringify({
|
|
285
|
-
schema_version: mode === 'invalid-schema' ? 'invalid' : '
|
|
306
|
+
schema_version: mode === 'invalid-schema' ? 'invalid' : 'gotry_benchmark_environment_bridge_v3',
|
|
286
307
|
enabled: true,
|
|
287
308
|
executable: mode === 'spawn-failed' ? spawnTarget : process.execPath,
|
|
288
309
|
cwd,
|
|
@@ -291,15 +312,24 @@ async function runCase(mode, executableOverride, extraEnv = {}) {
|
|
|
291
312
|
] : [
|
|
292
313
|
runner
|
|
293
314
|
],
|
|
294
|
-
|
|
295
|
-
|
|
315
|
+
tools: [
|
|
316
|
+
{
|
|
317
|
+
name: 'lookup',
|
|
318
|
+
description: 'Lookup.',
|
|
319
|
+
input_schema: LOOKUP_INPUT_SCHEMA,
|
|
320
|
+
output_keys: [
|
|
321
|
+
'marker',
|
|
322
|
+
'leaked'
|
|
323
|
+
],
|
|
324
|
+
domain_outcomes: [
|
|
325
|
+
{
|
|
326
|
+
status: 'miss',
|
|
327
|
+
code: 'NOT_FOUND',
|
|
328
|
+
recovery: domainRecoveryMode ? 'revise_arguments' : 'none'
|
|
329
|
+
}
|
|
330
|
+
]
|
|
331
|
+
}
|
|
296
332
|
],
|
|
297
|
-
allowed_output_keys: {
|
|
298
|
-
lookup: [
|
|
299
|
-
'marker',
|
|
300
|
-
'leaked'
|
|
301
|
-
]
|
|
302
|
-
},
|
|
303
333
|
timeout_ms: mode === 'timeout' ? 50 : 10_000,
|
|
304
334
|
max_output_bytes: mode === 'output-truncated' ? 1_024 : 4_096,
|
|
305
335
|
terminal_output: {
|
|
@@ -403,13 +433,16 @@ async function runCase(mode, executableOverride, extraEnv = {}) {
|
|
|
403
433
|
calendar: 0,
|
|
404
434
|
map: 0
|
|
405
435
|
});
|
|
436
|
+
const runnerArguments = existsSync(runnerTrace) ? readFileSync(runnerTrace, 'utf8').split('\n').filter(Boolean).map((line)=>JSON.parse(line)) : [];
|
|
406
437
|
return {
|
|
407
438
|
exit,
|
|
408
439
|
stdout,
|
|
409
440
|
stderr,
|
|
410
441
|
output: stdout + stderr,
|
|
411
442
|
requests,
|
|
412
|
-
optionalResolutionHits
|
|
443
|
+
optionalResolutionHits,
|
|
444
|
+
servedToolCalls,
|
|
445
|
+
runnerArguments
|
|
413
446
|
};
|
|
414
447
|
} finally{
|
|
415
448
|
await new Promise((resolve)=>server.close(()=>resolve()));
|
|
@@ -442,6 +475,21 @@ async function assertRuntimeContract(executableOverride) {
|
|
|
442
475
|
assert.deepEqual(enabledToolNames, [
|
|
443
476
|
TOOL
|
|
444
477
|
], `${target} enabled runtime must expose exactly the benchmark tool; observed tool names=${JSON.stringify(enabledToolNames)}`);
|
|
478
|
+
const bridgeTool = enabled.requests.find((request)=>names(request).includes(TOOL))?.tools?.find((tool)=>names({
|
|
479
|
+
tools: [
|
|
480
|
+
tool
|
|
481
|
+
]
|
|
482
|
+
}).includes(TOOL));
|
|
483
|
+
const flatSchema = bridgeTool?.function?.parameters;
|
|
484
|
+
assert.ok(Array.isArray(flatSchema?.oneOf), `${target} bridge exposes flat oneOf schema`);
|
|
485
|
+
const lookupBranch = flatSchema.oneOf.find((branch)=>branch?.properties?.tool?.const === 'lookup');
|
|
486
|
+
assert.deepEqual(lookupBranch?.required, [
|
|
487
|
+
'action',
|
|
488
|
+
'tool',
|
|
489
|
+
'arguments'
|
|
490
|
+
]);
|
|
491
|
+
assert.equal(lookupBranch?.additionalProperties, false);
|
|
492
|
+
assert.deepEqual(lookupBranch?.properties?.arguments, LOOKUP_INPUT_SCHEMA);
|
|
445
493
|
assert.equal(enabledToolNames.some((name)=>name.startsWith('calendar_') || name.startsWith('map_')), false, `${target} benchmark projection must not expose calendar/map tools`);
|
|
446
494
|
assert.deepEqual(enabled.optionalResolutionHits, {
|
|
447
495
|
calendar: 0,
|
|
@@ -465,6 +513,37 @@ async function assertRuntimeContract(executableOverride) {
|
|
|
465
513
|
return (prompt.match(/You are GoTry, a task-agnostic travel planning assistant\./g) ?? []).length === 1 && (prompt.match(/Use only the current conversation and tools available in this benchmark session\./g) ?? []).length === 1;
|
|
466
514
|
}), `${target} benchmark persona has each stable sentence exactly once per request`);
|
|
467
515
|
assert.match(enabled.output, /benchmark_terminal/);
|
|
516
|
+
const recovered = await runCase('domain-recovery', executableOverride);
|
|
517
|
+
assert.equal(recovered.exit, 0, `${target} model-driven domain miss recovery exits successfully; output=${recovered.output.slice(-2_000)}`);
|
|
518
|
+
assert.equal(recovered.servedToolCalls, 2, `${target} model emits exactly two tool calls around one declared miss`);
|
|
519
|
+
assert.deepEqual(recovered.runnerArguments, [
|
|
520
|
+
{
|
|
521
|
+
city: 'Dubai'
|
|
522
|
+
},
|
|
523
|
+
{
|
|
524
|
+
city: 'Singapore'
|
|
525
|
+
}
|
|
526
|
+
], `${target} model revises the declared city before the second adapter invocation`);
|
|
527
|
+
assert.ok(recovered.requests.some((request)=>(request.messages ?? []).some((message)=>{
|
|
528
|
+
if (message.role !== 'tool') return false;
|
|
529
|
+
const serialized = JSON.stringify(message);
|
|
530
|
+
return /status\\?":\\?"miss/.test(serialized) && /recovery\\?":\\?"revise_arguments/.test(serialized);
|
|
531
|
+
})), `${target} declared typed miss reaches model history`);
|
|
532
|
+
assert.match(recovered.stdout, /<benchmark_terminal>/, `${target} corrected second call reaches tagged terminal output`);
|
|
533
|
+
const failedRecovery = await runCase('domain-recovery-failed', executableOverride);
|
|
534
|
+
assert.equal(failedRecovery.exit, 1, `${target} infrastructure failure after a declared miss cannot be masked by the earlier domain outcome`);
|
|
535
|
+
assert.equal(failedRecovery.servedToolCalls, 2, `${target} failed recovery still exercises exactly two model-owned tool calls`);
|
|
536
|
+
assert.deepEqual(failedRecovery.runnerArguments, [
|
|
537
|
+
{
|
|
538
|
+
city: 'Dubai'
|
|
539
|
+
},
|
|
540
|
+
{
|
|
541
|
+
city: 'Singapore'
|
|
542
|
+
}
|
|
543
|
+
], `${target} failed recovery reaches the revised second adapter invocation`);
|
|
544
|
+
assert.equal(failedRecovery.stdout, '', `${target} failed recovery releases no terminal stdout`);
|
|
545
|
+
assert.match(failedRecovery.stderr, /benchmark terminal output unavailable \(child_bridge_runner_failed\)/, `${target} failed recovery preserves the second runner failure classification`);
|
|
546
|
+
assert.equal(failedRecovery.output.includes('PRIVATE_RECOVERY_RUNNER_DIAGNOSTIC_DO_NOT_REFLECT'), false, `${target} failed recovery never reflects private runner stderr`);
|
|
468
547
|
const debugRedaction = await runCase('debug-redaction', executableOverride);
|
|
469
548
|
assert.equal(debugRedaction.exit, 0, `${target} benchmark debug mode preserves successful execution`);
|
|
470
549
|
assert.equal(debugRedaction.output.includes('PRIVATE_QUERY_SENTINEL_DO_NOT_REFLECT'), false, `${target} benchmark debug output never reflects the private task`);
|
|
@@ -734,23 +813,32 @@ async function runConformanceCase(mode, executableOverride) {
|
|
|
734
813
|
const runner = join(cwd, 'synthetic-runner.js');
|
|
735
814
|
const runnerCount = join(cwd, 'runner-count.txt');
|
|
736
815
|
const configPath = join(cwd, 'benchmark-env-config.json');
|
|
737
|
-
writeFileSync(runner, `const fs = require('node:fs'); const path = ${JSON.stringify(runnerCount)}; const count = fs.existsSync(path) ? Number(fs.readFileSync(path, 'utf8')) : 0; fs.writeFileSync(path, String(count + 1)); process.stdout.write(JSON.stringify({ result: { marker: '${MARKER}' } }))`);
|
|
816
|
+
writeFileSync(runner, `const fs = require('node:fs'); const path = ${JSON.stringify(runnerCount)}; const count = fs.existsSync(path) ? Number(fs.readFileSync(path, 'utf8')) : 0; fs.writeFileSync(path, String(count + 1)); process.stdout.write(JSON.stringify({ schema_version: 'gotry_benchmark_tool_result_v1', status: 'ok', result: { marker: '${MARKER}' } }))`);
|
|
738
817
|
writeFileSync(configPath, JSON.stringify({
|
|
739
|
-
schema_version: '
|
|
818
|
+
schema_version: 'gotry_benchmark_environment_bridge_v3',
|
|
740
819
|
enabled: true,
|
|
741
820
|
executable: process.execPath,
|
|
742
821
|
cwd,
|
|
743
822
|
argv_prefix: [
|
|
744
823
|
runner
|
|
745
824
|
],
|
|
746
|
-
|
|
747
|
-
|
|
825
|
+
tools: [
|
|
826
|
+
{
|
|
827
|
+
name: 'lookup',
|
|
828
|
+
description: 'Lookup.',
|
|
829
|
+
input_schema: LOOKUP_INPUT_SCHEMA,
|
|
830
|
+
output_keys: [
|
|
831
|
+
'marker'
|
|
832
|
+
],
|
|
833
|
+
domain_outcomes: [
|
|
834
|
+
{
|
|
835
|
+
status: 'miss',
|
|
836
|
+
code: 'NOT_FOUND',
|
|
837
|
+
recovery: 'none'
|
|
838
|
+
}
|
|
839
|
+
]
|
|
840
|
+
}
|
|
748
841
|
],
|
|
749
|
-
allowed_output_keys: {
|
|
750
|
-
lookup: [
|
|
751
|
-
'marker'
|
|
752
|
-
]
|
|
753
|
-
},
|
|
754
842
|
timeout_ms: 2_000,
|
|
755
843
|
max_output_bytes: 4_096,
|
|
756
844
|
terminal_output: {
|
|
@@ -944,7 +1032,7 @@ if (packaged) {
|
|
|
944
1032
|
try {
|
|
945
1033
|
const configPath = join(missingServiceRoot, 'bridge.json');
|
|
946
1034
|
writeFileSync(configPath, JSON.stringify({
|
|
947
|
-
schema_version: '
|
|
1035
|
+
schema_version: 'gotry_benchmark_environment_bridge_v3',
|
|
948
1036
|
enabled: true,
|
|
949
1037
|
executable: process.execPath,
|
|
950
1038
|
cwd: missingServiceRoot,
|
|
@@ -952,8 +1040,22 @@ if (packaged) {
|
|
|
952
1040
|
'-e',
|
|
953
1041
|
'process.exit(0)'
|
|
954
1042
|
],
|
|
955
|
-
|
|
956
|
-
|
|
1043
|
+
tools: [
|
|
1044
|
+
{
|
|
1045
|
+
name: 'lookup',
|
|
1046
|
+
description: 'Lookup.',
|
|
1047
|
+
input_schema: LOOKUP_INPUT_SCHEMA,
|
|
1048
|
+
output_keys: [
|
|
1049
|
+
'marker'
|
|
1050
|
+
],
|
|
1051
|
+
domain_outcomes: [
|
|
1052
|
+
{
|
|
1053
|
+
status: 'miss',
|
|
1054
|
+
code: 'NOT_FOUND',
|
|
1055
|
+
recovery: 'none'
|
|
1056
|
+
}
|
|
1057
|
+
]
|
|
1058
|
+
}
|
|
957
1059
|
],
|
|
958
1060
|
timeout_ms: 100,
|
|
959
1061
|
max_output_bytes: 4_096,
|