@danceiny/gotry 0.0.1-rc.19 → 0.0.1-rc.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/README.zh-CN.md +1 -1
- package/bin/gotry-bootstrap.js +83 -37
- package/bin/gotry-inner.js +11 -6
- package/cordis.gotry-patch.yml +7 -3
- package/dist/capabilities/anything.js +62 -29
- package/dist/capabilities/doctor.js +25 -3
- package/dist/capabilities/hbcli.js +28 -2
- package/dist/scripts/agent-planning-turn-deadline-e2e.js +2 -1
- package/dist/scripts/anything-tests.js +70 -21
- package/dist/scripts/benchmark-environment-bridge-e2e.js +136 -29
- package/dist/scripts/benchmark-environment-bridge-tests.js +1056 -269
- package/dist/scripts/bootstrap-tests.js +28 -2
- package/dist/scripts/doctor-tests.js +55 -1
- package/dist/scripts/hbcli-tests.js +38 -1
- package/dist/src/benchmark-agent-conformance.js +32 -12
- package/dist/src/benchmark-environment-bridge.js +317 -114
- package/dist/src/index.js +5 -2
- package/extension/manifest.json +2 -2
- package/package.json +2 -1
- package/ts/capabilities/anything.ts +89 -38
- package/ts/capabilities/hbcli.ts +53 -4
- package/ts/dsh-runtime/vendor/README.md +60 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/LICENSE +21 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/README.en.md +200 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/README.md +202 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/client/client.js +216 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/cordis.patch.yml +5 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/clients/amap.js +371 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/clients/nominatim.js +63 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/clients/osrm.js +56 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/clients/photon.js +55 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/config-file.js +87 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/config-route.js +126 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/config.js +16 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/index.js +109 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/settings-ns.js +27 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/tools/geocode.js +127 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/tools/poi.js +84 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/tools/routes.js +170 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/clients/amap.d.ts +53 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/clients/nominatim.d.ts +17 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/clients/osrm.d.ts +15 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/clients/photon.d.ts +22 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/config-file.d.ts +33 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/config-route.d.ts +20 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/config.d.ts +23 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/index.d.ts +18 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/settings-ns.d.ts +14 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/tools/geocode.d.ts +11 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/tools/poi.d.ts +9 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/tools/routes.d.ts +16 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/types.d.ts +55 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types.js +16 -0
- package/ts/dsh-runtime/vendor/dsh-map-tools/package.json +91 -0
- package/ts/package.json +9 -1
- package/ts/src/benchmark-agent-conformance.ts +40 -9
- package/ts/src/benchmark-environment-bridge.ts +235 -66
- package/ts/src/index.ts +4 -2
- package/ts/src/turn-deadline.ts +4 -0
|
@@ -11,11 +11,17 @@ try {
|
|
|
11
11
|
await writeFile(p, `#!/bin/sh
|
|
12
12
|
cat <<'JSON'
|
|
13
13
|
{"candidates":[
|
|
14
|
-
{"type":"hotel","
|
|
15
|
-
{"type":"city","
|
|
16
|
-
{"type":"place","
|
|
14
|
+
{"type":"hotel","matchScore":180,"hotel":{"id":"h1","name":{"en":"Park Hyatt","zh":"柏悦酒店"},"star":5,"destinationId":"d1","latlngCoordinator":{"google":{"lat":22.31,"lng":114.16}}}},
|
|
15
|
+
{"type":"city","matchScore":250,"region":{"id":"d1","name":{"en":"Dali","zh":"大理市"},"countryCode":"CN","coordinates":{"centerLat":25.58,"centerLng":100.21}}},
|
|
16
|
+
{"type":"place","matchScore":90,"region":{"id":"r1","name":{"zh":"洱海"}},"place":{"latlngCoordinator":{"google":{"lat":25.74,"lng":100.25}}}}
|
|
17
17
|
]}
|
|
18
18
|
JSON
|
|
19
|
+
`, {
|
|
20
|
+
mode: 0o755
|
|
21
|
+
});
|
|
22
|
+
} else if (behaviour === 'echo-args') {
|
|
23
|
+
await writeFile(p, `#!/bin/sh
|
|
24
|
+
printf '{"candidates":[{"type":"city","region":{"id":"%s"}}]}' "$*"
|
|
19
25
|
`, {
|
|
20
26
|
mode: 0o755
|
|
21
27
|
});
|
|
@@ -31,6 +37,13 @@ JSON
|
|
|
31
37
|
await writeFile(p, `#!/bin/sh
|
|
32
38
|
echo 'hotelbe down' >&2
|
|
33
39
|
exit 1
|
|
40
|
+
`, {
|
|
41
|
+
mode: 0o755
|
|
42
|
+
});
|
|
43
|
+
} else if (behaviour === 'unknown-command') {
|
|
44
|
+
await writeFile(p, `#!/bin/sh
|
|
45
|
+
echo "error: unknown command 'anything'" >&2
|
|
46
|
+
exit 1
|
|
34
47
|
`, {
|
|
35
48
|
mode: 0o755
|
|
36
49
|
});
|
|
@@ -46,44 +59,80 @@ while :; do sleep 5; done
|
|
|
46
59
|
}
|
|
47
60
|
const ok = await fakeBin('hbcli-ok', 'ok');
|
|
48
61
|
const r1 = await anythingSearch({
|
|
49
|
-
keyword: '
|
|
62
|
+
keyword: '大理',
|
|
50
63
|
hbcliBin: ok
|
|
51
64
|
});
|
|
52
65
|
assert.equal(r1.verdict, 'hit');
|
|
53
66
|
assert.equal(r1.hits.length, 3);
|
|
54
67
|
assert.ok(r1.evidence.includes('hbcli-anything@'));
|
|
55
|
-
|
|
56
|
-
|
|
68
|
+
const [hotelHit, cityHit, placeHit] = r1.hits;
|
|
69
|
+
assert.equal(hotelHit.type, 'hotel');
|
|
70
|
+
assert.equal(hotelHit.name, '柏悦酒店', 'hotel i18n name 应 zh 优先');
|
|
71
|
+
assert.equal(hotelHit.hotelId, 'h1');
|
|
72
|
+
assert.equal(hotelHit.destinationId, 'd1', 'hotel 候选 destinationId 取所属目的地');
|
|
73
|
+
assert.equal(hotelHit.star, 5);
|
|
74
|
+
assert.equal(hotelHit.latitude, 22.31);
|
|
75
|
+
assert.equal(hotelHit.longitude, 114.16);
|
|
76
|
+
assert.equal(hotelHit.score, 180);
|
|
77
|
+
assert.equal(cityHit.type, 'city');
|
|
78
|
+
assert.equal(cityHit.name, '大理市', 'region i18n name 应 zh 优先');
|
|
79
|
+
assert.equal(cityHit.destinationId, 'd1');
|
|
80
|
+
assert.equal(cityHit.latitude, 25.58, 'city 坐标取 region.coordinates.centerLat');
|
|
81
|
+
assert.equal(cityHit.longitude, 100.21);
|
|
82
|
+
assert.equal(placeHit.type, 'place');
|
|
83
|
+
assert.equal(placeHit.name, '洱海');
|
|
84
|
+
assert.equal(placeHit.latitude, 25.74, 'place 坐标回退 place.latlngCoordinator');
|
|
85
|
+
console.log('1. 真实候选 wire 形状(hotel/city/place 逐字段)OK');
|
|
86
|
+
const echo = await fakeBin('hbcli-echo-args', 'echo-args');
|
|
57
87
|
const r2 = await anythingSearch({
|
|
88
|
+
keyword: '大理',
|
|
89
|
+
contentType: 'city',
|
|
90
|
+
parentDestinationId: 'd1',
|
|
91
|
+
hbcliBin: echo
|
|
92
|
+
});
|
|
93
|
+
assert.equal(r2.verdict, 'hit');
|
|
94
|
+
assert.equal(r2.hits[0].destinationId, '--json search anything 大理 --content-type city --destination-id d1', 'spawn 旗标形态(--json 前置/--destination-id 映射)应稳定,实际 ' + r2.hits[0].destinationId);
|
|
95
|
+
console.log('2. 旗标回归(--json 前置 + --destination-id 映射)OK');
|
|
96
|
+
const empty = await fakeBin('hbcli-empty', 'empty');
|
|
97
|
+
const r3 = await anythingSearch({
|
|
58
98
|
keyword: 'nothing',
|
|
59
99
|
hbcliBin: empty
|
|
60
100
|
});
|
|
61
|
-
assert.equal(
|
|
62
|
-
assert.equal(
|
|
63
|
-
console.log('
|
|
101
|
+
assert.equal(r3.verdict, 'miss');
|
|
102
|
+
assert.equal(r3.hits.length, 0);
|
|
103
|
+
console.log('3. nothing → miss OK');
|
|
64
104
|
const fail = await fakeBin('hbcli-fail', 'fail');
|
|
65
|
-
const
|
|
105
|
+
const r4 = await anythingSearch({
|
|
66
106
|
keyword: 'x',
|
|
67
107
|
hbcliBin: fail
|
|
68
108
|
});
|
|
69
|
-
assert.equal(
|
|
70
|
-
assert.equal(
|
|
71
|
-
assert.match(
|
|
72
|
-
console.log('
|
|
109
|
+
assert.equal(r4.verdict, 'error');
|
|
110
|
+
assert.equal(r4.ok, false);
|
|
111
|
+
assert.match(r4.evidence, /error/);
|
|
112
|
+
console.log('4. fail → error (降级) OK');
|
|
113
|
+
const old = await fakeBin('hbcli-old', 'unknown-command');
|
|
114
|
+
const r5 = await anythingSearch({
|
|
115
|
+
keyword: 'x',
|
|
116
|
+
hbcliBin: old
|
|
117
|
+
});
|
|
118
|
+
assert.equal(r5.verdict, 'error');
|
|
119
|
+
assert.match(r5.error ?? '', /hbcli update|gotry setup/, 'error 应带升级指引,实际 ' + r5.error);
|
|
120
|
+
assert.match(r5.evidence, /unknown command/, 'evidence 保留上游原话(溯源)');
|
|
121
|
+
console.log(`5. 旧 CLI unknown command → 升级指引 OK(${r5.error})`);
|
|
73
122
|
const hang = await fakeBin('hbcli-hang', 'hang');
|
|
74
|
-
const
|
|
123
|
+
const r6 = await anythingSearch({
|
|
75
124
|
keyword: 'x',
|
|
76
125
|
hbcliBin: hang,
|
|
77
126
|
timeoutMs: 800
|
|
78
127
|
});
|
|
79
|
-
assert.equal(
|
|
80
|
-
console.log(`
|
|
81
|
-
const
|
|
128
|
+
assert.equal(r6.verdict, 'error', 'timeout 应判 error');
|
|
129
|
+
console.log(`6. hang/timeout → error OK (latency=${r6.latencyMs}ms)`);
|
|
130
|
+
const r7 = await anythingSearch({
|
|
82
131
|
keyword: ' '
|
|
83
132
|
});
|
|
84
|
-
assert.equal(
|
|
85
|
-
console.log('
|
|
86
|
-
console.log('\nANYTHING TESTS:
|
|
133
|
+
assert.equal(r7.verdict, 'error');
|
|
134
|
+
console.log('7. empty keyword → error OK');
|
|
135
|
+
console.log('\nANYTHING TESTS: 7/7 OK(hbcli fake + wire 对齐 + 降级诚实)');
|
|
87
136
|
} finally{
|
|
88
137
|
await rm(tmp, {
|
|
89
138
|
recursive: true,
|
|
@@ -12,6 +12,22 @@ const BIN = join(ROOT, 'bin', 'gotry-inner.js');
|
|
|
12
12
|
const TOOL = 'gotry_benchmark_environment';
|
|
13
13
|
const MARKER = 'BENCHMARK_BRIDGE_LOOKUP_OK';
|
|
14
14
|
const TIMEOUT_MS = 30_000;
|
|
15
|
+
const LOOKUP_INPUT_SCHEMA = {
|
|
16
|
+
type: 'object',
|
|
17
|
+
properties: {
|
|
18
|
+
city: {
|
|
19
|
+
type: 'string',
|
|
20
|
+
enum: [
|
|
21
|
+
'Dubai',
|
|
22
|
+
'Singapore'
|
|
23
|
+
]
|
|
24
|
+
}
|
|
25
|
+
},
|
|
26
|
+
required: [
|
|
27
|
+
'city'
|
|
28
|
+
],
|
|
29
|
+
additionalProperties: false
|
|
30
|
+
};
|
|
15
31
|
const TSX_LOADER = pathToFileURL(createRequire(import.meta.url).resolve('tsx')).href;
|
|
16
32
|
function runRuntimeProbe(options) {
|
|
17
33
|
const fixture = mkdtempSync(join(tmpdir(), 'gotry-runtime-probe-'));
|
|
@@ -61,7 +77,7 @@ function assertRuntimeSelectionAndVersionGuards() {
|
|
|
61
77
|
const legacyFallback = runRuntimeProbe({
|
|
62
78
|
vendorVersion: '0.1.2-alpha.1'
|
|
63
79
|
});
|
|
64
|
-
assert.
|
|
80
|
+
assert.deepEqual(legacyFallback, null, 'non-benchmark source checkout fail-closes instead of using the removed legacy vendored dsh fallback');
|
|
65
81
|
const wrongBenchmarkVersion = runRuntimeProbe({
|
|
66
82
|
rootVersion: '0.1.2-alpha.1',
|
|
67
83
|
vendorVersion: '0.1.2-alpha.1',
|
|
@@ -183,7 +199,7 @@ function finalText(text) {
|
|
|
183
199
|
]
|
|
184
200
|
}) + 'data: [DONE]\n\n';
|
|
185
201
|
}
|
|
186
|
-
function toolCall(callId = 'bridge-call-1') {
|
|
202
|
+
function toolCall(callId = 'bridge-call-1', city = 'Dubai') {
|
|
187
203
|
return sse({
|
|
188
204
|
id: `bridge-${callId}`,
|
|
189
205
|
object: 'chat.completion.chunk',
|
|
@@ -202,7 +218,7 @@ function toolCall(callId = 'bridge-call-1') {
|
|
|
202
218
|
action: 'call',
|
|
203
219
|
tool: 'lookup',
|
|
204
220
|
arguments: {
|
|
205
|
-
city
|
|
221
|
+
city
|
|
206
222
|
}
|
|
207
223
|
})
|
|
208
224
|
}
|
|
@@ -237,7 +253,9 @@ function anyToolResultPresent(body) {
|
|
|
237
253
|
}
|
|
238
254
|
async function runCase(mode, executableOverride, extraEnv = {}) {
|
|
239
255
|
const requests = [];
|
|
256
|
+
let servedToolCalls = 0;
|
|
240
257
|
let spawnTarget = '';
|
|
258
|
+
const domainRecoveryMode = mode === 'domain-recovery' || mode === 'domain-recovery-failed';
|
|
241
259
|
const server = createServer((req, res)=>{
|
|
242
260
|
const chunks = [];
|
|
243
261
|
req.on('data', (c)=>chunks.push(Buffer.from(c)));
|
|
@@ -253,8 +271,15 @@ async function runCase(mode, executableOverride, extraEnv = {}) {
|
|
|
253
271
|
if (mode === 'spawn-failed' && names(body).includes(TOOL) && !anyToolResultPresent(body) && spawnTarget) rmSync(spawnTarget, {
|
|
254
272
|
force: true
|
|
255
273
|
});
|
|
256
|
-
if (
|
|
257
|
-
|
|
274
|
+
if (domainRecoveryMode && servedToolCalls === 1 && names(body).includes(TOOL) && anyToolResultPresent(body) && !toolResultPresent(body)) {
|
|
275
|
+
servedToolCalls += 1;
|
|
276
|
+
res.end(toolCall('bridge-call-2', 'Singapore'));
|
|
277
|
+
} else if (mode !== 'disabled' && mode !== 'invalid-path' && mode !== 'invalid-schema' && mode !== 'unsafe-config' && names(body).includes(TOOL) && !anyToolResultPresent(body)) {
|
|
278
|
+
servedToolCalls += 1;
|
|
279
|
+
res.end(toolCall());
|
|
280
|
+
} else {
|
|
281
|
+
res.end(finalText('<benchmark_terminal>{"status":"succeeded"}</benchmark_terminal>'));
|
|
282
|
+
}
|
|
258
283
|
});
|
|
259
284
|
});
|
|
260
285
|
await new Promise((resolve)=>server.listen(0, '127.0.0.1', resolve));
|
|
@@ -269,15 +294,16 @@ async function runCase(mode, executableOverride, extraEnv = {}) {
|
|
|
269
294
|
});
|
|
270
295
|
writeResolutionProbe(probe, probeResult, mode === 'disabled');
|
|
271
296
|
const runner = join(cwd, 'synthetic-runner.js');
|
|
297
|
+
const runnerTrace = join(cwd, 'synthetic-runner-trace.jsonl');
|
|
272
298
|
spawnTarget = join(cwd, 'synthetic-spawn-target.js');
|
|
273
299
|
const configPath = join(cwd, mode === 'invalid-path' ? 'benchmark-env-config-\n.json' : 'benchmark-env-config.json');
|
|
274
|
-
const runnerBody = mode === 'timeout' ? `setTimeout(() => {}, 60_000)` : mode === 'runner-failed' ? `process.stderr.write('PRIVATE_RUNNER_DIAGNOSTIC_DO_NOT_REFLECT'); process.exit(17)` : mode === 'output-truncated' ? `process.stdout.write('x'.repeat(20_000))` : mode === 'unexpected-output' ? `process.stdout.write(JSON.stringify({ result: { marker: '${MARKER}', leaked: [], unexpected: 'must-not-reflect' } }))` : `const forbidden = ['GOTRY_BENCHMARK_ENV_CONFIG', 'GOTRY_BENCHMARK_BRIDGE_PARENT_SECRET', 'LLM_API_KEY', 'LLM_BASE_URL', 'LLM_MODEL', 'DEEPSEEK_BASE_URL', 'GOTRY_LLM_MODEL', 'DATABASE_URL', 'SSH_AUTH_SOCK', 'AWS_PROFILE', 'HTTPS_PROXY']; const leaked = forbidden.filter(name => process.env[name] !== undefined); process.stdout.write(JSON.stringify({ result: { marker: '${MARKER}', leaked } }))`;
|
|
275
|
-
writeFileSync(runner, `if (process.argv.length !== 5 || process.argv[2] !== 'call' || process.argv[3] !== 'lookup' || JSON.parse(process.argv[4]).city !== 'Dubai') process.exit(2); ${runnerBody}`);
|
|
300
|
+
const runnerBody = mode === 'domain-recovery-failed' ? `if (args.city === 'Dubai') process.stdout.write(JSON.stringify({ schema_version: 'gotry_benchmark_tool_result_v1', status: 'miss', code: 'NOT_FOUND', recovery: 'revise_arguments' })); else { process.stderr.write('PRIVATE_RECOVERY_RUNNER_DIAGNOSTIC_DO_NOT_REFLECT'); process.exit(17) }` : mode === 'domain-recovery' ? `if (args.city === 'Dubai') process.stdout.write(JSON.stringify({ schema_version: 'gotry_benchmark_tool_result_v1', status: 'miss', code: 'NOT_FOUND', recovery: 'revise_arguments' })); else process.stdout.write(JSON.stringify({ schema_version: 'gotry_benchmark_tool_result_v1', status: 'ok', result: { marker: '${MARKER}', leaked: [] } }))` : mode === 'timeout' ? `setTimeout(() => {}, 60_000)` : mode === 'runner-failed' ? `process.stderr.write('PRIVATE_RUNNER_DIAGNOSTIC_DO_NOT_REFLECT'); process.exit(17)` : mode === 'output-truncated' ? `process.stdout.write('x'.repeat(20_000))` : mode === 'unexpected-output' ? `process.stdout.write(JSON.stringify({ schema_version: 'gotry_benchmark_tool_result_v1', status: 'ok', result: { marker: '${MARKER}', leaked: [], unexpected: 'must-not-reflect' } }))` : `const forbidden = ['GOTRY_BENCHMARK_ENV_CONFIG', 'GOTRY_BENCHMARK_BRIDGE_PARENT_SECRET', 'LLM_API_KEY', 'LLM_BASE_URL', 'LLM_MODEL', 'DEEPSEEK_BASE_URL', 'GOTRY_LLM_MODEL', 'DATABASE_URL', 'SSH_AUTH_SOCK', 'AWS_PROFILE', 'HTTPS_PROXY']; const leaked = forbidden.filter(name => process.env[name] !== undefined); process.stdout.write(JSON.stringify({ schema_version: 'gotry_benchmark_tool_result_v1', status: 'ok', result: { marker: '${MARKER}', leaked } }))`;
|
|
301
|
+
writeFileSync(runner, `if (process.argv.length !== 5 || process.argv[2] !== 'call' || process.argv[3] !== 'lookup' || (!${JSON.stringify(domainRecoveryMode)} && JSON.parse(process.argv[4]).city !== 'Dubai') || (${JSON.stringify(domainRecoveryMode)} && !['Dubai', 'Singapore'].includes(JSON.parse(process.argv[4]).city))) process.exit(2); const fs = require('node:fs'); const args = JSON.parse(process.argv[4]); fs.appendFileSync(${JSON.stringify(runnerTrace)}, JSON.stringify({ city: args.city }) + '\\n'); ${runnerBody}`);
|
|
276
302
|
writeFileSync(spawnTarget, '#!/usr/bin/env node\nprocess.exit(0)\n', {
|
|
277
303
|
mode: 0o700
|
|
278
304
|
});
|
|
279
305
|
writeFileSync(configPath, JSON.stringify({
|
|
280
|
-
schema_version: mode === 'invalid-schema' ? 'invalid' : '
|
|
306
|
+
schema_version: mode === 'invalid-schema' ? 'invalid' : 'gotry_benchmark_environment_bridge_v3',
|
|
281
307
|
enabled: true,
|
|
282
308
|
executable: mode === 'spawn-failed' ? spawnTarget : process.execPath,
|
|
283
309
|
cwd,
|
|
@@ -286,15 +312,24 @@ async function runCase(mode, executableOverride, extraEnv = {}) {
|
|
|
286
312
|
] : [
|
|
287
313
|
runner
|
|
288
314
|
],
|
|
289
|
-
|
|
290
|
-
|
|
315
|
+
tools: [
|
|
316
|
+
{
|
|
317
|
+
name: 'lookup',
|
|
318
|
+
description: 'Lookup.',
|
|
319
|
+
input_schema: LOOKUP_INPUT_SCHEMA,
|
|
320
|
+
output_keys: [
|
|
321
|
+
'marker',
|
|
322
|
+
'leaked'
|
|
323
|
+
],
|
|
324
|
+
domain_outcomes: [
|
|
325
|
+
{
|
|
326
|
+
status: 'miss',
|
|
327
|
+
code: 'NOT_FOUND',
|
|
328
|
+
recovery: domainRecoveryMode ? 'revise_arguments' : 'none'
|
|
329
|
+
}
|
|
330
|
+
]
|
|
331
|
+
}
|
|
291
332
|
],
|
|
292
|
-
allowed_output_keys: {
|
|
293
|
-
lookup: [
|
|
294
|
-
'marker',
|
|
295
|
-
'leaked'
|
|
296
|
-
]
|
|
297
|
-
},
|
|
298
333
|
timeout_ms: mode === 'timeout' ? 50 : 10_000,
|
|
299
334
|
max_output_bytes: mode === 'output-truncated' ? 1_024 : 4_096,
|
|
300
335
|
terminal_output: {
|
|
@@ -398,13 +433,16 @@ async function runCase(mode, executableOverride, extraEnv = {}) {
|
|
|
398
433
|
calendar: 0,
|
|
399
434
|
map: 0
|
|
400
435
|
});
|
|
436
|
+
const runnerArguments = existsSync(runnerTrace) ? readFileSync(runnerTrace, 'utf8').split('\n').filter(Boolean).map((line)=>JSON.parse(line)) : [];
|
|
401
437
|
return {
|
|
402
438
|
exit,
|
|
403
439
|
stdout,
|
|
404
440
|
stderr,
|
|
405
441
|
output: stdout + stderr,
|
|
406
442
|
requests,
|
|
407
|
-
optionalResolutionHits
|
|
443
|
+
optionalResolutionHits,
|
|
444
|
+
servedToolCalls,
|
|
445
|
+
runnerArguments
|
|
408
446
|
};
|
|
409
447
|
} finally{
|
|
410
448
|
await new Promise((resolve)=>server.close(()=>resolve()));
|
|
@@ -437,6 +475,21 @@ async function assertRuntimeContract(executableOverride) {
|
|
|
437
475
|
assert.deepEqual(enabledToolNames, [
|
|
438
476
|
TOOL
|
|
439
477
|
], `${target} enabled runtime must expose exactly the benchmark tool; observed tool names=${JSON.stringify(enabledToolNames)}`);
|
|
478
|
+
const bridgeTool = enabled.requests.find((request)=>names(request).includes(TOOL))?.tools?.find((tool)=>names({
|
|
479
|
+
tools: [
|
|
480
|
+
tool
|
|
481
|
+
]
|
|
482
|
+
}).includes(TOOL));
|
|
483
|
+
const flatSchema = bridgeTool?.function?.parameters;
|
|
484
|
+
assert.ok(Array.isArray(flatSchema?.oneOf), `${target} bridge exposes flat oneOf schema`);
|
|
485
|
+
const lookupBranch = flatSchema.oneOf.find((branch)=>branch?.properties?.tool?.const === 'lookup');
|
|
486
|
+
assert.deepEqual(lookupBranch?.required, [
|
|
487
|
+
'action',
|
|
488
|
+
'tool',
|
|
489
|
+
'arguments'
|
|
490
|
+
]);
|
|
491
|
+
assert.equal(lookupBranch?.additionalProperties, false);
|
|
492
|
+
assert.deepEqual(lookupBranch?.properties?.arguments, LOOKUP_INPUT_SCHEMA);
|
|
440
493
|
assert.equal(enabledToolNames.some((name)=>name.startsWith('calendar_') || name.startsWith('map_')), false, `${target} benchmark projection must not expose calendar/map tools`);
|
|
441
494
|
assert.deepEqual(enabled.optionalResolutionHits, {
|
|
442
495
|
calendar: 0,
|
|
@@ -460,6 +513,37 @@ async function assertRuntimeContract(executableOverride) {
|
|
|
460
513
|
return (prompt.match(/You are GoTry, a task-agnostic travel planning assistant\./g) ?? []).length === 1 && (prompt.match(/Use only the current conversation and tools available in this benchmark session\./g) ?? []).length === 1;
|
|
461
514
|
}), `${target} benchmark persona has each stable sentence exactly once per request`);
|
|
462
515
|
assert.match(enabled.output, /benchmark_terminal/);
|
|
516
|
+
const recovered = await runCase('domain-recovery', executableOverride);
|
|
517
|
+
assert.equal(recovered.exit, 0, `${target} model-driven domain miss recovery exits successfully; output=${recovered.output.slice(-2_000)}`);
|
|
518
|
+
assert.equal(recovered.servedToolCalls, 2, `${target} model emits exactly two tool calls around one declared miss`);
|
|
519
|
+
assert.deepEqual(recovered.runnerArguments, [
|
|
520
|
+
{
|
|
521
|
+
city: 'Dubai'
|
|
522
|
+
},
|
|
523
|
+
{
|
|
524
|
+
city: 'Singapore'
|
|
525
|
+
}
|
|
526
|
+
], `${target} model revises the declared city before the second adapter invocation`);
|
|
527
|
+
assert.ok(recovered.requests.some((request)=>(request.messages ?? []).some((message)=>{
|
|
528
|
+
if (message.role !== 'tool') return false;
|
|
529
|
+
const serialized = JSON.stringify(message);
|
|
530
|
+
return /status\\?":\\?"miss/.test(serialized) && /recovery\\?":\\?"revise_arguments/.test(serialized);
|
|
531
|
+
})), `${target} declared typed miss reaches model history`);
|
|
532
|
+
assert.match(recovered.stdout, /<benchmark_terminal>/, `${target} corrected second call reaches tagged terminal output`);
|
|
533
|
+
const failedRecovery = await runCase('domain-recovery-failed', executableOverride);
|
|
534
|
+
assert.equal(failedRecovery.exit, 1, `${target} infrastructure failure after a declared miss cannot be masked by the earlier domain outcome`);
|
|
535
|
+
assert.equal(failedRecovery.servedToolCalls, 2, `${target} failed recovery still exercises exactly two model-owned tool calls`);
|
|
536
|
+
assert.deepEqual(failedRecovery.runnerArguments, [
|
|
537
|
+
{
|
|
538
|
+
city: 'Dubai'
|
|
539
|
+
},
|
|
540
|
+
{
|
|
541
|
+
city: 'Singapore'
|
|
542
|
+
}
|
|
543
|
+
], `${target} failed recovery reaches the revised second adapter invocation`);
|
|
544
|
+
assert.equal(failedRecovery.stdout, '', `${target} failed recovery releases no terminal stdout`);
|
|
545
|
+
assert.match(failedRecovery.stderr, /benchmark terminal output unavailable \(child_bridge_runner_failed\)/, `${target} failed recovery preserves the second runner failure classification`);
|
|
546
|
+
assert.equal(failedRecovery.output.includes('PRIVATE_RECOVERY_RUNNER_DIAGNOSTIC_DO_NOT_REFLECT'), false, `${target} failed recovery never reflects private runner stderr`);
|
|
463
547
|
const debugRedaction = await runCase('debug-redaction', executableOverride);
|
|
464
548
|
assert.equal(debugRedaction.exit, 0, `${target} benchmark debug mode preserves successful execution`);
|
|
465
549
|
assert.equal(debugRedaction.output.includes('PRIVATE_QUERY_SENTINEL_DO_NOT_REFLECT'), false, `${target} benchmark debug output never reflects the private task`);
|
|
@@ -729,23 +813,32 @@ async function runConformanceCase(mode, executableOverride) {
|
|
|
729
813
|
const runner = join(cwd, 'synthetic-runner.js');
|
|
730
814
|
const runnerCount = join(cwd, 'runner-count.txt');
|
|
731
815
|
const configPath = join(cwd, 'benchmark-env-config.json');
|
|
732
|
-
writeFileSync(runner, `const fs = require('node:fs'); const path = ${JSON.stringify(runnerCount)}; const count = fs.existsSync(path) ? Number(fs.readFileSync(path, 'utf8')) : 0; fs.writeFileSync(path, String(count + 1)); process.stdout.write(JSON.stringify({ result: { marker: '${MARKER}' } }))`);
|
|
816
|
+
writeFileSync(runner, `const fs = require('node:fs'); const path = ${JSON.stringify(runnerCount)}; const count = fs.existsSync(path) ? Number(fs.readFileSync(path, 'utf8')) : 0; fs.writeFileSync(path, String(count + 1)); process.stdout.write(JSON.stringify({ schema_version: 'gotry_benchmark_tool_result_v1', status: 'ok', result: { marker: '${MARKER}' } }))`);
|
|
733
817
|
writeFileSync(configPath, JSON.stringify({
|
|
734
|
-
schema_version: '
|
|
818
|
+
schema_version: 'gotry_benchmark_environment_bridge_v3',
|
|
735
819
|
enabled: true,
|
|
736
820
|
executable: process.execPath,
|
|
737
821
|
cwd,
|
|
738
822
|
argv_prefix: [
|
|
739
823
|
runner
|
|
740
824
|
],
|
|
741
|
-
|
|
742
|
-
|
|
825
|
+
tools: [
|
|
826
|
+
{
|
|
827
|
+
name: 'lookup',
|
|
828
|
+
description: 'Lookup.',
|
|
829
|
+
input_schema: LOOKUP_INPUT_SCHEMA,
|
|
830
|
+
output_keys: [
|
|
831
|
+
'marker'
|
|
832
|
+
],
|
|
833
|
+
domain_outcomes: [
|
|
834
|
+
{
|
|
835
|
+
status: 'miss',
|
|
836
|
+
code: 'NOT_FOUND',
|
|
837
|
+
recovery: 'none'
|
|
838
|
+
}
|
|
839
|
+
]
|
|
840
|
+
}
|
|
743
841
|
],
|
|
744
|
-
allowed_output_keys: {
|
|
745
|
-
lookup: [
|
|
746
|
-
'marker'
|
|
747
|
-
]
|
|
748
|
-
},
|
|
749
842
|
timeout_ms: 2_000,
|
|
750
843
|
max_output_bytes: 4_096,
|
|
751
844
|
terminal_output: {
|
|
@@ -939,7 +1032,7 @@ if (packaged) {
|
|
|
939
1032
|
try {
|
|
940
1033
|
const configPath = join(missingServiceRoot, 'bridge.json');
|
|
941
1034
|
writeFileSync(configPath, JSON.stringify({
|
|
942
|
-
schema_version: '
|
|
1035
|
+
schema_version: 'gotry_benchmark_environment_bridge_v3',
|
|
943
1036
|
enabled: true,
|
|
944
1037
|
executable: process.execPath,
|
|
945
1038
|
cwd: missingServiceRoot,
|
|
@@ -947,8 +1040,22 @@ if (packaged) {
|
|
|
947
1040
|
'-e',
|
|
948
1041
|
'process.exit(0)'
|
|
949
1042
|
],
|
|
950
|
-
|
|
951
|
-
|
|
1043
|
+
tools: [
|
|
1044
|
+
{
|
|
1045
|
+
name: 'lookup',
|
|
1046
|
+
description: 'Lookup.',
|
|
1047
|
+
input_schema: LOOKUP_INPUT_SCHEMA,
|
|
1048
|
+
output_keys: [
|
|
1049
|
+
'marker'
|
|
1050
|
+
],
|
|
1051
|
+
domain_outcomes: [
|
|
1052
|
+
{
|
|
1053
|
+
status: 'miss',
|
|
1054
|
+
code: 'NOT_FOUND',
|
|
1055
|
+
recovery: 'none'
|
|
1056
|
+
}
|
|
1057
|
+
]
|
|
1058
|
+
}
|
|
952
1059
|
],
|
|
953
1060
|
timeout_ms: 100,
|
|
954
1061
|
max_output_bytes: 4_096,
|