@danceiny/gotry 0.0.1-rc.19 → 0.0.1-rc.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/README.md +1 -1
  2. package/README.zh-CN.md +1 -1
  3. package/bin/gotry-bootstrap.js +83 -37
  4. package/bin/gotry-inner.js +11 -6
  5. package/cordis.gotry-patch.yml +7 -3
  6. package/dist/capabilities/anything.js +62 -29
  7. package/dist/capabilities/doctor.js +25 -3
  8. package/dist/capabilities/hbcli.js +28 -2
  9. package/dist/scripts/agent-planning-turn-deadline-e2e.js +2 -1
  10. package/dist/scripts/anything-tests.js +70 -21
  11. package/dist/scripts/benchmark-environment-bridge-e2e.js +136 -29
  12. package/dist/scripts/benchmark-environment-bridge-tests.js +1056 -269
  13. package/dist/scripts/bootstrap-tests.js +28 -2
  14. package/dist/scripts/doctor-tests.js +55 -1
  15. package/dist/scripts/hbcli-tests.js +38 -1
  16. package/dist/src/benchmark-agent-conformance.js +32 -12
  17. package/dist/src/benchmark-environment-bridge.js +317 -114
  18. package/dist/src/index.js +5 -2
  19. package/extension/manifest.json +2 -2
  20. package/package.json +2 -1
  21. package/ts/capabilities/anything.ts +89 -38
  22. package/ts/capabilities/hbcli.ts +53 -4
  23. package/ts/dsh-runtime/vendor/README.md +60 -0
  24. package/ts/dsh-runtime/vendor/dsh-map-tools/LICENSE +21 -0
  25. package/ts/dsh-runtime/vendor/dsh-map-tools/README.en.md +200 -0
  26. package/ts/dsh-runtime/vendor/dsh-map-tools/README.md +202 -0
  27. package/ts/dsh-runtime/vendor/dsh-map-tools/client/client.js +216 -0
  28. package/ts/dsh-runtime/vendor/dsh-map-tools/cordis.patch.yml +5 -0
  29. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/clients/amap.js +371 -0
  30. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/clients/nominatim.js +63 -0
  31. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/clients/osrm.js +56 -0
  32. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/clients/photon.js +55 -0
  33. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/config-file.js +87 -0
  34. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/config-route.js +126 -0
  35. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/config.js +16 -0
  36. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/index.js +109 -0
  37. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/settings-ns.js +27 -0
  38. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/tools/geocode.js +127 -0
  39. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/tools/poi.js +84 -0
  40. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/tools/routes.js +170 -0
  41. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/clients/amap.d.ts +53 -0
  42. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/clients/nominatim.d.ts +17 -0
  43. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/clients/osrm.d.ts +15 -0
  44. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/clients/photon.d.ts +22 -0
  45. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/config-file.d.ts +33 -0
  46. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/config-route.d.ts +20 -0
  47. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/config.d.ts +23 -0
  48. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/index.d.ts +18 -0
  49. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/settings-ns.d.ts +14 -0
  50. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/tools/geocode.d.ts +11 -0
  51. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/tools/poi.d.ts +9 -0
  52. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/tools/routes.d.ts +16 -0
  53. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/types.d.ts +55 -0
  54. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types.js +16 -0
  55. package/ts/dsh-runtime/vendor/dsh-map-tools/package.json +91 -0
  56. package/ts/package.json +9 -1
  57. package/ts/src/benchmark-agent-conformance.ts +40 -9
  58. package/ts/src/benchmark-environment-bridge.ts +235 -66
  59. package/ts/src/index.ts +4 -2
  60. package/ts/src/turn-deadline.ts +4 -0
@@ -11,11 +11,17 @@ try {
11
11
  await writeFile(p, `#!/bin/sh
12
12
  cat <<'JSON'
13
13
  {"candidates":[
14
- {"type":"hotel","name":"Park Hyatt","hotel":{"id":"h1","name":"Park Hyatt","latitude":22.31,"longitude":114.16}},
15
- {"type":"city","name":"大理市","region":{"id":"d1","name":"Dali","latitude":25.58,"longitude":100.21}},
16
- {"type":"place","name":"洱海","region":{"id":"r1","name":"Erhai Lake","latitude":25.74,"longitude":100.25}}
14
+ {"type":"hotel","matchScore":180,"hotel":{"id":"h1","name":{"en":"Park Hyatt","zh":"柏悦酒店"},"star":5,"destinationId":"d1","latlngCoordinator":{"google":{"lat":22.31,"lng":114.16}}}},
15
+ {"type":"city","matchScore":250,"region":{"id":"d1","name":{"en":"Dali","zh":"大理市"},"countryCode":"CN","coordinates":{"centerLat":25.58,"centerLng":100.21}}},
16
+ {"type":"place","matchScore":90,"region":{"id":"r1","name":{"zh":"洱海"}},"place":{"latlngCoordinator":{"google":{"lat":25.74,"lng":100.25}}}}
17
17
  ]}
18
18
  JSON
19
+ `, {
20
+ mode: 0o755
21
+ });
22
+ } else if (behaviour === 'echo-args') {
23
+ await writeFile(p, `#!/bin/sh
24
+ printf '{"candidates":[{"type":"city","region":{"id":"%s"}}]}' "$*"
19
25
  `, {
20
26
  mode: 0o755
21
27
  });
@@ -31,6 +37,13 @@ JSON
31
37
  await writeFile(p, `#!/bin/sh
32
38
  echo 'hotelbe down' >&2
33
39
  exit 1
40
+ `, {
41
+ mode: 0o755
42
+ });
43
+ } else if (behaviour === 'unknown-command') {
44
+ await writeFile(p, `#!/bin/sh
45
+ echo "error: unknown command 'anything'" >&2
46
+ exit 1
34
47
  `, {
35
48
  mode: 0o755
36
49
  });
@@ -46,44 +59,80 @@ while :; do sleep 5; done
46
59
  }
47
60
  const ok = await fakeBin('hbcli-ok', 'ok');
48
61
  const r1 = await anythingSearch({
49
- keyword: 'Park Hyatt',
62
+ keyword: '大理',
50
63
  hbcliBin: ok
51
64
  });
52
65
  assert.equal(r1.verdict, 'hit');
53
66
  assert.equal(r1.hits.length, 3);
54
67
  assert.ok(r1.evidence.includes('hbcli-anything@'));
55
- console.log(`1. Park Hyatt hit (${r1.hits.length} candidates) OK`);
56
- const empty = await fakeBin('hbcli-empty', 'empty');
68
+ const [hotelHit, cityHit, placeHit] = r1.hits;
69
+ assert.equal(hotelHit.type, 'hotel');
70
+ assert.equal(hotelHit.name, '柏悦酒店', 'hotel i18n name 应 zh 优先');
71
+ assert.equal(hotelHit.hotelId, 'h1');
72
+ assert.equal(hotelHit.destinationId, 'd1', 'hotel 候选 destinationId 取所属目的地');
73
+ assert.equal(hotelHit.star, 5);
74
+ assert.equal(hotelHit.latitude, 22.31);
75
+ assert.equal(hotelHit.longitude, 114.16);
76
+ assert.equal(hotelHit.score, 180);
77
+ assert.equal(cityHit.type, 'city');
78
+ assert.equal(cityHit.name, '大理市', 'region i18n name 应 zh 优先');
79
+ assert.equal(cityHit.destinationId, 'd1');
80
+ assert.equal(cityHit.latitude, 25.58, 'city 坐标取 region.coordinates.centerLat');
81
+ assert.equal(cityHit.longitude, 100.21);
82
+ assert.equal(placeHit.type, 'place');
83
+ assert.equal(placeHit.name, '洱海');
84
+ assert.equal(placeHit.latitude, 25.74, 'place 坐标回退 place.latlngCoordinator');
85
+ console.log('1. 真实候选 wire 形状(hotel/city/place 逐字段)OK');
86
+ const echo = await fakeBin('hbcli-echo-args', 'echo-args');
57
87
  const r2 = await anythingSearch({
88
+ keyword: '大理',
89
+ contentType: 'city',
90
+ parentDestinationId: 'd1',
91
+ hbcliBin: echo
92
+ });
93
+ assert.equal(r2.verdict, 'hit');
94
+ assert.equal(r2.hits[0].destinationId, '--json search anything 大理 --content-type city --destination-id d1', 'spawn 旗标形态(--json 前置/--destination-id 映射)应稳定,实际 ' + r2.hits[0].destinationId);
95
+ console.log('2. 旗标回归(--json 前置 + --destination-id 映射)OK');
96
+ const empty = await fakeBin('hbcli-empty', 'empty');
97
+ const r3 = await anythingSearch({
58
98
  keyword: 'nothing',
59
99
  hbcliBin: empty
60
100
  });
61
- assert.equal(r2.verdict, 'miss');
62
- assert.equal(r2.hits.length, 0);
63
- console.log('2. nothing → miss OK');
101
+ assert.equal(r3.verdict, 'miss');
102
+ assert.equal(r3.hits.length, 0);
103
+ console.log('3. nothing → miss OK');
64
104
  const fail = await fakeBin('hbcli-fail', 'fail');
65
- const r3 = await anythingSearch({
105
+ const r4 = await anythingSearch({
66
106
  keyword: 'x',
67
107
  hbcliBin: fail
68
108
  });
69
- assert.equal(r3.verdict, 'error');
70
- assert.equal(r3.ok, false);
71
- assert.match(r3.evidence, /error/);
72
- console.log('3. fail → error (降级) OK');
109
+ assert.equal(r4.verdict, 'error');
110
+ assert.equal(r4.ok, false);
111
+ assert.match(r4.evidence, /error/);
112
+ console.log('4. fail → error (降级) OK');
113
+ const old = await fakeBin('hbcli-old', 'unknown-command');
114
+ const r5 = await anythingSearch({
115
+ keyword: 'x',
116
+ hbcliBin: old
117
+ });
118
+ assert.equal(r5.verdict, 'error');
119
+ assert.match(r5.error ?? '', /hbcli update|gotry setup/, 'error 应带升级指引,实际 ' + r5.error);
120
+ assert.match(r5.evidence, /unknown command/, 'evidence 保留上游原话(溯源)');
121
+ console.log(`5. 旧 CLI unknown command → 升级指引 OK(${r5.error})`);
73
122
  const hang = await fakeBin('hbcli-hang', 'hang');
74
- const r4 = await anythingSearch({
123
+ const r6 = await anythingSearch({
75
124
  keyword: 'x',
76
125
  hbcliBin: hang,
77
126
  timeoutMs: 800
78
127
  });
79
- assert.equal(r4.verdict, 'error', 'timeout 应判 error');
80
- console.log(`4. hang/timeout → error OK (latency=${r4.latencyMs}ms)`);
81
- const r5 = await anythingSearch({
128
+ assert.equal(r6.verdict, 'error', 'timeout 应判 error');
129
+ console.log(`6. hang/timeout → error OK (latency=${r6.latencyMs}ms)`);
130
+ const r7 = await anythingSearch({
82
131
  keyword: ' '
83
132
  });
84
- assert.equal(r5.verdict, 'error');
85
- console.log('5. empty keyword → error OK');
86
- console.log('\nANYTHING TESTS: 5/5 OK(hbcli fake + graceful degrade)');
133
+ assert.equal(r7.verdict, 'error');
134
+ console.log('7. empty keyword → error OK');
135
+ console.log('\nANYTHING TESTS: 7/7 OK(hbcli fake + wire 对齐 + 降级诚实)');
87
136
  } finally{
88
137
  await rm(tmp, {
89
138
  recursive: true,
@@ -12,6 +12,22 @@ const BIN = join(ROOT, 'bin', 'gotry-inner.js');
12
12
  const TOOL = 'gotry_benchmark_environment';
13
13
  const MARKER = 'BENCHMARK_BRIDGE_LOOKUP_OK';
14
14
  const TIMEOUT_MS = 30_000;
15
+ const LOOKUP_INPUT_SCHEMA = {
16
+ type: 'object',
17
+ properties: {
18
+ city: {
19
+ type: 'string',
20
+ enum: [
21
+ 'Dubai',
22
+ 'Singapore'
23
+ ]
24
+ }
25
+ },
26
+ required: [
27
+ 'city'
28
+ ],
29
+ additionalProperties: false
30
+ };
15
31
  const TSX_LOADER = pathToFileURL(createRequire(import.meta.url).resolve('tsx')).href;
16
32
  function runRuntimeProbe(options) {
17
33
  const fixture = mkdtempSync(join(tmpdir(), 'gotry-runtime-probe-'));
@@ -61,7 +77,7 @@ function assertRuntimeSelectionAndVersionGuards() {
61
77
  const legacyFallback = runRuntimeProbe({
62
78
  vendorVersion: '0.1.2-alpha.1'
63
79
  });
64
- assert.equal(legacyFallback, null, 'D-27 removal (#120): legacy vendored fallback no longer resolves even outside benchmark — fail-closed');
80
+ assert.deepEqual(legacyFallback, null, 'non-benchmark source checkout fail-closes instead of using the removed legacy vendored dsh fallback');
65
81
  const wrongBenchmarkVersion = runRuntimeProbe({
66
82
  rootVersion: '0.1.2-alpha.1',
67
83
  vendorVersion: '0.1.2-alpha.1',
@@ -183,7 +199,7 @@ function finalText(text) {
183
199
  ]
184
200
  }) + 'data: [DONE]\n\n';
185
201
  }
186
- function toolCall(callId = 'bridge-call-1') {
202
+ function toolCall(callId = 'bridge-call-1', city = 'Dubai') {
187
203
  return sse({
188
204
  id: `bridge-${callId}`,
189
205
  object: 'chat.completion.chunk',
@@ -202,7 +218,7 @@ function toolCall(callId = 'bridge-call-1') {
202
218
  action: 'call',
203
219
  tool: 'lookup',
204
220
  arguments: {
205
- city: 'Dubai'
221
+ city
206
222
  }
207
223
  })
208
224
  }
@@ -237,7 +253,9 @@ function anyToolResultPresent(body) {
237
253
  }
238
254
  async function runCase(mode, executableOverride, extraEnv = {}) {
239
255
  const requests = [];
256
+ let servedToolCalls = 0;
240
257
  let spawnTarget = '';
258
+ const domainRecoveryMode = mode === 'domain-recovery' || mode === 'domain-recovery-failed';
241
259
  const server = createServer((req, res)=>{
242
260
  const chunks = [];
243
261
  req.on('data', (c)=>chunks.push(Buffer.from(c)));
@@ -253,8 +271,15 @@ async function runCase(mode, executableOverride, extraEnv = {}) {
253
271
  if (mode === 'spawn-failed' && names(body).includes(TOOL) && !anyToolResultPresent(body) && spawnTarget) rmSync(spawnTarget, {
254
272
  force: true
255
273
  });
256
- if (mode !== 'disabled' && mode !== 'invalid-path' && mode !== 'invalid-schema' && mode !== 'unsafe-config' && names(body).includes(TOOL) && !anyToolResultPresent(body)) res.end(toolCall());
257
- else res.end(finalText(mode === 'enabled' ? '<benchmark_terminal>{"status":"succeeded"}</benchmark_terminal>' : '<benchmark_terminal>{"status":"succeeded"}</benchmark_terminal>'));
274
+ if (domainRecoveryMode && servedToolCalls === 1 && names(body).includes(TOOL) && anyToolResultPresent(body) && !toolResultPresent(body)) {
275
+ servedToolCalls += 1;
276
+ res.end(toolCall('bridge-call-2', 'Singapore'));
277
+ } else if (mode !== 'disabled' && mode !== 'invalid-path' && mode !== 'invalid-schema' && mode !== 'unsafe-config' && names(body).includes(TOOL) && !anyToolResultPresent(body)) {
278
+ servedToolCalls += 1;
279
+ res.end(toolCall());
280
+ } else {
281
+ res.end(finalText('<benchmark_terminal>{"status":"succeeded"}</benchmark_terminal>'));
282
+ }
258
283
  });
259
284
  });
260
285
  await new Promise((resolve)=>server.listen(0, '127.0.0.1', resolve));
@@ -269,15 +294,16 @@ async function runCase(mode, executableOverride, extraEnv = {}) {
269
294
  });
270
295
  writeResolutionProbe(probe, probeResult, mode === 'disabled');
271
296
  const runner = join(cwd, 'synthetic-runner.js');
297
+ const runnerTrace = join(cwd, 'synthetic-runner-trace.jsonl');
272
298
  spawnTarget = join(cwd, 'synthetic-spawn-target.js');
273
299
  const configPath = join(cwd, mode === 'invalid-path' ? 'benchmark-env-config-\n.json' : 'benchmark-env-config.json');
274
- const runnerBody = mode === 'timeout' ? `setTimeout(() => {}, 60_000)` : mode === 'runner-failed' ? `process.stderr.write('PRIVATE_RUNNER_DIAGNOSTIC_DO_NOT_REFLECT'); process.exit(17)` : mode === 'output-truncated' ? `process.stdout.write('x'.repeat(20_000))` : mode === 'unexpected-output' ? `process.stdout.write(JSON.stringify({ result: { marker: '${MARKER}', leaked: [], unexpected: 'must-not-reflect' } }))` : `const forbidden = ['GOTRY_BENCHMARK_ENV_CONFIG', 'GOTRY_BENCHMARK_BRIDGE_PARENT_SECRET', 'LLM_API_KEY', 'LLM_BASE_URL', 'LLM_MODEL', 'DEEPSEEK_BASE_URL', 'GOTRY_LLM_MODEL', 'DATABASE_URL', 'SSH_AUTH_SOCK', 'AWS_PROFILE', 'HTTPS_PROXY']; const leaked = forbidden.filter(name => process.env[name] !== undefined); process.stdout.write(JSON.stringify({ result: { marker: '${MARKER}', leaked } }))`;
275
- writeFileSync(runner, `if (process.argv.length !== 5 || process.argv[2] !== 'call' || process.argv[3] !== 'lookup' || JSON.parse(process.argv[4]).city !== 'Dubai') process.exit(2); ${runnerBody}`);
300
+ const runnerBody = mode === 'domain-recovery-failed' ? `if (args.city === 'Dubai') process.stdout.write(JSON.stringify({ schema_version: 'gotry_benchmark_tool_result_v1', status: 'miss', code: 'NOT_FOUND', recovery: 'revise_arguments' })); else { process.stderr.write('PRIVATE_RECOVERY_RUNNER_DIAGNOSTIC_DO_NOT_REFLECT'); process.exit(17) }` : mode === 'domain-recovery' ? `if (args.city === 'Dubai') process.stdout.write(JSON.stringify({ schema_version: 'gotry_benchmark_tool_result_v1', status: 'miss', code: 'NOT_FOUND', recovery: 'revise_arguments' })); else process.stdout.write(JSON.stringify({ schema_version: 'gotry_benchmark_tool_result_v1', status: 'ok', result: { marker: '${MARKER}', leaked: [] } }))` : mode === 'timeout' ? `setTimeout(() => {}, 60_000)` : mode === 'runner-failed' ? `process.stderr.write('PRIVATE_RUNNER_DIAGNOSTIC_DO_NOT_REFLECT'); process.exit(17)` : mode === 'output-truncated' ? `process.stdout.write('x'.repeat(20_000))` : mode === 'unexpected-output' ? `process.stdout.write(JSON.stringify({ schema_version: 'gotry_benchmark_tool_result_v1', status: 'ok', result: { marker: '${MARKER}', leaked: [], unexpected: 'must-not-reflect' } }))` : `const forbidden = ['GOTRY_BENCHMARK_ENV_CONFIG', 'GOTRY_BENCHMARK_BRIDGE_PARENT_SECRET', 'LLM_API_KEY', 'LLM_BASE_URL', 'LLM_MODEL', 'DEEPSEEK_BASE_URL', 'GOTRY_LLM_MODEL', 'DATABASE_URL', 'SSH_AUTH_SOCK', 'AWS_PROFILE', 'HTTPS_PROXY']; const leaked = forbidden.filter(name => process.env[name] !== undefined); process.stdout.write(JSON.stringify({ schema_version: 'gotry_benchmark_tool_result_v1', status: 'ok', result: { marker: '${MARKER}', leaked } }))`;
301
+ writeFileSync(runner, `if (process.argv.length !== 5 || process.argv[2] !== 'call' || process.argv[3] !== 'lookup' || (!${JSON.stringify(domainRecoveryMode)} && JSON.parse(process.argv[4]).city !== 'Dubai') || (${JSON.stringify(domainRecoveryMode)} && !['Dubai', 'Singapore'].includes(JSON.parse(process.argv[4]).city))) process.exit(2); const fs = require('node:fs'); const args = JSON.parse(process.argv[4]); fs.appendFileSync(${JSON.stringify(runnerTrace)}, JSON.stringify({ city: args.city }) + '\\n'); ${runnerBody}`);
276
302
  writeFileSync(spawnTarget, '#!/usr/bin/env node\nprocess.exit(0)\n', {
277
303
  mode: 0o700
278
304
  });
279
305
  writeFileSync(configPath, JSON.stringify({
280
- schema_version: mode === 'invalid-schema' ? 'invalid' : 'gotry_benchmark_environment_bridge_v2',
306
+ schema_version: mode === 'invalid-schema' ? 'invalid' : 'gotry_benchmark_environment_bridge_v3',
281
307
  enabled: true,
282
308
  executable: mode === 'spawn-failed' ? spawnTarget : process.execPath,
283
309
  cwd,
@@ -286,15 +312,24 @@ async function runCase(mode, executableOverride, extraEnv = {}) {
286
312
  ] : [
287
313
  runner
288
314
  ],
289
- allowed_tools: [
290
- 'lookup'
315
+ tools: [
316
+ {
317
+ name: 'lookup',
318
+ description: 'Lookup.',
319
+ input_schema: LOOKUP_INPUT_SCHEMA,
320
+ output_keys: [
321
+ 'marker',
322
+ 'leaked'
323
+ ],
324
+ domain_outcomes: [
325
+ {
326
+ status: 'miss',
327
+ code: 'NOT_FOUND',
328
+ recovery: domainRecoveryMode ? 'revise_arguments' : 'none'
329
+ }
330
+ ]
331
+ }
291
332
  ],
292
- allowed_output_keys: {
293
- lookup: [
294
- 'marker',
295
- 'leaked'
296
- ]
297
- },
298
333
  timeout_ms: mode === 'timeout' ? 50 : 10_000,
299
334
  max_output_bytes: mode === 'output-truncated' ? 1_024 : 4_096,
300
335
  terminal_output: {
@@ -398,13 +433,16 @@ async function runCase(mode, executableOverride, extraEnv = {}) {
398
433
  calendar: 0,
399
434
  map: 0
400
435
  });
436
+ const runnerArguments = existsSync(runnerTrace) ? readFileSync(runnerTrace, 'utf8').split('\n').filter(Boolean).map((line)=>JSON.parse(line)) : [];
401
437
  return {
402
438
  exit,
403
439
  stdout,
404
440
  stderr,
405
441
  output: stdout + stderr,
406
442
  requests,
407
- optionalResolutionHits
443
+ optionalResolutionHits,
444
+ servedToolCalls,
445
+ runnerArguments
408
446
  };
409
447
  } finally{
410
448
  await new Promise((resolve)=>server.close(()=>resolve()));
@@ -437,6 +475,21 @@ async function assertRuntimeContract(executableOverride) {
437
475
  assert.deepEqual(enabledToolNames, [
438
476
  TOOL
439
477
  ], `${target} enabled runtime must expose exactly the benchmark tool; observed tool names=${JSON.stringify(enabledToolNames)}`);
478
+ const bridgeTool = enabled.requests.find((request)=>names(request).includes(TOOL))?.tools?.find((tool)=>names({
479
+ tools: [
480
+ tool
481
+ ]
482
+ }).includes(TOOL));
483
+ const flatSchema = bridgeTool?.function?.parameters;
484
+ assert.ok(Array.isArray(flatSchema?.oneOf), `${target} bridge exposes flat oneOf schema`);
485
+ const lookupBranch = flatSchema.oneOf.find((branch)=>branch?.properties?.tool?.const === 'lookup');
486
+ assert.deepEqual(lookupBranch?.required, [
487
+ 'action',
488
+ 'tool',
489
+ 'arguments'
490
+ ]);
491
+ assert.equal(lookupBranch?.additionalProperties, false);
492
+ assert.deepEqual(lookupBranch?.properties?.arguments, LOOKUP_INPUT_SCHEMA);
440
493
  assert.equal(enabledToolNames.some((name)=>name.startsWith('calendar_') || name.startsWith('map_')), false, `${target} benchmark projection must not expose calendar/map tools`);
441
494
  assert.deepEqual(enabled.optionalResolutionHits, {
442
495
  calendar: 0,
@@ -460,6 +513,37 @@ async function assertRuntimeContract(executableOverride) {
460
513
  return (prompt.match(/You are GoTry, a task-agnostic travel planning assistant\./g) ?? []).length === 1 && (prompt.match(/Use only the current conversation and tools available in this benchmark session\./g) ?? []).length === 1;
461
514
  }), `${target} benchmark persona has each stable sentence exactly once per request`);
462
515
  assert.match(enabled.output, /benchmark_terminal/);
516
+ const recovered = await runCase('domain-recovery', executableOverride);
517
+ assert.equal(recovered.exit, 0, `${target} model-driven domain miss recovery exits successfully; output=${recovered.output.slice(-2_000)}`);
518
+ assert.equal(recovered.servedToolCalls, 2, `${target} model emits exactly two tool calls around one declared miss`);
519
+ assert.deepEqual(recovered.runnerArguments, [
520
+ {
521
+ city: 'Dubai'
522
+ },
523
+ {
524
+ city: 'Singapore'
525
+ }
526
+ ], `${target} model revises the declared city before the second adapter invocation`);
527
+ assert.ok(recovered.requests.some((request)=>(request.messages ?? []).some((message)=>{
528
+ if (message.role !== 'tool') return false;
529
+ const serialized = JSON.stringify(message);
530
+ return /status\\?":\\?"miss/.test(serialized) && /recovery\\?":\\?"revise_arguments/.test(serialized);
531
+ })), `${target} declared typed miss reaches model history`);
532
+ assert.match(recovered.stdout, /<benchmark_terminal>/, `${target} corrected second call reaches tagged terminal output`);
533
+ const failedRecovery = await runCase('domain-recovery-failed', executableOverride);
534
+ assert.equal(failedRecovery.exit, 1, `${target} infrastructure failure after a declared miss cannot be masked by the earlier domain outcome`);
535
+ assert.equal(failedRecovery.servedToolCalls, 2, `${target} failed recovery still exercises exactly two model-owned tool calls`);
536
+ assert.deepEqual(failedRecovery.runnerArguments, [
537
+ {
538
+ city: 'Dubai'
539
+ },
540
+ {
541
+ city: 'Singapore'
542
+ }
543
+ ], `${target} failed recovery reaches the revised second adapter invocation`);
544
+ assert.equal(failedRecovery.stdout, '', `${target} failed recovery releases no terminal stdout`);
545
+ assert.match(failedRecovery.stderr, /benchmark terminal output unavailable \(child_bridge_runner_failed\)/, `${target} failed recovery preserves the second runner failure classification`);
546
+ assert.equal(failedRecovery.output.includes('PRIVATE_RECOVERY_RUNNER_DIAGNOSTIC_DO_NOT_REFLECT'), false, `${target} failed recovery never reflects private runner stderr`);
463
547
  const debugRedaction = await runCase('debug-redaction', executableOverride);
464
548
  assert.equal(debugRedaction.exit, 0, `${target} benchmark debug mode preserves successful execution`);
465
549
  assert.equal(debugRedaction.output.includes('PRIVATE_QUERY_SENTINEL_DO_NOT_REFLECT'), false, `${target} benchmark debug output never reflects the private task`);
@@ -729,23 +813,32 @@ async function runConformanceCase(mode, executableOverride) {
729
813
  const runner = join(cwd, 'synthetic-runner.js');
730
814
  const runnerCount = join(cwd, 'runner-count.txt');
731
815
  const configPath = join(cwd, 'benchmark-env-config.json');
732
- writeFileSync(runner, `const fs = require('node:fs'); const path = ${JSON.stringify(runnerCount)}; const count = fs.existsSync(path) ? Number(fs.readFileSync(path, 'utf8')) : 0; fs.writeFileSync(path, String(count + 1)); process.stdout.write(JSON.stringify({ result: { marker: '${MARKER}' } }))`);
816
+ writeFileSync(runner, `const fs = require('node:fs'); const path = ${JSON.stringify(runnerCount)}; const count = fs.existsSync(path) ? Number(fs.readFileSync(path, 'utf8')) : 0; fs.writeFileSync(path, String(count + 1)); process.stdout.write(JSON.stringify({ schema_version: 'gotry_benchmark_tool_result_v1', status: 'ok', result: { marker: '${MARKER}' } }))`);
733
817
  writeFileSync(configPath, JSON.stringify({
734
- schema_version: 'gotry_benchmark_environment_bridge_v2',
818
+ schema_version: 'gotry_benchmark_environment_bridge_v3',
735
819
  enabled: true,
736
820
  executable: process.execPath,
737
821
  cwd,
738
822
  argv_prefix: [
739
823
  runner
740
824
  ],
741
- allowed_tools: [
742
- 'lookup'
825
+ tools: [
826
+ {
827
+ name: 'lookup',
828
+ description: 'Lookup.',
829
+ input_schema: LOOKUP_INPUT_SCHEMA,
830
+ output_keys: [
831
+ 'marker'
832
+ ],
833
+ domain_outcomes: [
834
+ {
835
+ status: 'miss',
836
+ code: 'NOT_FOUND',
837
+ recovery: 'none'
838
+ }
839
+ ]
840
+ }
743
841
  ],
744
- allowed_output_keys: {
745
- lookup: [
746
- 'marker'
747
- ]
748
- },
749
842
  timeout_ms: 2_000,
750
843
  max_output_bytes: 4_096,
751
844
  terminal_output: {
@@ -939,7 +1032,7 @@ if (packaged) {
939
1032
  try {
940
1033
  const configPath = join(missingServiceRoot, 'bridge.json');
941
1034
  writeFileSync(configPath, JSON.stringify({
942
- schema_version: 'gotry_benchmark_environment_bridge_v2',
1035
+ schema_version: 'gotry_benchmark_environment_bridge_v3',
943
1036
  enabled: true,
944
1037
  executable: process.execPath,
945
1038
  cwd: missingServiceRoot,
@@ -947,8 +1040,22 @@ if (packaged) {
947
1040
  '-e',
948
1041
  'process.exit(0)'
949
1042
  ],
950
- allowed_tools: [
951
- 'lookup'
1043
+ tools: [
1044
+ {
1045
+ name: 'lookup',
1046
+ description: 'Lookup.',
1047
+ input_schema: LOOKUP_INPUT_SCHEMA,
1048
+ output_keys: [
1049
+ 'marker'
1050
+ ],
1051
+ domain_outcomes: [
1052
+ {
1053
+ status: 'miss',
1054
+ code: 'NOT_FOUND',
1055
+ recovery: 'none'
1056
+ }
1057
+ ]
1058
+ }
952
1059
  ],
953
1060
  timeout_ms: 100,
954
1061
  max_output_bytes: 4_096,