@danceiny/gotry 0.0.1-rc.19 → 0.0.1-rc.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/README.md +14 -3
  2. package/README.zh-CN.md +15 -4
  3. package/bin/gotry-bootstrap.js +136 -45
  4. package/bin/gotry-inner.js +11 -6
  5. package/cordis.gotry-patch.yml +19 -8
  6. package/dist/capabilities/anything.js +62 -29
  7. package/dist/capabilities/doctor.js +113 -19
  8. package/dist/capabilities/hbcli.js +28 -2
  9. package/dist/capabilities/session/health-watch.js +10 -3
  10. package/dist/scripts/agent-planning-turn-deadline-e2e.js +2 -1
  11. package/dist/scripts/anything-tests.js +70 -21
  12. package/dist/scripts/benchmark-environment-bridge-e2e.js +178 -33
  13. package/dist/scripts/benchmark-environment-bridge-tests.js +1480 -271
  14. package/dist/scripts/booking-copilot-dsh-planner-proof-tests.js +211 -2
  15. package/dist/scripts/booking-copilot-runtime-proof-tests.js +132 -2
  16. package/dist/scripts/bootstrap-tests.js +30 -3
  17. package/dist/scripts/doctor-tests.js +55 -1
  18. package/dist/scripts/hbcli-e2e-tests.js +1 -1
  19. package/dist/scripts/hbcli-tests.js +38 -1
  20. package/dist/scripts/health-watch-cli.js +9 -1
  21. package/dist/scripts/map-tools-vendor-package-proof.js +276 -0
  22. package/dist/scripts/persona-surface-guard-tests.js +8 -3
  23. package/dist/src/benchmark-agent-conformance.js +187 -17
  24. package/dist/src/benchmark-environment-bridge.js +287 -102
  25. package/dist/src/booking-surface/dsh-planner.js +24 -8
  26. package/dist/src/index.js +5 -2
  27. package/extension/manifest.json +2 -2
  28. package/package.json +3 -1
  29. package/ts/capabilities/anything.ts +89 -38
  30. package/ts/capabilities/hbcli.ts +53 -4
  31. package/ts/capabilities/session/health-watch.ts +9 -2
  32. package/ts/dsh-runtime/vendor/README.md +60 -0
  33. package/ts/dsh-runtime/vendor/dsh-map-tools/LICENSE +21 -0
  34. package/ts/dsh-runtime/vendor/dsh-map-tools/README.en.md +200 -0
  35. package/ts/dsh-runtime/vendor/dsh-map-tools/README.md +202 -0
  36. package/ts/dsh-runtime/vendor/dsh-map-tools/client/client.js +216 -0
  37. package/ts/dsh-runtime/vendor/dsh-map-tools/cordis.patch.yml +5 -0
  38. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/clients/amap.js +371 -0
  39. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/clients/nominatim.js +63 -0
  40. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/clients/osrm.js +56 -0
  41. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/clients/photon.js +55 -0
  42. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/config-file.js +87 -0
  43. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/config-route.js +126 -0
  44. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/config.js +16 -0
  45. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/index.js +109 -0
  46. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/settings-ns.js +27 -0
  47. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/tools/geocode.js +127 -0
  48. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/tools/poi.js +84 -0
  49. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/tools/routes.js +170 -0
  50. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/clients/amap.d.ts +53 -0
  51. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/clients/nominatim.d.ts +17 -0
  52. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/clients/osrm.d.ts +15 -0
  53. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/clients/photon.d.ts +22 -0
  54. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/config-file.d.ts +33 -0
  55. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/config-route.d.ts +20 -0
  56. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/config.d.ts +23 -0
  57. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/index.d.ts +18 -0
  58. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/settings-ns.d.ts +14 -0
  59. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/tools/geocode.d.ts +11 -0
  60. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/tools/poi.d.ts +9 -0
  61. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/tools/routes.d.ts +16 -0
  62. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types/types.d.ts +55 -0
  63. package/ts/dsh-runtime/vendor/dsh-map-tools/lib/types.js +16 -0
  64. package/ts/dsh-runtime/vendor/dsh-map-tools/package.json +91 -0
  65. package/ts/package.json +17 -1
  66. package/ts/scripts/map-tools-vendor-package-proof.ts +268 -0
  67. package/ts/src/benchmark-agent-conformance.ts +175 -13
  68. package/ts/src/benchmark-environment-bridge.ts +220 -66
  69. package/ts/src/booking-surface/dsh-planner.ts +25 -13
  70. package/ts/src/index.ts +4 -2
  71. package/ts/src/turn-deadline.ts +4 -0
@@ -0,0 +1,276 @@
1
+ import { execFileSync } from 'node:child_process';
2
+ import { createHash } from 'node:crypto';
3
+ import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, rmSync, symlinkSync } from 'node:fs';
4
+ import { createRequire } from 'node:module';
5
+ import { basename, dirname, isAbsolute, join, resolve } from 'node:path';
6
+ import { fileURLToPath, pathToFileURL } from 'node:url';
7
+ import assert from 'node:assert/strict';
8
+ const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), '../..');
9
+ const upstreamLicenseSha256 = 'b6bbb0c73a02cf8d2c304e9f208b41c32f0a810fabe366986e2b14ac3338f618';
10
+ const upstreamVendorFileCount = 32;
11
+ const upstreamVendorAggregateSha256 = 'd82a38adace8bfe574dcecb4222da5c994a1b9e8792f78dc2dad3a7f50c9925a';
12
+ const expectedTools = [
13
+ 'map_bicycling_route',
14
+ 'map_driving_route',
15
+ 'map_geocode',
16
+ 'map_poi_search',
17
+ 'map_reverse_geocode',
18
+ 'map_transit_route',
19
+ 'map_walking_route'
20
+ ];
21
+ function sha256(path) {
22
+ return createHash('sha256').update(readFileSync(path)).digest('hex');
23
+ }
24
+ function vendorAggregate(root) {
25
+ const files = [];
26
+ const walk = (directory)=>{
27
+ for (const entry of readdirSync(directory, {
28
+ withFileTypes: true
29
+ })){
30
+ const path = join(directory, entry.name);
31
+ if (entry.isDirectory()) walk(path);
32
+ else files.push(path.slice(root.length + 1).replaceAll('\\', '/'));
33
+ }
34
+ };
35
+ walk(root);
36
+ files.sort();
37
+ const digest = createHash('sha256');
38
+ for (const relative of files){
39
+ digest.update(relative);
40
+ digest.update('\0');
41
+ digest.update(readFileSync(join(root, relative)));
42
+ digest.update('\0');
43
+ }
44
+ return {
45
+ count: files.length,
46
+ sha256: digest.digest('hex')
47
+ };
48
+ }
49
+ function assertAlpha3LockClosure() {
50
+ const manifest = JSON.parse(readFileSync(join(repoRoot, 'ts/package.json'), 'utf8'));
51
+ assert.equal(manifest.dependencies?.['dsh-map-tools'], undefined, 'external map-tools dependency must stay removed');
52
+ const lock = JSON.parse(readFileSync(join(repoRoot, 'ts/package-lock.json'), 'utf8'));
53
+ assert.equal(lock.packages?.['']?.dependencies?.['dsh-map-tools'], undefined, 'lock root must not resolve external map-tools');
54
+ assert.equal(lock.packages?.['node_modules/dsh-map-tools'], undefined, 'lock must not carry external map-tools');
55
+ const dshEntries = Object.entries(lock.packages ?? {}).filter(([path])=>/(?:^|\/)node_modules\/@deepseek-ai\/(?:dsh|dsh-[^/]+)$/.test(path));
56
+ assert.ok(dshEntries.length >= 10, 'expected dsh-tools peer closure in the ts lock');
57
+ for (const [path, entry] of dshEntries){
58
+ assert.equal(entry.version, '0.1.2-alpha.3', `${path} drifted outside the alpha.3 closure`);
59
+ }
60
+ return {
61
+ packages: dshEntries.length,
62
+ version: '0.1.2-alpha.3'
63
+ };
64
+ }
65
+ function findPackageRoot(entry) {
66
+ let cursor = dirname(entry);
67
+ for(;;){
68
+ const manifestPath = join(cursor, 'package.json');
69
+ if (existsSync(manifestPath)) {
70
+ const manifest = JSON.parse(readFileSync(manifestPath, 'utf8'));
71
+ if (manifest.name === '@danceiny/gotry') return cursor;
72
+ }
73
+ const parent = dirname(cursor);
74
+ if (parent === cursor) throw new Error(`cannot locate @danceiny/gotry package root from ${entry}`);
75
+ cursor = parent;
76
+ }
77
+ }
78
+ function assertBinTargetsPackage(binPath, expectedTarget) {
79
+ const resolvedExpectedTarget = realpathSync(expectedTarget);
80
+ if (realpathSync(binPath) === resolvedExpectedTarget) return;
81
+ const shim = readFileSync(binPath, 'utf8');
82
+ assert.ok(Buffer.byteLength(shim, 'utf8') <= 32_768, 'clean-install gotry shim is unexpectedly large');
83
+ const lines = shim.trimEnd().split(/\r?\n/);
84
+ const markers = lines.filter((line)=>line.startsWith('# cmd-shim-target='));
85
+ assert.equal(markers.length, 1, 'pnpm gotry shim must contain exactly one target marker');
86
+ assert.equal(lines.at(-1), markers[0], 'pnpm gotry shim target marker must be the final line');
87
+ const markerTarget = markers[0].slice('# cmd-shim-target='.length);
88
+ assert.ok(isAbsolute(markerTarget), 'pnpm gotry shim target must be absolute');
89
+ assert.equal(realpathSync(markerTarget), resolvedExpectedTarget, 'GOTRY_MAP_TOOLS_E2E_BIN shim must target this installed GoTry package');
90
+ }
91
+ function prepareProofPackage(cleanInstalledBin) {
92
+ if (cleanInstalledBin) {
93
+ const binPath = resolve(cleanInstalledBin);
94
+ assert.equal(basename(binPath), 'gotry', 'clean-install executable must use the gotry bin name');
95
+ assert.equal(basename(dirname(binPath)), '.bin', 'clean-install executable must come from node_modules/.bin');
96
+ assert.equal(basename(dirname(dirname(binPath))), 'node_modules', 'clean-install executable must come from a consumer install');
97
+ const consumerRoot = dirname(dirname(dirname(binPath)));
98
+ const packageMain = createRequire(join(consumerRoot, 'package.json')).resolve('@danceiny/gotry');
99
+ const packageRoot = findPackageRoot(packageMain);
100
+ const installedManifest = JSON.parse(readFileSync(join(packageRoot, 'package.json'), 'utf8'));
101
+ assert.equal(installedManifest.name, '@danceiny/gotry', 'clean-install executable must belong to GoTry');
102
+ const declaredBin = typeof installedManifest.bin === 'string' ? installedManifest.bin : installedManifest.bin?.gotry;
103
+ assert.equal(declaredBin, 'bin/gotry-inner.js', 'installed package must declare the production gotry bin');
104
+ assertBinTargetsPackage(binPath, join(packageRoot, declaredBin));
105
+ return {
106
+ packageRoot
107
+ };
108
+ }
109
+ const proofRoot = mkdtempSync(join(repoRoot, '.tmp-map-tools-proof-'));
110
+ try {
111
+ const packOutput = execFileSync('npm', [
112
+ 'pack',
113
+ '--ignore-scripts',
114
+ '--json',
115
+ '--pack-destination',
116
+ proofRoot
117
+ ], {
118
+ cwd: repoRoot,
119
+ encoding: 'utf8'
120
+ });
121
+ const packInfo = JSON.parse(packOutput);
122
+ assert.equal(packInfo.length, 1, 'npm pack must produce one artifact');
123
+ const tarball = join(proofRoot, packInfo[0].filename);
124
+ assert.ok(existsSync(tarball), `packed artifact missing: ${tarball}`);
125
+ const unpackRoot = join(proofRoot, 'unpacked');
126
+ mkdirSync(unpackRoot);
127
+ execFileSync('tar', [
128
+ '-xzf',
129
+ tarball,
130
+ '-C',
131
+ unpackRoot
132
+ ]);
133
+ const packageRoot = join(unpackRoot, 'package');
134
+ symlinkSync(join(repoRoot, 'node_modules'), join(packageRoot, 'node_modules'), process.platform === 'win32' ? 'junction' : 'dir');
135
+ return {
136
+ packageRoot,
137
+ proofRoot
138
+ };
139
+ } catch (error) {
140
+ rmSync(proofRoot, {
141
+ recursive: true,
142
+ force: true
143
+ });
144
+ throw error;
145
+ }
146
+ }
147
+ const cleanInstalledBin = process.env.GOTRY_MAP_TOOLS_E2E_BIN;
148
+ const { packageRoot, proofRoot } = prepareProofPackage(cleanInstalledBin);
149
+ try {
150
+ const lockedDsh = assertAlpha3LockClosure();
151
+ const vendorRoot = join(packageRoot, 'ts/dsh-runtime/vendor/dsh-map-tools');
152
+ const requiredFiles = [
153
+ 'LICENSE',
154
+ 'package.json',
155
+ 'cordis.patch.yml',
156
+ 'lib/index.js',
157
+ 'lib/settings-ns.js',
158
+ 'lib/config.js',
159
+ 'lib/tools/geocode.js',
160
+ 'lib/tools/routes.js',
161
+ 'lib/tools/poi.js'
162
+ ];
163
+ for (const relative of requiredFiles){
164
+ assert.ok(existsSync(join(vendorRoot, relative)), `required vendor file missing: ${relative}`);
165
+ }
166
+ const vendorTree = vendorAggregate(vendorRoot);
167
+ assert.equal(vendorTree.count, upstreamVendorFileCount, 'vendored payload file count drifted');
168
+ assert.equal(vendorTree.sha256, upstreamVendorAggregateSha256, 'vendored payload aggregate drifted');
169
+ const vendorPackage = JSON.parse(readFileSync(join(vendorRoot, 'package.json'), 'utf8'));
170
+ assert.equal(vendorPackage.name, 'dsh-map-tools');
171
+ assert.equal(vendorPackage.version, '0.5.1');
172
+ assert.equal(vendorPackage.license, 'MIT');
173
+ assert.equal(sha256(join(vendorRoot, 'LICENSE')), upstreamLicenseSha256, 'upstream MIT license changed');
174
+ const packageRequire = createRequire(join(packageRoot, 'package.json'));
175
+ const dshTools = JSON.parse(readFileSync(packageRequire.resolve('@deepseek-ai/dsh-tools/package.json'), 'utf8'));
176
+ const dshSettings = JSON.parse(readFileSync(packageRequire.resolve('@deepseek-ai/dsh-settings/package.json'), 'utf8'));
177
+ assert.equal(dshTools.version, '0.1.2-alpha.3');
178
+ assert.equal(dshSettings.version, '0.1.2-alpha.3');
179
+ const settingsApi = await import(pathToFileURL(packageRequire.resolve('@deepseek-ai/dsh-settings')).href);
180
+ assert.equal(typeof settingsApi.SettingsProvider?.prototype?.installSection, 'function', 'real alpha.3 SettingsProvider.installSection API must be present');
181
+ const plugin = await import(pathToFileURL(join(vendorRoot, 'lib/index.js')).href);
182
+ const registered = [];
183
+ const settingsCalls = [];
184
+ const context = {
185
+ tools: {
186
+ register (tool) {
187
+ registered.push(tool);
188
+ return ()=>undefined;
189
+ }
190
+ },
191
+ effect (effect) {
192
+ effect();
193
+ },
194
+ inject (dependencies, callback) {
195
+ if (dependencies.includes('settings')) {
196
+ callback({
197
+ settings: {
198
+ installSection: (...args)=>{
199
+ settingsCalls.push(args);
200
+ }
201
+ }
202
+ });
203
+ } else if (dependencies.includes('webServer')) {
204
+ callback({
205
+ webServer: {
206
+ register: ()=>undefined
207
+ }
208
+ });
209
+ } else {
210
+ throw new Error(`unexpected dependency injection: ${dependencies.join(',')}`);
211
+ }
212
+ }
213
+ };
214
+ plugin.apply(context, {
215
+ provider: 'osm',
216
+ amapKey: '',
217
+ timeoutMs: 1000,
218
+ maxQps: 2,
219
+ defaultMode: 'driving',
220
+ language: 'zh'
221
+ });
222
+ assert.deepEqual(registered.map((tool)=>tool.name).sort(), expectedTools, 'exactly seven map tools must register');
223
+ assert.equal(settingsCalls.length, 1, 'alpha.3 settings section must be installed once');
224
+ assert.equal(settingsCalls[0][1], 'dsh-map-tools', 'settings namespace must be stable');
225
+ assert.equal(typeof settingsCalls[0][4], 'object', 'alpha.3 installSection hooks must be supplied');
226
+ const originalFetch = globalThis.fetch;
227
+ globalThis.fetch = async ()=>{
228
+ throw new Error('network forbidden by package proof');
229
+ };
230
+ try {
231
+ const geocode = registered.find((tool)=>tool.name === 'map_geocode');
232
+ assert.ok(geocode, 'map_geocode must be registered');
233
+ const result = await geocode.execute({
234
+ address: '116.397428,39.90923'
235
+ }, {
236
+ signal: new AbortController().signal
237
+ });
238
+ assert.equal(result.provider, 'inline');
239
+ assert.deepEqual(result.location, [
240
+ 116.397428,
241
+ 39.90923
242
+ ]);
243
+ } finally{
244
+ globalThis.fetch = originalFetch;
245
+ }
246
+ console.log(JSON.stringify({
247
+ proof: 'map-tools-vendor-package',
248
+ status: 'PASS',
249
+ artifactMode: cleanInstalledBin ? 'clean-installed-tarball' : 'packed-unpacked-focused',
250
+ vendored: {
251
+ sourcePackage: 'dsh-map-tools@0.5.1',
252
+ licenseSha256: upstreamLicenseSha256,
253
+ vendorFileCount: vendorTree.count,
254
+ vendorAggregateSha256: vendorTree.sha256
255
+ },
256
+ artifactVendor: vendorRoot,
257
+ dshClosure: {
258
+ tools: dshTools.version,
259
+ settings: dshSettings.version,
260
+ ...lockedDsh
261
+ },
262
+ tools: registered.map((tool)=>tool.name).sort(),
263
+ inlineCoordinate: {
264
+ provider: 'inline',
265
+ network: 'forbidden'
266
+ }
267
+ }, null, 2));
268
+ } finally{
269
+ if (proofRoot) rmSync(proofRoot, {
270
+ recursive: true,
271
+ force: true
272
+ });
273
+ }
274
+
275
+
276
+ //# sourceURL=ts/scripts/map-tools-vendor-package-proof.ts
@@ -8,12 +8,17 @@ assert.ok(patch.includes('绝不把 gotry_* 名字传给 skill 加载器'), '应
8
8
  const markers = new Set([
9
9
  ...patch.matchAll(/\((\d{1,2})\)/g)
10
10
  ].map((m)=>Number(m[1])));
11
- for(let i = 1; i <= 22; i += 1){
11
+ for(let i = 1; i <= 23; i += 1){
12
12
  assert.ok(markers.has(i), `行为契约 (${i}) 应存在`);
13
13
  }
14
- assert.equal(markers.size, 22, `契约条目应恰为 22 条,实际 ${markers.size}`);
14
+ assert.equal(markers.size, 23, `契约条目应恰为 23 条,实际 ${markers.size}`);
15
15
  assert.ok(patch.includes('改回 tool call'), '应指引失败后改回 tool call');
16
- console.log('PERSONA SURFACE GUARD TESTS: 3/3 OK(表层规则句 / 22 条契约完整 / skill 失败行为指引)');
16
+ assert.ok(patch.includes('澄清卡/访谈/选项里向用户列出的候选时段示例'), '#2 回归:候选时段示例条款应存在');
17
+ assert.ok(patch.includes('已过的节日不进示例枚举'), '应禁止已过节日进澄清卡示例枚举');
18
+ assert.ok(patch.includes('(23)子任务等待纪律'), '#194 回归:子任务等待纪律条款应存在');
19
+ assert.ok(patch.includes('对子代理绝不调用 job_output/job_kill'), '应禁止对子代理调用 job_output/job_kill');
20
+ assert.ok(patch.includes("job_output/job_kill 只用于"), '应限定 job_output/job_kill 的适用面');
21
+ console.log('PERSONA SURFACE GUARD TESTS: 5/5 OK(表层规则句 / 23 条契约完整 / skill 失败行为指引 / 澄清卡示例过锚点卡 / 子任务等待纪律)');
17
22
 
18
23
 
19
24
  //# sourceURL=ts/scripts/persona-surface-guard-tests.ts
@@ -12,6 +12,26 @@ export const BENCHMARK_BRIDGE_RETRY_CALL_NOT_ALLOWED = 'BENCHMARK_BRIDGE_RETRY_C
12
12
  export const BENCHMARK_CONFORMANCE_STATE_UNAVAILABLE = 'BENCHMARK_CONFORMANCE_STATE_UNAVAILABLE';
13
13
  const IDENTIFIER = /^[A-Za-z][A-Za-z0-9_.-]*$/;
14
14
  const MAX_TERMINAL_BYTES = 1024 * 1024;
15
+ const BENCHMARK_TOOL_RESULT_SCHEMA_VERSION = 'gotry_benchmark_tool_result_v1';
16
+ const BENCHMARK_DOMAIN_RECOVERIES = new Set([
17
+ 'none',
18
+ 'retry_same',
19
+ 'revise_arguments',
20
+ 'choose_alternative'
21
+ ]);
22
+ const TERMINAL_BODY_SCHEMA_MAX_BYTES = 16 * 1024;
23
+ const TERMINAL_BODY_SCHEMA_MAX_DEPTH = 8;
24
+ const TERMINAL_BODY_SCHEMA_MAX_NODES = 256;
25
+ const TERMINAL_BODY_SCHEMA_MAX_PROPERTIES = 256;
26
+ const TERMINAL_BODY_SCALAR_TYPES = new Set([
27
+ 'string',
28
+ 'number',
29
+ 'integer',
30
+ 'boolean',
31
+ 'null'
32
+ ]);
33
+ const TERMINAL_BODY_VALUE_MAX_NODES = 10_000;
34
+ const TERMINAL_BODY_VALUE_MAX_DEPTH = 24;
15
35
  function plainObject(value) {
16
36
  return typeof value === 'object' && value !== null && !Array.isArray(value);
17
37
  }
@@ -19,9 +39,143 @@ export function validateTerminalOutputConfig(value) {
19
39
  if (!plainObject(value)) return false;
20
40
  const keys = Object.keys(value).sort();
21
41
  return JSON.stringify(keys) === JSON.stringify([
42
+ 'body_schema',
22
43
  'max_bytes',
23
44
  'tag'
24
- ]) && typeof value.tag === 'string' && IDENTIFIER.test(value.tag) && typeof value.max_bytes === 'number' && Number.isInteger(value.max_bytes) && value.max_bytes >= 1 && value.max_bytes <= MAX_TERMINAL_BYTES;
45
+ ]) && typeof value.tag === 'string' && IDENTIFIER.test(value.tag) && typeof value.max_bytes === 'number' && Number.isInteger(value.max_bytes) && value.max_bytes >= 1 && value.max_bytes <= MAX_TERMINAL_BYTES && validateTerminalBodySchema(value.body_schema);
46
+ }
47
+ export function validateTerminalBodySchema(value) {
48
+ let serialized;
49
+ try {
50
+ serialized = JSON.stringify(value);
51
+ } catch {
52
+ return false;
53
+ }
54
+ if (typeof serialized !== 'string' || Buffer.byteLength(serialized, 'utf8') > TERMINAL_BODY_SCHEMA_MAX_BYTES) return false;
55
+ const pending = [
56
+ {
57
+ node: value,
58
+ depth: 0
59
+ }
60
+ ];
61
+ const seen = new Set();
62
+ let nodes = 0;
63
+ let properties = 0;
64
+ while(pending.length > 0){
65
+ const { node, depth } = pending.pop();
66
+ if (!plainObject(node) || seen.has(node) || ++nodes > TERMINAL_BODY_SCHEMA_MAX_NODES || depth > TERMINAL_BODY_SCHEMA_MAX_DEPTH) return false;
67
+ seen.add(node);
68
+ const type = node.type;
69
+ if (type === 'object') {
70
+ if (!exactKeys(node, [
71
+ 'type',
72
+ 'properties',
73
+ 'required',
74
+ 'additionalProperties'
75
+ ]) || !plainObject(node.properties) || !Array.isArray(node.required) || node.additionalProperties !== false) return false;
76
+ const entries = Object.entries(node.properties);
77
+ properties += entries.length;
78
+ if (properties > TERMINAL_BODY_SCHEMA_MAX_PROPERTIES || entries.some(([key, child])=>key.length === 0 || key.length > 64 || !plainObject(child))) return false;
79
+ if (node.required.length > entries.length || new Set(node.required).size !== node.required.length || node.required.some((key)=>typeof key !== 'string' || !Object.hasOwn(node.properties, key))) return false;
80
+ for (const [, child] of entries)pending.push({
81
+ node: child,
82
+ depth: depth + 1
83
+ });
84
+ continue;
85
+ }
86
+ if (type === 'array') {
87
+ if (!exactKeys(node, [
88
+ 'type',
89
+ 'items'
90
+ ]) || !plainObject(node.items)) return false;
91
+ pending.push({
92
+ node: node.items,
93
+ depth: depth + 1
94
+ });
95
+ continue;
96
+ }
97
+ if (typeof type !== 'string' || !TERMINAL_BODY_SCALAR_TYPES.has(type) || !exactKeys(node, [
98
+ 'type'
99
+ ])) return false;
100
+ }
101
+ return plainObject(value) && value.type === 'object';
102
+ }
103
+ function exactKeys(value, allowed) {
104
+ return Object.keys(value).every((key)=>allowed.includes(key));
105
+ }
106
+ export function validateTerminalBodyValue(value, schema) {
107
+ const pending = [
108
+ {
109
+ value,
110
+ schema,
111
+ depth: 0
112
+ }
113
+ ];
114
+ let nodes = 0;
115
+ while(pending.length > 0){
116
+ const current = pending.pop();
117
+ if (++nodes > TERMINAL_BODY_VALUE_MAX_NODES || current.depth > TERMINAL_BODY_VALUE_MAX_DEPTH) return false;
118
+ const { schema: node } = current;
119
+ if (node.type === 'object') {
120
+ const record = current.value;
121
+ if (!plainObject(record)) return false;
122
+ const properties = node.properties;
123
+ for (const key of node.required){
124
+ if (!Object.hasOwn(record, key)) return false;
125
+ }
126
+ for (const [key, child] of Object.entries(record)){
127
+ const childSchema = properties[key];
128
+ if (!childSchema) return false;
129
+ pending.push({
130
+ value: child,
131
+ schema: childSchema,
132
+ depth: current.depth + 1
133
+ });
134
+ }
135
+ continue;
136
+ }
137
+ if (node.type === 'array') {
138
+ if (!Array.isArray(current.value)) return false;
139
+ for (const item of current.value)pending.push({
140
+ value: item,
141
+ schema: node.items,
142
+ depth: current.depth + 1
143
+ });
144
+ continue;
145
+ }
146
+ if (node.type === 'string') {
147
+ if (typeof current.value !== 'string') return false;
148
+ continue;
149
+ }
150
+ if (node.type === 'boolean') {
151
+ if (typeof current.value !== 'boolean') return false;
152
+ continue;
153
+ }
154
+ if (node.type === 'null') {
155
+ if (current.value !== null) return false;
156
+ continue;
157
+ }
158
+ if (node.type === 'integer') {
159
+ if (typeof current.value !== 'number' || !Number.isSafeInteger(current.value)) return false;
160
+ continue;
161
+ }
162
+ if (typeof current.value !== 'number' || !Number.isFinite(current.value)) return false;
163
+ }
164
+ return true;
165
+ }
166
+ export function terminalSchemaOutline(schema) {
167
+ const render = (node, depth)=>{
168
+ if (depth > TERMINAL_BODY_SCHEMA_MAX_DEPTH) return '…';
169
+ if (node.type === 'object') {
170
+ const properties = node.properties;
171
+ const required = new Set(node.required);
172
+ const parts = Object.entries(properties).map(([key, child])=>`${required.has(key) ? '' : '?'}${key}:${render(child, depth + 1)}`);
173
+ return `object{${parts.join(',')}}`;
174
+ }
175
+ if (node.type === 'array') return `array<${render(node.items, depth + 1)}>`;
176
+ return node.type;
177
+ };
178
+ return render(schema, 0);
25
179
  }
26
180
  function invalidTerminal() {
27
181
  return {
@@ -44,10 +198,12 @@ export function parseBenchmarkTerminal(raw, config) {
44
198
  if (body.length === 0 || body.includes('```')) return invalidTerminal();
45
199
  try {
46
200
  const value = JSON.parse(body);
47
- return plainObject(value) ? {
201
+ if (!plainObject(value)) return invalidTerminal();
202
+ if (!validateTerminalBodyValue(value, config.body_schema)) return invalidTerminal();
203
+ return {
48
204
  ok: true,
49
205
  value
50
- } : invalidTerminal();
206
+ };
51
207
  } catch {
52
208
  return invalidTerminal();
53
209
  }
@@ -90,13 +246,23 @@ function bridgeResultStatus(data, callId) {
90
246
  try {
91
247
  const parsed = JSON.parse(text);
92
248
  if (!plainObject(parsed)) return undefined;
93
- if (parsed.ok === true) return {
94
- ok: true
249
+ const keys = Object.keys(parsed).sort().join(',');
250
+ if (parsed.ok === true && keys === 'ok,result' && (plainObject(parsed.result) || Array.isArray(parsed.result))) return {
251
+ kind: 'result'
95
252
  };
253
+ if (parsed.ok === true && keys === 'ok,outcome' && plainObject(parsed.outcome)) {
254
+ const outcomeKeys = Object.keys(parsed.outcome).sort().join(',');
255
+ if (outcomeKeys === 'code,recovery,schema_version,status' && parsed.outcome.schema_version === BENCHMARK_TOOL_RESULT_SCHEMA_VERSION && (parsed.outcome.status === 'miss' || parsed.outcome.status === 'error') && typeof parsed.outcome.code === 'string' && IDENTIFIER.test(parsed.outcome.code) && typeof parsed.outcome.recovery === 'string' && BENCHMARK_DOMAIN_RECOVERIES.has(parsed.outcome.recovery)) return {
256
+ kind: 'domain'
257
+ };
258
+ return undefined;
259
+ }
96
260
  if (parsed.ok === false && typeof parsed.error === 'string') {
261
+ const exactFailure = parsed.error === 'runner_failed' ? keys === 'error,ok' || keys === 'error,exit_code,ok,signal' && (parsed.exit_code === null || typeof parsed.exit_code === 'number' && Number.isInteger(parsed.exit_code)) && (parsed.signal === null || typeof parsed.signal === 'string') : parsed.error === 'invalid_arguments' ? keys === 'error,ok,reason' && typeof parsed.reason === 'string' : keys === 'error,ok';
262
+ if (!exactFailure) return undefined;
97
263
  const code = parsed.error === 'timed_out' ? BENCHMARK_BRIDGE_TIMED_OUT : parsed.error === 'runner_failed' ? BENCHMARK_BRIDGE_RUNNER_FAILED : parsed.error === 'spawn_failed' ? BENCHMARK_BRIDGE_SPAWN_FAILED : parsed.error === 'output_truncated' ? BENCHMARK_BRIDGE_OUTPUT_TRUNCATED : BENCHMARK_BRIDGE_CALL_FAILED;
98
264
  return {
99
- ok: false,
265
+ kind: 'failure',
100
266
  code
101
267
  };
102
268
  }
@@ -164,8 +330,11 @@ export function createBenchmarkAgentConformance(projection) {
164
330
  const callId = event.data.message.source.callId;
165
331
  if (typeof callId !== 'string' || !state.validCallIds.has(callId)) return;
166
332
  const resultStatus = bridgeResultStatus(event.data, callId);
167
- if (resultStatus?.ok === true) {
333
+ if (typeof event.data.step === 'number') state.lastBridgeResponseStep = event.data.step;
334
+ if (resultStatus?.kind === 'result') {
168
335
  state.successfulResultStep = typeof event.data.step === 'number' ? event.data.step : state.successfulResultStep;
336
+ } else if (resultStatus?.kind === 'domain') {
337
+ state.domainOutcomeStep = typeof event.data.step === 'number' ? event.data.step : state.domainOutcomeStep;
169
338
  } else {
170
339
  state.bridgeCallFailed = true;
171
340
  state.bridgeFailureCode = resultStatus?.code ?? BENCHMARK_BRIDGE_CALL_FAILED;
@@ -194,13 +363,13 @@ export function createBenchmarkAgentConformance(projection) {
194
363
  code: BENCHMARK_BRIDGE_RETRY_CALL_NOT_ALLOWED
195
364
  };
196
365
  }
197
- if (state.successfulResultStep === undefined) {
198
- if (state.bridgeCallFailed) {
199
- return {
200
- kind: 'reject',
201
- code: state.bridgeFailureCode ?? BENCHMARK_BRIDGE_CALL_FAILED
202
- };
203
- }
366
+ if (state.bridgeCallFailed && state.successfulResultStep === undefined) {
367
+ return {
368
+ kind: 'reject',
369
+ code: state.bridgeFailureCode ?? BENCHMARK_BRIDGE_CALL_FAILED
370
+ };
371
+ }
372
+ if (state.successfulResultStep === undefined && state.domainOutcomeStep === undefined) {
204
373
  if (state.retryCount >= MAX_CONFORMANCE_RETRIES) {
205
374
  return {
206
375
  kind: 'reject',
@@ -215,7 +384,7 @@ export function createBenchmarkAgentConformance(projection) {
215
384
  };
216
385
  }
217
386
  const terminal = state.lastAssistant;
218
- const terminalIsValid = terminal !== undefined && !terminal.interrupted && terminal.step > state.successfulResultStep && parseBenchmarkTerminal(terminal.text, projection.terminal).ok;
387
+ const terminalIsValid = terminal !== undefined && !terminal.interrupted && terminal.step > state.lastBridgeResponseStep && parseBenchmarkTerminal(terminal.text, projection.terminal).ok;
219
388
  if (terminalIsValid) return {
220
389
  kind: 'accept'
221
390
  };
@@ -243,7 +412,7 @@ function requireDisposer(value, capability) {
243
412
  return value;
244
413
  }
245
414
  function correctionMessage(mode, projection) {
246
- const text = mode === 'call' ? `BENCHMARK_CONFORMANCE_CALL: Execute exactly one native ${projection.toolName} action:"call" now. Describing an intended CLI, shell, or Python command does not execute it.` : `BENCHMARK_CONFORMANCE_TERMINAL: Reuse the existing successful tool result. Do not call any tool. Reply only <${projection.terminal.tag}>{"result":"..."}</${projection.terminal.tag}> with one JSON object.`;
415
+ const text = mode === 'call' ? `BENCHMARK_CONFORMANCE_CALL: Execute exactly one native ${projection.toolName} action:"call" now. Describing an intended CLI, shell, or Python command does not execute it.` : `BENCHMARK_CONFORMANCE_TERMINAL: Reuse the existing successful tool result. Do not call any tool. Reply only <${projection.terminal.tag}> with one JSON object matching exactly ${terminalSchemaOutline(projection.terminal.body_schema)} no extra keys, no missing keys, exact types.`;
247
416
  return createUserMessage({
248
417
  content: [
249
418
  {
@@ -259,6 +428,7 @@ function correctionMessage(mode, projection) {
259
428
  }
260
429
  function systemSection(projection) {
261
430
  const allowed = projection.allowedTools.join(', ');
431
+ const outline = terminalSchemaOutline(projection.terminal.body_schema);
262
432
  return {
263
433
  name: 'benchmark:agent-conformance',
264
434
  text: [
@@ -266,7 +436,7 @@ function systemSection(projection) {
266
436
  `- Translate every task instruction to use a CLI, shell, Python, or agent_env.cli into the native tool ${projection.toolName}; do not merely describe the intended command.`,
267
437
  `- Call it with exactly {"action":"call","tool":"<one of: ${allowed}>","arguments":{...}}.`,
268
438
  '- action:"tools" is discovery only and does not satisfy the required environment call.',
269
- `- After a successful tool result, reply only <${projection.terminal.tag}>{...one JSON object...}</${projection.terminal.tag}> with no prose or code fence.`,
439
+ `- After a successful tool result, reply only <${projection.terminal.tag}> with one JSON object matching exactly ${outline} no extra keys, no missing keys, exact types; no prose or code fence.`,
270
440
  '- If a terminal-format correction arrives, reuse the existing result and do not call the tool again.'
271
441
  ].join('\n')
272
442
  };