@danceiny/gotry 0.0.1-rc.19 → 0.0.1-rc.21

package/dist/scripts/benchmark-environment-bridge-e2e.js

@@ -12,6 +12,22 @@ const BIN = join(ROOT, 'bin', 'gotry-inner.js');
 const TOOL = 'gotry_benchmark_environment';
 const MARKER = 'BENCHMARK_BRIDGE_LOOKUP_OK';
 const TIMEOUT_MS = 30_000;
+const LOOKUP_INPUT_SCHEMA = {
+    type: 'object',
+    properties: {
+        city: {
+            type: 'string',
+            enum: [
+                'Dubai',
+                'Singapore'
+            ]
+        }
+    },
+    required: [
+        'city'
+    ],
+    additionalProperties: false
+};
 const TSX_LOADER = pathToFileURL(createRequire(import.meta.url).resolve('tsx')).href;
 function runRuntimeProbe(options) {
     const fixture = mkdtempSync(join(tmpdir(), 'gotry-runtime-probe-'));
@@ -61,7 +77,7 @@ function assertRuntimeSelectionAndVersionGuards() {
     const legacyFallback = runRuntimeProbe({
         vendorVersion: '0.1.2-alpha.1'
     });
-    assert.equal(legacyFallback, null, 'D-27 removal (#120): legacy vendored fallback no longer resolves even outside benchmark — fail-closed');
+    assert.deepEqual(legacyFallback, null, 'non-benchmark source checkout fail-closes instead of using the removed legacy vendored dsh fallback');
     const wrongBenchmarkVersion = runRuntimeProbe({
         rootVersion: '0.1.2-alpha.1',
         vendorVersion: '0.1.2-alpha.1',
@@ -183,7 +199,7 @@ function finalText(text) {
         ]
     }) + 'data: [DONE]\n\n';
 }
-function toolCall(callId = 'bridge-call-1') {
+function toolCall(callId = 'bridge-call-1', city = 'Dubai') {
     return sse({
         id: `bridge-${callId}`,
         object: 'chat.completion.chunk',
@@ -202,7 +218,7 @@ function toolCall(callId = 'bridge-call-1') {
                                     action: 'call',
                                     tool: 'lookup',
                                     arguments: {
-                                        city: 'Dubai'
+                                        city
                                     }
                                 })
                             }
@@ -235,9 +251,25 @@ function toolResultPresent(body) {
 function anyToolResultPresent(body) {
     return (body.messages ?? []).some((m)=>m.role === 'tool');
 }
+const BRIDGE_E2E_BODY_SCHEMA = {
+    type: 'object',
+    properties: {
+        status: {
+            type: 'string'
+        },
+        payload: {
+            type: 'string'
+        }
+    },
+    required: [],
+    additionalProperties: false
+};
+const TERMINAL_OUTLINE = 'object{?status:string,?payload:string}';
 async function runCase(mode, executableOverride, extraEnv = {}) {
     const requests = [];
+    let servedToolCalls = 0;
     let spawnTarget = '';
+    const domainRecoveryMode = mode === 'domain-recovery' || mode === 'domain-recovery-failed';
     const server = createServer((req, res)=>{
         const chunks = [];
         req.on('data', (c)=>chunks.push(Buffer.from(c)));
@@ -253,8 +285,15 @@ async function runCase(mode, executableOverride, extraEnv = {}) {
             if (mode === 'spawn-failed' && names(body).includes(TOOL) && !anyToolResultPresent(body) && spawnTarget) rmSync(spawnTarget, {
                 force: true
             });
-            if (mode !== 'disabled' && mode !== 'invalid-path' && mode !== 'invalid-schema' && mode !== 'unsafe-config' && names(body).includes(TOOL) && !anyToolResultPresent(body)) res.end(toolCall());
-            else res.end(finalText(mode === 'enabled' ? '<benchmark_terminal>{"status":"succeeded"}</benchmark_terminal>' : '<benchmark_terminal>{"status":"succeeded"}</benchmark_terminal>'));
+            if (domainRecoveryMode && servedToolCalls === 1 && names(body).includes(TOOL) && anyToolResultPresent(body) && !toolResultPresent(body)) {
+                servedToolCalls += 1;
+                res.end(toolCall('bridge-call-2', 'Singapore'));
+            } else if (mode !== 'disabled' && mode !== 'invalid-path' && mode !== 'invalid-schema' && mode !== 'unsafe-config' && names(body).includes(TOOL) && !anyToolResultPresent(body)) {
+                servedToolCalls += 1;
+                res.end(toolCall());
+            } else {
+                res.end(finalText('<benchmark_terminal>{"status":"succeeded"}</benchmark_terminal>'));
+            }
         });
     });
     await new Promise((resolve)=>server.listen(0, '127.0.0.1', resolve));
@@ -269,15 +308,16 @@ async function runCase(mode, executableOverride, extraEnv = {}) {
     });
     writeResolutionProbe(probe, probeResult, mode === 'disabled');
     const runner = join(cwd, 'synthetic-runner.js');
+    const runnerTrace = join(cwd, 'synthetic-runner-trace.jsonl');
     spawnTarget = join(cwd, 'synthetic-spawn-target.js');
     const configPath = join(cwd, mode === 'invalid-path' ? 'benchmark-env-config-\n.json' : 'benchmark-env-config.json');
-    const runnerBody = mode === 'timeout' ? `setTimeout(() => {}, 60_000)` : mode === 'runner-failed' ? `process.stderr.write('PRIVATE_RUNNER_DIAGNOSTIC_DO_NOT_REFLECT'); process.exit(17)` : mode === 'output-truncated' ? `process.stdout.write('x'.repeat(20_000))` : mode === 'unexpected-output' ? `process.stdout.write(JSON.stringify({ result: { marker: '${MARKER}', leaked: [], unexpected: 'must-not-reflect' } }))` : `const forbidden = ['GOTRY_BENCHMARK_ENV_CONFIG', 'GOTRY_BENCHMARK_BRIDGE_PARENT_SECRET', 'LLM_API_KEY', 'LLM_BASE_URL', 'LLM_MODEL', 'DEEPSEEK_BASE_URL', 'GOTRY_LLM_MODEL', 'DATABASE_URL', 'SSH_AUTH_SOCK', 'AWS_PROFILE', 'HTTPS_PROXY']; const leaked = forbidden.filter(name => process.env[name] !== undefined); process.stdout.write(JSON.stringify({ result: { marker: '${MARKER}', leaked } }))`;
-    writeFileSync(runner, `if (process.argv.length !== 5 || process.argv[2] !== 'call' || process.argv[3] !== 'lookup' || JSON.parse(process.argv[4]).city !== 'Dubai') process.exit(2); ${runnerBody}`);
+    const runnerBody = mode === 'domain-recovery-failed' ? `if (args.city === 'Dubai') process.stdout.write(JSON.stringify({ schema_version: 'gotry_benchmark_tool_result_v1', status: 'miss', code: 'NOT_FOUND', recovery: 'revise_arguments' })); else { process.stderr.write('PRIVATE_RECOVERY_RUNNER_DIAGNOSTIC_DO_NOT_REFLECT'); process.exit(17) }` : mode === 'domain-recovery' ? `if (args.city === 'Dubai') process.stdout.write(JSON.stringify({ schema_version: 'gotry_benchmark_tool_result_v1', status: 'miss', code: 'NOT_FOUND', recovery: 'revise_arguments' })); else process.stdout.write(JSON.stringify({ schema_version: 'gotry_benchmark_tool_result_v1', status: 'ok', result: { marker: '${MARKER}', leaked: [] } }))` : mode === 'timeout' ? `setTimeout(() => {}, 60_000)` : mode === 'runner-failed' ? `process.stderr.write('PRIVATE_RUNNER_DIAGNOSTIC_DO_NOT_REFLECT'); process.exit(17)` : mode === 'output-truncated' ? `process.stdout.write('x'.repeat(20_000))` : mode === 'unexpected-output' ? `process.stdout.write(JSON.stringify({ schema_version: 'gotry_benchmark_tool_result_v1', status: 'ok', result: { marker: '${MARKER}', leaked: [], unexpected: 'must-not-reflect' } }))` : `const forbidden = ['GOTRY_BENCHMARK_ENV_CONFIG', 'GOTRY_BENCHMARK_BRIDGE_PARENT_SECRET', 'LLM_API_KEY', 'LLM_BASE_URL', 'LLM_MODEL', 'DEEPSEEK_BASE_URL', 'GOTRY_LLM_MODEL', 'DATABASE_URL', 'SSH_AUTH_SOCK', 'AWS_PROFILE', 'HTTPS_PROXY']; const leaked = forbidden.filter(name => process.env[name] !== undefined); process.stdout.write(JSON.stringify({ schema_version: 'gotry_benchmark_tool_result_v1', status: 'ok', result: { marker: '${MARKER}', leaked } }))`;
+    writeFileSync(runner, `if (process.argv.length !== 5 || process.argv[2] !== 'call' || process.argv[3] !== 'lookup' || (!${JSON.stringify(domainRecoveryMode)} && JSON.parse(process.argv[4]).city !== 'Dubai') || (${JSON.stringify(domainRecoveryMode)} && !['Dubai', 'Singapore'].includes(JSON.parse(process.argv[4]).city))) process.exit(2); const fs = require('node:fs'); const args = JSON.parse(process.argv[4]); fs.appendFileSync(${JSON.stringify(runnerTrace)}, JSON.stringify({ city: args.city }) + '\\n'); ${runnerBody}`);
     writeFileSync(spawnTarget, '#!/usr/bin/env node\nprocess.exit(0)\n', {
         mode: 0o700
     });
     writeFileSync(configPath, JSON.stringify({
-        schema_version: mode === 'invalid-schema' ? 'invalid' : 'gotry_benchmark_environment_bridge_v2',
+        schema_version: mode === 'invalid-schema' ? 'invalid' : 'gotry_benchmark_environment_bridge_v4',
         enabled: true,
         executable: mode === 'spawn-failed' ? spawnTarget : process.execPath,
         cwd,
@@ -286,20 +326,30 @@ async function runCase(mode, executableOverride, extraEnv = {}) {
         ] : [
             runner
         ],
-        allowed_tools: [
-            'lookup'
+        tools: [
+            {
+                name: 'lookup',
+                description: 'Lookup.',
+                input_schema: LOOKUP_INPUT_SCHEMA,
+                output_keys: [
+                    'marker',
+                    'leaked'
+                ],
+                domain_outcomes: [
+                    {
+                        status: 'miss',
+                        code: 'NOT_FOUND',
+                        recovery: domainRecoveryMode ? 'revise_arguments' : 'none'
+                    }
+                ]
+            }
         ],
-        allowed_output_keys: {
-            lookup: [
-                'marker',
-                'leaked'
-            ]
-        },
         timeout_ms: mode === 'timeout' ? 50 : 10_000,
         max_output_bytes: mode === 'output-truncated' ? 1_024 : 4_096,
         terminal_output: {
             tag: 'benchmark_terminal',
-            max_bytes: 4_096
+            max_bytes: 4_096,
+            body_schema: BRIDGE_E2E_BODY_SCHEMA
         },
         isolation: {
             mode: 'host-enforced',
@@ -398,13 +448,16 @@ async function runCase(mode, executableOverride, extraEnv = {}) {
             calendar: 0,
             map: 0
         });
+        const runnerArguments = existsSync(runnerTrace) ? readFileSync(runnerTrace, 'utf8').split('\n').filter(Boolean).map((line)=>JSON.parse(line)) : [];
         return {
             exit,
             stdout,
             stderr,
             output: stdout + stderr,
             requests,
-            optionalResolutionHits
+            optionalResolutionHits,
+            servedToolCalls,
+            runnerArguments
         };
     } finally{
         await new Promise((resolve)=>server.close(()=>resolve()));
@@ -437,6 +490,31 @@ async function assertRuntimeContract(executableOverride) {
     assert.deepEqual(enabledToolNames, [
         TOOL
     ], `${target} enabled runtime must expose exactly the benchmark tool; observed tool names=${JSON.stringify(enabledToolNames)}`);
+    const bridgeTool = enabled.requests.find((request)=>names(request).includes(TOOL))?.tools?.find((tool)=>names({
+            tools: [
+                tool
+            ]
+        }).includes(TOOL));
+    const flatSchema = bridgeTool?.function?.parameters;
+    assert.equal(flatSchema?.type, 'object', `${target} bridge exposes an object-root wire schema`);
+    assert.equal(flatSchema?.oneOf, undefined, `${target} bridge wire has no top-level oneOf`);
+    assert.deepEqual(flatSchema?.required, [
+        'action'
+    ]);
+    assert.equal(flatSchema?.additionalProperties, false);
+    assert.deepEqual(flatSchema?.properties?.action?.enum, [
+        'tools',
+        'call',
+        'errors'
+    ]);
+    assert.deepEqual(flatSchema?.properties?.tool?.enum, [
+        'lookup'
+    ]);
+    assert.deepEqual(flatSchema?.properties?.arguments, {
+        type: 'object',
+        additionalProperties: true,
+        description: 'action=call 时传给工具的参数对象'
+    });
     assert.equal(enabledToolNames.some((name)=>name.startsWith('calendar_') || name.startsWith('map_')), false, `${target} benchmark projection must not expose calendar/map tools`);
     assert.deepEqual(enabled.optionalResolutionHits, {
         calendar: 0,
@@ -460,6 +538,37 @@ async function assertRuntimeContract(executableOverride) {
         return (prompt.match(/You are GoTry, a task-agnostic travel planning assistant\./g) ?? []).length === 1 && (prompt.match(/Use only the current conversation and tools available in this benchmark session\./g) ?? []).length === 1;
     }), `${target} benchmark persona has each stable sentence exactly once per request`);
     assert.match(enabled.output, /benchmark_terminal/);
+    const recovered = await runCase('domain-recovery', executableOverride);
+    assert.equal(recovered.exit, 0, `${target} model-driven domain miss recovery exits successfully; output=${recovered.output.slice(-2_000)}`);
+    assert.equal(recovered.servedToolCalls, 2, `${target} model emits exactly two tool calls around one declared miss`);
+    assert.deepEqual(recovered.runnerArguments, [
+        {
+            city: 'Dubai'
+        },
+        {
+            city: 'Singapore'
+        }
+    ], `${target} model revises the declared city before the second adapter invocation`);
+    assert.ok(recovered.requests.some((request)=>(request.messages ?? []).some((message)=>{
+            if (message.role !== 'tool') return false;
+            const serialized = JSON.stringify(message);
+            return /status\\?":\\?"miss/.test(serialized) && /recovery\\?":\\?"revise_arguments/.test(serialized);
+        })), `${target} declared typed miss reaches model history`);
+    assert.match(recovered.stdout, /<benchmark_terminal>/, `${target} corrected second call reaches tagged terminal output`);
+    const failedRecovery = await runCase('domain-recovery-failed', executableOverride);
+    assert.equal(failedRecovery.exit, 1, `${target} infrastructure failure after a declared miss cannot be masked by the earlier domain outcome`);
+    assert.equal(failedRecovery.servedToolCalls, 2, `${target} failed recovery still exercises exactly two model-owned tool calls`);
+    assert.deepEqual(failedRecovery.runnerArguments, [
+        {
+            city: 'Dubai'
+        },
+        {
+            city: 'Singapore'
+        }
+    ], `${target} failed recovery reaches the revised second adapter invocation`);
+    assert.equal(failedRecovery.stdout, '', `${target} failed recovery releases no terminal stdout`);
+    assert.match(failedRecovery.stderr, /benchmark terminal output unavailable \(child_bridge_runner_failed\)/, `${target} failed recovery preserves the second runner failure classification`);
+    assert.equal(failedRecovery.output.includes('PRIVATE_RECOVERY_RUNNER_DIAGNOSTIC_DO_NOT_REFLECT'), false, `${target} failed recovery never reflects private runner stderr`);
     const debugRedaction = await runCase('debug-redaction', executableOverride);
     assert.equal(debugRedaction.exit, 0, `${target} benchmark debug mode preserves successful execution`);
     assert.equal(debugRedaction.output.includes('PRIVATE_QUERY_SENTINEL_DO_NOT_REFLECT'), false, `${target} benchmark debug output never reflects the private task`);
@@ -661,7 +770,8 @@ function conformanceResponse(mode, request, plannerCount) {
         'b',
         'd',
         'f',
-        'large'
+        'large',
+        'schema'
     ].includes(mode) && plannerCount === 1;
     if (call && !hasToolResult) return toolCall();
     if (mode === 'f' && plannerCount === 3) return toolCall('bridge-call-retry');
@@ -670,6 +780,9 @@ function conformanceResponse(mode, request, plannerCount) {
             payload: LARGE_TERMINAL_PAYLOAD
         })}</benchmark_terminal>`);
     }
+    if (mode === 'schema') {
+        return finalText('<benchmark_terminal>{"status":"succeeded","budget":{"total_cost":1}}</benchmark_terminal>');
+    }
     const valid = mode === 'a' ? hasToolResult : mode === 'b' ? plannerCount >= 3 : mode === 'e' ? true : false;
     return finalText(mode === 'c' || mode === 'd' ? 'bad benchmark body' : taggedTerminal(valid));
 }
@@ -729,28 +842,38 @@ async function runConformanceCase(mode, executableOverride) {
     const runner = join(cwd, 'synthetic-runner.js');
     const runnerCount = join(cwd, 'runner-count.txt');
     const configPath = join(cwd, 'benchmark-env-config.json');
-    writeFileSync(runner, `const fs = require('node:fs'); const path = ${JSON.stringify(runnerCount)}; const count = fs.existsSync(path) ? Number(fs.readFileSync(path, 'utf8')) : 0; fs.writeFileSync(path, String(count + 1)); process.stdout.write(JSON.stringify({ result: { marker: '${MARKER}' } }))`);
+    writeFileSync(runner, `const fs = require('node:fs'); const path = ${JSON.stringify(runnerCount)}; const count = fs.existsSync(path) ? Number(fs.readFileSync(path, 'utf8')) : 0; fs.writeFileSync(path, String(count + 1)); process.stdout.write(JSON.stringify({ schema_version: 'gotry_benchmark_tool_result_v1', status: 'ok', result: { marker: '${MARKER}' } }))`);
     writeFileSync(configPath, JSON.stringify({
-        schema_version: 'gotry_benchmark_environment_bridge_v2',
+        schema_version: 'gotry_benchmark_environment_bridge_v4',
         enabled: true,
         executable: process.execPath,
         cwd,
         argv_prefix: [
             runner
         ],
-        allowed_tools: [
-            'lookup'
+        tools: [
+            {
+                name: 'lookup',
+                description: 'Lookup.',
+                input_schema: LOOKUP_INPUT_SCHEMA,
+                output_keys: [
+                    'marker'
+                ],
+                domain_outcomes: [
+                    {
+                        status: 'miss',
+                        code: 'NOT_FOUND',
+                        recovery: 'none'
+                    }
+                ]
+            }
         ],
-        allowed_output_keys: {
-            lookup: [
-                'marker'
-            ]
-        },
         timeout_ms: 2_000,
         max_output_bytes: 4_096,
         terminal_output: {
             tag: 'benchmark_terminal',
-            max_bytes: mode === 'large' ? 128 * 1024 : 4_096
+            max_bytes: mode === 'large' ? 128 * 1024 : 4_096,
+            body_schema: BRIDGE_E2E_BODY_SCHEMA
         },
         isolation: {
             mode: 'host-enforced',
@@ -888,6 +1011,7 @@ async function assertOutputConformance(executableOverride) {
     assert.equal(a.servedToolCalls, 1, 'A exposes exactly one bridge call');
     assert.equal(a.runnerInvocations, 1, 'A executes the bridge subprocess exactly once');
     assert.ok(a.stdout.includes('<benchmark_terminal>'), 'A forwards only tagged terminal output');
+    assert.ok(a.requests.some((request)=>JSON.stringify(request).includes(`matching exactly ${TERMINAL_OUTLINE}`)), 'A system prompt projects the exact terminal schema outline');
     const b = await runConformanceCase('b', executableOverride);
     assert.equal(b.exit, 0, 'B malformed terminal correction then valid terminal exits 0');
     assert.equal(b.servedToolCalls, 1, `B exposes exactly one bridge call; request shapes=${JSON.stringify(b.requests.map((request)=>({
@@ -895,6 +1019,12 @@ async function assertOutputConformance(executableOverride) {
             roles: (request.messages ?? []).map((message)=>message.role)
         })))}`);
     assert.equal(b.runnerInvocations, 1, 'B format-only correction does not rerun the bridge subprocess');
+    assert.ok(b.requests.some((request)=>JSON.stringify(request).includes('BENCHMARK_CONFORMANCE_TERMINAL') && JSON.stringify(request).includes(`matching exactly ${TERMINAL_OUTLINE}`)), 'B terminal correction projects the same schema outline as the system prompt');
+    const schema = await runConformanceCase('schema', executableOverride);
+    assert.notEqual(schema.exit, 0, 'schema-invalid terminal body (extra root key) is rejected after the single correction');
+    assert.match(schema.stderr, /benchmark terminal output unavailable \(child_conformance_failure\)/, 'schema-invalid terminal emits the stable conformance reason code');
+    assert.equal(schema.runnerInvocations, 1, 'schema-invalid terminal still executed the bridge exactly once');
+    assert.equal(schema.stdout.includes('<benchmark_terminal>'), false, 'schema-invalid terminal body is never released to stdout');
     for (const mode of [
         'c',
         'd'
@@ -939,7 +1069,7 @@ if (packaged) {
     try {
         const configPath = join(missingServiceRoot, 'bridge.json');
         writeFileSync(configPath, JSON.stringify({
-            schema_version: 'gotry_benchmark_environment_bridge_v2',
+            schema_version: 'gotry_benchmark_environment_bridge_v4',
             enabled: true,
             executable: process.execPath,
             cwd: missingServiceRoot,
@@ -947,14 +1077,29 @@ if (packaged) {
                 '-e',
                 'process.exit(0)'
             ],
-            allowed_tools: [
-                'lookup'
+            tools: [
+                {
+                    name: 'lookup',
+                    description: 'Lookup.',
+                    input_schema: LOOKUP_INPUT_SCHEMA,
+                    output_keys: [
+                        'marker'
+                    ],
+                    domain_outcomes: [
+                        {
+                            status: 'miss',
+                            code: 'NOT_FOUND',
+                            recovery: 'none'
+                        }
+                    ]
+                }
             ],
             timeout_ms: 100,
             max_output_bytes: 4_096,
             terminal_output: {
                 tag: 'benchmark_terminal',
-                max_bytes: 4_096
+                max_bytes: 4_096,
+                body_schema: BRIDGE_E2E_BODY_SCHEMA
             },
             isolation: {
                 mode: 'host-enforced',