@danceiny/gotry 0.0.1-rc.20 → 0.0.1-rc.21

package/dist/scripts/benchmark-environment-bridge-tests.js

@@ -4,9 +4,9 @@ import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { Context } from '@deepseek-ai/cordis';
 import { apply } from '../src/index.js';
-import { BENCHMARK_TOOL_RESULT_SCHEMA_VERSION, registerBenchmarkEnvironmentBridge } from '../src/benchmark-environment-bridge.js';
+import { BRIDGE_ERROR_CONTRACT, BENCHMARK_TOOL_RESULT_SCHEMA_VERSION, registerBenchmarkEnvironmentBridge } from '../src/benchmark-environment-bridge.js';
 import { installBenchmarkToolIsolation } from '../src/benchmark-tool-isolation.js';
-import { BENCHMARK_BRIDGE_CALL_FAILED, BENCHMARK_BRIDGE_CALL_REQUIRED, BENCHMARK_BRIDGE_RETRY_CALL_NOT_ALLOWED, BENCHMARK_BRIDGE_OUTPUT_TRUNCATED, BENCHMARK_BRIDGE_RUNNER_FAILED, BENCHMARK_BRIDGE_SPAWN_FAILED, BENCHMARK_BRIDGE_TIMED_OUT, BENCHMARK_CONFORMANCE_STATE_UNAVAILABLE, BENCHMARK_TERMINAL_INVALID, MAX_CONFORMANCE_RETRIES, benchmarkChildFailureForConformanceCode, createBenchmarkAgentConformance, installBenchmarkAgentConformance, parseBenchmarkTerminal, validateTerminalOutputConfig } from '../src/benchmark-agent-conformance.js';
+import { BENCHMARK_BRIDGE_CALL_FAILED, BENCHMARK_BRIDGE_CALL_REQUIRED, BENCHMARK_BRIDGE_RETRY_CALL_NOT_ALLOWED, BENCHMARK_BRIDGE_OUTPUT_TRUNCATED, BENCHMARK_BRIDGE_RUNNER_FAILED, BENCHMARK_BRIDGE_SPAWN_FAILED, BENCHMARK_BRIDGE_TIMED_OUT, BENCHMARK_CONFORMANCE_STATE_UNAVAILABLE, BENCHMARK_TERMINAL_INVALID, MAX_CONFORMANCE_RETRIES, benchmarkChildFailureForConformanceCode, createBenchmarkAgentConformance, installBenchmarkAgentConformance, parseBenchmarkTerminal, terminalSchemaOutline, validateTerminalBodySchema, validateTerminalBodyValue, validateTerminalOutputConfig } from '../src/benchmark-agent-conformance.js';
 import { BENCHMARK_CHILD_DIAGNOSTIC_MAX_BYTES, BENCHMARK_CHILD_DIAGNOSTIC_SCHEMA, appendBoundedChildDiagnostic, classifyBenchmarkChildFailure, classifyBenchmarkTurnEnd, createBenchmarkDiagnosticArbiter, parseBenchmarkChildDiagnostic } from '../src/benchmark-headless-child-diagnostics.js';
 {
     const exactFamilies = [
@@ -262,6 +262,22 @@ import { BENCHMARK_CHILD_DIAGNOSTIC_MAX_BYTES, BENCHMARK_CHILD_DIAGNOSTIC_SCHEMA
         'child_model_server'
     ]);
 }assert.equal(MAX_CONFORMANCE_RETRIES, 1);
+const OK_BODY_SCHEMA = {
+    type: 'object',
+    properties: {
+        ok: {
+            type: 'boolean'
+        },
+        status: {
+            type: 'string'
+        },
+        x: {
+            type: 'string'
+        }
+    },
+    required: [],
+    additionalProperties: false
+};
 const projection = {
     toolName: 'gotry_benchmark_environment',
     allowedTools: [
@@ -269,7 +285,8 @@ const projection = {
     ],
     terminal: {
         tag: 'done',
-        max_bytes: 1024
+        max_bytes: 1024,
+        body_schema: OK_BODY_SCHEMA
     }
 };
 assert.equal(validateTerminalOutputConfig(projection.terminal), true);
@@ -294,6 +311,20 @@ for (const invalid of [
         tag: 'done',
         max_bytes: 1024,
         extra: true
+    },
+    {
+        tag: 'done',
+        max_bytes: 1024
+    },
+    {
+        tag: 'done',
+        max_bytes: 1024,
+        body_schema: {
+            type: 'object',
+            properties: {},
+            required: [],
+            additionalProperties: true
+        }
     }
 ])assert.equal(validateTerminalOutputConfig(invalid), false);
 assert.deepEqual(parseBenchmarkTerminal(' \n<done>{"ok":true}</done>\n', projection.terminal), {
@@ -338,7 +369,284 @@ for (const [raw, expectOk] of [
 ]){
     assert.equal(parseBenchmarkTerminal(raw, projection.terminal).ok, expectOk, raw);
 }
-function turnStart(turn = 1) {
+{
+    const travelSchema = {
+        type: 'object',
+        properties: {
+            people_number: {
+                type: 'integer'
+            },
+            start_city: {
+                type: 'string'
+            },
+            target_city: {
+                type: 'string'
+            },
+            itinerary: {
+                type: 'array',
+                items: {
+                    type: 'object',
+                    properties: {
+                        day: {
+                            type: 'integer'
+                        },
+                        activities: {
+                            type: 'array',
+                            items: {
+                                type: 'object',
+                                properties: {
+                                    type: {
+                                        type: 'string'
+                                    },
+                                    start_time: {
+                                        type: 'string'
+                                    },
+                                    cost: {
+                                        type: 'number'
+                                    },
+                                    transports: {
+                                        type: 'array',
+                                        items: {
+                                            type: 'object',
+                                            properties: {
+                                                start: {
+                                                    type: 'string'
+                                                },
+                                                mode: {
+                                                    type: 'string'
+                                                }
+                                            },
+                                            required: [
+                                                'start',
+                                                'mode'
+                                            ],
+                                            additionalProperties: false
+                                        }
+                                    }
+                                },
+                                required: [
+                                    'type',
+                                    'start_time',
+                                    'cost',
+                                    'transports'
+                                ],
+                                additionalProperties: false
+                            }
+                        }
+                    },
+                    required: [
+                        'day',
+                        'activities'
+                    ],
+                    additionalProperties: false
+                }
+            }
+        },
+        required: [
+            'people_number',
+            'start_city',
+            'target_city',
+            'itinerary'
+        ],
+        additionalProperties: false
+    };
+    const travelConfig = {
+        tag: 'done',
+        max_bytes: 65_536,
+        body_schema: travelSchema
+    };
+    assert.equal(validateTerminalOutputConfig(travelConfig), true, 'ChinaTravel-like closed schema is a valid terminal config');
+    const legalBody = {
+        people_number: 2,
+        start_city: 'Dubai',
+        target_city: 'Chengdu',
+        itinerary: [
+            {
+                day: 1,
+                activities: [
+                    {
+                        type: 'attraction',
+                        start_time: '09:00',
+                        cost: 0,
+                        transports: [
+                            {
+                                start: 'hotel',
+                                mode: 'walk'
+                            }
+                        ]
+                    }
+                ]
+            }
+        ]
+    };
+    assert.equal(validateTerminalBodySchema(travelSchema), true);
+    const mutate = (fn)=>{
+        const copy = JSON.parse(JSON.stringify(travelSchema));
+        fn(copy);
+        return copy;
+    };
+    for (const [label, schema] of [
+        [
+            'extra root key',
+            mutate((s)=>{
+                s.description = 'the plan';
+            })
+        ],
+        [
+            'enum on scalar',
+            mutate((s)=>{
+                s.properties.people_number.enum = [
+                    2
+                ];
+            })
+        ],
+        [
+            'const on scalar',
+            mutate((s)=>{
+                s.properties.start_city.const = 'Dubai';
+            })
+        ],
+        [
+            'default on scalar',
+            mutate((s)=>{
+                s.properties.target_city.default = 'Chengdu';
+            })
+        ],
+        [
+            'examples key',
+            mutate((s)=>{
+                s.properties.itinerary.examples = [];
+            })
+        ],
+        [
+            'pattern on scalar',
+            mutate((s)=>{
+                s.properties.start_city.pattern = '^[A-Z]';
+            })
+        ],
+        [
+            'open object',
+            mutate((s)=>{
+                s.additionalProperties = true;
+            })
+        ],
+        [
+            'missing additionalProperties',
+            mutate((s)=>{
+                delete s.additionalProperties;
+            })
+        ],
+        [
+            'array without items',
+            mutate((s)=>{
+                s.properties.itinerary = {
+                    type: 'array'
+                };
+            })
+        ],
+        [
+            'object without required',
+            mutate((s)=>{
+                delete s.required;
+            })
+        ],
+        [
+            'unknown type',
+            mutate((s)=>{
+                s.properties.people_number.type = 'float';
+            })
+        ],
+        [
+            'allOf composition',
+            mutate((s)=>{
+                s.allOf = [];
+            })
+        ]
+    ]){
+        assert.equal(validateTerminalBodySchema(schema), false, `schema dialect rejects: ${label}`);
+    }
+    assert.equal(validateTerminalBodyValue(legalBody, travelSchema), true, 'legal ChinaTravel-like hierarchy passes');
+    assert.equal(parseBenchmarkTerminal(`<done>${JSON.stringify(legalBody)}</done>`, travelConfig).ok, true, 'legal body parses through the terminal gate');
+    for (const [label, body] of [
+        [
+            'extra root key (Round 11 budget/total_cost case)',
+            {
+                ...legalBody,
+                budget: {
+                    total_cost: 1
+                }
+            }
+        ],
+        [
+            'itinerary item written as direct activity (missing day)',
+            {
+                ...legalBody,
+                itinerary: [
+                    {
+                        type: 'attraction',
+                        start_time: '09:00',
+                        cost: 0,
+                        transports: []
+                    }
+                ]
+            }
+        ],
+        [
+            'missing activities array',
+            {
+                ...legalBody,
+                itinerary: [
+                    {
+                        day: 1
+                    }
+                ]
+            }
+        ],
+        [
+            'wrong type (string day)',
+            {
+                ...legalBody,
+                itinerary: [
+                    {
+                        day: '1',
+                        activities: []
+                    }
+                ]
+            }
+        ],
+        [
+            'nested extra field',
+            {
+                ...legalBody,
+                itinerary: [
+                    {
+                        day: 1,
+                        activities: [
+                            {
+                                type: 'attraction',
+                                start_time: '09:00',
+                                cost: 0,
+                                transports: [],
+                                unexpected: 1
+                            }
+                        ]
+                    }
+                ]
+            }
+        ]
+    ]){
+        assert.equal(validateTerminalBodyValue(body, travelSchema), false, `value validation rejects: ${label}`);
+        assert.equal(parseBenchmarkTerminal(`<done>${JSON.stringify(body)}</done>`, travelConfig).ok, false, `terminal gate rejects: ${label}`);
+    }
+    assert.equal(parseBenchmarkTerminal(`<done>${' '.repeat(2000)}${JSON.stringify({
+        ok: true
+    })}</done>`, projection.terminal).ok, false, 'schema-valid body over max_bytes still rejects');
+    const outline = terminalSchemaOutline(travelSchema);
+    assert.equal(outline, terminalSchemaOutline(JSON.parse(JSON.stringify(travelSchema))), 'outline is deterministic');
+    assert.equal(outline, 'object{people_number:integer,start_city:string,target_city:string,itinerary:array<object{day:integer,activities:array<object{type:string,start_time:string,cost:number,transports:array<object{start:string,mode:string}>}>}>}', 'outline renders the exact structural contract');
+    assert.match(outline, /object\{/, 'outline renders object nodes');
+    assert.ok(!outline.includes('optional') && !outline.includes('?'), 'all keys required render without optional markers');
+}function turnStart(turn = 1) {
     return {
         type: 'turn/start',
         data: {
@@ -874,6 +1182,8 @@ function toolResult(callId = 'call-1', options = {}) {
     assert.match(assembled.sections[0].text, /\"action\":\"call\"/);
     assert.match(assembled.sections[0].text, /<done>/);
     assert.equal(assembled.sections[0].text.includes('/tmp/'), false);
+    const round12Outline = terminalSchemaOutline(OK_BODY_SCHEMA);
+    assert.ok(assembled.sections[0].text.includes(`matching exactly ${round12Outline}`), 'system section projects the terminal body schema outline');
     rootListeners.get('session/event')[0](session, turnEnd());
     assert.deepEqual(runtimeWrites, []);
     rootListeners.get('session/event')[0](session, turnStart(2));
@@ -1206,7 +1516,7 @@ try {
     assert.equal(parseBenchmarkChildDiagnostic(boundedControl.toString('utf8')), 'child_bridge_timed_out', 'rolling tail preserves a final structured reason after bounded noise');
     const configPath = join(root, 'bridge.json');
     writeFileSync(configPath, JSON.stringify({
-        schema_version: 'gotry_benchmark_environment_bridge_v3',
+        schema_version: 'gotry_benchmark_environment_bridge_v4',
         enabled: true,
         executable: process.execPath,
         cwd: root,
@@ -1268,7 +1578,8 @@ try {
         max_output_bytes: 4_096,
         terminal_output: {
             tag: 'done',
-            max_bytes: 4_096
+            max_bytes: 4_096,
+            body_schema: OK_BODY_SCHEMA
         },
         isolation: {
             mode: 'host-enforced',
@@ -1891,29 +2202,28 @@ try {
     const bridge = registered.find((tool)=>tool.name === 'gotry_benchmark_environment');
     assert.ok(bridge.execute, 'registered bridge exposes execute');
     const parameters = bridge.parameters;
-    assert.equal(Array.isArray(parameters.oneOf), true, 'v3 bridge exposes flat descriptor-derived model branches');
-    const lookupBranch = parameters.oneOf.find((branch)=>branch?.properties?.tool?.const === 'lookup');
-    assert.ok(lookupBranch, 'model schema contains the configured lookup tool branch');
-    assert.equal(lookupBranch.description, 'Lookup one declared city.', 'descriptor description reaches the model-visible call branch');
-    assert.deepEqual(lookupBranch.required, [
-        'action',
-        'tool',
-        'arguments'
-    ]);
-    assert.equal(lookupBranch.additionalProperties, false);
-    assert.deepEqual(lookupBranch.properties.arguments.required, [
-        'city'
-    ]);
-    assert.equal(lookupBranch.properties.arguments.additionalProperties, false);
-    assert.equal(lookupBranch.properties.arguments.properties.city.type, 'string');
-    assert.deepEqual(lookupBranch.properties.arguments.properties.city.enum, [
-        'Dubai',
-        'Abu Dhabi'
+    assert.equal(parameters.type, 'object', 'flat bridge uses an object root for model-facing parameters');
+    assert.equal(parameters.oneOf, undefined, 'model-facing wire has no top-level oneOf');
+    assert.deepEqual(parameters.required, [
+        'action'
+    ], 'action is the only universally required wire field');
+    assert.equal(parameters.additionalProperties, false);
+    assert.deepEqual(parameters.properties?.action?.enum, [
+        'tools',
+        'call',
+        'errors'
     ]);
+    assert.deepEqual(parameters.properties?.tool?.enum, [
+        'lookup',
+        'constructor',
+        'toString'
+    ], 'tool enum exposes frozen descriptor names');
+    assert.equal(parameters.properties?.arguments?.type, 'object', 'arguments remains a generic object on the open wire');
+    assert.equal(parameters.properties?.arguments?.additionalProperties, true);
     assert.equal(Object.isFrozen(bridge.parameters), true, 'registered raw parameters are frozen');
-    assert.equal(Object.isFrozen(lookupBranch.properties.arguments.properties.city.enum), true, 'descriptor-derived nested schema is frozen');
+    assert.equal(Object.isFrozen(parameters.properties?.tool?.enum), true, 'descriptor-derived tool enum is frozen');
     assert.throws(()=>{
-        lookupBranch.properties.arguments.properties.city.enum.push('escape');
+        parameters.properties?.tool?.enum.push('escape');
     }, TypeError);
     const beforeProtocolRejected = spawnSpecs.length;
     await assert.rejects(()=>bridge.execute({
@@ -1926,15 +2236,54 @@ try {
             action: 'tools',
             tool: 'lookup'
         }, null), /invalid arguments/, 'mixed action/tool fields are rejected by the flat protocol');
+    await assert.rejects(()=>bridge.execute({
+            action: 'tools',
+            arguments: {}
+        }, null), /invalid arguments/, 'tools rejects mixed arguments');
+    await assert.rejects(()=>bridge.execute({
+            action: 'errors',
+            tool: 'lookup'
+        }, null), /invalid arguments/, 'errors rejects mixed tool fields');
+    await assert.rejects(()=>bridge.execute({
+            action: 'errors',
+            arguments: {}
+        }, null), /invalid arguments/, 'errors rejects mixed arguments');
     await assert.rejects(()=>bridge.execute({
             ...bridgeCall('lookup', {
                 city: 'Dubai'
             }),
             extra: true
         }, null), /invalid arguments/, 'extra top-level fields are rejected by the flat protocol');
+    await assert.rejects(()=>bridge.execute({
+            action: 'call'
+        }, null), /invalid arguments/, 'call without tool or arguments is rejected');
+    await assert.rejects(()=>bridge.execute({
+            action: 'call',
+            tool: 'lookup'
+        }, null), /invalid arguments/, 'call without arguments is rejected');
+    await assert.rejects(()=>bridge.execute({
+            action: 'call',
+            arguments: {
+                city: 'Dubai'
+            }
+        }, null), /invalid arguments/, 'call without tool is rejected');
     await assert.rejects(()=>bridge.execute(bridgeCall('lookup', {
             city: 'Sharjah'
         }), null), /invalid arguments/, 'invalid descriptor argument values are rejected by the flat protocol');
+    await assert.rejects(()=>bridge.execute({
+            action: 'call',
+            tool: 'lookup',
+            arguments: {}
+        }, null), /invalid arguments/, 'missing required descriptor arguments are rejected by the flat protocol');
+    for (const tool of [
+        'lookup',
+        'constructor',
+        'toString'
+    ]){
+        await assert.rejects(()=>bridge.execute(bridgeCall(tool, {
+                city: 'Sharjah'
+            }), null), /invalid arguments/, `${tool} rejects invalid descriptor arguments`);
+    }
     assert.equal(spawnSpecs.length, beforeProtocolRejected, 'flat protocol rejections happen before spawn');
     const args = {
         city: 'Dubai',
@@ -2297,6 +2646,64 @@ try {
     }
     assert.equal(errorsTable.errors?.invalid_arguments?.recoverable, true, 'invalid caller arguments are recoverable');
     assert.equal(errorsTable.errors?.forbidden_output?.recoverable, false, 'policy-boundary output is not recoverable');
+    assert.deepEqual(errorsTable, {
+        ok: true,
+        errors: BRIDGE_ERROR_CONTRACT
+    }, 'errors action returns the exact closed failure contract');
+    const toolsTable = await bridge.execute({
+        action: 'tools'
+    }, null);
+    assert.deepEqual(toolsTable, {
+        ok: true,
+        tools: [
+            {
+                name: 'lookup',
+                description: 'Lookup one declared city.',
+                input_schema: lookupInputSchema(),
+                output_keys: [
+                    'city',
+                    'nested'
+                ],
+                domain_outcomes: [
+                    {
+                        status: 'miss',
+                        code: 'NOT_FOUND',
+                        recovery: 'revise_arguments'
+                    },
+                    {
+                        status: 'error',
+                        code: 'AMBIGUOUS',
+                        recovery: 'choose_alternative'
+                    }
+                ]
+            },
+            {
+                name: 'constructor',
+                description: 'Construct one declared city.',
+                input_schema: lookupInputSchema(),
+                output_keys: [
+                    'legacy'
+                ],
+                domain_outcomes: [
+                    {
+                        status: 'error',
+                        code: 'INVALID',
+                        recovery: 'retry_same'
+                    }
+                ]
+            },
+            {
+                name: 'toString',
+                description: 'Stringify one declared city.',
+                input_schema: lookupInputSchema(),
+                output_keys: [
+                    'city',
+                    'nested'
+                ],
+                domain_outcomes: []
+            }
+        ]
+    }, 'tools action returns every exact frozen descriptor, including its input schema');
     for (const stdout of [
         'not-json',
         '{"result":1}{"result":2}'
@@ -2669,7 +3076,7 @@ try {
         else process.env.GOTRY_LLM_MODEL = originalModelOverride;
     }
     const validConfig = {
-        schema_version: 'gotry_benchmark_environment_bridge_v3',
+        schema_version: 'gotry_benchmark_environment_bridge_v4',
         enabled: true,
         executable: process.execPath,
         cwd: root,
@@ -2716,7 +3123,8 @@ try {
         max_output_bytes: 4_096,
         terminal_output: {
             tag: 'done',
-            max_bytes: 4_096
+            max_bytes: 4_096,
+            body_schema: OK_BODY_SCHEMA
         },
         isolation: {
             mode: 'host-enforced',
@@ -2891,6 +3299,20 @@ try {
             ...validConfig,
             schema_version: 'gotry_benchmark_environment_bridge_v1'
         }), /benchmark environment bridge configuration unavailable/, 'v1 config cannot silently omit the Round 3 terminal semantics');
+    assert.throws(()=>bridgeRegistrationFor({
+            ...validConfig,
+            schema_version: 'gotry_benchmark_environment_bridge_v3'
+        }), /benchmark environment bridge configuration unavailable/, 'v3 config cannot silently omit the Round 12 terminal body schema (#215)');
+    assert.throws(()=>bridgeRegistrationFor({
+            ...validConfig,
+            terminal_output: {
+                ...validConfig.terminal_output,
+                body_schema: {
+                    ...validConfig.terminal_output.body_schema,
+                    additionalProperties: true
+                }
+            }
+        }), /benchmark environment bridge configuration unavailable/, 'non-closed terminal body schema fails the config load closed');
     assert.throws(()=>bridgeRegistrationFor({
             ...validConfig,
             tools: [