backend-manager 5.6.4 → 5.7.0

package/test/helpers/ai-test-provider.js

@@ -0,0 +1,202 @@
+/**
+ * Test: deterministic AI test provider (libraries/ai/providers/test.js)
+ *
+ * The test provider is a first-class provider (the `test` payment-processor
+ * precedent) that consumer suites drive with directives embedded in the last
+ * user message. These tests exercise the full request() surface directly —
+ * no network involved by design.
+ */
+const TestProvider = require('../../src/manager/libraries/ai/providers/test.js');
+const { parseScript } = TestProvider._internals;
+
+function makeProvider() {
+  // No Manager — the provider falls back to the BEM_TESTING signal, which the
+  // test runner sets
+  return new TestProvider({});
+}
+
+module.exports = {
+  description: 'AI test provider (scripted sequences)',
+  type: 'group',
+  tests: [
+    // ─── Script parsing ───
+
+    {
+      name: 'parse-script-orders-steps-and-strips-directives',
+      async run({ assert }) {
+        const { steps, cleanText } = parseScript(
+          'Check my order [[tool:check_order {"orderNumber":"1"}]] please [[reply:{"message":"done"}]]',
+        );
+
+        assert.equal(steps.length, 2, 'two steps');
+        assert.equal(steps[0].type, 'tools', 'first step is a tool call');
+        assert.equal(steps[0].calls[0].name, 'check_order', 'tool name parsed');
+        assert.deepEqual(steps[0].calls[0].arguments, { orderNumber: '1' }, 'tool args parsed');
+        assert.equal(steps[1].type, 'reply', 'second step is the reply');
+        assert.equal(cleanText, 'Check my order  please', 'directives stripped from echo text');
+      },
+    },
+
+    {
+      name: 'parse-script-delay-attaches-to-next-step',
+      async run({ assert }) {
+        const { steps } = parseScript('[[delay:50]][[tool:slow_tool]]');
+
+        assert.equal(steps.length, 1, 'one step');
+        assert.equal(steps[0].delay, 50, 'delay attached to the tool step');
+      },
+    },
+
+    // ─── Echo default ───
+
+    {
+      name: 'echo-reply-without-directives',
+      async run({ assert }) {
+        const provider = makeProvider();
+        const result = await provider.request({
+          messages: [
+            { role: 'system', content: 'sys' },
+            { role: 'user', content: 'hello there' },
+          ],
+        });
+
+        assert.equal(result.content, 'Echo: hello there', 'echoes the user message');
+        assert.equal(result.stopReason, 'end', 'final turn');
+        assert.deepEqual(result.toolCalls, [], 'no tool calls');
+        assert.equal(result.tokens.total.count > 0, true, 'tokens accounted');
+      },
+    },
+
+    {
+      name: 'echo-reply-json-mode-wraps-message',
+      async run({ assert }) {
+        const provider = makeProvider();
+        const result = await provider.request({
+          response: 'json',
+          messages: [{ role: 'user', content: 'hi' }],
+        });
+
+        assert.equal(typeof result.content, 'object', 'parsed object');
+        assert.equal(result.content.message, 'Echo: hi', 'wrapped message');
+      },
+    },
+
+    // ─── Scripted reply ───
+
+    {
+      name: 'scripted-json-reply',
+      async run({ assert }) {
+        const provider = makeProvider();
+        const result = await provider.request({
+          response: 'json',
+          messages: [
+            { role: 'user', content: 'whatever [[reply:{"message":"Hi!","actions":[{"type":"reply","label":"More"}]}]]' },
+          ],
+        });
+
+        assert.equal(result.content.message, 'Hi!', 'scripted message');
+        assert.equal(result.content.actions[0].label, 'More', 'scripted actions');
+        assert.equal(result.stopReason, 'end', 'final turn');
+      },
+    },
+
+    // ─── Tool loop sequence ───
+
+    {
+      name: 'tool-then-reply-sequence-across-turns',
+      async run({ assert }) {
+        const provider = makeProvider();
+        const script = 'check it [[tool:check_order {"orderNumber":"42"}]] [[reply:{"message":"Order 42 shipped"}]]';
+
+        // Turn 1: the provider emits the tool call
+        const first = await provider.request({
+          response: 'json',
+          messages: [
+            { role: 'system', content: 'sys' },
+            { role: 'user', content: script },
+          ],
+        });
+
+        assert.equal(first.stopReason, 'tool_use', 'first turn requests the tool');
+        assert.equal(first.toolCalls.length, 1, 'one call');
+        assert.equal(first.toolCalls[0].name, 'check_order', 'tool name');
+        assert.deepEqual(first.toolCalls[0].arguments, { orderNumber: '42' }, 'tool args');
+
+        // Turn 2: caller appended the assistant turn + tool result — provider
+        // moves to the next step (the reply)
+        const second = await provider.request({
+          response: 'json',
+          messages: [
+            { role: 'system', content: 'sys' },
+            { role: 'user', content: script },
+            { role: 'assistant', toolCalls: first.toolCalls },
+            { role: 'tool', toolCallId: first.toolCalls[0].id, content: '{"status":"shipped"}' },
+          ],
+        });
+
+        assert.equal(second.stopReason, 'end', 'second turn is final');
+        assert.equal(second.content.message, 'Order 42 shipped', 'scripted final reply');
+      },
+    },
+
+    {
+      name: 'parallel-tools-directive-emits-multiple-calls',
+      async run({ assert }) {
+        const provider = makeProvider();
+        const result = await provider.request({
+          messages: [
+            { role: 'user', content: '[[tools:[{"name":"a","arguments":{"x":1}},{"name":"b","arguments":{"y":2}}]]]' },
+          ],
+        });
+
+        assert.equal(result.stopReason, 'tool_use', 'tool turn');
+        assert.equal(result.toolCalls.length, 2, 'two parallel calls');
+        assert.equal(result.toolCalls[0].name, 'a', 'first call');
+        assert.equal(result.toolCalls[1].name, 'b', 'second call');
+        assert.deepEqual(result.toolCalls[1].arguments, { y: 2 }, 'second args');
+      },
+    },
+
+    // ─── Scripted error ───
+
+    {
+      name: 'scripted-error-throws',
+      async run({ assert }) {
+        const provider = makeProvider();
+        let threw = false;
+
+        try {
+          await provider.request({
+            messages: [{ role: 'user', content: '[[error:boom]]' }],
+          });
+        } catch (e) {
+          threw = true;
+          assert.equal(e.message, 'boom', 'scripted error message');
+        }
+
+        assert.equal(threw, true, 'throws the scripted error');
+      },
+    },
+
+    // ─── Script exhaustion ───
+
+    {
+      name: 'exhausted-script-falls-back-to-echo',
+      async run({ assert }) {
+        const provider = makeProvider();
+        const script = 'hi [[tool:t1]]';
+
+        const second = await provider.request({
+          messages: [
+            { role: 'user', content: script },
+            { role: 'assistant', toolCalls: [{ id: 'c', name: 't1', arguments: {} }] },
+            { role: 'tool', toolCallId: 'c', content: 'r' },
+          ],
+        });
+
+        assert.equal(second.stopReason, 'end', 'falls back to a final turn');
+        assert.equal(String(second.content).startsWith('Echo:'), true, 'echo fallback');
+      },
+    },
+  ],
+};

package/test/helpers/ai-tools-format.js

@@ -0,0 +1,350 @@
+/**
+ * Test: cross-provider AI tools formatting (libraries/ai)
+ *
+ * Verifies the unified tools interface added for agentic tool loops:
+ *   - anthropic-format.js — tool defs, tool choice, unified-message → Claude
+ *     Messages API mapping (tool_use / tool_result blocks), stop reasons
+ *   - openai.js formatMessages/normalizeToolEntry/normalizeToolChoice — the
+ *     direct-messages path and Responses API tool envelopes
+ *   - ai/index.js normalizeOptions — structured conversations are NOT
+ *     string-flattened; system injections still apply
+ *
+ * All pure helpers — no network, no assistant.
+ */
+const format = require('../../src/manager/libraries/ai/providers/anthropic-format.js');
+const OpenAI = require('../../src/manager/libraries/ai/providers/openai.js');
+const AI = require('../../src/manager/libraries/ai/index.js');
+
+const { formatMessages, normalizeToolEntry, normalizeToolChoice } = OpenAI._internals;
+const { normalizeOptions, isStructuredMessages, SYSTEM_PROMPT_INJECTIONS } = AI._internals;
+
+function noopLog() {}
+
+const SAMPLE_TOOL = {
+  name: 'check_order',
+  description: 'Look up an order',
+  parameters: { type: 'object', properties: { orderNumber: { type: 'string' } }, required: ['orderNumber'] },
+};
+
+module.exports = {
+  description: 'AI cross-provider tools formatting',
+  type: 'group',
+  tests: [
+    // ─── anthropic-format: tool definitions ───
+
+    {
+      name: 'anthropic-tool-defs-map-parameters-to-input-schema',
+      async run({ assert }) {
+        const defs = format.buildToolDefs([SAMPLE_TOOL]);
+
+        assert.equal(defs.length, 1, 'one def');
+        assert.equal(defs[0].name, 'check_order', 'name preserved');
+        assert.equal(defs[0].description, 'Look up an order', 'description preserved');
+        assert.deepEqual(defs[0].input_schema, SAMPLE_TOOL.parameters, 'parameters → input_schema');
+      },
+    },
+
+    {
+      name: 'anthropic-tool-defs-accept-explicit-function-type',
+      async run({ assert }) {
+        const defs = format.buildToolDefs([{ ...SAMPLE_TOOL, type: 'function' }]);
+
+        assert.equal(defs[0].name, 'check_order', 'function-typed entry accepted');
+      },
+    },
+
+    {
+      name: 'anthropic-tool-defs-reject-hosted-tools',
+      async run({ assert }) {
+        let threw = false;
+
+        try {
+          format.buildToolDefs([{ type: 'web_search' }]);
+        } catch (e) {
+          threw = true;
+        }
+
+        assert.equal(threw, true, 'hosted tool types throw on Anthropic');
+      },
+    },
+
+    {
+      name: 'anthropic-tool-defs-empty-list-returns-empty',
+      async run({ assert }) {
+        assert.deepEqual(format.buildToolDefs(undefined), [], 'undefined → []');
+        assert.deepEqual(format.buildToolDefs([]), [], '[] → []');
+      },
+    },
+
+    // ─── anthropic-format: tool choice ───
+
+    {
+      name: 'anthropic-tool-choice-mapping',
+      async run({ assert }) {
+        assert.deepEqual(format.buildToolChoice('auto'), { type: 'auto' }, 'auto');
+        assert.deepEqual(format.buildToolChoice('required'), { type: 'any' }, 'required → any');
+        assert.deepEqual(format.buildToolChoice('none'), { type: 'none' }, 'none');
+        assert.deepEqual(format.buildToolChoice({ name: 'check_order' }), { type: 'tool', name: 'check_order' }, 'specific tool');
+        assert.equal(format.buildToolChoice(undefined), undefined, 'undefined passes through');
+      },
+    },
+
+    // ─── anthropic-format: message building ───
+
+    {
+      name: 'anthropic-messages-assistant-toolcalls-become-tool-use-blocks',
+      async run({ assert }) {
+        const { system, messages } = format.buildMessages({
+          messages: [
+            { role: 'system', content: 'sys' },
+            { role: 'user', content: 'check my order 123' },
+            { role: 'assistant', content: 'Let me check.', toolCalls: [{ id: 'call_1', name: 'check_order', arguments: { orderNumber: '123' } }] },
+          ],
+        });
+
+        assert.equal(system, 'sys', 'system extracted');
+        assert.equal(messages.length, 2, 'user + assistant');
+
+        const assistantTurn = messages[1];
+        assert.equal(assistantTurn.role, 'assistant', 'assistant role');
+        assert.equal(assistantTurn.content[0].type, 'text', 'text block first');
+        assert.equal(assistantTurn.content[0].text, 'Let me check.', 'text content');
+        assert.equal(assistantTurn.content[1].type, 'tool_use', 'tool_use block');
+        assert.equal(assistantTurn.content[1].id, 'call_1', 'call id');
+        assert.deepEqual(assistantTurn.content[1].input, { orderNumber: '123' }, 'arguments → input');
+      },
+    },
+
+    {
+      name: 'anthropic-messages-consecutive-tool-results-merge-into-one-user-turn',
+      async run({ assert }) {
+        const { messages } = format.buildMessages({
+          messages: [
+            { role: 'user', content: 'hi' },
+            { role: 'assistant', toolCalls: [
+              { id: 'call_1', name: 'a', arguments: {} },
+              { id: 'call_2', name: 'b', arguments: {} },
+            ] },
+            { role: 'tool', toolCallId: 'call_1', content: 'result A' },
+            { role: 'tool', toolCallId: 'call_2', content: 'result B' },
+          ],
+        });
+
+        assert.equal(messages.length, 3, 'user + assistant + ONE merged tool-result user turn');
+
+        const resultTurn = messages[2];
+        assert.equal(resultTurn.role, 'user', 'tool results ride a user turn');
+        assert.equal(resultTurn.content.length, 2, 'both results in one turn');
+        assert.equal(resultTurn.content[0].type, 'tool_result', 'tool_result block');
+        assert.equal(resultTurn.content[0].tool_use_id, 'call_1', 'first result id');
+        assert.equal(resultTurn.content[1].tool_use_id, 'call_2', 'second result id');
+      },
+    },
+
+    {
+      name: 'anthropic-messages-developer-role-folds-into-system',
+      async run({ assert }) {
+        const { system, messages } = format.buildMessages({
+          messages: [
+            { role: 'system', content: 'platform' },
+            { role: 'developer', content: 'operator' },
+            { role: 'user', content: 'hi' },
+          ],
+        });
+
+        assert.equal(system, 'platform\n\noperator', 'developer folded into system');
+        assert.equal(messages.length, 1, 'only the user turn remains');
+      },
+    },
+
+    {
+      name: 'anthropic-messages-raw-block-arrays-pass-through',
+      async run({ assert }) {
+        const rawBlocks = [{ type: 'text', text: 'hello' }, { type: 'tool_use', id: 'x', name: 't', input: {} }];
+        const { messages } = format.buildMessages({
+          messages: [
+            { role: 'user', content: 'hi' },
+            { role: 'assistant', content: rawBlocks },
+          ],
+        });
+
+        assert.deepEqual(messages[1].content, rawBlocks, 'raw blocks untouched');
+      },
+    },
+
+    {
+      name: 'anthropic-messages-legacy-prompt-message-form-unchanged',
+      async run({ assert }) {
+        const { system, messages } = format.buildMessages({
+          prompt: { content: 'sys prompt' },
+          message: { content: 'user msg' },
+        });
+
+        assert.equal(system, 'sys prompt', 'system from prompt');
+        assert.deepEqual(messages, [{ role: 'user', content: 'user msg' }], 'single user turn');
+      },
+    },
+
+    // ─── anthropic-format: response extraction ───
+
+    {
+      name: 'anthropic-extract-tool-calls-and-stop-reasons',
+      async run({ assert }) {
+        const calls = format.extractToolCalls([
+          { type: 'text', text: 'thinking...' },
+          { type: 'tool_use', id: 'call_9', name: 'check_order', input: { orderNumber: '9' } },
+        ]);
+
+        assert.equal(calls.length, 1, 'one call');
+        assert.deepEqual(calls[0], { id: 'call_9', name: 'check_order', arguments: { orderNumber: '9' } }, 'normalized shape');
+
+        assert.equal(format.mapStopReason('tool_use'), 'tool_use', 'tool_use');
+        assert.equal(format.mapStopReason('max_tokens'), 'max_tokens', 'max_tokens');
+        assert.equal(format.mapStopReason('end_turn'), 'end', 'end_turn → end');
+        assert.equal(format.mapStopReason('stop_sequence'), 'end', 'stop_sequence → end');
+      },
+    },
+
+    // ─── openai: direct-messages path ───
+
+    {
+      name: 'openai-format-messages-maps-toolcalls-to-function-call-items',
+      async run({ assert }) {
+        const input = formatMessages([
+          { role: 'system', content: 'sys' },
+          { role: 'user', content: 'check order 123' },
+          { role: 'assistant', content: 'Checking.', toolCalls: [{ id: 'call_1', name: 'check_order', arguments: { orderNumber: '123' } }] },
+          { role: 'tool', toolCallId: 'call_1', content: '{"status":"shipped"}' },
+        ], noopLog);
+
+        assert.equal(input.length, 5, 'system + user + assistant text + function_call + function_call_output');
+        assert.equal(input[0].role, 'system', 'system turn');
+        assert.equal(input[1].role, 'user', 'user turn');
+        assert.equal(input[2].role, 'assistant', 'assistant text turn');
+        assert.equal(input[2].content[0].type, 'output_text', 'assistant uses output_text');
+
+        const callItem = input[3];
+        assert.equal(callItem.type, 'function_call', 'function_call item');
+        assert.equal(callItem.call_id, 'call_1', 'call id');
+        assert.equal(callItem.name, 'check_order', 'name');
+        assert.equal(callItem.arguments, JSON.stringify({ orderNumber: '123' }), 'arguments stringified');
+
+        const outputItem = input[4];
+        assert.equal(outputItem.type, 'function_call_output', 'function_call_output item');
+        assert.equal(outputItem.call_id, 'call_1', 'output call id');
+        assert.equal(outputItem.output, '{"status":"shipped"}', 'output string');
+      },
+    },
+
+    {
+      name: 'openai-format-messages-toolcall-turn-without-text-emits-no-message-item',
+      async run({ assert }) {
+        const input = formatMessages([
+          { role: 'user', content: 'hi' },
+          { role: 'assistant', toolCalls: [{ id: 'c1', name: 't', arguments: {} }] },
+        ], noopLog);
+
+        assert.equal(input.length, 2, 'user + function_call only');
+        assert.equal(input[1].type, 'function_call', 'no empty assistant message item');
+      },
+    },
+
+    // ─── openai: tool envelopes ───
+
+    {
+      name: 'openai-tool-entry-normalization',
+      async run({ assert }) {
+        const normalized = normalizeToolEntry(SAMPLE_TOOL);
+
+        assert.equal(normalized.type, 'function', 'function envelope added');
+        assert.equal(normalized.name, 'check_order', 'name preserved');
+        assert.deepEqual(normalized.parameters, SAMPLE_TOOL.parameters, 'parameters preserved');
+
+        const hosted = normalizeToolEntry({ type: 'web_search' });
+        assert.deepEqual(hosted, { type: 'web_search' }, 'hosted tools pass verbatim');
+      },
+    },
+
+    {
+      name: 'openai-tool-choice-normalization',
+      async run({ assert }) {
+        assert.equal(normalizeToolChoice('auto'), 'auto', 'auto passes');
+        assert.equal(normalizeToolChoice('required'), 'required', 'required passes');
+        assert.equal(normalizeToolChoice('none'), 'none', 'none passes');
+        assert.deepEqual(normalizeToolChoice({ name: 'check_order' }), { type: 'function', name: 'check_order' }, 'specific tool');
+      },
+    },
+
+    // ─── index.js: normalizeOptions on structured conversations ───
+
+    {
+      name: 'structured-detection',
+      async run({ assert }) {
+        assert.equal(isStructuredMessages([{ role: 'user', content: 'hi' }]), false, 'plain text is not structured');
+        assert.equal(isStructuredMessages([{ role: 'tool', toolCallId: 'x', content: 'r' }]), true, 'tool turn is structured');
+        assert.equal(isStructuredMessages([{ role: 'assistant', toolCalls: [{ id: 'x', name: 't' }] }]), true, 'toolCalls turn is structured');
+        assert.equal(isStructuredMessages([{ role: 'assistant', content: [{ type: 'tool_use', id: 'x' }] }]), true, 'raw tool_use blocks are structured');
+      },
+    },
+
+    {
+      name: 'normalize-options-structured-keeps-turns-and-injects-system-rules',
+      async run({ assert }) {
+        const messages = [
+          { role: 'system', content: 'agent instructions' },
+          { role: 'user', content: 'check order 1' },
+          { role: 'assistant', toolCalls: [{ id: 'c1', name: 'check_order', arguments: {} }] },
+          { role: 'tool', toolCallId: 'c1', content: 'shipped' },
+        ];
+        const out = normalizeOptions({ messages });
+
+        assert.equal(out.messages.length, 4, 'no turns added or dropped');
+        assert.equal(
+          out.messages[0].content.includes(SYSTEM_PROMPT_INJECTIONS[0]),
+          true,
+          'system rules injected into the system turn',
+        );
+        assert.equal(
+          out.messages[0].content.includes('agent instructions'),
+          true,
+          'original system content preserved',
+        );
+        assert.deepEqual(out.messages[2], messages[2], 'toolCalls turn untouched');
+        assert.deepEqual(out.messages[3], messages[3], 'tool result turn untouched');
+        assert.equal(out.prompt, undefined, 'prompt NOT synthesized in structured mode');
+      },
+    },
+
+    {
+      name: 'normalize-options-structured-without-system-prepends-rules-turn',
+      async run({ assert }) {
+        const out = normalizeOptions({
+          messages: [
+            { role: 'user', content: 'hi' },
+            { role: 'tool', toolCallId: 'c1', content: 'r' },
+          ],
+        });
+
+        assert.equal(out.messages.length, 3, 'rules turn prepended');
+        assert.equal(out.messages[0].role, 'system', 'first turn is system');
+        assert.equal(out.messages[0].content.includes(SYSTEM_PROMPT_INJECTIONS[1]), true, 'rules content');
+      },
+    },
+
+    {
+      name: 'normalize-options-legacy-plain-messages-unchanged-behavior',
+      async run({ assert }) {
+        const out = normalizeOptions({
+          messages: [
+            { role: 'system', content: 'sys' },
+            { role: 'user', content: 'hello' },
+          ],
+        });
+
+        assert.equal(typeof out.prompt?.content, 'string', 'prompt synthesized for legacy mode');
+        assert.equal(out.prompt.content.includes('sys'), true, 'prompt carries system content');
+        assert.equal(out.message?.content, 'hello', 'message carries last user turn');
+      },
+    },
+  ],
+};

package/test/mcp/discovery.js

@@ -0,0 +1,53 @@
+/**
+ * Test: MCP OAuth discovery endpoints
+ * Tests .well-known/oauth-authorization-server and .well-known/oauth-protected-resource
+ *
+ * Run: npx mgr test bem:mcp/discovery
+ */
+const fetch = require('wonderful-fetch');
+
+const BASE_URL = 'http://localhost:5002';
+
+module.exports = {
+  description: 'MCP OAuth discovery endpoints',
+  type: 'group',
+
+  tests: [
+    {
+      name: 'oauth-authorization-server returns valid metadata',
+      async run({ assert }) {
+        const response = await fetch(`${BASE_URL}/.well-known/oauth-authorization-server`, {
+          method: 'GET',
+          response: 'json',
+          timeout: 10000,
+        });
+
+        assert.ok(response, 'Discovery endpoint should return a response');
+        assert.ok(response.issuer, 'Should have issuer');
+        assert.ok(response.authorization_endpoint, 'Should have authorization_endpoint');
+        assert.ok(response.token_endpoint, 'Should have token_endpoint');
+        assert.ok(response.authorization_endpoint.includes('/backend-manager/mcp/authorize'), 'authorization_endpoint should point to mcp/authorize');
+        assert.ok(response.token_endpoint.includes('/backend-manager/mcp/token'), 'token_endpoint should point to mcp/token');
+        assert.ok(response.response_types_supported.includes('code'), 'Should support code response type');
+        assert.ok(response.code_challenge_methods_supported.includes('S256'), 'Should support PKCE S256');
+      },
+    },
+
+    {
+      name: 'oauth-protected-resource returns valid metadata',
+      async run({ assert }) {
+        const response = await fetch(`${BASE_URL}/.well-known/oauth-protected-resource`, {
+          method: 'GET',
+          response: 'json',
+          timeout: 10000,
+        });
+
+        assert.ok(response, 'Protected resource endpoint should return a response');
+        assert.ok(response.resource, 'Should have resource');
+        assert.ok(response.resource.includes('/backend-manager/mcp'), 'resource should point to MCP endpoint');
+        assert.ok(Array.isArray(response.authorization_servers), 'Should have authorization_servers array');
+        assert.ok(response.authorization_servers.length > 0, 'Should have at least one authorization server');
+      },
+    },
+  ],
+};