@xalia/agent 0.6.9 → 0.6.11

package/src/test/agent.test.ts

@@ -1,590 +0,0 @@
-import { expect } from "vitest";
-import { strict as assert } from "assert";
-
-import { Schema } from "@xalia/xmcp/sdk";
-
-import {
-  Agent,
-  AgentEx,
-  AgentProfile,
-  IAgentToolProvider,
-  ToolCallResult,
-  completionToAssistantMessageParam,
-} from "../agent/agent";
-import {
-  AssistantMessageParam,
-  Choice,
-  MessageParam,
-  MessageToolCall,
-  ToolDescriptor,
-  ToolMessageParam,
-} from "../agent/llm";
-import { ContextManager } from "../agent/context";
-import { LOCAL_SERVER_URL, SkillManager } from "../agent/sudoMcpServerManager";
-import { SystemPromptProvider } from "../agent/promptProvider";
-import {
-  DUMMY_PLATFORM,
-  TestAgentEventHandler,
-  TestDummyLLM,
-  TestErrorLLM,
-} from "./testTools";
-import { DummyLLMScriptEntry } from "../agent/dummyLLM";
-
-function clone<T>(a: T): T {
-  return JSON.parse(JSON.stringify(a)) as T;
-}
-
-const DUMMY_SCRIPT: Choice[] = [
-  {
-    index: 0,
-    finish_reason: "stop",
-    message: {
-      content: "message1",
-      refusal: null,
-      role: "assistant",
-    },
-    logprobs: null,
-  },
-  {
-    index: 0,
-    finish_reason: "stop",
-    message: {
-      content: "message2",
-      refusal: null,
-      role: "assistant",
-    },
-    logprobs: null,
-  },
-];
-
-function createCallTestToolScript(
-  tool_call_ids: string[],
-  param1: string,
-  param2: number
-): {
-  script: Choice[];
-  expectCompletions: AssistantMessageParam[];
-  expectAgentMessages: string[];
-  expectToolResults: ToolMessageParam[];
-  tool0_descriptor: ToolDescriptor;
-  tool0_fn: (agent: AgentEx, args: unknown) => Promise<ToolCallResult>;
-  test_tool_descriptor: ToolDescriptor;
-  test_tool_fn: (agent: AgentEx, args: unknown) => Promise<ToolCallResult>;
-} {
-  // A tool with no args
-
-  const tool0_params: Schema = {
-    type: "object",
-    properties: {},
-  };
-
-  const tool0_descriptor: ToolDescriptor = {
-    type: "function",
-    function: {
-      name: "tool0",
-      parameters: tool0_params,
-      strict: true,
-    },
-  };
-
-  const tool0_fn = (_: AgentEx, _args: unknown): Promise<ToolCallResult> => {
-    return Promise.resolve({ response: "0" });
-  };
-
-  // A trivial tool
-
-  const test_tool_params: Schema = {
-    type: "object",
-    properties: {
-      param1: {
-        type: "string",
-      },
-      param2: {
-        type: "number",
-      },
-    },
-    required: ["param1", "param2"],
-  };
-
-  const test_tool_descriptor: ToolDescriptor = {
-    type: "function",
-    function: {
-      name: "test_tool",
-      parameters: test_tool_params,
-      strict: true,
-    },
-  };
-
-  const test_tool_fn = async (
-    _: AgentEx,
-    args: unknown
-  ): Promise<ToolCallResult> => {
-    const { param1, param2 } = args as { param1: string; param2: number };
-    return new Promise((r) => {
-      r({
-        response: `tool_result: '${param1}' '${String(param2)}'`,
-        _meta: { type: "text/plain" },
-      });
-    });
-  };
-
-  const tool_calls: MessageToolCall[] = [];
-  const expectToolResults: ToolMessageParam[] = [];
-
-  // First tool call (if requested) is test_tool
-  if (tool_call_ids.length > 0) {
-    const id = tool_call_ids[0];
-    tool_calls.push({
-      id,
-      function: {
-        name: "test_tool",
-        arguments: JSON.stringify({ param1, param2 }),
-      },
-      type: "function",
-    });
-    expectToolResults.push({
-      content: `tool_result: '${param1}' '${String(param2)}'`,
-      role: "tool",
-      tool_call_id: id,
-      _meta: { type: "text/plain" },
-    });
-  }
-
-  // Second tool call (if requested) is tool0
-  if (tool_call_ids.length > 1) {
-    const id = tool_call_ids[1];
-    tool_calls.push({
-      id,
-      function: {
-        name: "tool0",
-        arguments: "",
-      },
-      type: "function",
-    });
-    expectToolResults.push({
-      content: "0",
-      role: "tool",
-      tool_call_id: id,
-    });
-  }
-
-  // 3 calls not supported
-  if (tool_call_ids.length > 2) {
-    throw new Error("3 toolc alls not supported in this test");
-  }
-
-  // A script that uses the tools
-
-  const script: Choice[] = [
-    {
-      index: 0,
-      finish_reason: "tool_calls",
-      message: {
-        content: "calling test_tool.",
-        refusal: null,
-        role: "assistant",
-        tool_calls,
-      },
-      logprobs: null,
-    },
-    {
-      index: 0,
-      finish_reason: "stop",
-      message: {
-        content: "message after tools calls.",
-        refusal: null,
-        role: "assistant",
-      },
-      logprobs: null,
-    },
-  ];
-
-  // Expected output
-
-  const expectAgentMessages = [
-    "calling test_tool.",
-    "message after tools calls.",
-  ];
-
-  return {
-    script,
-    expectCompletions: script.map((s) =>
-      completionToAssistantMessageParam(s.message)
-    ),
-    expectAgentMessages,
-    expectToolResults,
-    tool0_descriptor,
-    tool0_fn,
-    test_tool_descriptor,
-    test_tool_fn,
-  };
-}
-
-/// Return a dummy agent and a TestAgentEventHandler for tracking messages.
-async function createTestAgent(script: Choice[]): Promise<{
-  agent: Agent;
-  skillManager: SkillManager;
-  eventHandler: TestAgentEventHandler;
-}> {
-  const agentProfile: AgentProfile = {
-    model: "dummy:" + JSON.stringify(script),
-    system_prompt: "SYSTEM_PROMPT",
-    mcp_settings: {},
-  };
-
-  const eventHandler = new TestAgentEventHandler();
-
-  const skillManager = await SkillManager.initialize(
-    (url: string, authResultP: Promise<boolean>, displayName: string) => {
-      DUMMY_PLATFORM.openUrl(url, authResultP, displayName);
-    },
-    LOCAL_SERVER_URL,
-    "dummy_key",
-    undefined
-  );
-
-  const agent = Agent.initializeWithLLM(
-    eventHandler,
-    new TestDummyLLM(script),
-    new ContextManager(agentProfile.system_prompt, []),
-    skillManager
-  );
-
-  return { agent, skillManager, eventHandler };
-}
-
-describe("Agent", () => {
-  it("correctly calls custom tools", async function () {
-    const tool_call_id: string = "tool_call_1";
-    const param1: string = "first param";
-    const param2: number = 2;
-
-    const {
-      script,
-      expectAgentMessages,
-      expectToolResults,
-      tool0_descriptor,
-      tool0_fn,
-      test_tool_descriptor,
-      test_tool_fn,
-    } = createCallTestToolScript([tool_call_id], param1, param2);
-    const { agent /*, skillManager */, eventHandler } =
-      await createTestAgent(script);
-
-    agent.addAgentTool(tool0_descriptor, tool0_fn);
-    agent.addAgentTool(test_tool_descriptor, test_tool_fn);
-
-    await agent.userMessageEx("user message 1");
-
-    expect(eventHandler.getAgentMessages()).eql(expectAgentMessages);
-    expect(eventHandler.getToolCallResults()).eql(expectToolResults);
-
-    // Check the ordering
-
-    const allExpect = [
-      completionToAssistantMessageParam(script[0].message),
-      ...expectToolResults,
-      completionToAssistantMessageParam(script[1].message),
-    ];
-    const all = eventHandler.getAll();
-    expect(all).eql(allExpect);
-
-    const allToolCalls = eventHandler.getToolCalls();
-    const agentToolCalls = eventHandler.getAgentToolCalls();
-    const expectToolCalls = script[0].message.tool_calls;
-
-    expect(allToolCalls).eql(expectToolCalls);
-    expect(agentToolCalls).eql(expectToolCalls);
-  });
-
-  it("correctly invokes tool providers", async function () {
-    const tool_call_id: string = "tool_call_1";
-    const param1: string = "asdf";
-    const param2: number = 3;
-    const {
-      script,
-      expectCompletions,
-      expectAgentMessages,
-      expectToolResults,
-      test_tool_descriptor,
-      test_tool_fn,
-    } = createCallTestToolScript([tool_call_id], param1, param2);
-    const { agent /*, skillManager */, eventHandler } =
-      await createTestAgent(script);
-
-    const toolProvider: IAgentToolProvider = {
-      setup: (agent: AgentEx) => {
-        // Add the tool async to test this mechanism
-        return new Promise<void>((r) => {
-          setTimeout(() => {
-            agent.addAgentTool(test_tool_descriptor, test_tool_fn);
-            r();
-          });
-        });
-      },
-    };
-    await agent.addAgentToolProvider(toolProvider);
-
-    await agent.userMessageEx("user message 1");
-
-    expect(eventHandler.getCompletions()).eql(expectCompletions);
-    expect(eventHandler.getAgentMessages()).eql(expectAgentMessages);
-    expect(eventHandler.getToolCallResults()).eql(expectToolResults);
-  });
-
-  it("correctly reflects system prompt", async function () {
-    const llm = new TestDummyLLM(DUMMY_SCRIPT);
-    const contextManager = new ContextManager("agent_prompt", []);
-    const agent = Agent.initializeWithLLM(
-      new TestAgentEventHandler(),
-      llm,
-      contextManager
-    );
-
-    SystemPromptProvider.setGlobalPrompt("global_prompt");
-
-    // NOTE: SystemPromptProvider/ContextManager is unit tested elsewhere so
-    // we assume it computes the correct prompt.  Here we check that this is
-    // reflected in the request to LLM.
-
-    // Check the prompt
-
-    await agent.userMessageEx("user message 1");
-    let lastRequest = llm.getLastRequest();
-    assert(lastRequest);
-    expect(lastRequest[0].content).eql(
-      contextManager.getLLMContext()[0].content
-    );
-    expect(agent.getSystemPrompt()).eql("agent_prompt");
-
-    // Update and re-check the prompt
-
-    agent.setSystemPrompt("agent_prompt_2");
-    await agent.userMessageEx("user message 1");
-    lastRequest = llm.getLastRequest();
-    assert(lastRequest);
-    expect(lastRequest[0].content).eql(
-      contextManager.getLLMContext()[0].content
-    );
-    expect(agent.getSystemPrompt()).eql("agent_prompt_2");
-  });
-
-  it("correctly orders messages for multiple tool calls", async function () {
-    const tool_call_ids: string[] = ["tool_call_1", "tool_call_2"];
-    const param1: string = "asdf";
-    const param2: number = 3;
-    const {
-      script,
-      expectToolResults,
-      test_tool_descriptor,
-      test_tool_fn,
-      tool0_descriptor,
-      tool0_fn,
-    } = createCallTestToolScript(tool_call_ids, param1, param2);
-    const { agent /*, skillManager */, eventHandler } =
-      await createTestAgent(script);
-    agent.addAgentTool(test_tool_descriptor, test_tool_fn);
-    agent.addAgentTool(tool0_descriptor, tool0_fn);
-
-    await agent.userMessageEx("user message 1");
-
-    // Check the event handler was called with all completions and tool
-    // results, in the correct order.
-
-    const allExpect = [
-      completionToAssistantMessageParam(script[0].message),
-      ...expectToolResults,
-      completionToAssistantMessageParam(script[1].message),
-    ];
-    const all = eventHandler.getAll();
-    expect(all).eql(allExpect);
-
-    // The conversation (context) messages (order and number) should also
-    // match this.
-
-    const conv = agent.getConversation();
-    expect(conv.slice(1)).eql(all);
-  });
-
-  it("correctly updates tool call args", async function () {
-    const tool_call_ids: string[] = ["tool_call_1", "tool_call_2"];
-    const param1: string = "asdf";
-    const param2: number = 3;
-    const {
-      script,
-      expectToolResults,
-      test_tool_descriptor,
-      test_tool_fn,
-      tool0_descriptor,
-      tool0_fn,
-    } = createCallTestToolScript(tool_call_ids, param1, param2);
-    const { agent /*, skillManager */, eventHandler } =
-      await createTestAgent(script);
-
-    // Take a copy of the script now, since this will be updated when the
-    // tools redacted their args.
-
-    const scriptCopy: Choice[] = clone(script);
-
-    // Define the tool to update the arg.  The transform is to up-case
-    // `param1: string` and add 100 to `param2: number`.
-
-    type Args = { param1: string; param2: number };
-    const transformArgs = (args: Args): Args => {
-      return { param1: args.param1.toUpperCase(), param2: args.param2 + 100 };
-    };
-    const transformArgsStr = (argsStr: string): string => {
-      const args = JSON.parse(argsStr) as Args;
-      return JSON.stringify(transformArgs(args));
-    };
-    const newToolFn = async (
-      agent: AgentEx,
-      args: unknown
-    ): Promise<ToolCallResult> => {
-      const result = await test_tool_fn(agent, args);
-      result.overwriteArgs = JSON.stringify(transformArgs(args as Args));
-      return result;
-    };
-    agent.addAgentTool(test_tool_descriptor, newToolFn);
-    agent.addAgentTool(tool0_descriptor, tool0_fn);
-
-    // Send a message and trigger the tool calls
-
-    await agent.userMessageEx("user message 1");
-
-    // Check the event handler was called with all completions and tool
-    // results, in the correct order.
-
-    const transformToolCall = (toolCall: MessageToolCall): MessageToolCall => {
-      return {
-        ...toolCall,
-        function: {
-          ...toolCall.function,
-          arguments: transformArgsStr(toolCall.function.arguments),
-        },
-      };
-    };
-
-    assert(scriptCopy[0].message.tool_calls);
-    const transformedFirstMsg = completionToAssistantMessageParam({
-      ...scriptCopy[0].message,
-      tool_calls: [
-        transformToolCall(scriptCopy[0].message.tool_calls[0]),
-        scriptCopy[0].message.tool_calls[1],
-      ],
-    });
-
-    const allExpect: MessageParam[] = [
-      transformedFirstMsg,
-      ...expectToolResults,
-      completionToAssistantMessageParam(scriptCopy[1].message),
-    ];
-    const all = eventHandler.getAll();
-    const conv = agent.getConversation();
-
-    expect(all).eql(allExpect);
-    expect(conv.slice(1)).eql(all);
-  });
-
-  it("correctly updates tool call results", async function () {
-    const tool_call_ids: string[] = ["tool_call_1", "tool_call_2"];
-    const param1: string = "asdf";
-    const param2: number = 3;
-    const {
-      script,
-      expectToolResults,
-      test_tool_descriptor,
-      test_tool_fn,
-      tool0_descriptor,
-      tool0_fn,
-    } = createCallTestToolScript(tool_call_ids, param1, param2);
-    const { agent /*, skillManager */, eventHandler } =
-      await createTestAgent(script);
-
-    // Define the tool to update the results.  The transform is to up-case
-    // the whole result string.
-
-    const transformResult = (result: string): string => {
-      return result.toUpperCase();
-    };
-    const newToolFn = async (
-      agent: AgentEx,
-      args: unknown
-    ): Promise<ToolCallResult> => {
-      const result = await test_tool_fn(agent, args);
-      result.overwriteResponse = transformResult(result.response);
-      return result;
-    };
-    agent.addAgentTool(tool0_descriptor, tool0_fn);
-    agent.addAgentTool(test_tool_descriptor, newToolFn);
-
-    // Send a message and trigger the tool calls
-
-    await agent.userMessageEx("user message 1");
-
-    // Check the event handler was called with all completions and tool
-    // results as expected.
-
-    const expectToolCallResults: MessageParam[] = expectToolResults.map(
-      (tr) => {
-        assert(typeof tr.content === "string");
-        return {
-          ...tr,
-          content: transformResult(tr.content),
-        };
-      }
-    );
-
-    const allExpect: MessageParam[] = [
-      completionToAssistantMessageParam(script[0].message),
-      ...expectToolCallResults,
-      completionToAssistantMessageParam(script[1].message),
-    ];
-    const all = eventHandler.getAll();
-    const conv = agent.getConversation();
-
-    expect(all).eql(allExpect);
-    expect(conv.slice(1)).eql(all);
-  });
-
-  it("correctly handles LLM errors", async function () {
-    const eventHandler = new TestAgentEventHandler();
-    const llm = new TestErrorLLM();
-    const ctxMgr = new ContextManager("SYSTEM_PROMPT", []);
-    const agent = Agent.initializeWithLLM(eventHandler, llm, ctxMgr);
-
-    await expect(
-      agent.userMessagesRaw([{ role: "user", content: "message" }])
-    ).rejects.toThrow();
-
-    // Check the Agent has not added the user messages.
-    expect(ctxMgr.getLLMContext().length).eql(1);
-  });
-
-  it("supports reasoning token callbacks", async function () {
-    const script: DummyLLMScriptEntry[] = [
-      {
-        finish_reason: "reasoning",
-        message: "Some reasoning tokens...",
-      },
-      {
-        index: 0,
-        finish_reason: "stop",
-        message: {
-          content: "Agent Message",
-          refusal: null,
-          role: "assistant",
-        },
-        logprobs: null,
-      },
-    ];
-
-    const llm = new TestDummyLLM(script);
-    const eventHandler = new TestAgentEventHandler();
-    const ctxMgr = new ContextManager("SYSTEM_PROMPT", []);
-    const agent = Agent.initializeWithLLM(eventHandler, llm, ctxMgr);
-
-    await agent.userMessageEx("user message 1");
-    expect(eventHandler.getReasoning()).eql([script[0].message, ""]);
-  });
-});