@xalia/agent 0.6.1 → 0.6.2

package/scripts/test_chat

@@ -47,6 +47,7 @@ stop_chat_server
 # Start the server with full logging and short heartbeat timeout.
 export DEFAULT_LLM_MODEL=repeat
 export DEVELOPMENT=1
+export GEN_IMAGE_MODEL=dummy:test_data/dummyllm_script_image_gen.json
 
 LOG_LEVEL=debug \
     COMPRESSION_TRIGGER_NUM_MESSAGES=3 \
@@ -94,87 +95,128 @@ pushd _test_chat
 
   # Create an agent for the single-user tests
 
-  echo '{"system_prompt":"You are a helpful agent in a multi-user chat session.  If you have nothing to contribute and are not being addressed, just say `No comment`.","model":"'${DEFAULT_LLM_MODEL}'","mcp_settings":{}}' > agent_profile.json
+  echo '{"system_prompt":"You are a helpful agent in a multi-user chat session.  If you have nothing to contribute and are not being addressed, just say `No comment`.","mcp_settings":{}}' > agent_profile.json
   ${client} agent-profile set profile0 --profile agent_profile.json | jq -r .uuid > agent_profile.uuid
   agent_profile_id=`cat agent_profile.uuid`
 
-  # User 0 tries to join with invalid API key - fails with "invalid api key" error
+  # # User 0 tries to join with invalid API key - fails with "invalid api key" error
 
-  invalid_apikey0="invalid_api_key"
-  ${agent} chat client --session-id no_such_session --api-key "${invalid_apikey0}" 2>&1 | tee invalid_apikey_test.log | grep -i "invalid api key" || \
-      (echo "Expected 'invalid api key' error message"; cat invalid_apikey_test.log; exit 1)
+  # invalid_apikey0="invalid_api_key"
+  # ${agent} chat client --session-id no_such_session --api-key "${invalid_apikey0}" 2>&1 | tee invalid_apikey_test.log | grep -i "invalid api key" || \
+  #     (echo "Expected 'invalid api key' error message"; cat invalid_apikey_test.log; exit 1)
 
-  # User 0 tries to join non-existant session - fails
+  # # User 0 tries to join non-existant session - fails
 
-  ${agent} chat clear-sessions
-  ${agent} chat client --session-id no_such_session && \
-      (echo "Should not be able to join"; exit 1)
+  # ${agent} chat clear-sessions
+  # ${agent} chat client --session-id no_such_session && \
+  #     (echo "Should not be able to join"; exit 1)
 
-  # User 0 creates a new session with `crash_agent_profile` and interacts with
-  # it, exercising LLM error conditions.
+  # # User 0 creates a new session with `crash_agent_profile` and interacts with
+  # # it, exercising LLM error conditions.
 
-  echo ':si crash_session.id' > crash_script
-  echo 'Hello' >> crash_script
-  echo 'Hello again' >> crash_script
-  ${agent} chat client --session-title crash_test_session \
-           --agent-profile-id ${crash_agent_profile_id} \
-           --script crash_script
+  # echo ':si crash_session.id' > crash_script
+  # echo 'Hello' >> crash_script
+  # echo 'Hello again' >> crash_script
+  # ${agent} chat client --session-title crash_test_session \
+  #          --agent-profile-id ${crash_agent_profile_id} \
+  #          --script crash_script
 
-  # User 0 creates empty session, which should not appear in the DB.
+  # # User 0 creates empty session, which should not appear in the DB.
 
-  echo ':si empty_session.id' > empty_session
-  ${agent} chat client --session-title test_session \
-           --agent-profile-id ${agent_profile_id} \
-           --script empty_session
-  empty_session_id=`cat empty_session.id`
-  ${agent} chat list-sessions | grep  `cat empty_session.id` && \
-      (echo "Empty session should not be persisted"; exit 1)
+  # echo ':si empty_session.id' > empty_session
+  # ${agent} chat client --session-title test_session \
+  #          --agent-profile-id ${agent_profile_id} \
+  #          --script empty_session
+  # empty_session_id=`cat empty_session.id`
+  # ${agent} chat list-sessions | grep  `cat empty_session.id` && \
+  #     (echo "Empty session should not be persisted"; exit 1)
 
   # User 0 creates a user session, sends message and writes session id to file
 
-  echo 'tell me a joke' > init_session
+  echo ':pause-agent 1' > init_session
+  echo 'tell me a joke' >> init_session
   echo ':si session.id' >> init_session
+  echo ':share-session guest.key' >> init_session
+  echo ':pause-agent 0' >> init_session
+  echo 'now please' >> init_session
   ${agent} chat client --session-title test_session \
            --agent-profile-id ${agent_profile_id} \
            --script init_session
   session_id=`cat session.id`
+  guest_key=`cat guest.key`
+
+  # # Check that the session got saved with the updated title
+
+  # ${agent} chat list-sessions | grep '"title":"tell me a joke"'
+
+  # # Check that a guest can join with the guest key
 
-  # Check that the session got saved with the updated title
+  # echo "Hello from the guest" > guest_script
+  # ${agent} chat client \
+  #          --session-id ${session_id} \
+  #          --api-key ${guest_key} \
+  #          --script guest_script | tee guest_output.txt
+  # grep "joke" guest_output.txt
+  # grep "[Guest]" guest_output.txt
 
-  ${agent} chat list-sessions | grep '"title":"tell me a joke"'
+  # # Dummy session to invoke the image generation
 
-  # User 1 tries to join the session.  Should be rejected.
+  # echo '{"system_prompt":"prompt","model":"dummy:test_data/dummyllm_script_invoke_image_gen_tool.json","mcp_settings":{}}' > invoke_gen_image_profile.json
+  # ${client} agent-profile set invoke_gen_image_agent_profile \
+  #           --profile invoke_gen_image_profile.json \
+  #     | jq -r .uuid > invoke_gen_image_profile.uuid
+  # invoke_gen_image_profile_id=`cat invoke_gen_image_profile.uuid`
 
-  ${agent} chat client --session-id ${session_id} --api-key ${apikey1} \
-           --script init_session > join_unauthorized.log 2>&1 && \
-      (echo "Should exit with error when joining invalid session"; exit 1)
+  # echo "Choose an animal and generate an image of it" > invoke_gen_image_script
+  # echo ":list-files" >> invoke_gen_image_script
+  # echo ":si invoke_gen_image.id" >> invoke_gen_image_script
 
-  grep -i "not authorized" join_unauthorized.log || \
-      (echo "Should include error message"; exit 1)
+  # LOG_LEVEL=debug \
+  #     ${agent} chat client \
+  #     --session-title invoke_gen_image_session \
+  #     --agent-profile-id ${invoke_gen_image_profile_id} \
+  #     --script invoke_gen_image_script | tee invoke_gen_image_output.txt
+  # grep 'frog.png' invoke_gen_image_output.txt
+  # grep 'metadata: {"type":"image/png"}' invoke_gen_image_output.txt
+
+  # # User 1 tries to join the session.  Should be rejected.
+
+  # ${agent} chat client --session-id ${session_id} --api-key ${apikey1} \
+  #          --script init_session > join_unauthorized.log 2>&1 && \
+  #     (echo "Should exit with error when joining invalid session"; exit 1)
+
+  # grep -i "not authorized" join_unauthorized.log || \
+  #     (echo "Should include error message"; exit 1)
 
   # User 0 create a team, and write team id to a file
 
   echo ':ct team0' > script_create_team
   echo ':ci team0.id' >> script_create_team
-  ${agent} chat client --session-id ${session_id} --script script_create_team
+  ${agent} chat client \
+           --session-id ${session_id} \
+           --script script_create_team
   team_id=`cat team0.id`
 
-  # User 0 create a new team session (including a new agent), and write
-  # session id to a file.
+  # User 0 create a new team session (including a new paused agent), and write
+  # session id to a file.  Extract the agent_profile UUID.
 
-  echo 'tell me a joke' > script_team_chat
+  echo ':pause-agent 1' > script_team_chat
+  echo 'tell me a joke' >> script_team_chat
   echo ':si team_session.id' >> script_team_chat
   ${agent} chat client --session-title test_team_session \
            --team-id ${team_id} \
-           --script script_team_chat
+           --script script_team_chat | tee team_chat_1
   team_session_id=`cat team_session.id`
 
-  # Extract the agent uuid form the new session
-
   get_profile_id_query='.[] | select(.uuid=="'${team_session_id}'") | .agent_profile_uuid'
   ${agent} chat list-sessions | jq -r "${get_profile_id_query}" > team_agent.id
   team_agent_id=`cat team_agent.id`
 
+  # Check that the agent has not said anything
+
+  grep AGENT team_chat_1 && \
+      (echo "Unexpected AGENT messages"; exit 1)
+
   # User 1 tries to join the team session.  Should be rejected.
 
   ${agent} chat client --session-id ${team_session_id} --api-key ${apikey1} \
@@ -183,36 +225,53 @@ pushd _test_chat
   grep -i "not authorized" join_unauthorized_team.log || \
       (echo "Should include error message (no auth team session)"; exit 1)
 
-  # User 0 adds user 1 as a participant in team and checks the DB
+  # User0 sets up the session
+  #   add charuser1 to the team
+  #   add duckduckgo-search
+  #   add a session file (md file)
+  #   add an image to the workspace
+  #   unpause the agent
+
+  echo ":ap ${team_id} chatuser1" > script_setup_team_session
+  echo ':lp' >> script_setup_team_session
+  echo ":as duckduckgo-search" >> script_setup_team_session
+  echo ':put-file plan.md data:text/markdown,#PLAN\n##STEP1\n##STEP2\n' \
+       >> script_setup_team_session
+  echo ':sw file:../test_data/frog.png For context, this image represents our shared workspace' >> script_setup_team_session
+  echo ':pause-agent 0' >> script_setup_team_session
+  ${agent} chat client \
+           --session-id ${team_session_id} \
+           --script script_setup_team_session
+
+  # checks the DB for chatuser2 participant and duckduckgo mcp server
 
-  echo ":ap ${team_id} chatuser1" > script_add_chatuser1
-  echo ':lp' >> script_add_chatuser1
-  ${agent} chat client  --session-id ${team_session_id} --script script_add_chatuser1
   ${agent} chat list-participants --session ${team_session_id} | grep chatuser1
+  ${client} agent-profile get ${team_agent_id} | grep duckduckgo
 
-  # Have a participant add duckduckgo-search
-  # Check that the duckduckgo-search server is available
-  # Have a participant remove it and check that got reflected in the db
+  # Check the state.  Clean the workspace, file and mcp servers
 
-  echo ":as duckduckgo-search" > script_add_duckduckgo
-  ${agent} chat client --session-id ${team_session_id} --script script_add_duckduckgo
-  ${client} agent-profile get ${team_agent_id} | grep duckduckgo
-  sleep 1
+  echo ':sleep 1000' > script_check_state
+  echo ':list-files filelist.json' >> script_check_state
+  echo 'Tell me about the image in our workspace?' >> script_check_state
+  echo ':sw' >> script_check_state
+  echo 'How many steps are in plan.md?' >> script_check_state
+  echo ':sleep 5000' >> script_check_state
+  echo ':delete-file plan.md' >> script_check_state
+  echo ':rs duckduckgo-search' >> script_check_state
+  ${agent} chat client \
+           --session-id ${team_session_id} \
+           --script script_check_state
 
-  echo ":rs duckduckgo-search" > script_rm_duckduckgo
-  ${agent} chat client --session-id ${team_session_id} --script script_rm_duckduckgo
   sleep 1
-  (${client} agent-profile get ${team_session_id} | grep duckduckgo) && ( \
+
+  # Check duckduckgo-search got removed
+  (${client} agent-profile get ${team_agent_id} | grep duckduckgo) && ( \
       echo "ERROR: expected duckduckgo to be removed from agent profile" ; \
       exit 1 \
       )
 
-  # Set an image to the workspace and asks a question
-
-  echo ':sw file:../test_data/frog.png For context, this image represents our shared workspace' > script_set_workspace
-  echo 'Tell me about the image in our workspace?' >> script_set_workspace
-  echo ':sw' >> script_set_workspace
-  ${agent} chat client --session-id ${session_id} --script script_set_workspace
+  # Check the file manager list
+  grep 'plan.md' filelist.json
 
   # Run both clients with a script that continues the conversation
 
@@ -222,12 +281,11 @@ pushd _test_chat
   ${agent} chat client --session-id ${team_session_id} --api-key ${apikey1} --script script1 >chatuser1.output
   wait ${pid0}
 
-  # The name chatuser0 should appear in chatuser1's session output
+  # The name chatuser0 and AGENT should appear in chatuser1's session output
 
-  if ! (grep chatuser0 chatuser1.output) ; then
-      echo "error: expected chatuser0 in chatuser1's output"
-      exit 1
-  fi
+  grep chatuser0 chatuser1.output
+  grep chatuser1 chatuser1.output
+  grep AGENT chatuser1.output
 
 popd
 

package/src/agent/agent.ts

@@ -1,15 +1,35 @@
-import * as dotenv from "dotenv";
 import { OpenAI } from "openai";
 import { McpServerManager } from "./mcpServerManager";
 import { strict as assert } from "assert";
-import { ILLM } from "./llm";
+import {
+  ILLM,
+  ChatCompletionMessageParam,
+  ChatCompletionUserMessageParam,
+  ChatCompletionMessageToolCall,
+  ChatCompletion,
+  ChatCompletionAssistantMessageParam,
+  ChatCompletionMessage,
+} from "./llm";
 import { AgentProfile, getLogger } from "@xalia/xmcp/sdk";
 import { IAgentEventHandler } from "./iAgentEventHandler";
 import { IContextManager } from "./context";
 export { AgentProfile } from "@xalia/xmcp/sdk";
 
+export const DEFAULT_LLM_URL = "http://localhost:5001/v1";
+
 const MAX_TOOL_CALL_RESPONSE_LENGTH = 4000;
 
+/**
+ * An agent's response, with optional extra image data.
+ * `ChatCompletionMessageParam` may one day be updated to allow image data (as
+ * it does for audio data), but for now image data is not included and so we
+ * keep it separate.
+ */
+export type AssistantResponse = {
+  message: ChatCompletionMessageParam;
+  images?: OpenAI.Chat.Completions.ChatCompletionContentPartImage[];
+};
+
 export interface IAgentToolProvider {
   /**
    * Any initial setup to be performed by the tool (loading data, etc).  This
@@ -25,6 +45,11 @@ export type ToolCallResult = {
    */
   response: string;
 
+  /**
+   * Application-specific meta data about the tool call result.
+   */
+  metadata?: Record<string, string>;
+
   /**
    * If set, `response` is used in the next round of the Agent loop (if any),
    * but `overwriteResponse` is passed to the ContextManager to be stored, and
@@ -49,21 +74,6 @@ export type ToolHandler = (
 
 export type McpServerUrls = (name: string) => string;
 
-export type ChatCompletionMessageParam = OpenAI.ChatCompletionMessageParam;
-
-export type ChatCompletionMessageToolCall =
-  OpenAI.ChatCompletionMessageToolCall;
-
-export type ChatCompletionAssistantMessageParam =
-  OpenAI.ChatCompletionAssistantMessageParam;
-
-export type ChatCompletionUserMessageParam =
-  OpenAI.ChatCompletionUserMessageParam;
-
-export type ChatCompletionToolMessageParam =
-  OpenAI.ChatCompletionToolMessageParam;
-
-dotenv.config();
 const logger = getLogger();
 
 export interface IConversation {
@@ -153,7 +163,7 @@ export class Agent implements IConversation {
     msg?: string,
     imageB64?: string,
     name?: string
-  ): Promise<ChatCompletionMessageParam | undefined> {
+  ): Promise<AssistantResponse | undefined> {
     const userMessage = createUserMessage(msg, imageB64, name);
     if (!userMessage) {
       return undefined;
@@ -163,13 +173,22 @@ export class Agent implements IConversation {
 
   public async userMessageRaw(
     userMessage: ChatCompletionUserMessageParam
-  ): Promise<ChatCompletionMessageParam | undefined> {
+  ): Promise<AssistantResponse | undefined> {
     return this.userMessagesRaw([userMessage]);
   }
 
   public async userMessagesRaw(
     userMessages: ChatCompletionUserMessageParam[]
-  ): Promise<ChatCompletionMessageParam | undefined> {
+  ): Promise<AssistantResponse | undefined> {
+    // Image and audio handling
+    //
+    // `ChatCompletions` (responses from the LLM) can contain `audio` and
+    // `images` tags.  However, the `ChatCompletionMessageParam` type does not
+    // allow for "assistant" messages with images / audio.
+    //
+    // As such, our current approach is to extract all assistant-generated
+    // media and return it separately.
+
     // Note: `getLLMContext` returns a copy to we can mutate this array
     const context = this.contextManager.getLLMContext();
     const newMessagesIdx = context.length;
@@ -177,8 +196,13 @@ export class Agent implements IConversation {
     // Add the new user messages
     context.push(...userMessages);
 
+    const images: OpenAI.Chat.Completions.ChatCompletionContentPartImage[] = [];
+
+    // We convert the `ChatCompletionsMessage` into a
+    // `ChatCompletionAssistantMessageParam` and extract image data.
+
     let completion = await this.chatCompletion(context);
-    let message = completion.choices[0].message;
+    let message = this.processCompletion(completion, images);
     context.push(message);
 
     // While there are tool calls to make, invoke them and loop
@@ -199,6 +223,7 @@ export class Agent implements IConversation {
           role: "tool",
           tool_call_id: toolCall.id,
           content: result.response,
+          metadata: result.metadata,
         });
 
         // If the tool call requested that its args be redacted, this can be
@@ -226,14 +251,14 @@ export class Agent implements IConversation {
 
       try {
         completion = await this.chatCompletion(context); // CAN THROW
-        message = completion.choices[0].message;
+        message = this.processCompletion(completion, images);
         context.push(message);
       } finally {
         // Now that the tool call results have been passed to the LLM, perform
         // any updates on them.  Pass the (updated) tool-call-result LLM
-        // messages to the event handler - note, we want to do this even if
-        // the an error occured, so that the caller has an up-to-date picture
-        // of the context state when the error occured.
+        // messages to the event handler - note, we want to do this even if an
+        // error occured, so that the caller has an up-to-date picture of the
+        // context state when the error occured.
 
         toolCallResults.forEach(([indexInContext, tcr]) => {
           const ctxMsg = context[indexInContext];
@@ -256,7 +281,7 @@ export class Agent implements IConversation {
     // Add all new new messages to the context
     this.contextManager.addMessages(context.slice(newMessagesIdx));
 
-    return completion.choices[0].message;
+    return { message, images: images.length === 0 ? undefined : images };
   }
 
   public userMessage(msg?: string, imageB64?: string): void {
@@ -285,19 +310,22 @@ export class Agent implements IConversation {
 
   async chatCompletion(
     context: ChatCompletionMessageParam[]
-  ): Promise<OpenAI.Chat.Completions.ChatCompletion> {
+  ): Promise<ChatCompletion> {
     // Compute the full list of available tools
 
     let tools: OpenAI.ChatCompletionTool[] | undefined;
     const mcpTools = this.mcpServerManager.getOpenAITools();
+    logger.debug(`[chatCompletion] mcpTools: ${JSON.stringify(mcpTools)}`);
     const enabledTools = this.tools.concat(mcpTools);
     if (enabledTools.length > 0) {
       tools = enabledTools;
     }
+    logger.debug(`[chatCompletion] tools: ${JSON.stringify(tools)}`);
     const completion = await this.llm.getConversationResponse(
       context,
       tools,
-      this.eventHandler.onAgentMessage.bind(this.eventHandler)
+      this.eventHandler.onAgentMessage.bind(this.eventHandler),
+      this.eventHandler.onReasoning.bind(this.eventHandler)
     );
     logger.debug(`Received chat completion ${JSON.stringify(completion)}`);
     return completion;
@@ -359,19 +387,26 @@ export class Agent implements IConversation {
       const toolName = toolCall.function.name;
       const agentTool = this.agentTools.get(toolName);
       const isAgentTool = !!agentTool;
-      const approve = await this.eventHandler.onToolCall(toolCall, isAgentTool);
-      if (!approve) {
-        result = { response: "User denied tool request." };
-      } else if (isAgentTool) {
+
+      if (isAgentTool) {
         // Internal (agent) tool
-        const args: unknown = JSON.parse(toolCall.function.arguments);
-        result = await agentTool.handler(this, args);
+        if (!(await this.eventHandler.onToolCall(toolCall, true))) {
+          result = { response: "User denied tool request." };
+        } else {
+          const args: unknown = JSON.parse(toolCall.function.arguments);
+          result = await agentTool.handler(this, args);
+        }
       } else {
-        // McpServer tool call (agentTool === undefined)
+        // McpServer tool call (agentTool === undefined).  Sanity check the
+        // tool call data, get approval, and then invoke.
+
         const args: unknown = JSON.parse(toolCall.function.arguments);
-        result = {
-          response: await this.mcpServerManager.invoke(toolName, args),
-        };
+        const tc = this.mcpServerManager.verifyToolCall(toolName, args);
+        if (!(await this.eventHandler.onToolCall(toolCall, false))) {
+          result = { response: "User denied tool request." };
+        } else {
+          result = { response: await this.mcpServerManager.invoke(tc) };
+        }
         logger.debug(`tool call result ${JSON.stringify(result)}`);
       }
     } catch (e) {
@@ -402,6 +437,23 @@ export class Agent implements IConversation {
 
     return result;
   }
+
+  private processCompletion(
+    completion: ChatCompletion,
+    images: OpenAI.Chat.Completions.ChatCompletionContentPartImage[]
+  ): ChatCompletionAssistantMessageParam {
+    // Add any images into the list, and call the event handler
+
+    const compMessage = completion.choices[0].message;
+    if (compMessage.images) {
+      for (const image of compMessage.images) {
+        this.eventHandler.onImage(image);
+        images.push(image);
+      }
+    }
+
+    return completionToAssistantMessageParam(compMessage);
+  }
 }
 
 /**
@@ -460,3 +512,58 @@ export function createUserMessageEnsure(
   assert(userMsg);
   return userMsg;
 }
+
+export function completionToAssistantMessageParam(
+  compMessage: ChatCompletionMessage
+): ChatCompletionAssistantMessageParam {
+  // Strip down the `ChatCompletionMessage` to a
+  // `ChatCompletionAssistantMessageParam`, only including the non-null
+  // elements.  For reference:
+  //
+  // Response from the LLM:
+  //
+  //   export interface ChatCompletionMessage {
+  //     role: 'assistant';
+  //     audio?: ChatCompletionAudio | null;
+  //     content: string | null;
+  //     refusal: string | null;
+  //     tool_calls?: Array<ChatCompletionMessageToolCall>;
+  //
+  //     annotations?: Array<ChatCompletionMessage.Annotation>;
+  //     // openrouter
+  //     images?: Array<ChatCompletionContentPartImage>
+  //   }
+  //
+  // Input to the LLM
+  //
+  //   export interface ChatCompletionAssistantMessageParam {
+  //     role: "assistant";
+  //     audio?: ChatCompletionAssistantMessageParam.Audio | null;
+  //     content?:
+  //       | string
+  //       | Array<ChatCompletionContentPartText |
+  //               ChatCompletionContentPartRefusal>
+  //       | null;
+  //     refusal?: string | null;
+  //     tool_calls?: Array<ChatCompletionMessageToolCall>;
+  //
+  //     name?: string;
+  //   }
+
+  const message: ChatCompletionAssistantMessageParam = {
+    role: "assistant",
+  };
+  if (compMessage.audio) {
+    message.audio = compMessage.audio;
+  }
+  if (compMessage.content) {
+    message.content = compMessage.content;
+  }
+  if (compMessage.refusal) {
+    message.refusal = compMessage.refusal;
+  }
+  if (compMessage.tool_calls) {
+    message.tool_calls = compMessage.tool_calls;
+  }
+  return message;
+}