@xalia/agent 0.5.7 → 0.5.8

package/src/chat/client.ts

@@ -1,20 +1,16 @@
 import { Tool } from "@modelcontextprotocol/sdk/types.js";
 import { strict as assert } from "assert";
-import * as websocket from "ws";
 
 import { AgentProfile, McpServerBrief, getLogger } from "@xalia/xmcp/sdk";
 
 import { ISkillManager } from "../agent/sudoMcpServerManager";
-import {
-  IMcpServerManager,
-  McpServerInfo,
-  McpServerInfoRW,
-} from "../agent/mcpServerManager";
-
-import { ServerToClient, ClientToServer } from "./messages";
+import { McpServerInfo, McpServerInfoRW } from "../agent/mcpServerManager";
 import { ChatCompletionMessageParam, IConversation } from "../agent/agent";
 import { IPlatform } from "../agent/iplatform";
 
+import { WebSocket } from "./websocket";
+import { ServerToClient, ClientToServer } from "./messages";
+
 const logger = getLogger();
 
 type OnMessageCallback = (msg: ServerToClient) => void;
@@ -25,10 +21,19 @@ interface IMessageSender {
   sendMessage(message: ClientToServer): void;
 }
 
-class RemoteMcpServerManager implements IMcpServerManager {
+class RemoteSudoMcpServerManager implements ISkillManager {
+  private sender: IMessageSender;
+  private briefs: McpServerBrief[];
+  private briefsMap: Record<string, McpServerBrief>;
   private mcpServers: { [serverName: string]: McpServerInfoRW } = {};
 
-  constructor(private sender: IMessageSender) {
+  constructor(sender: IMessageSender, briefs: McpServerBrief[]) {
+    this.sender = sender;
+    this.briefs = briefs;
+    this.briefsMap = {};
+    briefs.forEach((b) => {
+      this.briefsMap[b.name] = b;
+    });
     this.mcpServers = {};
   }
 
@@ -161,25 +166,6 @@ class RemoteMcpServerManager implements IMcpServerManager {
 
     server.disableTool(toolName);
   }
-}
-
-class RemoteSudoMcpServerManager implements ISkillManager {
-  private briefsMap: Record<string, McpServerBrief>;
-
-  constructor(
-    private sender: IMessageSender,
-    private briefs: McpServerBrief[],
-    private msm: RemoteMcpServerManager
-  ) {
-    this.briefsMap = {};
-    briefs.forEach((b) => {
-      this.briefsMap[b.name] = b;
-    });
-  }
-
-  getMcpServerManager(): IMcpServerManager {
-    return this.msm;
-  }
 
   getServerBriefs(): McpServerBrief[] {
     return this.briefs;
@@ -196,22 +182,23 @@ class RemoteSudoMcpServerManager implements ISkillManager {
       enable_all,
     });
   }
+
+  async shutdown(): Promise<void> {}
 }
 
 export class ChatClient implements IMessageSender, IConversation {
   private platform: IPlatform;
-  private ws: websocket.WebSocket;
+  private ws: WebSocket;
   private onMessageCB: OnMessageCallback;
   private onConnectionClosedCB: OnConnectionClosedCallback;
   private closed: boolean;
-  private msm: RemoteMcpServerManager;
   private smsm: RemoteSudoMcpServerManager;
   private systemPrompt: string;
   private model: string;
 
   private constructor(
     platform: IPlatform,
-    ws: websocket.WebSocket,
+    ws: WebSocket,
     onMessageCB: OnMessageCallback,
     onConnectionClosedCB: OnConnectionClosedCallback,
     serverBriefs: McpServerBrief[]
@@ -221,8 +208,7 @@ export class ChatClient implements IMessageSender, IConversation {
     this.onMessageCB = onMessageCB;
     this.onConnectionClosedCB = onConnectionClosedCB;
     this.closed = false;
-    this.msm = new RemoteMcpServerManager(this);
-    this.smsm = new RemoteSudoMcpServerManager(this, serverBriefs, this.msm);
+    this.smsm = new RemoteSudoMcpServerManager(this, serverBriefs);
     this.systemPrompt = "";
     this.model = "";
   }
@@ -240,7 +226,7 @@ export class ChatClient implements IMessageSender, IConversation {
       const urlParams = new URLSearchParams(params);
       const url = `ws://${host}:${port}?${urlParams}`;
 
-      const ws = new websocket.WebSocket(url, [token]);
+      const ws = new WebSocket(url, [token]);
       logger.info("created ws");
 
       let client: ChatClient | undefined = undefined;
@@ -271,7 +257,7 @@ export class ChatClient implements IMessageSender, IConversation {
       ws.onopen = async () => {
         logger.info("opened");
 
-        ws.onmessage = (ev: websocket.MessageEvent) => {
+        ws.onmessage = (ev: MessageEvent) => {
           try {
             const msgData = ev.data;
             if (typeof msgData !== "string") {
@@ -288,10 +274,10 @@ export class ChatClient implements IMessageSender, IConversation {
         };
       };
 
-      ws.onclose = async (err) => {
+      ws.onclose = async (ev: CloseEvent) => {
         logger.info("closed");
         logger.info(
-          `[client] WebSocket connection closed: ${JSON.stringify(err)}`
+          `[client] WebSocket connection closed: ${JSON.stringify(ev)}`
         );
         if (client) {
           client.closed = true;
@@ -299,9 +285,9 @@ export class ChatClient implements IMessageSender, IConversation {
         await onConnectionClosedCB();
       };
 
-      ws.onerror = async (error) => {
-        logger.error("[client] WebSocket error:", JSON.stringify(error));
-        e(error);
+      ws.onerror = async (ev: Event) => {
+        logger.error("[client] WebSocket error:", JSON.stringify(ev));
+        e(ev);
 
         if (client) {
           client.closed = true;
@@ -381,6 +367,10 @@ export class ChatClient implements IMessageSender, IConversation {
     throw "resetConversation not implemented for ChatClient";
   }
 
+  shutdown(): Promise<void> {
+    throw "shutdown not implemented for ChatClient";
+  }
+
   public sendMessage(message: ClientToServer): undefined {
     assert(!this.closed);
     const data = JSON.stringify(message);
@@ -401,20 +391,20 @@ export class ChatClient implements IMessageSender, IConversation {
       //
 
       case "mcp_server_added":
-        this.msm.onMcpServerAdded(
+        this.smsm.onMcpServerAdded(
           message.server_name,
           message.tools,
           message.enabled_tools
         );
         break;
       case "mcp_server_removed":
-        this.msm.onMcpServerRemoved(message.server_name);
+        this.smsm.onMcpServerRemoved(message.server_name);
         break;
       case "mcp_server_tool_enabled":
-        this.msm.onMcpServerToolEnabled(message.server_name, message.tool);
+        this.smsm.onMcpServerToolEnabled(message.server_name, message.tool);
         break;
       case "mcp_server_tool_disabled":
-        this.msm.onMcpServerToolDisabled(message.server_name, message.tool);
+        this.smsm.onMcpServerToolDisabled(message.server_name, message.tool);
         break;
       case "system_prompt_updated":
         this.systemPrompt = message.system_prompt;

package/src/chat/conversationManager.ts

@@ -8,6 +8,8 @@ import {
   Agent,
   ChatCompletionMessageParam,
   ChatCompletionMessageToolCall,
+  ChatCompletionUserMessageParam,
+  createUserMessage,
 } from "../agent/agent";
 import { SkillManager } from "../agent/sudoMcpServerManager";
 import { createAgentWithSkills } from "../agent/agentUtils";
@@ -21,6 +23,7 @@ import type {
   ServerMcpServerToolDisabled,
   ServerSystemPromptUpdated,
   ServerModelUpdated,
+  ServerUserMessage,
 } from "./messages";
 import { AsyncQueue } from "./asyncQueue";
 import { Database, UserData } from "./db";
@@ -81,27 +84,49 @@ export class OpenSession {
       throw new UserAlreadyConnected();
     }
 
-    // Inform any other participants, and add the WebSocket to the map.
+    // Inform any other participants, then add the new WebSocket to the map.
 
     this.broadcast({ type: "user_joined", user: userName });
     this.connections[userName] = ws;
 
-    // TODO:
-    //
-    //   - send conversation state (ServerHistory)
+    // Send MCP server briefs first (client expects this)
 
     const briefs = this.skillManager.getServerBriefs();
     this.sendTo(userName, { type: "mcp_server_briefs", server_briefs: briefs });
 
+    // Send conversation
+
+    const conversation = this.agent.getConversation();
+    const convMessages = conversationToChatMessages(conversation, userName);
+    for (const chatMsg of convMessages) {
+      this.sendTo(userName, chatMsg);
+    }
+
+    // Update the MSM state
+
+    const agentProfile = this.agent.getAgentProfile();
     const msm = this.agent.getMcpServerManager();
-    const mcpServerNames = msm.getMcpServerNames();
-    for (const serverName of mcpServerNames) {
-      const info = msm.getMcpServer(serverName);
+    for (const server_name in agentProfile.mcp_settings) {
+      const tools = msm.getMcpServer(server_name).getTools();
+      const enabled_tools = agentProfile.mcp_settings[server_name];
       this.sendTo(userName, {
         type: "mcp_server_added",
-        server_name: serverName,
-        tools: info.getTools(),
-        enabled_tools: Object.keys(info.getEnabledTools()),
+        server_name,
+        tools,
+        enabled_tools,
+      });
+    }
+
+    // Send system prompt and model
+
+    this.sendTo(userName, {
+      type: "system_prompt_updated",
+      system_prompt: agentProfile.system_prompt,
+    });
+    if (agentProfile.model) {
+      this.sendTo(userName, {
+        type: "model_updated",
+        model: agentProfile.model,
       });
     }
 
@@ -215,7 +240,11 @@ export class OpenSession {
     });
   }
 
-  async onChatMessage(message: string, userToken: string): Promise<undefined> {
+  async onChatMessage(
+    message: string,
+    // imageB64: string | undefined,
+    userToken: string
+  ): Promise<undefined> {
     // We manually broadcast the user's message here and start the agent
     // conversation, and then wait to get back all data from the agent before
     // processing further messages from clients.
@@ -225,11 +254,33 @@ export class OpenSession {
       type: "user_msg",
       message_id: msgId,
       message,
+      // imageB64,
       from: userToken,
     });
 
     this.curAgentMsgId = `${msgId}-resp`;
-    await this.agent.userMessageEx(message, undefined, userToken);
+    const userMessageParam = createUserMessage(
+      message,
+      /*imageB64*/ undefined,
+      userToken
+    );
+    if (!userMessageParam) {
+      logger.debug(`ignoring empty message: ${message}`);
+      return;
+    }
+
+    const assistantReply = await this.agent.userMessageRaw(userMessageParam);
+    if (assistantReply) {
+      // TODO: consider including all messages (including tool calls)
+      // const newEntries = agent.getTrailingEntries(prevConvLength);
+      // await this.db.sessionConversationAppend(this.sessionUUID, newEntries);
+
+      const newEntries: ChatCompletionMessageParam[] = [
+        userMessageParam,
+        assistantReply,
+      ];
+      await this.db.sessionConversationAppend(this.sessionUUID, newEntries);
+    }
   }
 
   async onAddMcpServer(
@@ -593,3 +644,87 @@ export class ConversationManager {
     return openSession;
   }
 }
+
+export function userMessageToChatMessage(
+  userMessage: ChatCompletionUserMessageParam,
+  message_id: string,
+  defaultName: string
+): ServerUserMessage | undefined {
+  const from = userMessage.name || defaultName;
+  if (typeof userMessage.content === "string") {
+    return {
+      type: "user_msg",
+      message_id,
+      message: userMessage.content,
+      from,
+    };
+  }
+
+  let message = "";
+  let image = undefined;
+  for (const content of userMessage.content) {
+    switch (content.type) {
+      case "text":
+        message += content.text;
+        break;
+      case "image_url":
+        assert(!image, "only one image per message supported");
+        image = content.image_url;
+        break;
+      case "input_audio":
+        throw "userMessageToChatMessage: audio content not supported";
+      case "file":
+        throw "userMessageToChatMessage: file content not supported";
+      default:
+        throw "userMessageToChatMessage: unexpected content.type";
+    }
+  }
+
+  if (image) {
+    throw "unimpl: image content";
+  }
+
+  return {
+    type: "user_msg",
+    message_id,
+    message,
+    from,
+  };
+}
+
+export function conversationToChatMessages(
+  conversation: ChatCompletionMessageParam[],
+  defaultName: string
+): ServerToClient[] {
+  const msgs: ServerToClient[] = [];
+  for (const ccmp of conversation) {
+    const message_id = `message_${msgs.length}`;
+    switch (ccmp.role) {
+      case "developer":
+        throw "developer messages not handled yet";
+      case "assistant":
+        assert(!ccmp.audio);
+        if (ccmp.content) {
+          msgs.push({
+            type: "agent_msg",
+            message: ccmp,
+            message_id,
+          });
+        }
+        // TODO: do we want to convert tool calls etc?
+        break;
+      case "user":
+        {
+          const msg = userMessageToChatMessage(ccmp, message_id, defaultName);
+          if (msg) {
+            msgs.push(msg);
+          }
+        }
+        break;
+      default:
+        break;
+    }
+  }
+
+  return msgs;
+}

package/src/chat/db.ts

@@ -6,6 +6,7 @@ import {
   getLogger,
   SavedAgentProfile,
 } from "@xalia/xmcp/sdk";
+import { ChatCompletionMessageParam } from "../agent/agent";
 
 const logger = getLogger();
 
@@ -284,6 +285,19 @@ export class Database {
     return data[0].uuid;
   }
 
+  async sessionConversationAppend(
+    session_id: string,
+    newEntries: ChatCompletionMessageParam[]
+  ): Promise<void> {
+    const { error } = await this.client.rpc("session_append_to_conversation", {
+      session_id,
+      messages: newEntries,
+    });
+    if (error) {
+      throw error;
+    }
+  }
+
   async clearSessions(): Promise<void> {
     await this.client.from("sessions").delete().neq("uuid", "");
   }

package/src/chat/messages.ts

@@ -233,3 +233,34 @@ export type ServerToClient =
   | ServerTyping
   | ServerToClientStateUpdate
   | ServerToClientActions;
+
+export function decodeAssistantMessageParam(
+  msg: ChatCompletionAssistantMessageParam
+): string {
+  let text = "";
+
+  if (msg.audio) {
+    throw "decodeAssistantMessageParam; audio unimplemented";
+  }
+
+  if (msg.content) {
+    if (typeof msg.content === "string") {
+      text = msg.content;
+    } else if (msg.content instanceof Array) {
+      for (const c of msg.content) {
+        switch (c.type) {
+          case "text":
+            text += c.text;
+            break;
+          case "refusal":
+            text += c.refusal;
+            break;
+          default:
+            throw Error("unexpected AssistantMessageParam.content entry");
+        }
+      }
+    }
+  }
+
+  return text;
+}

package/src/chat/websocket.ts

@@ -0,0 +1,14 @@
+// Browser and Node.js Universal WebSocket wrapper
+
+let WSClass: typeof WebSocket;
+
+if (typeof window !== "undefined" && typeof window.WebSocket !== "undefined") {
+  // Running in browser
+  WSClass = window.WebSocket;
+} else {
+  // eslint-disable-next-line @typescript-eslint/no-require-imports
+  const ws = require("ws");
+  WSClass = ws.WebSocket as unknown as typeof WebSocket;
+}
+
+export { WSClass as WebSocket };

package/src/test/db.test.ts

@@ -1,6 +1,6 @@
 import { expect } from "chai";
 import { Database, SUPABASE_LOCAL_KEY, SUPABASE_LOCAL_URL } from "../chat/db";
-import { AgentProfile } from "../agent/agent";
+import { AgentProfile, ChatCompletionMessageParam } from "../agent/agent";
 import { strict as assert } from "assert";
 import { ApiClient } from "@xalia/xmcp/sdk";
 import { LOCAL_SERVER_URL } from "../agent/sudoMcpServerManager";
@@ -108,6 +108,15 @@ describe("DB", () => {
   });
 
   it("should create and retrieve sessions", async function () {
+    const CONV_0: ChatCompletionMessageParam[] = [
+      { role: "user", content: "message 0" },
+      { role: "assistant", content: "message 1" },
+    ];
+    const CONV_1: ChatCompletionMessageParam[] = [
+      { role: "user", content: "message 2" },
+      { role: "assistant", content: "message 3" },
+    ];
+
     const db = getLocalDB();
 
     await db.clearAgentProfiles();
@@ -132,5 +141,17 @@ describe("DB", () => {
     expect(session.title).eql("test_session");
     expect(session.user_uuid).eql("dummy_user");
     expect(session.uuid).eql(sessionId);
+
+    // Append messages to empty conversation
+
+    await db.sessionConversationAppend(sessionId, CONV_0);
+    const session0 = (await db.getSessionById(sessionId))!;
+    expect(session0.conversation).eql(CONV_0);
+
+    // Append further messages
+
+    await db.sessionConversationAppend(sessionId, CONV_1);
+    const session1 = (await db.getSessionById(sessionId))!;
+    expect(session1.conversation).eql(CONV_0.concat(CONV_1));
   });
 });

package/src/test/mcpServerManager.test.ts

@@ -28,7 +28,7 @@ async function shutdownAll() {
 describe("McpServerManager", async () => {
   it("should initialize with correct descriptions", async (): Promise<void> => {
     const tm = getMcpServerManager();
-    await tm.addMcpServer(
+    await tm.addMcpServerWithSSEUrl(
       "simplecalc",
       "http://localhost:5001/mcpservers/sudomcp/simplecalc/session",
       "dummy_key"
@@ -79,7 +79,7 @@ describe("McpServerManager", async () => {
   it("add / remove servers", async () => {
     const tm = getMcpServerManager();
 
-    await tm.addMcpServer(
+    await tm.addMcpServerWithSSEUrl(
       "simplecalc",
       "http://localhost:5001/mcpservers/sudomcp/simplecalc/session",
       "dummy_key"

package/src/test/openaiStreaming.test.ts

@@ -6,7 +6,9 @@ import {
   updateCompletion,
 } from "../agent/openAILLMStreaming";
 
-const TEST_STANDARD: OpenAI.Chat.Completions.ChatCompletionChunk[] = [
+type ChatCompletionChunk = OpenAI.Chat.Completions.ChatCompletionChunk;
+
+const TEST_STANDARD: ChatCompletionChunk[] = [
   {
     id: "chatcmpl-BzBqzVs5w0kU5we3KIUN1Q4FAZXjg",
     choices: [
@@ -34,7 +36,7 @@ const TEST_STANDARD: OpenAI.Chat.Completions.ChatCompletionChunk[] = [
   },
 ];
 
-const TEST_TRAILING_USAGE: OpenAI.Chat.Completions.ChatCompletionChunk[] = [
+const TEST_TRAILING_USAGE: ChatCompletionChunk[] = [
   {
     id: "gen-1753923406-nsIKHyFRoJqkUntBnQTw",
     choices: [
@@ -73,6 +75,55 @@ const TEST_TRAILING_USAGE: OpenAI.Chat.Completions.ChatCompletionChunk[] = [
   },
 ];
 
+const AGGREGATED_MESSAGE: OpenAI.Chat.Completions.ChatCompletion = {
+  id: "gen-1753923406-nsIKHyFRoJqkUntBnQTw",
+  choices: [
+    {
+      message: {
+        content: "test",
+        role: "assistant",
+        refusal: null,
+        tool_calls: undefined,
+      },
+      finish_reason: "stop",
+      index: 0,
+      logprobs: null,
+    },
+  ],
+  created: 1753923406,
+  model: "openai/gpt-4o",
+  object: "chat.completion",
+  service_tier: undefined,
+  system_fingerprint: "fp_a288987b44",
+  usage: {
+    completion_tokens: 50,
+    prompt_tokens: 271,
+    total_tokens: 321,
+    completion_tokens_details: { reasoning_tokens: 0 },
+    prompt_tokens_details: { cached_tokens: 0 },
+  },
+};
+
+const TEST_TRAILING_USAGE_WITH_3_CHUNKS: ChatCompletionChunk[] = [
+  {
+    ...TEST_TRAILING_USAGE[0],
+    choices: [
+      {
+        delta: { content: "test", role: "assistant" },
+        finish_reason: null,
+        index: 0,
+        logprobs: null,
+      },
+    ],
+  },
+  {
+    ...TEST_TRAILING_USAGE[1],
+    choices: [{ ...TEST_TRAILING_USAGE[1].choices[0], finish_reason: "stop" }],
+    usage: undefined,
+  },
+  { ...TEST_TRAILING_USAGE[1], choices: [] },
+];
+
 describe("OpenAI Streaming", () => {
   it("should support standard termination", async function () {
     const chunks = TEST_STANDARD;
@@ -110,33 +161,16 @@ describe("OpenAI Streaming", () => {
     const { initMessage } = initializeCompletion(chunks[0]);
     updateCompletion(initMessage, chunks[1]);
 
-    expect(initMessage).eql({
-      id: "gen-1753923406-nsIKHyFRoJqkUntBnQTw",
-      choices: [
-        {
-          message: {
-            content: "test",
-            role: "assistant",
-            refusal: null,
-            tool_calls: undefined,
-          },
-          finish_reason: "stop",
-          index: 0,
-          logprobs: null,
-        },
-      ],
-      created: 1753923406,
-      model: "openai/gpt-4o",
-      object: "chat.completion",
-      service_tier: undefined,
-      system_fingerprint: "fp_a288987b44",
-      usage: {
-        completion_tokens: 50,
-        prompt_tokens: 271,
-        total_tokens: 321,
-        completion_tokens_details: { reasoning_tokens: 0 },
-        prompt_tokens_details: { cached_tokens: 0 },
-      },
-    });
+    expect(initMessage).eql(AGGREGATED_MESSAGE);
+  });
+
+  it("should support trailing usage with 3 chunks", async function () {
+    const chunks = TEST_TRAILING_USAGE_WITH_3_CHUNKS;
+    expect(chunks.length).eql(3);
+    const { initMessage } = initializeCompletion(chunks[0]);
+    updateCompletion(initMessage, chunks[1]);
+    updateCompletion(initMessage, chunks[2]);
+
+    expect(initMessage).eql(AGGREGATED_MESSAGE);
   });
 });