@xalia/agent 1.0.19

package/src/agent.ts

@@ -0,0 +1,357 @@
+import * as dotenv from "dotenv";
+import { OpenAI } from "openai";
+import { McpServerManager, McpServerManagerSettings } from "./mcpServerManager";
+import {
+  ChatCompletionContentPart,
+  ChatCompletionUserMessageParam,
+} from "openai/resources.mjs";
+import { strict as assert } from "assert";
+import { OpenAILLM } from "./openAILLM";
+import { ILLM } from "./llm";
+import { DummyLLM } from "./dummyLLM";
+import { getLogger } from "@xalia/xmcp/sdk";
+
+export type ToolHandler = (args: unknown) => string;
+
+export type McpServerUrls = (name: string) => string;
+
+// Role: If content, give it to UI
+export type OnMessageCB = {
+  (msg: OpenAI.ChatCompletionMessageParam, msgEnd: boolean): Promise<void>;
+};
+
+// Role: If tool calls, prompt for permission to handle them
+export type OnToolCallCB = {
+  (msg: OpenAI.ChatCompletionMessageToolCall): Promise<boolean>;
+};
+
+dotenv.config();
+const logger = getLogger();
+
+export class AgentProfile {
+  constructor(
+    /// The llm provider endpoint, or dummy llm filename.  `undefined` means
+    /// openai.
+    public llm_url: string | undefined,
+
+    /// "dummy" means use the dummy LLM, in which case llmUrl refers to the
+    /// filename.  `undefined` means default for the provider.
+    public model: string | undefined,
+
+    /// System prompt
+    public system_prompt: string,
+
+    /// MCP server settings.
+    public mcp_settings: McpServerManagerSettings
+  ) {}
+
+  public static fromJSONObj(obj: Record<string, unknown>): AgentProfile {
+    assert(typeof obj === "object");
+    assert(
+      typeof obj.llm_url === "string" || typeof obj.llm_url === "undefined"
+    );
+    assert(typeof obj.model === "string" || typeof obj.model === "undefined");
+    assert(typeof obj.system_prompt === "string");
+    assert(typeof obj.mcp_settings === "object");
+    return new AgentProfile(
+      obj.llm_url,
+      obj.model,
+      obj.system_prompt,
+      obj.mcp_settings as McpServerManagerSettings
+    );
+  }
+}
+
+export class Agent {
+  private toolHandlers: { [toolName: string]: ToolHandler } = {};
+
+  private constructor(
+    public onMessage: OnMessageCB,
+    public onToolCall: OnToolCallCB,
+    private messages: OpenAI.ChatCompletionMessageParam[],
+    private mcpServerManager: McpServerManager,
+    private tools: OpenAI.ChatCompletionTool[],
+    private llm: ILLM
+  ) {}
+
+  public static async initializeWithLLM(
+    onMessage: OnMessageCB,
+    onToolCall: OnToolCallCB,
+    systemPrompt: string | undefined,
+    llm: ILLM
+  ): Promise<Agent> {
+    // Initialize messages with system prompt
+    const messages = [
+      {
+        role: "system",
+        content: systemPrompt ?? "You are a helpful assistant",
+      } as OpenAI.ChatCompletionMessageParam,
+    ];
+
+    // Create the server manager
+    const mcpServerManager = new McpServerManager();
+
+    return new Agent(
+      onMessage,
+      onToolCall,
+      messages,
+      mcpServerManager,
+      [],
+      llm
+    );
+  }
+
+  public static async initialize(
+    onMessage: OnMessageCB,
+    onToolCall: OnToolCallCB,
+    systemPrompt: string,
+    openaiApiUrl: string | undefined,
+    openaiApiKey: string,
+    model: string | undefined
+  ): Promise<Agent> {
+    return Agent.initializeWithLLM(
+      onMessage,
+      onToolCall,
+      systemPrompt,
+      new OpenAILLM(openaiApiKey, openaiApiUrl, model)
+    );
+  }
+
+  public static async initializeDummy(
+    onMessage: OnMessageCB,
+    onToolCall: OnToolCallCB,
+    systemPrompt: string,
+    responses: OpenAI.Chat.Completions.ChatCompletion.Choice[]
+  ): Promise<Agent> {
+    return Agent.initializeWithLLM(
+      onMessage,
+      onToolCall,
+      systemPrompt,
+      new DummyLLM(responses)
+    );
+  }
+
+  public async shutdown(): Promise<void> {
+    return this.mcpServerManager.shutdown();
+  }
+
+  public getAgentProfile(): AgentProfile {
+    return new AgentProfile(
+      this.llm.getUrl(),
+      this.llm.getModel(),
+      this.getSystemMessage(),
+      this.mcpServerManager.getMcpServerSettings()
+    );
+  }
+
+  public getConversation(): OpenAI.ChatCompletionMessageParam[] {
+    assert(
+      this.messages[0].role == "system",
+      "first message must have system role"
+    );
+    // Return a copy so future modifications to `this.messages` don't impact
+    // the callers copy.
+    return structuredClone(this.messages.slice(1));
+  }
+
+  public setConversation(messages: OpenAI.ChatCompletionMessageParam[]) {
+    assert(this.messages[0].role == "system");
+    assert(messages[0].role != "system", "conversation contains system msg");
+
+    const newMessages: OpenAI.ChatCompletionMessageParam[] = [this.messages[0]];
+    this.messages = newMessages.concat(structuredClone(messages));
+  }
+
+  public getMcpServerManager(): McpServerManager {
+    return this.mcpServerManager;
+  }
+
+  public async userMessage(
+    msg?: string,
+    imageB64?: string
+  ): Promise<OpenAI.ChatCompletionMessageParam | undefined> {
+    const userMessage = createUserMessage(msg, imageB64);
+    if (!userMessage) {
+      return undefined;
+    }
+
+    this.messages.push(userMessage);
+    let completion = await this.chatCompletion();
+
+    let message = completion.choices[0].message;
+    this.messages.push(message);
+
+    if (message.content) {
+      await this.onMessage(message, true);
+    }
+
+    // While there are tool calls to make, make them and loop
+
+    while (message.tool_calls && message.tool_calls.length > 0) {
+      for (const toolCall of message.tool_calls ?? []) {
+        const approval = await this.onToolCall(toolCall);
+        if (approval) {
+          try {
+            const result = await this.doToolCall(toolCall);
+            logger.debug(`tool call result ${JSON.stringify(result)}`);
+            this.messages.push(result);
+          } catch (e) {
+            logger.error(`tool call error: ${e}`);
+            this.messages.push({
+              role: "tool",
+              tool_call_id: toolCall.id,
+              content: "Tool call failed.",
+            });
+          }
+        } else {
+          this.messages.push({
+            role: "tool",
+            tool_call_id: toolCall.id,
+            content: "User denied tool use request.",
+          });
+        }
+      }
+
+      completion = await this.chatCompletion();
+      message = completion.choices[0].message;
+      this.messages.push(message);
+
+      if (message.content) {
+        await this.onMessage(message, true);
+      }
+    }
+
+    return completion.choices[0].message;
+  }
+
+  public chooseModel(model: string) {
+    logger.debug(`Set model ${model}`);
+    assert(this.llm instanceof OpenAILLM);
+    this.llm.setModel(model);
+  }
+
+  /**
+   * Clear the conversation.
+   */
+  public resetConversation() {
+    assert(this.messages.length > 0);
+    // Keep only the system message
+    this.messages.splice(1);
+  }
+
+  public getSystemMessage(): string {
+    assert(this.messages[0].role === "system");
+    return this.messages[0].content as string;
+  }
+
+  /**
+   * Set the system prompt
+   */
+  public setSystemMessage(systemMsg: string) {
+    assert(this.messages[0].role === "system");
+    this.messages[0].content = systemMsg;
+  }
+
+  async chatCompletion(): Promise<OpenAI.Chat.Completions.ChatCompletion> {
+    let tools: OpenAI.ChatCompletionTool[] | undefined;
+    const enabledTools = this.tools.concat(
+      this.mcpServerManager.getOpenAITools()
+    );
+    if (enabledTools.length > 0) {
+      tools = enabledTools;
+    }
+    // logger.debug(
+    //   `chatCompletion: tools: ${JSON.stringify(tools, undefined, 2)}`
+    // );
+    const completion = await this.llm.getConversationResponse(
+      this.messages,
+      tools
+    );
+    logger.debug(`Received chat completion ${JSON.stringify(completion)}`);
+    return completion;
+  }
+
+  public toolNames(): string[] {
+    return this.mcpServerManager
+      .getOpenAITools()
+      .map((tool) => tool.function.name);
+  }
+
+  public addTool(tool: OpenAI.ChatCompletionTool, handler: ToolHandler) {
+    const name = tool.function.name;
+    if (this.toolHandlers[name]) {
+      throw `tool ${name} already added`;
+    }
+
+    logger.debug(`Adding tool ${name}`);
+
+    this.tools.push(tool);
+    this.toolHandlers[name] = handler;
+  }
+
+  async doToolCall(
+    toolCall: OpenAI.ChatCompletionMessageToolCall
+  ): Promise<OpenAI.ChatCompletionToolMessageParam> {
+    const name = toolCall.function.name;
+    const args = JSON.parse(toolCall.function.arguments);
+
+    let result: string | undefined = undefined;
+    const handler = this.toolHandlers[name];
+    if (handler) {
+      logger.debug(`  found agent tool ${name} ...`);
+      result = handler(args);
+    } else {
+      result = await this.mcpServerManager.invoke(name, args);
+    }
+    return {
+      role: "tool",
+      tool_call_id: toolCall.id,
+      content: result.toString(),
+    };
+  }
+}
+
+/**
+ * Returns the ChatCompletionMessageParam constructed from (optional) text and
+ * (optional) image.  If neither is given (null message), then undefined is
+ * returned.
+ **/
+export function createUserMessage(
+  msg?: string,
+  imageB64?: string
+): ChatCompletionUserMessageParam | undefined {
+  const content = (() => {
+    if (!imageB64) {
+      if (!msg) {
+        return undefined;
+      }
+      return msg;
+    }
+
+    const content: ChatCompletionContentPart[] = [];
+    if (msg) {
+      content.push({
+        type: "text",
+        text: msg,
+      });
+    }
+    if (imageB64) {
+      content.push({
+        type: "image_url",
+        image_url: {
+          url: imageB64,
+        },
+      });
+    }
+    return content;
+  })();
+
+  if (!content) {
+    return undefined;
+  }
+
+  return {
+    role: "user",
+    content,
+  };
+}

package/src/agentUtils.ts

@@ -0,0 +1,188 @@
+import { getLogger } from "@xalia/xmcp/sdk";
+import { Agent, AgentProfile, OnMessageCB, OnToolCallCB } from "./agent";
+import { IPlatform } from "./iplatform";
+import { SudoMcpServerManager } from "./sudoMcpServerManager";
+import OpenAI from "openai";
+import { Configuration as SudoMcpConfiguration } from "@xalia/xmcp/sdk";
+
+const logger = getLogger();
+
+/**
+ * Util function to create an Agent from some config information.
+ */
+async function createAgent(
+  llmUrl: string | undefined,
+  model: string | undefined,
+  systemPrompt: string,
+  onMessage: OnMessageCB,
+  onToolCall: OnToolCallCB,
+  platform: IPlatform,
+  openaiApiKey: string | undefined
+): Promise<Agent> {
+  if (model === "dummy") {
+    if (!llmUrl) {
+      throw "AgentProfile.llmUrl must be set for dummy LLM";
+    }
+    logger.debug(`dummy model with script: ${llmUrl}`);
+    const script = await platform.load(llmUrl);
+    logger.debug(` script: ${script}`);
+    const responses: OpenAI.ChatCompletion.Choice[] = JSON.parse(script);
+    logger.debug(`Initializing Dummy Agent: ${llmUrl}`);
+    return Agent.initializeDummy(
+      onMessage,
+      onToolCall,
+      systemPrompt,
+      responses
+    );
+  }
+
+  if (!openaiApiKey) {
+    throw "Missing OpenAI API Key";
+  }
+
+  logger.debug(`Initializing Agent: ${llmUrl} - ${model}`);
+
+  return Agent.initialize(
+    onMessage,
+    onToolCall,
+    systemPrompt,
+    llmUrl,
+    openaiApiKey,
+    model
+  );
+}
+
+/**
+ * Util function to create and initialize an Agent given an AgentProfile.
+ */
+export async function createAgentAndSudoMcpServerManager(
+  agentProfile: AgentProfile,
+  onMessage: OnMessageCB,
+  onToolCall: OnToolCallCB,
+  platform: IPlatform,
+  openaiApiKey: string | undefined,
+  sudomcpConfig: SudoMcpConfiguration,
+  authorizedUrl: string | undefined,
+  conversation: OpenAI.ChatCompletionMessageParam[] | undefined
+): Promise<[Agent, SudoMcpServerManager]> {
+  // Create agent
+  logger.debug("[createAgentAndSudoMcpServerManager] creating agent ...");
+  const agent = await createAgent(
+    agentProfile.llm_url,
+    agentProfile.model,
+    agentProfile.system_prompt,
+    onMessage,
+    onToolCall,
+    platform,
+    openaiApiKey
+  );
+  if (conversation) {
+    agent.setConversation(conversation);
+  }
+
+  // Init SudoMcpServerManager
+  logger.debug(
+    "[createAgentAndSudoMcpServerManager] creating SudoMcpServerManager."
+  );
+  const sudoMcpServerManager = await SudoMcpServerManager.initialize(
+    agent.getMcpServerManager(),
+    platform.openUrl,
+    sudomcpConfig.backend_url,
+    sudomcpConfig.api_key,
+    authorizedUrl
+  );
+  logger.debug(
+    "[createAgentAndSudoMcpServerManager] restore mcp settings:" +
+      JSON.stringify(agentProfile.mcp_settings)
+  );
+  await sudoMcpServerManager.restoreMcpSettings(
+    agentProfile.mcp_settings,
+    sudomcpConfig.server_configs
+  );
+
+  logger.debug("[createAgentAndSudoMcpServerManager] done");
+  return [agent, sudoMcpServerManager];
+}
+
+/**
+ * An "non-interactive" agent is one which is not intended to be used
+ * interactively (settings cannot be dyanmically adjusted, intermediate
+ * messages are not used by the caller, the user does not need to approve tool
+ * calls, etc).
+ */
+export async function createNonInteractiveAgent(
+  agentProfile: AgentProfile,
+  conversation: OpenAI.ChatCompletionMessageParam[] | undefined,
+  platform: IPlatform,
+  openaiApiKey: string | undefined,
+  sudomcpConfig: SudoMcpConfiguration,
+  approveToolsUpTo: number
+): Promise<Agent> {
+  let remainingToolCalls = approveToolsUpTo;
+  const onMessage = async () => {};
+  const onToolCall = async () => {
+    if (remainingToolCalls !== 0) {
+      --remainingToolCalls;
+      return true;
+    }
+    return false;
+  };
+
+  const [agent, _] = await createAgentAndSudoMcpServerManager(
+    agentProfile,
+    onMessage,
+    onToolCall,
+    platform,
+    openaiApiKey,
+    sudomcpConfig,
+    undefined,
+    conversation
+  );
+
+  return agent;
+}
+
+/**
+ * Create an Agent (from the AgentProfile), pass it a single prompt and output
+ * the response.
+ */
+export async function runOneShot(
+  agentProfile: AgentProfile,
+  conversation: OpenAI.ChatCompletionMessageParam[] | undefined,
+  platform: IPlatform,
+  prompt: string,
+  image: string | undefined,
+  openaiApiKey: string | undefined,
+  sudomcpConfig: SudoMcpConfiguration,
+  approveToolsUpTo: number
+): Promise<{
+  response: string;
+  conversation: OpenAI.ChatCompletionMessageParam[];
+}> {
+  logger.debug("[runOneShot]: start");
+
+  // Create a non-interactive agent and pass any prompt/ image to it.  Return
+  // the first answer.
+
+  const agent = await createNonInteractiveAgent(
+    agentProfile,
+    conversation,
+    platform,
+    openaiApiKey,
+    sudomcpConfig,
+    approveToolsUpTo
+  );
+
+  const response = await agent.userMessage(prompt, image);
+  await agent.shutdown();
+  logger.debug("[runOneShot]: shutdown done");
+
+  if (!response) {
+    throw "No message returned from agent";
+  }
+
+  return {
+    response: "" + response.content,
+    conversation: agent.getConversation(),
+  };
+}