@xalia/agent 0.6.1 → 0.6.2

package/src/agent/agentUtils.ts

@@ -8,26 +8,18 @@ import { Configuration as SudoMcpConfiguration } from "@xalia/xmcp/sdk";
 import { OpenAILLM } from "./openAILLM";
 import { OpenAILLMStreaming } from "./openAILLMStreaming";
 import { DummyLLM } from "./dummyLLM";
-import { ILLM } from "./llm";
+import { ChatCompletionMessageParam, ILLM } from "./llm";
 import { strict as assert } from "assert";
 import { RepeatLLM } from "./repeatLLM";
 import { ContextManager, IContextManager } from "./context";
+import { DEFAULT_IMAGE_GEN_MODEL, ImageGenLLM } from "./imageGenLLM";
 
 const logger = getLogger();
 
-export const DEFAULT_LLM_URL = "http://localhost:5001/v1";
-// uses openrouter
-export const DEFAULT_LLM_MODEL =
-  process.env["DEFAULT_LLM_MODEL"] || "openai/gpt-4o";
-
-export const XALIA_APP_HEADER = {
-  "HTTP-Referer": "xalia.ai",
-  "X-Title": "Xalia",
-};
-
 export async function createAgentWithoutSkills(
   llmUrl: string,
   agentProfile: AgentProfile,
+  defaultModel: string,
   eventHandler: IAgentEventHandler,
   platform: IPlatform,
   contextManager: IContextManager,
@@ -55,6 +47,7 @@ export async function createAgentWithoutSkills(
   const agent = await createAgentFromSkillManager(
     llmUrl,
     agentProfile,
+    defaultModel,
     eventHandler,
     platform,
     contextManager,
@@ -74,6 +67,7 @@ export async function createAgentWithoutSkills(
 export async function createAgentWithSkills(
   llmUrl: string,
   agentProfile: AgentProfile,
+  defaultModel: string,
   eventHandler: IAgentEventHandler,
   platform: IPlatform,
   contextManager: IContextManager,
@@ -85,6 +79,7 @@ export async function createAgentWithSkills(
   const [agent, sudoMcpServerManager] = await createAgentWithoutSkills(
     llmUrl,
     agentProfile,
+    defaultModel,
     eventHandler,
     platform,
     contextManager,
@@ -103,11 +98,12 @@ export async function createAgentWithSkills(
 export async function createAgentFromSkillManager(
   llmUrl: string,
   agentProfile: AgentProfile,
+  defaultModel: string,
   eventHandler: IAgentEventHandler,
   platform: IPlatform,
   contextManager: IContextManager,
   llmApiKey: string | undefined,
-  skillManager: SkillManager,
+  skillManager: SkillManager | undefined,
   stream: boolean = false
 ): Promise<Agent> {
   // Create agent
@@ -115,7 +111,7 @@ export async function createAgentFromSkillManager(
   const llm = await createLLM(
     llmUrl,
     llmApiKey,
-    agentProfile.model,
+    agentProfile.model || defaultModel,
     stream,
     platform
   );
@@ -134,7 +130,7 @@ export async function createAgentFromSkillManager(
 export async function createLLM(
   llmUrl: string | undefined,
   llmApiKey: string | undefined,
-  model: string | undefined,
+  model: string,
   stream: boolean = false,
   platform: IPlatform
 ): Promise<ILLM> {
@@ -144,15 +140,16 @@ export async function createLLM(
     llm = await DummyLLM.initFromModelUrl(model, platform);
   } else if (model === "repeat") {
     llm = new RepeatLLM();
+  } else if (model == DEFAULT_IMAGE_GEN_MODEL) {
+    logger.info("ImageGenLLM");
+    llm = new ImageGenLLM(llmApiKey, llmUrl, model);
   } else {
     // Regular Agent
     if (!llmApiKey) {
       throw new Error("Missing OpenAI API Key");
     }
 
-    logger.debug(
-      `Initializing Agent: ${llmUrl ?? "unknown"} - ${model ?? "unknown"}`
-    );
+    logger.debug(`Initializing Agent: ${llmUrl ?? "unknown"} - ${model}`);
     if (stream) {
       llm = new OpenAILLMStreaming(llmApiKey, llmUrl, model);
     } else {
@@ -173,7 +170,8 @@ export async function createLLM(
 export async function createNonInteractiveAgent(
   url: string,
   agentProfile: AgentProfile,
-  conversation: OpenAI.ChatCompletionMessageParam[] | undefined,
+  defaultModel: string,
+  conversation: ChatCompletionMessageParam[] | undefined,
   platform: IPlatform,
   openaiApiKey: string | undefined,
   sudomcpConfig: SudoMcpConfiguration,
@@ -182,7 +180,9 @@ export async function createNonInteractiveAgent(
   let remainingToolCalls = approveToolsUpTo;
   const eventHandler: IAgentEventHandler = {
     onCompletion: () => {},
+    onImage: () => {},
     onAgentMessage: async () => {},
+    onReasoning: async () => {},
     // eslint-disable-next-line @typescript-eslint/require-await
     onToolCall: async () => {
       if (remainingToolCalls !== 0) {
@@ -201,6 +201,7 @@ export async function createNonInteractiveAgent(
   const [agent, _] = await createAgentWithSkills(
     url,
     agentProfile,
+    defaultModel,
     eventHandler,
     platform,
     contextManager,
@@ -219,7 +220,8 @@ export async function createNonInteractiveAgent(
 export async function runOneShot(
   url: string,
   agentProfile: AgentProfile,
-  conversation: OpenAI.ChatCompletionMessageParam[] | undefined,
+  defaultModel: string,
+  conversation: ChatCompletionMessageParam[] | undefined,
   platform: IPlatform,
   prompt: string,
   image: string | undefined,
@@ -229,6 +231,7 @@ export async function runOneShot(
 ): Promise<{
   response: string;
   conversation: OpenAI.ChatCompletionMessageParam[];
+  images: OpenAI.Chat.Completions.ChatCompletionContentPartImage[] | undefined;
 }> {
   logger.debug("[runOneShot]: start");
 
@@ -238,6 +241,7 @@ export async function runOneShot(
   const agent = await createNonInteractiveAgent(
     url,
     agentProfile,
+    defaultModel,
     conversation,
     platform,
     llmApiKey,
@@ -245,15 +249,16 @@ export async function runOneShot(
     approveToolsUpTo
   );
 
-  const response = await agent.userMessageEx(prompt, image);
+  const agentResponse = await agent.userMessageEx(prompt, image);
   await agent.shutdown();
   logger.debug("[runOneShot]: shutdown done");
 
-  if (!response) {
+  if (!agentResponse) {
     throw new Error("No message returned from agent");
   }
 
   // Handle different content types
+  const response = agentResponse.message;
   let responseText = "";
   if (typeof response.content === "string") {
     responseText = response.content;
@@ -277,5 +282,6 @@ export async function runOneShot(
   return {
     response: responseText,
     conversation: agent.getConversation(),
+    images: agentResponse.images,
   };
 }

package/src/agent/compressingContextManager.ts

@@ -1,11 +1,11 @@
 import { strict as assert } from "assert";
 import { getLogger } from "@xalia/xmcp/sdk";
 
+import { Agent } from "./agent";
 import {
-  Agent,
   ChatCompletionUserMessageParam,
   ChatCompletionMessageParam,
-} from "./agent";
+} from "./llm";
 import { NULL_PLATFORM } from "./nullPlatform";
 import { createLLM } from "./agentUtils";
 import { ContextManager, ContextManagerWithCommit } from "./context";
@@ -68,11 +68,12 @@ export async function createSummary(
     compressionAgentApiKey
   );
 
-  const resp = await agent.userMessageEx(JSON.stringify(conversation));
-  if (!resp) {
+  const agentResp = await agent.userMessageEx(JSON.stringify(conversation));
+  if (!agentResp) {
     throw new Error("compression agent returned null");
   }
 
+  const resp = agentResp.message;
   assert(resp.role === "assistant");
   assert(
     typeof resp.content === "string",

package/src/agent/context.ts

@@ -2,7 +2,7 @@ import { strict as assert } from "assert";
 import {
   ChatCompletionMessageParam,
   ChatCompletionUserMessageParam,
-} from "./agent";
+} from "./llm";
 import { SystemPromptProvider } from "./promptProvider";
 
 /**

package/src/agent/dummyLLM.ts

@@ -92,9 +92,7 @@ export class DummyLLM implements ILLM {
 
     if (onMessage) {
       const message = response.message;
-      if (message.content) {
-        void onMessage(message.content, true);
-      }
+      void onMessage(message.content || "", true);
     }
 
     return {

package/src/agent/iAgentEventHandler.ts

@@ -2,7 +2,8 @@ import { OpenAI } from "openai";
 import {
   ChatCompletionAssistantMessageParam,
   ChatCompletionMessageToolCall,
-} from "./agent";
+  ChatCompletionToolMessageParam,
+} from "./llm";
 
 /**
  * Interface for handling events from the Agent.
@@ -18,11 +19,18 @@ export interface IAgentEventHandler {
    */
   onCompletion(result: ChatCompletionAssistantMessageParam): void;
 
+  /**
+   * Images do not appear as part of `ChatCompletionAssistantMessageParam`,
+   * although they do appear in the final returned result from the Agent.
+   * This callback gives clients the chance to see them earlier.
+   */
+  onImage(image: OpenAI.Chat.Completions.ChatCompletionContentPartImage): void;
+
   /**
    * Called when a tool call execution completes (success, error, or denial).
    * These messages are insertes into the LLM context.
    */
-  onToolCallResult(result: OpenAI.ChatCompletionToolMessageParam): void;
+  onToolCallResult(result: ChatCompletionToolMessageParam): void;
 
   /**
    * Called when the agent produces a message chunk (streaming).  Calls here
@@ -33,6 +41,11 @@ export interface IAgentEventHandler {
    */
   onAgentMessage(chunk: string, isEnd: boolean): Promise<void>;
 
+  /**
+   * Called when the agent produces reasoning tokens
+   */
+  onReasoning(chunk: string): Promise<void>;
+
   /**
    * Called when the agent wants to execute a tool call.  Calls do NOT
    * correspond to context entries.  (A completion from the LLM may correspond

package/src/agent/imageGenLLM.ts

@@ -0,0 +1,99 @@
+import { OpenAI } from "openai";
+import { strict as assert } from "assert";
+import { writeFileSync } from "fs";
+
+import { getLogger } from "@xalia/xmcp/sdk";
+
+import { ILLM, ChatCompletion, XALIA_APP_HEADER } from "./llm";
+
+const logger = getLogger();
+
+export const DEFAULT_IMAGE_GEN_MODEL = "google/gemini-2.5-flash-image-preview";
+
+type ChatCompletionCreateParams = Omit<
+  OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming,
+  "modalities"
+> & {
+  modalities: Array<"text" | "audio" | "image">;
+};
+
+export class ImageGenLLM implements ILLM {
+  private readonly openai: OpenAI;
+  private model: string;
+
+  constructor(
+    apiKey: string | undefined,
+    apiUrl: string | undefined,
+    model: string | undefined
+  ) {
+    logger.debug(`here`);
+    this.openai = new OpenAI({
+      apiKey,
+      baseURL: apiUrl,
+      dangerouslyAllowBrowser: true,
+      defaultHeaders: XALIA_APP_HEADER,
+    });
+    this.model = model || DEFAULT_IMAGE_GEN_MODEL;
+  }
+
+  public setModel(model: string) {
+    this.model = model;
+  }
+
+  getModel(): string {
+    return this.model;
+  }
+
+  getUrl(): string {
+    return this.openai.baseURL;
+  }
+
+  public async getConversationResponse(
+    messages: OpenAI.Chat.Completions.ChatCompletionMessageParam[],
+    tools?: OpenAI.Chat.Completions.ChatCompletionTool[],
+    onMessage?: (msg: string, end: boolean) => Promise<void>
+  ): Promise<ChatCompletion> {
+    assert(!tools || tools.length === 0, "tools not supported in ImageGenLLM");
+
+    // Designed for image generation using openrouter, which tweaks the Create
+    const params: ChatCompletionCreateParams = {
+      model: this.model,
+      messages,
+      tools,
+      modalities: ["image", "text"],
+    };
+
+    logger.info(`[ImageGenLLM] params; ${JSON.stringify(params)}`);
+
+    const completion = (await this.openai.chat.completions.create(
+      params as OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming
+    )) as ChatCompletion;
+
+    // const completion = {} as unknown as ChatCompletion;
+
+    const filePath: string = "./completion.json";
+    logger.info(`[ImageGenLLM] writing ${filePath}`);
+    writeFileSync(filePath, JSON.stringify(completion), "utf-8");
+    logger.info(`[ImageGenLLM] written`);
+
+    // logger.debug(
+    //   `[ImageGenLLM.getConversationResponse] completion:
+    //   ${JSON.stringify(completion)}`
+    // );
+
+    if (onMessage) {
+      const message = completion.choices[0].message;
+      if (message.content) {
+        await onMessage(message.content, true);
+      }
+      if (message.images) {
+        message.images.forEach((image, index) => {
+          const imageUrl = image.image_url.url; // Base64 data URL
+          const truncated = imageUrl.substring(0, 50);
+          logger.info(`[ImageGenLLM] ${String(index + 1)}: ${truncated}...`);
+        });
+      }
+    }
+    return completion;
+  }
+}

package/src/agent/imageGenerator.ts

@@ -0,0 +1,60 @@
+import { Agent, createUserMessage } from "./agent";
+import { createLLM } from "./agentUtils";
+import { ContextManager } from "./context";
+import { NULL_AGENT_EVENT_HANDLER } from "./nullAgentEventHandler";
+import { NULL_PLATFORM } from "./nullPlatform";
+import { NODE_PLATFORM } from "../tool/nodePlatform";
+import { DEFAULT_IMAGE_GEN_MODEL } from "./imageGenLLM";
+
+const IMAGE_GEN_SYSTEM_PROMPT = "You are an image generator";
+
+export class ImageGenerator {
+  readonly agent: Agent;
+  readonly contextManager: ContextManager;
+
+  constructor(agent: Agent, contextManager: ContextManager) {
+    this.agent = agent;
+    this.contextManager = contextManager;
+  }
+
+  public static async init(
+    llmUrl: string,
+    llmApiKey: string,
+    model?: string
+  ): Promise<ImageGenerator> {
+    const development = !!process.env.DEVELOPMENT;
+    const llm = await createLLM(
+      llmUrl,
+      llmApiKey,
+      model || DEFAULT_IMAGE_GEN_MODEL,
+      false /* stream */,
+      development ? NODE_PLATFORM : NULL_PLATFORM // allow file loading
+    );
+    const contextManager = new ContextManager(IMAGE_GEN_SYSTEM_PROMPT, []);
+    const agent = Agent.initializeWithLLM(
+      NULL_AGENT_EVENT_HANDLER,
+      llm,
+      contextManager
+    );
+    return new ImageGenerator(agent, contextManager);
+  }
+
+  public async generate(prompt: string, image?: string): Promise<string> {
+    const userMessage = createUserMessage(prompt, image);
+    if (!userMessage) {
+      throw new Error("invalid user message / input image");
+    }
+    const agentResponse = await this.agent.userMessageRaw(userMessage);
+    if (!agentResponse) {
+      throw new Error("invalid response from image gen agent");
+    }
+    if (!agentResponse.images || agentResponse.images.length === 0) {
+      throw new Error("invalid response from image gen agent");
+    }
+
+    // Clear the context
+    while (this.contextManager.popMessage());
+
+    return agentResponse.images[0].image_url.url;
+  }
+}

package/src/agent/llm.ts

@@ -1,15 +1,139 @@
 import { OpenAI } from "openai";
 
+export const XALIA_APP_HEADER = {
+  "HTTP-Referer": "xalia.ai",
+  "X-Title": "Xalia",
+};
+
+// Extend the ChatCompletionMessage type with an `images` value, compatible
+// with google/gemini-2.5-flash-image-preview.
+
+export type ChatCompletionMessage =
+  OpenAI.Chat.Completions.ChatCompletionMessage & {
+    images?: OpenAI.Chat.Completions.ChatCompletionContentPartImage[];
+  };
+
+export type ChatCompletionChoice = Omit<
+  OpenAI.Chat.Completions.ChatCompletion.Choice,
+  "message"
+> & { message: ChatCompletionMessage };
+
+export type ChatCompletion = Omit<
+  OpenAI.Chat.Completions.ChatCompletion,
+  "choices"
+> & { choices: Array<ChatCompletionChoice> };
+
+// Shortcuts to other useful OpenAI types.
+
+export type ChatCompletionMessageToolCall =
+  OpenAI.ChatCompletionMessageToolCall;
+
+export type ChatCompletionAssistantMessageParam =
+  OpenAI.ChatCompletionAssistantMessageParam;
+
+export type ChatCompletionUserMessageParam =
+  OpenAI.ChatCompletionUserMessageParam;
+
+export type ChatCompletionToolMessageParam =
+  OpenAI.ChatCompletionToolMessageParam & {
+    metadata?: Record<string, string>;
+  };
+
+export type ChatCompletionMessageParam =
+  | OpenAI.Chat.Completions.ChatCompletionSystemMessageParam
+  | ChatCompletionAssistantMessageParam
+  | ChatCompletionUserMessageParam
+  | ChatCompletionToolMessageParam;
+
+// The tool description type
+
+export type ChatCompletionTool = OpenAI.Chat.Completions.ChatCompletionTool;
+
+// CompletionCreate params
+
+// openrouter reasoning type
+
+export type ReasoningEffort = {
+  effort?: OpenAI.ReasoningEffort;
+  max_tokens?: never;
+};
+
+export type ReasoningMaxTokens = { effort?: never; max_tokens?: number };
+
+export type ReasoningExclude = { exclude?: boolean; enabled?: never };
+
+export type ReasoningEnabled = {
+  exclude?: never;
+  enabled?: boolean;
+};
+
+export type Reasoning = (ReasoningEffort | ReasoningMaxTokens) &
+  (ReasoningExclude | ReasoningEnabled);
+
+export type ReasoningDetails = {
+  type: "reasoning.text" | "<unknown>";
+  text?: string;
+  signature?: string;
+  format?: string;
+  index?: number;
+};
+
+/**
+ * A (openrouter-specific) stream chunk possibly containing reasoning tokens.
+ */
+export type ChatCompletionChunkChoiceDeltaWithReasoning =
+  OpenAI.Chat.Completions.ChatCompletionChunk.Choice.Delta & {
+    reasoning?: string;
+    reasoning_details?: ReasoningDetails[];
+  };
+
+export function choiceDeltaExtractReasoning(
+  delta: ChatCompletionChunkChoiceDeltaWithReasoning
+): string | undefined {
+  if (delta.reasoning) {
+    return delta.reasoning;
+  }
+
+  if (delta.reasoning_details) {
+    let reasoning = "";
+    for (const details of delta.reasoning_details) {
+      if (details.type !== "reasoning.text") {
+        throw new Error(`unexpected details.type: ${details.type}`);
+      }
+      if (details.text) {
+        if (typeof details.text !== "string") {
+          throw new Error(
+            `unexpected typeof details.text: ${typeof details.text}`
+          );
+        }
+        reasoning += details.text;
+      }
+    }
+    return reasoning;
+  }
+
+  return undefined;
+}
+
+/**
+ * A chat completion message with extra reasoning tokens.
+ */
+export type ChatCompletionMessageWithReasoning =
+  OpenAI.Chat.Completions.ChatCompletionMessage & {
+    reasoning?: string;
+  };
+
 export interface ILLM {
   getModel(): string;
 
   getUrl(): string;
 
   getConversationResponse(
-    messages: OpenAI.Chat.Completions.ChatCompletionMessageParam[],
-    tools?: OpenAI.Chat.Completions.ChatCompletionTool[],
-    onMessage?: (msg: string, end: boolean) => Promise<void>
-  ): Promise<OpenAI.Chat.Completions.ChatCompletion>;
+    messages: ChatCompletionMessageParam[],
+    tools?: ChatCompletionTool[],
+    onMessage?: (msg: string, end: boolean) => Promise<void>,
+    onReasoning?: (reasoning: string) => Promise<void>
+  ): Promise<ChatCompletion>;
 
   setModel(model: string): void;
 }

package/src/agent/mcpServerManager.ts

@@ -12,6 +12,12 @@ export type { McpServerSettings } from "@xalia/xmcp/sdk";
 
 const logger = getLogger();
 
+export type VerifiedMcpToolCall = {
+  mcpServerName: string;
+  toolName: string;
+  args: unknown;
+};
+
 /// Callback into an Mcp server
 export type McpCallback = { (args: string): Promise<string> };
 
@@ -292,14 +298,10 @@ export class McpServerManager implements IMcpServerManager {
     return this.enabledOpenAITools;
   }
 
-  /**
-   * Note the `qualifiedToolName` is the full `{mcpServerName}/{toolName}` as
-   * in the openai spec.
-   */
-  public async invoke(
+  public verifyToolCall(
     qualifiedToolName: string,
     args: unknown
-  ): Promise<string> {
+  ): VerifiedMcpToolCall {
     const [mcpServerName, toolName] = splitQualifiedName(qualifiedToolName);
     logger.debug(`invoke: qualified: ${qualifiedToolName}`);
     logger.debug(
@@ -312,8 +314,25 @@ export class McpServerManager implements IMcpServerManager {
     if (!cb) {
       throw new Error(`Unknown tool ${qualifiedToolName}`);
     }
+    return {
+      mcpServerName,
+      toolName,
+      args,
+    };
+  }
+
+  /**
+   * Note the `qualifiedToolName` is the full `{mcpServerName}/{toolName}` as
+   * in the openai spec.
+   */
+  public async invoke(toolCall: VerifiedMcpToolCall): Promise<string> {
+    const server = this.getMcpServerInternal(toolCall.mcpServerName);
+    const cb = server.getCallback(toolCall.toolName);
+    if (!cb) {
+      throw new Error(`Unknown tool ${toolCall.toolName}`);
+    }
 
-    return cb(JSON.stringify(args));
+    return cb(JSON.stringify(toolCall.args));
   }
 
   /**

package/src/agent/nullAgentEventHandler.ts

@@ -6,12 +6,18 @@ import { IAgentEventHandler } from "./iAgentEventHandler";
  */
 export const NULL_AGENT_EVENT_HANDLER: IAgentEventHandler = {
   onCompletion: (): void => {},
+  onImage: (): void => {},
   onToolCallResult: (): void => {},
   onAgentMessage: (): Promise<void> => {
     return new Promise<void>((r) => {
       r();
     });
   },
+  onReasoning: (): Promise<void> => {
+    return new Promise<void>((r) => {
+      r();
+    });
+  },
   onToolCall: (): Promise<boolean> => {
     return new Promise((r) => {
       r(false);

package/src/agent/openAILLM.ts

@@ -1,23 +1,18 @@
-import { DEFAULT_LLM_MODEL, XALIA_APP_HEADER } from "./agentUtils";
-import { ILLM } from "./llm";
+import { ILLM, XALIA_APP_HEADER } from "./llm";
 import { OpenAI } from "openai";
 
 export class OpenAILLM implements ILLM {
   private readonly openai: OpenAI;
   private model: string;
 
-  constructor(
-    apiKey: string,
-    apiUrl: string | undefined,
-    model: string | undefined
-  ) {
+  constructor(apiKey: string, apiUrl: string | undefined, model: string) {
     this.openai = new OpenAI({
       apiKey,
       baseURL: apiUrl,
       dangerouslyAllowBrowser: true,
       defaultHeaders: XALIA_APP_HEADER,
     });
-    this.model = model || DEFAULT_LLM_MODEL;
+    this.model = model;
   }
 
   public setModel(model: string) {