@xalia/agent 0.5.0

package/src/openAILLMStreaming.ts

@@ -0,0 +1,528 @@
+import { getLogger } from "@xalia/xmcp/sdk";
+import { ILLM } from "./llm";
+import { OpenAI } from "openai";
+import { strict as assert } from "assert";
+
+const logger = getLogger();
+
+function initialToolCallFunction(
+  deltaFn:
+    | OpenAI.Chat.Completions.ChatCompletionChunk.Choice.Delta.ToolCall.Function
+    | undefined
+): OpenAI.Chat.Completions.ChatCompletionMessageToolCall.Function {
+  //   export interface ChatCompletionChunk.Choice.Delta.ToolCall.Function {
+  //     arguments?: string;
+  //     name?: string;
+  //   }
+  //
+  // ->
+  //
+  //   export interface Function {
+  //     arguments: string;
+  //     name: string;
+  //   }
+
+  return {
+    arguments: deltaFn?.arguments || "",
+    name: deltaFn?.name || "",
+  };
+}
+
+function updateToolCallFunction(
+  existingFn: OpenAI.Chat.Completions.ChatCompletionMessageToolCall.Function,
+  deltaFn: OpenAI.Chat.Completions.ChatCompletionChunk.Choice.Delta.ToolCall.Function // eslint-disable-line
+) {
+  //   export interface ChatCompletionChunk.Choice.Delta.ToolCall.Function {
+  //     arguments?: string;
+  //     name?: string;
+  //   }
+  // ->
+  //   export interface Function {
+  //     arguments: string;
+  //     name: string;
+  //   }
+
+  // The function can have either (or possibly both) field(s) empty.
+  // `arguments` has been observered to arrive in chunks.  The same is
+  // probably true of `name`.
+
+  if (!deltaFn) {
+    return;
+  }
+  if (deltaFn.name) {
+    existingFn.name += deltaFn.name;
+  }
+  if (deltaFn.arguments) {
+    existingFn.arguments += deltaFn.arguments;
+  }
+}
+
+function initialToolCall(
+  delta: OpenAI.Chat.Completions.ChatCompletionChunk.Choice.Delta.ToolCall
+): OpenAI.Chat.Completions.ChatCompletionMessageToolCall {
+  return {
+    id: delta.id || "",
+    function: initialToolCallFunction(delta.function),
+    type: "function",
+  };
+}
+
+function updateToolCall(
+  existing: OpenAI.Chat.Completions.ChatCompletionMessageToolCall,
+  delta: OpenAI.Chat.Completions.ChatCompletionChunk.Choice.Delta.ToolCall
+) {
+  //   export interface ChatCompletionChunk.Choice.Delta.ToolCall {
+  //     index: number;
+  //     id?: string;
+  //     function?: ToolCall.Function;
+  //     type?: 'function';
+  //   }
+  //
+  // ->
+  //
+  //   export interface ChatCompletionMessageToolCall {
+  //     id: string;
+  //     function: ChatCompletionMessageToolCall.Function;
+  //     type: 'function';
+  //   }
+
+  if (delta.id) {
+    if (existing.id.length > 0) {
+      assert(delta.id == existing.id);
+    } else {
+      existing.id = delta.id;
+    }
+  }
+
+  if (delta.function) {
+    updateToolCallFunction(existing.function, delta.function);
+  }
+
+  if (delta.type) {
+    assert((delta.type = "function"));
+  }
+}
+
+function updateToolCalls(
+  toolCalls:
+    | OpenAI.Chat.Completions.ChatCompletionMessageToolCall[]
+    | undefined,
+  deltaToolCall: OpenAI.Chat.Completions.ChatCompletionChunk.Choice.Delta.ToolCall // eslint-disable-line
+): OpenAI.Chat.Completions.ChatCompletionMessageToolCall[] {
+  //   export interface ChatCompletionChunk.Choice.Delta.ToolCall {
+  //     index: number;
+  //     id?: string;
+  //     function?: ToolCall.Function;
+  //     type?: 'function';
+  //   }
+  //
+  // ->
+  //
+  //   export interface ChatCompletionMessageToolCall {
+  //     id: string;
+  //     function: ChatCompletionMessageToolCall.Function;
+  //     type: 'function';
+  //   }
+
+  // The delta can arrive with any or none of the given fields.  Only `index`
+  // can be relied upon.
+
+  if (typeof toolCalls === "undefined") {
+    toolCalls = [];
+  }
+  const existing = toolCalls[deltaToolCall.index];
+  if (!existing) {
+    toolCalls[deltaToolCall.index] = initialToolCall(deltaToolCall);
+  } else {
+    updateToolCall(toolCalls[deltaToolCall.index], deltaToolCall);
+  }
+  return toolCalls;
+}
+
+function initializeCompletionMessage(
+  delta: OpenAI.Chat.Completions.ChatCompletionChunk.Choice.Delta
+): OpenAI.Chat.Completions.ChatCompletionMessage {
+  assert(delta.role === undefined || delta.role == "assistant");
+  assert(!delta.function_call);
+
+  //   export interface ChatCompletionChunk.Choice.Delta {
+  //     content?: string | null;
+  //     function_call?: Delta.FunctionCall;
+  //     refusal?: string | null;
+  //     role?: 'developer' | 'system' | 'user' | 'assistant' | 'tool';
+  //     tool_calls?: Array<Delta.ToolCall>;
+  //   }
+  //
+  // ->
+  //
+  //   export interface ChatCompletionMessage {
+  //     content: string | null;
+  //     refusal: string | null;
+  //     role: 'assistant';
+  //     annotations?: Array<ChatCompletionMessage.Annotation>;
+  //     audio?: ChatCompletionAudio | null;
+  //     function_call?: ChatCompletionMessage.FunctionCall | null;
+  //     tool_calls?: Array<ChatCompletionMessageToolCall>;
+  //   }
+
+  let toolCalls:
+    | OpenAI.Chat.Completions.ChatCompletionMessageToolCall[]
+    | undefined = undefined;
+  if (delta.tool_calls) {
+    for (const t of delta.tool_calls) {
+      toolCalls = updateToolCalls(toolCalls, t);
+    }
+  }
+
+  return {
+    content: delta.content || null,
+    refusal: delta.refusal || null,
+    role: "assistant",
+    // annotations?: Array<ChatCompletionMessage.Annotation>;
+    // audio?: ChatCompletionAudio | null;
+    // function_call: delta.function_call,
+    tool_calls: toolCalls,
+  };
+}
+
+function updateCompletionMessage(
+  message: OpenAI.Chat.Completions.ChatCompletionMessage,
+  delta: OpenAI.Chat.Completions.ChatCompletionChunk.Choice.Delta
+) {
+  assert(message.role === "assistant");
+  assert(!message.function_call);
+  assert(!message.audio);
+  assert(
+    message.tool_calls instanceof Array ||
+      typeof message.tool_calls === "undefined"
+  );
+
+  //   export interface ChatCompletionChunk.Choice.Delta {
+  //     content?: string | null;
+  //     function_call?: Delta.FunctionCall;
+  //     refusal?: string | null;
+  //     role?: 'developer' | 'system' | 'user' | 'assistant' | 'tool';
+  //     tool_calls?: Array<Delta.ToolCall>;
+  //   }
+  //
+  // ->
+  //
+  //   export interface ChatCompletionMessage {
+  //     content: string | null;
+  //     refusal: string | null;
+  //     role: 'assistant';
+  //     annotations?: Array<ChatCompletionMessage.Annotation>;
+  //     audio?: ChatCompletionAudio | null;
+  //     function_call?: ChatCompletionMessage.FunctionCall | null;
+  //     tool_calls?: Array<ChatCompletionMessageToolCall>;
+  //   }
+
+  if (delta.content) {
+    if (message.content) {
+      message.content += delta.content;
+    } else {
+      message.content = delta.content;
+    }
+  }
+  assert(!delta.function_call);
+  if (delta.refusal) {
+    if (message.refusal) {
+      message.refusal += delta.refusal;
+    } else {
+      message.refusal = delta.refusal;
+    }
+  }
+  assert(delta.role === undefined || delta.role == "assistant");
+  if (delta.tool_calls) {
+    for (const t of delta.tool_calls) {
+      message.tool_calls = updateToolCalls(message.tool_calls, t);
+    }
+  }
+}
+
+function initializeCompletionChoice(
+  chunkChoice: OpenAI.Chat.Completions.ChatCompletionChunk.Choice
+): { choice: OpenAI.Chat.Completions.ChatCompletion.Choice; done: boolean } {
+  //   export interface ChatCompletionChunk.Choice {
+  //     delta: Choice.Delta;
+  //     finish_reason:
+  //       'stop'|'length'|'tool_calls'|'content_filter'|'function_call'|null;
+  //     index: number;
+  //     logprobs?: Choice.Logprobs | null;
+  //   }
+  //
+  // ->
+  //
+  //   export interface ChatCompletion.Choice {
+  //     message: CompletionsCompletionsAPI.ChatCompletionMessage;
+  //     finish_reason:
+  //         'stop'|'length'|'tool_calls'|'content_filter'|'function_call';
+  //     index: number;
+  //     logprobs: Choice.Logprobs | null;
+  //   }
+
+  const message = initializeCompletionMessage(chunkChoice.delta);
+  return {
+    choice: {
+      message,
+      finish_reason: chunkChoice.finish_reason || "stop",
+      index: chunkChoice.index,
+      logprobs: chunkChoice.logprobs || null,
+    },
+    done: !!chunkChoice.finish_reason,
+  };
+}
+
+function updateCompletionChoice(
+  completionChoice: OpenAI.Chat.Completions.ChatCompletion.Choice,
+  chunkChoice: OpenAI.Chat.Completions.ChatCompletionChunk.Choice
+): boolean {
+  //   export interface ChatCompletionChunk.Choice {
+  //     delta: Choice.Delta;
+  //     finish_reason:
+  //       'stop'|'length'|'tool_calls'|'content_filter'|'function_call'|null;
+  //     index: number;
+  //     logprobs?: Choice.Logprobs | null;
+  //   }
+  //
+  // ->
+  //
+  //   export interface ChatCompletion.Choice {
+  //     message: CompletionsCompletionsAPI.ChatCompletionMessage;
+  //     finish_reason:
+  //         'stop'|'length'|'tool_calls'|'content_filter'|'function_call';
+  //     index: number;
+  //     logprobs: Choice.Logprobs | null;
+  //   }
+
+  // TODO: logprobs
+
+  assert(completionChoice.index === chunkChoice.index);
+  updateCompletionMessage(completionChoice.message, chunkChoice.delta);
+  if (chunkChoice.finish_reason) {
+    completionChoice.finish_reason = chunkChoice.finish_reason;
+    return true;
+  }
+  return false;
+}
+
+function initializeCompletionChoices(
+  chunkChoices: OpenAI.Chat.Completions.ChatCompletionChunk.Choice[]
+): { choices: OpenAI.Chat.Completions.ChatCompletion.Choice[]; done: boolean } {
+  // Technically, one choice could be done and the other still have some
+  // content to stream.  We keep it simple for now and assume only single
+  // choices, which allows us to mark everything as done if any choice we hit is
+  // done.
+  assert(chunkChoices.length === 1);
+
+  let msgDone = false;
+  const choices: OpenAI.Chat.Completions.ChatCompletion.Choice[] = [];
+  for (const chunkChoice of chunkChoices) {
+    const { choice, done } = initializeCompletionChoice(chunkChoice);
+    if (done) {
+      msgDone = true;
+    }
+    choices[chunkChoice.index] = choice;
+  }
+
+  return { choices, done: msgDone };
+}
+
+function updateCompletionChoices(
+  completionChoices: OpenAI.Chat.Completions.ChatCompletion.Choice[],
+  chunkChoices: OpenAI.Chat.Completions.ChatCompletionChunk.Choice[]
+): boolean {
+  // Technically, one choice could be done and the other still have some
+  // content to stream.  We keep it simple for now and assume only single
+  // choices, which allows us to mark everything as done if any choice we hit is
+  // done.
+  assert(chunkChoices.length === 1);
+  assert(completionChoices.length === 1);
+
+  let msgDone = false;
+  for (const chunkChoice of chunkChoices) {
+    const choiceIdx = chunkChoice.index;
+    const done = updateCompletionChoice(
+      completionChoices[choiceIdx],
+      chunkChoice
+    );
+    if (done) {
+      msgDone = true;
+    }
+  }
+
+  return msgDone;
+}
+
+function initializeCompletion(
+  chunk: OpenAI.Chat.Completions.ChatCompletionChunk
+): { initMessage: OpenAI.Chat.Completions.ChatCompletion; done: boolean } {
+  //   export interface ChatCompletionChunk {
+  //     id: string;
+  //     choices: Array<ChatCompletionChunk.Choice>;
+  //     created: number;
+  //     model: string;
+  //     object: 'chat.completion.chunk';
+  //     service_tier?: 'auto' | 'default' | 'flex' | null;
+  //     system_fingerprint?: string;
+  //     usage?: CompletionsAPI.CompletionUsage | null;
+  //   }
+  //
+  // ->
+  //
+  //   export interface ChatCompletion {
+  //     id: string;
+  //     choices: Array<ChatCompletion.Choice>;
+  //     created: number;
+  //     model: string;
+  //     object: 'chat.completion';
+  //     service_tier?: 'auto'|'default'|'flex'|null;
+  //     system_fingerprint?: string;
+  //     usage?: CompletionsAPI.CompletionUsage;
+  //   }
+
+  const { choices, done } = initializeCompletionChoices(chunk.choices);
+  return {
+    initMessage: {
+      id: chunk.id,
+      choices,
+      created: chunk.created,
+      model: chunk.model,
+      object: "chat.completion",
+      service_tier: chunk.service_tier,
+      system_fingerprint: chunk.system_fingerprint,
+      usage: chunk.usage ?? undefined,
+    },
+    done,
+  };
+}
+
+function updateCompletion(
+  completion: OpenAI.Chat.Completions.ChatCompletion,
+  chunk: OpenAI.Chat.Completions.ChatCompletionChunk
+): boolean {
+  //   export interface ChatCompletionChunk {
+  //     id: string;
+  //     choices: Array<ChatCompletionChunk.Choice>;
+  //     created: number;
+  //     model: string;
+  //     object: 'chat.completion.chunk';
+  //     service_tier?: 'auto' | 'default' | 'flex' | null;
+  //     system_fingerprint?: string;
+  //     usage?: CompletionsAPI.CompletionUsage | null;
+  //   }
+  //
+  // ->
+  //
+  //   export interface ChatCompletion {
+  //     id: string;
+  //     choices: Array<ChatCompletion.Choice>;
+  //     created: number;
+  //     model: string;
+  //     object: 'chat.completion';
+  //     service_tier?: 'auto'|'default'|'flex'|null;
+  //     system_fingerprint?: string;
+  //     usage?: CompletionsAPI.CompletionUsage;
+  //   }
+
+  assert(completion.id === chunk.id);
+  assert(completion.model === chunk.model);
+  completion.service_tier = completion.service_tier || chunk.service_tier;
+  completion.system_fingerprint =
+    completion.system_fingerprint || chunk.system_fingerprint;
+  completion.usage = completion.usage || chunk.usage || undefined;
+
+  return updateCompletionChoices(completion.choices, chunk.choices);
+}
+
+export class OpenAILLMStreaming implements ILLM {
+  private readonly openai: OpenAI;
+  private model: string;
+
+  constructor(
+    apiKey: string,
+    apiUrl: string | undefined,
+    model: string | undefined
+  ) {
+    this.openai = new OpenAI({
+      apiKey,
+      baseURL: apiUrl,
+      dangerouslyAllowBrowser: true,
+    });
+    this.model = model || "gpt-4o-mini";
+  }
+
+  public setModel(model: string) {
+    this.model = model;
+  }
+
+  getModel(): string {
+    return this.model;
+  }
+
+  getUrl(): string {
+    return this.openai.baseURL;
+  }
+
+  public async getConversationResponse(
+    messages: OpenAI.Chat.Completions.ChatCompletionMessageParam[],
+    tools?: OpenAI.Chat.Completions.ChatCompletionTool[],
+    onMessage?: (msg: string, end: boolean) => Promise<void>
+  ): Promise<OpenAI.Chat.Completions.ChatCompletion> {
+    const chunks = await this.openai.chat.completions.create({
+      model: this.model,
+      messages,
+      tools,
+      stream: true,
+    });
+
+    // Check the type casting above
+    if (!(chunks as unknown as { iterator: unknown }).iterator) {
+      throw "not a stream";
+    }
+
+    let aggregatedMessage: OpenAI.Chat.Completions.ChatCompletion | undefined;
+    let done = false;
+
+    for await (const chunk of chunks) {
+      logger.debug(`[stream] chunk: ${JSON.stringify(chunk)}`);
+      assert(!done);
+
+      if (chunk.object !== "chat.completion.chunk") {
+        // logger.warn("[stream]: unexpected message");
+        continue;
+      }
+
+      done = (() => {
+        if (!aggregatedMessage) {
+          logger.debug(`[stream] first}`);
+          const { initMessage, done } = initializeCompletion(chunk);
+          aggregatedMessage = initMessage;
+          return done;
+        } else {
+          return updateCompletion(aggregatedMessage, chunk);
+        }
+      })();
+
+      if (onMessage) {
+        // Inform the call of a message fragment.  Note that even if there is
+        // no content, we must call `onMessage` once `done` is true.
+
+        const delta = chunk.choices[0].delta;
+        if (delta.content) {
+          await onMessage(delta.content, done);
+        } else if (done) {
+          await onMessage("", true);
+        }
+      }
+    }
+
+    logger.debug(
+      `[stream] final message: ${JSON.stringify(aggregatedMessage)}`
+    );
+
+    assert(done);
+    assert(aggregatedMessage);
+    return aggregatedMessage;
+  }
+}

package/src/options.ts

@@ -0,0 +1,103 @@
+import { boolean, option, optional, string, flag, number } from "cmd-ts";
+import { ArgParser } from "cmd-ts/dist/cjs/argparser";
+import { Descriptive, ProvidesHelp } from "cmd-ts/dist/cjs/helpdoc";
+import { DEFAULT_LLM_URL } from "./agentUtils";
+
+export type Option = ReturnType<typeof option>;
+export type OptionalOption = ArgParser<string | undefined> &
+  ProvidesHelp &
+  Partial<Descriptive>;
+
+/// Prevents env content from being displayed in the help text.
+export function secretOption({
+  long,
+  short,
+  env,
+  description,
+}: {
+  long: string;
+  short?: string;
+  env?: string;
+  description: string;
+}): OptionalOption {
+  if (env) {
+    return option({
+      type: optional(string),
+      long,
+      short,
+      description: `${description} [env: ${env}]`,
+      defaultValue: () => process.env[env],
+      defaultValueIsSerializable: false, // hides the value from --help
+    });
+  }
+
+  return option({
+    type: optional(string),
+    long,
+    short,
+    description: `${description} (can also be set via ${env} env var)`,
+  });
+}
+
+export const promptFile = option({
+  type: optional(string),
+  long: "prompt",
+  short: "p",
+  description: "File containing user's first prompt to LLM",
+});
+
+export const imageFile = option({
+  type: optional(string),
+  long: "image",
+  short: "i",
+  description: "File containing image input",
+});
+
+export const systemPromptFile = option({
+  type: optional(string),
+  long: "sysprompt",
+  short: "s",
+  description: "File containing system prompt",
+});
+
+export const llmModel = option({
+  type: optional(string),
+  long: "model",
+  short: "m",
+  description: "LLM model",
+  env: "LLM_MODEL",
+});
+
+export const oneShot = flag({
+  type: boolean,
+  long: "one-shot",
+  short: "1",
+  description: "Exit after first reply (implies --approve-tools)",
+});
+
+export const approveTools = flag({
+  type: boolean,
+  long: "approve-tools",
+  short: "y",
+  description: "Automatically approve all tool calls",
+});
+
+export const approveToolsUpTo = option({
+  type: optional(number),
+  long: "approve-tools-up-to",
+  description: "Automatically approve all tool calls up to some number",
+});
+
+export const llmApiKey = secretOption({
+  long: "llm-api-key",
+  description: "API Key for LLM provider",
+  env: "LLM_API_KEY",
+});
+
+export const llmUrl = option({
+  type: string,
+  long: "llm-url",
+  description: "LLM provider URL (OpenAI compatible)",
+  env: "LLM_URL",
+  defaultValue: () => DEFAULT_LLM_URL,
+});