mu-core 0.13.0 → 0.16.1

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "mu-core",
-  "version": "0.13.0",
-  "description": "Agent loop orchestration core: types, plugin SDK, channels, sessions",
+  "version": "0.16.1",
+  "description": "Standalone multimodal agentic loop: content, messages, tools, provider interface, createAgent",
   "type": "module",
   "main": "./src/index.ts",
   "types": "./src/index.ts",

package/src/agent.test.ts

@@ -1,91 +1,45 @@
-/**
- * Focused tests for `runAgent`. Most coverage lives in higher layers
- * (session, mu-agents subagent, integration) but we keep a few pinpoint
- * cases here to lock down behaviour that's easy to break:
- *
- *  - `display.llmHidden` strips messages from the network payload but
- *    keeps them in the streamed transcript surface (regression target
- *    for the `@`-mention dispatch flow that injects a UI-only subagent
- *    header alongside a real synthetic tool flow).
- */
-import { describe, expect, it } from 'bun:test';
-import { runAgent } from './agent';
-import { createProviderRegistry } from './provider/registry';
-import { PluginRegistry } from './registry';
-import type { ChatMessage, ProviderConfig, StreamChunk } from './types/llm';
-
-interface CapturedCall {
-  messages: ChatMessage[];
-}
-
-function fakeRegistry(captured: CapturedCall[]): PluginRegistry {
-  const providers = createProviderRegistry();
-  providers.register({
-    id: 'openai',
-    async *streamChat(messages: ChatMessage[]): AsyncIterable<StreamChunk> {
-      // Snapshot the exact message list the provider receives so the test
-      // can assert on what `streamTurn` actually sent over the wire.
-      captured.push({ messages: messages.map((m) => ({ ...m })) });
-      yield { type: 'content', text: 'ok' };
+import { assertEquals } from '@std/assert';
+import { createAgent } from './agent';
+import { image } from './types';
+import type { ContentPart, Provider, Tool } from './types';
+
+const scripted = (turns: ContentPart[][]): Provider => {
+  let i = 0;
+  return {
+    async *stream() {
+      for (const event of turns[i++]) yield event;
     },
-    async listModels() {
-      return [];
-    },
-  });
-  return new PluginRegistry({ cwd: '/tmp', config: {}, providers });
-}
-
-const CFG: ProviderConfig = { providerId: 'openai' };
-
-describe('runAgent — display.llmHidden filter', () => {
-  it('strips llmHidden messages from the provider payload', async () => {
-    const captured: CapturedCall[] = [];
-    const registry = fakeRegistry(captured);
-
-    const messages: ChatMessage[] = [
-      { role: 'system', content: 'sys' },
-      // UI-only marker — must be filtered before the network call.
-      {
-        role: 'assistant',
-        content: 'phantom header',
-        display: { llmHidden: true },
-      },
-      { role: 'user', content: 'hi' },
-    ];
-
-    for await (const _ of runAgent(messages, CFG, 'm', new AbortController().signal, registry)) {
-      // drain
-    }
-
-    expect(captured.length).toBe(1);
-    const sent = captured[0].messages;
-    // Only system + user reach the provider.
-    expect(sent.length).toBe(2);
-    expect(sent[0].role).toBe('system');
-    expect(sent[1].role).toBe('user');
-    expect(sent.find((m) => m.content === 'phantom header')).toBeUndefined();
+  };
+};
+
+Deno.test('loops on a tool_call and returns the final message', async () => {
+  const provider = scripted([
+    [{ type: 'tool_call', id: '1', name: 'snap', input: {} }],
+    [{ type: 'text', text: 'voici' }],
+  ]);
+
+  const snap: Tool = {
+    name: 'snap',
+    description: 'returns an image',
+    parameters: {},
+    run: async () => [image('image/png', new Uint8Array([1, 2, 3]))],
+  };
+
+  const agent = createAgent({ provider, model: 'mock', tools: [snap] });
+  const { message, messages } = await agent.run('photo ?');
+
+  assertEquals(message.content, [{ type: 'text', text: 'voici' }]);
+  const tool = messages.find((m) => m.role === 'user' && m.content[0]?.type === 'tool_result');
+  assertEquals(tool?.content[0], {
+    type: 'tool_result',
+    id: '1',
+    content: [{ type: 'image', mime: 'image/png', data: new Uint8Array([1, 2, 3]) }],
   });
+});
 
-  it('keeps non-llmHidden messages even when `display.hidden` is set', async () => {
-    // Inverse flag: `display.hidden` is UI-only suppression and must NOT
-    // affect the LLM payload. This test pins that the two flags do not
-    // accidentally collapse into the same filter.
-    const captured: CapturedCall[] = [];
-    const registry = fakeRegistry(captured);
-
-    const messages: ChatMessage[] = [
-      { role: 'system', content: 'sys' },
-      { role: 'user', content: 'silent reminder', display: { hidden: true } },
-      { role: 'user', content: 'hi' },
-    ];
-
-    for await (const _ of runAgent(messages, CFG, 'm', new AbortController().signal, registry)) {
-      // drain
-    }
-
-    expect(captured.length).toBe(1);
-    const sent = captured[0].messages;
-    expect(sent.length).toBe(3);
-    expect(sent.find((m) => m.content === 'silent reminder')).toBeDefined();
-  });
+Deno.test('merges streamed text deltas', async () => {
+  const provider = scripted([[{ type: 'text', text: 'bon' }, { type: 'text', text: 'jour' }]]);
+  const agent = createAgent({ provider, model: 'mock' });
+  const { message } = await agent.run('salut');
+  assertEquals(message.content, [{ type: 'text', text: 'bonjour' }]);
 });

package/src/agent.ts

@@ -1,247 +1,140 @@
-import {
-  runAfterAgentRunHooks,
-  runAfterLlmHooks,
-  runAfterToolExecHook,
-  runBeforeLlmHooks,
-  runBeforeToolExecHook,
-  runDecorateMessageHooks,
-} from './hooks';
-import type { AgentEvent, PluginTool, ToolResult, TurnResult } from './plugin';
-import type { PluginRegistry } from './registry';
-import type { ChatMessage, ProviderConfig, StreamChunk, StreamOptions, ToolCall } from './types/llm';
-
-const DEFAULT_PROVIDER_ID = 'openai';
-
-/**
- * Stream a chat completion through the host's provider registry. Resolved
- * by `config.providerId` (default `'openai'`). When no provider matches, we
- * throw — `mu-core` is provider-agnostic, the host is responsible for
- * registering at least one (e.g. via `mu-openai-provider`).
- */
-function streamChatViaRegistry(
-  registry: PluginRegistry,
-  messages: ChatMessage[],
-  config: ProviderConfig,
-  model: string,
-  options: StreamOptions,
-): AsyncIterable<StreamChunk> {
-  const id = config.providerId ?? DEFAULT_PROVIDER_ID;
-  const providers = registry.getProviders();
-  const provider = providers?.get(id);
-  if (!provider) {
-    throw new Error(
-      `No provider registered for id "${id}". Register one (e.g. via mu-openai-provider) before calling runAgent.`,
-    );
-  }
-  return provider.streamChat(messages, config, model, options);
-}
-
-async function executeTool(call: ToolCall, tools: PluginTool[], signal?: AbortSignal): Promise<ToolResult> {
-  let args: Record<string, unknown>;
-  try {
-    args = JSON.parse(call.function.arguments);
-  } catch {
-    return { tool_call_id: call.id, name: call.function.name, content: 'Error: Invalid JSON arguments', error: true };
-  }
-
-  const tool = tools.find((t) => t.definition.function.name === call.function.name);
-  if (!tool) {
-    return {
-      tool_call_id: call.id,
-      name: call.function.name,
-      content: `Error: Unknown tool: ${call.function.name}`,
-      error: true,
-    };
+import type { ContentPart, Message, Provider, Tool, Usage } from './types';
+
+type ToolCallPart = Extract<ContentPart, { type: 'tool_call' }>;
+
+export type LoopEvent =
+  | ContentPart
+  | { type: 'usage'; usage: Usage }
+  | { type: 'reasoning'; text: string }
+  | { type: 'message'; message: Message }
+  | { type: 'done'; messages: Message[] };
+
+const append = (parts: ContentPart[], part: ContentPart): void => {
+  const last = parts[parts.length - 1];
+  if (part.type === 'text' && last?.type === 'text') {
+    last.text += part.text;
+  } else if (part.type === 'audio' && last?.type === 'audio' && last.mime === part.mime) {
+    const merged = new Uint8Array(last.data.length + part.data.length);
+    merged.set(last.data);
+    merged.set(part.data, last.data.length);
+    last.data = merged;
+  } else if (part.type === 'text') {
+    parts.push({ type: 'text', text: part.text });
+  } else if (part.type === 'audio') {
+    parts.push({ type: 'audio', mime: part.mime, data: part.data });
+  } else {
+    parts.push(part);
   }
+};
 
+const execute = async (tools: Map<string, Tool>, call: ToolCallPart, signal?: AbortSignal): Promise<ContentPart[]> => {
+  const tool = tools.get(call.name);
+  if (!tool) return [{ type: 'text', text: `Unknown tool: ${call.name}` }];
   try {
-    const result = await tool.execute(args, signal);
-    // Tools may return either a plain string (error inferred from "Error:"
-    // prefix — legacy/convenience form) or a typed `{ content, error }`
-    // object that makes the error flag explicit. Both forms are accepted to
-    // avoid breaking existing plugin authors.
-    if (typeof result === 'string') {
-      return { tool_call_id: call.id, name: call.function.name, content: result, error: result.startsWith('Error:') };
-    }
-    return {
-      tool_call_id: call.id,
-      name: call.function.name,
-      content: result.content,
-      error: result.error ?? false,
-    };
+    return await tool.run(call.input, { signal });
   } catch (err) {
-    const msg = err instanceof Error ? err.message : 'Unknown error';
-    return { tool_call_id: call.id, name: call.function.name, content: `Error: ${msg}`, error: true };
+    return [{ type: 'text', text: err instanceof Error ? err.message : String(err) }];
   }
+};
+
+export interface RunOptions {
+  provider: Provider;
+  model: string;
+  messages: Message[];
+  tools?: Tool[];
+  signal?: AbortSignal;
 }
 
-async function* streamTurn(
-  messages: ChatMessage[],
-  config: ProviderConfig,
-  model: string,
-  signal: AbortSignal,
-  registry: PluginRegistry,
-  toolDefs: PluginTool[],
-): AsyncGenerator<AgentEvent, TurnResult> {
-  let content = '';
-  let reasoning = '';
-  let usage = 0;
-  let cachedPromptTokens = 0;
-  const toolCalls: ToolCall[] = [];
-
-  const hooks = registry.getHooks();
-  const hookedMessages = await runBeforeLlmHooks(hooks, messages, config);
-  // `display.llmHidden` keeps a message in the on-screen transcript but
-  // strips it from the LLM payload. Applied AFTER `beforeLlmHooks` so plugin
-  // hooks still see the full transcript if they want it; this is the very
-  // last filter before the network call. Inverse of `display.hidden`, which
-  // hides from UI but keeps in the LLM payload.
-  const llmMessages = hookedMessages.filter((m) => !m.display?.llmHidden);
-  const toolDefinitions = toolDefs.map((t) => t.definition);
-
-  for await (const chunk of streamChatViaRegistry(registry, llmMessages, config, model, {
-    signal,
-    tools: toolDefinitions,
-    onUsage: (u) => {
-      usage = u.totalTokens;
-      cachedPromptTokens = u.cachedPromptTokens ?? 0;
-    },
-  })) {
-    if (signal.aborted) {
-      break;
-    }
-    if (chunk.type === 'reasoning') {
-      reasoning += chunk.text;
-      yield { type: 'reasoning', text: reasoning };
-    } else if (chunk.type === 'content') {
-      content += chunk.text;
-      yield { type: 'content', text: content };
-    } else if (chunk.type === 'tool_call') {
-      toolCalls.push({ id: chunk.toolCall.id, function: chunk.toolCall.function });
+export async function* run(opts: RunOptions): AsyncIterable<LoopEvent> {
+  const { provider, model, signal } = opts;
+  const tools = opts.tools ?? [];
+  const registry = new Map(tools.map((t) => [t.name, t]));
+  const messages = [...opts.messages];
+
+  while (true) {
+    const content: ContentPart[] = [];
+    const calls: ToolCallPart[] = [];
+
+    for await (const event of provider.stream({ model, messages, tools, signal })) {
+      if (event.type === 'usage' || event.type === 'reasoning') {
+        yield event;
+        continue;
+      }
+      yield event;
+      append(content, event);
+      if (event.type === 'tool_call') calls.push(event);
     }
-  }
 
-  const result: TurnResult = { content, reasoning, toolCalls, usage, cachedPromptTokens };
-  return await runAfterLlmHooks(hooks, result);
-}
+    const message: Message = { role: 'assistant', content };
+    messages.push(message);
+    yield { type: 'message', message };
+
+    if (calls.length === 0) break;
 
-async function executeOneToolCall(
-  tc: ToolCall,
-  tools: PluginTool[],
-  signal: AbortSignal,
-  registry: PluginRegistry,
-): Promise<ChatMessage> {
-  const hooks = registry.getHooks();
-  const hookOutcome = await runBeforeToolExecHook(hooks, tc);
-
-  if ('blocked' in hookOutcome) {
-    const content = await runAfterToolExecHook(hooks, tc, hookOutcome.content);
-    return runDecorateMessageHooks(hooks, {
-      role: 'tool',
-      content,
-      toolCallId: tc.id,
-      toolResult: { name: tc.function.name, content, error: hookOutcome.error ?? true },
-      toolCallArgs: { [tc.function.name]: tc.function.arguments },
-    });
+    const results: ContentPart[] = await Promise.all(
+      calls.map(async (call) => ({
+        type: 'tool_result' as const,
+        id: call.id,
+        content: await execute(registry, call, signal),
+      })),
+    );
+    const toolMessage: Message = { role: 'user', content: results };
+    messages.push(toolMessage);
+    yield { type: 'message', message: toolMessage };
   }
 
-  const result = await executeTool(hookOutcome, tools, signal);
-  const content = await runAfterToolExecHook(hooks, hookOutcome, result.content);
-  return runDecorateMessageHooks(hooks, {
-    role: 'tool',
-    content,
-    toolCallId: result.tool_call_id,
-    toolResult: { name: result.name, content, error: result.error },
-    toolCallArgs: { [result.name]: tc.function.arguments },
-  });
+  yield { type: 'done', messages };
 }
 
-async function* executeToolCalls(
-  calls: ToolCall[],
-  start: ChatMessage[],
-  signal: AbortSignal,
-  registry: PluginRegistry,
-  tools: PluginTool[],
-): AsyncGenerator<AgentEvent, ChatMessage[]> {
-  let current = start;
-
-  for (const tc of calls) {
-    if (signal.aborted) break;
-    const toolMessage = await executeOneToolCall(tc, tools, signal, registry);
-    if (signal.aborted) break;
-    current = [...current, toolMessage];
-    yield { type: 'messages', messages: current };
-  }
-  return current;
+export interface AgentConfig {
+  provider: Provider;
+  model: string;
+  tools?: Tool[];
+  system?: string;
+  signal?: AbortSignal;
 }
 
-async function buildMergedConfig(config: ProviderConfig, registry: PluginRegistry): Promise<ProviderConfig> {
-  const pluginPrompts = await registry.getSystemPrompts();
-  const merged = [config.systemPrompt, ...pluginPrompts].filter(Boolean).join('\n\n');
-  const transformed = await registry.applySystemPromptTransforms(merged);
-  if (!transformed) return config;
-  return { ...config, systemPrompt: transformed };
+export type Input = string | ContentPart[] | Message[];
+
+export interface AgentResult {
+  message: Message;
+  messages: Message[];
 }
 
-export async function* runAgent(
-  initialMessages: ChatMessage[],
-  config: ProviderConfig,
-  model: string,
-  signal: AbortSignal,
-  registry: PluginRegistry,
-): AsyncGenerator<AgentEvent> {
-  const mergedConfig = await buildMergedConfig(config, registry);
-  const hooks = registry.getHooks();
-
-  // Wrap the body in try/finally so `afterAgentRun` fires exactly once per
-  // `runAgent` call, regardless of how it terminates: normal completion (LLM
-  // produced a final response), abort via `signal.aborted`, or generator
-  // cancellation by the consumer (Ink unmount, manual `.return()`).
-  try {
-    const customLoop = registry.getAgentLoop();
-    if (customLoop) {
-      const filtered = await registry.getFilteredTools();
-      yield* customLoop.run(initialMessages, mergedConfig, model, signal, filtered, hooks);
-      return;
-    }
+export interface Agent {
+  stream(input: Input): AsyncIterable<LoopEvent>;
+  run(input: Input): Promise<AgentResult>;
+}
 
-    let current = initialMessages;
-
-    while (!signal.aborted) {
-      // Re-evaluate every turn so per-turn agent switches are honoured.
-      const tools = await registry.getFilteredTools();
-      const { content, reasoning, toolCalls, usage, cachedPromptTokens } = yield* streamTurn(
-        current,
-        mergedConfig,
-        model,
-        signal,
-        registry,
-        tools,
-      );
-
-      if (usage > 0) {
-        yield { type: 'usage', totalTokens: usage, cachedTokens: cachedPromptTokens };
-      }
-      if (signal.aborted) {
-        break;
-      }
+const isMessages = (input: ContentPart[] | Message[]): input is Message[] => input.length > 0 && 'role' in input[0];
 
-      const reasoningField = reasoning || undefined;
-      const assistantBase: ChatMessage = { role: 'assistant', content, reasoning: reasoningField };
-      if (toolCalls.length === 0) {
-        const assistant = await runDecorateMessageHooks(registry.getHooks(), assistantBase);
-        current = [...current, assistant];
-        yield { type: 'messages', messages: current };
-        return;
-      }
+const toMessages = (input: Input): Message[] => {
+  if (typeof input === 'string') return [{ role: 'user', content: [{ type: 'text', text: input }] }];
+  if (isMessages(input)) return input;
+  return [{ role: 'user', content: input }];
+};
+
+export const createAgent = (config: AgentConfig): Agent => {
+  const tools = config.tools ?? [];
+
+  const build = (input: Input): Message[] => {
+    const messages = toMessages(input);
+    if (!config.system) return messages;
+    return [{ role: 'system', content: [{ type: 'text', text: config.system }] }, ...messages];
+  };
+
+  const stream = (input: Input): AsyncIterable<LoopEvent> =>
+    run({ provider: config.provider, model: config.model, tools, messages: build(input), signal: config.signal });
 
-      const assistantWithCalls = await runDecorateMessageHooks(registry.getHooks(), { ...assistantBase, toolCalls });
-      current = [...current, assistantWithCalls];
-      yield { type: 'messages', messages: current };
-      current = yield* executeToolCalls(toolCalls, current, signal, registry, tools);
-      yield { type: 'turn_end' };
+  const runToEnd = async (input: Input): Promise<AgentResult> => {
+    let message: Message = { role: 'assistant', content: [] };
+    let messages: Message[] = [];
+    for await (const event of stream(input)) {
+      if (event.type === 'message' && event.message.role === 'assistant') message = event.message;
+      else if (event.type === 'done') messages = event.messages;
     }
-  } finally {
-    await runAfterAgentRunHooks(hooks, signal.aborted ? 'aborted' : 'complete');
-  }
-}
+    return { message, messages };
+  };
+
+  return { stream, run: runToEnd };
+};

package/src/index.ts

@@ -1,74 +1,5 @@
-export type { ActivityBus, ActivityEvent, ActivityKind, SubAgentEvent, SubAgentEventKind } from './activity';
-export { createActivityBus } from './activity';
-export { runAgent } from './agent';
-export type { Channel, ChannelRegistry, ChannelResponder, InboundKind, InboundMessage, ResponseMode } from './channel';
-export { createChannelRegistry } from './channel';
-export { runDecorateMessageHooks, runTransformUserInputHooks } from './hooks';
-export type { MuConfigShape, MuHandle, StartMuOptions } from './host/index';
-export { startMu } from './host/index';
-export type {
-  AgentEndReason,
-  AgentEvent,
-  AgentLoopStrategy,
-  AgentSourceRegistry,
-  BeforeToolExecResult,
-  CommandContext,
-  InputInfoSegment,
-  LifecycleHooks,
-  MentionCompletion,
-  MentionProvider,
-  MessageBus,
-  MessageRenderer,
-  Plugin,
-  PluginContext,
-  PluginExtras,
-  PluginRegistryView,
-  PluginTool,
-  PluginToolPermission,
-  ShortcutHandler,
-  SlashCommand,
-  StatusSegment,
-  ToolBlock,
-  ToolDisplayHint,
-  ToolExecutor,
-  ToolExecutorResult,
-  ToolResult,
-  TurnResult,
-  UserInputTransform,
-} from './plugin';
-export type {
-  ChatRequestInput,
-  ModelsRequestInput,
-  ParsedChatEvent,
-  Provider,
-  ProviderAdapter,
-  RequestSpec,
-} from './provider/adapter';
-export { createProvider } from './provider/adapter';
-export type { ProviderRegistry } from './provider/registry';
-export { createProviderRegistry } from './provider/registry';
-export { fetchWithIdleTimeout, readNDJSON, readSSE } from './provider/transport';
-export { PluginRegistry, type PluginRegistryOptions } from './registry';
-export type {
-  CreateSessionManagerOptions,
-  RunTurnOptions,
-  Session,
-  SessionEvent,
-  SessionInit,
-  SessionManager,
-} from './session';
-export { createSessionManager } from './session';
-export type {
-  ApiModel,
-  ChatMessage,
-  ImageAttachment,
-  MessageDisplay,
-  ProviderConfig,
-  StreamChunk,
-  StreamOptions,
-  ToolCall,
-  ToolDefinition,
-  ToolResultInfo,
-  Usage,
-} from './types/llm';
-export { ConsoleUIService, type UINotifyLevel, type UIService } from './ui';
+export type { ContentPart, Message, Provider, Role, StreamEvent, Tool, Usage } from './types';
+export { audio, image, text } from './types';
+
+export type { Agent, AgentConfig, AgentResult, Input, LoopEvent, RunOptions } from './agent';
+export { createAgent, run } from './agent';

package/src/types.ts

@@ -0,0 +1,37 @@
+export type ContentPart =
+  | { type: 'text'; text: string }
+  | { type: 'image'; mime: string; data: Uint8Array }
+  | { type: 'audio'; mime: string; data: Uint8Array }
+  | { type: 'tool_call'; id: string; name: string; input: unknown }
+  | { type: 'tool_result'; id: string; content: ContentPart[] };
+
+export type Role = 'system' | 'user' | 'assistant';
+export type Message = { role: Role; content: ContentPart[] };
+
+export const text = (value: string): ContentPart => ({ type: 'text', text: value });
+export const image = (mime: string, data: Uint8Array): ContentPart => ({ type: 'image', mime, data });
+export const audio = (mime: string, data: Uint8Array): ContentPart => ({ type: 'audio', mime, data });
+
+export interface Tool {
+  name: string;
+  description: string;
+  parameters: Record<string, unknown>;
+  prompt?: string;
+  run(input: unknown, ctx: { signal?: AbortSignal }): Promise<ContentPart[]>;
+}
+
+export interface Usage {
+  input?: number;
+  output?: number;
+  total?: number;
+  contextWindow?: number;
+}
+
+export type StreamEvent =
+  | ContentPart
+  | { type: 'usage'; usage: Usage }
+  | { type: 'reasoning'; text: string };
+
+export interface Provider {
+  stream(req: { model: string; messages: Message[]; tools: Tool[]; signal?: AbortSignal }): AsyncIterable<StreamEvent>;
+}