mu-harness 0.20.0 → 0.22.0

package/esm/channels/ws/client.js

@@ -1,5 +1,5 @@
 import { createWsClient } from './ws-client.js';
-import { textOf } from './wire.js';
+import { partsToAttachments, textOf } from './wire.js';
 function deferred() {
     let resolve;
     const promise = new Promise((r) => (resolve = r));
@@ -31,6 +31,9 @@ export async function connectHarness(opts) {
     let models = [];
     let modelSelected = '';
     let commands = [];
+    // Mutated in place by the connect-time `capabilities` frame so the host (read once by
+    // ChatApp after connect) reflects the server's actual model modalities.
+    const features = { ...(opts.features ?? {}) };
     const sessionsById = new Map();
     const subSessions = new Map();
     const approvalListeners = new Set();
@@ -58,7 +61,18 @@ export async function connectHarness(opts) {
             tools: [],
             send: (input) => {
                 if (!readonly) {
-                    client.send({ type: 'chat', sessionId: id, text: typeof input === 'string' ? input : textOf(input) });
+                    if (typeof input === 'string') {
+                        client.send({ type: 'chat', sessionId: id, text: input });
+                    }
+                    else {
+                        const attachments = partsToAttachments(input);
+                        client.send({
+                            type: 'chat',
+                            sessionId: id,
+                            text: textOf(input),
+                            ...(attachments.length > 0 ? { attachments } : {}),
+                        });
+                    }
                 }
                 return Promise.resolve();
             },
@@ -126,6 +140,10 @@ export async function connectHarness(opts) {
             case 'commands':
                 commands = frame.commands;
                 return;
+            case 'capabilities':
+                features.vision = frame.vision;
+                features.audio = frame.audio;
+                return;
             case 'models:listed': {
                 models = frame.models.map((m) => ({ id: m.id, ownedBy: m.ownedBy }));
                 modelSelected = frame.selected;
@@ -281,7 +299,7 @@ export async function connectHarness(opts) {
         initialThinking: opts.initialThinking ?? false,
         saveThinking: opts.saveThinking ?? (() => { }),
         history: opts.history,
-        features: opts.features,
+        features,
         banner: opts.banner,
         minimal: opts.minimal,
         commands: () => commands.map((c) => ({ name: c.command.replace(/^\//, ''), description: c.description })),

package/esm/channels/ws/index.d.ts

@@ -1,6 +1,6 @@
 export * from './protocol.js';
 export { type SessionSummaryWire, type PersistedSessionWire, type SessionService, createSessionService } from './session-service.js';
-export { type WireMessage, messagesToWire } from './wire.js';
+export { attachmentsToParts, messagesToWire, partsToAttachments, type WireAttachment, type WireMessage } from './wire.js';
 export { type WebSocketAdapter, type WebSocketAdapterOptions, webSocketAdapter } from './server.js';
 export { type ConnectHarnessOptions, connectHarness, type RemoteHarness } from './client.js';
 export { createWsClient, type WsClient, type WsClientOptions } from './ws-client.js';

package/esm/channels/ws/index.js

@@ -1,6 +1,6 @@
 export * from './protocol.js';
 export { createSessionService } from './session-service.js';
-export { messagesToWire } from './wire.js';
+export { attachmentsToParts, messagesToWire, partsToAttachments } from './wire.js';
 export { webSocketAdapter } from './server.js';
 export { connectHarness } from './client.js';
 export { createWsClient } from './ws-client.js';

package/esm/channels/ws/protocol.d.ts

@@ -1,5 +1,5 @@
 import type { PersistedSessionWire, SessionSummaryWire } from './session-service.js';
-import type { WireMessage } from './wire.js';
+import type { WireAttachment, WireMessage } from './wire.js';
 import type { ApprovalAction, PendingApproval } from '../../permissions/index.js';
 import type { Message, Usage } from 'mu-core';
 export interface WireModel {
@@ -10,6 +10,7 @@ export type WsInbound = {
     type: 'chat';
     sessionId?: string;
     text: string;
+    attachments?: WireAttachment[];
 } | {
     type: 'command';
     sessionId?: string;
@@ -163,6 +164,10 @@ export type WireSchedulerEvent = {
 export type WsOutbound = {
     type: 'commands';
     commands: WireCommand[];
+} | {
+    type: 'capabilities';
+    vision: boolean;
+    audio: boolean;
 } | {
     type: 'agents';
     agents: WireAgent[];

package/esm/channels/ws/protocol.js

@@ -1,5 +1,19 @@
 const APPROVAL_ACTIONS = new Set(['approve', 'approve_always', 'deny']);
 const optionalString = (v) => (typeof v === 'string' && v ? v : undefined);
+function parseAttachments(v) {
+    if (!Array.isArray(v))
+        return undefined;
+    const out = [];
+    for (const item of v) {
+        if (!item || typeof item !== 'object')
+            continue;
+        const a = item;
+        if ((a.kind === 'image' || a.kind === 'audio') && typeof a.mime === 'string' && typeof a.data === 'string') {
+            out.push({ kind: a.kind, mime: a.mime, data: a.data });
+        }
+    }
+    return out.length > 0 ? out : undefined;
+}
 export function parseInbound(raw) {
     if (!raw || typeof raw !== 'object')
         return { error: 'not an object' };
@@ -9,7 +23,8 @@ export function parseInbound(raw) {
         case 'chat': {
             if (typeof o.text !== 'string')
                 return { error: 'chat requires text:string' };
-            return { type: 'chat', sessionId: optionalString(o.sessionId), text: o.text };
+            const attachments = parseAttachments(o.attachments);
+            return { type: 'chat', sessionId: optionalString(o.sessionId), text: o.text, ...(attachments ? { attachments } : {}) };
         }
         case 'command': {
             if (typeof o.text !== 'string')

package/esm/channels/ws/server.d.ts

@@ -6,10 +6,20 @@ export interface WebSocketAdapterOptions {
     authToken?: string;
     activeAgentId?: string;
     listModels?: () => Promise<WireModel[]>;
+    /** Modalities the configured model accepts. Image/audio attachments are dropped when off. */
+    capabilities?: {
+        vision?: boolean;
+        audio?: boolean;
+    };
     maxPayloadBytes?: number;
     log?: (msg: string) => void;
 }
 export type WebSocketAdapter = ChannelAdapter & {
     push(frame: WsOutbound): void;
+    /** Update the advertised model capabilities and broadcast a fresh `capabilities` frame. */
+    setCapabilities(caps: {
+        vision: boolean;
+        audio: boolean;
+    }): void;
 };
 export declare function webSocketAdapter(opts: WebSocketAdapterOptions): WebSocketAdapter;

package/esm/channels/ws/server.js

@@ -1,6 +1,8 @@
 import { WebSocket, WebSocketServer } from 'ws';
+import { text as textPart } from 'mu-core';
 import { createSessionService } from './session-service.js';
 import { observeSubAgent } from './sub-agent.js';
+import { attachmentsToParts } from './wire.js';
 import { emitSessionEvent } from './wire-events.js';
 import { approvalRequestToWire, parseInbound, } from './protocol.js';
 const DEFAULT_MAX_PAYLOAD_BYTES = 1024 * 1024;
@@ -11,9 +13,15 @@ function toWireCommands(commands) {
 export function webSocketAdapter(opts) {
     const log = opts.log ?? (() => { });
     let pushFn = () => { };
+    const caps = { vision: opts.capabilities?.vision === true, audio: opts.capabilities?.audio === true };
     const adapter = {
         name: 'websocket',
         push: (frame) => pushFn(frame),
+        setCapabilities: (next) => {
+            caps.vision = next.vision;
+            caps.audio = next.audio;
+            pushFn({ type: 'capabilities', vision: caps.vision, audio: caps.audio });
+        },
         start: (ctx) => start(ctx),
     };
     return adapter;
@@ -74,6 +82,20 @@ export function webSocketAdapter(opts) {
             push({ type: 'sessions:changed', sessionId, kind });
             push({ type: 'sessions:listed', sessions: await service.list() });
         }
+        // Build the channel payload from text + attachments, dropping any modality the model lacks.
+        function buildChatPayload(sessionId, body, attachments) {
+            if (!attachments || attachments.length === 0)
+                return body;
+            const allowed = attachments.filter((a) => (a.kind === 'image' ? caps.vision : caps.audio));
+            if (allowed.length < attachments.length) {
+                const kinds = [...new Set(attachments.filter((a) => !allowed.includes(a)).map((a) => a.kind))].join('/');
+                push({ type: 'error', sessionId, message: `the active model has no ${kinds} capability — attachment(s) dropped` });
+            }
+            if (allowed.length === 0)
+                return body;
+            const parts = attachmentsToParts(allowed);
+            return body ? [textPart(body), ...parts] : parts;
+        }
         async function dispatch(client, msg) {
             switch (msg.type) {
                 case 'chat': {
@@ -81,7 +103,8 @@ export function webSocketAdapter(opts) {
                     client.sessionId = sessionId;
                     currentApprovalSessionId = sessionId;
                     const channel = manager.get(sessionId) ?? manager.open({ id: sessionId });
-                    void channel.send(msg.text).catch((err) => {
+                    const payload = buildChatPayload(sessionId, msg.text, msg.attachments);
+                    void channel.send(payload).catch((err) => {
                         push({ type: 'error', sessionId, message: err instanceof Error ? err.message : String(err) });
                     });
                     return;
@@ -239,6 +262,7 @@ export function webSocketAdapter(opts) {
             const client = { ws, sessionId };
             clients.set(ws, client);
             send(ws, { type: 'commands', commands: toWireCommands(commands.list()) });
+            send(ws, { type: 'capabilities', vision: caps.vision, audio: caps.audio });
             send(ws, agentsFrame());
             void modelsFrame().then((frame) => frame && send(ws, frame));
             void service.list().then((sessions) => send(ws, { type: 'sessions:listed', sessions }));

package/esm/channels/ws/sub-agent.js

@@ -29,7 +29,7 @@ export function observeSubAgent(session, meta, broadcast) {
                 });
                 return;
             case 'message': {
-                if (ev.message.role !== 'user')
+                if (ev.message.role !== 'tool')
                     return;
                 for (const part of ev.message.content) {
                     if (part.type !== 'tool_result')

package/esm/channels/ws/wire.d.ts

@@ -1,5 +1,13 @@
-import type { ContentPart, Message } from 'mu-core';
+import { type ContentPart, type Message } from 'mu-core';
 export type WireRole = 'user' | 'assistant' | 'system' | 'tool';
+/** A non-text content part carried over the wire as base64. */
+export interface WireAttachment {
+    kind: 'image' | 'audio';
+    mime: string;
+    data: string;
+}
+export declare const partsToAttachments: (parts: readonly ContentPart[]) => WireAttachment[];
+export declare const attachmentsToParts: (attachments: readonly WireAttachment[]) => ContentPart[];
 export interface WireToolCall {
     id: string;
     function: {
@@ -27,6 +35,7 @@ export interface WireMessage {
     toolCalls?: WireToolCall[];
     toolCallId?: string;
     toolResult?: WireToolResultInfo;
+    attachments?: WireAttachment[];
     meta?: WireMessageMeta;
 }
 export declare const textOf: (parts: readonly ContentPart[]) => string;

package/esm/channels/ws/wire.js

@@ -1,3 +1,16 @@
+import { Buffer } from 'node:buffer';
+import { audio, image } from 'mu-core';
+export const partsToAttachments = (parts) => parts.flatMap((part) => part.type === 'image'
+    ? [{ kind: 'image', mime: part.mime, data: Buffer.from(part.data).toString('base64') }]
+    : part.type === 'audio'
+        ? [{ kind: 'audio', mime: part.mime, data: Buffer.from(part.data).toString('base64') }]
+        : part.type === 'tool_result'
+            ? partsToAttachments(part.content)
+            : []);
+export const attachmentsToParts = (attachments) => attachments.map((a) => {
+    const data = new Uint8Array(Buffer.from(a.data, 'base64'));
+    return a.kind === 'image' ? image(a.mime, data) : audio(a.mime, data);
+});
 export const textOf = (parts) => parts.map((part) => (part.type === 'text' ? part.text : '')).join('');
 export const argsToString = (input) => (typeof input === 'string' ? input : JSON.stringify(input ?? {}));
 export const toolResultText = (parts) => parts
@@ -24,7 +37,8 @@ export function messageToWire(message, idBase, ts, toolNames) {
     if (results.length > 0) {
         return results.map((result, index) => {
             const content = toolResultText(result.content);
-            return {
+            const attachments = partsToAttachments(result.content);
+            const out = {
                 id: `${idBase}:t${index}`,
                 ts,
                 role: 'tool',
@@ -32,9 +46,16 @@ export function messageToWire(message, idBase, ts, toolNames) {
                 toolCallId: result.id,
                 toolResult: { name: toolNames.get(result.id) ?? '', content },
             };
+            if (attachments.length > 0)
+                out.attachments = attachments;
+            return out;
         });
     }
-    return [{ id: `${idBase}:u`, ts, role: 'user', content: textOf(message.content) }];
+    const userOut = { id: `${idBase}:u`, ts, role: 'user', content: textOf(message.content) };
+    const userAttachments = partsToAttachments(message.content);
+    if (userAttachments.length > 0)
+        userOut.attachments = userAttachments;
+    return [userOut];
 }
 export function messagesToWire(messages, baseTs = 0) {
     const toolNames = new Map();

package/esm/tui/chat/ChatApp.d.ts

@@ -10,6 +10,10 @@ export interface ChatFeatures {
     subAgents?: boolean;
     sessionPicker?: boolean;
     modelPicker?: boolean;
+    /** The active model accepts image input. Default off — must be opted in. */
+    vision?: boolean;
+    /** The active model accepts audio input. Default off — must be opted in. */
+    audio?: boolean;
 }
 export interface ChatHost {
     session?: AgentSession;
@@ -77,6 +81,8 @@ export declare class ChatApp {
     private readonly pendingShell;
     private readonly pastes;
     private pasteSeq;
+    private readonly attachments;
+    private attachSeq;
     private models;
     private paletteCursor;
     private paletteDismissedFor;
@@ -121,6 +127,12 @@ export declare class ChatApp {
     private send;
     private capturePaste;
     private expandPastes;
+    private capable;
+    private pasteClipboardImage;
+    private attachFromFile;
+    private addAttachment;
+    private stripAttachmentPlaceholders;
+    private clearAttachments;
     private flushShellContext;
     private submit;
     private clearPastes;

package/esm/tui/chat/ChatApp.js

@@ -1,7 +1,8 @@
 import { spawn } from 'node:child_process';
-import { mkdir, writeFile } from 'node:fs/promises';
-import { dirname, isAbsolute, relative, resolve } from 'node:path';
-import { box, column, flex, measure, ProcessTerminal, scrollView, truncateToWidth, TUI, visibleWidth, } from 'mu-tui';
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import { dirname, extname, isAbsolute, relative, resolve } from 'node:path';
+import { audio, image } from 'mu-core';
+import { box, column, flex, measure, ProcessTerminal, readClipboardImage, scrollView, truncateToWidth, TUI, visibleWidth, } from 'mu-tui';
 import { buildCommands, filterCommands } from './commands.js';
 import { MultilineEditor } from './editor.js';
 import { activeMention, collectCandidates, mentionRanges, rank } from './picker.js';
@@ -21,6 +22,23 @@ const APPROVAL_OPTIONS = [
     { label: 'Deny', value: 'deny' },
 ];
 const DIM = '\x1b[2m';
+const IMAGE_MIME = {
+    '.png': 'image/png',
+    '.jpg': 'image/jpeg',
+    '.jpeg': 'image/jpeg',
+    '.gif': 'image/gif',
+    '.webp': 'image/webp',
+    '.bmp': 'image/bmp',
+};
+const AUDIO_MIME = {
+    '.wav': 'audio/wav',
+    '.mp3': 'audio/mpeg',
+    '.m4a': 'audio/mp4',
+    '.ogg': 'audio/ogg',
+    '.flac': 'audio/flac',
+    '.webm': 'audio/webm',
+};
+const ATTACHMENT_PLACEHOLDER = /\[(?:image|audio) #\d+\]/g;
 const lastAssistantText = (messages) => {
     for (let i = messages.length - 1; i >= 0; i--) {
         const message = messages[i];
@@ -90,6 +108,8 @@ export class ChatApp {
     pendingShell = [];
     pastes = new Map();
     pasteSeq = 0;
+    attachments = new Map();
+    attachSeq = 0;
     models = [];
     paletteCursor = 0;
     paletteDismissedFor = '__none__';
@@ -167,6 +187,7 @@ export class ChatApp {
         this.tui.addGlobalKeybinding({ chord: { key: 't', ctrl: true }, handler: () => this.toggleTheme() });
         this.tui.addGlobalKeybinding({ chord: { key: 'o', ctrl: true }, handler: () => this.toggleExpand() });
         this.tui.addGlobalKeybinding({ chord: { key: 'end', ctrl: true }, handler: () => this.jumpToBottom() });
+        this.tui.addGlobalKeybinding({ chord: { key: 'v', ctrl: true }, handler: () => void this.pasteClipboardImage() });
         this.unsubscribeTheme = this.themeProvider.subscribe(() => {
             this.tui.setBackgroundColor(this.theme().colors.background);
             this.tui.setToastBackground(this.theme().colors.surface);
@@ -465,10 +486,13 @@ export class ChatApp {
         const agents = new Set(this.host.agentNames());
         return text.replace(/(^|\s)@(\S+)/g, (match, pre, token) => (agents.has(token) ? match : `${pre}${token}`));
     }
-    send(value) {
+    send(value, attachments = []) {
         const session = this.ensureSession();
         this.transcript.appendUser(value);
-        const content = this.flushShellContext(this.stripFileMentions(value));
+        const text = this.flushShellContext(this.stripFileMentions(this.stripAttachmentPlaceholders(value)));
+        const content = attachments.length > 0
+            ? [...(text ? [{ type: 'text', text }] : []), ...attachments]
+            : text;
         this.running = true;
         this.status.busy = true;
         this.setStatus('thinking…');
@@ -482,6 +506,20 @@ export class ChatApp {
         });
     }
     capturePaste(text) {
+        // Empty bracketed paste: the terminal swallowed binary clipboard data (e.g. an image).
+        if (!text.trim()) {
+            void this.pasteClipboardImage();
+            return '';
+        }
+        // A pasted path to a local image/audio file becomes an attachment, not literal text.
+        const filePath = text.trim().replace(/^['"]|['"]$/g, '').replace(/\\ /g, ' ');
+        if (!filePath.includes('\n')) {
+            const ext = extname(filePath).toLowerCase();
+            if (IMAGE_MIME[ext] || AUDIO_MIME[ext]) {
+                void this.attachFromFile(filePath, ext);
+                return '';
+            }
+        }
         const lines = text.split('\n').length;
         if (lines < 2 && text.length <= 200)
             return undefined;
@@ -498,6 +536,50 @@ export class ChatApp {
         }
         return out;
     }
+    capable(kind) {
+        return (kind === 'image' ? this.features.vision : this.features.audio) === true;
+    }
+    async pasteClipboardImage() {
+        const img = await readClipboardImage().catch(() => undefined);
+        if (!img) {
+            this.setStatus('no image in clipboard');
+            this.tui.requestRender();
+            return;
+        }
+        this.addAttachment('image', img.mime, img.data);
+    }
+    async attachFromFile(path, ext) {
+        const kind = IMAGE_MIME[ext] ? 'image' : 'audio';
+        const mime = IMAGE_MIME[ext] ?? AUDIO_MIME[ext];
+        try {
+            const buf = await readFile(resolve(this.host.cwd, path));
+            this.addAttachment(kind, mime, new Uint8Array(buf));
+        }
+        catch {
+            this.setStatus(`could not read ${path}`);
+            this.tui.requestRender();
+        }
+    }
+    addAttachment(kind, mime, data) {
+        if (!this.capable(kind)) {
+            this.setStatus(`the active model has no ${kind} capability — ${kind} not attached`);
+            this.tui.requestRender();
+            return;
+        }
+        const placeholder = `[${kind} #${++this.attachSeq}]`;
+        this.attachments.set(placeholder, kind === 'image' ? image(mime, data) : audio(mime, data));
+        const value = this.editor.getValue();
+        this.editor.setValue(value ? `${value} ${placeholder} ` : `${placeholder} `);
+        this.onInputChange(this.editor.getValue());
+        this.tui.requestRender();
+    }
+    stripAttachmentPlaceholders(text) {
+        return text.replace(ATTACHMENT_PLACEHOLDER, '').replace(/[ \t]{2,}/g, ' ').trim();
+    }
+    clearAttachments() {
+        this.attachments.clear();
+        this.attachSeq = 0;
+    }
     flushShellContext(userText) {
         if (this.pendingShell.length === 0)
             return userText;
@@ -515,8 +597,10 @@ export class ChatApp {
             return;
         this.clearError();
         const text = this.expandPastes(trimmed);
+        const attachments = [...this.attachments.values()];
         this.editor.setValue('');
         this.clearPastes();
+        this.clearAttachments();
         this.pushHistory(text);
         if (text.startsWith('!') || text.startsWith('$')) {
             this.runShell(text.slice(1).trim());
@@ -529,11 +613,13 @@ export class ChatApp {
         if (this.tryDispatch(text))
             return;
         if (this.running) {
+            if (attachments.length > 0)
+                this.setStatus('attachments are dropped for messages queued mid-turn');
             this.queue.push(text);
             this.tui.requestRender();
             return;
         }
-        this.send(text);
+        this.send(text, attachments);
     }
     clearPastes() {
         this.pastes.clear();
@@ -546,6 +632,7 @@ export class ChatApp {
         const text = this.expandPastes(value);
         this.editor.setValue('');
         this.clearPastes();
+        this.clearAttachments();
         this.pushHistory(text);
         this.queue.push(text);
         this.tui.requestRender();
@@ -555,6 +642,10 @@ export class ChatApp {
             if (!value.includes(placeholder))
                 this.pastes.delete(placeholder);
         }
+        for (const placeholder of this.attachments.keys()) {
+            if (!value.includes(placeholder))
+                this.attachments.delete(placeholder);
+        }
         if (value !== this.paletteDismissedFor)
             this.paletteDismissedFor = '__none__';
         const items = this.paletteItems();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mu-harness",
-  "version": "0.20.0",
+  "version": "0.22.0",
   "description": "Agent harness: createHarness wires mu-core into a host — XDG paths, model registry, plugins, disk-loaded agents & skills, sub-agents, sessions (JSONL + SQLite catalog), slash commands, permission/approval hooks, an optional scheduler, and a composable TUI chat app",
   "license": "MIT",
   "main": "./script/index.js",
@@ -23,8 +23,8 @@
     "@swc/wasm-typescript": "^1.15.0",
     "cli-highlight": "^2.1.11",
     "croner": "^9.0.0",
-    "mu-core": "^0.20.0",
-    "mu-tui": "^0.20.0",
+    "mu-core": "^0.22.0",
+    "mu-tui": "^0.22.0",
     "ws": "^8.18.0"
   },
   "_generatedBy": "dnt@dev",

package/script/channels/ws/client.js

@@ -34,6 +34,9 @@ async function connectHarness(opts) {
     let models = [];
     let modelSelected = '';
     let commands = [];
+    // Mutated in place by the connect-time `capabilities` frame so the host (read once by
+    // ChatApp after connect) reflects the server's actual model modalities.
+    const features = { ...(opts.features ?? {}) };
     const sessionsById = new Map();
     const subSessions = new Map();
     const approvalListeners = new Set();
@@ -61,7 +64,18 @@ async function connectHarness(opts) {
             tools: [],
             send: (input) => {
                 if (!readonly) {
-                    client.send({ type: 'chat', sessionId: id, text: typeof input === 'string' ? input : (0, wire_js_1.textOf)(input) });
+                    if (typeof input === 'string') {
+                        client.send({ type: 'chat', sessionId: id, text: input });
+                    }
+                    else {
+                        const attachments = (0, wire_js_1.partsToAttachments)(input);
+                        client.send({
+                            type: 'chat',
+                            sessionId: id,
+                            text: (0, wire_js_1.textOf)(input),
+                            ...(attachments.length > 0 ? { attachments } : {}),
+                        });
+                    }
                 }
                 return Promise.resolve();
             },
@@ -129,6 +143,10 @@ async function connectHarness(opts) {
             case 'commands':
                 commands = frame.commands;
                 return;
+            case 'capabilities':
+                features.vision = frame.vision;
+                features.audio = frame.audio;
+                return;
             case 'models:listed': {
                 models = frame.models.map((m) => ({ id: m.id, ownedBy: m.ownedBy }));
                 modelSelected = frame.selected;
@@ -284,7 +302,7 @@ async function connectHarness(opts) {
         initialThinking: opts.initialThinking ?? false,
         saveThinking: opts.saveThinking ?? (() => { }),
         history: opts.history,
-        features: opts.features,
+        features,
         banner: opts.banner,
         minimal: opts.minimal,
         commands: () => commands.map((c) => ({ name: c.command.replace(/^\//, ''), description: c.description })),

package/script/channels/ws/index.d.ts

@@ -1,6 +1,6 @@
 export * from './protocol.js';
 export { type SessionSummaryWire, type PersistedSessionWire, type SessionService, createSessionService } from './session-service.js';
-export { type WireMessage, messagesToWire } from './wire.js';
+export { attachmentsToParts, messagesToWire, partsToAttachments, type WireAttachment, type WireMessage } from './wire.js';
 export { type WebSocketAdapter, type WebSocketAdapterOptions, webSocketAdapter } from './server.js';
 export { type ConnectHarnessOptions, connectHarness, type RemoteHarness } from './client.js';
 export { createWsClient, type WsClient, type WsClientOptions } from './ws-client.js';

package/script/channels/ws/index.js

@@ -14,12 +14,14 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.createWsClient = exports.connectHarness = exports.webSocketAdapter = exports.messagesToWire = exports.createSessionService = void 0;
+exports.createWsClient = exports.connectHarness = exports.webSocketAdapter = exports.partsToAttachments = exports.messagesToWire = exports.attachmentsToParts = exports.createSessionService = void 0;
 __exportStar(require("./protocol.js"), exports);
 var session_service_js_1 = require("./session-service.js");
 Object.defineProperty(exports, "createSessionService", { enumerable: true, get: function () { return session_service_js_1.createSessionService; } });
 var wire_js_1 = require("./wire.js");
+Object.defineProperty(exports, "attachmentsToParts", { enumerable: true, get: function () { return wire_js_1.attachmentsToParts; } });
 Object.defineProperty(exports, "messagesToWire", { enumerable: true, get: function () { return wire_js_1.messagesToWire; } });
+Object.defineProperty(exports, "partsToAttachments", { enumerable: true, get: function () { return wire_js_1.partsToAttachments; } });
 var server_js_1 = require("./server.js");
 Object.defineProperty(exports, "webSocketAdapter", { enumerable: true, get: function () { return server_js_1.webSocketAdapter; } });
 var client_js_1 = require("./client.js");

package/script/channels/ws/protocol.d.ts

@@ -1,5 +1,5 @@
 import type { PersistedSessionWire, SessionSummaryWire } from './session-service.js';
-import type { WireMessage } from './wire.js';
+import type { WireAttachment, WireMessage } from './wire.js';
 import type { ApprovalAction, PendingApproval } from '../../permissions/index.js';
 import type { Message, Usage } from 'mu-core';
 export interface WireModel {
@@ -10,6 +10,7 @@ export type WsInbound = {
     type: 'chat';
     sessionId?: string;
     text: string;
+    attachments?: WireAttachment[];
 } | {
     type: 'command';
     sessionId?: string;
@@ -163,6 +164,10 @@ export type WireSchedulerEvent = {
 export type WsOutbound = {
     type: 'commands';
     commands: WireCommand[];
+} | {
+    type: 'capabilities';
+    vision: boolean;
+    audio: boolean;
 } | {
     type: 'agents';
     agents: WireAgent[];

package/script/channels/ws/protocol.js

@@ -4,6 +4,20 @@ exports.parseInbound = parseInbound;
 exports.approvalRequestToWire = approvalRequestToWire;
 const APPROVAL_ACTIONS = new Set(['approve', 'approve_always', 'deny']);
 const optionalString = (v) => (typeof v === 'string' && v ? v : undefined);
+function parseAttachments(v) {
+    if (!Array.isArray(v))
+        return undefined;
+    const out = [];
+    for (const item of v) {
+        if (!item || typeof item !== 'object')
+            continue;
+        const a = item;
+        if ((a.kind === 'image' || a.kind === 'audio') && typeof a.mime === 'string' && typeof a.data === 'string') {
+            out.push({ kind: a.kind, mime: a.mime, data: a.data });
+        }
+    }
+    return out.length > 0 ? out : undefined;
+}
 function parseInbound(raw) {
     if (!raw || typeof raw !== 'object')
         return { error: 'not an object' };
@@ -13,7 +27,8 @@ function parseInbound(raw) {
         case 'chat': {
             if (typeof o.text !== 'string')
                 return { error: 'chat requires text:string' };
-            return { type: 'chat', sessionId: optionalString(o.sessionId), text: o.text };
+            const attachments = parseAttachments(o.attachments);
+            return { type: 'chat', sessionId: optionalString(o.sessionId), text: o.text, ...(attachments ? { attachments } : {}) };
         }
         case 'command': {
             if (typeof o.text !== 'string')

package/script/channels/ws/server.d.ts

@@ -6,10 +6,20 @@ export interface WebSocketAdapterOptions {
     authToken?: string;
     activeAgentId?: string;
     listModels?: () => Promise<WireModel[]>;
+    /** Modalities the configured model accepts. Image/audio attachments are dropped when off. */
+    capabilities?: {
+        vision?: boolean;
+        audio?: boolean;
+    };
     maxPayloadBytes?: number;
     log?: (msg: string) => void;
 }
 export type WebSocketAdapter = ChannelAdapter & {
     push(frame: WsOutbound): void;
+    /** Update the advertised model capabilities and broadcast a fresh `capabilities` frame. */
+    setCapabilities(caps: {
+        vision: boolean;
+        audio: boolean;
+    }): void;
 };
 export declare function webSocketAdapter(opts: WebSocketAdapterOptions): WebSocketAdapter;

package/script/channels/ws/server.js

@@ -2,8 +2,10 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.webSocketAdapter = webSocketAdapter;
 const ws_1 = require("ws");
+const mu_core_1 = require("mu-core");
 const session_service_js_1 = require("./session-service.js");
 const sub_agent_js_1 = require("./sub-agent.js");
+const wire_js_1 = require("./wire.js");
 const wire_events_js_1 = require("./wire-events.js");
 const protocol_js_1 = require("./protocol.js");
 const DEFAULT_MAX_PAYLOAD_BYTES = 1024 * 1024;
@@ -14,9 +16,15 @@ function toWireCommands(commands) {
 function webSocketAdapter(opts) {
     const log = opts.log ?? (() => { });
     let pushFn = () => { };
+    const caps = { vision: opts.capabilities?.vision === true, audio: opts.capabilities?.audio === true };
     const adapter = {
         name: 'websocket',
         push: (frame) => pushFn(frame),
+        setCapabilities: (next) => {
+            caps.vision = next.vision;
+            caps.audio = next.audio;
+            pushFn({ type: 'capabilities', vision: caps.vision, audio: caps.audio });
+        },
         start: (ctx) => start(ctx),
     };
     return adapter;
@@ -77,6 +85,20 @@ function webSocketAdapter(opts) {
             push({ type: 'sessions:changed', sessionId, kind });
             push({ type: 'sessions:listed', sessions: await service.list() });
         }
+        // Build the channel payload from text + attachments, dropping any modality the model lacks.
+        function buildChatPayload(sessionId, body, attachments) {
+            if (!attachments || attachments.length === 0)
+                return body;
+            const allowed = attachments.filter((a) => (a.kind === 'image' ? caps.vision : caps.audio));
+            if (allowed.length < attachments.length) {
+                const kinds = [...new Set(attachments.filter((a) => !allowed.includes(a)).map((a) => a.kind))].join('/');
+                push({ type: 'error', sessionId, message: `the active model has no ${kinds} capability — attachment(s) dropped` });
+            }
+            if (allowed.length === 0)
+                return body;
+            const parts = (0, wire_js_1.attachmentsToParts)(allowed);
+            return body ? [(0, mu_core_1.text)(body), ...parts] : parts;
+        }
         async function dispatch(client, msg) {
             switch (msg.type) {
                 case 'chat': {
@@ -84,7 +106,8 @@ function webSocketAdapter(opts) {
                     client.sessionId = sessionId;
                     currentApprovalSessionId = sessionId;
                     const channel = manager.get(sessionId) ?? manager.open({ id: sessionId });
-                    void channel.send(msg.text).catch((err) => {
+                    const payload = buildChatPayload(sessionId, msg.text, msg.attachments);
+                    void channel.send(payload).catch((err) => {
                         push({ type: 'error', sessionId, message: err instanceof Error ? err.message : String(err) });
                     });
                     return;
@@ -242,6 +265,7 @@ function webSocketAdapter(opts) {
             const client = { ws, sessionId };
             clients.set(ws, client);
             send(ws, { type: 'commands', commands: toWireCommands(commands.list()) });
+            send(ws, { type: 'capabilities', vision: caps.vision, audio: caps.audio });
             send(ws, agentsFrame());
             void modelsFrame().then((frame) => frame && send(ws, frame));
             void service.list().then((sessions) => send(ws, { type: 'sessions:listed', sessions }));

package/script/channels/ws/sub-agent.js

@@ -32,7 +32,7 @@ function observeSubAgent(session, meta, broadcast) {
                 });
                 return;
             case 'message': {
-                if (ev.message.role !== 'user')
+                if (ev.message.role !== 'tool')
                     return;
                 for (const part of ev.message.content) {
                     if (part.type !== 'tool_result')

package/script/channels/ws/wire.d.ts

@@ -1,5 +1,13 @@
-import type { ContentPart, Message } from 'mu-core';
+import { type ContentPart, type Message } from 'mu-core';
 export type WireRole = 'user' | 'assistant' | 'system' | 'tool';
+/** A non-text content part carried over the wire as base64. */
+export interface WireAttachment {
+    kind: 'image' | 'audio';
+    mime: string;
+    data: string;
+}
+export declare const partsToAttachments: (parts: readonly ContentPart[]) => WireAttachment[];
+export declare const attachmentsToParts: (attachments: readonly WireAttachment[]) => ContentPart[];
 export interface WireToolCall {
     id: string;
     function: {
@@ -27,6 +35,7 @@ export interface WireMessage {
     toolCalls?: WireToolCall[];
     toolCallId?: string;
     toolResult?: WireToolResultInfo;
+    attachments?: WireAttachment[];
     meta?: WireMessageMeta;
 }
 export declare const textOf: (parts: readonly ContentPart[]) => string;

package/script/channels/ws/wire.js

@@ -1,8 +1,23 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.toolResultText = exports.argsToString = exports.textOf = void 0;
+exports.toolResultText = exports.argsToString = exports.textOf = exports.attachmentsToParts = exports.partsToAttachments = void 0;
 exports.messageToWire = messageToWire;
 exports.messagesToWire = messagesToWire;
+const node_buffer_1 = require("node:buffer");
+const mu_core_1 = require("mu-core");
+const partsToAttachments = (parts) => parts.flatMap((part) => part.type === 'image'
+    ? [{ kind: 'image', mime: part.mime, data: node_buffer_1.Buffer.from(part.data).toString('base64') }]
+    : part.type === 'audio'
+        ? [{ kind: 'audio', mime: part.mime, data: node_buffer_1.Buffer.from(part.data).toString('base64') }]
+        : part.type === 'tool_result'
+            ? (0, exports.partsToAttachments)(part.content)
+            : []);
+exports.partsToAttachments = partsToAttachments;
+const attachmentsToParts = (attachments) => attachments.map((a) => {
+    const data = new Uint8Array(node_buffer_1.Buffer.from(a.data, 'base64'));
+    return a.kind === 'image' ? (0, mu_core_1.image)(a.mime, data) : (0, mu_core_1.audio)(a.mime, data);
+});
+exports.attachmentsToParts = attachmentsToParts;
 const textOf = (parts) => parts.map((part) => (part.type === 'text' ? part.text : '')).join('');
 exports.textOf = textOf;
 const argsToString = (input) => (typeof input === 'string' ? input : JSON.stringify(input ?? {}));
@@ -32,7 +47,8 @@ function messageToWire(message, idBase, ts, toolNames) {
     if (results.length > 0) {
         return results.map((result, index) => {
             const content = (0, exports.toolResultText)(result.content);
-            return {
+            const attachments = (0, exports.partsToAttachments)(result.content);
+            const out = {
                 id: `${idBase}:t${index}`,
                 ts,
                 role: 'tool',
@@ -40,9 +56,16 @@ function messageToWire(message, idBase, ts, toolNames) {
                 toolCallId: result.id,
                 toolResult: { name: toolNames.get(result.id) ?? '', content },
             };
+            if (attachments.length > 0)
+                out.attachments = attachments;
+            return out;
         });
     }
-    return [{ id: `${idBase}:u`, ts, role: 'user', content: (0, exports.textOf)(message.content) }];
+    const userOut = { id: `${idBase}:u`, ts, role: 'user', content: (0, exports.textOf)(message.content) };
+    const userAttachments = (0, exports.partsToAttachments)(message.content);
+    if (userAttachments.length > 0)
+        userOut.attachments = userAttachments;
+    return [userOut];
 }
 function messagesToWire(messages, baseTs = 0) {
     const toolNames = new Map();

package/script/tui/chat/ChatApp.d.ts

@@ -10,6 +10,10 @@ export interface ChatFeatures {
     subAgents?: boolean;
     sessionPicker?: boolean;
     modelPicker?: boolean;
+    /** The active model accepts image input. Default off — must be opted in. */
+    vision?: boolean;
+    /** The active model accepts audio input. Default off — must be opted in. */
+    audio?: boolean;
 }
 export interface ChatHost {
     session?: AgentSession;
@@ -77,6 +81,8 @@ export declare class ChatApp {
     private readonly pendingShell;
     private readonly pastes;
     private pasteSeq;
+    private readonly attachments;
+    private attachSeq;
     private models;
     private paletteCursor;
     private paletteDismissedFor;
@@ -121,6 +127,12 @@ export declare class ChatApp {
     private send;
     private capturePaste;
     private expandPastes;
+    private capable;
+    private pasteClipboardImage;
+    private attachFromFile;
+    private addAttachment;
+    private stripAttachmentPlaceholders;
+    private clearAttachments;
     private flushShellContext;
     private submit;
     private clearPastes;

package/script/tui/chat/ChatApp.js

@@ -4,6 +4,7 @@ exports.ChatApp = void 0;
 const node_child_process_1 = require("node:child_process");
 const promises_1 = require("node:fs/promises");
 const node_path_1 = require("node:path");
+const mu_core_1 = require("mu-core");
 const mu_tui_1 = require("mu-tui");
 const commands_js_1 = require("./commands.js");
 const editor_js_1 = require("./editor.js");
@@ -24,6 +25,23 @@ const APPROVAL_OPTIONS = [
     { label: 'Deny', value: 'deny' },
 ];
 const DIM = '\x1b[2m';
+const IMAGE_MIME = {
+    '.png': 'image/png',
+    '.jpg': 'image/jpeg',
+    '.jpeg': 'image/jpeg',
+    '.gif': 'image/gif',
+    '.webp': 'image/webp',
+    '.bmp': 'image/bmp',
+};
+const AUDIO_MIME = {
+    '.wav': 'audio/wav',
+    '.mp3': 'audio/mpeg',
+    '.m4a': 'audio/mp4',
+    '.ogg': 'audio/ogg',
+    '.flac': 'audio/flac',
+    '.webm': 'audio/webm',
+};
+const ATTACHMENT_PLACEHOLDER = /\[(?:image|audio) #\d+\]/g;
 const lastAssistantText = (messages) => {
     for (let i = messages.length - 1; i >= 0; i--) {
         const message = messages[i];
@@ -93,6 +111,8 @@ class ChatApp {
     pendingShell = [];
     pastes = new Map();
     pasteSeq = 0;
+    attachments = new Map();
+    attachSeq = 0;
     models = [];
     paletteCursor = 0;
     paletteDismissedFor = '__none__';
@@ -170,6 +190,7 @@ class ChatApp {
         this.tui.addGlobalKeybinding({ chord: { key: 't', ctrl: true }, handler: () => this.toggleTheme() });
         this.tui.addGlobalKeybinding({ chord: { key: 'o', ctrl: true }, handler: () => this.toggleExpand() });
         this.tui.addGlobalKeybinding({ chord: { key: 'end', ctrl: true }, handler: () => this.jumpToBottom() });
+        this.tui.addGlobalKeybinding({ chord: { key: 'v', ctrl: true }, handler: () => void this.pasteClipboardImage() });
         this.unsubscribeTheme = this.themeProvider.subscribe(() => {
             this.tui.setBackgroundColor(this.theme().colors.background);
             this.tui.setToastBackground(this.theme().colors.surface);
@@ -468,10 +489,13 @@ class ChatApp {
         const agents = new Set(this.host.agentNames());
         return text.replace(/(^|\s)@(\S+)/g, (match, pre, token) => (agents.has(token) ? match : `${pre}${token}`));
     }
-    send(value) {
+    send(value, attachments = []) {
         const session = this.ensureSession();
         this.transcript.appendUser(value);
-        const content = this.flushShellContext(this.stripFileMentions(value));
+        const text = this.flushShellContext(this.stripFileMentions(this.stripAttachmentPlaceholders(value)));
+        const content = attachments.length > 0
+            ? [...(text ? [{ type: 'text', text }] : []), ...attachments]
+            : text;
         this.running = true;
         this.status.busy = true;
         this.setStatus('thinking…');
@@ -485,6 +509,20 @@ class ChatApp {
         });
     }
     capturePaste(text) {
+        // Empty bracketed paste: the terminal swallowed binary clipboard data (e.g. an image).
+        if (!text.trim()) {
+            void this.pasteClipboardImage();
+            return '';
+        }
+        // A pasted path to a local image/audio file becomes an attachment, not literal text.
+        const filePath = text.trim().replace(/^['"]|['"]$/g, '').replace(/\\ /g, ' ');
+        if (!filePath.includes('\n')) {
+            const ext = (0, node_path_1.extname)(filePath).toLowerCase();
+            if (IMAGE_MIME[ext] || AUDIO_MIME[ext]) {
+                void this.attachFromFile(filePath, ext);
+                return '';
+            }
+        }
         const lines = text.split('\n').length;
         if (lines < 2 && text.length <= 200)
             return undefined;
@@ -501,6 +539,50 @@ class ChatApp {
         }
         return out;
     }
+    capable(kind) {
+        return (kind === 'image' ? this.features.vision : this.features.audio) === true;
+    }
+    async pasteClipboardImage() {
+        const img = await (0, mu_tui_1.readClipboardImage)().catch(() => undefined);
+        if (!img) {
+            this.setStatus('no image in clipboard');
+            this.tui.requestRender();
+            return;
+        }
+        this.addAttachment('image', img.mime, img.data);
+    }
+    async attachFromFile(path, ext) {
+        const kind = IMAGE_MIME[ext] ? 'image' : 'audio';
+        const mime = IMAGE_MIME[ext] ?? AUDIO_MIME[ext];
+        try {
+            const buf = await (0, promises_1.readFile)((0, node_path_1.resolve)(this.host.cwd, path));
+            this.addAttachment(kind, mime, new Uint8Array(buf));
+        }
+        catch {
+            this.setStatus(`could not read ${path}`);
+            this.tui.requestRender();
+        }
+    }
+    addAttachment(kind, mime, data) {
+        if (!this.capable(kind)) {
+            this.setStatus(`the active model has no ${kind} capability — ${kind} not attached`);
+            this.tui.requestRender();
+            return;
+        }
+        const placeholder = `[${kind} #${++this.attachSeq}]`;
+        this.attachments.set(placeholder, kind === 'image' ? (0, mu_core_1.image)(mime, data) : (0, mu_core_1.audio)(mime, data));
+        const value = this.editor.getValue();
+        this.editor.setValue(value ? `${value} ${placeholder} ` : `${placeholder} `);
+        this.onInputChange(this.editor.getValue());
+        this.tui.requestRender();
+    }
+    stripAttachmentPlaceholders(text) {
+        return text.replace(ATTACHMENT_PLACEHOLDER, '').replace(/[ \t]{2,}/g, ' ').trim();
+    }
+    clearAttachments() {
+        this.attachments.clear();
+        this.attachSeq = 0;
+    }
     flushShellContext(userText) {
         if (this.pendingShell.length === 0)
             return userText;
@@ -518,8 +600,10 @@ class ChatApp {
             return;
         this.clearError();
         const text = this.expandPastes(trimmed);
+        const attachments = [...this.attachments.values()];
         this.editor.setValue('');
         this.clearPastes();
+        this.clearAttachments();
         this.pushHistory(text);
         if (text.startsWith('!') || text.startsWith('$')) {
             this.runShell(text.slice(1).trim());
@@ -532,11 +616,13 @@ class ChatApp {
         if (this.tryDispatch(text))
             return;
         if (this.running) {
+            if (attachments.length > 0)
+                this.setStatus('attachments are dropped for messages queued mid-turn');
             this.queue.push(text);
             this.tui.requestRender();
             return;
         }
-        this.send(text);
+        this.send(text, attachments);
     }
     clearPastes() {
         this.pastes.clear();
@@ -549,6 +635,7 @@ class ChatApp {
         const text = this.expandPastes(value);
         this.editor.setValue('');
         this.clearPastes();
+        this.clearAttachments();
         this.pushHistory(text);
         this.queue.push(text);
         this.tui.requestRender();
@@ -558,6 +645,10 @@ class ChatApp {
             if (!value.includes(placeholder))
                 this.pastes.delete(placeholder);
         }
+        for (const placeholder of this.attachments.keys()) {
+            if (!value.includes(placeholder))
+                this.attachments.delete(placeholder);
+        }
         if (value !== this.paletteDismissedFor)
             this.paletteDismissedFor = '__none__';
         const items = this.paletteItems();