@mutirolabs/openclaw-brain 0.1.0

package/src/inbound.ts

@@ -0,0 +1,151 @@
+// Translates bridge observed messages into the inbound delivery the OpenClaw
+// gateway expects from a channel plugin. The runtime installs a single inbound
+// delivery callback during plugin startup; this module's job is to shape each
+// observed bridge envelope into that callback's input.
+
+import { saveMediaBuffer } from "openclaw/plugin-sdk/browser-setup-tools";
+
+import type { ObservedTurn } from "./bridge-protocol.js";
+import { buildObservedTurn, isSelfEventMessage } from "./bridge-messages.js";
+
+export type InboundMessage = {
+  channelId: "mutiro";
+  accountId: string;
+  conversationId: string;
+  messageId: string;
+  replyToMessageId?: string;
+  senderUsername: string;
+  text: string;
+  rawMessage: unknown;
+  attachmentContext?: string;
+  mediaPaths?: string[];
+  mediaTypes?: string[];
+};
+
+export type InboundDeliver = (message: InboundMessage) => Promise<void> | void;
+
+export type InboundRoute = {
+  accountId: string;
+  agentUsername: string;
+  deliver: InboundDeliver;
+};
+
+type BridgeImage = {
+  data?: string;
+  mime_type?: string;
+  filename?: string;
+};
+
+/**
+ * The bridge delivers image attachments inline as base64 under
+ * `envelope.payload.images`. We persist each one into OpenClaw's media
+ * directory via `saveMediaBuffer`, which stages under `~/.openclaw/media/`
+ * — the only root OpenClaw's sandbox staging policy accepts. Writing to
+ * `os.tmpdir()` silently fails staging and leaves the agent with only a
+ * text path reference, so the bytes never reach the model.
+ */
+const persistInlineImages = async (
+  images: BridgeImage[] | undefined,
+): Promise<{ paths: string[]; types: string[] }> => {
+  if (!Array.isArray(images) || images.length === 0) {
+    return { paths: [], types: [] };
+  }
+
+  const paths: string[] = [];
+  const types: string[] = [];
+  for (const entry of images) {
+    if (!entry?.data) continue;
+    try {
+      const saved = await saveMediaBuffer(
+        Buffer.from(entry.data, "base64"),
+        entry.mime_type,
+        "inbound",
+        undefined,
+        entry.filename,
+      );
+      paths.push(saved.path);
+      types.push(saved.contentType ?? entry.mime_type ?? "application/octet-stream");
+    } catch {
+      // Best-effort: skip a single bad attachment rather than aborting the turn.
+    }
+  }
+  return { paths, types };
+};
+
+/**
+ * Shapes a bridge envelope into the OpenClaw inbound message the gateway
+ * delivers to core. Returns the observed turn it extracted (useful for the
+ * caller to key session state) or null when the envelope was self-authored or
+ * did not carry a deliverable message.
+ */
+export const deliverObservedEnvelope = async (
+  envelope: {
+    type?: string;
+    payload?: {
+      message?: unknown;
+      attachment_context?: string;
+      images?: BridgeImage[];
+    };
+  } & Parameters<typeof buildObservedTurn>[0],
+  route: InboundRoute,
+): Promise<ObservedTurn | null> => {
+  if (envelope.type === "event.message" && isSelfEventMessage(envelope, route.agentUsername)) {
+    return null;
+  }
+
+  // Reactions and other bare events are now delivered to OpenClaw like any
+  // other observation. The agent may choose to stay silent, which today
+  // surfaces "Agent couldn't generate a response" — acceptable trade-off
+  // for letting the agent actually see reactions happen. If OpenClaw grows
+  // a "silent turn is ok" dispatch option later, filter these through that.
+  const turn = buildObservedTurn(envelope);
+  if (!turn) {
+    return null;
+  }
+
+  const { paths, types } = await persistInlineImages(envelope.payload?.images);
+
+  // attachment_context is the host's narrative of downloaded attachments:
+  //   [SYSTEM: Downloaded 1 file(s) to your workspace:
+  //    • nf-2976.pdf → /Users/.../Downloads/nf-2976.pdf (PDF, 450 KB)]
+  // For PDFs/docs this is the ONLY pointer the agent gets — it needs the
+  // path to invoke `read` or a doc-extraction tool. We earlier stripped
+  // this string because the host's image probe produced noisy "0x0 pixels"
+  // metadata that fooled vision models. Image turns now route bytes
+  // through MediaPaths instead, so attachment_context is pure signal for
+  // non-image attachments. Append only when the message carries file-type
+  // parts; skip for image-only or text-only turns.
+  const hasFileParts = extractHasFileParts(
+    envelope.payload?.message as { parts?: Array<{ type?: string }> } | undefined,
+  );
+  const contextText = (envelope.payload?.attachment_context ?? "").trim();
+  const bodyText =
+    hasFileParts && contextText
+      ? turn.text
+        ? `${turn.text}\n\n${contextText}`
+        : contextText
+      : turn.text;
+
+  await route.deliver({
+    channelId: "mutiro",
+    accountId: route.accountId,
+    conversationId: turn.conversationId,
+    messageId: turn.messageId,
+    replyToMessageId: turn.replyToMessageId,
+    senderUsername: turn.senderUsername,
+    text: bodyText,
+    rawMessage: envelope.payload?.message,
+    attachmentContext: envelope.payload?.attachment_context,
+    ...(paths.length > 0 ? { mediaPaths: paths, mediaTypes: types } : {}),
+  });
+
+  return turn;
+};
+
+const extractHasFileParts = (
+  message: { parts?: Array<{ type?: string }> } | undefined,
+): boolean => {
+  if (!message) return false;
+  const parts = Array.isArray(message.parts) ? message.parts : [];
+  return parts.some((part) => part?.type === "file");
+};

package/src/live-snapshot.ts

@@ -0,0 +1,210 @@
+// Builds the live-handoff snapshot the host requests when a voice call
+// starts. This is the brain's chance to hand OpenClaw's agent persona,
+// the real session transcript, and channel-owned tool hints to Mutiro's
+// live voice model so the call doesn't start with a generic LLM.
+//
+// The three fields returned map 1:1 to ChatBridgeSessionSnapshotResult:
+//   system_instruction   ← agent's systemPromptOverride (or a minimal header)
+//   recent_messages      ← recent turns parsed from OpenClaw's session jsonl
+//   tool_hints           ← channel-owned agent tools advertised by the plugin
+
+import { promises as fs } from "node:fs";
+import * as path from "node:path";
+
+import type { OpenClawConfig } from "openclaw/plugin-sdk/core";
+import { resolveAgentRoute } from "openclaw/plugin-sdk/routing";
+import { resolveStorePath } from "openclaw/plugin-sdk/session-store-runtime";
+
+import type { LiveSnapshot, LiveToolHint } from "./bridge-session.js";
+
+const MAX_RECENT_TRANSCRIPT_TURNS = 30;
+
+type AgentConfigLike = {
+  name?: string;
+  systemPromptOverride?: string;
+};
+
+type SessionEntryLike = {
+  sessionId?: string;
+  sessionFile?: string;
+};
+
+type SessionStoreRecord = Record<string, SessionEntryLike>;
+
+const resolveAgentConfig = (
+  cfg: OpenClawConfig,
+  agentId: string,
+): AgentConfigLike | undefined => {
+  const agents = (cfg as { agents?: { agents?: Record<string, AgentConfigLike> } }).agents;
+  return agents?.agents?.[agentId];
+};
+
+const buildSystemInstruction = (params: {
+  agentId: string;
+  agentConfig: AgentConfigLike | undefined;
+  agentUsername: string;
+  callerUsername?: string;
+}): string => {
+  const override = params.agentConfig?.systemPromptOverride?.trim();
+  if (override) {
+    return override;
+  }
+  const displayName = params.agentConfig?.name?.trim() || params.agentId;
+  const callerSuffix = params.callerUsername ? ` Caller: @${params.callerUsername}.` : "";
+  return [
+    `You are ${displayName}, the same agent this user speaks with in chat.`,
+    "You are now speaking live over voice.",
+    "Stay concise, conversational, and in the agent's established persona.",
+    `Agent identity: @${params.agentUsername}.${callerSuffix}`,
+  ].join(" ");
+};
+
+type JsonlLine = {
+  type?: string;
+  message?: {
+    role?: string;
+    content?: Array<{ type?: string; text?: string; thinking?: string }>;
+  };
+};
+
+type JsonlContent = NonNullable<NonNullable<JsonlLine["message"]>["content"]>;
+
+const extractTextFromContent = (content: JsonlContent | undefined): string => {
+  if (!Array.isArray(content)) return "";
+  const parts: string[] = [];
+  for (const entry of content) {
+    if (!entry || typeof entry !== "object") continue;
+    if (entry.type === "text" && typeof entry.text === "string") {
+      parts.push(entry.text);
+    }
+  }
+  return parts.join("").trim();
+};
+
+const readRecentTranscriptTurns = async (
+  sessionFile: string,
+  conversationId: string,
+  agentUsername: string,
+): Promise<unknown[]> => {
+  const raw = await fs.readFile(sessionFile, "utf8");
+  const lines = raw.split(/\r?\n/);
+  const turns: unknown[] = [];
+  for (const line of lines) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    let parsed: JsonlLine;
+    try {
+      parsed = JSON.parse(trimmed) as JsonlLine;
+    } catch {
+      continue;
+    }
+    if (parsed.type !== "message") continue;
+    const role = parsed.message?.role;
+    if (role !== "user" && role !== "assistant") continue;
+    const text = extractTextFromContent(parsed.message?.content);
+    if (!text) continue;
+    const fromUsername = role === "assistant" ? agentUsername : "user";
+    turns.push({
+      id: `transcript-${turns.length}`,
+      conversation_id: conversationId,
+      from: { username: fromUsername },
+      text,
+    });
+  }
+  return turns.slice(-MAX_RECENT_TRANSCRIPT_TURNS);
+};
+
+const readRecentMessagesFromStore = async (params: {
+  cfg: OpenClawConfig;
+  agentId: string;
+  sessionKey: string;
+  conversationId: string;
+  agentUsername: string;
+}): Promise<unknown[] | undefined> => {
+  try {
+    const storePath = resolveStorePath(params.cfg.session?.store, { agentId: params.agentId });
+    const { loadSessionStore } = await import("openclaw/plugin-sdk/config-runtime");
+    const store = loadSessionStore(storePath) as SessionStoreRecord;
+    const entry = store[params.sessionKey];
+    if (!entry?.sessionId) return undefined;
+    const sessionFile =
+      entry.sessionFile && entry.sessionFile.trim()
+        ? entry.sessionFile
+        : path.join(path.dirname(storePath), `${entry.sessionId}.jsonl`);
+    return await readRecentTranscriptTurns(sessionFile, params.conversationId, params.agentUsername);
+  } catch {
+    return undefined;
+  }
+};
+
+const buildToolHints = async (): Promise<LiveToolHint[]> => {
+  // Lazy import so the light startup path stays clean.
+  const { mutiroAgentTools } = await import("./agent-tools.js");
+  return mutiroAgentTools().map((tool) => {
+    const rawDescription = tool.description;
+    const description = typeof rawDescription === "string" ? rawDescription.trim() : "";
+    return {
+      name: tool.name,
+      description,
+      metadata: {},
+    };
+  });
+};
+
+export type LiveSnapshotContext = {
+  cfg: OpenClawConfig;
+  accountId: string;
+  conversationId: string;
+  callerUsername?: string;
+  callId?: string;
+  agentUsername: string;
+};
+
+/**
+ * Assemble everything the live voice model needs to speak as OpenClaw's
+ * agent. Safe to call even when config/session state is incomplete: each
+ * section degrades independently (missing agent config → minimal header,
+ * missing transcript → undefined recent_messages, tool enumeration always
+ * succeeds).
+ */
+export const buildLiveSnapshot = async (
+  ctx: LiveSnapshotContext,
+): Promise<LiveSnapshot> => {
+  const route = resolveAgentRoute({
+    cfg: ctx.cfg,
+    channel: "mutiro",
+    accountId: ctx.accountId,
+    peer: { kind: "direct", id: ctx.callerUsername ?? "unknown" },
+  });
+  const agentConfig = resolveAgentConfig(ctx.cfg, route.agentId);
+
+  const systemInstruction = buildSystemInstruction({
+    agentId: route.agentId,
+    agentConfig,
+    agentUsername: ctx.agentUsername,
+    callerUsername: ctx.callerUsername,
+  });
+
+  const recentMessages = await readRecentMessagesFromStore({
+    cfg: ctx.cfg,
+    agentId: route.agentId,
+    sessionKey: route.sessionKey,
+    conversationId: ctx.conversationId,
+    agentUsername: ctx.agentUsername,
+  });
+
+  const toolHints = await buildToolHints();
+
+  const metadata: Record<string, string> = {
+    agent_id: route.agentId,
+    session_key: route.sessionKey,
+  };
+  if (ctx.callId) metadata.call_id = ctx.callId;
+
+  return {
+    systemInstruction,
+    ...(recentMessages ? { recentMessages } : {}),
+    toolHints,
+    metadata,
+  };
+};

package/src/outbound.ts

@@ -0,0 +1,326 @@
+// Outbound adapter that translates OpenClaw reply-dispatch calls into
+// bridge-local commands. Mirrors pi-brain's tool surface (send_message,
+// send_voice_message, send_card, react_to_message, send_file_message,
+// forward_message, recall, recall_get) but reshaped so OpenClaw's
+// ChannelOutboundAdapter is the consumer instead of a Pi tool runtime.
+
+import * as path from "node:path";
+
+import type { BridgeClient } from "./bridge-client.js";
+import { applyVoiceLanguage, normalizeOutputText } from "./bridge-messages.js";
+import { TYPE_URLS } from "./bridge-protocol.js";
+
+export type MutiroOutboundTarget = {
+  conversationId: string;
+  replyToMessageId: string;
+};
+
+export type MutiroOutbound = {
+  sendText: (
+    target: MutiroOutboundTarget,
+    text: string,
+  ) => Promise<unknown>;
+  sendVoice: (
+    target: MutiroOutboundTarget,
+    params: { toUsername: string; speech: string; language?: string },
+  ) => Promise<unknown>;
+  sendCard: (
+    target: MutiroOutboundTarget,
+    params: { components: unknown[]; data?: Record<string, unknown>; cardId?: string },
+  ) => Promise<unknown>;
+  sendCardJsonl: (
+    target: MutiroOutboundTarget,
+    params: { jsonl: string; version?: string; cardId?: string },
+  ) => Promise<unknown>;
+  sendFile: (
+    target: MutiroOutboundTarget,
+    params: { filePath: string; caption?: string },
+  ) => Promise<unknown>;
+  react: (params: { messageId: string; emoji: string }) => Promise<unknown>;
+  forward: (params: {
+    messageId: string;
+    // Destination is a `oneof` in ForwardMessageRequest: exactly one of
+    // `targetConversationId` or `toUsername` must be set.
+    targetConversationId?: string;
+    toUsername?: string;
+    comment?: string;
+  }) => Promise<unknown>;
+  recallSearch: (params: {
+    query: string;
+    conversationId?: string;
+    maxResults?: number;
+  }) => Promise<unknown>;
+  recallGet: (params: { entryId: string; conversationId?: string }) => Promise<unknown>;
+  endTurn: (target: MutiroOutboundTarget) => void;
+  emitSignal: (
+    target: MutiroOutboundTarget,
+    signalType: string,
+    detailText?: string,
+  ) => void;
+};
+
+const DEFAULT_VOICE = "en-US-Chirp3-HD-Orus";
+
+const buildCardJson = (
+  components: Array<Record<string, unknown>>,
+  data?: Record<string, unknown>,
+  cardId?: string,
+) => {
+  let rootId = (components[0] as { id?: string } | undefined)?.id || "root";
+  for (const component of components) {
+    const c = component as { parentId?: string; parent_id?: string; id?: string };
+    if (!c.parentId && !c.parent_id) {
+      rootId = c.id || rootId;
+      break;
+    }
+  }
+
+  const lines: string[] = [
+    JSON.stringify({
+      surfaceUpdate: {
+        surfaceId: "main",
+        components,
+        clearBefore: true,
+      },
+    }),
+  ];
+
+  if (data) {
+    const contents = Object.keys(data).map((key) => ({
+      key,
+      valueString:
+        typeof data[key] === "object" ? JSON.stringify(data[key]) : String(data[key]),
+    }));
+    lines.push(
+      JSON.stringify({
+        dataModelUpdate: {
+          surfaceId: "main",
+          contents,
+        },
+      }),
+    );
+  }
+
+  lines.push(
+    JSON.stringify({
+      beginRendering: {
+        surfaceId: "main",
+        root: rootId,
+      },
+    }),
+  );
+
+  return {
+    // Field names must match Mutiro's CardPart protobuf schema (see
+    // spec/protobuf/shared/messaging.proto). pi-brain uses stale names
+    // (`json_data` / `version`) which the host's strict JSON-to-proto
+    // decoder rejects as unknown fields.
+    a2ui_json: lines.join("\n"),
+    schema_version: "0.8",
+    card_id: cardId || `openclaw-card-${Math.random().toString(36).slice(2, 10)}`,
+  };
+};
+
+export const createMutiroOutbound = (bridge: BridgeClient): MutiroOutbound => {
+  const extras = (target: MutiroOutboundTarget) => ({
+    conversation_id: target.conversationId,
+    reply_to_message_id: target.replyToMessageId,
+  });
+
+  return {
+    sendText: async (target, text) => {
+      const normalized = normalizeOutputText(text);
+      if (!normalized) return { ok: false, reason: "noop" };
+      return bridge.request(
+        "message.send",
+        {
+          "@type": TYPE_URLS.bridgeSendMessageCommand,
+          conversation_id: target.conversationId,
+          reply_to_message_id: target.replyToMessageId,
+          text: { text: normalized },
+        },
+        extras(target),
+      );
+    },
+
+    sendVoice: async (target, params) => {
+      const normalized = normalizeOutputText(params.speech);
+      if (!normalized) return { ok: false, reason: "noop" };
+      const voiceName = params.language
+        ? applyVoiceLanguage(DEFAULT_VOICE, params.language)
+        : DEFAULT_VOICE;
+      return bridge.request(
+        "message.send_voice",
+        {
+          "@type": TYPE_URLS.bridgeSendVoiceMessageCommand,
+          to_username: params.toUsername.replace(/^@/, ""),
+          speech: normalized,
+          voice_name: voiceName,
+          reply_to_message_id: target.replyToMessageId,
+        },
+        extras(target),
+      );
+    },
+
+    sendCard: async (target, params) => {
+      return bridge.request(
+        "message.send",
+        {
+          "@type": TYPE_URLS.bridgeSendMessageCommand,
+          conversation_id: target.conversationId,
+          reply_to_message_id: target.replyToMessageId,
+          parts: {
+            parts: [
+              {
+                card: buildCardJson(
+                  params.components as Array<Record<string, unknown>>,
+                  params.data,
+                  params.cardId,
+                ),
+              },
+            ],
+          },
+        },
+        extras(target),
+      );
+    },
+
+    // Ship pre-built A2UI JSONL (same format the canvas tool's a2ui_push
+    // action emits). Skips the components-to-JSONL conversion entirely, so
+    // the agent can reuse its canvas mental model verbatim.
+    sendCardJsonl: async (target, params) => {
+      const jsonl = (params.jsonl ?? "").trim();
+      if (!jsonl) return { ok: false, reason: "empty_jsonl" };
+      return bridge.request(
+        "message.send",
+        {
+          "@type": TYPE_URLS.bridgeSendMessageCommand,
+          conversation_id: target.conversationId,
+          reply_to_message_id: target.replyToMessageId,
+          parts: {
+            parts: [
+              {
+                card: {
+                  a2ui_json: jsonl,
+                  schema_version: params.version ?? "0.8",
+                  card_id:
+                    params.cardId || `openclaw-card-${Math.random().toString(36).slice(2, 10)}`,
+                },
+              },
+            ],
+          },
+        },
+        extras(target),
+      );
+    },
+
+    sendFile: async (target, params) => {
+      const uploadRes = (await bridge.request(
+        "media.upload",
+        {
+          "@type": TYPE_URLS.bridgeMediaUploadCommand,
+          local_path: params.filePath,
+          filename: path.basename(params.filePath),
+          mime_type: "application/octet-stream",
+        },
+        extras(target),
+      )) as { media?: unknown } | null;
+
+      if (!uploadRes?.media) {
+        throw new Error(`failed to upload media: ${JSON.stringify(uploadRes)}`);
+      }
+
+      return bridge.request(
+        "message.send",
+        {
+          "@type": TYPE_URLS.bridgeSendMessageCommand,
+          conversation_id: target.conversationId,
+          reply_to_message_id: target.replyToMessageId,
+          parts: {
+            parts: [
+              {
+                file: uploadRes.media,
+                ...(params.caption ? { metadata: { caption: params.caption } } : {}),
+              },
+            ],
+          },
+        },
+        extras(target),
+      );
+    },
+
+    react: async (params) => {
+      return bridge.request(
+        "message.react",
+        {
+          "@type": TYPE_URLS.addReactionRequest,
+          message_id: params.messageId,
+          emoji: params.emoji,
+        },
+        { message_id: params.messageId },
+      );
+    },
+
+    forward: async (params) => {
+      const toUsername = params.toUsername?.trim().replace(/^@/, "");
+      const targetConversationId = params.targetConversationId?.trim();
+      if (!toUsername && !targetConversationId) {
+        throw new Error(
+          "forward requires exactly one of { toUsername, targetConversationId }",
+        );
+      }
+      // `destination` is a proto `oneof`; set exactly one of the two fields.
+      return bridge.request("message.forward", {
+        "@type": TYPE_URLS.forwardMessageRequest,
+        message_id: params.messageId,
+        ...(toUsername
+          ? { to_username: toUsername }
+          : { conversation_id: targetConversationId }),
+        comment: params.comment || "",
+      });
+    },
+
+    recallSearch: async (params) => {
+      return bridge.request("recall.search", {
+        "@type": TYPE_URLS.recallSearchRequest,
+        query: params.query,
+        conversation_id: params.conversationId,
+        limit: params.maxResults,
+      });
+    },
+
+    recallGet: async (params) => {
+      return bridge.request("recall.get", {
+        "@type": TYPE_URLS.recallGetRequest,
+        entry_id: params.entryId,
+        conversation_id: params.conversationId,
+      });
+    },
+
+    endTurn: (target) => {
+      bridge.send(
+        "turn.end",
+        {
+          "@type": TYPE_URLS.bridgeTurnEndCommand,
+          status: "completed",
+        },
+        extras(target),
+      );
+    },
+
+    emitSignal: (target, signalType, detailText = "") => {
+      if (!target.conversationId) return;
+      bridge.send(
+        "signal.emit",
+        {
+          "@type": TYPE_URLS.sendSignalRequest,
+          conversation_id: target.conversationId,
+          signal_type: signalType,
+          detail_text: detailText,
+          in_reply_to: target.replyToMessageId,
+        },
+        extras(target),
+      );
+    },
+  };
+};