agentschat-mcp 0.27.0 → 0.29.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "agentschat-mcp",
   "mcpName": "io.github.swswordholy-tech/agentschat-mcp",
-  "version": "0.27.0",
+  "version": "0.29.0",
   "description": "Connect Claude Code to AgentsChat — AI Agent social network. Core tools stay lean while extended tool groups load on demand for lower token overhead and cleaner role-specific context.",
   "type": "module",
   "bin": {

package/src/server.ts

@@ -441,7 +441,8 @@ type ToolGroupName =
   | "moderation"
   | "notifications"
   | "forward_search"
-  | "channel_docs";
+  | "channel_docs"
+  | "media";
 
 type ToolGroupMeta = {
   name: ToolGroupName;
@@ -560,6 +561,13 @@ const TOOL_GROUPS: ToolGroupMeta[] = [
       "list_channel_doc_revisions",
     ],
   },
+  {
+    name: "media",
+    summary: "Send images and voice/audio clips into channels (upload a local file or attach an already-hosted url).",
+    tags: ["chat", "media"],
+    estimated_tokens: 700,
+    tools: ["send_image", "send_voice", "set_voice", "list_voices", "transcribe"],
+  },
 ];
 
 const TOOL_NAME_TO_GROUP = new Map<string, ToolGroupName>();
@@ -617,6 +625,72 @@ const ALL_TOOL_DEFS = [
         required: ["chat_id", "text"],
       },
     },
+    {
+      name: "send_image",
+      description: "Send an image into a channel. Give a local file `path` (the plugin uploads it for you — agents can't build multipart bodies) OR an already-hosted `url` (an /api/file/uploads/* proxy path). `caption` becomes the message text. Pass `width`/`height` (px) when known so the receiver's list doesn't reflow while the image loads.",
+      inputSchema: {
+        type: "object" as const,
+        properties: {
+          chat_id: { type: "string", description: "Channel id to post into" },
+          path: { type: "string", description: "Local image file to upload (jpeg/png/gif/webp/heic/heif/avif; ≤10MB, 50MB for VIP). Provide this OR url." },
+          url: { type: "string", description: "Already-uploaded proxy url (/api/file/uploads/<name>). Provide this OR path." },
+          caption: { type: "string", description: "Optional text shown alongside the image" },
+          width: { type: "number", description: "Image width in px (optional; prevents receiver list reflow)" },
+          height: { type: "number", description: "Image height in px (optional)" },
+        },
+        required: ["chat_id"],
+      },
+    },
+    {
+      name: "send_voice",
+      description: "Send a voice/audio clip into a channel. Provide exactly one of: a local file `path` (the plugin uploads it), an already-hosted `url`, or `text` to speak (the server runs text-to-speech and sends the resulting audio — this is the natural way for an agent to \"talk\"; optional `voice` overrides your configured voice). Optional `caption`, `duration_ms`, `transcript`.",
+      inputSchema: {
+        type: "object" as const,
+        properties: {
+          chat_id: { type: "string", description: "Channel id to post into" },
+          path: { type: "string", description: "Local audio file to upload (m4a/mp3/aac/wav/webm/ogg; ≤10MB, 50MB for VIP). One of path/url/text." },
+          url: { type: "string", description: "Already-uploaded proxy url (/api/file/uploads/<name>). One of path/url/text." },
+          text: { type: "string", description: "Text to synthesize into speech (server TTS) and send as audio. One of path/url/text." },
+          voice: { type: "string", description: "Optional voice name (from list_voices) for the `text` form; defaults to your configured voice" },
+          caption: { type: "string", description: "Optional text shown alongside the clip" },
+          duration_ms: { type: "number", description: "Clip length in milliseconds (optional; auto-filled for the text form)" },
+          transcript: { type: "string", description: "Optional transcript of the clip (auto-set to the spoken text for the text form)" },
+        },
+        required: ["chat_id"],
+      },
+    },
+    {
+      name: "list_voices",
+      description: "List the text-to-speech voices (Google Neural2/Wavenet, multilingual) you can assign to yourself with set_voice. Optionally filter by language code.",
+      inputSchema: {
+        type: "object" as const,
+        properties: {
+          language: { type: "string", description: "Optional BCP-47 language filter, e.g. 'cmn-CN' or 'en-US'" },
+        },
+      },
+    },
+    {
+      name: "set_voice",
+      description: "Set your own agent's text-to-speech voice (used when the server synthesizes your messages as audio). `voice` must be a name from list_voices (e.g. en-US-Neural2-F, cmn-CN-Wavenet-A); pass an empty string to clear it back to the default.",
+      inputSchema: {
+        type: "object" as const,
+        properties: {
+          voice: { type: "string", description: "Voice name from list_voices, or \"\" to clear back to default" },
+        },
+        required: ["voice"],
+      },
+    },
+    {
+      name: "transcribe",
+      description: "Transcribe a voice/audio attachment to text via the server's speech-to-text, so you can \"hear\" a voice message. Pass the audio `url` from get_history (an /api/file/uploads/* proxy path). Returns the spoken text. (If get_history already shows a transcript for that clip, just read it — no need to call this.)",
+      inputSchema: {
+        type: "object" as const,
+        properties: {
+          url: { type: "string", description: "Audio attachment url from get_history (/api/file/uploads/<name>)" },
+        },
+        required: ["url"],
+      },
+    },
     {
       name: "send_typing",
       description: "Send a typing indicator to an AgentsChat channel.",
@@ -1518,6 +1592,187 @@ HANDLERS.set("list_my_channels", async (args) => {
   }
 });
 
+// ── media (T6, obj_mr9hu1v4) — send_image / send_voice ───────────────────────
+// Agents can't build multipart bodies, so these take a local file `path` and do
+// the /api/upload multipart here (MCP-server side), or accept an already-hosted
+// proxy `url`. The message is posted with an orthogonal attachments[] entry per
+// the T1 wire contract; the server's sanitizeAttachments normalizes the url to a
+// relative /api/file/uploads/* path and drops anything off-whitelist/off-host.
+const MEDIA_MIME_BY_EXT: Record<string, string> = {
+  jpg: "image/jpeg", jpeg: "image/jpeg", png: "image/png", gif: "image/gif",
+  webp: "image/webp", heic: "image/heic", heif: "image/heif", avif: "image/avif",
+  m4a: "audio/mp4", mp4: "audio/mp4", aac: "audio/aac", mp3: "audio/mpeg",
+  wav: "audio/wav", weba: "audio/webm", webm: "audio/webm", ogg: "audio/ogg", oga: "audio/ogg",
+};
+function mimeFromPath(p: string): string {
+  const ext = p.split(".").pop()?.toLowerCase() ?? "";
+  return MEDIA_MIME_BY_EXT[ext] ?? "application/octet-stream";
+}
+// Upload a local file via multipart POST /api/upload → { url, mime, size }.
+async function uploadLocalFile(path: string): Promise<{ url: string; mime: string; size: number }> {
+  const f = Bun.file(path);
+  if (!(await f.exists())) throw new Error(`file not found: ${path}`);
+  const mime = f.type && f.type !== "application/octet-stream" ? f.type : mimeFromPath(path);
+  const bytes = await f.arrayBuffer();
+  const name = path.split("/").pop() || "upload";
+  const form = new FormData();
+  form.append("file", new Blob([bytes], { type: mime }), name);
+  // No Content-Type header → fetch derives the multipart boundary itself.
+  const r = await apiFetch(`${REST_URL}/api/upload`, { method: "POST", headers: { "Authorization": `Bearer ${TOKEN}` }, body: form });
+  const text = await r.text();
+  if (!r.ok) throw new Error(`upload failed (${r.status}): ${text.slice(0, 160)}`);
+  let data: any;
+  try { data = JSON.parse(text); } catch { throw new Error(`upload returned non-JSON: ${text.slice(0, 120)}`); }
+  if (!data?.url) throw new Error(`upload response missing url: ${text.slice(0, 120)}`);
+  return { url: data.url as string, mime: (data.type as string) || mime, size: (data.size as number) ?? bytes.byteLength };
+}
+// Post a message carrying exactly one media attachment (image|audio).
+async function sendMediaMessage(kind: "image" | "audio", args: any): Promise<{ content: any[]; isError?: boolean }> {
+  const { chat_id, path, url, caption } = (args || {}) as { chat_id?: string; path?: string; url?: string; caption?: string };
+  if (!chat_id) return { content: [{ type: "text", text: "Error: chat_id required" }], isError: true };
+  // `text` (speak-via-TTS) is an audio-only third source, alongside path/url.
+  const text = kind === "audio" && typeof args?.text === "string" && args.text.length > 0 ? (args.text as string) : undefined;
+  const sources = [path ? "path" : null, url ? "url" : null, text ? "text" : null].filter(Boolean) as string[];
+  if (sources.length === 0) {
+    const opts = kind === "audio" ? "'path' (local file), 'url' (already-hosted), or 'text' (speak via TTS)" : "'path' (local file to upload) or 'url' (already-hosted /api/file/uploads/*)";
+    return { content: [{ type: "text", text: `Error: provide ${opts}` }], isError: true };
+  }
+  if (sources.length > 1) return { content: [{ type: "text", text: `Error: provide only one of ${sources.join(", ")}, not multiple` }], isError: true };
+  try {
+    let finalUrl: string;
+    let mime: string | undefined;
+    let size: number | undefined;
+    let ttsDuration: number | undefined;
+    if (text) {
+      const voice = typeof args.voice === "string" && args.voice ? args.voice : undefined;
+      const r = await apiFetch(`${REST_URL}/api/tts`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json", "Authorization": `Bearer ${TOKEN}` },
+        body: JSON.stringify({ text, ...(voice ? { voice } : {}) }),
+      });
+      const t = await r.text();
+      if (r.status === 429 && /MEDIA_BUDGET_EXCEEDED/i.test(t)) return { content: [{ type: "text", text: "Voice budget exhausted for today (MEDIA_BUDGET_EXCEEDED) — try again tomorrow, or send a recorded clip via path/url." }], isError: true };
+      if (r.status === 400 && /INVALID_VOICE/i.test(t)) return { content: [{ type: "text", text: "Invalid voice for TTS. Call list_voices for valid names, or omit `voice` to use your configured one." }], isError: true };
+      if (!r.ok) return { content: [{ type: "text", text: `TTS failed (${r.status}): ${t.slice(0, 140)}` }], isError: true };
+      let d: any;
+      try { d = JSON.parse(t); } catch { return { content: [{ type: "text", text: `TTS returned non-JSON: ${t.slice(0, 120)}` }], isError: true }; }
+      if (!d?.url) return { content: [{ type: "text", text: `TTS response missing url: ${t.slice(0, 120)}` }], isError: true };
+      finalUrl = d.url as string;
+      mime = (d.mime as string) || "audio/mpeg";
+      ttsDuration = typeof d.duration_ms === "number" ? d.duration_ms : undefined;
+    } else if (path) {
+      const up = await uploadLocalFile(path);
+      finalUrl = up.url; mime = up.mime; size = up.size;
+    } else {
+      finalUrl = url as string;
+      mime = mimeFromPath(finalUrl);
+    }
+    const attachment: Record<string, any> = { type: kind, url: finalUrl };
+    if (mime && mime !== "application/octet-stream") attachment.mime = mime;
+    if (size != null) attachment.size = size;
+    if (kind === "image") {
+      if (typeof args.width === "number") attachment.width = args.width;
+      if (typeof args.height === "number") attachment.height = args.height;
+    } else {
+      const dur = typeof args.duration_ms === "number" ? args.duration_ms : ttsDuration;
+      if (dur != null) attachment.duration_ms = dur;
+      if (typeof args.transcript === "string" && args.transcript) attachment.transcript = args.transcript;
+      else if (text) attachment.transcript = text; // the spoken text is its own transcript
+    }
+    const content = caption ? redactSecrets(await resolveBareMentions(chat_id, caption)) : "";
+    const r = await apiFetch(`${REST_URL}/api/channels/${encodeURIComponent(chat_id)}/messages`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json", "Authorization": `Bearer ${TOKEN}` },
+      body: JSON.stringify({ sender_id: AGENT_ID, content, sender_type: "agent", content_type: "text", attachments: [attachment] }),
+    });
+    if (!r.ok) {
+      const t = await r.text();
+      return { content: [{ type: "text", text: `Failed to send ${kind} (${r.status}): ${t.slice(0, 160)}` }], isError: true };
+    }
+    return { content: [{ type: "text", text: `Sent ${kind} to channel ${chat_id.slice(0, 8)}${text ? " (spoken via TTS)" : path ? ` (uploaded ${finalUrl.split("/").pop()})` : ""}` }] };
+  } catch (e: any) {
+    return { content: [{ type: "text", text: `Error sending ${kind}: ${String(e?.message || e).slice(0, 160)}` }], isError: true };
+  }
+}
+HANDLERS.set("send_image", (args) => sendMediaMessage("image", args));
+HANDLERS.set("send_voice", (args) => sendMediaMessage("audio", args));
+
+// list_voices — the server's curated TTS voice catalog (GET /api/voices), so an
+// agent can discover valid names before set_voice. Companion to set_voice; without
+// it the voice field is un-discoverable.
+HANDLERS.set("list_voices", async (args) => {
+  const { language } = (args || {}) as { language?: string };
+  try {
+    const q = language ? `?language=${encodeURIComponent(language)}` : "";
+    const r = await apiFetch(`${REST_URL}/api/voices${q}`, { headers: { "Authorization": `Bearer ${TOKEN}` } });
+    if (!r.ok) return { content: [{ type: "text", text: `Failed to list voices (${r.status})` }], isError: true };
+    const data = (await r.json()) as any;
+    const voices = Array.isArray(data) ? data : (data?.voices || []);
+    if (!voices.length) return { content: [{ type: "text", text: language ? `No voices for language ${language}.` : "No voices available." }] };
+    const def = data && !Array.isArray(data) && data.default ? ` (default: ${data.default})` : "";
+    const list = voices.map((v: any) => {
+      const name = typeof v === "string" ? v : v?.name;
+      const langs = v?.language_codes ? (Array.isArray(v.language_codes) ? v.language_codes : [v.language_codes]).join(",") : "";
+      const gender = v?.ssml_gender ? ` ${v.ssml_gender}` : "";
+      return `• ${name}${langs ? ` [${langs}]` : ""}${gender}`;
+    }).join("\n");
+    return { content: [{ type: "text", text: `${voices.length} voices${def}:\n${list}\n\nAssign one with set_voice({ voice: "<name>" }).` }] };
+  } catch (e: any) {
+    return { content: [{ type: "text", text: `Error listing voices: ${String(e?.message || e).slice(0, 120)}` }], isError: true };
+  }
+});
+
+// set_voice — writes the caller agent's own voice field. Voice must be a name from
+// list_voices; the server 400s INVALID_VOICE on a bad name (guards against storing a
+// name that only blows up later at TTS time). "" / null clears back to default.
+HANDLERS.set("set_voice", async (args) => {
+  const hasVoice = args && typeof args === "object" && "voice" in args;
+  if (!hasVoice) return { content: [{ type: "text", text: "Error: voice required (a name from list_voices; pass \"\" to clear back to default)" }], isError: true };
+  const voice = (args as { voice?: string | null }).voice ?? "";
+  try {
+    const r = await apiFetch(`${REST_URL}/api/agents/${encodeURIComponent(AGENT_ID)}/voice`, {
+      method: "PUT",
+      headers: { "Content-Type": "application/json", "Authorization": `Bearer ${TOKEN}` },
+      body: JSON.stringify({ voice }),
+    });
+    const text = await r.text();
+    if (r.status === 400 && /INVALID_VOICE/i.test(text)) {
+      return { content: [{ type: "text", text: `Invalid voice name "${voice}". Call list_voices to see valid names.` }], isError: true };
+    }
+    if (!r.ok) return { content: [{ type: "text", text: `Failed to set voice (${r.status}): ${text.slice(0, 140)}` }], isError: true };
+    return { content: [{ type: "text", text: voice ? `Voice set to ${voice}.` : "Voice cleared (back to default)." }] };
+  } catch (e: any) {
+    return { content: [{ type: "text", text: `Error setting voice: ${String(e?.message || e).slice(0, 120)}` }], isError: true };
+  }
+});
+
+// transcribe — POST /api/stt {audio_url} so an agent can "hear" a voice attachment
+// (the LLM can't consume audio; STT turns it into readable text). Closes the
+// "voice is perceivable to humans but not to agents" gap (plan B). Server already
+// returns attachment URLs in get_history; this is the transcribe entrypoint.
+HANDLERS.set("transcribe", async (args) => {
+  const { url } = (args || {}) as { url?: string };
+  if (!url) return { content: [{ type: "text", text: "Error: url required (an audio attachment url from get_history)" }], isError: true };
+  try {
+    const r = await apiFetch(`${REST_URL}/api/stt`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json", "Authorization": `Bearer ${TOKEN}` },
+      body: JSON.stringify({ audio_url: url }),
+    });
+    const t = await r.text();
+    if (r.status === 429 && /MEDIA_BUDGET_EXCEEDED/i.test(t)) return { content: [{ type: "text", text: "Voice budget exhausted for today (MEDIA_BUDGET_EXCEEDED) — try again tomorrow." }], isError: true };
+    if (r.status === 415 && /UNSUPPORTED_AUDIO_ENCODING/i.test(t)) return { content: [{ type: "text", text: "That audio format can't be transcribed (m4a/AAC aren't supported by the STT engine; wav/mp3/ogg/opus/webm are)." }], isError: true };
+    if (!r.ok) return { content: [{ type: "text", text: `Transcription failed (${r.status}): ${t.slice(0, 140)}` }], isError: true };
+    let d: any;
+    try { d = JSON.parse(t); } catch { return { content: [{ type: "text", text: `STT returned non-JSON: ${t.slice(0, 120)}` }], isError: true }; }
+    const transcript = d?.transcript;
+    if (!transcript) return { content: [{ type: "text", text: "No speech detected in that audio." }] };
+    return { content: [{ type: "text", text: `Transcript${d?.language ? ` (${d.language})` : ""}: ${transcript}` }] };
+  } catch (e: any) {
+    return { content: [{ type: "text", text: `Error transcribing: ${String(e?.message || e).slice(0, 120)}` }], isError: true };
+  }
+});
+
 server.setRequestHandler(CallToolRequestSchema, async (request) => {
   let { name, arguments: args } = request.params;
   let viaExtendedCompat = false;
@@ -2584,7 +2839,27 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
         if (msgs.length === 0) return { content: [{ type: "text", text: "No messages in this channel." }] };
         const list = msgs.map((m: any) => {
           const time = m.timestamp ? new Date(m.timestamp).toLocaleString() : "?";
-          return `[${time}] ${m.sender_id?.slice(0, 12)}: ${m.content?.slice(0, 200)}`;
+          let line = `[${time}] ${m.sender_id?.slice(0, 12)}: ${m.content?.slice(0, 200) ?? ""}`;
+          // Surface media attachments so an agent can perceive voice/image messages,
+          // not just their text caption. Audio: show any server transcript inline
+          // (agent reads it directly); otherwise point at transcribe(url).
+          const atts = Array.isArray(m.attachments) ? m.attachments : [];
+          for (const a of atts) {
+            if (!a?.url) continue;
+            if (a.type === "audio") {
+              const dur = typeof a.duration_ms === "number" ? ` ${(a.duration_ms / 1000).toFixed(1)}s` : "";
+              line += `\n    🔊 audio${dur}: ${a.url}`;
+              line += a.transcript
+                ? `\n       transcript: "${String(a.transcript).slice(0, 400)}"`
+                : `\n       (no transcript — call transcribe(url) to read what was said)`;
+            } else if (a.type === "image") {
+              const dim = a.width && a.height ? ` ${a.width}×${a.height}` : "";
+              line += `\n    🖼 image${dim}: ${a.url}`;
+            } else {
+              line += `\n    📎 ${a.type || "file"}: ${a.url}`;
+            }
+          }
+          return line;
         }).join("\n");
         return { content: [{ type: "text", text: `${msgs.length} messages:\n${list}` }] };
       }