@tractorscorch/clank 1.2.0 → 1.3.0

package/CHANGELOG.md

@@ -6,6 +6,30 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/).
 
 ---
 
+## [1.3.0] — 2026-03-23
+
+### Added
+- **ElevenLabs integration** — text-to-speech via ElevenLabs API, configurable during onboarding
+- **Whisper integration** — speech-to-text via OpenAI Whisper API or local whisper.cpp
+- **Voice tools (3):** `text_to_speech`, `speech_to_text`, `list_voices` — agent can generate and transcribe audio
+- **Telegram voice messages** — send a voice message → auto-transcribed via Whisper → routed to agent → response as voice (ElevenLabs) or text
+- **Integrations config section** — unified config for third-party API services (ElevenLabs, Whisper, image gen, extensible)
+- **Setup wizard integrations step** — configure ElevenLabs, Whisper, and other API services during onboarding
+
+### Changed
+- Tool count: 21 total (10 core + 11 self-config/voice)
+- Setup wizard now asks about integrations for all users (not just advanced mode)
+
+---
+
+## [1.2.1] — 2026-03-23
+
+### Fixed
+- **Gateway crash on restart** — stale Telegram messages queued while offline no longer flood the model. Messages older than 30s before startup are dropped.
+- **Parallel model overload** — Telegram messages from the same chat are now processed sequentially (per-chat queue) instead of all at once.
+
+---
+
 ## [1.2.0] — 2026-03-22
 
 ### Added

package/dist/index.js

@@ -1901,7 +1901,8 @@ function defaultConfig() {
     },
     safety: {
       confirmExternal: true
-    }
+    },
+    integrations: {}
   };
 }
 function substituteEnvVars(obj) {
@@ -3919,6 +3920,264 @@ var init_message_tool = __esm({
   }
 });
 
+// src/voice/tts.ts
+var TTSEngine, STTEngine;
+var init_tts = __esm({
+  "src/voice/tts.ts"() {
+    "use strict";
+    init_esm_shims();
+    TTSEngine = class {
+      config;
+      constructor(config) {
+        this.config = config;
+      }
+      /** Check if TTS is available */
+      isAvailable() {
+        return !!(this.config.integrations.elevenlabs?.enabled && this.config.integrations.elevenlabs?.apiKey);
+      }
+      /** Convert text to speech */
+      async synthesize(text, opts) {
+        const elevenlabs = this.config.integrations.elevenlabs;
+        if (!elevenlabs?.enabled || !elevenlabs.apiKey) return null;
+        const voiceId = opts?.voiceId || elevenlabs.voiceId || "JBFqnCBsd6RMkjVDRZzb";
+        const model = elevenlabs.model || "eleven_multilingual_v2";
+        try {
+          const res = await fetch(
+            `https://api.elevenlabs.io/v1/text-to-speech/${voiceId}`,
+            {
+              method: "POST",
+              headers: {
+                "Content-Type": "application/json",
+                "xi-api-key": elevenlabs.apiKey
+              },
+              body: JSON.stringify({
+                text,
+                model_id: model,
+                voice_settings: {
+                  stability: 0.5,
+                  similarity_boost: 0.75
+                }
+              })
+            }
+          );
+          if (!res.ok) {
+            const err = await res.text().catch(() => "");
+            console.error(`ElevenLabs TTS error ${res.status}: ${err}`);
+            return null;
+          }
+          const arrayBuffer = await res.arrayBuffer();
+          return {
+            audioBuffer: Buffer.from(arrayBuffer),
+            format: "mp3"
+          };
+        } catch (err) {
+          console.error(`TTS error: ${err instanceof Error ? err.message : err}`);
+          return null;
+        }
+      }
+      /** List available voices from ElevenLabs */
+      async listVoices() {
+        const elevenlabs = this.config.integrations.elevenlabs;
+        if (!elevenlabs?.enabled || !elevenlabs.apiKey) return [];
+        try {
+          const res = await fetch("https://api.elevenlabs.io/v1/voices", {
+            headers: { "xi-api-key": elevenlabs.apiKey }
+          });
+          if (!res.ok) return [];
+          const data = await res.json();
+          return (data.voices || []).map((v) => ({ id: v.voice_id, name: v.name }));
+        } catch {
+          return [];
+        }
+      }
+    };
+    STTEngine = class {
+      config;
+      constructor(config) {
+        this.config = config;
+      }
+      /** Check if STT is available */
+      isAvailable() {
+        const whisper = this.config.integrations.whisper;
+        if (whisper?.enabled) {
+          if (whisper.provider === "openai" && whisper.apiKey) return true;
+          if (whisper.provider === "local") return true;
+        }
+        if (this.config.models.providers.openai?.apiKey) return true;
+        return false;
+      }
+      /** Transcribe audio to text */
+      async transcribe(audioBuffer, format = "ogg") {
+        const whisper = this.config.integrations.whisper;
+        const apiKey = whisper?.apiKey || this.config.models.providers.openai?.apiKey;
+        if (apiKey && whisper?.provider !== "local") {
+          return this.transcribeOpenAI(audioBuffer, format, apiKey);
+        }
+        return this.transcribeLocal(audioBuffer, format);
+      }
+      /** Transcribe via OpenAI Whisper API */
+      async transcribeOpenAI(audioBuffer, format, apiKey) {
+        try {
+          const blob = new Blob([new Uint8Array(audioBuffer)], { type: `audio/${format}` });
+          const formData = new FormData();
+          formData.append("file", blob, `audio.${format}`);
+          formData.append("model", "whisper-1");
+          const res = await fetch("https://api.openai.com/v1/audio/transcriptions", {
+            method: "POST",
+            headers: { "Authorization": `Bearer ${apiKey}` },
+            body: formData
+          });
+          if (!res.ok) return null;
+          const data = await res.json();
+          return data.text ? { text: data.text, language: data.language } : null;
+        } catch {
+          return null;
+        }
+      }
+      /** Transcribe via local whisper.cpp */
+      async transcribeLocal(audioBuffer, format) {
+        try {
+          const { writeFile: writeFile9, unlink: unlink5 } = await import("fs/promises");
+          const { execSync: execSync3 } = await import("child_process");
+          const { join: join19 } = await import("path");
+          const { tmpdir } = await import("os");
+          const tmpFile = join19(tmpdir(), `clank-stt-${Date.now()}.${format}`);
+          await writeFile9(tmpFile, audioBuffer);
+          const output = execSync3(`whisper "${tmpFile}" --model base.en --output-txt`, {
+            encoding: "utf-8",
+            timeout: 6e4
+          });
+          await unlink5(tmpFile).catch(() => {
+          });
+          return output.trim() ? { text: output.trim() } : null;
+        } catch {
+          return null;
+        }
+      }
+    };
+  }
+});
+
+// src/voice/index.ts
+var voice_exports = {};
+__export(voice_exports, {
+  STTEngine: () => STTEngine,
+  TTSEngine: () => TTSEngine
+});
+var init_voice = __esm({
+  "src/voice/index.ts"() {
+    "use strict";
+    init_esm_shims();
+    init_tts();
+  }
+});
+
+// src/tools/self-config/voice-tool.ts
+var ttsTool, sttTool, voiceListTool;
+var init_voice_tool = __esm({
+  "src/tools/self-config/voice-tool.ts"() {
+    "use strict";
+    init_esm_shims();
+    init_voice();
+    init_config2();
+    ttsTool = {
+      definition: {
+        name: "text_to_speech",
+        description: "Convert text to speech audio using ElevenLabs. Returns the audio file path. Requires ElevenLabs integration to be configured.",
+        parameters: {
+          type: "object",
+          properties: {
+            text: { type: "string", description: "Text to convert to speech" },
+            voice_id: { type: "string", description: "ElevenLabs voice ID (optional, uses default)" }
+          },
+          required: ["text"]
+        }
+      },
+      safetyLevel: "low",
+      readOnly: true,
+      validate(args) {
+        if (!args.text || typeof args.text !== "string") return { ok: false, error: "text is required" };
+        if (args.text.length > 5e3) return { ok: false, error: "text too long (max 5000 chars)" };
+        return { ok: true };
+      },
+      async execute(args) {
+        const config = await loadConfig();
+        const engine = new TTSEngine(config);
+        if (!engine.isAvailable()) {
+          return "Error: ElevenLabs not configured. Tell me to set it up, or run: clank setup --section integrations";
+        }
+        const result = await engine.synthesize(args.text, {
+          voiceId: args.voice_id
+        });
+        if (!result) return "Error: TTS synthesis failed";
+        const { writeFile: writeFile9 } = await import("fs/promises");
+        const { join: join19 } = await import("path");
+        const { tmpdir } = await import("os");
+        const outPath = join19(tmpdir(), `clank-tts-${Date.now()}.${result.format}`);
+        await writeFile9(outPath, result.audioBuffer);
+        return `Audio generated: ${outPath} (${result.format}, ${Math.round(result.audioBuffer.length / 1024)}KB)`;
+      }
+    };
+    sttTool = {
+      definition: {
+        name: "speech_to_text",
+        description: "Transcribe an audio file to text using Whisper (OpenAI API or local whisper.cpp). Provide a file path to an audio file.",
+        parameters: {
+          type: "object",
+          properties: {
+            file_path: { type: "string", description: "Path to audio file (.mp3, .wav, .ogg, .m4a)" }
+          },
+          required: ["file_path"]
+        }
+      },
+      safetyLevel: "low",
+      readOnly: true,
+      validate(args) {
+        if (!args.file_path || typeof args.file_path !== "string") return { ok: false, error: "file_path is required" };
+        return { ok: true };
+      },
+      async execute(args) {
+        const { readFile: readFile12 } = await import("fs/promises");
+        const { existsSync: existsSync10 } = await import("fs");
+        const filePath = args.file_path;
+        if (!existsSync10(filePath)) return `Error: File not found: ${filePath}`;
+        const config = await loadConfig();
+        const engine = new STTEngine(config);
+        if (!engine.isAvailable()) {
+          return "Error: Speech-to-text not configured. Need OpenAI API key or local whisper.cpp installed.";
+        }
+        const audioBuffer = await readFile12(filePath);
+        const ext = filePath.split(".").pop() || "wav";
+        const result = await engine.transcribe(audioBuffer, ext);
+        if (!result) return "Error: Transcription failed";
+        return result.text;
+      }
+    };
+    voiceListTool = {
+      definition: {
+        name: "list_voices",
+        description: "List available ElevenLabs voices for text-to-speech.",
+        parameters: { type: "object", properties: {} }
+      },
+      safetyLevel: "low",
+      readOnly: true,
+      validate() {
+        return { ok: true };
+      },
+      async execute() {
+        const config = await loadConfig();
+        const engine = new TTSEngine(config);
+        if (!engine.isAvailable()) {
+          return "Error: ElevenLabs not configured.";
+        }
+        const voices = await engine.listVoices();
+        if (voices.length === 0) return "No voices found or API error.";
+        return voices.map((v) => `${v.name}: ${v.id}`).join("\n");
+      }
+    };
+  }
+});
+
 // src/tools/self-config/index.ts
 function registerSelfConfigTools(registry) {
   registry.register(configTool);
@@ -3929,6 +4188,9 @@ function registerSelfConfigTools(registry) {
   registry.register(cronTool);
   registry.register(gatewayTool);
   registry.register(messageTool);
+  registry.register(ttsTool);
+  registry.register(sttTool);
+  registry.register(voiceListTool);
 }
 var init_self_config = __esm({
   "src/tools/self-config/index.ts"() {
@@ -3942,6 +4204,7 @@ var init_self_config = __esm({
     init_cron_tool();
     init_gateway_tool();
     init_message_tool();
+    init_voice_tool();
     init_config_tool();
     init_channel_tool();
     init_agent_tool();
@@ -3950,6 +4213,7 @@ var init_self_config = __esm({
     init_cron_tool();
     init_gateway_tool();
     init_message_tool();
+    init_voice_tool();
   }
 });
 
@@ -4732,11 +4996,17 @@ var init_telegram = __esm({
         try {
           this.bot = new Bot(telegramConfig.botToken);
           const bot = this.bot;
+          const startupTime = Math.floor(Date.now() / 1e3);
+          const chatLocks = /* @__PURE__ */ new Map();
           bot.on("message:text", async (ctx) => {
             const msg = ctx.message;
             const chatId = msg.chat.id;
             const userId = msg.from?.id;
             const isGroup = msg.chat.type === "group" || msg.chat.type === "supergroup";
+            if (msg.date < startupTime - 30) {
+              console.log(`  Telegram: dropping stale message from ${userId} (${startupTime - msg.date}s old)`);
+              return;
+            }
             if (telegramConfig.allowFrom && telegramConfig.allowFrom.length > 0) {
               const username = msg.from?.username ? `@${msg.from.username}` : "";
               const userIdStr = String(userId || "");
@@ -4760,27 +5030,103 @@ var init_telegram = __esm({
               }
               return;
             }
-            if (!this.gateway) return;
-            try {
-              await ctx.api.sendChatAction(chatId, "typing");
-              const response = await this.gateway.handleInboundMessage(
-                {
-                  channel: "telegram",
-                  peerId: chatId,
-                  peerKind: isGroup ? "group" : "dm"
-                },
-                msg.text
-              );
-              if (response) {
-                const chunks = splitMessage(response, 4e3);
-                for (const chunk of chunks) {
-                  await ctx.api.sendMessage(chatId, chunk);
+            const processMessage = async () => {
+              if (!this.gateway) return;
+              try {
+                await ctx.api.sendChatAction(chatId, "typing");
+                const response = await this.gateway.handleInboundMessage(
+                  {
+                    channel: "telegram",
+                    peerId: chatId,
+                    peerKind: isGroup ? "group" : "dm"
+                  },
+                  msg.text
+                );
+                if (response) {
+                  const chunks = splitMessage(response, 4e3);
+                  for (const chunk of chunks) {
+                    await ctx.api.sendMessage(chatId, chunk);
+                  }
                 }
+              } catch (err) {
+                const errMsg = err instanceof Error ? err.message : String(err);
+                await ctx.api.sendMessage(chatId, `Error: ${errMsg.slice(0, 200)}`);
               }
-            } catch (err) {
-              const errMsg = err instanceof Error ? err.message : String(err);
-              await ctx.api.sendMessage(chatId, `Error: ${errMsg.slice(0, 200)}`);
+            };
+            const prev = chatLocks.get(chatId) || Promise.resolve();
+            const next = prev.then(processMessage).catch(() => {
+            });
+            chatLocks.set(chatId, next);
+          });
+          bot.on("message:voice", async (ctx) => {
+            const msg = ctx.message;
+            const chatId = msg.chat.id;
+            const userId = msg.from?.id;
+            if (telegramConfig.allowFrom && telegramConfig.allowFrom.length > 0) {
+              const username = msg.from?.username ? `@${msg.from.username}` : "";
+              const userIdStr = String(userId || "");
+              const allowed = telegramConfig.allowFrom.map(String);
+              const isAllowed = allowed.some(
+                (a) => a === userIdStr || a.toLowerCase() === username.toLowerCase() || a.toLowerCase() === (msg.from?.username || "").toLowerCase()
+              );
+              if (!isAllowed) return;
             }
+            if (msg.date < startupTime - 30) return;
+            const processVoice = async () => {
+              if (!this.gateway || !this.config) return;
+              try {
+                await ctx.api.sendChatAction(chatId, "typing");
+                const file = await ctx.api.getFile(msg.voice.file_id);
+                const fileUrl = `https://api.telegram.org/file/bot${telegramConfig.botToken}/${file.file_path}`;
+                const res = await fetch(fileUrl);
+                if (!res.ok) {
+                  await ctx.api.sendMessage(chatId, "Error: could not download voice message");
+                  return;
+                }
+                const audioBuffer = Buffer.from(await res.arrayBuffer());
+                const { STTEngine: STTEngine2 } = await Promise.resolve().then(() => (init_voice(), voice_exports));
+                const { loadConfig: loadConfig3 } = await Promise.resolve().then(() => (init_config2(), config_exports));
+                const config = await loadConfig3();
+                const stt = new STTEngine2(config);
+                if (!stt.isAvailable()) {
+                  await ctx.api.sendMessage(chatId, "Voice messages require speech-to-text. Set up Whisper: /help");
+                  return;
+                }
+                const transcription = await stt.transcribe(audioBuffer, "ogg");
+                if (!transcription?.text) {
+                  await ctx.api.sendMessage(chatId, "Could not transcribe voice message.");
+                  return;
+                }
+                const isGroup = msg.chat.type === "group" || msg.chat.type === "supergroup";
+                const response = await this.gateway.handleInboundMessage(
+                  { channel: "telegram", peerId: chatId, peerKind: isGroup ? "group" : "dm" },
+                  `[Voice message transcription]: ${transcription.text}`
+                );
+                if (response) {
+                  const { TTSEngine: TTSEngine2 } = await Promise.resolve().then(() => (init_voice(), voice_exports));
+                  const tts = new TTSEngine2(config);
+                  if (tts.isAvailable() && response.length < 2e3) {
+                    const audio = await tts.synthesize(response);
+                    if (audio) {
+                      const { InputFile } = await import("grammy");
+                      await ctx.api.sendVoice(chatId, new InputFile(audio.audioBuffer, "reply.mp3"));
+                      return;
+                    }
+                  }
+                  const chunks = splitMessage(response, 4e3);
+                  for (const chunk of chunks) {
+                    await ctx.api.sendMessage(chatId, chunk);
+                  }
+                }
+              } catch (err) {
+                const errMsg = err instanceof Error ? err.message : String(err);
+                await ctx.api.sendMessage(chatId, `Error: ${errMsg.slice(0, 200)}`);
+              }
+            };
+            const prev = chatLocks.get(chatId) || Promise.resolve();
+            const next = prev.then(processVoice).catch(() => {
+            });
+            chatLocks.set(chatId, next);
           });
           bot.start({
             onStart: () => {
@@ -5327,7 +5673,7 @@ var init_server = __esm({
           res.writeHead(200, { "Content-Type": "application/json" });
           res.end(JSON.stringify({
             status: "ok",
-            version: "1.2.0",
+            version: "1.3.0",
             uptime: process.uptime(),
             clients: this.clients.size,
             agents: this.engines.size
@@ -5435,7 +5781,7 @@ var init_server = __esm({
         const hello = {
           type: "hello",
           protocol: PROTOCOL_VERSION,
-          version: "1.2.0",
+          version: "1.3.0",
           agents: this.config.agents.list.map((a) => ({
             id: a.id,
             name: a.name || a.id,
@@ -6278,21 +6624,44 @@ async function runSetup(opts) {
         console.log(green4("    Brave Search configured"));
       }
     }
-    if (isAdvanced) {
-      console.log("");
-      const addVoice = await ask(rl, cyan2("  Set up voice (TTS/STT)? [y/N] "));
-      if (addVoice.toLowerCase() === "y") {
-        console.log(dim4("    1. ElevenLabs (cloud)"));
-        console.log(dim4("    2. Local (whisper.cpp + piper)"));
-        const voiceChoice = await ask(rl, cyan2("    Choice [1]: "));
-        if (voiceChoice === "1" || !voiceChoice) {
-          const key = await ask(rl, cyan2("    ElevenLabs API key: "));
+    console.log("");
+    console.log("  API Integrations:");
+    console.log(dim4("    Add third-party services for voice, image gen, etc."));
+    console.log(dim4("    You can also configure these later through conversation."));
+    console.log("");
+    const addElevenLabs = await ask(rl, cyan2("  Set up ElevenLabs (text-to-speech)? [y/N] "));
+    if (addElevenLabs.toLowerCase() === "y") {
+      console.log(dim4("    Get an API key at: https://elevenlabs.io/"));
+      const key = await ask(rl, cyan2("    ElevenLabs API key: "));
+      if (key.trim()) {
+        config.integrations.elevenlabs = { enabled: true, apiKey: key.trim() };
+        const voiceId = await ask(rl, cyan2("    Default voice ID (Enter to skip): "));
+        if (voiceId.trim()) {
+          config.integrations.elevenlabs.voiceId = voiceId.trim();
+        }
+        console.log(green4("    ElevenLabs configured (TTS available)"));
+      }
+    }
+    const addWhisper = await ask(rl, cyan2("  Set up speech-to-text (Whisper)? [y/N] "));
+    if (addWhisper.toLowerCase() === "y") {
+      console.log(dim4("    1. OpenAI Whisper API (cloud, uses OpenAI key)"));
+      console.log(dim4("    2. Local whisper.cpp (requires whisper installed)"));
+      const whisperChoice = await ask(rl, cyan2("    Choice [1]: "));
+      if (whisperChoice === "2") {
+        config.integrations.whisper = { enabled: true, provider: "local" };
+        console.log(green4("    Local whisper.cpp configured"));
+        console.log(dim4("    Make sure whisper is installed and in PATH"));
+      } else {
+        const existingKey = config.models.providers.openai?.apiKey;
+        if (existingKey) {
+          config.integrations.whisper = { enabled: true, provider: "openai", apiKey: existingKey };
+          console.log(green4("    Whisper configured (using existing OpenAI key)"));
+        } else {
+          const key = await ask(rl, cyan2("    OpenAI API key for Whisper: "));
           if (key.trim()) {
-            console.log(green4("    ElevenLabs configured"));
+            config.integrations.whisper = { enabled: true, provider: "openai", apiKey: key.trim() };
+            console.log(green4("    Whisper configured"));
           }
-        } else {
-          console.log(dim4("    Local voice will use whisper.cpp (STT) + piper (TTS)"));
-          console.log(dim4("    Make sure they're installed: whisper, piper"));
         }
       }
     }
@@ -6769,7 +7138,7 @@ async function runTui(opts) {
   ws.on("open", () => {
     ws.send(JSON.stringify({
       type: "connect",
-      params: { auth: { token }, mode: "tui", version: "1.2.0" }
+      params: { auth: { token }, mode: "tui", version: "1.3.0" }
     }));
   });
   ws.on("message", (data) => {
@@ -7198,7 +7567,7 @@ import { fileURLToPath as fileURLToPath5 } from "url";
 import { dirname as dirname5, join as join18 } from "path";
 var __filename3 = fileURLToPath5(import.meta.url);
 var __dirname3 = dirname5(__filename3);
-var version = "1.2.0";
+var version = "1.3.0";
 try {
   const pkg = JSON.parse(readFileSync(join18(__dirname3, "..", "package.json"), "utf-8"));
   version = pkg.version;