clearclaw 0.8.0 → 0.9.0

package/dist/channel/telegram.d.ts

@@ -41,7 +41,7 @@ export declare class TelegramChannel extends EventEmitter implements Channel {
     private extractSender;
     /** Extract reply context from a Telegram message, if it's a reply. */
     private extractReplyContext;
-    /** Shared handler for photo/document messages: download, emit attachment or fallback. */
+    /** Shared handler for file messages: download, emit attachment or fallback. */
     private handleFileMessage;
     sendFile(chatId: string, buffer: Buffer, filename: string, opts?: SendFileOpts): Promise<void>;
     reactToMessage(chatId: string, messageId: string, emoji: string): Promise<void>;

package/dist/channel/telegram.js

@@ -82,10 +82,17 @@ export class TelegramChannel extends EventEmitter {
             const doc = ctx.message.document;
             await this.handleFileMessage(ctx, doc.file_id, doc.mime_type ?? "application/octet-stream", doc.file_name, ctx.message.caption);
         });
+        // Voice notes and audio files: the orchestrator transcribes them
+        this.bot.on("message:voice", async (ctx) => {
+            const voice = ctx.message.voice;
+            await this.handleFileMessage(ctx, voice.file_id, voice.mime_type ?? "audio/ogg", "voice.ogg", ctx.message.caption);
+        });
+        this.bot.on("message:audio", async (ctx) => {
+            const audio = ctx.message.audio;
+            await this.handleFileMessage(ctx, audio.file_id, audio.mime_type ?? "audio/mpeg", audio.file_name, ctx.message.caption);
+        });
         // Placeholder handlers for unsupported media types
         for (const [filter, label] of [
-            ["message:voice", "Voice message"],
-            ["message:audio", "Audio message"],
             ["message:video", "Video"],
             ["message:sticker", "Sticker"],
         ]) {
@@ -399,7 +406,7 @@ export class TelegramChannel extends EventEmitter {
             mediaType,
         };
     }
-    /** Shared handler for photo/document messages: download, emit attachment or fallback. */
+    /** Shared handler for file messages: download, emit attachment or fallback. */
     async handleFileMessage(ctx, fileId, mimeType, filename, caption) {
         const sender = this.extractSender(ctx);
         if (!sender)

package/dist/orchestrator.d.ts

@@ -1,9 +1,10 @@
 import type { Config } from "./config.js";
-import type { Channel, Engine, InboundMessage, MessageOrigin } from "./types.js";
+import type { Attachment, Channel, Engine, InboundMessage, MessageOrigin } from "./types.js";
 export interface OrchestratorOpts {
     channel: Channel;
     engines: Map<string, Engine>;
     config: Config;
+    transcribe?: (att: Attachment) => Promise<string>;
 }
 export declare class Orchestrator {
     private channel;
@@ -13,6 +14,7 @@ export declare class Orchestrator {
     private creatingWorkspaces;
     private scheduler;
     private planUsage;
+    private transcribe;
     constructor(opts: OrchestratorOpts);
     private chat;
     private engineFor;
@@ -46,6 +48,11 @@ export declare class Orchestrator {
     private persistSessionId;
     /** Native selection works even when the workspace's engine cannot run. */
     private selectEngine;
+    /**
+     * Turn audio attachments into message text. Engines can't consume audio, so it never
+     * reaches them; the echo lets the user catch a bad transcription before the agent acts.
+     */
+    private transcribeVoice;
     private routeMessage;
     private routeEngineEvent;
     private routeTextChunk;

package/dist/orchestrator.js

@@ -5,6 +5,7 @@ import { createSdkMcpServer, tool } from "@anthropic-ai/claude-agent-sdk";
 import { z } from "zod";
 import log from "./logger.js";
 import { saveFile } from "./files.js";
+import { isAudio, transcribeAudio } from "./transcribe.js";
 import { assemblePrompt } from "./prompt.js";
 import { formatPlanUsage, recordPlanUsage } from "./plan-usage.js";
 import { formatToolStatusLine, formatToolCallSummary, formatPermissionPrompt, formatTodoList, timeAgo } from "./format.js";
@@ -28,10 +29,12 @@ export class Orchestrator {
     creatingWorkspaces = new Set();
     scheduler = null;
     planUsage = new Map();
+    transcribe;
     constructor(opts) {
         this.channel = opts.channel;
         this.engines = opts.engines;
         this.config = opts.config;
+        this.transcribe = opts.transcribe ?? transcribeAudio;
     }
     chat(chatId) {
         let s = this.chats.get(chatId);
@@ -539,6 +542,33 @@ export class Orchestrator {
         await this.channel.sendMessage(msg.chatId, `Engine set to ${selected}.${changed ? " Session cleared." : ""}${handoffNote}`);
         await this.updateStatusMessage(msg.chatId, state);
     }
+    /**
+     * Turn audio attachments into message text. Engines can't consume audio, so it never
+     * reaches them; the echo lets the user catch a bad transcription before the agent acts.
+     */
+    async transcribeVoice(msg) {
+        const attachments = msg.attachments ?? [];
+        const rest = attachments.filter((att) => !isAudio(att));
+        let spoken;
+        try {
+            const transcripts = [];
+            for (const att of attachments.filter(isAudio))
+                transcripts.push(await this.transcribe(att));
+            spoken = transcripts.join("\n");
+            const echo = spoken.length > 1000 ? `${spoken.slice(0, 1000)}…` : spoken;
+            await this.channel.sendMessage(msg.chatId, `🎙 heard: ${echo}`, { format: "plain" })
+                .catch((err) => log.warn({ err }, "[orchestrator] failed to echo transcript"));
+        }
+        catch (err) {
+            log.warn({ err }, "[orchestrator] voice transcription failed");
+            spoken = "[Voice message — transcription failed]";
+        }
+        return {
+            ...msg,
+            text: msg.text ? `${msg.text}\n${spoken}` : spoken,
+            attachments: rest.length ? rest : undefined,
+        };
+    }
     async routeMessage(msg) {
         try {
             // routeMessage only handles channel-emitted messages, which are always user-originated.
@@ -547,6 +577,8 @@ export class Orchestrator {
             const { user } = msg.origin;
             if (!this.config.isAuthorized(user.id))
                 return;
+            if (msg.attachments?.some(isAudio))
+                msg = await this.transcribeVoice(msg);
             if (this.channel.isRootDM(msg.chatId, user.id) && !this.config.workspaceByChat(msg.chatId)) {
                 this.config.connectHomeWorkspace(msg.chatId);
             }

package/dist/prompt.js

@@ -1,12 +1,11 @@
-import fs from "node:fs";
 import fsp from "node:fs/promises";
 import path from "node:path";
 import log from "./logger.js";
 // A cloud-synced instructions dir (OneDrive/iCloud "dataless" placeholder) can
 // block a read indefinitely. Reading it synchronously froze the whole relay.
 const READ_TIMEOUT_MS = 5000;
-/** Read a file, giving up after READ_TIMEOUT_MS. Returns null on timeout or error. */
-async function readFileOrSkip(file) {
+/** Settle with `op`, or with null once READ_TIMEOUT_MS passes first. */
+async function withReadTimeout(op) {
     // ponytail: Promise.race, not AbortSignal — an abort can't interrupt a read
     // already blocked in libuv's threadpool, which is exactly what a
     // cloud-placeholder file does.
@@ -15,7 +14,16 @@ async function readFileOrSkip(file) {
         timer = setTimeout(() => resolve(null), READ_TIMEOUT_MS);
     });
     try {
-        const content = await Promise.race([fsp.readFile(file, "utf-8"), timeout]);
+        return await Promise.race([op, timeout]);
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+/** Read a file, giving up after READ_TIMEOUT_MS. Returns null on timeout or error. */
+async function readFileOrSkip(file) {
+    try {
+        const content = await withReadTimeout(fsp.readFile(file, "utf-8"));
         if (content === null) {
             log.warn("[prompt] timed out reading %s after %dms — skipping", file, READ_TIMEOUT_MS);
             return null;
@@ -26,26 +34,29 @@ async function readFileOrSkip(file) {
         log.warn({ err }, "[prompt] could not read %s — skipping", file);
         return null;
     }
-    finally {
-        clearTimeout(timer);
-    }
 }
 /**
  * Read all .md files from a directory, sorted alphabetically.
  * Unreadable files are reported by name instead of silently dropped.
  */
 async function readMarkdownFiles(dir) {
-    if (!fs.existsSync(dir))
-        return { contents: [], skipped: [] };
     let entries;
     try {
-        entries = await fsp.readdir(dir);
+        // Listing a cloud-synced dir can block just like reading one (seen with
+        // Google Drive under a launchd daemon before macOS granted access).
+        entries = await withReadTimeout(fsp.readdir(dir));
     }
     catch (err) {
+        if (err.code === "ENOENT")
+            return { contents: [], skipped: [] };
         // Cloud-synced dirs (Google Drive File Provider) can transiently refuse a listing with EPERM.
         log.warn({ err }, "[prompt] could not list %s — skipping", dir);
         return { contents: [], skipped: [dir] };
     }
+    if (entries === null) {
+        log.warn("[prompt] timed out listing %s after %dms — skipping", dir, READ_TIMEOUT_MS);
+        return { contents: [], skipped: [dir] };
+    }
     const names = entries.filter((f) => f.endsWith(".md")).sort();
     const results = await Promise.all(names.map(async (f) => ({ name: f, text: await readFileOrSkip(path.join(dir, f)) })));
     return {

package/dist/transcribe.d.ts

@@ -0,0 +1,13 @@
+import type { Attachment } from "./types.js";
+export declare function isAudio(att: Attachment): boolean;
+/** Interim/Text frames replace the current segment; Endpoint finalizes it. */
+export declare function reduceTranscriptFrames(frames: Array<{
+    type: string;
+    data?: string;
+}>): string;
+/** The service's last word on a stream is an Endpoint; anything else means text may still be coming. */
+export declare function isComplete(frames: Array<{
+    type: string;
+}>): boolean;
+/** Speech-to-text via the stream Claude Code's /voice uses, authenticated with the Claude.ai OAuth token. */
+export declare function transcribeAudio(att: Attachment): Promise<string>;

package/dist/transcribe.js

@@ -0,0 +1,157 @@
+import { execFileSync, spawn } from "node:child_process";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+const RATE = 16000;
+const FRAME_BYTES = RATE * 2 / 10; // 100 ms of s16le mono
+const STREAM_URL = "wss://api.anthropic.com/api/ws/speech_to_text/voice_stream?" + new URLSearchParams({
+    encoding: "linear16", sample_rate: String(RATE), channels: "1", endpointing_ms: "300", utterance_end_ms: "1000",
+    language: "en", use_conversation_engine: "true", stt_provider: "deepgram-nova3",
+});
+export function isAudio(att) {
+    return att.mimeType.startsWith("audio/");
+}
+/** Interim/Text frames replace the current segment; Endpoint finalizes it. */
+export function reduceTranscriptFrames(frames) {
+    const finals = [];
+    let current = "";
+    for (const f of frames) {
+        if (f.type === "TranscriptInterim" || f.type === "TranscriptText")
+            current = f.data ?? "";
+        else if (f.type === "TranscriptEndpoint") {
+            if (current)
+                finals.push(current);
+            current = "";
+        }
+    }
+    return [...finals, current].filter(Boolean).join(" ").trim();
+}
+/** The service's last word on a stream is an Endpoint; anything else means text may still be coming. */
+export function isComplete(frames) {
+    return frames.at(-1)?.type === "TranscriptEndpoint";
+}
+/** Speech-to-text via the stream Claude Code's /voice uses, authenticated with the Claude.ai OAuth token. */
+export async function transcribeAudio(att) {
+    const token = readClaudeToken();
+    if (!token)
+        throw new Error("no Claude.ai credentials");
+    const pcm = await decodeToPcm16k(att.buffer);
+    const text = await voiceStream(pcm, token);
+    if (!text)
+        throw new Error("empty transcript");
+    return text;
+}
+function readClaudeToken() {
+    try {
+        const raw = process.platform === "darwin"
+            ? execFileSync("security", ["find-generic-password", "-s", "Claude Code-credentials", "-w"], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"], timeout: 5000 })
+            : fs.readFileSync(path.join(os.homedir(), ".claude", ".credentials.json"), "utf8");
+        const oauth = JSON.parse(raw).claudeAiOauth;
+        if (!oauth?.accessToken || !(oauth.expiresAt > Date.now()))
+            return null;
+        return oauth.accessToken;
+    }
+    catch {
+        return null;
+    }
+}
+// The service requires an explicit PCM encoding; it does not accept OGG/Opus.
+function decodeToPcm16k(input) {
+    return new Promise((resolve, reject) => {
+        const ff = spawn("ffmpeg", ["-loglevel", "error", "-i", "pipe:0", "-f", "s16le", "-ac", "1", "-ar", String(RATE), "pipe:1"], { timeout: 30_000 });
+        const out = [];
+        let stderr = "";
+        ff.stdout.on("data", (c) => out.push(c));
+        ff.stderr.on("data", (c) => (stderr += c));
+        ff.on("error", (e) => reject(new Error(`ffmpeg failed: ${e.message}`)));
+        ff.on("close", (code) => {
+            const pcm = Buffer.concat(out);
+            if (code === 0 && pcm.length > 0)
+                resolve(pcm);
+            else
+                reject(new Error(`ffmpeg failed: ${stderr.trim().split("\n").pop() || `exit ${code}`}`));
+        });
+        ff.stdin.on("error", () => { }); // ffmpeg can exit before reading all input; "close" reports why
+        ff.stdin.end(input);
+    });
+}
+function voiceStream(pcm, token) {
+    // The service trims ~0.5 s at each end; pad with silence so the first and last words survive.
+    const pad = Buffer.alloc(RATE); // 0.5 s
+    const audio = Buffer.concat([pad, pcm, pad]);
+    const durationMs = (pcm.length / 2 / RATE) * 1000;
+    return new Promise((resolve, reject) => {
+        const ws = new WebSocket(STREAM_URL, { headers: {
+                Authorization: `Bearer ${token}`, "User-Agent": "claude-cli/2.1.284 (external, cli)",
+                "x-app": "cli", "anthropic-client-platform": "claude_code_cli",
+            } }); // Node (undici) accepts { headers }; the DOM typings do not
+        ws.binaryType = "arraybuffer";
+        const frames = [];
+        let closing = false;
+        let done = false;
+        let keepalive;
+        let idle;
+        let safety;
+        const finish = (err) => {
+            if (done)
+                return;
+            done = true;
+            clearInterval(keepalive);
+            clearTimeout(idle);
+            clearTimeout(safety);
+            try {
+                ws.close();
+            }
+            catch { /* already closed */ }
+            if (err)
+                reject(err);
+            else
+                resolve(reduceTranscriptFrames(frames));
+        };
+        // Finish on idle rather than on the first Endpoint after CloseStream: with unpaced
+        // sending, that endpoint can arrive before the tail of the audio is transcribed.
+        // A partial transcript is never a success: an incomplete stream waits for the safety timeout.
+        const bump = () => {
+            if (!closing)
+                return;
+            clearTimeout(idle);
+            idle = setTimeout(() => { if (isComplete(frames))
+                finish(); }, 1500);
+        };
+        safety = setTimeout(() => finish(new Error("timed out")), 15_000 + durationMs);
+        ws.onopen = () => {
+            const keepAlive = () => ws.send(JSON.stringify({ type: "KeepAlive" }));
+            keepAlive();
+            keepalive = setInterval(keepAlive, 8000);
+            for (let off = 0; off < audio.length; off += FRAME_BYTES)
+                ws.send(audio.subarray(off, off + FRAME_BYTES));
+            closing = true;
+            ws.send(JSON.stringify({ type: "CloseStream" }));
+            bump();
+        };
+        ws.onmessage = (ev) => {
+            let msg;
+            try {
+                msg = JSON.parse(typeof ev.data === "string" ? ev.data : Buffer.from(ev.data).toString());
+            }
+            catch {
+                return;
+            }
+            bump();
+            if (msg.type === "TranscriptError" || msg.type === "error") {
+                finish(new Error(`voice stream error: ${JSON.stringify(msg.data ?? msg).slice(0, 200)}`));
+            }
+            else if (msg.type) {
+                frames.push({ type: msg.type, data: typeof msg.data === "string" ? msg.data : undefined });
+            }
+        };
+        ws.onerror = () => finish(new Error("voice stream error: connection failed"));
+        ws.onclose = (ev) => {
+            if (closing && isComplete(frames))
+                finish();
+            else
+                finish(new Error(`voice stream error: closed ${ev.code} ${ev.reason}`.trim()));
+        };
+    });
+}
+//# sourceMappingURL=transcribe.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clearclaw",
-  "version": "0.8.0",
+  "version": "0.9.0",
   "description": "Your coding agent, from your phone — personal assistant + project workspaces on your machine, built on the CLI's permissions, config, and memory instead of replacing them.",
   "type": "module",
   "license": "MIT",