@rubytech/taskmaster 1.2.1 → 1.4.0

package/dist/gateway/server-methods/chat.js

@@ -379,8 +379,12 @@ export const chatHandlers = {
                 }
             }
         }
-        // Save document attachments to workspace uploads dir (persistent, accessible by agent)
+        // Save document attachments to workspace uploads dir (persistent, accessible by agent).
+        // Audio files are separated so they can be routed through the media understanding
+        // pipeline (STT) instead of being treated as generic file attachments.
         const savedDocPaths = [];
+        const savedAudioPaths = [];
+        const savedAudioTypes = [];
         if (documentAttachments.length > 0 && uploadsDir) {
             for (const doc of documentAttachments) {
                 if (!doc.content || typeof doc.content !== "string")
@@ -389,7 +393,14 @@ export const chatHandlers = {
                 const destPath = path.join(uploadsDir, safeName);
                 try {
                     fs.writeFileSync(destPath, Buffer.from(doc.content, "base64"));
-                    savedDocPaths.push(destPath);
+                    const mimeBase = doc.mimeType?.split(";")[0]?.trim() ?? "";
+                    if (mimeBase.startsWith("audio/")) {
+                        savedAudioPaths.push(destPath);
+                        savedAudioTypes.push(doc.mimeType ?? "audio/webm");
+                    }
+                    else {
+                        savedDocPaths.push(destPath);
+                    }
                 }
                 catch (err) {
                     context.logGateway.warn(`chat document save failed: ${String(err)}`);
@@ -460,18 +471,29 @@ export const chatHandlers = {
             const trimmedMessage = p.message.trim();
             const injectThinking = Boolean(p.thinking && trimmedMessage && !trimmedMessage.startsWith("/"));
             const commandBody = injectThinking ? `/think ${p.thinking} ${p.message}` : p.message;
-            // If documents were saved, prepend file paths to message so the agent knows about them
+            // If non-audio documents were saved, prepend file paths to message.
+            // Audio files are NOT annotated here — they go through MediaPaths so the
+            // media understanding pipeline (STT) handles them, and buildInboundMediaNote
+            // generates the proper [media attached: ...] annotation.
             const docNote = savedDocPaths.length > 0
                 ? savedDocPaths.map((p) => `[file: ${p}]`).join("\n") + "\n\n"
                 : "";
-            const messageWithDocs = docNote + p.message;
+            // Audio-only message (voice note, no text): use placeholder so
+            // applyMediaUnderstanding knows to replace with transcript or error.
+            const hasAudioMedia = savedAudioPaths.length > 0;
+            const effectiveBody = hasAudioMedia && !trimmedMessage ? "<media:audio>" : p.message;
+            const messageWithDocs = docNote + effectiveBody;
+            const effectiveCommandBody = hasAudioMedia && !trimmedMessage ? "<media:audio>" : commandBody;
+            // Merge image and audio paths so the media understanding pipeline sees both.
+            const allMediaPaths = [...savedImagePaths, ...savedAudioPaths];
+            const allMediaTypes = [...savedImageTypes, ...savedAudioTypes];
             const clientInfo = client?.connect?.client;
             const ctx = {
                 Body: messageWithDocs,
                 BodyForAgent: messageWithDocs,
-                BodyForCommands: docNote + commandBody,
+                BodyForCommands: docNote + effectiveCommandBody,
                 RawBody: messageWithDocs,
-                CommandBody: docNote + commandBody,
+                CommandBody: docNote + effectiveCommandBody,
                 SessionKey: p.sessionKey,
                 Provider: INTERNAL_MESSAGE_CHANNEL,
                 Surface: INTERNAL_MESSAGE_CHANNEL,
@@ -485,10 +507,10 @@ export const chatHandlers = {
                 // Image/media paths — same pattern as WhatsApp. buildInboundMediaNote()
                 // will generate [media attached: ...] annotations that the agent runner
                 // detects and loads from disk at inference time.
-                MediaPaths: savedImagePaths.length > 0 ? savedImagePaths : undefined,
-                MediaPath: savedImagePaths[0],
-                MediaTypes: savedImageTypes.length > 0 ? savedImageTypes : undefined,
-                MediaType: savedImageTypes[0],
+                MediaPaths: allMediaPaths.length > 0 ? allMediaPaths : undefined,
+                MediaPath: allMediaPaths[0],
+                MediaTypes: allMediaTypes.length > 0 ? allMediaTypes : undefined,
+                MediaType: allMediaTypes[0],
             };
             const agentId = resolveSessionAgentId({
                 sessionKey: p.sessionKey,
@@ -496,16 +518,26 @@ export const chatHandlers = {
             });
             // Fire message:inbound hook for conversation archiving.
             // Include image paths so the archive references the attached media.
+            // Audio archive is deferred until after media understanding resolves (see
+            // onMediaResolved below) so the transcript is available instead of the
+            // raw <media:audio> placeholder.
             const imageNote = savedImagePaths.length > 0 ? savedImagePaths.map((ip) => `[image: ${ip}]`).join("\n") : "";
-            const archiveText = [p.message, imageNote].filter(Boolean).join("\n").trim();
-            void triggerInternalHook(createInternalHookEvent("message", "inbound", p.sessionKey, {
-                text: archiveText || undefined,
-                timestamp: now,
-                chatType: "direct",
-                agentId,
-                channel: "webchat",
-                cfg,
-            }));
+            const fireArchiveHook = (resolvedBody) => {
+                const body = resolvedBody ?? p.message;
+                const archiveText = [body, imageNote].filter(Boolean).join("\n").trim();
+                void triggerInternalHook(createInternalHookEvent("message", "inbound", p.sessionKey, {
+                    text: archiveText || undefined,
+                    timestamp: now,
+                    chatType: "direct",
+                    agentId,
+                    channel: "webchat",
+                    cfg,
+                }));
+            };
+            if (!hasAudioMedia) {
+                // No audio — fire immediately (no STT to wait for).
+                fireArchiveHook();
+            }
             let prefixContext = {
                 identityName: resolveIdentityName(cfg, agentId),
             };
@@ -541,7 +573,7 @@ export const chatHandlers = {
                 },
             });
             let agentRunStarted = false;
-            context.logGateway.info(`webchat dispatch: sessionKey=${p.sessionKey} runId=${clientRunId} body=${messageWithDocs.length}ch images=${savedImagePaths.length} docs=${savedDocPaths.length}`);
+            context.logGateway.info(`webchat dispatch: sessionKey=${p.sessionKey} runId=${clientRunId} body=${messageWithDocs.length}ch images=${savedImagePaths.length} audio=${savedAudioPaths.length} docs=${savedDocPaths.length}`);
             void dispatchInboundMessage({
                 ctx,
                 cfg,
@@ -554,11 +586,18 @@ export const chatHandlers = {
                         agentRunStarted = true;
                         context.logGateway.info(`webchat agent run started: sessionKey=${p.sessionKey} runId=${runId}`);
                     },
-                    onModelSelected: (ctx) => {
-                        prefixContext.provider = ctx.provider;
-                        prefixContext.model = extractShortModelName(ctx.model);
-                        prefixContext.modelFull = `${ctx.provider}/${ctx.model}`;
-                        prefixContext.thinkingLevel = ctx.thinkLevel ?? "off";
+                    onMediaResolved: hasAudioMedia
+                        ? () => {
+                            // STT complete — archive the resolved body (transcript) instead
+                            // of the raw <media:audio> placeholder.
+                            fireArchiveHook(ctx.Body);
+                        }
+                        : undefined,
+                    onModelSelected: (modelCtx) => {
+                        prefixContext.provider = modelCtx.provider;
+                        prefixContext.model = extractShortModelName(modelCtx.model);
+                        prefixContext.modelFull = `${modelCtx.provider}/${modelCtx.model}`;
+                        prefixContext.thinkingLevel = modelCtx.thinkLevel ?? "off";
                     },
                 },
             })

package/dist/gateway/server-methods/tts.js

@@ -80,8 +80,11 @@ export const ttsHandlers = {
     },
     "tts.setProvider": async ({ params, respond }) => {
         const provider = typeof params.provider === "string" ? params.provider.trim() : "";
-        if (provider !== "openai" && provider !== "elevenlabs" && provider !== "edge") {
-            respond(false, undefined, errorShape(ErrorCodes.INVALID_REQUEST, "Invalid provider. Use openai, elevenlabs, or edge."));
+        if (provider !== "openai" &&
+            provider !== "elevenlabs" &&
+            provider !== "edge" &&
+            provider !== "hume") {
+            respond(false, undefined, errorShape(ErrorCodes.INVALID_REQUEST, "Invalid provider. Use openai, elevenlabs, hume, or edge."));
             return;
         }
         try {
@@ -115,6 +118,12 @@ export const ttsHandlers = {
                         configured: Boolean(resolveTtsApiKey(config, "elevenlabs")),
                         models: ["eleven_multilingual_v2", "eleven_turbo_v2_5", "eleven_monolingual_v1"],
                     },
+                    {
+                        id: "hume",
+                        name: "Hume",
+                        configured: Boolean(resolveTtsApiKey(config, "hume")),
+                        models: [],
+                    },
                     {
                         id: "edge",
                         name: "Edge TTS",

package/dist/gateway/server.impl.js

@@ -9,6 +9,7 @@ import { CONFIG_PATH_TASKMASTER, isNixMode, loadConfig, migrateLegacyConfig, rea
 import { VERSION } from "../version.js";
 import { isDiagnosticsEnabled } from "../infra/diagnostic-events.js";
 import { logAcceptedEnvOption } from "../infra/env.js";
+import { reconcileAgentContactTools } from "../config/agent-tools-reconcile.js";
 import { applyPluginAutoEnable } from "../config/plugin-auto-enable.js";
 import { clearAgentRunContext, onAgentEvent } from "../infra/agent-events.js";
 import { onHeartbeatEvent } from "../infra/heartbeat-events.js";
@@ -53,6 +54,7 @@ import { ensureWatchdogUnitOnStartup, scheduleWatchdogStabilityConfirmation, } f
 import { startGatewayTailscaleExposure } from "./server-tailscale.js";
 import { startWifiWatchdog } from "./server-wifi-watchdog.js";
 import { loadGatewayTlsRuntime } from "./server/tls.js";
+import { isLoopbackHost } from "./net.js";
 import { createWizardSessionTracker } from "./server-wizard-sessions.js";
 import { attachGatewayWsHandlers } from "./server-ws-runtime.js";
 import { isLicenseValid } from "../license/validate.js";
@@ -121,6 +123,20 @@ export async function startGatewayServer(port = 18789, opts = {}) {
             log.warn(`gateway: failed to persist plugin auto-enable changes: ${String(err)}`);
         }
     }
+    // Reconcile agent tool groups (e.g. individual contact tools → group:contacts).
+    const toolReconcile = reconcileAgentContactTools({ config: configSnapshot.config });
+    if (toolReconcile.changes.length > 0) {
+        try {
+            await writeConfigFile(toolReconcile.config);
+            configSnapshot = await readConfigFileSnapshot();
+            log.info(`gateway: reconciled agent tools:\n${toolReconcile.changes
+                .map((entry) => `- ${entry}`)
+                .join("\n")}`);
+        }
+        catch (err) {
+            log.warn(`gateway: failed to persist agent tools reconciliation: ${String(err)}`);
+        }
+    }
     // Stamp config with running version on startup so upgrades keep the stamp current.
     const storedVersion = configSnapshot.config.meta?.lastTouchedVersion;
     if (configSnapshot.exists && storedVersion !== VERSION) {
@@ -211,10 +227,30 @@ export async function startGatewayServer(port = 18789, opts = {}) {
     const { wizardSessions, findRunningWizard, purgeWizardSession } = createWizardSessionTracker();
     const deps = createDefaultDeps();
     let canvasHostServer = null;
-    const gatewayTls = await loadGatewayTlsRuntime(cfgAtStart.gateway?.tls, log.child("tls"));
-    if (cfgAtStart.gateway?.tls?.enabled && !gatewayTls.enabled) {
+    // Auto-enable TLS when binding to a non-loopback address (LAN, custom, etc.)
+    // so that browser secure-context APIs (getUserMedia, etc.) work over .local.
+    // Only auto-enable when the user hasn't explicitly configured tls.enabled.
+    const tlsExplicit = cfgAtStart.gateway?.tls?.enabled;
+    const tlsAutoEnable = tlsExplicit === undefined && !isLoopbackHost(bindHost);
+    const effectiveTlsConfig = tlsAutoEnable
+        ? { ...cfgAtStart.gateway?.tls, enabled: true }
+        : cfgAtStart.gateway?.tls;
+    if (tlsAutoEnable) {
+        log.child("tls").info("gateway tls: auto-enabled for non-loopback bind");
+    }
+    const bonjourHostname = cfgAtStart.discovery?.bonjourHostname || "taskmaster";
+    const tlsHostnames = [bonjourHostname];
+    const gatewayTls = await loadGatewayTlsRuntime(effectiveTlsConfig, log.child("tls"), tlsHostnames);
+    if (tlsExplicit === true && !gatewayTls.enabled) {
+        // User explicitly enabled TLS — fail hard if it can't start.
         throw new Error(gatewayTls.error ?? "gateway tls: failed to enable");
     }
+    if (tlsAutoEnable && !gatewayTls.enabled) {
+        // Auto-enabled TLS failed — fall back to HTTP with a warning.
+        log
+            .child("tls")
+            .warn(`gateway tls: auto-enable failed (${gatewayTls.error ?? "unknown"}), continuing with HTTP`);
+    }
     const { canvasHost, httpServer, httpServers, httpBindHosts, wss, clients, broadcast, agentRunSeq, dedupe, chatRunState, chatRunBuffers, chatDeltaSentAt, addChatRun, removeChatRun, chatAbortControllers, } = await createGatewayRuntimeState({
         cfg: cfgAtStart,
         bindHost,
@@ -268,9 +304,6 @@ export async function startGatewayServer(port = 18789, opts = {}) {
     });
     const { getRuntimeSnapshot, startChannels, startChannel, stopChannel, markChannelLoggedOut } = channelManager;
     const machineDisplayName = await getMachineDisplayName();
-    // Default to "taskmaster" hostname for mDNS so taskmaster.local works out of the box.
-    // Users can override via discovery.bonjourHostname config if needed.
-    const bonjourHostname = cfgAtStart.discovery?.bonjourHostname || "taskmaster";
     const discovery = await startGatewayDiscovery({
         machineDisplayName,
         port,

package/dist/infra/tls/gateway.js

@@ -1,6 +1,7 @@
 import { execFile } from "node:child_process";
 import { X509Certificate } from "node:crypto";
 import fs from "node:fs/promises";
+import os from "node:os";
 import path from "node:path";
 import { promisify } from "node:util";
 import { CONFIG_DIR, ensureDir, resolveUserPath, shortenHomeInString } from "../../utils.js";
@@ -15,6 +16,18 @@ async function fileExists(filePath) {
         return false;
     }
 }
+function buildSanString(hostnames) {
+    const sans = new Set(["DNS:localhost", "IP:127.0.0.1"]);
+    const raw = hostnames?.length ? hostnames : [os.hostname()];
+    for (const h of raw) {
+        const name = h.replace(/\.local$/i, "").trim();
+        if (!name)
+            continue;
+        sans.add(`DNS:${name}`);
+        sans.add(`DNS:${name}.local`);
+    }
+    return [...sans].join(",");
+}
 async function generateSelfSignedCert(params) {
     const certDir = path.dirname(params.certPath);
     const keyDir = path.dirname(params.keyPath);
@@ -22,6 +35,7 @@ async function generateSelfSignedCert(params) {
     if (keyDir !== certDir) {
         await ensureDir(keyDir);
     }
+    const san = buildSanString(params.hostnames);
     await execFileAsync("openssl", [
         "req",
         "-x509",
@@ -37,12 +51,14 @@ async function generateSelfSignedCert(params) {
         params.certPath,
         "-subj",
         "/CN=taskmaster-gateway",
+        "-addext",
+        `subjectAltName=${san}`,
     ]);
     await fs.chmod(params.keyPath, 0o600).catch(() => { });
     await fs.chmod(params.certPath, 0o600).catch(() => { });
-    params.log?.info?.(`gateway tls: generated self-signed cert at ${shortenHomeInString(params.certPath)}`);
+    params.log?.info?.(`gateway tls: generated self-signed cert at ${shortenHomeInString(params.certPath)} (SAN: ${san})`);
 }
-export async function loadGatewayTlsRuntime(cfg, log) {
+export async function loadGatewayTlsRuntime(cfg, log, hostnames) {
     if (!cfg || cfg.enabled !== true)
         return { enabled: false, required: false };
     const autoGenerate = cfg.autoGenerate !== false;
@@ -54,7 +70,7 @@ export async function loadGatewayTlsRuntime(cfg, log) {
     const hasKey = await fileExists(keyPath);
     if (!hasCert && !hasKey && autoGenerate) {
         try {
-            await generateSelfSignedCert({ certPath, keyPath, log });
+            await generateSelfSignedCert({ certPath, keyPath, hostnames, log });
         }
         catch (err) {
             return {

package/dist/media-understanding/apply.js

@@ -1,4 +1,5 @@
 import { finalizeInboundContext } from "../auto-reply/reply/inbound-context.js";
+import { logVerbose } from "../globals.js";
 import { extractMediaUserText, formatAudioTranscripts, formatMediaUnderstandingBody, } from "./format.js";
 import { runWithConcurrency } from "./concurrency.js";
 import { resolveConcurrency } from "./resolve.js";
@@ -42,6 +43,40 @@ export async function applyMediaUnderstanding(params) {
         if (decisions.length > 0) {
             ctx.MediaUnderstandingDecisions = [...(ctx.MediaUnderstandingDecisions ?? []), ...decisions];
         }
+        // Surface audio failures so the agent can inform the user instead of receiving
+        // a bare <media:audio> placeholder with no context about what went wrong.
+        const audioDecision = decisions.find((d) => d.capability === "audio");
+        const audioTranscribed = outputs.some((o) => o.kind === "audio.transcription");
+        const bodyHint = ctx.CommandBody ?? ctx.RawBody ?? ctx.Body ?? "";
+        const isAudioPlaceholder = /^<media:audio>/i.test(bodyHint.trim());
+        if (isAudioPlaceholder && !audioTranscribed) {
+            let reason;
+            if (ctx.MediaDownloadFailed) {
+                reason = "media download failed — the voice note could not be retrieved from WhatsApp";
+            }
+            else if (audioDecision?.outcome === "no-attachment") {
+                reason = "no audio file available for transcription";
+            }
+            else if (audioDecision?.outcome === "skipped") {
+                // Distinguish between "no providers at all" (empty attempts) and "providers tried but all failed"
+                const hasAttempts = audioDecision.attachments?.some((a) => a.attempts.length > 0);
+                reason = hasAttempts
+                    ? "all transcription attempts failed"
+                    : "no transcription provider configured (add an OpenAI, Google, Groq, or Deepgram API key)";
+            }
+            else if (audioDecision?.outcome === "disabled") {
+                reason = "audio transcription is disabled in config";
+            }
+            else {
+                reason = `transcription ${audioDecision?.outcome ?? "unavailable"}`;
+            }
+            const note = `[Voice note received but could not be transcribed: ${reason}]`;
+            logVerbose(`applyMediaUnderstanding: ${note}`);
+            ctx.Body = note;
+            ctx.CommandBody = note;
+            ctx.RawBody = note;
+            finalizeInboundContext(ctx, { forceBodyForAgent: true, forceBodyForCommands: true });
+        }
         if (outputs.length > 0) {
             ctx.Body = formatMediaUnderstandingBody({ body: ctx.Body, outputs });
             const audioOutputs = outputs.filter((output) => output.kind === "audio.transcription");

package/dist/media-understanding/providers/deepgram/audio.js

@@ -22,7 +22,7 @@ export async function transcribeDeepgramAudio(params) {
     }
     const headers = new Headers(params.headers);
     if (!headers.has("authorization")) {
-        headers.set("authorization", `Token ${params.apiKey}`);
+        headers.set("authorization", `Token ${params.apiKey ?? ""}`);
     }
     if (!headers.has("content-type")) {
         headers.set("content-type", params.mime ?? "application/octet-stream");

package/dist/media-understanding/providers/google/audio.js

@@ -23,7 +23,7 @@ export async function transcribeGeminiAudio(params) {
         headers.set("content-type", "application/json");
     }
     if (!headers.has("x-goog-api-key")) {
-        headers.set("x-goog-api-key", params.apiKey);
+        headers.set("x-goog-api-key", params.apiKey ?? "");
     }
     const body = {
         contents: [

package/dist/media-understanding/providers/google/video.js

@@ -23,7 +23,7 @@ export async function describeGeminiVideo(params) {
         headers.set("content-type", "application/json");
     }
     if (!headers.has("x-goog-api-key")) {
-        headers.set("x-goog-api-key", params.apiKey);
+        headers.set("x-goog-api-key", params.apiKey ?? "");
     }
     const body = {
         contents: [

package/dist/media-understanding/providers/index.js

@@ -5,6 +5,7 @@ import { googleProvider } from "./google/index.js";
 import { groqProvider } from "./groq/index.js";
 import { minimaxProvider } from "./minimax/index.js";
 import { openaiProvider } from "./openai/index.js";
+import { sherpaOnnxProvider } from "./sherpa-onnx/index.js";
 const PROVIDERS = [
     groqProvider,
     openaiProvider,
@@ -12,6 +13,7 @@ const PROVIDERS = [
     anthropicProvider,
     minimaxProvider,
     deepgramProvider,
+    sherpaOnnxProvider,
 ];
 export function normalizeMediaProviderId(id) {
     const normalized = normalizeProviderId(id);

package/dist/media-understanding/providers/openai/audio.js

@@ -25,7 +25,7 @@ export async function transcribeOpenAiCompatibleAudio(params) {
         form.append("prompt", params.prompt.trim());
     const headers = new Headers(params.headers);
     if (!headers.has("authorization")) {
-        headers.set("authorization", `Bearer ${params.apiKey}`);
+        headers.set("authorization", `Bearer ${params.apiKey ?? ""}`);
     }
     const res = await fetchWithTimeout(url, {
         method: "POST",

package/dist/media-understanding/providers/sherpa-onnx/index.js

@@ -0,0 +1,10 @@
+import { transcribeLocal, MODEL_LABEL } from "../../sherpa-onnx-local.js";
+export const sherpaOnnxProvider = {
+    id: "sherpa-onnx",
+    isLocal: true,
+    capabilities: ["audio"],
+    transcribeAudio: async (req) => {
+        const result = await transcribeLocal(req.buffer, req.fileName);
+        return { text: result.text, model: result.model ?? MODEL_LABEL };
+    },
+};