@swarmclawai/swarmclaw 0.6.3 → 0.6.6

package/src/lib/server/connectors/inbound-audio-transcription.test.ts

@@ -0,0 +1,191 @@
+import { afterEach, beforeEach, describe, it } from 'node:test'
+import assert from 'node:assert/strict'
+import fs from 'node:fs'
+import path from 'node:path'
+import { enrichInboundMessageWithAudioTranscript } from './inbound-audio-transcription'
+import type { InboundMessage } from './types'
+import { UPLOAD_DIR, loadSettings, saveSettings } from '../storage'
+
+const ENV_KEYS = [
+  'OPENAI_API_KEY',
+  'SWARMCLAW_OPENAI_STT_API_KEY',
+  'SWARMCLAW_OPENAI_STT_BASE_URL',
+  'SWARMCLAW_OPENAI_STT_MODEL',
+  'SWARMCLAW_ELEVENLABS_STT_MODEL',
+  'SWARMCLAW_CONNECTOR_AUDIO_TRANSCRIBE',
+  'SWARMCLAW_CONNECTOR_AUDIO_TRANSCRIBE_TIMEOUT_MS',
+  'SWARMCLAW_CONNECTOR_AUDIO_TRANSCRIBE_MAX_BYTES',
+  'ELEVENLABS_API_KEY',
+] as const
+
+type EnvSnapshot = Record<(typeof ENV_KEYS)[number], string | undefined>
+
+let originalFetch: typeof fetch
+let originalSettings: Record<string, unknown>
+let originalEnv: EnvSnapshot
+let tempFiles: string[] = []
+
+function setEnv(name: (typeof ENV_KEYS)[number], value: string | undefined): void {
+  if (value === undefined) delete process.env[name]
+  else process.env[name] = value
+}
+
+function createAudioFixture(name: string): string {
+  fs.mkdirSync(UPLOAD_DIR, { recursive: true })
+  const filePath = path.join(UPLOAD_DIR, `${Date.now()}-${name}.ogg`)
+  fs.writeFileSync(filePath, Buffer.from('voice-note-bytes'))
+  tempFiles.push(filePath)
+  return filePath
+}
+
+function buildInboundMessage(localPath: string, text = '(media message)'): InboundMessage {
+  return {
+    platform: 'whatsapp',
+    channelId: '15550001111@s.whatsapp.net',
+    senderId: '15550001111@s.whatsapp.net',
+    senderName: 'Tester',
+    text,
+    media: [{ type: 'audio', localPath, mimeType: 'audio/ogg', fileName: 'voice.ogg' }],
+  }
+}
+
+beforeEach(() => {
+  originalFetch = global.fetch
+  originalSettings = loadSettings()
+  originalEnv = Object.fromEntries(
+    ENV_KEYS.map((key) => [key, process.env[key]]),
+  ) as EnvSnapshot
+  tempFiles = []
+})
+
+afterEach(() => {
+  global.fetch = originalFetch
+  saveSettings(originalSettings)
+  for (const key of ENV_KEYS) setEnv(key, originalEnv[key])
+  for (const filePath of tempFiles) fs.rmSync(filePath, { force: true })
+})
+
+describe('enrichInboundMessageWithAudioTranscript', () => {
+  it('transcribes placeholder audio messages with OpenAI STT', async () => {
+    const audioPath = createAudioFixture('openai')
+    setEnv('OPENAI_API_KEY', 'openai-test-key')
+    setEnv('SWARMCLAW_CONNECTOR_AUDIO_TRANSCRIBE_TIMEOUT_MS', '5000')
+    saveSettings({
+      ...originalSettings,
+      elevenLabsEnabled: false,
+      elevenLabsApiKey: null,
+    })
+
+    let called = 0
+    global.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
+      called += 1
+      const url = String(input)
+      assert.ok(url.endsWith('/audio/transcriptions'))
+      assert.equal(init?.method, 'POST')
+      assert.equal((init?.headers as Record<string, string>)?.Authorization, 'Bearer openai-test-key')
+      return new Response(JSON.stringify({ text: 'Please move this task to tomorrow morning.' }), {
+        status: 200,
+        headers: { 'Content-Type': 'application/json' },
+      })
+    }) as typeof fetch
+
+    const inbound = buildInboundMessage(audioPath)
+    const enriched = await enrichInboundMessageWithAudioTranscript({ msg: inbound })
+
+    assert.equal(called, 1)
+    assert.equal(enriched.text, 'Please move this task to tomorrow morning.')
+  })
+
+  it('tries ElevenLabs first and falls back to OpenAI when ElevenLabs fails', async () => {
+    const audioPath = createAudioFixture('fallback')
+    setEnv('OPENAI_API_KEY', 'openai-fallback-key')
+    saveSettings({
+      ...originalSettings,
+      elevenLabsEnabled: true,
+      elevenLabsApiKey: 'el-test-key',
+    })
+
+    const calledUrls: string[] = []
+    global.fetch = (async (input: RequestInfo | URL) => {
+      const url = String(input)
+      calledUrls.push(url)
+      if (url.includes('api.elevenlabs.io/v1/speech-to-text')) {
+        return new Response(JSON.stringify({ detail: 'upstream unavailable' }), {
+          status: 503,
+          headers: { 'Content-Type': 'application/json' },
+        })
+      }
+      if (url.endsWith('/audio/transcriptions')) {
+        return new Response(JSON.stringify({ text: 'Fallback transcription succeeded.' }), {
+          status: 200,
+          headers: { 'Content-Type': 'application/json' },
+        })
+      }
+      return new Response('unexpected url', { status: 404 })
+    }) as typeof fetch
+
+    const inbound = buildInboundMessage(audioPath)
+    const enriched = await enrichInboundMessageWithAudioTranscript({ msg: inbound })
+
+    assert.equal(enriched.text, 'Fallback transcription succeeded.')
+    assert.equal(calledUrls.length, 2)
+    assert.ok(calledUrls[0].includes('api.elevenlabs.io/v1/speech-to-text'))
+    assert.ok(calledUrls[1].endsWith('/audio/transcriptions'))
+  })
+
+  it('skips transcription when the inbound message already has non-placeholder text', async () => {
+    const audioPath = createAudioFixture('skip')
+    setEnv('OPENAI_API_KEY', 'openai-test-key')
+
+    let called = false
+    global.fetch = (async () => {
+      called = true
+      return new Response(JSON.stringify({ text: 'should not be used' }), {
+        status: 200,
+        headers: { 'Content-Type': 'application/json' },
+      })
+    }) as typeof fetch
+
+    const inbound = buildInboundMessage(audioPath, 'Already typed this manually')
+    const enriched = await enrichInboundMessageWithAudioTranscript({ msg: inbound })
+
+    assert.equal(enriched.text, 'Already typed this manually')
+    assert.equal(called, false)
+  })
+
+  it('returns a clear failure note when STT providers error out', async () => {
+    const audioPath = createAudioFixture('provider-error')
+    setEnv('OPENAI_API_KEY', 'openai-error-key')
+    saveSettings({
+      ...originalSettings,
+      elevenLabsEnabled: true,
+      elevenLabsApiKey: 'el-error-key',
+    })
+
+    global.fetch = (async () => {
+      return new Response(JSON.stringify({ error: 'upstream down' }), {
+        status: 500,
+        headers: { 'Content-Type': 'application/json' },
+      })
+    }) as typeof fetch
+
+    const inbound = buildInboundMessage(audioPath)
+    const enriched = await enrichInboundMessageWithAudioTranscript({ msg: inbound })
+
+    assert.ok(enriched.text.toLowerCase().includes('automatic transcription failed'))
+  })
+
+  it('returns a clear note when inbound audio cannot be loaded from disk', async () => {
+    const inbound: InboundMessage = {
+      platform: 'whatsapp',
+      channelId: '15550001111@s.whatsapp.net',
+      senderId: '15550001111@s.whatsapp.net',
+      senderName: 'Tester',
+      text: '(media message)',
+      media: [{ type: 'audio', localPath: '/tmp/nonexistent-voice-note.ogg', mimeType: 'audio/ogg', fileName: 'voice.ogg' }],
+    }
+
+    const enriched = await enrichInboundMessageWithAudioTranscript({ msg: inbound })
+    assert.ok(enriched.text.toLowerCase().includes('audio attachment could not be loaded'))
+  })
+})

package/src/lib/server/connectors/inbound-audio-transcription.ts

@@ -0,0 +1,261 @@
+import fs from 'node:fs'
+import path from 'node:path'
+import { decryptKey, loadCredentials, loadSettings } from '../storage'
+import { mimeFromPath } from './media'
+import type { InboundMessage, InboundMedia } from './types'
+
+const PLACEHOLDER_TEXT = new Set([
+  '',
+  '(media message)',
+  '(audio message)',
+  '(voice message)',
+  '<media:attachment>',
+])
+
+const DEFAULT_MAX_AUDIO_BYTES = 25 * 1024 * 1024
+const DEFAULT_TIMEOUT_MS = 30_000
+const TRANSCRIPTION_UNAVAILABLE_NOTE = '[Voice note received — automatic transcription is unavailable (no STT provider key configured).]'
+const TRANSCRIPTION_FAILED_NOTE = '[Voice note received — automatic transcription failed. Please check STT provider configuration/logs.]'
+const AUDIO_DOWNLOAD_FAILED_NOTE = '[Voice note received — audio attachment could not be loaded for transcription.]'
+
+function boolFromEnv(name: string, fallback: boolean): boolean {
+  const raw = String(process.env[name] || '').trim().toLowerCase()
+  if (!raw) return fallback
+  if (['1', 'true', 'yes', 'on', 'enabled'].includes(raw)) return true
+  if (['0', 'false', 'no', 'off', 'disabled'].includes(raw)) return false
+  return fallback
+}
+
+function numberFromEnv(name: string, fallback: number): number {
+  const raw = Number.parseInt(String(process.env[name] || '').trim(), 10)
+  if (!Number.isFinite(raw) || raw <= 0) return fallback
+  return raw
+}
+
+function normalizeLanguageCode(raw: unknown): string | undefined {
+  const normalized = typeof raw === 'string' ? raw.trim() : ''
+  if (!normalized) return undefined
+  const token = normalized.split(/[-_]/)[0]?.toLowerCase() || ''
+  return /^[a-z]{2,3}$/.test(token) ? token : undefined
+}
+
+function isAudioPlaceholder(text: string): boolean {
+  return PLACEHOLDER_TEXT.has(text.trim().toLowerCase())
+}
+
+function pickInboundAudio(msg: InboundMessage): InboundMedia | null {
+  if (!Array.isArray(msg.media) || msg.media.length === 0) return null
+  for (const media of msg.media) {
+    if (!media) continue
+    const isAudio = media.type === 'audio'
+      || (typeof media.mimeType === 'string' && media.mimeType.toLowerCase().startsWith('audio/'))
+    if (!isAudio) continue
+    const localPath = typeof media.localPath === 'string' ? media.localPath.trim() : ''
+    if (!localPath || !fs.existsSync(localPath)) continue
+    return media
+  }
+  return null
+}
+
+function hasInboundAudio(msg: InboundMessage): boolean {
+  if (!Array.isArray(msg.media) || msg.media.length === 0) return false
+  return msg.media.some((media) => media?.type === 'audio'
+    || (typeof media?.mimeType === 'string' && media.mimeType.toLowerCase().startsWith('audio/')))
+}
+
+function extractTranscriptText(payload: unknown): string {
+  if (!payload || typeof payload !== 'object') return ''
+  const obj = payload as Record<string, unknown>
+  if (typeof obj.text === 'string' && obj.text.trim()) return obj.text.trim()
+  if (Array.isArray(obj.transcripts)) {
+    const merged = obj.transcripts
+      .map((entry) => {
+        if (!entry || typeof entry !== 'object') return ''
+        const text = (entry as Record<string, unknown>).text
+        return typeof text === 'string' ? text.trim() : ''
+      })
+      .filter(Boolean)
+      .join(' ')
+      .trim()
+    if (merged) return merged
+  }
+  return ''
+}
+
+function resolveOpenAiApiKey(preferredCredentialId?: string | null): string | null {
+  const envKey = String(process.env.SWARMCLAW_OPENAI_STT_API_KEY || process.env.OPENAI_API_KEY || '').trim()
+  if (envKey) return envKey
+
+  const creds = loadCredentials() as Record<string, { provider?: string; encryptedKey?: string }>
+  const candidates: string[] = []
+  if (preferredCredentialId) candidates.push(preferredCredentialId)
+  for (const [id, cred] of Object.entries(creds)) {
+    const provider = String(cred?.provider || '').trim().toLowerCase()
+    if (provider === 'openai') candidates.push(id)
+  }
+  const seen = new Set<string>()
+  for (const id of candidates) {
+    if (!id || seen.has(id)) continue
+    seen.add(id)
+    const cred = creds[id]
+    const provider = String(cred?.provider || '').trim().toLowerCase()
+    if (provider !== 'openai') continue
+    if (!cred?.encryptedKey) continue
+    try {
+      const decrypted = decryptKey(cred.encryptedKey).trim()
+      if (decrypted) return decrypted
+    } catch { /* ignore invalid credential */ }
+  }
+
+  return null
+}
+
+function resolveElevenLabsKey(): string | null {
+  const settings = loadSettings()
+  const key = String(settings.elevenLabsApiKey || process.env.ELEVENLABS_API_KEY || '').trim()
+  return key || null
+}
+
+async function transcribeWithElevenLabs(params: {
+  apiKey: string
+  audioPath: string
+  fileName: string
+  mimeType: string
+  language?: string
+  timeoutMs: number
+}): Promise<string> {
+  const form = new FormData()
+  const modelId = String(process.env.SWARMCLAW_ELEVENLABS_STT_MODEL || 'scribe_v1').trim() || 'scribe_v1'
+  form.set('model_id', modelId)
+  if (params.language) form.set('language_code', params.language)
+  const fileBuffer = fs.readFileSync(params.audioPath)
+  const blob = new Blob([fileBuffer], { type: params.mimeType })
+  form.set('file', blob, params.fileName)
+
+  const response = await fetch('https://api.elevenlabs.io/v1/speech-to-text', {
+    method: 'POST',
+    headers: { 'xi-api-key': params.apiKey },
+    body: form,
+    signal: AbortSignal.timeout(params.timeoutMs),
+  })
+  if (!response.ok) {
+    const body = await response.text().catch(() => '')
+    throw new Error(`elevenlabs stt ${response.status}: ${body.slice(0, 160)}`)
+  }
+  const json = await response.json().catch(() => null)
+  return extractTranscriptText(json)
+}
+
+async function transcribeWithOpenAI(params: {
+  apiKey: string
+  audioPath: string
+  fileName: string
+  mimeType: string
+  language?: string
+  timeoutMs: number
+}): Promise<string> {
+  const form = new FormData()
+  const model = String(process.env.SWARMCLAW_OPENAI_STT_MODEL || 'gpt-4o-mini-transcribe').trim() || 'gpt-4o-mini-transcribe'
+  form.set('model', model)
+  if (params.language) form.set('language', params.language)
+  const fileBuffer = fs.readFileSync(params.audioPath)
+  const blob = new Blob([fileBuffer], { type: params.mimeType })
+  form.set('file', blob, params.fileName)
+
+  const base = String(process.env.SWARMCLAW_OPENAI_STT_BASE_URL || 'https://api.openai.com/v1').trim().replace(/\/+$/, '')
+  const response = await fetch(`${base}/audio/transcriptions`, {
+    method: 'POST',
+    headers: { Authorization: `Bearer ${params.apiKey}` },
+    body: form,
+    signal: AbortSignal.timeout(params.timeoutMs),
+  })
+  if (!response.ok) {
+    const body = await response.text().catch(() => '')
+    throw new Error(`openai stt ${response.status}: ${body.slice(0, 160)}`)
+  }
+  const json = await response.json().catch(() => null)
+  return extractTranscriptText(json)
+}
+
+/**
+ * Convert inbound audio media into text before routing to the agent.
+ * This prevents "(media message)" placeholders from reaching the model.
+ */
+export async function enrichInboundMessageWithAudioTranscript(params: {
+  msg: InboundMessage
+  preferredCredentialId?: string | null
+}): Promise<InboundMessage> {
+  const { preferredCredentialId } = params
+  const msg = params.msg
+  if (!boolFromEnv('SWARMCLAW_CONNECTOR_AUDIO_TRANSCRIBE', true)) return msg
+
+  const originalText = String(msg.text || '').trim()
+  if (!isAudioPlaceholder(originalText)) return msg
+
+  const inboundAudio = pickInboundAudio(msg)
+  if (!inboundAudio) {
+    if (hasInboundAudio(msg)) return { ...msg, text: AUDIO_DOWNLOAD_FAILED_NOTE }
+    return msg
+  }
+
+  const localPath = String(inboundAudio.localPath || '').trim()
+  if (!localPath || !fs.existsSync(localPath)) return { ...msg, text: AUDIO_DOWNLOAD_FAILED_NOTE }
+
+  const maxBytes = numberFromEnv('SWARMCLAW_CONNECTOR_AUDIO_TRANSCRIBE_MAX_BYTES', DEFAULT_MAX_AUDIO_BYTES)
+  const stat = fs.statSync(localPath)
+  if (!stat.isFile() || stat.size <= 0 || stat.size > maxBytes) {
+    return { ...msg, text: TRANSCRIPTION_FAILED_NOTE }
+  }
+
+  const mimeType = (inboundAudio.mimeType || mimeFromPath(localPath) || 'application/octet-stream').split(';')[0].trim()
+  const fileName = inboundAudio.fileName || path.basename(localPath)
+  const timeoutMs = numberFromEnv('SWARMCLAW_CONNECTOR_AUDIO_TRANSCRIBE_TIMEOUT_MS', DEFAULT_TIMEOUT_MS)
+  const language = normalizeLanguageCode(loadSettings().speechRecognitionLang)
+
+  const attempts: Array<{ provider: 'elevenlabs' | 'openai'; run: () => Promise<string> }> = []
+  const elevenKey = resolveElevenLabsKey()
+  if (elevenKey) {
+    attempts.push({
+      provider: 'elevenlabs',
+      run: () => transcribeWithElevenLabs({
+        apiKey: elevenKey,
+        audioPath: localPath,
+        fileName,
+        mimeType,
+        language,
+        timeoutMs,
+      }),
+    })
+  }
+
+  const openAiKey = resolveOpenAiApiKey(preferredCredentialId)
+  if (openAiKey) {
+    attempts.push({
+      provider: 'openai',
+      run: () => transcribeWithOpenAI({
+        apiKey: openAiKey,
+        audioPath: localPath,
+        fileName,
+        mimeType,
+        language,
+        timeoutMs,
+      }),
+    })
+  }
+
+  if (attempts.length === 0) return { ...msg, text: TRANSCRIPTION_UNAVAILABLE_NOTE }
+
+  for (const attempt of attempts) {
+    try {
+      const transcript = (await attempt.run()).replace(/\s+/g, ' ').trim()
+      if (!transcript) continue
+      console.log(`[connector] Inbound audio transcribed via ${attempt.provider}: ${path.basename(localPath)}`)
+      return { ...msg, text: transcript }
+    } catch (err: unknown) {
+      const reason = err instanceof Error ? err.message : String(err)
+      console.warn(`[connector] Inbound audio transcription failed via ${attempt.provider}: ${reason}`)
+    }
+  }
+
+  return { ...msg, text: TRANSCRIPTION_FAILED_NOTE }
+}

package/src/lib/server/connectors/manager.ts

@@ -16,12 +16,16 @@ import { requestHeartbeatNow } from '../heartbeat-wake'
 import { buildCurrentDateTimePromptContext } from '../prompt-runtime-context'
 import {
   parseMentions,
+  compactChatroomMessages,
   buildChatroomSystemPrompt,
   buildSyntheticSession,
   buildAgentSystemPromptForChatroom,
   buildHistoryForAgent,
   resolveApiKey as resolveApiKeyHelper,
 } from '../chatroom-helpers'
+import { filterHealthyChatroomAgents } from '../chatroom-health'
+import { markProviderFailure, markProviderSuccess } from '../provider-health'
+import { getProvider } from '@/lib/providers'
 import type { Connector, MessageSource, Chatroom, ChatroomMessage } from '@/types'
 import type { ConnectorInstance, InboundMessage, InboundMedia } from './types'
 import {
@@ -35,6 +39,7 @@ import {
   parsePairingPolicy,
   type PairingPolicy,
 } from './pairing'
+import { enrichInboundMessageWithAudioTranscript } from './inbound-audio-transcription'
 
 function resolveUploadPathFromUrl(rawUrl: string): string | null {
   if (!rawUrl) return null
@@ -657,10 +662,27 @@ async function routeMessageToChatroom(connector: Connector, msg: InboundMessage)
   if (!chatroom) return '[Error] Chatroom not found.'
 
   const agents = loadAgents()
+  const preferredCredentialId = (() => {
+    if (connector.agentId && agents[connector.agentId]?.credentialId) {
+      return agents[connector.agentId].credentialId as string
+    }
+    for (const agentId of chatroom.agentIds) {
+      const credentialId = agents[agentId]?.credentialId
+      if (credentialId) return credentialId as string
+    }
+    return null
+  })()
+  msg = await enrichInboundMessageWithAudioTranscript({
+    msg,
+    preferredCredentialId,
+  })
+
   const source: MessageSource = {
     platform: connector.platform,
     connectorId: connector.id,
     connectorName: connector.name,
+    channelId: msg.channelId,
+    senderId: msg.senderId,
     senderName: msg.senderName,
   }
   const inboundText = formatInboundUserText(msg)
@@ -673,6 +695,8 @@ async function routeMessageToChatroom(connector: Connector, msg: InboundMessage)
   if (chatroom.autoAddress && mentions.length === 0) {
     mentions = [...chatroom.agentIds]
   }
+  const mentionHealth = filterHealthyChatroomAgents(mentions, agents)
+  mentions = mentionHealth.healthyAgentIds
 
   // Create and persist the user message in the chatroom
   const userMessage: ChatroomMessage = {
@@ -689,12 +713,23 @@ async function routeMessageToChatroom(connector: Connector, msg: InboundMessage)
     source,
   }
   chatroom.messages.push(userMessage)
+  compactChatroomMessages(chatroom)
   chatroom.updatedAt = Date.now()
   chatrooms[chatroomId] = chatroom
   saveChatrooms(chatrooms)
   notify('chatrooms')
   notify(`chatroom:${chatroomId}`)
 
+  if (mentions.length === 0) {
+    if (mentionHealth.skipped.length > 0) {
+      const skippedSummary = mentionHealth.skipped
+        .map((row) => `${agents[row.agentId]?.name || row.agentId}: ${row.reason}`)
+        .join(', ')
+      return `[Error] No healthy agents were available for this request. Skipped: ${skippedSummary}`
+    }
+    return '[Error] No agents were selected for this request.'
+  }
+
   // Process mentioned agents sequentially and collect responses
   const responses: string[] = []
   for (const agentId of mentions) {
@@ -704,6 +739,23 @@ async function routeMessageToChatroom(connector: Connector, msg: InboundMessage)
     const apiKey = resolveApiKeyHelper(agent.credentialId)
     const freshChatrooms = loadChatrooms()
     const freshChatroom = freshChatrooms[chatroomId] as Chatroom
+    if (compactChatroomMessages(freshChatroom)) {
+      freshChatrooms[chatroomId] = freshChatroom
+      saveChatrooms(freshChatrooms)
+      notify(`chatroom:${chatroomId}`)
+    }
+
+    const providerInfo = getProvider(agent.provider)
+    if (providerInfo?.requiresApiKey && !apiKey) {
+      markProviderFailure(agent.provider, 'missing_api_credentials')
+      responses.push(`[${agent.name}] [Error] Missing API credentials.`)
+      continue
+    }
+    if (providerInfo?.requiresEndpoint && !agent.apiEndpoint) {
+      markProviderFailure(agent.provider, 'missing_api_endpoint')
+      responses.push(`[${agent.name}] [Error] Missing endpoint configuration.`)
+      continue
+    }
 
     const syntheticSession = buildSyntheticSession(agent, chatroomId)
     const agentSystemPrompt = buildAgentSystemPromptForChatroom(agent)
@@ -730,6 +782,7 @@ async function routeMessageToChatroom(connector: Connector, msg: InboundMessage)
           platform: connector.platform,
           connectorId: connector.id,
           connectorName: connector.name,
+          channelId: msg.channelId,
         }
         const agentMessage: ChatroomMessage = {
           id: genId(),
@@ -737,7 +790,10 @@ async function routeMessageToChatroom(connector: Connector, msg: InboundMessage)
           senderName: agent.name,
           role: 'assistant',
           text: responseText,
-          mentions: parseMentions(responseText, agents, freshChatroom.agentIds),
+          mentions: filterHealthyChatroomAgents(
+            parseMentions(responseText, agents, freshChatroom.agentIds),
+            agents,
+          ).healthyAgentIds,
           reactions: [],
           time: Date.now(),
           source: agentSource,
@@ -750,10 +806,14 @@ async function routeMessageToChatroom(connector: Connector, msg: InboundMessage)
         saveChatrooms(latestChatrooms)
         notify(`chatroom:${chatroomId}`)
 
+        markProviderSuccess(agent.provider)
         responses.push(`[${agent.name}] ${responseText}`)
+      } else {
+        markProviderSuccess(agent.provider)
       }
     } catch (err: unknown) {
       const errMsg = err instanceof Error ? err.message : String(err)
+      markProviderFailure(agent.provider, errMsg)
       console.error(`[connector] Chatroom agent ${agent.name} error:`, errMsg)
     }
   }
@@ -798,6 +858,10 @@ async function routeMessage(connector: Connector, msg: InboundMessage): Promise<
   if (!effectiveAgentId) return '[Error] Connector has no agent configured.'
   const agent = agents[effectiveAgentId]
   if (!agent) return '[Error] Connector agent not found.'
+  msg = await enrichInboundMessageWithAudioTranscript({
+    msg,
+    preferredCredentialId: agent.credentialId || null,
+  })
 
   // Enqueue system event + heartbeat wake for the agent
   const preview = (msg.text || '').slice(0, 80)
@@ -931,9 +995,14 @@ async function routeMessage(connector: Connector, msg: InboundMessage): Promise<
     return commandResult
   }
 
-  // Build system prompt: [userPrompt] \n\n [soul] \n\n [systemPrompt]
+  // Build system prompt: [identity] \n\n [userPrompt] \n\n [soul] \n\n [systemPrompt]
   const settings = loadSettings()
   const promptParts: string[] = []
+  // Identity block — agent needs to know who it is
+  const identityLines = [`## My Identity`, `My name is ${agent.name}.`]
+  if (agent.description) identityLines.push(agent.description)
+  identityLines.push('I should always refer to myself by this name. I am not "Assistant" — I have my own name and identity.')
+  promptParts.push(identityLines.join(' '))
   if (settings.userPrompt) promptParts.push(settings.userPrompt)
   promptParts.push(buildCurrentDateTimePromptContext())
   if (agent.soul) promptParts.push(agent.soul)
@@ -960,6 +1029,11 @@ Do not end every reply with a question.
 Only ask a question when a specific missing detail blocks progress.
 When a task is complete, state the result plainly and stop.
 
+## Async Update Routing
+When you start work that may finish later (task, schedule, delegated run), tell the user where updates will be sent.
+Default to this same ${msg.platform} chat unless the user requested another destination.
+If channel preference is ambiguous and there are multiple reasonable destinations, ask one short routing question.
+
 ## Knowing When Not to Reply
 Real conversations have natural pauses — not every message needs a response. Reply with exactly "NO_MESSAGE" (nothing else) to stay silent when replying would feel unnatural or forced.
 Stay silent for simple acknowledgments ("okay", "alright", "cool", "got it", "sounds good"), conversation closers ("thanks", "bye", "night", "ttyl"), reactions (emoji, "haha", "lol"), and forwarded content with no question attached.
@@ -987,6 +1061,8 @@ If media sending fails, report the exact error and retry with a corrected path/t
     platform: connector.platform,
     connectorId: connector.id,
     connectorName: connector.name,
+    channelId: msg.channelId,
+    senderId: msg.senderId,
     senderName: msg.senderName,
   }
   session.messages.push({
@@ -1002,6 +1078,7 @@ If media sending fails, report the exact error and retry with a corrected path/t
   const s1 = loadSessions()
   s1[session.id] = session
   saveSessions(s1)
+  notify(`messages:${session.id}`)
 
   // Stream the response
   let fullText = ''
@@ -1109,6 +1186,7 @@ If media sending fails, report the exact error and retry with a corrected path/t
     platform: connector.platform,
     connectorId: connector.id,
     connectorName: connector.name,
+    channelId: msg.channelId,
   }
   if (fullText.trim()) {
     session.messages.push({ role: 'assistant', text: fullText.trim(), time: Date.now(), source: assistantSource })

package/src/lib/server/connectors/whatsapp-text.test.ts

@@ -0,0 +1,29 @@
+import { describe, it } from 'node:test'
+import assert from 'node:assert/strict'
+import { formatTextForWhatsApp } from './whatsapp-text'
+
+describe('formatTextForWhatsApp', () => {
+  it('converts markdown links to readable whatsapp text', () => {
+    const input = 'See [Google](https://google.com) and [https://x.com](https://x.com)'
+    const output = formatTextForWhatsApp(input)
+    assert.equal(output, 'See Google: https://google.com and https://x.com')
+  })
+
+  it('converts common markdown emphasis syntax', () => {
+    const input = '**Bold** __Italic__ ~~Strike~~'
+    const output = formatTextForWhatsApp(input)
+    assert.equal(output, 'Bold Italic Strike')
+  })
+
+  it('removes headings and preserves body text', () => {
+    const input = '# Title\n\n## Subtitle\nBody line'
+    const output = formatTextForWhatsApp(input)
+    assert.equal(output, 'Title\n\nSubtitle\nBody line')
+  })
+
+  it('converts code fences to plain text content', () => {
+    const input = '```ts\nconst x = 1\n```\n\nDone.'
+    const output = formatTextForWhatsApp(input)
+    assert.equal(output, 'const x = 1\n\nDone.')
+  })
+})