@swarmclawai/swarmclaw 0.6.2 → 0.6.3

package/src/lib/server/connectors/whatsapp.ts

@@ -10,12 +10,13 @@ import path from 'path'
 import fs from 'fs'
 import type { Connector } from '@/types'
 import type { PlatformConnector, ConnectorInstance, InboundMessage } from './types'
-import { saveInboundMediaBuffer, mimeFromPath, isImageMime } from './media'
+import { saveInboundMediaBuffer, mimeFromPath, isImageMime, isAudioMime } from './media'
 import { isNoMessage } from './manager'
 
 import { DATA_DIR } from '../data-dir'
 
 const AUTH_DIR = path.join(DATA_DIR, 'whatsapp-auth')
+const INBOUND_DEDUPE_TTL_MS = 2 * 60 * 1000
 
 /** Normalize a phone number for JID matching — strip leading 0 or + */
 function normalizeNumber(num: string): string {
@@ -57,6 +58,7 @@ const whatsapp: PlatformConnector = {
     let sock: ReturnType<typeof makeWASocket> | null = null
     let stopped = false
     let socketGen = 0 // Track socket generation to ignore stale events
+    const seenInboundMessageIds = new Map<string, number>()
 
     const instance: ConnectorInstance = {
       connector,
@@ -74,7 +76,15 @@ const whatsapp: PlatformConnector = {
           const fName = options.fileName || path.basename(options.mediaPath)
           let sent
           if (isImageMime(mime)) {
-            sent = await sock.sendMessage(channelId, { image: buf, caption, mimetype: mime })
+            try {
+              sent = await sock.sendMessage(channelId, { image: buf, caption, mimetype: mime })
+            } catch (err: unknown) {
+              const errMsg = err instanceof Error ? err.message : String(err)
+              console.warn(`[whatsapp] Image send failed (${errMsg}); retrying as document: ${fName}`)
+              sent = await sock.sendMessage(channelId, { document: buf, fileName: fName, mimetype: mime, caption })
+            }
+          } else if (isAudioMime(mime)) {
+            sent = await sock.sendMessage(channelId, { audio: buf, mimetype: mime, ptt: options.ptt !== false })
           } else {
             sent = await sock.sendMessage(channelId, { document: buf, fileName: fName, mimetype: mime, caption })
           }
@@ -228,6 +238,22 @@ const whatsapp: PlatformConnector = {
 
           if (msg.key.remoteJid === 'status@broadcast') continue
 
+          const msgId = msg.key.id || ''
+          if (msgId) {
+            const now = Date.now()
+            const seenAt = seenInboundMessageIds.get(msgId)
+            if (typeof seenAt === 'number' && now - seenAt <= INBOUND_DEDUPE_TTL_MS) {
+              console.log(`[whatsapp] Skipping duplicate inbound message id: ${msgId}`)
+              continue
+            }
+            seenInboundMessageIds.set(msgId, now)
+            if (seenInboundMessageIds.size > 5000) {
+              for (const [id, ts] of seenInboundMessageIds.entries()) {
+                if (now - ts > INBOUND_DEDUPE_TTL_MS) seenInboundMessageIds.delete(id)
+              }
+            }
+          }
+
           // Skip messages sent by the bot itself (tracked by ID to prevent infinite loops)
           if (msg.key.id && sentMessageIds.has(msg.key.id)) {
             console.log(`[whatsapp] Skipping own bot reply: ${msg.key.id}`)

package/src/lib/server/elevenlabs.test.ts

@@ -0,0 +1,60 @@
+import { describe, it } from 'node:test'
+import assert from 'node:assert/strict'
+import { requestElevenLabsMp3Stream, synthesizeElevenLabsMp3 } from './elevenlabs'
+
+describe('elevenlabs helpers', () => {
+  it('synthesizeElevenLabsMp3 posts TTS request and returns audio bytes', async () => {
+    const originalFetch = global.fetch
+    const originalKey = process.env.ELEVENLABS_API_KEY
+    const originalVoice = process.env.ELEVENLABS_VOICE
+    process.env.ELEVENLABS_API_KEY = 'test-key'
+    process.env.ELEVENLABS_VOICE = 'voice-123'
+
+    let called = false
+    global.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
+      called = true
+      assert.equal(String(input), 'https://api.elevenlabs.io/v1/text-to-speech/voice-123')
+      assert.equal(init?.method, 'POST')
+      assert.equal((init?.headers as Record<string, string>)['xi-api-key'], 'test-key')
+      return new Response(Buffer.from('abc'), { status: 200, headers: { 'Content-Type': 'audio/mpeg' } })
+    }) as typeof fetch
+
+    try {
+      const out = await synthesizeElevenLabsMp3({ text: 'hello world' })
+      assert.ok(called)
+      assert.equal(out.toString('utf8'), 'abc')
+    } finally {
+      global.fetch = originalFetch
+      if (originalKey === undefined) delete process.env.ELEVENLABS_API_KEY
+      else process.env.ELEVENLABS_API_KEY = originalKey
+      if (originalVoice === undefined) delete process.env.ELEVENLABS_VOICE
+      else process.env.ELEVENLABS_VOICE = originalVoice
+    }
+  })
+
+  it('requestElevenLabsMp3Stream calls streaming endpoint', async () => {
+    const originalFetch = global.fetch
+    const originalKey = process.env.ELEVENLABS_API_KEY
+    const originalVoice = process.env.ELEVENLABS_VOICE
+    process.env.ELEVENLABS_API_KEY = 'test-key'
+    process.env.ELEVENLABS_VOICE = 'voice-xyz'
+
+    global.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
+      assert.equal(String(input), 'https://api.elevenlabs.io/v1/text-to-speech/voice-xyz/stream')
+      assert.equal(init?.method, 'POST')
+      return new Response('stream', { status: 200, headers: { 'Content-Type': 'audio/mpeg' } })
+    }) as typeof fetch
+
+    try {
+      const res = await requestElevenLabsMp3Stream({ text: 'streaming text' })
+      assert.equal(res.status, 200)
+      assert.equal(await res.text(), 'stream')
+    } finally {
+      global.fetch = originalFetch
+      if (originalKey === undefined) delete process.env.ELEVENLABS_API_KEY
+      else process.env.ELEVENLABS_API_KEY = originalKey
+      if (originalVoice === undefined) delete process.env.ELEVENLABS_VOICE
+      else process.env.ELEVENLABS_VOICE = originalVoice
+    }
+  })
+})

package/src/lib/server/elevenlabs.ts

@@ -0,0 +1,103 @@
+import { loadSettings } from './storage'
+
+const DEFAULT_VOICE_ID = 'JBFqnCBsd6RMkjVDRZzb'
+const DEFAULT_MODEL_ID = 'eleven_multilingual_v2'
+
+function getErrorMessage(err: unknown): string {
+  if (err instanceof Error && err.message) return err.message
+  return String(err)
+}
+
+export function resolveElevenLabsConfig(voiceId?: string | null): {
+  apiKey: string
+  voiceId: string
+} {
+  const settings = loadSettings()
+  const apiKey = String(settings.elevenLabsApiKey || process.env.ELEVENLABS_API_KEY || '').trim()
+  if (!apiKey) {
+    throw new Error('No ElevenLabs API key. Set one in Settings > Voice.')
+  }
+
+  const resolvedVoiceId = String(
+    voiceId
+    || settings.elevenLabsVoiceId
+    || process.env.ELEVENLABS_VOICE
+    || DEFAULT_VOICE_ID,
+  ).trim()
+
+  return { apiKey, voiceId: resolvedVoiceId || DEFAULT_VOICE_ID }
+}
+
+export async function synthesizeElevenLabsMp3(params: {
+  text: string
+  voiceId?: string | null
+  stability?: number
+  similarityBoost?: number
+}): Promise<Buffer> {
+  const text = params.text.trim()
+  if (!text) throw new Error('No text provided for ElevenLabs synthesis.')
+
+  const { apiKey, voiceId } = resolveElevenLabsConfig(params.voiceId)
+  const stability = Number.isFinite(params.stability) ? Math.max(0, Math.min(1, Number(params.stability))) : 0.5
+  const similarityBoost = Number.isFinite(params.similarityBoost) ? Math.max(0, Math.min(1, Number(params.similarityBoost))) : 0.75
+
+  const apiRes = await fetch(`https://api.elevenlabs.io/v1/text-to-speech/${voiceId}`, {
+    method: 'POST',
+    headers: {
+      'xi-api-key': apiKey,
+      'Content-Type': 'application/json',
+      Accept: 'audio/mpeg',
+    },
+    body: JSON.stringify({
+      text,
+      model_id: DEFAULT_MODEL_ID,
+      voice_settings: {
+        stability,
+        similarity_boost: similarityBoost,
+      },
+    }),
+  })
+
+  if (!apiRes.ok) {
+    const errBody = await apiRes.text().catch(() => '')
+    throw new Error(errBody || `ElevenLabs request failed (${apiRes.status})`)
+  }
+
+  const audioBuffer = await apiRes.arrayBuffer()
+  return Buffer.from(audioBuffer)
+}
+
+export async function requestElevenLabsMp3Stream(params: {
+  text: string
+  voiceId?: string | null
+}): Promise<Response> {
+  const text = params.text.trim()
+  if (!text) throw new Error('No text provided for ElevenLabs stream.')
+
+  const { apiKey, voiceId } = resolveElevenLabsConfig(params.voiceId)
+  const apiRes = await fetch(`https://api.elevenlabs.io/v1/text-to-speech/${voiceId}/stream`, {
+    method: 'POST',
+    headers: {
+      'xi-api-key': apiKey,
+      'Content-Type': 'application/json',
+      Accept: 'audio/mpeg',
+    },
+    body: JSON.stringify({
+      text: text.slice(0, 2000),
+      model_id: DEFAULT_MODEL_ID,
+      voice_settings: { stability: 0.5, similarity_boost: 0.75 },
+      output_format: 'mp3_22050_32',
+    }),
+  })
+
+  if (!apiRes.ok) {
+    const errBody = await apiRes.text().catch(() => '')
+    throw new Error(errBody || `ElevenLabs streaming request failed (${apiRes.status})`)
+  }
+
+  return apiRes
+}
+
+export function explainElevenLabsError(err: unknown): string {
+  return getErrorMessage(err)
+}

package/src/lib/server/queue.ts

@@ -1,5 +1,7 @@
 import { genId } from '@/lib/id'
-import { loadTasks, saveTasks, loadQueue, saveQueue, loadAgents, loadSchedules, saveSchedules, loadSessions, saveSessions, loadSettings } from './storage'
+import fs from 'node:fs'
+import path from 'node:path'
+import { loadTasks, saveTasks, loadQueue, saveQueue, loadAgents, loadSchedules, saveSchedules, loadSessions, saveSessions, loadSettings, loadConnectors, UPLOAD_DIR } from './storage'
 import { notify } from './ws-hub'
 import { WORKSPACE_DIR } from './data-dir'
 import { createOrchestratorSession, executeOrchestrator } from './orchestrator'
@@ -114,6 +116,54 @@ function latestAssistantText(session: SessionLike | null | undefined): string {
   return ''
 }
 
+function isEnabledFlag(value: unknown): boolean {
+  if (typeof value === 'boolean') return value
+  if (typeof value !== 'string') return false
+  const normalized = value.trim().toLowerCase()
+  return normalized === '1'
+    || normalized === 'true'
+    || normalized === 'yes'
+    || normalized === 'on'
+    || normalized === 'enabled'
+}
+
+function normalizeWhatsappTarget(raw: string): string {
+  const trimmed = raw.trim()
+  if (!trimmed) return trimmed
+  if (trimmed.includes('@')) return trimmed
+  let cleaned = trimmed.replace(/[^\d+]/g, '')
+  if (cleaned.startsWith('+')) cleaned = cleaned.slice(1)
+  if (cleaned.startsWith('0') && cleaned.length >= 10) {
+    cleaned = `44${cleaned.slice(1)}`
+  }
+  cleaned = cleaned.replace(/[^\d]/g, '')
+  return cleaned ? `${cleaned}@s.whatsapp.net` : trimmed
+}
+
+function fillTaskFollowupTemplate(template: string, data: {
+  status: string
+  title: string
+  summary: string
+  taskId: string
+}): string {
+  return template
+    .replaceAll('{status}', data.status)
+    .replaceAll('{title}', data.title)
+    .replaceAll('{summary}', data.summary)
+    .replaceAll('{taskId}', data.taskId)
+}
+
+function maybeResolveUploadMediaPathFromUrl(url: string | undefined): string | undefined {
+  if (!url || !url.startsWith('/api/uploads/')) return undefined
+  const rawName = url.slice('/api/uploads/'.length).split(/[?#]/)[0] || ''
+  let decoded: string
+  try { decoded = decodeURIComponent(rawName) } catch { decoded = rawName }
+  const safeName = decoded.replace(/[^a-zA-Z0-9._-]/g, '')
+  if (!safeName) return undefined
+  const fullPath = path.join(UPLOAD_DIR, safeName)
+  return fs.existsSync(fullPath) ? fullPath : undefined
+}
+
 // Task result extraction now uses Zod-validated structured data
 // from ./task-result.ts (extractTaskResult, formatResultBody)
 
@@ -215,6 +265,78 @@ function notifyMainChatScheduleResult(task: BoardTask): void {
   if (changed) saveSessions(sessions)
 }
 
+async function notifyConnectorTaskFollowups(params: {
+  task: BoardTask
+  statusLabel: string
+  summaryText: string
+  imageUrl?: string
+}) {
+  const { task, statusLabel, summaryText, imageUrl } = params
+
+  const connectors = loadConnectors()
+  const running = (await import('./connectors/manager')).listRunningConnectors()
+  const manager = await import('./connectors/manager')
+
+  const candidates = running.filter((entry) => {
+    if (!entry.supportsSend || !entry.id) return false
+    const connector = connectors[entry.id]
+    if (!connector) return false
+    if (connector.agentId !== task.agentId) return false
+    return isEnabledFlag(connector.config?.taskFollowups)
+  })
+  if (!candidates.length) return
+
+  const summary = summaryText.trim().slice(0, 1400)
+  for (const candidate of candidates) {
+    const connector = connectors[candidate.id]
+    if (!connector) continue
+
+    const channelTargetRaw = candidate.recentChannelId
+      || candidate.configuredTargets[0]
+      || connector.config?.outboundJid
+      || connector.config?.outboundTarget
+      || ''
+    if (!channelTargetRaw) continue
+
+    const channelId = connector.platform === 'whatsapp'
+      ? normalizeWhatsappTarget(channelTargetRaw)
+      : channelTargetRaw
+
+    const template = typeof connector.config?.taskFollowupTemplate === 'string'
+      ? connector.config.taskFollowupTemplate.trim()
+      : ''
+    const message = template
+      ? fillTaskFollowupTemplate(template, {
+          status: statusLabel,
+          title: task.title || task.id,
+          summary,
+          taskId: task.id,
+        })
+      : [
+          `Task ${statusLabel}: ${task.title}`,
+          summary || 'No summary provided.',
+        ].join('\n\n')
+
+    const resolvedMediaPath = maybeResolveUploadMediaPathFromUrl(imageUrl)
+    try {
+      await manager.sendConnectorMessage({
+        connectorId: candidate.id,
+        channelId,
+        text: message,
+        ...(resolvedMediaPath
+          ? {
+              mediaPath: resolvedMediaPath,
+              caption: message,
+            }
+          : {}),
+      })
+    } catch (err: unknown) {
+      const errMsg = err instanceof Error ? err.message : String(err)
+      console.warn(`[queue] Failed task follow-up send on connector ${candidate.id}: ${errMsg}`)
+    }
+  }
+}
+
 /**
  * Notify agent thread sessions when a task completes or fails.
  * - Always pushes to the executing agent's thread
@@ -321,6 +443,13 @@ function notifyAgentThreadTaskResult(task: BoardTask): void {
   }
 
   if (changed) saveSessions(sessions)
+
+  void notifyConnectorTaskFollowups({
+    task,
+    statusLabel,
+    summaryText: resultBody || '',
+    imageUrl: firstImage?.url,
+  })
 }
 
 /** Disable heartbeat on a task's session when the task finishes. */