@swarmclawai/swarmclaw 0.7.7 → 0.8.0

package/src/lib/server/elevenlabs.test.ts

@@ -1,12 +1,61 @@
 import { describe, it } from 'node:test'
 import assert from 'node:assert/strict'
-import { requestElevenLabsMp3Stream, synthesizeElevenLabsMp3 } from './elevenlabs'
+import { requestElevenLabsMp3Stream, resolveElevenLabsConfig, synthesizeElevenLabsMp3 } from './elevenlabs'
+import { encryptKey, loadSecrets, loadSettings, saveSecrets, saveSettings } from './storage'
 
 describe('elevenlabs helpers', () => {
+  it('prefers agent override first, then settings default, then env fallback', () => {
+    const originalSettings = loadSettings()
+    const originalKey = process.env.ELEVENLABS_API_KEY
+    const originalVoice = process.env.ELEVENLABS_VOICE
+
+    try {
+      saveSettings({
+        ...originalSettings,
+        elevenLabsApiKey: 'settings-key',
+        elevenLabsVoiceId: 'settings-voice',
+      })
+      process.env.ELEVENLABS_API_KEY = 'env-key'
+      process.env.ELEVENLABS_VOICE = 'env-voice'
+
+      assert.deepEqual(resolveElevenLabsConfig('agent-voice'), {
+        apiKey: 'settings-key',
+        voiceId: 'agent-voice',
+      })
+      assert.deepEqual(resolveElevenLabsConfig(null), {
+        apiKey: 'settings-key',
+        voiceId: 'settings-voice',
+      })
+
+      saveSettings({
+        ...originalSettings,
+        elevenLabsApiKey: 'settings-key',
+        elevenLabsVoiceId: null,
+      })
+
+      assert.deepEqual(resolveElevenLabsConfig(undefined), {
+        apiKey: 'settings-key',
+        voiceId: 'env-voice',
+      })
+    } finally {
+      saveSettings(originalSettings)
+      if (originalKey === undefined) delete process.env.ELEVENLABS_API_KEY
+      else process.env.ELEVENLABS_API_KEY = originalKey
+      if (originalVoice === undefined) delete process.env.ELEVENLABS_VOICE
+      else process.env.ELEVENLABS_VOICE = originalVoice
+    }
+  })
+
   it('synthesizeElevenLabsMp3 posts TTS request and returns audio bytes', async () => {
     const originalFetch = global.fetch
+    const originalSettings = loadSettings()
     const originalKey = process.env.ELEVENLABS_API_KEY
     const originalVoice = process.env.ELEVENLABS_VOICE
+    saveSettings({
+      ...originalSettings,
+      elevenLabsApiKey: null,
+      elevenLabsVoiceId: null,
+    })
     process.env.ELEVENLABS_API_KEY = 'test-key'
     process.env.ELEVENLABS_VOICE = 'voice-123'
 
@@ -25,6 +74,150 @@ describe('elevenlabs helpers', () => {
       assert.equal(out.toString('utf8'), 'abc')
     } finally {
       global.fetch = originalFetch
+      saveSettings(originalSettings)
+      if (originalKey === undefined) delete process.env.ELEVENLABS_API_KEY
+      else process.env.ELEVENLABS_API_KEY = originalKey
+      if (originalVoice === undefined) delete process.env.ELEVENLABS_VOICE
+      else process.env.ELEVENLABS_VOICE = originalVoice
+    }
+  })
+
+  it('synthesizeElevenLabsMp3 retries with the built-in fallback voice when the configured voice is paid-only', async () => {
+    const originalFetch = global.fetch
+    const originalSettings = loadSettings()
+    const originalKey = process.env.ELEVENLABS_API_KEY
+    const originalVoice = process.env.ELEVENLABS_VOICE
+    saveSettings({
+      ...originalSettings,
+      elevenLabsApiKey: 'settings-key',
+      elevenLabsVoiceId: 'paid-only-voice',
+    })
+    delete process.env.ELEVENLABS_API_KEY
+    delete process.env.ELEVENLABS_VOICE
+
+    const calls: string[] = []
+    global.fetch = (async (input: RequestInfo | URL) => {
+      const url = String(input)
+      calls.push(url)
+      if (url.endsWith('/paid-only-voice')) {
+        return new Response(
+          '{"detail":{"type":"payment_required","code":"paid_plan_required","message":"Free users cannot use library voices via the API."}}',
+          { status: 402, headers: { 'Content-Type': 'application/json' } },
+        )
+      }
+      assert.equal(url, 'https://api.elevenlabs.io/v1/text-to-speech/JBFqnCBsd6RMkjVDRZzb')
+      return new Response(Buffer.from('fallback-audio'), { status: 200, headers: { 'Content-Type': 'audio/mpeg' } })
+    }) as typeof fetch
+
+    try {
+      const out = await synthesizeElevenLabsMp3({ text: 'hello world' })
+      assert.equal(out.toString('utf8'), 'fallback-audio')
+      assert.deepEqual(calls, [
+        'https://api.elevenlabs.io/v1/text-to-speech/paid-only-voice',
+        'https://api.elevenlabs.io/v1/text-to-speech/JBFqnCBsd6RMkjVDRZzb',
+      ])
+    } finally {
+      global.fetch = originalFetch
+      saveSettings(originalSettings)
+      if (originalKey === undefined) delete process.env.ELEVENLABS_API_KEY
+      else process.env.ELEVENLABS_API_KEY = originalKey
+      if (originalVoice === undefined) delete process.env.ELEVENLABS_VOICE
+      else process.env.ELEVENLABS_VOICE = originalVoice
+    }
+  })
+
+  it('synthesizeElevenLabsMp3 retries with the built-in fallback voice when the configured voice id is missing', async () => {
+    const originalFetch = global.fetch
+    const originalSettings = loadSettings()
+    const originalKey = process.env.ELEVENLABS_API_KEY
+    const originalVoice = process.env.ELEVENLABS_VOICE
+    saveSettings({
+      ...originalSettings,
+      elevenLabsApiKey: 'settings-key',
+      elevenLabsVoiceId: 'missing-voice',
+    })
+    delete process.env.ELEVENLABS_API_KEY
+    delete process.env.ELEVENLABS_VOICE
+
+    const calls: string[] = []
+    global.fetch = (async (input: RequestInfo | URL) => {
+      const url = String(input)
+      calls.push(url)
+      if (url.endsWith('/missing-voice')) {
+        return new Response(
+          '{"detail":{"type":"not_found","code":"voice_not_found","message":"A voice with voice_id \'missing-voice\' was not found."}}',
+          { status: 404, headers: { 'Content-Type': 'application/json' } },
+        )
+      }
+      assert.equal(url, 'https://api.elevenlabs.io/v1/text-to-speech/JBFqnCBsd6RMkjVDRZzb')
+      return new Response(Buffer.from('fallback-audio'), { status: 200, headers: { 'Content-Type': 'audio/mpeg' } })
+    }) as typeof fetch
+
+    try {
+      const out = await synthesizeElevenLabsMp3({ text: 'hello world' })
+      assert.equal(out.toString('utf8'), 'fallback-audio')
+      assert.deepEqual(calls, [
+        'https://api.elevenlabs.io/v1/text-to-speech/missing-voice',
+        'https://api.elevenlabs.io/v1/text-to-speech/JBFqnCBsd6RMkjVDRZzb',
+      ])
+    } finally {
+      global.fetch = originalFetch
+      saveSettings(originalSettings)
+      if (originalKey === undefined) delete process.env.ELEVENLABS_API_KEY
+      else process.env.ELEVENLABS_API_KEY = originalKey
+      if (originalVoice === undefined) delete process.env.ELEVENLABS_VOICE
+      else process.env.ELEVENLABS_VOICE = originalVoice
+    }
+  })
+
+  it('synthesizeElevenLabsMp3 falls back from an invalid settings key to a stored ElevenLabs secret', async () => {
+    const originalFetch = global.fetch
+    const originalSettings = loadSettings()
+    const originalSecrets = loadSecrets()
+    const originalKey = process.env.ELEVENLABS_API_KEY
+    const originalVoice = process.env.ELEVENLABS_VOICE
+    saveSettings({
+      ...originalSettings,
+      elevenLabsApiKey: 'invalid-settings-key',
+      elevenLabsVoiceId: 'voice-123',
+    })
+    saveSecrets({
+      eleven_secret: {
+        id: 'eleven_secret',
+        name: 'ElevenLabs API Key',
+        service: 'custom',
+        scope: 'global',
+        agentIds: [],
+        createdAt: Date.now(),
+        updatedAt: Date.now(),
+        encryptedValue: encryptKey('secret-fallback-key'),
+      },
+    })
+    delete process.env.ELEVENLABS_API_KEY
+    delete process.env.ELEVENLABS_VOICE
+
+    const calls: string[] = []
+    global.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => {
+      const key = String((init?.headers as Record<string, string>)['xi-api-key'] || '')
+      calls.push(key)
+      if (key === 'invalid-settings-key') {
+        return new Response(
+          '{"detail":{"status":"invalid_api_key","message":"Invalid API key"}}',
+          { status: 401, headers: { 'Content-Type': 'application/json' } },
+        )
+      }
+      assert.equal(key, 'secret-fallback-key')
+      return new Response(Buffer.from('secret-audio'), { status: 200, headers: { 'Content-Type': 'audio/mpeg' } })
+    }) as typeof fetch
+
+    try {
+      const out = await synthesizeElevenLabsMp3({ text: 'hello world' })
+      assert.equal(out.toString('utf8'), 'secret-audio')
+      assert.deepEqual(calls, ['invalid-settings-key', 'secret-fallback-key'])
+    } finally {
+      global.fetch = originalFetch
+      saveSettings(originalSettings)
+      saveSecrets(originalSecrets)
       if (originalKey === undefined) delete process.env.ELEVENLABS_API_KEY
       else process.env.ELEVENLABS_API_KEY = originalKey
       if (originalVoice === undefined) delete process.env.ELEVENLABS_VOICE
@@ -34,14 +227,21 @@ describe('elevenlabs helpers', () => {
 
   it('requestElevenLabsMp3Stream calls streaming endpoint', async () => {
     const originalFetch = global.fetch
+    const originalSettings = loadSettings()
     const originalKey = process.env.ELEVENLABS_API_KEY
     const originalVoice = process.env.ELEVENLABS_VOICE
+    saveSettings({
+      ...originalSettings,
+      elevenLabsApiKey: null,
+      elevenLabsVoiceId: null,
+    })
     process.env.ELEVENLABS_API_KEY = 'test-key'
     process.env.ELEVENLABS_VOICE = 'voice-xyz'
 
     global.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
       assert.equal(String(input), 'https://api.elevenlabs.io/v1/text-to-speech/voice-xyz/stream')
       assert.equal(init?.method, 'POST')
+      assert.equal((init?.headers as Record<string, string>)['xi-api-key'], 'test-key')
       return new Response('stream', { status: 200, headers: { 'Content-Type': 'audio/mpeg' } })
     }) as typeof fetch
 
@@ -51,6 +251,52 @@ describe('elevenlabs helpers', () => {
       assert.equal(await res.text(), 'stream')
     } finally {
       global.fetch = originalFetch
+      saveSettings(originalSettings)
+      if (originalKey === undefined) delete process.env.ELEVENLABS_API_KEY
+      else process.env.ELEVENLABS_API_KEY = originalKey
+      if (originalVoice === undefined) delete process.env.ELEVENLABS_VOICE
+      else process.env.ELEVENLABS_VOICE = originalVoice
+    }
+  })
+
+  it('requestElevenLabsMp3Stream retries with the built-in fallback voice when the configured voice is paid-only', async () => {
+    const originalFetch = global.fetch
+    const originalSettings = loadSettings()
+    const originalKey = process.env.ELEVENLABS_API_KEY
+    const originalVoice = process.env.ELEVENLABS_VOICE
+    saveSettings({
+      ...originalSettings,
+      elevenLabsApiKey: 'settings-key',
+      elevenLabsVoiceId: 'paid-only-voice',
+    })
+    delete process.env.ELEVENLABS_API_KEY
+    delete process.env.ELEVENLABS_VOICE
+
+    const calls: string[] = []
+    global.fetch = (async (input: RequestInfo | URL) => {
+      const url = String(input)
+      calls.push(url)
+      if (url.endsWith('/paid-only-voice/stream')) {
+        return new Response(
+          '{"detail":{"type":"payment_required","code":"paid_plan_required","message":"Free users cannot use library voices via the API."}}',
+          { status: 402, headers: { 'Content-Type': 'application/json' } },
+        )
+      }
+      assert.equal(url, 'https://api.elevenlabs.io/v1/text-to-speech/JBFqnCBsd6RMkjVDRZzb/stream')
+      return new Response('fallback-stream', { status: 200, headers: { 'Content-Type': 'audio/mpeg' } })
+    }) as typeof fetch
+
+    try {
+      const res = await requestElevenLabsMp3Stream({ text: 'streaming text' })
+      assert.equal(res.status, 200)
+      assert.equal(await res.text(), 'fallback-stream')
+      assert.deepEqual(calls, [
+        'https://api.elevenlabs.io/v1/text-to-speech/paid-only-voice/stream',
+        'https://api.elevenlabs.io/v1/text-to-speech/JBFqnCBsd6RMkjVDRZzb/stream',
+      ])
+    } finally {
+      global.fetch = originalFetch
+      saveSettings(originalSettings)
       if (originalKey === undefined) delete process.env.ELEVENLABS_API_KEY
       else process.env.ELEVENLABS_API_KEY = originalKey
       if (originalVoice === undefined) delete process.env.ELEVENLABS_VOICE

package/src/lib/server/elevenlabs.ts

@@ -1,4 +1,4 @@
-import { loadSettings } from './storage'
+import { decryptKey, loadSettings, loadSecrets } from './storage'
 
 const DEFAULT_VOICE_ID = 'JBFqnCBsd6RMkjVDRZzb'
 const DEFAULT_MODEL_ID = 'eleven_multilingual_v2'
@@ -12,12 +12,12 @@ export function resolveElevenLabsConfig(voiceId?: string | null): {
   apiKey: string
   voiceId: string
 } {
-  const settings = loadSettings()
-  const apiKey = String(settings.elevenLabsApiKey || process.env.ELEVENLABS_API_KEY || '').trim()
-  if (!apiKey) {
+  const apiKeys = resolveElevenLabsApiKeyCandidates()
+  if (!apiKeys.length) {
     throw new Error('No ElevenLabs API key. Set one in Settings > Voice.')
   }
 
+  const settings = loadSettings()
   const resolvedVoiceId = String(
     voiceId
     || settings.elevenLabsVoiceId
@@ -25,44 +25,161 @@ export function resolveElevenLabsConfig(voiceId?: string | null): {
     || DEFAULT_VOICE_ID,
   ).trim()
 
-  return { apiKey, voiceId: resolvedVoiceId || DEFAULT_VOICE_ID }
+  return { apiKey: apiKeys[0], voiceId: resolvedVoiceId || DEFAULT_VOICE_ID }
 }
 
-export async function synthesizeElevenLabsMp3(params: {
-  text: string
-  voiceId?: string | null
-  stability?: number
-  similarityBoost?: number
-}): Promise<Buffer> {
-  const text = params.text.trim()
-  if (!text) throw new Error('No text provided for ElevenLabs synthesis.')
+function resolveElevenLabsApiKeyCandidates(): string[] {
+  const settings = loadSettings()
+  const candidates: string[] = []
 
-  const { apiKey, voiceId } = resolveElevenLabsConfig(params.voiceId)
-  const stability = Number.isFinite(params.stability) ? Math.max(0, Math.min(1, Number(params.stability))) : 0.5
-  const similarityBoost = Number.isFinite(params.similarityBoost) ? Math.max(0, Math.min(1, Number(params.similarityBoost))) : 0.75
+  const pushCandidate = (value: unknown) => {
+    if (typeof value !== 'string') return
+    const trimmed = value.trim()
+    if (trimmed) candidates.push(trimmed)
+  }
+
+  pushCandidate(settings.elevenLabsApiKey)
+  pushCandidate(process.env.ELEVENLABS_API_KEY)
+
+  for (const secret of Object.values(loadSecrets()) as Array<Record<string, unknown>>) {
+    const label = [
+      typeof secret.id === 'string' ? secret.id : '',
+      typeof secret.name === 'string' ? secret.name : '',
+      typeof secret.service === 'string' ? secret.service : '',
+    ].join(' ').toLowerCase()
+    if (!label.includes('eleven')) continue
+    if (typeof secret.encryptedValue !== 'string' || !secret.encryptedValue) continue
+    try {
+      pushCandidate(decryptKey(secret.encryptedValue))
+    } catch {
+      // Ignore undecryptable secrets and continue.
+    }
+  }
 
-  const apiRes = await fetch(`https://api.elevenlabs.io/v1/text-to-speech/${voiceId}`, {
+  return [...new Set(candidates)]
+}
+
+function shouldRetryWithFallbackVoice(voiceId: string, errBody: string): boolean {
+  if (!voiceId || voiceId === DEFAULT_VOICE_ID) return false
+  return /paid_plan_required|library voices via the api|voice_not_found|voice with voice_id .* was not found/i.test(errBody)
+}
+
+async function postElevenLabsTts(params: {
+  apiKey: string
+  voiceId: string
+  text: string
+  stability: number
+  similarityBoost: number
+  stream?: boolean
+}): Promise<Response> {
+  const endpoint = params.stream
+    ? `https://api.elevenlabs.io/v1/text-to-speech/${params.voiceId}/stream`
+    : `https://api.elevenlabs.io/v1/text-to-speech/${params.voiceId}`
+  return fetch(endpoint, {
     method: 'POST',
     headers: {
-      'xi-api-key': apiKey,
+      'xi-api-key': params.apiKey,
       'Content-Type': 'application/json',
       Accept: 'audio/mpeg',
     },
     body: JSON.stringify({
-      text,
+      text: params.stream ? params.text.slice(0, 2000) : params.text,
       model_id: DEFAULT_MODEL_ID,
       voice_settings: {
-        stability,
-        similarity_boost: similarityBoost,
+        stability: params.stability,
+        similarity_boost: params.similarityBoost,
       },
+      ...(params.stream ? { output_format: 'mp3_22050_32' } : {}),
     }),
   })
+}
+
+async function requestElevenLabsAudioWithFallback(params: {
+  text: string
+  voiceId?: string | null
+  stability: number
+  similarityBoost: number
+  stream?: boolean
+}): Promise<Response> {
+  const settings = loadSettings()
+  const voiceId = String(
+    params.voiceId
+    || settings.elevenLabsVoiceId
+    || process.env.ELEVENLABS_VOICE
+    || DEFAULT_VOICE_ID,
+  ).trim() || DEFAULT_VOICE_ID
+  const apiKeys = resolveElevenLabsApiKeyCandidates()
+  if (!apiKeys.length) {
+    throw new Error('No ElevenLabs API key. Set one in Settings > Voice.')
+  }
+
+  let lastError = ''
+  for (const apiKey of apiKeys) {
+    const firstRes = await postElevenLabsTts({
+      apiKey,
+      voiceId,
+      text: params.text,
+      stability: params.stability,
+      similarityBoost: params.similarityBoost,
+      stream: params.stream,
+    })
+    if (firstRes.ok) return firstRes
+
+    const firstErr = await firstRes.text().catch(() => '')
+    lastError = firstErr || `ElevenLabs request failed (${firstRes.status})`
+
+    if (shouldRetryWithFallbackVoice(voiceId, firstErr)) {
+      const fallbackRes = await postElevenLabsTts({
+        apiKey,
+        voiceId: DEFAULT_VOICE_ID,
+        text: params.text,
+        stability: params.stability,
+        similarityBoost: params.similarityBoost,
+        stream: params.stream,
+      })
+      if (fallbackRes.ok) return fallbackRes
 
-  if (!apiRes.ok) {
-    const errBody = await apiRes.text().catch(() => '')
-    throw new Error(errBody || `ElevenLabs request failed (${apiRes.status})`)
+      const fallbackErr = await fallbackRes.text().catch(() => '')
+      lastError = fallbackErr
+        ? `${lastError} | fallback voice failed: ${fallbackErr}`
+        : lastError
+      if (!shouldTryNextApiKey(lastError)) {
+        throw new Error(lastError)
+      }
+      continue
+    }
+
+    if (!shouldTryNextApiKey(firstErr)) {
+      throw new Error(lastError)
+    }
   }
 
+  throw new Error(lastError || 'ElevenLabs request failed')
+}
+
+function shouldTryNextApiKey(errBody: string): boolean {
+  return /invalid_api_key|missing_permissions|detected_unusual_activity/i.test(errBody)
+}
+
+export async function synthesizeElevenLabsMp3(params: {
+  text: string
+  voiceId?: string | null
+  stability?: number
+  similarityBoost?: number
+}): Promise<Buffer> {
+  const text = params.text.trim()
+  if (!text) throw new Error('No text provided for ElevenLabs synthesis.')
+
+  const stability = Number.isFinite(params.stability) ? Math.max(0, Math.min(1, Number(params.stability))) : 0.5
+  const similarityBoost = Number.isFinite(params.similarityBoost) ? Math.max(0, Math.min(1, Number(params.similarityBoost))) : 0.75
+
+  const apiRes = await requestElevenLabsAudioWithFallback({
+    text,
+    voiceId: params.voiceId,
+    stability,
+    similarityBoost,
+  })
+
   const audioBuffer = await apiRes.arrayBuffer()
   return Buffer.from(audioBuffer)
 }
@@ -74,27 +191,14 @@ export async function requestElevenLabsMp3Stream(params: {
   const text = params.text.trim()
   if (!text) throw new Error('No text provided for ElevenLabs stream.')
 
-  const { apiKey, voiceId } = resolveElevenLabsConfig(params.voiceId)
-  const apiRes = await fetch(`https://api.elevenlabs.io/v1/text-to-speech/${voiceId}/stream`, {
-    method: 'POST',
-    headers: {
-      'xi-api-key': apiKey,
-      'Content-Type': 'application/json',
-      Accept: 'audio/mpeg',
-    },
-    body: JSON.stringify({
-      text: text.slice(0, 2000),
-      model_id: DEFAULT_MODEL_ID,
-      voice_settings: { stability: 0.5, similarity_boost: 0.75 },
-      output_format: 'mp3_22050_32',
-    }),
+  const apiRes = await requestElevenLabsAudioWithFallback({
+    text,
+    voiceId: params.voiceId,
+    stability: 0.5,
+    similarityBoost: 0.75,
+    stream: true,
   })
 
-  if (!apiRes.ok) {
-    const errBody = await apiRes.text().catch(() => '')
-    throw new Error(errBody || `ElevenLabs streaming request failed (${apiRes.status})`)
-  }
-
   return apiRes
 }