@pikku/core 0.12.34 → 0.12.35

package/src/wirings/ai-agent/index.ts

@@ -6,6 +6,8 @@ export {
 } from './ai-agent-helpers.js'
 export { runAIAgent, resumeAIAgentSync } from './ai-agent-runner.js'
 export { streamAIAgent, resumeAIAgent } from './ai-agent-stream.js'
+export { voiceInput } from './voice-input.js'
+export { voiceOutput } from './voice-output.js'
 export {
   type RunAIAgentParams,
   type StreamAIAgentOptions,

package/src/wirings/ai-agent/voice-input.ts

@@ -0,0 +1,132 @@
+import type { AIAgentRunnerService } from '../../services/ai-agent-runner-service.js'
+import { pikkuAIMiddleware } from '../../types/core.types.js'
+import type { AIContentPart } from './ai-agent.types.js'
+
+function base64ToUint8Array(base64: string): Uint8Array {
+  const binary = atob(base64)
+  const bytes = new Uint8Array(binary.length)
+  for (let i = 0; i < binary.length; i++) {
+    bytes[i] = binary.charCodeAt(i)
+  }
+  return bytes
+}
+
+const MAX_AUDIO_SIZE = 50 * 1024 * 1024
+
+// Portable SSRF guard: @pikku/core runs in edge runtimes (CF Workers) with no
+// Node `dns`, so we cannot resolve hostnames to check for private targets.
+// Reject the obvious internal literals; callers wanting stricter control pass
+// an explicit `allowedAudioHosts` allowlist. (Does not defend against a public
+// hostname that resolves to a private IP / DNS rebinding — out of reach here.)
+function isPrivateHost(hostname: string): boolean {
+  const host = hostname.replace(/^\[|\]$/g, '').toLowerCase()
+  if (host === 'localhost' || host === '0.0.0.0' || host === '::1') return true
+  if (host.startsWith('fe80:') || host.startsWith('fc') || host.startsWith('fd'))
+    return true
+  const v4 = host.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.\d{1,3}$/)
+  if (v4) {
+    const [a, b] = [Number(v4[1]), Number(v4[2])]
+    if (a === 127 || a === 10 || a === 0) return true
+    if (a === 169 && b === 254) return true // link-local incl. cloud metadata
+    if (a === 172 && b >= 16 && b <= 31) return true
+    if (a === 192 && b === 168) return true
+  }
+  return false
+}
+
+async function fetchAsUint8Array(
+  url: string,
+  allowedAudioHosts?: string[]
+): Promise<Uint8Array> {
+  const parsed = new URL(url)
+  if (parsed.protocol !== 'https:' && parsed.protocol !== 'http:') {
+    throw new Error('Only HTTP(S) URLs are supported for audio')
+  }
+  if (allowedAudioHosts) {
+    if (!allowedAudioHosts.includes(parsed.hostname)) {
+      throw new Error(`Audio URL host is not allowed: ${parsed.hostname}`)
+    }
+  } else if (isPrivateHost(parsed.hostname)) {
+    throw new Error(
+      `Refusing to fetch audio from a private/internal host: ${parsed.hostname}`
+    )
+  }
+  const response = await fetch(url)
+  const contentLength = response.headers.get('content-length')
+  if (contentLength && parseInt(contentLength, 10) > MAX_AUDIO_SIZE) {
+    throw new Error('Audio file exceeds maximum size')
+  }
+  const buffer = await response.arrayBuffer()
+  if (buffer.byteLength > MAX_AUDIO_SIZE) {
+    throw new Error('Audio file exceeds maximum size')
+  }
+  return new Uint8Array(buffer)
+}
+
+export const voiceInput = (config?: {
+  language?: string
+  model?: string
+  allowedAudioHosts?: string[]
+}) =>
+  pikkuAIMiddleware({
+    modifyInput: async (services, { messages, instructions }) => {
+      const transcribeAudio = (services as {
+        aiAgentRunner?: AIAgentRunnerService
+      }).aiAgentRunner?.transcribe
+      if (!transcribeAudio) return { messages, instructions }
+
+      const last = messages[messages.length - 1]
+      if (!last || last.role !== 'user' || typeof last.content === 'string') {
+        return { messages, instructions }
+      }
+
+      const parts = last.content as AIContentPart[]
+      if (!parts) return { messages, instructions }
+
+      const hasAudio = parts.some(
+        (p) => p.type === 'file' && !!p.mediaType?.startsWith('audio/')
+      )
+      if (!hasAudio) return { messages, instructions }
+
+      // Process parts in order, replacing each audio part with its transcription
+      // in place. Sequential, so no unbounded parallel downloads/transcriptions
+      // and original content ordering is preserved.
+      const updatedContent: AIContentPart[] = []
+      for (const p of parts) {
+        if (!(p.type === 'file' && p.mediaType?.startsWith('audio/'))) {
+          updatedContent.push(p)
+          continue
+        }
+        if (!config?.model) {
+          throw new Error(
+            'voiceInput requires a transcription model (e.g. openai/whisper-1)'
+          )
+        }
+        const audioData = p.data
+          ? base64ToUint8Array(p.data)
+          : await fetchAsUint8Array(p.url!, config.allowedAudioHosts)
+        const result = await transcribeAudio({
+          model: config.model,
+          audio: audioData,
+          ...(config.language
+            ? {
+                providerOptions: {
+                  openai: {
+                    language: config.language,
+                  },
+                },
+              }
+            : {}),
+        })
+        updatedContent.push({ type: 'text' as const, text: result.text })
+      }
+
+      return {
+        messages: [
+          ...messages.slice(0, -1),
+          { ...last, content: updatedContent },
+        ],
+        instructions,
+      }
+    },
+  })

package/src/wirings/ai-agent/voice-output.ts

@@ -0,0 +1,134 @@
+import type { AIAgentRunnerService } from '../../services/ai-agent-runner-service.js'
+import { pikkuAIMiddleware } from '../../types/core.types.js'
+
+type VoiceOutputState = {
+  textBuffer?: string
+}
+
+function bufferToBase64(data: Uint8Array): string {
+  let binary = ''
+  for (let i = 0; i < data.length; i++) {
+    binary += String.fromCharCode(data[i]!)
+  }
+  return btoa(binary)
+}
+
+const SENTENCE_BOUNDARY = /[.!?]\s*$/
+
+function isSentenceBoundary(text: string): boolean {
+  return SENTENCE_BOUNDARY.test(text)
+}
+
+async function synthesizeAudio(
+  aiAgentRunner: AIAgentRunnerService,
+  input: {
+    model: string
+    text: string
+    voice?: string
+    format?: string
+    instructions?: string
+    speed?: number
+    language?: string
+  }
+): Promise<{ bytes: Uint8Array; format: string }> {
+  const result = await aiAgentRunner.generateSpeech?.({
+    model: input.model,
+    text: input.text,
+    voice: input.voice,
+    outputFormat: input.format,
+    instructions: input.instructions,
+    speed: input.speed,
+    language: input.language,
+  })
+  if (!result) {
+    throw new Error(
+      'voiceOutput requires an aiAgentRunner with generateSpeech support'
+    )
+  }
+  // Label chunks with the format the provider actually returned (config.format
+  // is only a request), falling back to the requested format then pcm16.
+  return {
+    bytes: result.audio.uint8Array,
+    format: result.audio.format || input.format || 'pcm16',
+  }
+}
+
+export const voiceOutput = (config?: {
+  model?: string
+  format?: string
+  voice?: string
+  instructions?: string
+  speed?: number
+  language?: string
+}) =>
+  pikkuAIMiddleware<VoiceOutputState>({
+    modifyOutputStream: async (services, { event, state }) => {
+      const aiAgentRunner = (services as {
+        aiAgentRunner?: AIAgentRunnerService
+      }).aiAgentRunner
+      if (!aiAgentRunner?.generateSpeech) return event
+
+      if (event.type === 'done') {
+        const remaining = state.textBuffer ?? ''
+        if (remaining) {
+          state.textBuffer = ''
+          if (!config?.model) {
+            throw new Error(
+              'voiceOutput requires a speech model (e.g. openai/tts-1)'
+            )
+          }
+          const audio = await synthesizeAudio(aiAgentRunner, {
+            model: config.model,
+            text: remaining,
+            voice: config?.voice,
+            format: config?.format,
+            instructions: config?.instructions,
+            speed: config?.speed,
+            language: config?.language,
+          })
+          return [
+            {
+              type: 'audio-delta' as const,
+              data: bufferToBase64(audio.bytes),
+              format: audio.format,
+            },
+            { type: 'audio-done' as const },
+            event,
+          ]
+        }
+        return [{ type: 'audio-done' as const }, event]
+      }
+
+      if (event.type !== 'text-delta') return event
+
+      state.textBuffer = `${state.textBuffer ?? ''}${event.text}`
+      if (!isSentenceBoundary(state.textBuffer)) return event
+
+      const text = state.textBuffer
+      state.textBuffer = ''
+
+      if (!config?.model) {
+        throw new Error(
+          'voiceOutput requires a speech model (e.g. openai/tts-1)'
+        )
+      }
+      const audio = await synthesizeAudio(aiAgentRunner, {
+        model: config.model,
+        text,
+        voice: config?.voice,
+        format: config?.format,
+        instructions: config?.instructions,
+        speed: config?.speed,
+        language: config?.language,
+      })
+
+      return [
+        event,
+        {
+          type: 'audio-delta' as const,
+          data: bufferToBase64(audio.bytes),
+          format: audio.format,
+        },
+      ]
+    },
+  })