@pikku/core 0.12.34 → 0.12.36

package/src/services/ai-agent-runner-service.ts

@@ -5,6 +5,8 @@ import type {
   AIStreamChannel,
 } from '../wirings/ai-agent/ai-agent.types.js'
 
+export type AIProviderOptions = Record<string, Record<string, unknown>>
+
 export type AIAgentRunnerParams = {
   model: string
   temperature?: number
@@ -36,12 +38,169 @@ export type AIAgentStepResult = {
   reasoningContent?: string
 }
 
+export type AITranscriptionParams = {
+  model: string
+  audio: Uint8Array
+  providerOptions?: AIProviderOptions
+  maxRetries?: number
+  abortSignal?: AbortSignal
+  headers?: Record<string, string>
+}
+
+export type AITranscriptionResult = {
+  text: string
+  segments?: Array<{
+    text: string
+    startSecond: number
+    endSecond: number
+  }>
+  language?: string
+  durationInSeconds?: number
+  warnings?: unknown[]
+  providerMetadata?: Record<string, unknown>
+  responses?: unknown[]
+}
+
+export type AIGenerateSpeechParams = {
+  model: string
+  text: string
+  voice?: string
+  outputFormat?: string
+  instructions?: string
+  speed?: number
+  language?: string
+  providerOptions?: AIProviderOptions
+  maxRetries?: number
+  abortSignal?: AbortSignal
+  headers?: Record<string, string>
+}
+
+export type AIGenerateSpeechResult = {
+  audio: {
+    uint8Array: Uint8Array
+    base64: string
+    mediaType: string
+    format: string
+  }
+  warnings?: unknown[]
+  providerMetadata?: Record<string, unknown>
+  responses?: unknown[]
+}
+
+export type AIGenerateImagePrompt =
+  | string
+  | {
+      images: Array<Uint8Array | ArrayBuffer | string>
+      text?: string
+      mask?: Uint8Array | ArrayBuffer | string
+    }
+
+export type AIGenerateImageParams = {
+  model: string
+  prompt: AIGenerateImagePrompt
+  n?: number
+  maxImagesPerCall?: number
+  size?: `${number}x${number}`
+  aspectRatio?: `${number}:${number}`
+  seed?: number
+  providerOptions?: AIProviderOptions
+  maxRetries?: number
+  abortSignal?: AbortSignal
+  headers?: Record<string, string>
+}
+
+export type AIGenerateImageResult = {
+  images: Array<{
+    uint8Array: Uint8Array
+    base64: string
+    mediaType: string
+  }>
+  warnings?: unknown[]
+  providerMetadata?: Record<string, unknown>
+  responses?: unknown[]
+  usage?: {
+    inputTokens?: number
+    outputTokens?: number
+    totalTokens?: number
+  }
+}
+
+export type AIEmbedParams = {
+  model: string
+  value: string
+  providerOptions?: AIProviderOptions
+  maxRetries?: number
+  abortSignal?: AbortSignal
+  headers?: Record<string, string>
+}
+
+export type AIEmbedResult = {
+  value: string
+  embedding: number[]
+  usage?: { tokens?: number }
+  warnings?: unknown[]
+  providerMetadata?: Record<string, unknown>
+  response?: unknown
+}
+
+export type AIEmbedManyParams = {
+  model: string
+  values: string[]
+  providerOptions?: AIProviderOptions
+  maxRetries?: number
+  abortSignal?: AbortSignal
+  headers?: Record<string, string>
+  maxParallelCalls?: number
+}
+
+export type AIEmbedManyResult = {
+  values: string[]
+  embeddings: number[][]
+  usage?: { tokens?: number }
+  warnings?: unknown[]
+  providerMetadata?: Record<string, unknown>
+  responses?: unknown[]
+}
+
+export type AIRerankParams<VALUE extends string | Record<string, unknown>> = {
+  model: string
+  query: string
+  documents: VALUE[]
+  topK?: number
+  providerOptions?: AIProviderOptions
+  maxRetries?: number
+  abortSignal?: AbortSignal
+  headers?: Record<string, string>
+}
+
+export type AIRerankResult<VALUE extends string | Record<string, unknown>> = {
+  ranking: Array<{
+    index: number
+    document: VALUE
+    score: number
+  }>
+  rerankedDocuments: VALUE[]
+  originalDocuments: VALUE[]
+  providerMetadata?: Record<string, unknown>
+  response?: unknown
+}
+
 export interface AIAgentRunnerService {
   stream(
     params: AIAgentRunnerParams,
     channel: AIStreamChannel
   ): Promise<AIAgentStepResult>
   run(params: AIAgentRunnerParams): Promise<AIAgentStepResult>
+  transcribe?(params: AITranscriptionParams): Promise<AITranscriptionResult>
+  generateSpeech?(
+    params: AIGenerateSpeechParams
+  ): Promise<AIGenerateSpeechResult>
+  generateImage?(params: AIGenerateImageParams): Promise<AIGenerateImageResult>
+  embed?(params: AIEmbedParams): Promise<AIEmbedResult>
+  embedMany?(params: AIEmbedManyParams): Promise<AIEmbedManyResult>
+  rerank?<VALUE extends string | Record<string, unknown>>(
+    params: AIRerankParams<VALUE>
+  ): Promise<AIRerankResult<VALUE>>
   /** Return a new runner that uses the given API key for every LLM call.
    *  Optional — runners that don't support per-key scoping leave this undefined. */
   withApiKey?(apiKey: string): AIAgentRunnerService

package/src/services/index.ts

@@ -69,6 +69,20 @@ export type {
   AIAgentRunnerParams,
   AIAgentRunnerResult,
   AIAgentStepResult,
+  AIEmbedManyParams,
+  AIEmbedManyResult,
+  AIEmbedParams,
+  AIEmbedResult,
+  AIGenerateImageParams,
+  AIGenerateImagePrompt,
+  AIGenerateImageResult,
+  AIGenerateSpeechParams,
+  AIGenerateSpeechResult,
+  AIProviderOptions,
+  AIRerankParams,
+  AIRerankResult,
+  AITranscriptionParams,
+  AITranscriptionResult,
   AIAgentRunnerService,
 } from './ai-agent-runner-service.js'
 export type {

package/src/services/meta-service.ts

@@ -212,7 +212,6 @@ export class LocalMetaService implements MetaService {
   private secretsMetaCache: SecretDefinitionsMeta | null = null
   private credentialsMetaCache: CredentialDefinitionsMeta | null = null
   private variablesMetaCache: VariableDefinitionsMeta | null = null
-  private emailMetaCache: EmailsMeta | null = null
   private middlewareGroupsMetaCache: MiddlewareGroupsMeta | null = null
   private permissionsGroupsMetaCache: PermissionsGroupsMeta | null = null
   private agentsMetaCache: AgentsMeta | null = null
@@ -262,7 +261,6 @@ export class LocalMetaService implements MetaService {
     this.secretsMetaCache = null
     this.credentialsMetaCache = null
     this.variablesMetaCache = null
-    this.emailMetaCache = null
     this.middlewareGroupsMetaCache = null
     this.permissionsGroupsMetaCache = null
     this.agentsMetaCache = null
@@ -509,17 +507,20 @@ export class LocalMetaService implements MetaService {
   }
 
   async getEmailMeta(): Promise<EmailsMeta> {
-    if (this.emailMetaCache) return this.emailMetaCache
-
+    // Read fresh every call — never cache. The email meta file is generated by
+    // `pikku all`/`pikku emails generate` and is regenerated DURING a long-lived
+    // session (sandbox boots the orchestrator before the user project's codegen
+    // has written it). Caching here once returned an empty {templates:{}} that
+    // then stuck for the whole session, leaving the console emails screen blank
+    // even after the file appeared. A local JSON read is essentially free.
     const content = await this.readFile('email/pikku-emails-meta.gen.json')
-    this.emailMetaCache = content
+    return content
       ? JSON.parse(content)
       : {
           src: '',
           themeHash: '',
           templates: {},
         }
-    return this.emailMetaCache!
   }
 
   async getEmailTemplateAssets(

package/src/wirings/ai-agent/index.ts

@@ -6,6 +6,8 @@ export {
 } from './ai-agent-helpers.js'
 export { runAIAgent, resumeAIAgentSync } from './ai-agent-runner.js'
 export { streamAIAgent, resumeAIAgent } from './ai-agent-stream.js'
+export { voiceInput } from './voice-input.js'
+export { voiceOutput } from './voice-output.js'
 export {
   type RunAIAgentParams,
   type StreamAIAgentOptions,

package/src/wirings/ai-agent/voice-input.ts

@@ -0,0 +1,132 @@
+import type { AIAgentRunnerService } from '../../services/ai-agent-runner-service.js'
+import { pikkuAIMiddleware } from '../../types/core.types.js'
+import type { AIContentPart } from './ai-agent.types.js'
+
+function base64ToUint8Array(base64: string): Uint8Array {
+  const binary = atob(base64)
+  const bytes = new Uint8Array(binary.length)
+  for (let i = 0; i < binary.length; i++) {
+    bytes[i] = binary.charCodeAt(i)
+  }
+  return bytes
+}
+
+const MAX_AUDIO_SIZE = 50 * 1024 * 1024
+
+// Portable SSRF guard: @pikku/core runs in edge runtimes (CF Workers) with no
+// Node `dns`, so we cannot resolve hostnames to check for private targets.
+// Reject the obvious internal literals; callers wanting stricter control pass
+// an explicit `allowedAudioHosts` allowlist. (Does not defend against a public
+// hostname that resolves to a private IP / DNS rebinding — out of reach here.)
+function isPrivateHost(hostname: string): boolean {
+  const host = hostname.replace(/^\[|\]$/g, '').toLowerCase()
+  if (host === 'localhost' || host === '0.0.0.0' || host === '::1') return true
+  if (host.startsWith('fe80:') || host.startsWith('fc') || host.startsWith('fd'))
+    return true
+  const v4 = host.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.\d{1,3}$/)
+  if (v4) {
+    const [a, b] = [Number(v4[1]), Number(v4[2])]
+    if (a === 127 || a === 10 || a === 0) return true
+    if (a === 169 && b === 254) return true // link-local incl. cloud metadata
+    if (a === 172 && b >= 16 && b <= 31) return true
+    if (a === 192 && b === 168) return true
+  }
+  return false
+}
+
+async function fetchAsUint8Array(
+  url: string,
+  allowedAudioHosts?: string[]
+): Promise<Uint8Array> {
+  const parsed = new URL(url)
+  if (parsed.protocol !== 'https:' && parsed.protocol !== 'http:') {
+    throw new Error('Only HTTP(S) URLs are supported for audio')
+  }
+  if (allowedAudioHosts) {
+    if (!allowedAudioHosts.includes(parsed.hostname)) {
+      throw new Error(`Audio URL host is not allowed: ${parsed.hostname}`)
+    }
+  } else if (isPrivateHost(parsed.hostname)) {
+    throw new Error(
+      `Refusing to fetch audio from a private/internal host: ${parsed.hostname}`
+    )
+  }
+  const response = await fetch(url)
+  const contentLength = response.headers.get('content-length')
+  if (contentLength && parseInt(contentLength, 10) > MAX_AUDIO_SIZE) {
+    throw new Error('Audio file exceeds maximum size')
+  }
+  const buffer = await response.arrayBuffer()
+  if (buffer.byteLength > MAX_AUDIO_SIZE) {
+    throw new Error('Audio file exceeds maximum size')
+  }
+  return new Uint8Array(buffer)
+}
+
+export const voiceInput = (config?: {
+  language?: string
+  model?: string
+  allowedAudioHosts?: string[]
+}) =>
+  pikkuAIMiddleware({
+    modifyInput: async (services, { messages, instructions }) => {
+      const transcribeAudio = (services as {
+        aiAgentRunner?: AIAgentRunnerService
+      }).aiAgentRunner?.transcribe
+      if (!transcribeAudio) return { messages, instructions }
+
+      const last = messages[messages.length - 1]
+      if (!last || last.role !== 'user' || typeof last.content === 'string') {
+        return { messages, instructions }
+      }
+
+      const parts = last.content as AIContentPart[]
+      if (!parts) return { messages, instructions }
+
+      const hasAudio = parts.some(
+        (p) => p.type === 'file' && !!p.mediaType?.startsWith('audio/')
+      )
+      if (!hasAudio) return { messages, instructions }
+
+      // Process parts in order, replacing each audio part with its transcription
+      // in place. Sequential, so no unbounded parallel downloads/transcriptions
+      // and original content ordering is preserved.
+      const updatedContent: AIContentPart[] = []
+      for (const p of parts) {
+        if (!(p.type === 'file' && p.mediaType?.startsWith('audio/'))) {
+          updatedContent.push(p)
+          continue
+        }
+        if (!config?.model) {
+          throw new Error(
+            'voiceInput requires a transcription model (e.g. openai/whisper-1)'
+          )
+        }
+        const audioData = p.data
+          ? base64ToUint8Array(p.data)
+          : await fetchAsUint8Array(p.url!, config.allowedAudioHosts)
+        const result = await transcribeAudio({
+          model: config.model,
+          audio: audioData,
+          ...(config.language
+            ? {
+                providerOptions: {
+                  openai: {
+                    language: config.language,
+                  },
+                },
+              }
+            : {}),
+        })
+        updatedContent.push({ type: 'text' as const, text: result.text })
+      }
+
+      return {
+        messages: [
+          ...messages.slice(0, -1),
+          { ...last, content: updatedContent },
+        ],
+        instructions,
+      }
+    },
+  })

package/src/wirings/ai-agent/voice-output.ts

@@ -0,0 +1,134 @@
+import type { AIAgentRunnerService } from '../../services/ai-agent-runner-service.js'
+import { pikkuAIMiddleware } from '../../types/core.types.js'
+
+type VoiceOutputState = {
+  textBuffer?: string
+}
+
+function bufferToBase64(data: Uint8Array): string {
+  let binary = ''
+  for (let i = 0; i < data.length; i++) {
+    binary += String.fromCharCode(data[i]!)
+  }
+  return btoa(binary)
+}
+
+const SENTENCE_BOUNDARY = /[.!?]\s*$/
+
+function isSentenceBoundary(text: string): boolean {
+  return SENTENCE_BOUNDARY.test(text)
+}
+
+async function synthesizeAudio(
+  aiAgentRunner: AIAgentRunnerService,
+  input: {
+    model: string
+    text: string
+    voice?: string
+    format?: string
+    instructions?: string
+    speed?: number
+    language?: string
+  }
+): Promise<{ bytes: Uint8Array; format: string }> {
+  const result = await aiAgentRunner.generateSpeech?.({
+    model: input.model,
+    text: input.text,
+    voice: input.voice,
+    outputFormat: input.format,
+    instructions: input.instructions,
+    speed: input.speed,
+    language: input.language,
+  })
+  if (!result) {
+    throw new Error(
+      'voiceOutput requires an aiAgentRunner with generateSpeech support'
+    )
+  }
+  // Label chunks with the format the provider actually returned (config.format
+  // is only a request), falling back to the requested format then pcm16.
+  return {
+    bytes: result.audio.uint8Array,
+    format: result.audio.format || input.format || 'pcm16',
+  }
+}
+
+export const voiceOutput = (config?: {
+  model?: string
+  format?: string
+  voice?: string
+  instructions?: string
+  speed?: number
+  language?: string
+}) =>
+  pikkuAIMiddleware<VoiceOutputState>({
+    modifyOutputStream: async (services, { event, state }) => {
+      const aiAgentRunner = (services as {
+        aiAgentRunner?: AIAgentRunnerService
+      }).aiAgentRunner
+      if (!aiAgentRunner?.generateSpeech) return event
+
+      if (event.type === 'done') {
+        const remaining = state.textBuffer ?? ''
+        if (remaining) {
+          state.textBuffer = ''
+          if (!config?.model) {
+            throw new Error(
+              'voiceOutput requires a speech model (e.g. openai/tts-1)'
+            )
+          }
+          const audio = await synthesizeAudio(aiAgentRunner, {
+            model: config.model,
+            text: remaining,
+            voice: config?.voice,
+            format: config?.format,
+            instructions: config?.instructions,
+            speed: config?.speed,
+            language: config?.language,
+          })
+          return [
+            {
+              type: 'audio-delta' as const,
+              data: bufferToBase64(audio.bytes),
+              format: audio.format,
+            },
+            { type: 'audio-done' as const },
+            event,
+          ]
+        }
+        return [{ type: 'audio-done' as const }, event]
+      }
+
+      if (event.type !== 'text-delta') return event
+
+      state.textBuffer = `${state.textBuffer ?? ''}${event.text}`
+      if (!isSentenceBoundary(state.textBuffer)) return event
+
+      const text = state.textBuffer
+      state.textBuffer = ''
+
+      if (!config?.model) {
+        throw new Error(
+          'voiceOutput requires a speech model (e.g. openai/tts-1)'
+        )
+      }
+      const audio = await synthesizeAudio(aiAgentRunner, {
+        model: config.model,
+        text,
+        voice: config?.voice,
+        format: config?.format,
+        instructions: config?.instructions,
+        speed: config?.speed,
+        language: config?.language,
+      })
+
+      return [
+        event,
+        {
+          type: 'audio-delta' as const,
+          data: bufferToBase64(audio.bytes),
+          format: audio.format,
+        },
+      ]
+    },
+  })