@pikku/core 0.12.72 → 0.12.77

package/src/wirings/ai-agent/voice-output.ts

@@ -1,8 +1,21 @@
 import type { AIAgentRunnerService } from '../../services/ai-agent-runner-service.js'
 import { pikkuAIMiddleware } from '../../types/core.types.js'
+import { SPOKEN_TURN } from './voice-input.js'
 
 type VoiceOutputState = {
   textBuffer?: string
+  /**
+   * The audio emitted so far, as a chain.
+   *
+   * Synthesis for every finished sentence starts immediately, so the second
+   * sentence is being generated while the first is still being spoken. That
+   * also means the second can finish first — a short sentence after a long one
+   * routinely does. Chaining the *emissions* keeps the client receiving audio
+   * in the order it was written while leaving the generation overlapped.
+   */
+  tail?: Promise<void>
+  /** Whether this reply has already said it cannot be spoken aloud. */
+  reportedUnspeakable?: boolean
 }
 
 function bufferToBase64(data: Uint8Array): string {
@@ -19,6 +32,102 @@ function isSentenceBoundary(text: string): boolean {
   return SENTENCE_BOUNDARY.test(text)
 }
 
+/**
+ * Scripts a speech model can pronounce, as named by {@link voiceOutput}'s
+ * `speakableScripts`.
+ *
+ * Script rather than language because script is what is decidable from the text
+ * alone. Spanish, French and English are one test, not three — and the models
+ * that handle one Latin-script language generally handle the others. What a
+ * model cannot do is invent a phoneme set it was never given.
+ */
+const SCRIPT_RANGES: Record<string, RegExp> = {
+  latin: /[A-ɏ]/,
+  devanagari: /[ऀ-ॿ]/,
+  han: /[㐀-鿿]/,
+  kana: /[぀-ヿ]/,
+  arabic: /[؀-ۿݐ-ݿ]/,
+  cyrillic: /[Ѐ-ӿ]/,
+  hangul: /[가-힯ᄀ-ᇿ]/,
+  hebrew: /[֐-׿]/,
+  greek: /[Ͱ-Ͽ]/,
+  thai: /[฀-๿]/,
+}
+
+/**
+ * The scripts a model can pronounce, and optionally the voice each one needs.
+ *
+ * The array form is for a model whose voice is the same whatever the script.
+ * The record form exists because that is not the common case: Kokoro speaks
+ * Mandarin, Japanese and Hindi well, but only through a voice belonging to that
+ * language. Asked for Chinese in its default American-English voice it spells
+ * the characters out — 9.9 seconds of "Chinese letter, Chinese letter" for a
+ * sentence the right voice says in 3.5. Same model, same text, same price.
+ */
+export type SpeakableScripts = string[] | Record<string, string>
+
+/**
+ * Which script decides the voice when a sentence contains several.
+ *
+ * The ordering is by how much a script narrows down the language, not by
+ * preference. Kana appears only in Japanese, so a sentence holding any settles
+ * it — while Han is written in both Chinese and Japanese and so cannot, which
+ * matters because ordinary Japanese is mostly kanji. Latin comes last for the
+ * same reason from the other end: it turns up inside sentences in every other
+ * script, and it is the one whose voice is least often the one that counts. A
+ * Chinese voice reading an English word is accented; an English voice reading
+ * Chinese spells it out.
+ *
+ * Everything else — Han included — sits between the two in config order.
+ */
+const SCRIPT_PRECEDENCE: Record<string, number> = { kana: -1, latin: 1 }
+
+const scriptRank = (name: string): number => SCRIPT_PRECEDENCE[name] ?? 0
+
+const scriptNames = (speakable: SpeakableScripts): string[] =>
+  Array.isArray(speakable) ? speakable : Object.keys(speakable)
+
+/**
+ * The scripts present in `text` that the model was not said to handle.
+ *
+ * Silence would be the wrong answer here and so would a best effort. Handed
+ * Arabic — which it has no voice for at all — Kokoro does not fail and does not
+ * stay quiet: it reads out the *letter names*, twenty-four seconds of "Arabic
+ * meem, Arabic ra" for a one-line sentence, which is worse than either. Naming
+ * the gap lets the caller say so instead.
+ */
+export const unspeakableScripts = (
+  text: string,
+  speakable: SpeakableScripts
+): string[] => {
+  const allowed = new Set(scriptNames(speakable))
+  return Object.entries(SCRIPT_RANGES)
+    .filter(([name, range]) => !allowed.has(name) && range.test(text))
+    .map(([name]) => name)
+}
+
+/**
+ * The voice to speak `text` in, or `fallback` if no mapped script appears.
+ *
+ * A sentence is regularly in more than one script, so which one decides is
+ * settled by {@link SCRIPT_PRECEDENCE} rather than by the order the config
+ * happens to be written in.
+ */
+export const voiceForText = (
+  text: string,
+  speakable: SpeakableScripts,
+  fallback?: string
+): string | undefined => {
+  if (Array.isArray(speakable)) return fallback
+  const mapped = Object.entries(speakable).sort(
+    ([a], [b]) => scriptRank(a) - scriptRank(b)
+  )
+  for (const [name, voice] of mapped) {
+    if (SCRIPT_RANGES[name]?.test(text)) return voice
+  }
+  return fallback
+}
+
 async function synthesizeAudio(
   aiAgentRunner: AIAgentRunnerService,
   input: {
@@ -29,6 +138,7 @@ async function synthesizeAudio(
     instructions?: string
     speed?: number
     language?: string
+    abortSignal?: AbortSignal
   }
 ): Promise<{ bytes: Uint8Array; format: string }> {
   const result = await aiAgentRunner.generateSpeech?.({
@@ -39,14 +149,13 @@ async function synthesizeAudio(
     instructions: input.instructions,
     speed: input.speed,
     language: input.language,
+    abortSignal: input.abortSignal,
   })
   if (!result) {
     throw new Error(
       'voiceOutput requires an aiAgentRunner with generateSpeech support'
     )
   }
-  // Label chunks with the format the provider actually returned (config.format
-  // is only a request), falling back to the requested format then pcm16.
   return {
     bytes: result.audio.uint8Array,
     format: result.audio.format || input.format || 'pcm16',
@@ -60,75 +169,153 @@ export const voiceOutput = (config?: {
   instructions?: string
   speed?: number
   language?: string
+  /**
+   * Scripts the configured model can actually pronounce, and the voice each
+   * needs — see {@link SpeakableScripts}. Anything written in another script is
+   * left unspoken and reported once, as a `voice-unsupported` data event,
+   * rather than sent to a model that will mangle it.
+   *
+   * Omitted means no check: every model is trusted with every script, which is
+   * the right default for a provider that genuinely is multilingual.
+   */
+  speakableScripts?: SpeakableScripts
+  /**
+   * Speak every reply, including answers to turns that were typed.
+   *
+   * The default is to speak only what was spoken to. An agent is usually one
+   * agent serving both kinds of caller — the same thread is typed at from a
+   * desk and talked to from a phone — and synthesizing a reply nobody is
+   * listening to costs real money on every sentence of every turn. Whether the
+   * user is in a call is not something the client needs to declare: sending
+   * audio is what saying so looks like.
+   *
+   * Set this for the cases where speech is not a reply to speech at all — a
+   * read-aloud accessibility mode, a kiosk, anything whose whole output is
+   * meant to be heard.
+   */
+  always?: boolean
 }) =>
   pikkuAIMiddleware<VoiceOutputState>({
-    modifyOutputStream: async (services, { event, state }) => {
-      const aiAgentRunner = (services as {
+    modifyOutputStream: async (
+      services,
+      { event, state, shared, emit, signal }
+    ) => {
+      const { aiAgentRunner, logger } = services as {
         aiAgentRunner?: AIAgentRunnerService
-      }).aiAgentRunner
+        logger?: { error: (message: string) => void }
+      }
       if (!aiAgentRunner?.generateSpeech) return event
 
-      if (event.type === 'done') {
-        const remaining = state.textBuffer ?? ''
-        if (remaining) {
-          state.textBuffer = ''
-          if (!config?.model) {
-            throw new Error(
-              'voiceOutput requires a speech model (e.g. openai/tts-1)'
+      // Only an explicit `false` silences the reply. `voiceInput` sets this on
+      // every turn it handles, so `false` means a real user really typed; the
+      // key being absent means nothing reported either way — no voice input is
+      // wired — and that caller's replies are spoken exactly as they were before
+      // this option existed.
+      if (!config?.always && shared[SPOKEN_TURN] === false) return event
+
+      /**
+       * Queue a finished sentence for speech.
+       *
+       * Nothing here is awaited by the caller. Synthesis is kicked off at once
+       * and its emission is chained behind the previous sentence, which is
+       * what keeps the text flowing at the speed the model writes it instead
+       * of the speed the speech provider answers.
+       */
+      const speak = (text: string) => {
+        if (!config?.model) {
+          throw new Error(
+            'voiceOutput requires a speech model (e.g. openai/tts-1)'
+          )
+        }
+
+        // Checked per sentence but announced once per reply: a bilingual answer
+        // should still speak the half it can, and repeating the notice for
+        // every sentence of a long one would bury the reply itself.
+        //
+        // Per sentence is also what makes the voice right. A reply that answers
+        // in English and then quotes a Chinese title is two sentences in two
+        // scripts, and each is synthesized in the voice its own script needs.
+        let voice = config.voice
+        if (config.speakableScripts) {
+          const unspeakable = unspeakableScripts(text, config.speakableScripts)
+          voice = voiceForText(text, config.speakableScripts, config.voice)
+          if (unspeakable.length > 0) {
+            if (!state.reportedUnspeakable) {
+              state.reportedUnspeakable = true
+              state.tail = (state.tail ?? Promise.resolve()).then(() =>
+                emit({
+                  type: 'data',
+                  name: 'voice-unsupported',
+                  data: { model: config.model, scripts: unspeakable },
+                })
+              )
+            }
+            return
+          }
+        }
+        // Settled into a value rather than left as a rejectable promise: it
+        // sits unobserved until the chain reaches it, and an unhandled
+        // rejection in that window would take the process down.
+        const synthesis = synthesizeAudio(aiAgentRunner, {
+          model: config.model,
+          text,
+          voice,
+          format: config?.format,
+          instructions: config?.instructions,
+          speed: config?.speed,
+          language: config?.language,
+          // Barge-in should stop the bill, not just the playback. Without this
+          // every sentence already in flight is synthesized in full and paid
+          // for after the user has stopped listening.
+          abortSignal: signal,
+        }).then(
+          (audio) => ({ ok: true as const, audio }),
+          (error: unknown) => ({ ok: false as const, error })
+        )
+
+        state.tail = (state.tail ?? Promise.resolve()).then(async () => {
+          const result = await synthesis
+          if (!result.ok) {
+            // One sentence going unspoken is better than the rest of the reply
+            // never arriving, so the chain carries on.
+            logger?.error(
+              `voiceOutput could not synthesize a sentence: ${String(result.error)}`
             )
+            return
           }
-          const audio = await synthesizeAudio(aiAgentRunner, {
-            model: config.model,
-            text: remaining,
-            voice: config?.voice,
-            format: config?.format,
-            instructions: config?.instructions,
-            speed: config?.speed,
-            language: config?.language,
+          await emit({
+            type: 'audio-delta',
+            data: bufferToBase64(result.audio.bytes),
+            format: result.audio.format,
+            text,
           })
-          return [
-            {
-              type: 'audio-delta' as const,
-              data: bufferToBase64(audio.bytes),
-              format: audio.format,
-            },
-            { type: 'audio-done' as const },
-            event,
-          ]
+        })
+      }
+
+      if (event.type === 'done') {
+        const remaining = state.textBuffer ?? ''
+        state.textBuffer = ''
+        if (remaining.trim()) {
+          speak(remaining)
         }
+        // The only place this hook waits. The reply is already over, so the
+        // wait costs the reader nothing, and it is what makes `audio-done`
+        // mean what it says rather than racing the last chunk.
+        await state.tail
+        state.tail = undefined
         return [{ type: 'audio-done' as const }, event]
       }
 
       if (event.type !== 'text-delta') return event
 
       state.textBuffer = `${state.textBuffer ?? ''}${event.text}`
-      if (!isSentenceBoundary(state.textBuffer)) return event
-
-      const text = state.textBuffer
-      state.textBuffer = ''
-
-      if (!config?.model) {
-        throw new Error(
-          'voiceOutput requires a speech model (e.g. openai/tts-1)'
-        )
+      if (isSentenceBoundary(state.textBuffer)) {
+        const text = state.textBuffer
+        state.textBuffer = ''
+        speak(text)
       }
-      const audio = await synthesizeAudio(aiAgentRunner, {
-        model: config.model,
-        text,
-        voice: config?.voice,
-        format: config?.format,
-        instructions: config?.instructions,
-        speed: config?.speed,
-        language: config?.language,
-      })
-
-      return [
-        event,
-        {
-          type: 'audio-delta' as const,
-          data: bufferToBase64(audio.bytes),
-          format: audio.format,
-        },
-      ]
+
+      // Returned without waiting for any audio — the entire point.
+      return event
     },
   })

package/src/wirings/channel/channel-common.ts

@@ -8,30 +8,15 @@ import type {
 import type { CoreChannel, ChannelMessageMeta } from './channel.types.js'
 import { combineMiddleware, runMiddleware } from '../../middleware-runner.js'
 import { runPikkuFunc } from '../../function/function-runner.js'
+import {
+  type PikkuSessionService,
+  createMiddlewareSessionWireProps,
+} from '../../services/user-session-service.js'
 import {
   combineChannelMiddleware,
   wrapChannelWithMiddleware,
 } from './channel-middleware-runner.js'
 
-/**
- * Runs a channel lifecycle function (onConnect or onDisconnect) with proper middleware handling.
- *
- * This function:
- * 1. Extracts inline middleware from the lifecycle config if present
- * 2. Combines metadata middleware with inline middleware using combineMiddleware()
- * 3. Wraps the channel with channel middleware if present
- * 4. Runs the lifecycle function with middleware support
- *
- * @param channelConfig - The channel configuration
- * @param meta - Metadata for the lifecycle function (connect or disconnect)
- * @param lifecycleConfig - The onConnect or onDisconnect config (can be function or object with middleware)
- * @param lifecycleType - Type of lifecycle for cache key ('connect' or 'disconnect')
- * @param services - All services (singleton + session)
- * @param channel - The channel instance
- * @param data - Optional data to pass to the lifecycle function (for onConnect)
- * @param channelMiddlewareMeta - Optional channel middleware metadata for wrapping channel.send()
- * @returns Promise<unknown> - Result from the lifecycle function (if any)
- */
 export const runChannelLifecycleWithMiddleware = async ({
   channelConfig,
   meta,
@@ -41,6 +26,7 @@ export const runChannelLifecycleWithMiddleware = async ({
   channel,
   data,
   channelMiddlewareMeta,
+  userSession,
 }: {
   channelConfig: CoreChannel<unknown, any>
   meta: ChannelMessageMeta
@@ -50,6 +36,11 @@ export const runChannelLifecycleWithMiddleware = async ({
   channel: any
   data?: unknown
   channelMiddlewareMeta?: MiddlewareMetadata[]
+  /**
+   * Established while the socket was upgrading, so `onConnect` can tell who
+   * just connected.
+   */
+  userSession?: PikkuSessionService<any>
 }): Promise<unknown> => {
   const lifecycleMiddleware =
     typeof lifecycleConfig === 'object' && 'middleware' in lifecycleConfig
@@ -74,7 +65,10 @@ export const runChannelLifecycleWithMiddleware = async ({
     }
   )
 
-  let wire: PikkuRawWire = { channel }
+  let wire: PikkuRawWire = {
+    channel,
+    ...(userSession ? createMiddlewareSessionWireProps(userSession) : {}),
+  }
   if (allChannelMiddleware.length > 0) {
     wire = wrapChannelWithMiddleware(wire, services, allChannelMiddleware)
   }
@@ -84,6 +78,7 @@ export const runChannelLifecycleWithMiddleware = async ({
       singletonServices: services,
       data: () => data as any,
       wire,
+      sessionService: userSession,
       tags: meta.tags ?? [],
       packageName: meta.packageName ?? null,
     })

package/src/wirings/channel/channel-handler.test.ts

@@ -4,6 +4,8 @@ import assert from 'node:assert/strict'
 import { addFunction } from '../../function/function-runner.js'
 import { pikkuState, resetPikkuState } from '../../pikku-state.js'
 import { processMessageHandlers } from './channel-handler.js'
+import { getChannelHostRPC, releaseChannelHostRPC } from './channel-host-rpc.js'
+import { CHANNEL_RPC_RESPONSE } from './channel-rpc.js'
 
 const createLogger = () => {
   const errors: string[] = []
@@ -240,6 +242,54 @@ describe('processMessageHandlers', () => {
     assert.match(errors[1] || '', /Channel config name: chat/)
   })
 
+  test('takes a reverse-RPC answer off the socket before routing', async () => {
+    const { errors, logger } = createLogger()
+    const { sent, handler } = createChannelHandler()
+
+    pikkuState(null, 'channel', 'meta').chat = {
+      name: 'chat',
+      route: '/chat',
+      input: null,
+      connect: null,
+      disconnect: null,
+      message: { pikkuFuncId: 'defaultFunc', packageName: null },
+      messageWirings: {},
+    } as never
+    registerFunction('defaultFunc', async () => ({ ok: true }))
+
+    const { onMessage } = processMessageHandlers(
+      { logger } as never,
+      {
+        name: 'chat',
+        route: '/chat',
+        auth: false,
+        onMessage: { func: async () => ({ ok: true }) } as never,
+      } as never,
+      handler as never
+    )
+
+    const transport = getChannelHostRPC(handler.getChannel() as never)
+    const call = transport.invoke('localCheckout', {})
+    const request = sent[0] as { id: string }
+
+    const result = await onMessage(
+      JSON.stringify({
+        action: CHANNEL_RPC_RESPONSE,
+        id: request.id,
+        ok: true,
+        result: { sha: 'deadbeef' },
+      })
+    )
+
+    assert.deepEqual(await call, { sha: 'deadbeef' })
+    // The channel declares no route for replies and its default handler is
+    // never reached — the answer belongs to the waiting caller, not the app.
+    assert.equal(result, undefined)
+    assert.deepEqual(errors, [])
+
+    await releaseChannelHostRPC(handler.getChannel().channelId)
+  })
+
   test('blocks binary messages when a session is required and missing', async () => {
     const { errors, logger } = createLogger()
     const { handler } = createChannelHandler()

package/src/wirings/channel/channel-handler.ts

@@ -11,6 +11,8 @@ import type {
 } from './channel.types.js'
 import { pikkuState } from '../../pikku-state.js'
 import { runPikkuFunc } from '../../function/function-runner.js'
+import { isChannelRPCPending, isChannelRPCResponse } from './channel-rpc.js'
+import { handleChannelRPCResponse } from './channel-host-rpc.js'
 import type { SessionService } from '../../services/user-session-service.js'
 import { createMiddlewareSessionWireProps } from '../../services/user-session-service.js'
 const getChannelMeta = (channelName: string) => {
@@ -159,14 +161,26 @@ export const processMessageHandlers = (
     let result: unknown
     let processed = false
 
-    // Route-specific handling
-    if (typeof rawData === 'string' && channelConfig.onMessageWiring) {
-      let messageData: any
+    let frame: any
+    if (typeof rawData === 'string') {
       try {
-        messageData = JSON.parse(rawData)
+        frame = JSON.parse(rawData)
       } catch {
         // Most likely a json error.. ignore
       }
+    }
+
+    // Taken ahead of routing so every channel gets reverse RPC without
+    // declaring a route for replies, and so an answer to a call someone is
+    // still awaiting can never be mistaken for a new message.
+    if (isChannelRPCResponse(frame) || isChannelRPCPending(frame)) {
+      handleChannelRPCResponse(channelHandler.getChannel().channelId, frame)
+      return
+    }
+
+    // Route-specific handling
+    if (typeof rawData === 'string' && channelConfig.onMessageWiring) {
+      const messageData: any = frame
 
       if (messageData) {
         const entries = Object.entries(channelConfig.onMessageWiring)
@@ -176,14 +190,21 @@ export const processMessageHandlers = (
             const { [routingProperty]: _, ...data } = messageData
 
             processed = true
+            const routeResult = await processMessage(
+              data as any,
+              routes[routerValue],
+              routingProperty,
+              routerValue
+            )
+            // The routing key is echoed so the client can match the reply.
+            // Assigning it to a primitive throws under ESM strict mode, so a
+            // primitive result is wrapped rather than mutated.
             result =
-              (await processMessage(
-                data as any,
-                routes[routerValue],
-                routingProperty,
-                routerValue
-              )) || {}
-            ;(result as any)[routingProperty] = routerValue
+              routeResult !== null && typeof routeResult === 'object'
+                ? Object.assign(routeResult, { [routingProperty]: routerValue })
+                : routeResult === undefined
+                  ? { [routingProperty]: routerValue }
+                  : { [routingProperty]: routerValue, result: routeResult }
             break
           }
         }