@pikku/core 0.12.72 → 0.12.77

package/src/wirings/ai-agent/voice-input.test.ts

@@ -1,15 +1,15 @@
 import { describe, test } from 'node:test'
 import assert from 'node:assert/strict'
 
-import { voiceInput } from './voice-input.js'
+import {
+  voiceInput,
+  readsAsNonSpeech,
+  NoSpeechDetectedError,
+  SPOKEN_TURN,
+  SPOKEN_TRANSCRIPT,
+} from './voice-input.js'
 import type { AIContentPart, AIMessage } from './ai-agent.types.js'
 
-/**
- * A runner whose `transcribe` reads instance state through `this`, mirroring the
- * real `VercelAIAgentRunner` where `transcribe` calls `this.getModel(...)`. If
- * the middleware grabs the method as a bare reference the receiver is lost and
- * `this` is undefined — which is exactly the regression this guards.
- */
 class ThisDependentRunner {
   private readonly transcript = 'the transcribed spoken words'
 
@@ -41,6 +41,7 @@ describe('voiceInput', () => {
     const result = await mw.modifyInput!(services as any, {
       messages: [audioMessage()],
       instructions: 'sys',
+      shared: {},
     })
 
     const last = result.messages[result.messages.length - 1]!
@@ -69,6 +70,7 @@ describe('voiceInput', () => {
     const result = await mw.modifyInput!(services as any, {
       messages,
       instructions: 'sys',
+      shared: {},
     })
 
     assert.equal(result.messages[0]!.content, 'just text')
@@ -83,8 +85,136 @@ describe('voiceInput', () => {
         mw.modifyInput!(services as any, {
           messages: [audioMessage()],
           instructions: 'sys',
+          shared: {},
         }),
       /voiceInput requires a transcription model/
     )
   })
+
+  test('a turn with no speech in it never reaches the model', async () => {
+    // The failure this prevents: the user says nothing, the model answers
+    // anyway, and what it answers is a guess about audio it could not hear.
+    // An ASR that returns '' on non-speech is what makes this checkable — see
+    // readsAsNonSpeech for why nothing subtler than '' survived contact.
+    const silent = {
+      async transcribe() {
+        return { text: '', segments: [], warnings: [] }
+      },
+    }
+    const mw = voiceInput({
+      model: 'deepinfra/nvidia/Nemotron-3.5-ASR-Streaming-Multilingual-0.6b',
+    })
+
+    await assert.rejects(
+      () =>
+        mw.modifyInput!({ aiAgentRunner: silent } as any, {
+          messages: [audioMessage()],
+          instructions: 'sys',
+          shared: {},
+        }),
+      NoSpeechDetectedError
+    )
+  })
+
+  test('a transcript is passed through exactly as the provider wrote it', async () => {
+    // No reassembly, no trimming, no filtering — whatever the model heard is
+    // what the agent is asked about.
+    const spaced = {
+      async transcribe() {
+        return {
+          text: '  spacing   the provider chose  ',
+          segments: [],
+          warnings: [],
+        }
+      },
+    }
+    const mw = voiceInput({ model: 'mock/asr' })
+
+    const result = await mw.modifyInput!({ aiAgentRunner: spaced } as any, {
+      messages: [audioMessage()],
+      instructions: 'sys',
+      shared: {},
+    })
+
+    const parts = result.messages.at(-1)!.content as AIContentPart[]
+    assert.equal(
+      (parts[0] as { text: string }).text,
+      '  spacing   the provider chose  '
+    )
+  })
+})
+
+describe('readsAsNonSpeech', () => {
+  test('only an empty transcript counts as nothing said', () => {
+    assert.equal(readsAsNonSpeech({ text: '' }), true)
+    assert.equal(readsAsNonSpeech({ text: '   ' }), true)
+    assert.equal(readsAsNonSpeech({}), true)
+    assert.equal(readsAsNonSpeech({ text: 'add milk to the list' }), false)
+  })
+
+  test('a short or low-confidence-looking transcript is still speech', () => {
+    // The gate that used to live here would have been tempted by these. "Yes"
+    // is the single most important word in an approval flow, and dropping it
+    // because it is brief or quiet is far worse than keeping a stray one.
+    assert.equal(readsAsNonSpeech({ text: 'yes' }), false)
+    assert.equal(readsAsNonSpeech({ text: 'Thank you.' }), false)
+  })
+  test('records that the turn was spoken, for voiceOutput to read', async () => {
+    // The note has to be written here: the transcript this middleware puts in
+    // the message's place is indistinguishable from something typed, so no
+    // later middleware can tell what it started as.
+    const mw = voiceInput({ model: 'mock/whisper' })
+    const shared: Record<string, unknown> = {}
+
+    await mw.modifyInput!({ aiAgentRunner: new ThisDependentRunner() } as any, {
+      messages: [audioMessage()],
+      instructions: 'sys',
+      shared,
+    })
+
+    assert.equal(shared[SPOKEN_TURN], true)
+  })
+
+  test('records a typed turn as not spoken, rather than leaving it unsaid', async () => {
+    const mw = voiceInput({ model: 'mock/whisper' })
+    const shared: Record<string, unknown> = {}
+
+    await mw.modifyInput!({ aiAgentRunner: new ThisDependentRunner() } as any, {
+      messages: [{ role: 'user', content: 'typed' } as any],
+      instructions: 'sys',
+      shared,
+    })
+
+    // Explicitly false, not absent: absent is what a caller with no voice input
+    // at all looks like, and that one should still be spoken to.
+    assert.equal(shared[SPOKEN_TURN], false)
+  })
+
+  test('records what was heard, so the client can show the user their own turn', async () => {
+    const mw = voiceInput({ model: 'mock/whisper' })
+    const shared: Record<string, unknown> = {}
+
+    await mw.modifyInput!({ aiAgentRunner: new ThisDependentRunner() } as any, {
+      messages: [audioMessage()],
+      instructions: 'sys',
+      shared,
+    })
+
+    assert.equal(shared[SPOKEN_TRANSCRIPT], 'the transcribed spoken words')
+  })
+
+  test('a typed turn leaves no transcript behind', async () => {
+    // Absence is the signal: the stream only announces a transcript when there
+    // was one, and a typed message is already on the client's screen.
+    const mw = voiceInput({ model: 'mock/whisper' })
+    const shared: Record<string, unknown> = {}
+
+    await mw.modifyInput!({ aiAgentRunner: new ThisDependentRunner() } as any, {
+      messages: [{ role: 'user', content: 'typed' } as any],
+      instructions: 'sys',
+      shared,
+    })
+
+    assert.equal(SPOKEN_TRANSCRIPT in shared, false)
+  })
 })

package/src/wirings/ai-agent/voice-input.ts

@@ -14,6 +14,46 @@ function base64ToUint8Array(base64: string): Uint8Array {
 
 const MAX_AUDIO_SIZE = 50 * 1024 * 1024
 
+/**
+ * Raised when every audio part in a turn turned out to be non-speech.
+ *
+ * Distinct from a transcription failure, because the response is different: a
+ * failure is worth reporting, whereas this is the microphone doing its job on a
+ * pause. A voice loop should catch it and go back to listening without running
+ * the agent — answering a hallucinated sentence is worse than answering nothing.
+ */
+export class NoSpeechDetectedError extends Error {
+  constructor() {
+    super('No speech was detected in the audio.')
+    this.name = 'NoSpeechDetectedError'
+  }
+}
+
+/**
+ * Whether nothing usable was said — an empty transcript, and nothing cleverer
+ * than that.
+ *
+ * There was a confidence gate here once, and it is worth recording why it is
+ * gone. Whisper is trained on subtitles, so audio it cannot parse comes back
+ * not as nothing but as stock filler appended to real speech: "Hello, my name
+ * is Yasir. Thank you." where only the first sentence was spoken. The obvious
+ * defence is to drop low-confidence segments, and it does not work. Measured on
+ * a real turn, the invented "Thank you." scored `avg_logprob` -0.52 against
+ * -0.30 for the genuine sentence beside it, and `no_speech_prob` read 0.000 for
+ * both. Whisper is *confident* when it invents, because the text it invents is
+ * high-probability text. No threshold separates those two numbers, and one set
+ * low enough to try would take real quiet speech with it.
+ *
+ * The fix belongs at the model, not behind it: an ASR that is not a
+ * subtitle-trained autoregressive decoder returns an empty string on non-speech
+ * and there is nothing left to filter. Hence this reduces to asking whether
+ * anything came back at all — which is a property every provider reports
+ * honestly, unlike per-segment confidence (Nemotron, for one, hardcodes
+ * `avg_logprob` to 0.0).
+ */
+export const readsAsNonSpeech = (result: { text?: string }): boolean =>
+  !(result.text ?? '').trim()
+
 async function fetchAsUint8Array(
   url: string,
   allowedAudioHosts?: string[]
@@ -30,13 +70,35 @@ async function fetchAsUint8Array(
   return new Uint8Array(buffer)
 }
 
+/**
+ * Shared-notes key recording whether the user's turn arrived as audio.
+ *
+ * Written here because this is the last point at which it is knowable: the
+ * transcript that replaces the audio is indistinguishable from something typed.
+ * Read by `voiceOutput` to decide whether to answer aloud.
+ *
+ * Absent — rather than `false` — when this middleware is not wired at all, which
+ * is what lets `voiceOutput` still speak for a caller that has no voice input.
+ */
+export const SPOKEN_TURN = 'voice:spokenTurn'
+
+/**
+ * Shared-notes key holding what the user was heard to say, when they spoke.
+ *
+ * Absent on a typed turn. Read by the stream wiring, which forwards it to the
+ * client as a `transcript` event — a voice client has no idea what its own
+ * audio said, and without this the user's turn shows up in the UI as an empty
+ * bubble followed by an answer to a question they cannot see.
+ */
+export const SPOKEN_TRANSCRIPT = 'voice:transcript'
+
 export const voiceInput = (config?: {
   language?: string
   model?: string
   allowedAudioHosts?: string[]
 }) =>
   pikkuAIMiddleware({
-    modifyInput: async (services, { messages, instructions }) => {
+    modifyInput: async (services, { messages, instructions, shared }) => {
       const aiAgentRunner = (
         services as {
           aiAgentRunner?: AIAgentRunnerService
@@ -45,22 +107,22 @@ export const voiceInput = (config?: {
       if (!aiAgentRunner?.transcribe) return { messages, instructions }
 
       const last = messages[messages.length - 1]
-      if (!last || last.role !== 'user' || typeof last.content === 'string') {
-        return { messages, instructions }
-      }
-
-      const parts = last.content as AIContentPart[]
-      if (!parts) return { messages, instructions }
+      const parts =
+        last?.role === 'user' && typeof last.content !== 'string'
+          ? (last.content as AIContentPart[] | undefined)
+          : undefined
 
-      const hasAudio = parts.some(
+      const hasAudio = !!parts?.some(
         (p) => p.type === 'file' && !!p.mediaType?.startsWith('audio/')
       )
-      if (!hasAudio) return { messages, instructions }
+      // Recorded on every turn this middleware sees, spoken or not, so that a
+      // typed turn is an explicit `false` downstream rather than a silence that
+      // could equally mean nobody was asked.
+      shared[SPOKEN_TURN] = hasAudio
+      if (!hasAudio || !parts) return { messages, instructions }
 
-      // Process parts in order, replacing each audio part with its transcription
-      // in place. Sequential, so no unbounded parallel downloads/transcriptions
-      // and original content ordering is preserved.
       const updatedContent: AIContentPart[] = []
+      const heard: string[] = []
       for (const p of parts) {
         if (!(p.type === 'file' && p.mediaType?.startsWith('audio/'))) {
           updatedContent.push(p)
@@ -74,6 +136,9 @@ export const voiceInput = (config?: {
         const audioData = p.data
           ? base64ToUint8Array(p.data)
           : await fetchAsUint8Array(p.url!, config.allowedAudioHosts)
+        if (audioData.byteLength > MAX_AUDIO_SIZE) {
+          throw new Error('Audio file exceeds maximum size')
+        }
         const result = await aiAgentRunner.transcribe({
           model: config.model,
           audio: audioData,
@@ -87,9 +152,28 @@ export const voiceInput = (config?: {
               }
             : {}),
         })
+        // Dropped when nothing was said: an empty text part is still a turn the
+        // model will answer, and what it answers is a guess about what it could
+        // not hear.
+        if (readsAsNonSpeech(result)) continue
+        heard.push(result.text)
         updatedContent.push({ type: 'text' as const, text: result.text })
       }
 
+      // Every part was audio, and none of it was speech. There is nothing left
+      // to send, and a message with no content is not a question.
+      if (updatedContent.length === 0) throw new NoSpeechDetectedError()
+
+      // Only when something was actually heard. A turn can carry audio that all
+      // reads as non-speech and still have content — an image with a silent
+      // caption clip — which leaves nothing above to throw. Recording `''` here
+      // would send a transcript event saying the user said nothing, and a client
+      // that tells "not transcribed yet" from "transcribed" by the key being
+      // absent would render the turn as an empty bubble rather than a pending one.
+      if (heard.length > 0) {
+        shared[SPOKEN_TRANSCRIPT] = heard.join(' ')
+      }
+
       return {
         messages: [
           ...messages.slice(0, -1),