@pikku/core 0.12.107 → 0.12.112

package/src/wirings/agent/index.ts

@@ -1,58 +0,0 @@
-export {
-  agent,
-  agentStream,
-  agentResume,
-  agentApprove,
-  agentInterrupt,
-} from './agent-helpers.js'
-export { wrapChannelWithAGUI } from './agent-agui.js'
-export { runAgent, resumeAgentSync } from './agent-runner.js'
-export { resolveModelAlias } from './agent-model-config.js'
-export { streamAgent, resumeAgent, interruptAgent } from './agent-stream.js'
-export {
-  voiceInput,
-  NoSpeechDetectedError,
-  SPOKEN_TURN,
-  SPOKEN_TRANSCRIPT,
-} from './voice-input.js'
-export {
-  voiceOutput,
-  unspeakableScripts,
-  voiceForText,
-  type SpeakableScripts,
-  type VoiceOutputState,
-} from './voice-output.js'
-export { AgentInterruptedError, signalRunInterrupt } from './agent-interrupt.js'
-export type {
-  AgentInterruption,
-  AgentInterruptResult,
-  InterruptibleRunHandle,
-} from './agent-interrupt.js'
-export {
-  type RunAgentParams,
-  type StreamAgentOptions,
-  ToolApprovalRequired,
-  ToolCredentialRequired,
-  agentCallOptions,
-  canAccessThread,
-  isOwnedByPrincipal,
-  threadOwnerConstraint,
-} from './agent-prepare.js'
-export { addAgent } from './agent-registry.js'
-export type {
-  AgentInput,
-  AgentsMeta,
-  AgentMemoryConfig,
-  AgentStep,
-  AgentContentPart,
-  AgentRunRow,
-  AgentRunService,
-  AgentRunState,
-  AgentMessage,
-  AgentStreamChannel,
-  AgentStreamEvent,
-  AgentThread,
-  CoreAgent,
-  PendingApproval,
-  PikkuAgentMiddlewareHooks,
-} from './agent.types.js'

package/src/wirings/agent/voice-input.test.ts

@@ -1,220 +0,0 @@
-import { describe, test } from 'node:test'
-import assert from 'node:assert/strict'
-
-import {
-  voiceInput,
-  readsAsNonSpeech,
-  NoSpeechDetectedError,
-  SPOKEN_TURN,
-  SPOKEN_TRANSCRIPT,
-} from './voice-input.js'
-import type { AgentContentPart, AgentMessage } from './agent.types.js'
-
-class ThisDependentRunner {
-  private readonly transcript = 'the transcribed spoken words'
-
-  async transcribe() {
-    return {
-      text: this.transcript,
-      segments: [],
-      language: 'en',
-      durationInSeconds: 1,
-      warnings: [],
-    }
-  }
-}
-
-const audioMessage = (): AgentMessage => ({
-  id: 'm1',
-  role: 'user',
-  content: [
-    { type: 'file', mediaType: 'audio/wav', data: 'AAAA' },
-  ] as AgentContentPart[],
-  createdAt: new Date(0),
-})
-
-describe('voiceInput', () => {
-  test('transcribes an audio part by calling the runner with its receiver intact', async () => {
-    const mw = voiceInput({ model: 'mock/whisper' })
-    const services = { agentRunner: new ThisDependentRunner() }
-
-    const result = await mw.modifyInput!(services as any, {
-      messages: [audioMessage()],
-      instructions: 'sys',
-      shared: {},
-    })
-
-    const last = result.messages[result.messages.length - 1]!
-    const parts = last.content as AgentContentPart[]
-    assert.equal(parts.length, 1)
-    assert.equal(parts[0]!.type, 'text')
-    assert.equal(
-      (parts[0] as { text: string }).text,
-      'the transcribed spoken words'
-    )
-    assert.ok(!parts.some((p) => p.type === 'file'))
-  })
-
-  test('leaves the message untouched when there is no audio part', async () => {
-    const mw = voiceInput({ model: 'mock/whisper' })
-    const services = { agentRunner: new ThisDependentRunner() }
-    const messages: AgentMessage[] = [
-      {
-        id: 'm1',
-        role: 'user',
-        content: 'just text',
-        createdAt: new Date(0),
-      },
-    ]
-
-    const result = await mw.modifyInput!(services as any, {
-      messages,
-      instructions: 'sys',
-      shared: {},
-    })
-
-    assert.equal(result.messages[0]!.content, 'just text')
-  })
-
-  test('throws when an audio part is present but no model is configured', async () => {
-    const mw = voiceInput({})
-    const services = { agentRunner: new ThisDependentRunner() }
-
-    await assert.rejects(
-      () =>
-        mw.modifyInput!(services as any, {
-          messages: [audioMessage()],
-          instructions: 'sys',
-          shared: {},
-        }),
-      /voiceInput requires a transcription model/
-    )
-  })
-
-  test('a turn with no speech in it never reaches the model', async () => {
-    // The failure this prevents: the user says nothing, the model answers
-    // anyway, and what it answers is a guess about audio it could not hear.
-    // An ASR that returns '' on non-speech is what makes this checkable — see
-    // readsAsNonSpeech for why nothing subtler than '' survived contact.
-    const silent = {
-      async transcribe() {
-        return { text: '', segments: [], warnings: [] }
-      },
-    }
-    const mw = voiceInput({
-      model: 'deepinfra/nvidia/Nemotron-3.5-ASR-Streaming-Multilingual-0.6b',
-    })
-
-    await assert.rejects(
-      () =>
-        mw.modifyInput!({ agentRunner: silent } as any, {
-          messages: [audioMessage()],
-          instructions: 'sys',
-          shared: {},
-        }),
-      NoSpeechDetectedError
-    )
-  })
-
-  test('a transcript is passed through exactly as the provider wrote it', async () => {
-    // No reassembly, no trimming, no filtering — whatever the model heard is
-    // what the agent is asked about.
-    const spaced = {
-      async transcribe() {
-        return {
-          text: '  spacing   the provider chose  ',
-          segments: [],
-          warnings: [],
-        }
-      },
-    }
-    const mw = voiceInput({ model: 'mock/asr' })
-
-    const result = await mw.modifyInput!({ agentRunner: spaced } as any, {
-      messages: [audioMessage()],
-      instructions: 'sys',
-      shared: {},
-    })
-
-    const parts = result.messages.at(-1)!.content as AgentContentPart[]
-    assert.equal(
-      (parts[0] as { text: string }).text,
-      '  spacing   the provider chose  '
-    )
-  })
-})
-
-describe('readsAsNonSpeech', () => {
-  test('only an empty transcript counts as nothing said', () => {
-    assert.equal(readsAsNonSpeech({ text: '' }), true)
-    assert.equal(readsAsNonSpeech({ text: '   ' }), true)
-    assert.equal(readsAsNonSpeech({}), true)
-    assert.equal(readsAsNonSpeech({ text: 'add milk to the list' }), false)
-  })
-
-  test('a short or low-confidence-looking transcript is still speech', () => {
-    // The gate that used to live here would have been tempted by these. "Yes"
-    // is the single most important word in an approval flow, and dropping it
-    // because it is brief or quiet is far worse than keeping a stray one.
-    assert.equal(readsAsNonSpeech({ text: 'yes' }), false)
-    assert.equal(readsAsNonSpeech({ text: 'Thank you.' }), false)
-  })
-  test('records that the turn was spoken, for voiceOutput to read', async () => {
-    // The note has to be written here: the transcript this middleware puts in
-    // the message's place is indistinguishable from something typed, so no
-    // later middleware can tell what it started as.
-    const mw = voiceInput({ model: 'mock/whisper' })
-    const shared: Record<string, unknown> = {}
-
-    await mw.modifyInput!({ agentRunner: new ThisDependentRunner() } as any, {
-      messages: [audioMessage()],
-      instructions: 'sys',
-      shared,
-    })
-
-    assert.equal(shared[SPOKEN_TURN], true)
-  })
-
-  test('records a typed turn as not spoken, rather than leaving it unsaid', async () => {
-    const mw = voiceInput({ model: 'mock/whisper' })
-    const shared: Record<string, unknown> = {}
-
-    await mw.modifyInput!({ agentRunner: new ThisDependentRunner() } as any, {
-      messages: [{ role: 'user', content: 'typed' } as any],
-      instructions: 'sys',
-      shared,
-    })
-
-    // Explicitly false, not absent: absent is what a caller with no voice input
-    // at all looks like, and that one should still be spoken to.
-    assert.equal(shared[SPOKEN_TURN], false)
-  })
-
-  test('records what was heard, so the client can show the user their own turn', async () => {
-    const mw = voiceInput({ model: 'mock/whisper' })
-    const shared: Record<string, unknown> = {}
-
-    await mw.modifyInput!({ agentRunner: new ThisDependentRunner() } as any, {
-      messages: [audioMessage()],
-      instructions: 'sys',
-      shared,
-    })
-
-    assert.equal(shared[SPOKEN_TRANSCRIPT], 'the transcribed spoken words')
-  })
-
-  test('a typed turn leaves no transcript behind', async () => {
-    // Absence is the signal: the stream only announces a transcript when there
-    // was one, and a typed message is already on the client's screen.
-    const mw = voiceInput({ model: 'mock/whisper' })
-    const shared: Record<string, unknown> = {}
-
-    await mw.modifyInput!({ agentRunner: new ThisDependentRunner() } as any, {
-      messages: [{ role: 'user', content: 'typed' } as any],
-      instructions: 'sys',
-      shared,
-    })
-
-    assert.equal(SPOKEN_TRANSCRIPT in shared, false)
-  })
-})

package/src/wirings/agent/voice-input.ts

@@ -1,180 +0,0 @@
-import type { AgentRunnerService } from '../../services/agent-runner-service.js'
-import { pikkuAgentMiddleware } from '../../middleware/middleware-factories.js'
-import { safeFetch } from '../../utils/safe-fetch.js'
-import type { AgentContentPart } from './agent.types.js'
-
-function base64ToUint8Array(base64: string): Uint8Array {
-  const binary = atob(base64)
-  const bytes = new Uint8Array(binary.length)
-  for (let i = 0; i < binary.length; i++) {
-    bytes[i] = binary.charCodeAt(i)
-  }
-  return bytes
-}
-
-const MAX_AUDIO_SIZE = 50 * 1024 * 1024
-
-/**
- * Raised when every audio part in a turn turned out to be non-speech.
- *
- * Distinct from a transcription failure, because the response is different: a
- * failure is worth reporting, whereas this is the microphone doing its job on a
- * pause. A voice loop should catch it and go back to listening without running
- * the agent — answering a hallucinated sentence is worse than answering nothing.
- */
-export class NoSpeechDetectedError extends Error {
-  constructor() {
-    super('No speech was detected in the audio.')
-    this.name = 'NoSpeechDetectedError'
-  }
-}
-
-/**
- * Whether nothing usable was said — an empty transcript, and nothing cleverer
- * than that.
- *
- * There was a confidence gate here once, and it is worth recording why it is
- * gone. Whisper is trained on subtitles, so audio it cannot parse comes back
- * not as nothing but as stock filler appended to real speech: "Hello, my name
- * is Yasir. Thank you." where only the first sentence was spoken. The obvious
- * defence is to drop low-confidence segments, and it does not work. Measured on
- * a real turn, the invented "Thank you." scored `avg_logprob` -0.52 against
- * -0.30 for the genuine sentence beside it, and `no_speech_prob` read 0.000 for
- * both. Whisper is *confident* when it invents, because the text it invents is
- * high-probability text. No threshold separates those two numbers, and one set
- * low enough to try would take real quiet speech with it.
- *
- * The fix belongs at the model, not behind it: an ASR that is not a
- * subtitle-trained autoregressive decoder returns an empty string on non-speech
- * and there is nothing left to filter. Hence this reduces to asking whether
- * anything came back at all — which is a property every provider reports
- * honestly, unlike per-segment confidence (Nemotron, for one, hardcodes
- * `avg_logprob` to 0.0).
- */
-export const readsAsNonSpeech = (result: { text?: string }): boolean =>
-  !(result.text ?? '').trim()
-
-async function fetchAsUint8Array(
-  url: string,
-  allowedAudioHosts?: string[]
-): Promise<Uint8Array> {
-  const response = await safeFetch(url, {}, { allowedHosts: allowedAudioHosts })
-  const contentLength = response.headers.get('content-length')
-  if (contentLength && parseInt(contentLength, 10) > MAX_AUDIO_SIZE) {
-    throw new Error('Audio file exceeds maximum size')
-  }
-  const buffer = await response.arrayBuffer()
-  if (buffer.byteLength > MAX_AUDIO_SIZE) {
-    throw new Error('Audio file exceeds maximum size')
-  }
-  return new Uint8Array(buffer)
-}
-
-/**
- * Shared-notes key recording whether the user's turn arrived as audio.
- *
- * Written here because this is the last point at which it is knowable: the
- * transcript that replaces the audio is indistinguishable from something typed.
- * Read by `voiceOutput` to decide whether to answer aloud.
- *
- * Absent — rather than `false` — when this middleware is not wired at all, which
- * is what lets `voiceOutput` still speak for a caller that has no voice input.
- */
-export const SPOKEN_TURN = 'voice:spokenTurn'
-
-/**
- * Shared-notes key holding what the user was heard to say, when they spoke.
- *
- * Absent on a typed turn. Read by the stream wiring, which forwards it to the
- * client as a `transcript` event — a voice client has no idea what its own
- * audio said, and without this the user's turn shows up in the UI as an empty
- * bubble followed by an answer to a question they cannot see.
- */
-export const SPOKEN_TRANSCRIPT = 'voice:transcript'
-
-export const voiceInput = (config?: {
-  language?: string
-  model?: string
-  allowedAudioHosts?: string[]
-}) =>
-  pikkuAgentMiddleware({
-    modifyInput: async (services, { messages, instructions, shared }) => {
-      const agentRunner = (
-        services as {
-          agentRunner?: AgentRunnerService
-        }
-      ).agentRunner
-      if (!agentRunner?.transcribe) return { messages, instructions }
-
-      const last = messages[messages.length - 1]
-      const parts =
-        last?.role === 'user' && typeof last.content !== 'string'
-          ? (last.content as AgentContentPart[] | undefined)
-          : undefined
-
-      const hasAudio = !!parts?.some(
-        (p) => p.type === 'file' && !!p.mediaType?.startsWith('audio/')
-      )
-      // Recorded on every turn this middleware sees, spoken or not, so that a
-      // typed turn is an explicit `false` downstream rather than a silence that
-      // could equally mean nobody was asked.
-      shared[SPOKEN_TURN] = hasAudio
-      if (!hasAudio || !parts) return { messages, instructions }
-
-      const updatedContent: AgentContentPart[] = []
-      const heard: string[] = []
-      for (const p of parts) {
-        if (!(p.type === 'file' && p.mediaType?.startsWith('audio/'))) {
-          updatedContent.push(p)
-          continue
-        }
-        if (!config?.model) {
-          throw new Error(
-            'voiceInput requires a transcription model (e.g. openai/whisper-1)'
-          )
-        }
-        const audioData = p.data
-          ? base64ToUint8Array(p.data)
-          : await fetchAsUint8Array(p.url!, config.allowedAudioHosts)
-        if (audioData.byteLength > MAX_AUDIO_SIZE) {
-          throw new Error('Audio file exceeds maximum size')
-        }
-        const result = await agentRunner.transcribe({
-          model: config.model,
-          audio: audioData,
-          ...(config.language
-            ? {
-                providerOptions: {
-                  openai: {
-                    language: config.language,
-                  },
-                },
-              }
-            : {}),
-        })
-        // Dropped when nothing was said: an empty text part is still a turn the
-        // model will answer, and what it answers is a guess about what it could
-        // not hear.
-        if (readsAsNonSpeech(result)) continue
-        heard.push(result.text)
-        updatedContent.push({ type: 'text' as const, text: result.text })
-      }
-
-      // Every part was audio, and none of it was speech. There is nothing left
-      // to send, and a message with no content is not a question.
-      if (updatedContent.length === 0) throw new NoSpeechDetectedError()
-
-      // knowledge: decisions/internals/an-empty-transcript-is-not-recorded.md
-      if (heard.length > 0) {
-        shared[SPOKEN_TRANSCRIPT] = heard.join(' ')
-      }
-
-      return {
-        messages: [
-          ...messages.slice(0, -1),
-          { ...last, content: updatedContent },
-        ],
-        instructions,
-      }
-    },
-  })