@ai-ecoverse/pi-audio 1.17.0 → 1.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/echo.js +157 -0
- package/dist/echo.js.map +7 -0
- package/dist/gemini-live/session.js +32 -15
- package/dist/gemini-live/session.js.map +2 -2
- package/dist/index.js +16 -0
- package/dist/index.js.map +2 -2
- package/dist/openai-realtime/session.js +35 -9
- package/dist/openai-realtime/session.js.map +2 -2
- package/dist/openai-realtime/socket.js +12 -0
- package/dist/openai-realtime/socket.js.map +2 -2
- package/dist/talker.js +71 -27
- package/dist/talker.js.map +3 -3
- package/dist/turns.js +7 -1
- package/dist/turns.js.map +2 -2
- package/dist/types/echo.d.ts +29 -0
- package/dist/types/index.d.ts +2 -0
- package/dist/types/openai-realtime/socket.d.ts +1 -0
- package/dist/types/talker.d.ts +1 -1
- package/dist/types/turns.d.ts +2 -0
- package/package.json +1 -1
package/dist/turns.js.map
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../src/turns.ts"],
|
|
4
|
-
"sourcesContent": ["import type {\n AssistantMessageEvent,\n JsonObject,\n ProviderRequestOptions,\n ToolCall,\n UserMessage,\n} from '@earendil-works/pi-ai';\nimport {\n type AudioSink,\n type AudioSource,\n type AudioTag,\n chunkSeconds,\n type TextSpan,\n} from './audio.ts';\nimport { type SpokenClock, spokenOffset, utteranceIndex } from './captions.ts';\nimport { VoiceRequestError } from './errors.ts';\nimport type { SpeechModel, TranscriptionModel } from './models.ts';\nimport { pushQueue } from './queue.ts';\nimport {\n type AudioOut,\n type BedFollower,\n followBed,\n isAudioOut,\n type Soundscape,\n} from './soundscape.ts';\nimport type { SpeechEvent, SpeechInput } from './speech.ts';\nimport {\n createSpokenState,\n type SpokenPiece,\n spokenPieces,\n type VoiceAgentEvent,\n} from './spoken.ts';\nimport type { ListenOptions, TranscriptionEvent } from './transcription.ts';\n\nexport interface VoiceTurnMeta {\n source: 'voice';\n transcriptionModel: string;\n confidence?: number;\n}\n\nexport interface VoiceCut {\n entryId: string;\n contentIndex: number;\n heardUpTo: number;\n heardMs: number;\n}\n\nexport interface VoiceConversation {\n prompt(\n message: UserMessage,\n meta: VoiceTurnMeta,\n signal: AbortSignal\n ): AsyncIterable<VoiceAgentEvent>;\n truncate(cut: VoiceCut): Promise<void>;\n}\n\nexport type BargeIn = 'interrupt' | 'onWords' | 'ignore' | 'auto';\n\nexport type EchoRecommendation = 'interrupt' | 'onWords' | 'headphones';\n\nexport interface VoiceTurnOptions {\n ears: TranscriptionModel<string>;\n mouth: SpeechModel<string>;\n voice: string;\n speakable?: (event: AssistantMessageEvent) => string | undefined;\n announce?: (call: ToolCall) => string | undefined;\n bargeIn?: BargeIn;\n echoRecommendation?: EchoRecommendation;\n soundscape?: Soundscape;\n}\n\nexport interface VoiceModels {\n listen(\n model: TranscriptionModel<string>,\n audio: AudioSource,\n options?: ListenOptions\n ): AsyncIterable<TranscriptionEvent>;\n speak(\n model: SpeechModel<string>,\n input: SpeechInput,\n options?: ProviderRequestOptions\n ): AsyncIterable<SpeechEvent>;\n}\n\nexport type VoicePhase =\n | { phase: 'listening' }\n | { phase: 'heard'; text: string }\n | { phase: 'thinking' }\n | { phase: 'tool'; toolName: string; toolCallId: string; args: JsonObject }\n | { phase: 'retrying'; attempt: number; at: number }\n | { phase: 'compacting' }\n | { phase: 'speaking' }\n | { phase: 'idle' };\n\nexport type CaptionEvent =\n | { type: 'user_partial'; turnId: string; segmentId: string; text: string }\n | { type: 'user_final'; turnId: string; segmentId: string; text: string; confidence?: number }\n | { type: 'agent_generated'; turnId: string; utteranceId: string; text: string }\n | { type: 'agent_spoken'; turnId: string; utteranceId: string; upTo: number }\n | { type: 'agent_cut'; turnId: string; utteranceId: string; heardUpTo: number }\n | { type: 'announce'; turnId: string; text: string }\n | { type: 'thinker_progress'; turnId: string; callId: string; text: string }\n | { type: 'thinker_result'; turnId: string; callId: string; summary: string; isError: boolean };\n\nexport interface VoiceSession {\n phases: AsyncIterable<VoicePhase>;\n captions: AsyncIterable<CaptionEvent>;\n done: Promise<void>;\n}\n\ninterface SpokenMap {\n spokenStart: number;\n spokenEnd: number;\n contentIndex: number;\n start: number;\n end: number;\n entryId?: string;\n kind: SpokenPiece['kind'];\n}\n\ninterface ActiveTurn {\n controller: AbortController;\n unlink: () => void;\n done: Promise<void>;\n turnId: string;\n utteranceId: string;\n spokenUpTo: number;\n cut?: VoiceCut;\n}\n\nexport function voiceTurns(\n models: VoiceModels,\n conversation: VoiceConversation,\n mic: AudioSource,\n out: AudioSink | AudioOut,\n options: VoiceTurnOptions,\n signal: AbortSignal\n): VoiceSession {\n const phases = pushQueue<VoicePhase>();\n const captions = pushQueue<CaptionEvent>();\n const session: Session = {\n models,\n conversation,\n mic,\n out: speechOf(out),\n options,\n signal,\n phases,\n captions,\n maps: [],\n bed: attachBed(out, options.soundscape),\n };\n const done = runSession(session).finally(() => {\n phases.end();\n captions.end();\n });\n return { phases: phases.iterable, captions: captions.iterable, done };\n}\n\ninterface Session {\n models: VoiceModels;\n conversation: VoiceConversation;\n mic: AudioSource;\n out: AudioSink;\n options: VoiceTurnOptions;\n signal: AbortSignal;\n phases: ReturnType<typeof pushQueue<VoicePhase>>;\n captions: ReturnType<typeof pushQueue<CaptionEvent>>;\n maps: SpokenMap[];\n playing?: SpokenClock;\n bed: BedFollower;\n}\n\nasync function runSession(session: Session): Promise<void> {\n const mode = bargeMode(session.options);\n let active: ActiveTurn | undefined;\n let confidence: number | undefined;\n let reserved = 0;\n let openId = '';\n const openTurn = () => {\n if (!openId) {\n reserved += 1;\n openId = String(reserved);\n }\n return openId;\n };\n const stopPlayback = session.out.onPlayback((tag, offsetMs) => {\n if (!active) return;\n notePlayback(session, active, tag, offsetMs);\n });\n const setPhase = phaseSetter(session.phases, session.bed);\n try {\n if (session.signal.aborted) return;\n setPhase({ phase: 'listening' });\n for await (const event of session.models.listen(session.options.ears, session.mic, {\n signal: session.signal,\n })) {\n if (event.type === 'error') {\n if (event.reason === 'aborted' || session.signal.aborted) return;\n throw new VoiceRequestError(event.reason, event.error);\n }\n const captionId =\n event.type === 'partial' || event.type === 'final' || event.type === 'turn_end'\n ? openTurn()\n : openId;\n confidence = noteUser(session, event, captionId, confidence);\n if (shouldBarge(mode, event)) active = await interrupt(session, active);\n if (event.type !== 'turn_end') continue;\n const turnId = openId || openTurn();\n openId = '';\n if (mode === 'ignore') active = await finishTurn(active);\n else active = await interrupt(session, active);\n setPhase({ phase: 'heard', text: event.text });\n const meta = turnMeta(session.options, confidence);\n confidence = undefined;\n active = startTurn(session, setPhase, turnId, event, meta);\n }\n } finally {\n if (session.signal.aborted && active) await interrupt(session, active);\n else if (active) await active.done;\n stopPlayback();\n setPhase({ phase: 'idle' });\n session.bed.stop();\n }\n}\n\nfunction turnMeta(options: VoiceTurnOptions, confidence: number | undefined): VoiceTurnMeta {\n return {\n source: 'voice',\n transcriptionModel: options.ears.id,\n ...(confidence === undefined ? {} : { confidence }),\n };\n}\n\nfunction noteUser(\n session: Session,\n event: TranscriptionEvent,\n turnId: string,\n confidence: number | undefined\n): number | undefined {\n if (event.type === 'partial') {\n session.captions.push({\n type: 'user_partial',\n turnId,\n segmentId: event.segmentId,\n text: event.text,\n });\n }\n if (event.type === 'final') {\n session.captions.push({\n type: 'user_final',\n turnId,\n segmentId: event.segmentId,\n text: event.text,\n ...(event.confidence === undefined ? {} : { confidence: event.confidence }),\n });\n return event.confidence ?? confidence;\n }\n if (event.type === 'turn_end') {\n session.captions.push({ type: 'user_final', turnId, segmentId: 'turn', text: event.text });\n }\n return confidence;\n}\n\nasync function finishTurn(active: ActiveTurn | undefined): Promise<undefined> {\n if (!active) return undefined;\n await active.done;\n return undefined;\n}\n\nfunction startTurn(\n session: Session,\n setPhase: (phase: VoicePhase) => void,\n turnId: string,\n event: Extract<TranscriptionEvent, { type: 'turn_end' }>,\n meta: VoiceTurnMeta\n): ActiveTurn {\n const linked = linkSignal(session.signal);\n const active: ActiveTurn = {\n controller: linked.controller,\n unlink: linked.unlink,\n turnId,\n utteranceId: `${turnId}:agent`,\n spokenUpTo: 0,\n done: Promise.resolve(),\n };\n active.done = performTurn(session, setPhase, active, event.text, event.at, meta).finally(() => {\n linked.unlink();\n });\n return active;\n}\n\nasync function performTurn(\n session: Session,\n setPhase: (phase: VoicePhase) => void,\n active: ActiveTurn,\n text: string,\n at: number,\n meta: VoiceTurnMeta\n): Promise<void> {\n const signal = active.controller.signal;\n try {\n setPhase({ phase: 'thinking' });\n const message: UserMessage = { role: 'user', content: text, timestamp: at };\n const feed = textFeed(signal);\n const maps: SpokenMap[] = [];\n session.maps = maps;\n const producing = readAgent(session, setPhase, active, message, meta, feed, maps);\n const audio = session.models.speak(\n session.options.mouth,\n { text: feed.iterable, voice: session.options.voice },\n { signal }\n );\n await Promise.all([producing, pipeSpeech(session, setPhase, active, audio, maps)]);\n if (!signal.aborted) setPhase({ phase: 'listening' });\n } catch (error) {\n if (signal.aborted || session.signal.aborted) return;\n throw error;\n }\n}\n\nasync function readAgent(\n session: Session,\n setPhase: (phase: VoicePhase) => void,\n active: ActiveTurn,\n message: UserMessage,\n meta: VoiceTurnMeta,\n feed: TextFeed,\n maps: SpokenMap[]\n): Promise<void> {\n const state = createSpokenState();\n let spokenChars = 0;\n let generated = '';\n try {\n for await (const event of session.conversation.prompt(\n message,\n meta,\n active.controller.signal\n )) {\n if (active.controller.signal.aborted) break;\n notePhase(setPhase, event);\n for (const piece of spokenPieces(event, state, session.options)) {\n spokenChars += piece.text.length;\n generated = rememberPiece(\n session,\n active,\n piece,\n maps,\n spokenChars,\n generated,\n state.entryId\n );\n feed.push(piece.text);\n }\n if (state.entryId) bindEntry(maps, state.entryId);\n }\n } finally {\n feed.end();\n }\n}\n\nfunction rememberPiece(\n session: Session,\n active: ActiveTurn,\n piece: SpokenPiece,\n maps: SpokenMap[],\n spokenEnd: number,\n generated: string,\n entryId: string | undefined\n): string {\n const bound = piece.entryId ?? entryId;\n maps.push({\n spokenStart: spokenEnd - piece.text.length,\n spokenEnd,\n contentIndex: piece.contentIndex,\n start: piece.start,\n end: piece.end,\n kind: piece.kind,\n ...(bound ? { entryId: bound } : {}),\n });\n if (piece.kind === 'announce') {\n session.captions.push({ type: 'announce', turnId: active.turnId, text: piece.text });\n return generated;\n }\n const text = generated + piece.text;\n session.captions.push({\n type: 'agent_generated',\n turnId: active.turnId,\n utteranceId: active.utteranceId,\n text,\n });\n return text;\n}\n\nasync function pipeSpeech(\n session: Session,\n setPhase: (phase: VoicePhase) => void,\n active: ActiveTurn,\n audio: AsyncIterable<SpeechEvent>,\n maps: SpokenMap[]\n): Promise<void> {\n for await (const event of audio) {\n if (active.controller.signal.aborted) return;\n if (event.type === 'error') {\n if (event.reason === 'aborted') return;\n throw new VoiceRequestError(event.reason, event.error);\n }\n if (event.type !== 'audio') continue;\n setPhase({ phase: 'speaking' });\n const span = translateSpan(event.span, maps);\n const playing: SpokenClock = {\n start: event.span.start,\n end: event.span.end,\n durationMs: Math.round(chunkSeconds(event.chunk) * 1000),\n };\n if (event.alignment) playing.alignment = event.alignment;\n session.playing = playing;\n const tag: AudioTag = { span };\n if (event.alignment) tag.alignment = event.alignment;\n session.out.write(event.chunk, tag);\n }\n}\n\nfunction notePlayback(session: Session, active: ActiveTurn, tag: AudioTag, offsetMs: number): void {\n const playing = session.playing;\n if (playing) {\n const upTo = utteranceIndex(spokenOffset(playing, offsetMs), session.maps);\n if (upTo > active.spokenUpTo) {\n active.spokenUpTo = upTo;\n session.captions.push({\n type: 'agent_spoken',\n turnId: active.turnId,\n utteranceId: active.utteranceId,\n upTo,\n });\n }\n }\n if (!('entryId' in tag.span)) return;\n active.cut = {\n entryId: tag.span.entryId,\n contentIndex: tag.span.contentIndex,\n heardUpTo: tag.span.end,\n heardMs: session.out.playedMs,\n };\n}\n\nfunction translateSpan(span: TextSpan, maps: SpokenMap[]): TextSpan {\n if ('entryId' in span) return span;\n const map = answerMap(span, maps);\n if (!map?.entryId) return span;\n const spokenLen = map.spokenEnd - map.spokenStart;\n const sourceLen = map.end - map.start;\n const start = map.start + scale(Math.max(0, span.start - map.spokenStart), spokenLen, sourceLen);\n const end =\n map.start + scale(Math.min(spokenLen, span.end - map.spokenStart), spokenLen, sourceLen);\n return { entryId: map.entryId, contentIndex: map.contentIndex, start, end };\n}\n\nfunction answerMap(span: { start: number; end: number }, maps: SpokenMap[]): SpokenMap | undefined {\n for (let index = maps.length - 1; index >= 0; index -= 1) {\n const item = maps[index];\n if (item?.kind !== 'answer' || !item.entryId) continue;\n if (item.spokenStart < span.end && item.spokenEnd > span.start) return item;\n }\n return undefined;\n}\n\nfunction scale(offset: number, spokenLen: number, sourceLen: number): number {\n if (spokenLen <= 0) return 0;\n return Math.round((offset * sourceLen) / spokenLen);\n}\n\nfunction bindEntry(maps: SpokenMap[], entryId: string): void {\n for (const map of maps) {\n if (map.kind === 'answer' && map.entryId === undefined) map.entryId = entryId;\n }\n}\n\nfunction notePhase(setPhase: (phase: VoicePhase) => void, event: VoiceAgentEvent): void {\n if (event.type === 'run_start' || event.type === 'turn_start') setPhase({ phase: 'thinking' });\n if (event.type === 'compaction_start') setPhase({ phase: 'compacting' });\n if (event.type === 'auto_retry_start' && event.attempt !== undefined && event.at !== undefined) {\n setPhase({ phase: 'retrying', attempt: event.attempt, at: event.at });\n }\n if (event.type === 'tool_execution_start' && event.toolName && event.toolCallId) {\n setPhase({\n phase: 'tool',\n toolName: event.toolName,\n toolCallId: event.toolCallId,\n args: event.args ?? {},\n });\n }\n}\n\nasync function interrupt(session: Session, active: ActiveTurn | undefined): Promise<undefined> {\n if (!active) return undefined;\n active.controller.abort();\n session.out.flush();\n await active.done;\n if (active.cut) {\n session.captions.push({\n type: 'agent_cut',\n turnId: active.turnId,\n utteranceId: active.utteranceId,\n heardUpTo: active.cut.heardUpTo,\n });\n await session.conversation.truncate(active.cut);\n }\n return undefined;\n}\n\nfunction bargeMode(options: VoiceTurnOptions): 'interrupt' | 'onWords' | 'ignore' {\n const mode = options.bargeIn ?? 'interrupt';\n if (mode !== 'auto') return mode;\n if (options.echoRecommendation === 'interrupt') return 'interrupt';\n if (options.echoRecommendation === 'headphones') return 'ignore';\n return 'onWords';\n}\n\nfunction shouldBarge(mode: 'interrupt' | 'onWords' | 'ignore', event: TranscriptionEvent): boolean {\n if (mode === 'ignore') return false;\n if (mode === 'interrupt') return event.type === 'speech_start';\n return event.type === 'partial' && event.text.trim().length > 0;\n}\n\nfunction speechOf(out: AudioSink | AudioOut): AudioSink {\n if (isAudioOut(out)) return out.speech;\n return out;\n}\n\nfunction attachBed(out: AudioSink | AudioOut, soundscape: Soundscape | undefined): BedFollower {\n if (!soundscape) return idleBed();\n if (!isAudioOut(out)) return idleBed();\n return followBed(out.bed, soundscape);\n}\n\nfunction idleBed(): BedFollower {\n return {\n observe() {},\n stop() {},\n };\n}\n\nfunction phaseSetter(phases: Session['phases'], bed: BedFollower): (phase: VoicePhase) => void {\n let previous = '';\n return (phase) => {\n const key = JSON.stringify(phase);\n if (key === previous) return;\n previous = key;\n phases.push(phase);\n bed.observe(phase);\n };\n}\n\nfunction linkSignal(parent: AbortSignal): { controller: AbortController; unlink: () => void } {\n const controller = new AbortController();\n const abort = () => controller.abort();\n if (parent.aborted) controller.abort();\n else parent.addEventListener('abort', abort);\n return {\n controller,\n unlink: () => parent.removeEventListener('abort', abort),\n };\n}\n\ninterface TextFeed {\n push(text: string): void;\n end(): void;\n iterable: AsyncIterable<string>;\n}\n\nfunction textFeed(signal: AbortSignal): TextFeed {\n const queue: string[] = [];\n let ended = false;\n let wake: (() => void) | undefined;\n const notify = () => {\n const current = wake;\n wake = undefined;\n current?.();\n };\n const onAbort = () => notify();\n signal.addEventListener('abort', onAbort);\n return {\n push(text) {\n if (ended || signal.aborted || text.length === 0) return;\n queue.push(text);\n notify();\n },\n end() {\n ended = true;\n notify();\n },\n iterable: {\n async *[Symbol.asyncIterator]() {\n try {\n while (!signal.aborted) {\n const next = queue.shift();\n if (next !== undefined) {\n yield next;\n continue;\n }\n if (ended) return;\n await new Promise<void>((resolve) => {\n if (queue.length > 0 || ended || signal.aborted) resolve();\n else wake = resolve;\n });\n }\n } finally {\n signal.removeEventListener('abort', onAbort);\n }\n },\n },\n };\n}\n"],
|
|
5
|
-
"mappings": "AAOA;AAAA,EAIE;AAAA,OAEK;AACP,SAA2B,cAAc,sBAAsB;AAC/D,SAAS,yBAAyB;AAElC,SAAS,iBAAiB;AAC1B;AAAA,EAGE;AAAA,EACA;AAAA,OAEK;AAEP;AAAA,EACE;AAAA,EAEA;AAAA,OAEK;
|
|
4
|
+
"sourcesContent": ["import type {\n AssistantMessageEvent,\n JsonObject,\n ProviderRequestOptions,\n ToolCall,\n UserMessage,\n} from '@earendil-works/pi-ai';\nimport {\n type AudioSink,\n type AudioSource,\n type AudioTag,\n chunkSeconds,\n type TextSpan,\n} from './audio.ts';\nimport { type SpokenClock, spokenOffset, utteranceIndex } from './captions.ts';\nimport { type EchoControl, openEchoMic } from './echo.ts';\nimport { VoiceRequestError } from './errors.ts';\nimport type { SpeechModel, TranscriptionModel } from './models.ts';\nimport { pushQueue } from './queue.ts';\nimport {\n type AudioOut,\n type BedFollower,\n followBed,\n isAudioOut,\n type Soundscape,\n} from './soundscape.ts';\nimport type { SpeechEvent, SpeechInput } from './speech.ts';\nimport {\n createSpokenState,\n type SpokenPiece,\n spokenPieces,\n type VoiceAgentEvent,\n} from './spoken.ts';\nimport type { ListenOptions, TranscriptionEvent } from './transcription.ts';\n\nexport interface VoiceTurnMeta {\n source: 'voice';\n transcriptionModel: string;\n confidence?: number;\n}\n\nexport interface VoiceCut {\n entryId: string;\n contentIndex: number;\n heardUpTo: number;\n heardMs: number;\n}\n\nexport interface VoiceConversation {\n prompt(\n message: UserMessage,\n meta: VoiceTurnMeta,\n signal: AbortSignal\n ): AsyncIterable<VoiceAgentEvent>;\n truncate(cut: VoiceCut): Promise<void>;\n}\n\nexport type BargeIn = 'interrupt' | 'onWords' | 'ignore' | 'auto';\n\nexport type EchoRecommendation = 'interrupt' | 'onWords' | 'headphones';\n\nexport interface VoiceTurnOptions {\n ears: TranscriptionModel<string>;\n mouth: SpeechModel<string>;\n voice: string;\n speakable?: (event: AssistantMessageEvent) => string | undefined;\n announce?: (call: ToolCall) => string | undefined;\n bargeIn?: BargeIn;\n echoRecommendation?: EchoRecommendation;\n echo?: EchoControl;\n soundscape?: Soundscape;\n}\n\nexport interface VoiceModels {\n listen(\n model: TranscriptionModel<string>,\n audio: AudioSource,\n options?: ListenOptions\n ): AsyncIterable<TranscriptionEvent>;\n speak(\n model: SpeechModel<string>,\n input: SpeechInput,\n options?: ProviderRequestOptions\n ): AsyncIterable<SpeechEvent>;\n}\n\nexport type VoicePhase =\n | { phase: 'listening' }\n | { phase: 'heard'; text: string }\n | { phase: 'thinking' }\n | { phase: 'tool'; toolName: string; toolCallId: string; args: JsonObject }\n | { phase: 'retrying'; attempt: number; at: number }\n | { phase: 'compacting' }\n | { phase: 'speaking' }\n | { phase: 'idle' };\n\nexport type CaptionEvent =\n | { type: 'user_partial'; turnId: string; segmentId: string; text: string }\n | { type: 'user_final'; turnId: string; segmentId: string; text: string; confidence?: number }\n | { type: 'agent_generated'; turnId: string; utteranceId: string; text: string }\n | { type: 'agent_spoken'; turnId: string; utteranceId: string; upTo: number }\n | { type: 'agent_cut'; turnId: string; utteranceId: string; heardUpTo: number }\n | { type: 'announce'; turnId: string; text: string }\n | { type: 'thinker_progress'; turnId: string; callId: string; text: string }\n | { type: 'thinker_result'; turnId: string; callId: string; summary: string; isError: boolean };\n\nexport interface VoiceSession {\n phases: AsyncIterable<VoicePhase>;\n captions: AsyncIterable<CaptionEvent>;\n done: Promise<void>;\n}\n\ninterface SpokenMap {\n spokenStart: number;\n spokenEnd: number;\n contentIndex: number;\n start: number;\n end: number;\n entryId?: string;\n kind: SpokenPiece['kind'];\n}\n\ninterface ActiveTurn {\n controller: AbortController;\n unlink: () => void;\n done: Promise<void>;\n turnId: string;\n utteranceId: string;\n spokenUpTo: number;\n cut?: VoiceCut;\n}\n\nexport function voiceTurns(\n models: VoiceModels,\n conversation: VoiceConversation,\n mic: AudioSource,\n out: AudioSink | AudioOut,\n options: VoiceTurnOptions,\n signal: AbortSignal\n): VoiceSession {\n const phases = pushQueue<VoicePhase>();\n const captions = pushQueue<CaptionEvent>();\n const session: Session = {\n models,\n conversation,\n mic: heardMic(options, mic),\n out: speechOf(out),\n options,\n signal,\n phases,\n captions,\n maps: [],\n bed: attachBed(out, options.soundscape),\n };\n const done = runSession(session).finally(() => {\n phases.end();\n captions.end();\n });\n return { phases: phases.iterable, captions: captions.iterable, done };\n}\n\ninterface Session {\n models: VoiceModels;\n conversation: VoiceConversation;\n mic: AudioSource;\n out: AudioSink;\n options: VoiceTurnOptions;\n signal: AbortSignal;\n phases: ReturnType<typeof pushQueue<VoicePhase>>;\n captions: ReturnType<typeof pushQueue<CaptionEvent>>;\n maps: SpokenMap[];\n playing?: SpokenClock;\n bed: BedFollower;\n}\n\nasync function runSession(session: Session): Promise<void> {\n const mode = bargeMode(session.options);\n let active: ActiveTurn | undefined;\n let confidence: number | undefined;\n let reserved = 0;\n let openId = '';\n const openTurn = () => {\n if (!openId) {\n reserved += 1;\n openId = String(reserved);\n }\n return openId;\n };\n const stopPlayback = session.out.onPlayback((tag, offsetMs) => {\n if (!active) return;\n notePlayback(session, active, tag, offsetMs);\n });\n const setPhase = phaseSetter(session.phases, session.bed);\n try {\n if (session.signal.aborted) return;\n setPhase({ phase: 'listening' });\n for await (const event of session.models.listen(session.options.ears, session.mic, {\n signal: session.signal,\n })) {\n if (event.type === 'error') {\n if (event.reason === 'aborted' || session.signal.aborted) return;\n throw new VoiceRequestError(event.reason, event.error);\n }\n const captionId =\n event.type === 'partial' || event.type === 'final' || event.type === 'turn_end'\n ? openTurn()\n : openId;\n confidence = noteUser(session, event, captionId, confidence);\n if (shouldBarge(mode, event)) active = await interrupt(session, active);\n if (event.type !== 'turn_end') continue;\n const turnId = openId || openTurn();\n openId = '';\n if (mode === 'ignore') active = await finishTurn(active);\n else active = await interrupt(session, active);\n setPhase({ phase: 'heard', text: event.text });\n const meta = turnMeta(session.options, confidence);\n confidence = undefined;\n active = startTurn(session, setPhase, turnId, event, meta);\n }\n } finally {\n if (session.signal.aborted && active) await interrupt(session, active);\n else if (active) await active.done;\n stopPlayback();\n setPhase({ phase: 'idle' });\n session.bed.stop();\n }\n}\n\nfunction turnMeta(options: VoiceTurnOptions, confidence: number | undefined): VoiceTurnMeta {\n return {\n source: 'voice',\n transcriptionModel: options.ears.id,\n ...(confidence === undefined ? {} : { confidence }),\n };\n}\n\nfunction noteUser(\n session: Session,\n event: TranscriptionEvent,\n turnId: string,\n confidence: number | undefined\n): number | undefined {\n if (event.type === 'partial') {\n session.captions.push({\n type: 'user_partial',\n turnId,\n segmentId: event.segmentId,\n text: event.text,\n });\n }\n if (event.type === 'final') {\n session.captions.push({\n type: 'user_final',\n turnId,\n segmentId: event.segmentId,\n text: event.text,\n ...(event.confidence === undefined ? {} : { confidence: event.confidence }),\n });\n return event.confidence ?? confidence;\n }\n if (event.type === 'turn_end') {\n session.captions.push({ type: 'user_final', turnId, segmentId: 'turn', text: event.text });\n }\n return confidence;\n}\n\nasync function finishTurn(active: ActiveTurn | undefined): Promise<undefined> {\n if (!active) return undefined;\n await active.done;\n return undefined;\n}\n\nfunction startTurn(\n session: Session,\n setPhase: (phase: VoicePhase) => void,\n turnId: string,\n event: Extract<TranscriptionEvent, { type: 'turn_end' }>,\n meta: VoiceTurnMeta\n): ActiveTurn {\n const linked = linkSignal(session.signal);\n const active: ActiveTurn = {\n controller: linked.controller,\n unlink: linked.unlink,\n turnId,\n utteranceId: `${turnId}:agent`,\n spokenUpTo: 0,\n done: Promise.resolve(),\n };\n active.done = performTurn(session, setPhase, active, event.text, event.at, meta).finally(() => {\n linked.unlink();\n });\n return active;\n}\n\nasync function performTurn(\n session: Session,\n setPhase: (phase: VoicePhase) => void,\n active: ActiveTurn,\n text: string,\n at: number,\n meta: VoiceTurnMeta\n): Promise<void> {\n const signal = active.controller.signal;\n try {\n setPhase({ phase: 'thinking' });\n const message: UserMessage = { role: 'user', content: text, timestamp: at };\n const feed = textFeed(signal);\n const maps: SpokenMap[] = [];\n session.maps = maps;\n const producing = readAgent(session, setPhase, active, message, meta, feed, maps);\n const audio = session.models.speak(\n session.options.mouth,\n { text: feed.iterable, voice: session.options.voice },\n { signal }\n );\n await Promise.all([producing, pipeSpeech(session, setPhase, active, audio, maps)]);\n if (!signal.aborted) setPhase({ phase: 'listening' });\n } catch (error) {\n if (signal.aborted || session.signal.aborted) return;\n throw error;\n }\n}\n\nasync function readAgent(\n session: Session,\n setPhase: (phase: VoicePhase) => void,\n active: ActiveTurn,\n message: UserMessage,\n meta: VoiceTurnMeta,\n feed: TextFeed,\n maps: SpokenMap[]\n): Promise<void> {\n const state = createSpokenState();\n let spokenChars = 0;\n let generated = '';\n try {\n for await (const event of session.conversation.prompt(\n message,\n meta,\n active.controller.signal\n )) {\n if (active.controller.signal.aborted) break;\n notePhase(setPhase, event);\n for (const piece of spokenPieces(event, state, session.options)) {\n spokenChars += piece.text.length;\n generated = rememberPiece(\n session,\n active,\n piece,\n maps,\n spokenChars,\n generated,\n state.entryId\n );\n feed.push(piece.text);\n }\n if (state.entryId) bindEntry(maps, state.entryId);\n }\n } finally {\n feed.end();\n }\n}\n\nfunction rememberPiece(\n session: Session,\n active: ActiveTurn,\n piece: SpokenPiece,\n maps: SpokenMap[],\n spokenEnd: number,\n generated: string,\n entryId: string | undefined\n): string {\n const bound = piece.entryId ?? entryId;\n maps.push({\n spokenStart: spokenEnd - piece.text.length,\n spokenEnd,\n contentIndex: piece.contentIndex,\n start: piece.start,\n end: piece.end,\n kind: piece.kind,\n ...(bound ? { entryId: bound } : {}),\n });\n if (piece.kind === 'announce') {\n session.captions.push({ type: 'announce', turnId: active.turnId, text: piece.text });\n return generated;\n }\n const text = generated + piece.text;\n session.captions.push({\n type: 'agent_generated',\n turnId: active.turnId,\n utteranceId: active.utteranceId,\n text,\n });\n return text;\n}\n\nasync function pipeSpeech(\n session: Session,\n setPhase: (phase: VoicePhase) => void,\n active: ActiveTurn,\n audio: AsyncIterable<SpeechEvent>,\n maps: SpokenMap[]\n): Promise<void> {\n for await (const event of audio) {\n if (active.controller.signal.aborted) return;\n if (event.type === 'error') {\n if (event.reason === 'aborted') return;\n throw new VoiceRequestError(event.reason, event.error);\n }\n if (event.type !== 'audio') continue;\n setPhase({ phase: 'speaking' });\n const span = translateSpan(event.span, maps);\n const playing: SpokenClock = {\n start: event.span.start,\n end: event.span.end,\n durationMs: Math.round(chunkSeconds(event.chunk) * 1000),\n };\n if (event.alignment) playing.alignment = event.alignment;\n session.playing = playing;\n const tag: AudioTag = { span };\n if (event.alignment) tag.alignment = event.alignment;\n session.out.write(event.chunk, tag);\n }\n}\n\nfunction notePlayback(session: Session, active: ActiveTurn, tag: AudioTag, offsetMs: number): void {\n const playing = session.playing;\n if (playing) {\n const upTo = utteranceIndex(spokenOffset(playing, offsetMs), session.maps);\n if (upTo > active.spokenUpTo) {\n active.spokenUpTo = upTo;\n session.captions.push({\n type: 'agent_spoken',\n turnId: active.turnId,\n utteranceId: active.utteranceId,\n upTo,\n });\n }\n }\n if (!('entryId' in tag.span)) return;\n active.cut = {\n entryId: tag.span.entryId,\n contentIndex: tag.span.contentIndex,\n heardUpTo: tag.span.end,\n heardMs: session.out.playedMs,\n };\n}\n\nfunction translateSpan(span: TextSpan, maps: SpokenMap[]): TextSpan {\n if ('entryId' in span) return span;\n const map = answerMap(span, maps);\n if (!map?.entryId) return span;\n const spokenLen = map.spokenEnd - map.spokenStart;\n const sourceLen = map.end - map.start;\n const start = map.start + scale(Math.max(0, span.start - map.spokenStart), spokenLen, sourceLen);\n const end =\n map.start + scale(Math.min(spokenLen, span.end - map.spokenStart), spokenLen, sourceLen);\n return { entryId: map.entryId, contentIndex: map.contentIndex, start, end };\n}\n\nfunction answerMap(span: { start: number; end: number }, maps: SpokenMap[]): SpokenMap | undefined {\n for (let index = maps.length - 1; index >= 0; index -= 1) {\n const item = maps[index];\n if (item?.kind !== 'answer' || !item.entryId) continue;\n if (item.spokenStart < span.end && item.spokenEnd > span.start) return item;\n }\n return undefined;\n}\n\nfunction scale(offset: number, spokenLen: number, sourceLen: number): number {\n if (spokenLen <= 0) return 0;\n return Math.round((offset * sourceLen) / spokenLen);\n}\n\nfunction bindEntry(maps: SpokenMap[], entryId: string): void {\n for (const map of maps) {\n if (map.kind === 'answer' && map.entryId === undefined) map.entryId = entryId;\n }\n}\n\nfunction notePhase(setPhase: (phase: VoicePhase) => void, event: VoiceAgentEvent): void {\n if (event.type === 'run_start' || event.type === 'turn_start') setPhase({ phase: 'thinking' });\n if (event.type === 'compaction_start') setPhase({ phase: 'compacting' });\n if (event.type === 'auto_retry_start' && event.attempt !== undefined && event.at !== undefined) {\n setPhase({ phase: 'retrying', attempt: event.attempt, at: event.at });\n }\n if (event.type === 'tool_execution_start' && event.toolName && event.toolCallId) {\n setPhase({\n phase: 'tool',\n toolName: event.toolName,\n toolCallId: event.toolCallId,\n args: event.args ?? {},\n });\n }\n}\n\nasync function interrupt(session: Session, active: ActiveTurn | undefined): Promise<undefined> {\n if (!active) return undefined;\n active.controller.abort();\n session.out.flush();\n await active.done;\n if (active.cut) {\n session.captions.push({\n type: 'agent_cut',\n turnId: active.turnId,\n utteranceId: active.utteranceId,\n heardUpTo: active.cut.heardUpTo,\n });\n await session.conversation.truncate(active.cut);\n }\n return undefined;\n}\n\nfunction bargeMode(options: VoiceTurnOptions): 'interrupt' | 'onWords' | 'ignore' {\n const mode = options.bargeIn ?? 'interrupt';\n if (mode !== 'auto') return mode;\n if (options.echoRecommendation === 'interrupt') return 'interrupt';\n if (options.echoRecommendation === 'headphones') return 'ignore';\n return 'onWords';\n}\n\nfunction shouldBarge(mode: 'interrupt' | 'onWords' | 'ignore', event: TranscriptionEvent): boolean {\n if (mode === 'ignore') return false;\n if (mode === 'interrupt') return event.type === 'speech_start';\n return event.type === 'partial' && event.text.trim().length > 0;\n}\n\nfunction heardMic(options: VoiceTurnOptions, mic: AudioSource): AudioSource {\n const echo = options.echo;\n if (echo === undefined) return mic;\n return openEchoMic(echo, mic);\n}\n\nfunction speechOf(out: AudioSink | AudioOut): AudioSink {\n if (isAudioOut(out)) return out.speech;\n return out;\n}\n\nfunction attachBed(out: AudioSink | AudioOut, soundscape: Soundscape | undefined): BedFollower {\n if (!soundscape) return idleBed();\n if (!isAudioOut(out)) return idleBed();\n return followBed(out.bed, soundscape);\n}\n\nfunction idleBed(): BedFollower {\n return {\n observe() {},\n stop() {},\n };\n}\n\nfunction phaseSetter(phases: Session['phases'], bed: BedFollower): (phase: VoicePhase) => void {\n let previous = '';\n return (phase) => {\n const key = JSON.stringify(phase);\n if (key === previous) return;\n previous = key;\n phases.push(phase);\n bed.observe(phase);\n };\n}\n\nfunction linkSignal(parent: AbortSignal): { controller: AbortController; unlink: () => void } {\n const controller = new AbortController();\n const abort = () => controller.abort();\n if (parent.aborted) controller.abort();\n else parent.addEventListener('abort', abort);\n return {\n controller,\n unlink: () => parent.removeEventListener('abort', abort),\n };\n}\n\ninterface TextFeed {\n push(text: string): void;\n end(): void;\n iterable: AsyncIterable<string>;\n}\n\nfunction textFeed(signal: AbortSignal): TextFeed {\n const queue: string[] = [];\n let ended = false;\n let wake: (() => void) | undefined;\n const notify = () => {\n const current = wake;\n wake = undefined;\n current?.();\n };\n const onAbort = () => notify();\n signal.addEventListener('abort', onAbort);\n return {\n push(text) {\n if (ended || signal.aborted || text.length === 0) return;\n queue.push(text);\n notify();\n },\n end() {\n ended = true;\n notify();\n },\n iterable: {\n async *[Symbol.asyncIterator]() {\n try {\n while (!signal.aborted) {\n const next = queue.shift();\n if (next !== undefined) {\n yield next;\n continue;\n }\n if (ended) return;\n await new Promise<void>((resolve) => {\n if (queue.length > 0 || ended || signal.aborted) resolve();\n else wake = resolve;\n });\n }\n } finally {\n signal.removeEventListener('abort', onAbort);\n }\n },\n },\n };\n}\n"],
|
|
5
|
+
"mappings": "AAOA;AAAA,EAIE;AAAA,OAEK;AACP,SAA2B,cAAc,sBAAsB;AAC/D,SAA2B,mBAAmB;AAC9C,SAAS,yBAAyB;AAElC,SAAS,iBAAiB;AAC1B;AAAA,EAGE;AAAA,EACA;AAAA,OAEK;AAEP;AAAA,EACE;AAAA,EAEA;AAAA,OAEK;AAoGA,SAAS,WACd,QACA,cACA,KACA,KACA,SACA,QACc;AACd,QAAM,SAAS,UAAsB;AACrC,QAAM,WAAW,UAAwB;AACzC,QAAM,UAAmB;AAAA,IACvB;AAAA,IACA;AAAA,IACA,KAAK,SAAS,SAAS,GAAG;AAAA,IAC1B,KAAK,SAAS,GAAG;AAAA,IACjB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,MAAM,CAAC;AAAA,IACP,KAAK,UAAU,KAAK,QAAQ,UAAU;AAAA,EACxC;AACA,QAAM,OAAO,WAAW,OAAO,EAAE,QAAQ,MAAM;AAC7C,WAAO,IAAI;AACX,aAAS,IAAI;AAAA,EACf,CAAC;AACD,SAAO,EAAE,QAAQ,OAAO,UAAU,UAAU,SAAS,UAAU,KAAK;AACtE;AAgBA,eAAe,WAAW,SAAiC;AACzD,QAAM,OAAO,UAAU,QAAQ,OAAO;AACtC,MAAI;AACJ,MAAI;AACJ,MAAI,WAAW;AACf,MAAI,SAAS;AACb,QAAM,WAAW,MAAM;AACrB,QAAI,CAAC,QAAQ;AACX,kBAAY;AACZ,eAAS,OAAO,QAAQ;AAAA,IAC1B;AACA,WAAO;AAAA,EACT;AACA,QAAM,eAAe,QAAQ,IAAI,WAAW,CAAC,KAAK,aAAa;AAC7D,QAAI,CAAC,OAAQ;AACb,iBAAa,SAAS,QAAQ,KAAK,QAAQ;AAAA,EAC7C,CAAC;AACD,QAAM,WAAW,YAAY,QAAQ,QAAQ,QAAQ,GAAG;AACxD,MAAI;AACF,QAAI,QAAQ,OAAO,QAAS;AAC5B,aAAS,EAAE,OAAO,YAAY,CAAC;AAC/B,qBAAiB,SAAS,QAAQ,OAAO,OAAO,QAAQ,QAAQ,MAAM,QAAQ,KAAK;AAAA,MACjF,QAAQ,QAAQ;AAAA,IAClB,CAAC,GAAG;AACF,UAAI,MAAM,SAAS,SAAS;AAC1B,YAAI,MAAM,WAAW,aAAa,QAAQ,OAAO,QAAS;AAC1D,cAAM,IAAI,kBAAkB,MAAM,QAAQ,MAAM,KAAK;AAAA,MACvD;AACA,YAAM,YACJ,MAAM,SAAS,aAAa,MAAM,SAAS,WAAW,MAAM,SAAS,aACjE,SAAS,IACT;AACN,mBAAa,SAAS,SAAS,OAAO,WAAW,UAAU;AAC3D,UAAI,YAAY,MAAM,KAAK,EAAG,UAAS,MAAM,UAAU,SAAS,MAAM;AACtE,UAAI,MAAM,SAAS,WAAY;AAC/B,YAAM,SAAS,UAAU,SAAS;AAClC,eAAS;AACT,UAAI,SAAS,SAAU,UAAS,MAAM,WAAW,MAAM;AAAA,UAClD,UAAS,MAAM,UAAU,SAAS,MAAM;AAC7C,eAAS,EAAE,OAAO,SAAS,MAAM,MAAM,KAAK,CAAC;AAC7C,YAAM,OAAO,SAAS,QAAQ,SAAS,UAAU;AACjD,mBAAa;AACb,eAAS,UAAU,SAAS,UAAU,QAAQ,OAAO,IAAI;AAAA,IAC3D;AAAA,EACF,UAAE;AACA,QAAI,QAAQ,OAAO,WAAW,OAAQ,OAAM,UAAU,SAAS,MAAM;AAAA,aAC5D,OAAQ,OAAM,OAAO;AAC9B,iBAAa;AACb,aAAS,EAAE,OAAO,OAAO,CAAC;AAC1B,YAAQ,IAAI,KAAK;AAAA,EACnB;AACF;AAEA,SAAS,SAAS,SAA2B,YAA+C;AAC1F,SAAO;AAAA,IACL,QAAQ;AAAA,IACR,oBAAoB,QAAQ,KAAK;AAAA,IACjC,GAAI,eAAe,SAAY,CAAC,IAAI,EAAE,WAAW;AAAA,EACnD;AACF;AAEA,SAAS,SACP,SACA,OACA,QACA,YACoB;AACpB,MAAI,MAAM,SAAS,WAAW;AAC5B,YAAQ,SAAS,KAAK;AAAA,MACpB,MAAM;AAAA,MACN;AAAA,MACA,WAAW,MAAM;AAAA,MACjB,MAAM,MAAM;AAAA,IACd,CAAC;AAAA,EACH;AACA,MAAI,MAAM,SAAS,SAAS;AAC1B,YAAQ,SAAS,KAAK;AAAA,MACpB,MAAM;AAAA,MACN;AAAA,MACA,WAAW,MAAM;AAAA,MACjB,MAAM,MAAM;AAAA,MACZ,GAAI,MAAM,eAAe,SAAY,CAAC,IAAI,EAAE,YAAY,MAAM,WAAW;AAAA,IAC3E,CAAC;AACD,WAAO,MAAM,cAAc;AAAA,EAC7B;AACA,MAAI,MAAM,SAAS,YAAY;AAC7B,YAAQ,SAAS,KAAK,EAAE,MAAM,cAAc,QAAQ,WAAW,QAAQ,MAAM,MAAM,KAAK,CAAC;AAAA,EAC3F;AACA,SAAO;AACT;AAEA,eAAe,WAAW,QAAoD;AAC5E,MAAI,CAAC,OAAQ,QAAO;AACpB,QAAM,OAAO;AACb,SAAO;AACT;AAEA,SAAS,UACP,SACA,UACA,QACA,OACA,MACY;AACZ,QAAM,SAAS,WAAW,QAAQ,MAAM;AACxC,QAAM,SAAqB;AAAA,IACzB,YAAY,OAAO;AAAA,IACnB,QAAQ,OAAO;AAAA,IACf;AAAA,IACA,aAAa,GAAG,MAAM;AAAA,IACtB,YAAY;AAAA,IACZ,MAAM,QAAQ,QAAQ;AAAA,EACxB;AACA,SAAO,OAAO,YAAY,SAAS,UAAU,QAAQ,MAAM,MAAM,MAAM,IAAI,IAAI,EAAE,QAAQ,MAAM;AAC7F,WAAO,OAAO;AAAA,EAChB,CAAC;AACD,SAAO;AACT;AAEA,eAAe,YACb,SACA,UACA,QACA,MACA,IACA,MACe;AACf,QAAM,SAAS,OAAO,WAAW;AACjC,MAAI;AACF,aAAS,EAAE,OAAO,WAAW,CAAC;AAC9B,UAAM,UAAuB,EAAE,MAAM,QAAQ,SAAS,MAAM,WAAW,GAAG;AAC1E,UAAM,OAAO,SAAS,MAAM;AAC5B,UAAM,OAAoB,CAAC;AAC3B,YAAQ,OAAO;AACf,UAAM,YAAY,UAAU,SAAS,UAAU,QAAQ,SAAS,MAAM,MAAM,IAAI;AAChF,UAAM,QAAQ,QAAQ,OAAO;AAAA,MAC3B,QAAQ,QAAQ;AAAA,MAChB,EAAE,MAAM,KAAK,UAAU,OAAO,QAAQ,QAAQ,MAAM;AAAA,MACpD,EAAE,OAAO;AAAA,IACX;AACA,UAAM,QAAQ,IAAI,CAAC,WAAW,WAAW,SAAS,UAAU,QAAQ,OAAO,IAAI,CAAC,CAAC;AACjF,QAAI,CAAC,OAAO,QAAS,UAAS,EAAE,OAAO,YAAY,CAAC;AAAA,EACtD,SAAS,OAAO;AACd,QAAI,OAAO,WAAW,QAAQ,OAAO,QAAS;AAC9C,UAAM;AAAA,EACR;AACF;AAEA,eAAe,UACb,SACA,UACA,QACA,SACA,MACA,MACA,MACe;AACf,QAAM,QAAQ,kBAAkB;AAChC,MAAI,cAAc;AAClB,MAAI,YAAY;AAChB,MAAI;AACF,qBAAiB,SAAS,QAAQ,aAAa;AAAA,MAC7C;AAAA,MACA;AAAA,MACA,OAAO,WAAW;AAAA,IACpB,GAAG;AACD,UAAI,OAAO,WAAW,OAAO,QAAS;AACtC,gBAAU,UAAU,KAAK;AACzB,iBAAW,SAAS,aAAa,OAAO,OAAO,QAAQ,OAAO,GAAG;AAC/D,uBAAe,MAAM,KAAK;AAC1B,oBAAY;AAAA,UACV;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA,MAAM;AAAA,QACR;AACA,aAAK,KAAK,MAAM,IAAI;AAAA,MACtB;AACA,UAAI,MAAM,QAAS,WAAU,MAAM,MAAM,OAAO;AAAA,IAClD;AAAA,EACF,UAAE;AACA,SAAK,IAAI;AAAA,EACX;AACF;AAEA,SAAS,cACP,SACA,QACA,OACA,MACA,WACA,WACA,SACQ;AACR,QAAM,QAAQ,MAAM,WAAW;AAC/B,OAAK,KAAK;AAAA,IACR,aAAa,YAAY,MAAM,KAAK;AAAA,IACpC;AAAA,IACA,cAAc,MAAM;AAAA,IACpB,OAAO,MAAM;AAAA,IACb,KAAK,MAAM;AAAA,IACX,MAAM,MAAM;AAAA,IACZ,GAAI,QAAQ,EAAE,SAAS,MAAM,IAAI,CAAC;AAAA,EACpC,CAAC;AACD,MAAI,MAAM,SAAS,YAAY;AAC7B,YAAQ,SAAS,KAAK,EAAE,MAAM,YAAY,QAAQ,OAAO,QAAQ,MAAM,MAAM,KAAK,CAAC;AACnF,WAAO;AAAA,EACT;AACA,QAAM,OAAO,YAAY,MAAM;AAC/B,UAAQ,SAAS,KAAK;AAAA,IACpB,MAAM;AAAA,IACN,QAAQ,OAAO;AAAA,IACf,aAAa,OAAO;AAAA,IACpB;AAAA,EACF,CAAC;AACD,SAAO;AACT;AAEA,eAAe,WACb,SACA,UACA,QACA,OACA,MACe;AACf,mBAAiB,SAAS,OAAO;AAC/B,QAAI,OAAO,WAAW,OAAO,QAAS;AACtC,QAAI,MAAM,SAAS,SAAS;AAC1B,UAAI,MAAM,WAAW,UAAW;AAChC,YAAM,IAAI,kBAAkB,MAAM,QAAQ,MAAM,KAAK;AAAA,IACvD;AACA,QAAI,MAAM,SAAS,QAAS;AAC5B,aAAS,EAAE,OAAO,WAAW,CAAC;AAC9B,UAAM,OAAO,cAAc,MAAM,MAAM,IAAI;AAC3C,UAAM,UAAuB;AAAA,MAC3B,OAAO,MAAM,KAAK;AAAA,MAClB,KAAK,MAAM,KAAK;AAAA,MAChB,YAAY,KAAK,MAAM,aAAa,MAAM,KAAK,IAAI,GAAI;AAAA,IACzD;AACA,QAAI,MAAM,UAAW,SAAQ,YAAY,MAAM;AAC/C,YAAQ,UAAU;AAClB,UAAM,MAAgB,EAAE,KAAK;AAC7B,QAAI,MAAM,UAAW,KAAI,YAAY,MAAM;AAC3C,YAAQ,IAAI,MAAM,MAAM,OAAO,GAAG;AAAA,EACpC;AACF;AAEA,SAAS,aAAa,SAAkB,QAAoB,KAAe,UAAwB;AACjG,QAAM,UAAU,QAAQ;AACxB,MAAI,SAAS;AACX,UAAM,OAAO,eAAe,aAAa,SAAS,QAAQ,GAAG,QAAQ,IAAI;AACzE,QAAI,OAAO,OAAO,YAAY;AAC5B,aAAO,aAAa;AACpB,cAAQ,SAAS,KAAK;AAAA,QACpB,MAAM;AAAA,QACN,QAAQ,OAAO;AAAA,QACf,aAAa,OAAO;AAAA,QACpB;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AACA,MAAI,EAAE,aAAa,IAAI,MAAO;AAC9B,SAAO,MAAM;AAAA,IACX,SAAS,IAAI,KAAK;AAAA,IAClB,cAAc,IAAI,KAAK;AAAA,IACvB,WAAW,IAAI,KAAK;AAAA,IACpB,SAAS,QAAQ,IAAI;AAAA,EACvB;AACF;AAEA,SAAS,cAAc,MAAgB,MAA6B;AAClE,MAAI,aAAa,KAAM,QAAO;AAC9B,QAAM,MAAM,UAAU,MAAM,IAAI;AAChC,MAAI,CAAC,KAAK,QAAS,QAAO;AAC1B,QAAM,YAAY,IAAI,YAAY,IAAI;AACtC,QAAM,YAAY,IAAI,MAAM,IAAI;AAChC,QAAM,QAAQ,IAAI,QAAQ,MAAM,KAAK,IAAI,GAAG,KAAK,QAAQ,IAAI,WAAW,GAAG,WAAW,SAAS;AAC/F,QAAM,MACJ,IAAI,QAAQ,MAAM,KAAK,IAAI,WAAW,KAAK,MAAM,IAAI,WAAW,GAAG,WAAW,SAAS;AACzF,SAAO,EAAE,SAAS,IAAI,SAAS,cAAc,IAAI,cAAc,OAAO,IAAI;AAC5E;AAEA,SAAS,UAAU,MAAsC,MAA0C;AACjG,WAAS,QAAQ,KAAK,SAAS,GAAG,SAAS,GAAG,SAAS,GAAG;AACxD,UAAM,OAAO,KAAK,KAAK;AACvB,QAAI,MAAM,SAAS,YAAY,CAAC,KAAK,QAAS;AAC9C,QAAI,KAAK,cAAc,KAAK,OAAO,KAAK,YAAY,KAAK,MAAO,QAAO;AAAA,EACzE;AACA,SAAO;AACT;AAEA,SAAS,MAAM,QAAgB,WAAmB,WAA2B;AAC3E,MAAI,aAAa,EAAG,QAAO;AAC3B,SAAO,KAAK,MAAO,SAAS,YAAa,SAAS;AACpD;AAEA,SAAS,UAAU,MAAmB,SAAuB;AAC3D,aAAW,OAAO,MAAM;AACtB,QAAI,IAAI,SAAS,YAAY,IAAI,YAAY,OAAW,KAAI,UAAU;AAAA,EACxE;AACF;AAEA,SAAS,UAAU,UAAuC,OAA8B;AACtF,MAAI,MAAM,SAAS,eAAe,MAAM,SAAS,aAAc,UAAS,EAAE,OAAO,WAAW,CAAC;AAC7F,MAAI,MAAM,SAAS,mBAAoB,UAAS,EAAE,OAAO,aAAa,CAAC;AACvE,MAAI,MAAM,SAAS,sBAAsB,MAAM,YAAY,UAAa,MAAM,OAAO,QAAW;AAC9F,aAAS,EAAE,OAAO,YAAY,SAAS,MAAM,SAAS,IAAI,MAAM,GAAG,CAAC;AAAA,EACtE;AACA,MAAI,MAAM,SAAS,0BAA0B,MAAM,YAAY,MAAM,YAAY;AAC/E,aAAS;AAAA,MACP,OAAO;AAAA,MACP,UAAU,MAAM;AAAA,MAChB,YAAY,MAAM;AAAA,MAClB,MAAM,MAAM,QAAQ,CAAC;AAAA,IACvB,CAAC;AAAA,EACH;AACF;AAEA,eAAe,UAAU,SAAkB,QAAoD;AAC7F,MAAI,CAAC,OAAQ,QAAO;AACpB,SAAO,WAAW,MAAM;AACxB,UAAQ,IAAI,MAAM;AAClB,QAAM,OAAO;AACb,MAAI,OAAO,KAAK;AACd,YAAQ,SAAS,KAAK;AAAA,MACpB,MAAM;AAAA,MACN,QAAQ,OAAO;AAAA,MACf,aAAa,OAAO;AAAA,MACpB,WAAW,OAAO,IAAI;AAAA,IACxB,CAAC;AACD,UAAM,QAAQ,aAAa,SAAS,OAAO,GAAG;AAAA,EAChD;AACA,SAAO;AACT;AAEA,SAAS,UAAU,SAA+D;AAChF,QAAM,OAAO,QAAQ,WAAW;AAChC,MAAI,SAAS,OAAQ,QAAO;AAC5B,MAAI,QAAQ,uBAAuB,YAAa,QAAO;AACvD,MAAI,QAAQ,uBAAuB,aAAc,QAAO;AACxD,SAAO;AACT;AAEA,SAAS,YAAY,MAA0C,OAAoC;AACjG,MAAI,SAAS,SAAU,QAAO;AAC9B,MAAI,SAAS,YAAa,QAAO,MAAM,SAAS;AAChD,SAAO,MAAM,SAAS,aAAa,MAAM,KAAK,KAAK,EAAE,SAAS;AAChE;AAEA,SAAS,SAAS,SAA2B,KAA+B;AAC1E,QAAM,OAAO,QAAQ;AACrB,MAAI,SAAS,OAAW,QAAO;AAC/B,SAAO,YAAY,MAAM,GAAG;AAC9B;AAEA,SAAS,SAAS,KAAsC;AACtD,MAAI,WAAW,GAAG,EAAG,QAAO,IAAI;AAChC,SAAO;AACT;AAEA,SAAS,UAAU,KAA2B,YAAiD;AAC7F,MAAI,CAAC,WAAY,QAAO,QAAQ;AAChC,MAAI,CAAC,WAAW,GAAG,EAAG,QAAO,QAAQ;AACrC,SAAO,UAAU,IAAI,KAAK,UAAU;AACtC;AAEA,SAAS,UAAuB;AAC9B,SAAO;AAAA,IACL,UAAU;AAAA,IAAC;AAAA,IACX,OAAO;AAAA,IAAC;AAAA,EACV;AACF;AAEA,SAAS,YAAY,QAA2B,KAA+C;AAC7F,MAAI,WAAW;AACf,SAAO,CAAC,UAAU;AAChB,UAAM,MAAM,KAAK,UAAU,KAAK;AAChC,QAAI,QAAQ,SAAU;AACtB,eAAW;AACX,WAAO,KAAK,KAAK;AACjB,QAAI,QAAQ,KAAK;AAAA,EACnB;AACF;AAEA,SAAS,WAAW,QAA0E;AAC5F,QAAM,aAAa,IAAI,gBAAgB;AACvC,QAAM,QAAQ,MAAM,WAAW,MAAM;AACrC,MAAI,OAAO,QAAS,YAAW,MAAM;AAAA,MAChC,QAAO,iBAAiB,SAAS,KAAK;AAC3C,SAAO;AAAA,IACL;AAAA,IACA,QAAQ,MAAM,OAAO,oBAAoB,SAAS,KAAK;AAAA,EACzD;AACF;AAQA,SAAS,SAAS,QAA+B;AAC/C,QAAM,QAAkB,CAAC;AACzB,MAAI,QAAQ;AACZ,MAAI;AACJ,QAAM,SAAS,MAAM;AACnB,UAAM,UAAU;AAChB,WAAO;AACP,cAAU;AAAA,EACZ;AACA,QAAM,UAAU,MAAM,OAAO;AAC7B,SAAO,iBAAiB,SAAS,OAAO;AACxC,SAAO;AAAA,IACL,KAAK,MAAM;AACT,UAAI,SAAS,OAAO,WAAW,KAAK,WAAW,EAAG;AAClD,YAAM,KAAK,IAAI;AACf,aAAO;AAAA,IACT;AAAA,IACA,MAAM;AACJ,cAAQ;AACR,aAAO;AAAA,IACT;AAAA,IACA,UAAU;AAAA,MACR,QAAQ,OAAO,aAAa,IAAI;AAC9B,YAAI;AACF,iBAAO,CAAC,OAAO,SAAS;AACtB,kBAAM,OAAO,MAAM,MAAM;AACzB,gBAAI,SAAS,QAAW;AACtB,oBAAM;AACN;AAAA,YACF;AACA,gBAAI,MAAO;AACX,kBAAM,IAAI,QAAc,CAAC,YAAY;AACnC,kBAAI,MAAM,SAAS,KAAK,SAAS,OAAO,QAAS,SAAQ;AAAA,kBACpD,QAAO;AAAA,YACd,CAAC;AAAA,UACH;AAAA,QACF,UAAE;AACA,iBAAO,oBAAoB,SAAS,OAAO;AAAA,QAC7C;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { type AudioChunk, type AudioSource } from './audio.ts';
|
|
2
|
+
import type { AudioOut, Cue } from './soundscape.ts';
|
|
3
|
+
import type { EchoRecommendation } from './turns.ts';
|
|
4
|
+
export type EchoMode = 'browser' | 'wasm' | 'none';
|
|
5
|
+
export interface EchoCanceller {
|
|
6
|
+
cancel(mic: AudioChunk, reference: AudioChunk): AudioChunk;
|
|
7
|
+
}
|
|
8
|
+
export interface EchoControl {
|
|
9
|
+
mode: EchoMode;
|
|
10
|
+
reference: AudioSource;
|
|
11
|
+
canceller?: EchoCanceller;
|
|
12
|
+
}
|
|
13
|
+
export interface EchoReport {
|
|
14
|
+
delayMs: number;
|
|
15
|
+
cancellationDb: number;
|
|
16
|
+
residualDb: number;
|
|
17
|
+
recommendation: EchoRecommendation;
|
|
18
|
+
}
|
|
19
|
+
export interface MicOpen {
|
|
20
|
+
echoCancellation: boolean | 'all';
|
|
21
|
+
playback: 'webaudio' | 'rtc-loopback';
|
|
22
|
+
}
|
|
23
|
+
export declare function micOpen(mode: EchoMode, useAll: boolean): MicOpen;
|
|
24
|
+
export declare function echoRuntime(hasMediaDevices: boolean): 'node' | 'browser';
|
|
25
|
+
export declare function assertNodeEcho(mode: EchoMode, runtime: 'node' | 'browser'): void;
|
|
26
|
+
export declare function chooseCapture(tried: EchoMode, report: EchoReport): EchoMode;
|
|
27
|
+
export declare function openEchoMic(control: EchoControl, mic: AudioSource): AudioSource;
|
|
28
|
+
export declare function measureEcho(reference: Float32Array, mic: Float32Array, sampleRate: number): EchoReport;
|
|
29
|
+
export declare function calibrateEcho(out: AudioOut, mic: AudioSource, probe: Cue, durationMs: number, signal: AbortSignal): Promise<EchoReport>;
|
package/dist/types/index.d.ts
CHANGED
|
@@ -4,6 +4,8 @@ export { blankUsage } from './blank-usage.ts';
|
|
|
4
4
|
export type { CaptionHistoryEntry } from './captions.ts';
|
|
5
5
|
export { captionsFromHistory } from './captions.ts';
|
|
6
6
|
export { convertAudio, convertAudioSource, decodeAudioBase64, encodeAudioBase64, } from './convert.ts';
|
|
7
|
+
export type { EchoCanceller, EchoControl, EchoMode, EchoReport, MicOpen } from './echo.ts';
|
|
8
|
+
export { assertNodeEcho, calibrateEcho, chooseCapture, echoRuntime, measureEcho, micOpen, openEchoMic, } from './echo.ts';
|
|
7
9
|
export { VoiceRequestError } from './errors.ts';
|
|
8
10
|
export type { Delivery, DeliverySupport, KnownRealtimeApi, KnownSpeechApi, KnownTranscriptionApi, RealtimeApi, RealtimeModel, SpeechApi, SpeechModel, TranscriptionApi, TranscriptionModel, VoiceInfo, VoiceModel, VoiceModelType, } from './models.ts';
|
|
9
11
|
export { getVoiceModel, listVoiceModels, registerVoiceModel, } from './models.ts';
|
|
@@ -11,6 +11,7 @@ export interface VoiceWire {
|
|
|
11
11
|
export interface VoiceSocket {
|
|
12
12
|
send(data: string): void;
|
|
13
13
|
close(): void;
|
|
14
|
+
addEventListener(type: 'open', listener: () => void): void;
|
|
14
15
|
addEventListener(type: 'message', listener: (event: {
|
|
15
16
|
data: unknown;
|
|
16
17
|
}) => void): void;
|
package/dist/types/talker.d.ts
CHANGED
|
@@ -11,7 +11,7 @@ export type ThinkerUpdate = {
|
|
|
11
11
|
message: ToolResultMessage;
|
|
12
12
|
};
|
|
13
13
|
export interface Thinker {
|
|
14
|
-
ask(request: string, heard: string[], signal: AbortSignal): AsyncIterable<ThinkerUpdate>;
|
|
14
|
+
ask(request: string, heard: string[], signal: AbortSignal, context?: string): AsyncIterable<ThinkerUpdate>;
|
|
15
15
|
}
|
|
16
16
|
export interface BindThinkerOptions {
|
|
17
17
|
progress: Delivery;
|
package/dist/types/turns.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { AssistantMessageEvent, JsonObject, ProviderRequestOptions, ToolCall, UserMessage } from '@earendil-works/pi-ai';
|
|
2
2
|
import { type AudioSink, type AudioSource } from './audio.ts';
|
|
3
|
+
import { type EchoControl } from './echo.ts';
|
|
3
4
|
import type { SpeechModel, TranscriptionModel } from './models.ts';
|
|
4
5
|
import { type AudioOut, type Soundscape } from './soundscape.ts';
|
|
5
6
|
import type { SpeechEvent, SpeechInput } from './speech.ts';
|
|
@@ -30,6 +31,7 @@ export interface VoiceTurnOptions {
|
|
|
30
31
|
announce?: (call: ToolCall) => string | undefined;
|
|
31
32
|
bargeIn?: BargeIn;
|
|
32
33
|
echoRecommendation?: EchoRecommendation;
|
|
34
|
+
echo?: EchoControl;
|
|
33
35
|
soundscape?: Soundscape;
|
|
34
36
|
}
|
|
35
37
|
export interface VoiceModels {
|
package/package.json
CHANGED