@ai-ecoverse/pi-audio 1.19.0 → 1.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/dist/captions.js +16 -8
- package/dist/captions.js.map +2 -2
- package/dist/durable/index.js +3 -1
- package/dist/durable/index.js.map +2 -2
- package/dist/google/catalog.js +61 -0
- package/dist/google/catalog.js.map +7 -0
- package/dist/google/index.js +5 -0
- package/dist/google/index.js.map +7 -0
- package/dist/google/speak.js +160 -0
- package/dist/google/speak.js.map +7 -0
- package/dist/google/wire.js +43 -0
- package/dist/google/wire.js.map +7 -0
- package/dist/soundscape.js +14 -2
- package/dist/soundscape.js.map +2 -2
- package/dist/turns.js +78 -30
- package/dist/turns.js.map +2 -2
- package/dist/types/google/catalog.d.ts +1 -0
- package/dist/types/google/index.d.ts +1 -0
- package/dist/types/google/speak.d.ts +4 -0
- package/dist/types/google/wire.d.ts +8 -0
- package/dist/vad-listen.js +43 -12
- package/dist/vad-listen.js.map +2 -2
- package/dist/vad.js +15 -7
- package/dist/vad.js.map +2 -2
- package/package.json +5 -1
package/README.md
CHANGED
|
@@ -23,6 +23,8 @@ pi-audio does not apply that patch. Hosts such as slicc-agent record monkeypatch
|
|
|
23
23
|
|
|
24
24
|
`@ai-ecoverse/pi-audio/durable` appends a `voice.truncate` entry for a `VoiceCut`. The entry has no model messages. Its `edits` replace the target assistant message with the text heard so far and keep tool calls. The stored entry stays intact, and pi-durable derives the shorter message for the next model request. `@earendil-works/pi-durable` is an optional peer (`^1.1.0`); the root package does not import it. `VoiceCut.entryId` is the decimal string of pi-durable's numeric entry id.
|
|
25
25
|
|
|
26
|
+
`@ai-ecoverse/pi-audio/google` registers Gemini speech adapters when `registerGoogle()` is called. `gemini-3.8-flash-tts` and `gemini-3.8-flash-lite-tts` post JSON to `models/{id}:generateContent` with `responseModalities: ["AUDIO"]` and a prebuilt `voiceName`. Unary audio is WAV at 24 kHz unless the format asks for `AUDIO_L16`, `AUDIO_MULAW`, or `AUDIO_ALAW`. `textStreaming` is false. Output audio is priced from 25 tokens per second: $0.0135 per minute for Flash and $0.009 per minute for Flash-Lite through December 31, 2026. Text input is $0.50 per million tokens and is not on `audioCost`. Auth is `x-goog-api-key`.
|
|
27
|
+
|
|
26
28
|
`@ai-ecoverse/pi-audio/fish` registers Fish Audio speech adapters when `registerFish()` is called. `transcribe-1-pro` posts multipart field `audio` to `/asr` with header `model: transcribe-1-pro` ($0.36 per hour). `ignore_timestamps` is false, so word `segments` and `speaker_turns` come back. There is no realtime ASR. `s2.1-pro` posts JSON to `/tts` for a string and streams MessagePack `text` events on `wss://…/tts/live` for an async iterable (`textStreaming: true`). `s2.1-pro-free` is the same path at $0. Paid speech is $15 per million UTF-8 bytes, stored as $0.015 per thousand characters. Voices are caller `reference_id`s (`cloning: true`).
|
|
27
29
|
|
|
28
30
|
`@ai-ecoverse/pi-audio/mistral` registers Mistral speech adapters when `registerMistral()` is called. `voxtral-mini-latest` posts multipart audio to `/audio/transcriptions` ($0.003 per minute, diarization on). `voxtral-mini-transcribe-realtime-2602` streams base64 PCM on `wss://…/audio/transcriptions/realtime` after `session.created` ($0.006 per minute). Realtime has no server endpointing, so `turnDetection` is `none` and the registry VAD wrapper applies. `voxtral-mini-tts-2603` posts JSON to `/audio/speech` and reads SSE `speech.audio.delta` float32 PCM at 24 kHz ($0.016 per thousand characters). Hosted voice ids such as `en_paul_neutral` differ from the open-weight presets. `textStreaming` is false.
|
package/dist/captions.js
CHANGED
|
@@ -59,7 +59,7 @@ function captionsFromHistory(entries) {
|
|
|
59
59
|
type: "agent_generated",
|
|
60
60
|
turnId,
|
|
61
61
|
utteranceId,
|
|
62
|
-
text: blocks
|
|
62
|
+
text: joined(blocks)
|
|
63
63
|
});
|
|
64
64
|
}
|
|
65
65
|
return captions;
|
|
@@ -71,7 +71,7 @@ function firstMessage(entry) {
|
|
|
71
71
|
}
|
|
72
72
|
function pushUser(captions, entry, message, voice) {
|
|
73
73
|
const turnId = String(entry.id);
|
|
74
|
-
const text = textBlocks(message.content)
|
|
74
|
+
const text = joined(textBlocks(message.content));
|
|
75
75
|
if (voice.confidence === void 0) {
|
|
76
76
|
captions.push({ type: "user_final", turnId, segmentId: "turn", text });
|
|
77
77
|
return;
|
|
@@ -105,11 +105,10 @@ function noteCut(captions, agents, data) {
|
|
|
105
105
|
}
|
|
106
106
|
function heardInBlocks(blocks, contentIndex, heardUpTo) {
|
|
107
107
|
let prior = 0;
|
|
108
|
-
let index = 0;
|
|
109
108
|
for (const block of blocks) {
|
|
110
|
-
if (index === contentIndex) return prior + clampHeard(block, heardUpTo);
|
|
111
|
-
|
|
112
|
-
|
|
109
|
+
if (block.index === contentIndex) return prior + clampHeard(block.text, heardUpTo);
|
|
110
|
+
if (block.index > contentIndex) return prior;
|
|
111
|
+
prior += block.text.length;
|
|
113
112
|
}
|
|
114
113
|
return prior;
|
|
115
114
|
}
|
|
@@ -118,12 +117,21 @@ function clampHeard(block, heardUpTo) {
|
|
|
118
117
|
if (heardUpTo >= block.length) return block.length;
|
|
119
118
|
return heardUpTo;
|
|
120
119
|
}
|
|
120
|
+
function joined(blocks) {
|
|
121
|
+
let text = "";
|
|
122
|
+
for (const block of blocks) text += block.text;
|
|
123
|
+
return text;
|
|
124
|
+
}
|
|
121
125
|
function textBlocks(content) {
|
|
122
|
-
if (typeof content === "string") return [content];
|
|
126
|
+
if (typeof content === "string") return [{ index: 0, text: content }];
|
|
123
127
|
if (!content) return [];
|
|
124
128
|
const blocks = [];
|
|
129
|
+
let index = 0;
|
|
125
130
|
for (const block of content) {
|
|
126
|
-
if (block.type === "text" && typeof block.text === "string")
|
|
131
|
+
if (block.type === "text" && typeof block.text === "string") {
|
|
132
|
+
blocks.push({ index, text: block.text });
|
|
133
|
+
}
|
|
134
|
+
index += 1;
|
|
127
135
|
}
|
|
128
136
|
return blocks;
|
|
129
137
|
}
|
package/dist/captions.js.map
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../src/captions.ts"],
|
|
4
|
-
"sourcesContent": ["import type { CharAlignment } from './audio.ts';\nimport type { CaptionEvent } from './turns.ts';\n\nexport interface SpokenClock {\n start: number;\n end: number;\n durationMs: number;\n alignment?: CharAlignment;\n}\n\nexport interface UtteranceMap {\n spokenStart: number;\n spokenEnd: number;\n kind: string;\n}\n\nexport interface CaptionHistoryEntry {\n id: string | number;\n kind: string;\n model?: readonly HistoryMessage[];\n data?: unknown;\n}\n\ninterface HistoryMessage {\n role?: string;\n content?: string | readonly HistoryBlock[];\n}\n\ninterface HistoryBlock {\n type?: string;\n text?: string;\n}\n\ninterface StoredAgent {\n turnId: string;\n utteranceId: string;\n blocks:
|
|
5
|
-
"mappings": "
|
|
4
|
+
"sourcesContent": ["import type { CharAlignment } from './audio.ts';\nimport type { CaptionEvent } from './turns.ts';\n\nexport interface SpokenClock {\n start: number;\n end: number;\n durationMs: number;\n alignment?: CharAlignment;\n}\n\nexport interface UtteranceMap {\n spokenStart: number;\n spokenEnd: number;\n kind: string;\n}\n\nexport interface CaptionHistoryEntry {\n id: string | number;\n kind: string;\n model?: readonly HistoryMessage[];\n data?: unknown;\n}\n\ninterface HistoryMessage {\n role?: string;\n content?: string | readonly HistoryBlock[];\n}\n\ninterface HistoryBlock {\n type?: string;\n text?: string;\n}\n\ninterface IndexedBlock {\n index: number;\n text: string;\n}\n\ninterface StoredAgent {\n turnId: string;\n utteranceId: string;\n blocks: IndexedBlock[];\n}\n\nexport function spokenOffset(clock: SpokenClock, offsetMs: number): number {\n const length = clock.end - clock.start;\n if (length <= 0) return clock.start;\n const alignment = clock.alignment;\n if (alignment && alignment.startMs.length === length && alignment.durationMs.length === length) {\n return clock.start + alignedHeard(alignment, offsetMs, length);\n }\n if (clock.durationMs <= 0) {\n if (offsetMs <= 0) return clock.start;\n return clock.end;\n }\n const ratio = offsetMs <= 0 ? 0 : offsetMs / clock.durationMs;\n const fraction = ratio > 1 ? 1 : ratio;\n return clock.start + Math.round(fraction * length);\n}\n\nfunction alignedHeard(alignment: CharAlignment, offsetMs: number, length: number): number {\n const first = alignment.startMs[0];\n let origin = 0;\n if (first !== undefined) origin = first;\n const at = origin + offsetMs;\n let heard = 0;\n for (let index = 0; index < length; index += 1) {\n const start = alignment.startMs[index];\n if (start !== undefined && start <= at) heard = index + 1;\n }\n return heard;\n}\n\nexport function utteranceIndex(feedPos: number, maps: readonly UtteranceMap[]): number {\n let upTo = 0;\n for (const map of maps) {\n if (map.kind !== 'answer') continue;\n if (feedPos <= map.spokenStart) break;\n const end = feedPos < map.spokenEnd ? feedPos : map.spokenEnd;\n upTo += end - map.spokenStart;\n }\n return upTo;\n}\n\nexport function captionsFromHistory(entries: readonly CaptionHistoryEntry[]): CaptionEvent[] {\n const captions: CaptionEvent[] = [];\n const agents = new Map<string, StoredAgent>();\n for (const entry of entries) {\n if (entry.kind === 'voice.truncate') {\n noteCut(captions, agents, entry.data);\n continue;\n }\n const message = firstMessage(entry);\n if (!message) continue;\n const voice = voiceMeta(entry.data);\n if (message.role === 'user' && voice) {\n pushUser(captions, entry, message, voice);\n continue;\n }\n if (message.role !== 'assistant') continue;\n const blocks = textBlocks(message.content);\n const turnId = String(entry.id);\n const utteranceId = `${turnId}:agent`;\n agents.set(turnId, { turnId, utteranceId, blocks });\n captions.push({\n type: 'agent_generated',\n turnId,\n utteranceId,\n text: joined(blocks),\n });\n }\n return captions;\n}\n\nfunction firstMessage(entry: CaptionHistoryEntry): HistoryMessage | undefined {\n const model = entry.model;\n if (!model) return undefined;\n return model[0];\n}\n\nfunction pushUser(\n captions: CaptionEvent[],\n entry: CaptionHistoryEntry,\n message: HistoryMessage,\n voice: { confidence?: number }\n): void {\n const turnId = String(entry.id);\n const text = joined(textBlocks(message.content));\n if (voice.confidence === undefined) {\n captions.push({ type: 'user_final', turnId, segmentId: 'turn', text });\n return;\n }\n captions.push({\n type: 'user_final',\n turnId,\n segmentId: 'turn',\n text,\n confidence: voice.confidence,\n });\n}\n\nfunction noteCut(captions: CaptionEvent[], agents: Map<string, StoredAgent>, data: unknown): void {\n const cut = cutData(data);\n if (!cut) return;\n const agent = agents.get(cut.entryId);\n if (!agent) return;\n const heard = heardInBlocks(agent.blocks, cut.contentIndex, cut.heardUpTo);\n captions.push({\n type: 'agent_spoken',\n turnId: agent.turnId,\n utteranceId: agent.utteranceId,\n upTo: heard,\n });\n captions.push({\n type: 'agent_cut',\n turnId: agent.turnId,\n utteranceId: agent.utteranceId,\n heardUpTo: heard,\n });\n}\n\nfunction heardInBlocks(\n blocks: readonly IndexedBlock[],\n contentIndex: number,\n heardUpTo: number\n): number {\n let prior = 0;\n for (const block of blocks) {\n if (block.index === contentIndex) return prior + clampHeard(block.text, heardUpTo);\n if (block.index > contentIndex) return prior;\n prior += block.text.length;\n }\n return prior;\n}\n\nfunction clampHeard(block: string, heardUpTo: number): number {\n if (heardUpTo <= 0) return 0;\n if (heardUpTo >= block.length) return block.length;\n return heardUpTo;\n}\n\nfunction joined(blocks: readonly IndexedBlock[]): string {\n let text = '';\n for (const block of blocks) text += block.text;\n return text;\n}\n\nfunction textBlocks(content: HistoryMessage['content']): IndexedBlock[] {\n if (typeof content === 'string') return [{ index: 0, text: content }];\n if (!content) return [];\n const blocks: IndexedBlock[] = [];\n let index = 0;\n for (const block of content) {\n if (block.type === 'text' && typeof block.text === 'string') {\n blocks.push({ index, text: block.text });\n }\n index += 1;\n }\n return blocks;\n}\n\nfunction voiceMeta(data: unknown): { confidence?: number } | undefined {\n if (typeof data !== 'object' || data === null) return undefined;\n if (!('source' in data) || data.source !== 'voice') return undefined;\n if ('confidence' in data && typeof data.confidence === 'number') {\n return { confidence: data.confidence };\n }\n return {};\n}\n\nfunction cutData(\n data: unknown\n): { entryId: string; contentIndex: number; heardUpTo: number } | undefined {\n if (typeof data !== 'object' || data === null) return undefined;\n if (!('entryId' in data) || typeof data.entryId !== 'string') return undefined;\n if (!('contentIndex' in data) || typeof data.contentIndex !== 'number') return undefined;\n if (!('heardUpTo' in data) || typeof data.heardUpTo !== 'number') return undefined;\n return { entryId: data.entryId, contentIndex: data.contentIndex, heardUpTo: data.heardUpTo };\n}\n"],
|
|
5
|
+
"mappings": "AA4CO,SAAS,aAAa,OAAoB,UAA0B;AACzE,QAAM,SAAS,MAAM,MAAM,MAAM;AACjC,MAAI,UAAU,EAAG,QAAO,MAAM;AAC9B,QAAM,YAAY,MAAM;AACxB,MAAI,aAAa,UAAU,QAAQ,WAAW,UAAU,UAAU,WAAW,WAAW,QAAQ;AAC9F,WAAO,MAAM,QAAQ,aAAa,WAAW,UAAU,MAAM;AAAA,EAC/D;AACA,MAAI,MAAM,cAAc,GAAG;AACzB,QAAI,YAAY,EAAG,QAAO,MAAM;AAChC,WAAO,MAAM;AAAA,EACf;AACA,QAAM,QAAQ,YAAY,IAAI,IAAI,WAAW,MAAM;AACnD,QAAM,WAAW,QAAQ,IAAI,IAAI;AACjC,SAAO,MAAM,QAAQ,KAAK,MAAM,WAAW,MAAM;AACnD;AAEA,SAAS,aAAa,WAA0B,UAAkB,QAAwB;AACxF,QAAM,QAAQ,UAAU,QAAQ,CAAC;AACjC,MAAI,SAAS;AACb,MAAI,UAAU,OAAW,UAAS;AAClC,QAAM,KAAK,SAAS;AACpB,MAAI,QAAQ;AACZ,WAAS,QAAQ,GAAG,QAAQ,QAAQ,SAAS,GAAG;AAC9C,UAAM,QAAQ,UAAU,QAAQ,KAAK;AACrC,QAAI,UAAU,UAAa,SAAS,GAAI,SAAQ,QAAQ;AAAA,EAC1D;AACA,SAAO;AACT;AAEO,SAAS,eAAe,SAAiB,MAAuC;AACrF,MAAI,OAAO;AACX,aAAW,OAAO,MAAM;AACtB,QAAI,IAAI,SAAS,SAAU;AAC3B,QAAI,WAAW,IAAI,YAAa;AAChC,UAAM,MAAM,UAAU,IAAI,YAAY,UAAU,IAAI;AACpD,YAAQ,MAAM,IAAI;AAAA,EACpB;AACA,SAAO;AACT;AAEO,SAAS,oBAAoB,SAAyD;AAC3F,QAAM,WAA2B,CAAC;AAClC,QAAM,SAAS,oBAAI,IAAyB;AAC5C,aAAW,SAAS,SAAS;AAC3B,QAAI,MAAM,SAAS,kBAAkB;AACnC,cAAQ,UAAU,QAAQ,MAAM,IAAI;AACpC;AAAA,IACF;AACA,UAAM,UAAU,aAAa,KAAK;AAClC,QAAI,CAAC,QAAS;AACd,UAAM,QAAQ,UAAU,MAAM,IAAI;AAClC,QAAI,QAAQ,SAAS,UAAU,OAAO;AACpC,eAAS,UAAU,OAAO,SAAS,KAAK;AACxC;AAAA,IACF;AACA,QAAI,QAAQ,SAAS,YAAa;AAClC,UAAM,SAAS,WAAW,QAAQ,OAAO;AACzC,UAAM,SAAS,OAAO,MAAM,EAAE;AAC9B,UAAM,cAAc,GAAG,MAAM;AAC7B,WAAO,IAAI,QAAQ,EAAE,QAAQ,aAAa,OAAO,CAAC;AAClD,aAAS,KAAK;AAAA,MACZ,MAAM;AAAA,MACN;AAAA,MACA;AAAA,MACA,MAAM,OAAO,MAAM;AAAA,IACrB,CAAC;AAAA,EACH;AACA,SAAO;AACT;AAEA,SAAS,aAAa,OAAwD;AAC5E,QAAM,QAAQ,MAAM;AACpB,MAAI,CAAC,MAAO,QAAO;AACnB,SAAO,MAAM,CAAC;AAChB;AAEA,SAAS,SACP,UACA,OACA,SACA,OACM;AACN,QAAM,SAAS,OAAO,MAAM,EAAE;AAC9B,QAAM,OAAO,OAAO,WAAW,QAAQ,OAAO,CAAC;AAC/C,MAAI,MAAM,eAAe,QAAW;AAClC,aAAS,KAAK,EAAE,MAAM,cAAc,QAAQ,WAAW,QAAQ,KAAK,CAAC;AACrE;AAAA,EACF;AACA,WAAS,KAAK;AAAA,IACZ,MAAM;AAAA,IACN;AAAA,IACA,WAAW;AAAA,IACX;AAAA,IACA,YAAY,MAAM;AAAA,EACpB,CAAC;AACH;AAEA,SAAS,QAAQ,UAA0B,QAAkC,MAAqB;AAChG,QAAM,MAAM,QAAQ,IAAI;AACxB,MAAI,CAAC,IAAK;AACV,QAAM,QAAQ,OAAO,IAAI,IAAI,OAAO;AACpC,MAAI,CAAC,MAAO;AACZ,QAAM,QAAQ,cAAc,MAAM,QAAQ,IAAI,cAAc,IAAI,SAAS;AACzE,WAAS,KAAK;AAAA,IACZ,MAAM;AAAA,IACN,QAAQ,MAAM;AAAA,IACd,aAAa,MAAM;AAAA,IACnB,MAAM;AAAA,EACR,CAAC;AACD,WAAS,KAAK;AAAA,IACZ,MAAM;AAAA,IACN,QAAQ,MAAM;AAAA,IACd,aAAa,MAAM;AAAA,IACnB,WAAW;AAAA,EACb,CAAC;AACH;AAEA,SAAS,cACP,QACA,cACA,WACQ;AACR,MAAI,QAAQ;AACZ,aAAW,SAAS,QAAQ;AAC1B,QAAI,MAAM,UAAU,aAAc,QAAO,QAAQ,WAAW,MAAM,MAAM,SAAS;AACjF,QAAI,MAAM,QAAQ,aAAc,QAAO;AACvC,aAAS,MAAM,KAAK;AAAA,EACtB;AACA,SAAO;AACT;AAEA,SAAS,WAAW,OAAe,WAA2B;AAC5D,MAAI,aAAa,EAAG,QAAO;AAC3B,MAAI,aAAa,MAAM,OAAQ,QAAO,MAAM;AAC5C,SAAO;AACT;AAEA,SAAS,OAAO,QAAyC;AACvD,MAAI,OAAO;AACX,aAAW,SAAS,OAAQ,SAAQ,MAAM;AAC1C,SAAO;AACT;AAEA,SAAS,WAAW,SAAoD;AACtE,MAAI,OAAO,YAAY,SAAU,QAAO,CAAC,EAAE,OAAO,GAAG,MAAM,QAAQ,CAAC;AACpE,MAAI,CAAC,QAAS,QAAO,CAAC;AACtB,QAAM,SAAyB,CAAC;AAChC,MAAI,QAAQ;AACZ,aAAW,SAAS,SAAS;AAC3B,QAAI,MAAM,SAAS,UAAU,OAAO,MAAM,SAAS,UAAU;AAC3D,aAAO,KAAK,EAAE,OAAO,MAAM,MAAM,KAAK,CAAC;AAAA,IACzC;AACA,aAAS;AAAA,EACX;AACA,SAAO;AACT;AAEA,SAAS,UAAU,MAAoD;AACrE,MAAI,OAAO,SAAS,YAAY,SAAS,KAAM,QAAO;AACtD,MAAI,EAAE,YAAY,SAAS,KAAK,WAAW,QAAS,QAAO;AAC3D,MAAI,gBAAgB,QAAQ,OAAO,KAAK,eAAe,UAAU;AAC/D,WAAO,EAAE,YAAY,KAAK,WAAW;AAAA,EACvC;AACA,SAAO,CAAC;AACV;AAEA,SAAS,QACP,MAC0E;AAC1E,MAAI,OAAO,SAAS,YAAY,SAAS,KAAM,QAAO;AACtD,MAAI,EAAE,aAAa,SAAS,OAAO,KAAK,YAAY,SAAU,QAAO;AACrE,MAAI,EAAE,kBAAkB,SAAS,OAAO,KAAK,iBAAiB,SAAU,QAAO;AAC/E,MAAI,EAAE,eAAe,SAAS,OAAO,KAAK,cAAc,SAAU,QAAO;AACzE,SAAO,EAAE,SAAS,KAAK,SAAS,cAAc,KAAK,cAAc,WAAW,KAAK,UAAU;AAC7F;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
package/dist/durable/index.js
CHANGED
|
@@ -11,7 +11,9 @@ function keptBlock(block, index, cut) {
|
|
|
11
11
|
const heard = block.text.slice(0, cut.heardUpTo);
|
|
12
12
|
if (heard.length === 0) return void 0;
|
|
13
13
|
if (heard === block.text) return block;
|
|
14
|
-
|
|
14
|
+
const shortened = { ...block, text: heard };
|
|
15
|
+
delete shortened.textSignature;
|
|
16
|
+
return shortened;
|
|
15
17
|
}
|
|
16
18
|
function cutAssistantMessage(message, cut) {
|
|
17
19
|
const content = [];
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../src/durable/index.ts"],
|
|
4
|
-
"sourcesContent": ["import type { AssistantMessage } from '@earendil-works/pi-ai';\nimport type { ConversationId, EntryId, EntryRecord, Tx } from '@earendil-works/pi-durable';\nimport type { VoiceCut } from '../turns.ts';\n\ntype Block = AssistantMessage['content'][number];\n\ninterface TruncateSession<C> {\n commit<T>(change: (tx: Tx) => T | Promise<T>, context: C): Promise<T>;\n}\n\nfunction entryIdOf(entryId: string): EntryId {\n if (!/^\\d+$/.test(entryId)) {\n throw new Error(`Voice cut entry is not a durable id: ${entryId}`);\n }\n return Number(entryId) as EntryId;\n}\n\nfunction keptBlock(block: Block, index: number, cut: VoiceCut): Block | undefined {\n if (block.type === 'toolCall' || block.type === 'thinking') return block;\n if (index < cut.contentIndex) return block;\n if (index > cut.contentIndex) return undefined;\n const heard = block.text.slice(0, cut.heardUpTo);\n if (heard.length === 0) return undefined;\n if (heard === block.text) return block;\n
|
|
5
|
-
"mappings": "AAUA,SAAS,UAAU,SAA0B;AAC3C,MAAI,CAAC,QAAQ,KAAK,OAAO,GAAG;AAC1B,UAAM,IAAI,MAAM,wCAAwC,OAAO,EAAE;AAAA,EACnE;AACA,SAAO,OAAO,OAAO;AACvB;AAEA,SAAS,UAAU,OAAc,OAAe,KAAkC;AAChF,MAAI,MAAM,SAAS,cAAc,MAAM,SAAS,WAAY,QAAO;AACnE,MAAI,QAAQ,IAAI,aAAc,QAAO;AACrC,MAAI,QAAQ,IAAI,aAAc,QAAO;AACrC,QAAM,QAAQ,MAAM,KAAK,MAAM,GAAG,IAAI,SAAS;AAC/C,MAAI,MAAM,WAAW,EAAG,QAAO;AAC/B,MAAI,UAAU,MAAM,KAAM,QAAO;AACjC,
|
|
4
|
+
"sourcesContent": ["import type { AssistantMessage } from '@earendil-works/pi-ai';\nimport type { ConversationId, EntryId, EntryRecord, Tx } from '@earendil-works/pi-durable';\nimport type { VoiceCut } from '../turns.ts';\n\ntype Block = AssistantMessage['content'][number];\n\ninterface TruncateSession<C> {\n commit<T>(change: (tx: Tx) => T | Promise<T>, context: C): Promise<T>;\n}\n\nfunction entryIdOf(entryId: string): EntryId {\n if (!/^\\d+$/.test(entryId)) {\n throw new Error(`Voice cut entry is not a durable id: ${entryId}`);\n }\n return Number(entryId) as EntryId;\n}\n\nfunction keptBlock(block: Block, index: number, cut: VoiceCut): Block | undefined {\n if (block.type === 'toolCall' || block.type === 'thinking') return block;\n if (index < cut.contentIndex) return block;\n if (index > cut.contentIndex) return undefined;\n const heard = block.text.slice(0, cut.heardUpTo);\n if (heard.length === 0) return undefined;\n if (heard === block.text) return block;\n const shortened = { ...block, text: heard };\n delete shortened.textSignature;\n return shortened;\n}\n\nfunction cutAssistantMessage(message: AssistantMessage, cut: VoiceCut): AssistantMessage {\n const content: Block[] = [];\n for (const [index, block] of message.content.entries()) {\n const kept = keptBlock(block, index, cut);\n if (kept === undefined) continue;\n content.push(kept);\n }\n return { ...message, content };\n}\n\nexport async function appendVoiceTruncate<C>(\n session: TruncateSession<C>,\n conversationId: ConversationId,\n cut: VoiceCut,\n message: AssistantMessage,\n context: C\n): Promise<EntryRecord> {\n const replacement = cutAssistantMessage(message, cut);\n const target = entryIdOf(cut.entryId);\n return session.commit(\n (tx) =>\n tx.appendEntry(conversationId, {\n kind: 'voice.truncate',\n data: {\n entryId: cut.entryId,\n contentIndex: cut.contentIndex,\n heardUpTo: cut.heardUpTo,\n heardMs: cut.heardMs,\n },\n edits: [{ target, action: 'replace', messages: [replacement] }],\n }),\n context\n );\n}\n"],
|
|
5
|
+
"mappings": "AAUA,SAAS,UAAU,SAA0B;AAC3C,MAAI,CAAC,QAAQ,KAAK,OAAO,GAAG;AAC1B,UAAM,IAAI,MAAM,wCAAwC,OAAO,EAAE;AAAA,EACnE;AACA,SAAO,OAAO,OAAO;AACvB;AAEA,SAAS,UAAU,OAAc,OAAe,KAAkC;AAChF,MAAI,MAAM,SAAS,cAAc,MAAM,SAAS,WAAY,QAAO;AACnE,MAAI,QAAQ,IAAI,aAAc,QAAO;AACrC,MAAI,QAAQ,IAAI,aAAc,QAAO;AACrC,QAAM,QAAQ,MAAM,KAAK,MAAM,GAAG,IAAI,SAAS;AAC/C,MAAI,MAAM,WAAW,EAAG,QAAO;AAC/B,MAAI,UAAU,MAAM,KAAM,QAAO;AACjC,QAAM,YAAY,EAAE,GAAG,OAAO,MAAM,MAAM;AAC1C,SAAO,UAAU;AACjB,SAAO;AACT;AAEA,SAAS,oBAAoB,SAA2B,KAAiC;AACvF,QAAM,UAAmB,CAAC;AAC1B,aAAW,CAAC,OAAO,KAAK,KAAK,QAAQ,QAAQ,QAAQ,GAAG;AACtD,UAAM,OAAO,UAAU,OAAO,OAAO,GAAG;AACxC,QAAI,SAAS,OAAW;AACxB,YAAQ,KAAK,IAAI;AAAA,EACnB;AACA,SAAO,EAAE,GAAG,SAAS,QAAQ;AAC/B;AAEA,eAAsB,oBACpB,SACA,gBACA,KACA,SACA,SACsB;AACtB,QAAM,cAAc,oBAAoB,SAAS,GAAG;AACpD,QAAM,SAAS,UAAU,IAAI,OAAO;AACpC,SAAO,QAAQ;AAAA,IACb,CAAC,OACC,GAAG,YAAY,gBAAgB;AAAA,MAC7B,MAAM;AAAA,MACN,MAAM;AAAA,QACJ,SAAS,IAAI;AAAA,QACb,cAAc,IAAI;AAAA,QAClB,WAAW,IAAI;AAAA,QACf,SAAS,IAAI;AAAA,MACf;AAAA,MACA,OAAO,CAAC,EAAE,QAAQ,QAAQ,WAAW,UAAU,CAAC,WAAW,EAAE,CAAC;AAAA,IAChE,CAAC;AAAA,IACH;AAAA,EACF;AACF;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { registerVoiceModel } from "../models.js";
|
|
2
|
+
import { registerSpeechApiProvider } from "../registry.js";
|
|
3
|
+
import { speakGoogle } from "./speak.js";
|
|
4
|
+
const tokenCost = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 };
|
|
5
|
+
const google = "https://generativelanguage.googleapis.com/v1beta";
|
|
6
|
+
const format = (encoding, sampleRate) => ({
|
|
7
|
+
encoding,
|
|
8
|
+
sampleRate,
|
|
9
|
+
channels: 1
|
|
10
|
+
});
|
|
11
|
+
const outputFormats = [
|
|
12
|
+
format("wav", 24e3),
|
|
13
|
+
format("pcm16", 24e3),
|
|
14
|
+
format("mulaw", 8e3),
|
|
15
|
+
format("alaw", 8e3)
|
|
16
|
+
];
|
|
17
|
+
const voices = [
|
|
18
|
+
{ id: "Kore", name: "Kore" },
|
|
19
|
+
{ id: "Puck", name: "Puck" },
|
|
20
|
+
{ id: "Charon", name: "Charon" },
|
|
21
|
+
{ id: "Fenrir", name: "Fenrir" },
|
|
22
|
+
{ id: "Aoede", name: "Aoede" }
|
|
23
|
+
];
|
|
24
|
+
function speechModel(id, perOutputMinute) {
|
|
25
|
+
return {
|
|
26
|
+
id,
|
|
27
|
+
name: id,
|
|
28
|
+
api: "google-tts",
|
|
29
|
+
provider: "google",
|
|
30
|
+
baseUrl: google,
|
|
31
|
+
type: "speech",
|
|
32
|
+
textStreaming: false,
|
|
33
|
+
outputFormats,
|
|
34
|
+
voices,
|
|
35
|
+
cloning: true,
|
|
36
|
+
cost: tokenCost,
|
|
37
|
+
audioCost: { perOutputMinute }
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
function registerGoogle() {
|
|
41
|
+
const models = [
|
|
42
|
+
speechModel("gemini-3.8-flash-tts", 25 * 9 * 60 / 1e6),
|
|
43
|
+
speechModel("gemini-3.8-flash-lite-tts", 25 * 6 * 60 / 1e6)
|
|
44
|
+
];
|
|
45
|
+
const unregisterModels = models.map((model) => registerVoiceModel(model));
|
|
46
|
+
const unregisterSpeech = registerSpeechApiProvider({
|
|
47
|
+
api: "google-tts",
|
|
48
|
+
speak: speakGoogle
|
|
49
|
+
});
|
|
50
|
+
let released = false;
|
|
51
|
+
return () => {
|
|
52
|
+
if (released) return;
|
|
53
|
+
released = true;
|
|
54
|
+
for (const unregister of unregisterModels) unregister();
|
|
55
|
+
unregisterSpeech();
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
export {
|
|
59
|
+
registerGoogle
|
|
60
|
+
};
|
|
61
|
+
//# sourceMappingURL=catalog.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../src/google/catalog.ts"],
|
|
4
|
+
"sourcesContent": ["import type { AudioFormat } from '../audio.ts';\nimport { registerVoiceModel, type SpeechModel, type VoiceInfo } from '../models.ts';\nimport { registerSpeechApiProvider } from '../registry.ts';\nimport { speakGoogle } from './speak.ts';\n\nconst tokenCost = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 };\nconst google = 'https://generativelanguage.googleapis.com/v1beta';\n\nconst format = (encoding: 'wav' | 'pcm16' | 'mulaw' | 'alaw', sampleRate: number): AudioFormat => ({\n encoding,\n sampleRate,\n channels: 1,\n});\n\nconst outputFormats = [\n format('wav', 24000),\n format('pcm16', 24000),\n format('mulaw', 8000),\n format('alaw', 8000),\n];\n\nconst voices: VoiceInfo[] = [\n { id: 'Kore', name: 'Kore' },\n { id: 'Puck', name: 'Puck' },\n { id: 'Charon', name: 'Charon' },\n { id: 'Fenrir', name: 'Fenrir' },\n { id: 'Aoede', name: 'Aoede' },\n];\n\nfunction speechModel(id: string, perOutputMinute: number): SpeechModel<string> {\n return {\n id,\n name: id,\n api: 'google-tts',\n provider: 'google',\n baseUrl: google,\n type: 'speech',\n textStreaming: false,\n outputFormats,\n voices,\n cloning: true,\n cost: tokenCost,\n audioCost: { perOutputMinute },\n };\n}\n\nexport function registerGoogle(): () => void {\n const models = [\n speechModel('gemini-3.8-flash-tts', (25 * 9 * 60) / 1_000_000),\n speechModel('gemini-3.8-flash-lite-tts', (25 * 6 * 60) / 1_000_000),\n ];\n const unregisterModels = models.map((model) => registerVoiceModel(model));\n const unregisterSpeech = registerSpeechApiProvider({\n api: 'google-tts',\n speak: speakGoogle,\n });\n let released = false;\n return () => {\n if (released) return;\n released = true;\n for (const unregister of unregisterModels) unregister();\n unregisterSpeech();\n };\n}\n"],
|
|
5
|
+
"mappings": "AACA,SAAS,0BAA4D;AACrE,SAAS,iCAAiC;AAC1C,SAAS,mBAAmB;AAE5B,MAAM,YAAY,EAAE,OAAO,GAAG,QAAQ,GAAG,WAAW,GAAG,YAAY,EAAE;AACrE,MAAM,SAAS;AAEf,MAAM,SAAS,CAAC,UAA8C,gBAAqC;AAAA,EACjG;AAAA,EACA;AAAA,EACA,UAAU;AACZ;AAEA,MAAM,gBAAgB;AAAA,EACpB,OAAO,OAAO,IAAK;AAAA,EACnB,OAAO,SAAS,IAAK;AAAA,EACrB,OAAO,SAAS,GAAI;AAAA,EACpB,OAAO,QAAQ,GAAI;AACrB;AAEA,MAAM,SAAsB;AAAA,EAC1B,EAAE,IAAI,QAAQ,MAAM,OAAO;AAAA,EAC3B,EAAE,IAAI,QAAQ,MAAM,OAAO;AAAA,EAC3B,EAAE,IAAI,UAAU,MAAM,SAAS;AAAA,EAC/B,EAAE,IAAI,UAAU,MAAM,SAAS;AAAA,EAC/B,EAAE,IAAI,SAAS,MAAM,QAAQ;AAC/B;AAEA,SAAS,YAAY,IAAY,iBAA8C;AAC7E,SAAO;AAAA,IACL;AAAA,IACA,MAAM;AAAA,IACN,KAAK;AAAA,IACL,UAAU;AAAA,IACV,SAAS;AAAA,IACT,MAAM;AAAA,IACN,eAAe;AAAA,IACf;AAAA,IACA;AAAA,IACA,SAAS;AAAA,IACT,MAAM;AAAA,IACN,WAAW,EAAE,gBAAgB;AAAA,EAC/B;AACF;AAEO,SAAS,iBAA6B;AAC3C,QAAM,SAAS;AAAA,IACb,YAAY,wBAAyB,KAAK,IAAI,KAAM,GAAS;AAAA,IAC7D,YAAY,6BAA8B,KAAK,IAAI,KAAM,GAAS;AAAA,EACpE;AACA,QAAM,mBAAmB,OAAO,IAAI,CAAC,UAAU,mBAAmB,KAAK,CAAC;AACxE,QAAM,mBAAmB,0BAA0B;AAAA,IACjD,KAAK;AAAA,IACL,OAAO;AAAA,EACT,CAAC;AACD,MAAI,WAAW;AACf,SAAO,MAAM;AACX,QAAI,SAAU;AACd,eAAW;AACX,eAAW,cAAc,iBAAkB,YAAW;AACtD,qBAAiB;AAAA,EACnB;AACF;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
import { chunkSeconds } from "../audio.js";
|
|
2
|
+
import { decodeAudioBase64 } from "../convert.js";
|
|
3
|
+
import {
|
|
4
|
+
applyPayload,
|
|
5
|
+
asVoiceError,
|
|
6
|
+
endpoint,
|
|
7
|
+
isRecord,
|
|
8
|
+
readJson,
|
|
9
|
+
requestSignal,
|
|
10
|
+
stringField
|
|
11
|
+
} from "../openai/wire.js";
|
|
12
|
+
import { SpeechEventStream } from "../speech.js";
|
|
13
|
+
import { priceAudio } from "../usage.js";
|
|
14
|
+
import { googleHeaders, sameFormat, speechMime } from "./wire.js";
|
|
15
|
+
function resolveFormat(model, requested) {
|
|
16
|
+
const fallback = model.outputFormats[0];
|
|
17
|
+
if (!fallback) throw new Error("No output format");
|
|
18
|
+
if (!requested) return fallback;
|
|
19
|
+
const match = model.outputFormats.find((format) => sameFormat(format, requested));
|
|
20
|
+
if (!match) throw new Error(`Unsupported speech format ${requested.encoding}`);
|
|
21
|
+
return match;
|
|
22
|
+
}
|
|
23
|
+
function speechBody(text, voice, format) {
|
|
24
|
+
return {
|
|
25
|
+
contents: [{ role: "user", parts: [{ text }] }],
|
|
26
|
+
generationConfig: {
|
|
27
|
+
responseModalities: ["AUDIO"],
|
|
28
|
+
responseFormat: { audio: { mimeType: speechMime(format) } },
|
|
29
|
+
speechConfig: {
|
|
30
|
+
voiceConfig: { prebuiltVoiceConfig: { voiceName: voice } }
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
async function readText(text, signal) {
|
|
36
|
+
if (typeof text === "string") return text;
|
|
37
|
+
let combined = "";
|
|
38
|
+
for await (const part of text) {
|
|
39
|
+
if (signal?.aborted) throw new DOMException("aborted", "AbortError");
|
|
40
|
+
combined += part;
|
|
41
|
+
}
|
|
42
|
+
return combined;
|
|
43
|
+
}
|
|
44
|
+
function errorText(text, status) {
|
|
45
|
+
if (text === "") return `HTTP ${status}`;
|
|
46
|
+
let parsed;
|
|
47
|
+
try {
|
|
48
|
+
parsed = JSON.parse(text);
|
|
49
|
+
} catch {
|
|
50
|
+
return text;
|
|
51
|
+
}
|
|
52
|
+
if (!isRecord(parsed) || Array.isArray(parsed)) return text;
|
|
53
|
+
const direct = stringField(parsed, "message");
|
|
54
|
+
if (direct !== void 0) return direct;
|
|
55
|
+
const error = parsed.error;
|
|
56
|
+
if (typeof error === "string") return error;
|
|
57
|
+
if (isRecord(error) && !Array.isArray(error)) {
|
|
58
|
+
const nested = stringField(error, "message");
|
|
59
|
+
if (nested !== void 0) return nested;
|
|
60
|
+
}
|
|
61
|
+
return text;
|
|
62
|
+
}
|
|
63
|
+
function inlineData(part) {
|
|
64
|
+
const camel = part.inlineData;
|
|
65
|
+
const snake = part.inline_data;
|
|
66
|
+
const inline = isRecord(camel) && !Array.isArray(camel) ? camel : snake;
|
|
67
|
+
if (!isRecord(inline) || Array.isArray(inline)) return void 0;
|
|
68
|
+
return stringField(inline, "data");
|
|
69
|
+
}
|
|
70
|
+
function audioParts(body) {
|
|
71
|
+
const candidates = body.candidates;
|
|
72
|
+
if (!Array.isArray(candidates)) return [];
|
|
73
|
+
const first = candidates[0];
|
|
74
|
+
if (!isRecord(first) || Array.isArray(first)) return [];
|
|
75
|
+
const content = first.content;
|
|
76
|
+
if (!isRecord(content) || Array.isArray(content)) return [];
|
|
77
|
+
const parts = content.parts;
|
|
78
|
+
if (!Array.isArray(parts)) return [];
|
|
79
|
+
const chunks = [];
|
|
80
|
+
for (const part of parts) {
|
|
81
|
+
if (!isRecord(part) || Array.isArray(part)) continue;
|
|
82
|
+
const data = inlineData(part);
|
|
83
|
+
if (data === void 0 || data === "") continue;
|
|
84
|
+
chunks.push(data);
|
|
85
|
+
}
|
|
86
|
+
return chunks;
|
|
87
|
+
}
|
|
88
|
+
function concat(chunks) {
|
|
89
|
+
const length = chunks.reduce((sum, chunk) => sum + chunk.byteLength, 0);
|
|
90
|
+
const out = new Uint8Array(length);
|
|
91
|
+
let offset = 0;
|
|
92
|
+
for (const chunk of chunks) {
|
|
93
|
+
out.set(chunk, offset);
|
|
94
|
+
offset += chunk.byteLength;
|
|
95
|
+
}
|
|
96
|
+
return out;
|
|
97
|
+
}
|
|
98
|
+
async function googleFetch(model, options, url, body) {
|
|
99
|
+
const headers = googleHeaders(model, options);
|
|
100
|
+
headers["Content-Type"] = "application/json";
|
|
101
|
+
const fetchImpl = options?.fetch ?? globalThis.fetch;
|
|
102
|
+
const response = await fetchImpl(url, {
|
|
103
|
+
method: "POST",
|
|
104
|
+
headers,
|
|
105
|
+
body,
|
|
106
|
+
signal: requestSignal(options)
|
|
107
|
+
});
|
|
108
|
+
await options?.onResponse?.({ status: response.status, headers: {} }, model);
|
|
109
|
+
if (!response.ok) {
|
|
110
|
+
const text = await response.text();
|
|
111
|
+
throw new Error(errorText(text, response.status));
|
|
112
|
+
}
|
|
113
|
+
return response;
|
|
114
|
+
}
|
|
115
|
+
function speakGoogle(model, input, options) {
|
|
116
|
+
const stream = new SpeechEventStream();
|
|
117
|
+
void pumpSpeech(model, input, options, stream);
|
|
118
|
+
return stream;
|
|
119
|
+
}
|
|
120
|
+
async function pumpSpeech(model, input, options, stream) {
|
|
121
|
+
try {
|
|
122
|
+
if (options?.signal?.aborted) throw new DOMException("aborted", "AbortError");
|
|
123
|
+
const format = resolveFormat(model, input.format);
|
|
124
|
+
const text = await readText(input.text, options?.signal);
|
|
125
|
+
await postSpeech(model, text, input.voice, format, options, stream);
|
|
126
|
+
} catch (error) {
|
|
127
|
+
const voice = asVoiceError(error);
|
|
128
|
+
stream.push({ type: "error", reason: voice.reason, error: voice.error });
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
async function postSpeech(model, text, voice, format, options, stream) {
|
|
132
|
+
const payload = speechBody(text, voice, format);
|
|
133
|
+
const body = await applyPayload(payload, model, options);
|
|
134
|
+
if (!isRecord(body) || Array.isArray(body)) throw new Error("Speech payload was not JSON");
|
|
135
|
+
const response = await googleFetch(
|
|
136
|
+
model,
|
|
137
|
+
options,
|
|
138
|
+
endpoint(model.baseUrl, `models/${encodeURIComponent(model.id)}:generateContent`),
|
|
139
|
+
JSON.stringify(body)
|
|
140
|
+
);
|
|
141
|
+
const parsed = await readJson(response);
|
|
142
|
+
if (!isRecord(parsed) || Array.isArray(parsed)) throw new Error("Speech response had no audio");
|
|
143
|
+
const encoded = audioParts(parsed);
|
|
144
|
+
if (encoded.length === 0) throw new Error("Speech response had no audio");
|
|
145
|
+
const data = concat(encoded.map((item) => decodeAudioBase64(item)));
|
|
146
|
+
const seconds = chunkSeconds({ data, format, timestamp: 0 });
|
|
147
|
+
const meters = { characters: text.length };
|
|
148
|
+
if (seconds > 0) meters.outputSeconds = seconds;
|
|
149
|
+
stream.push({ type: "start", format });
|
|
150
|
+
stream.push({
|
|
151
|
+
type: "audio",
|
|
152
|
+
chunk: { data, format, timestamp: 0 },
|
|
153
|
+
span: { itemId: model.id, start: 0, end: text.length }
|
|
154
|
+
});
|
|
155
|
+
stream.push({ type: "done", usage: priceAudio(model.audioCost, meters) });
|
|
156
|
+
}
|
|
157
|
+
export {
|
|
158
|
+
speakGoogle
|
|
159
|
+
};
|
|
160
|
+
//# sourceMappingURL=speak.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../src/google/speak.ts"],
|
|
4
|
+
"sourcesContent": ["import type { ProviderRequestOptions } from '@earendil-works/pi-ai';\nimport type { AudioFormat } from '../audio.ts';\nimport { chunkSeconds } from '../audio.ts';\nimport { decodeAudioBase64 } from '../convert.ts';\nimport type { SpeechModel } from '../models.ts';\nimport {\n applyPayload,\n asVoiceError,\n endpoint,\n isRecord,\n readJson,\n requestSignal,\n stringField,\n} from '../openai/wire.ts';\nimport { SpeechEventStream, type SpeechInput } from '../speech.ts';\nimport { type AudioCounters, priceAudio } from '../usage.ts';\nimport { googleHeaders, sameFormat, speechMime } from './wire.ts';\n\nfunction resolveFormat(\n model: SpeechModel<string>,\n requested: AudioFormat | undefined\n): AudioFormat {\n const fallback = model.outputFormats[0];\n if (!fallback) throw new Error('No output format');\n if (!requested) return fallback;\n const match = model.outputFormats.find((format) => sameFormat(format, requested));\n if (!match) throw new Error(`Unsupported speech format ${requested.encoding}`);\n return match;\n}\n\nfunction speechBody(text: string, voice: string, format: AudioFormat): Record<string, unknown> {\n return {\n contents: [{ role: 'user', parts: [{ text }] }],\n generationConfig: {\n responseModalities: ['AUDIO'],\n responseFormat: { audio: { mimeType: speechMime(format) } },\n speechConfig: {\n voiceConfig: { prebuiltVoiceConfig: { voiceName: voice } },\n },\n },\n };\n}\n\nasync function readText(\n text: string | AsyncIterable<string>,\n signal: AbortSignal | undefined\n): Promise<string> {\n if (typeof text === 'string') return text;\n let combined = '';\n for await (const part of text) {\n if (signal?.aborted) throw new DOMException('aborted', 'AbortError');\n combined += part;\n }\n return combined;\n}\n\nfunction errorText(text: string, status: number): string {\n if (text === '') return `HTTP ${status}`;\n let parsed: unknown;\n try {\n parsed = JSON.parse(text);\n } catch {\n return text;\n }\n if (!isRecord(parsed) || Array.isArray(parsed)) return text;\n const direct = stringField(parsed, 'message');\n if (direct !== undefined) return direct;\n const error = parsed.error;\n if (typeof error === 'string') return error;\n if (isRecord(error) && !Array.isArray(error)) {\n const nested = stringField(error, 'message');\n if (nested !== undefined) return nested;\n }\n return text;\n}\n\nfunction inlineData(part: Record<string, unknown>): string | undefined {\n const camel = part.inlineData;\n const snake = part.inline_data;\n const inline = isRecord(camel) && !Array.isArray(camel) ? camel : snake;\n if (!isRecord(inline) || Array.isArray(inline)) return undefined;\n return stringField(inline, 'data');\n}\n\nfunction audioParts(body: Record<string, unknown>): string[] {\n const candidates = body.candidates;\n if (!Array.isArray(candidates)) return [];\n const first = candidates[0];\n if (!isRecord(first) || Array.isArray(first)) return [];\n const content = first.content;\n if (!isRecord(content) || Array.isArray(content)) return [];\n const parts = content.parts;\n if (!Array.isArray(parts)) return [];\n const chunks: string[] = [];\n for (const part of parts) {\n if (!isRecord(part) || Array.isArray(part)) continue;\n const data = inlineData(part);\n if (data === undefined || data === '') continue;\n chunks.push(data);\n }\n return chunks;\n}\n\nfunction concat(chunks: Uint8Array[]): Uint8Array {\n const length = chunks.reduce((sum, chunk) => sum + chunk.byteLength, 0);\n const out = new Uint8Array(length);\n let offset = 0;\n for (const chunk of chunks) {\n out.set(chunk, offset);\n offset += chunk.byteLength;\n }\n return out;\n}\n\nasync function googleFetch(\n model: SpeechModel<string>,\n options: ProviderRequestOptions | undefined,\n url: string,\n body: string\n): Promise<Response> {\n const headers = googleHeaders(model, options);\n headers['Content-Type'] = 'application/json';\n const fetchImpl = options?.fetch ?? globalThis.fetch;\n const response = await fetchImpl(url, {\n method: 'POST',\n headers,\n body,\n signal: requestSignal(options),\n });\n await options?.onResponse?.({ status: response.status, headers: {} }, model as never);\n if (!response.ok) {\n const text = await response.text();\n throw new Error(errorText(text, response.status));\n }\n return response;\n}\n\nexport function speakGoogle(\n model: SpeechModel<string>,\n input: SpeechInput,\n options?: ProviderRequestOptions\n): SpeechEventStream {\n const stream = new SpeechEventStream();\n void pumpSpeech(model, input, options, stream);\n return stream;\n}\n\nasync function pumpSpeech(\n model: SpeechModel<string>,\n input: SpeechInput,\n options: ProviderRequestOptions | undefined,\n stream: SpeechEventStream\n) {\n try {\n if (options?.signal?.aborted) throw new DOMException('aborted', 'AbortError');\n const format = resolveFormat(model, input.format);\n const text = await readText(input.text, options?.signal);\n await postSpeech(model, text, input.voice, format, options, stream);\n } catch (error) {\n const voice = asVoiceError(error);\n stream.push({ type: 'error', reason: voice.reason, error: voice.error });\n }\n}\n\nasync function postSpeech(\n model: SpeechModel<string>,\n text: string,\n voice: string,\n format: AudioFormat,\n options: ProviderRequestOptions | undefined,\n stream: SpeechEventStream\n) {\n const payload = speechBody(text, voice, format);\n const body = await applyPayload(payload, model, options);\n if (!isRecord(body) || Array.isArray(body)) throw new Error('Speech payload was not JSON');\n const response = await googleFetch(\n model,\n options,\n endpoint(model.baseUrl, `models/${encodeURIComponent(model.id)}:generateContent`),\n JSON.stringify(body)\n );\n const parsed = await readJson(response);\n if (!isRecord(parsed) || Array.isArray(parsed)) throw new Error('Speech response had no audio');\n const encoded = audioParts(parsed);\n if (encoded.length === 0) throw new Error('Speech response had no audio');\n const data = concat(encoded.map((item) => decodeAudioBase64(item)));\n const seconds = chunkSeconds({ data, format, timestamp: 0 });\n const meters: AudioCounters = { characters: text.length };\n if (seconds > 0) meters.outputSeconds = seconds;\n stream.push({ type: 'start', format });\n stream.push({\n type: 'audio',\n chunk: { data, format, timestamp: 0 },\n span: { itemId: model.id, start: 0, end: text.length },\n });\n stream.push({ type: 'done', usage: priceAudio(model.audioCost, meters) });\n}\n"],
|
|
5
|
+
"mappings": "AAEA,SAAS,oBAAoB;AAC7B,SAAS,yBAAyB;AAElC;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,yBAA2C;AACpD,SAA6B,kBAAkB;AAC/C,SAAS,eAAe,YAAY,kBAAkB;AAEtD,SAAS,cACP,OACA,WACa;AACb,QAAM,WAAW,MAAM,cAAc,CAAC;AACtC,MAAI,CAAC,SAAU,OAAM,IAAI,MAAM,kBAAkB;AACjD,MAAI,CAAC,UAAW,QAAO;AACvB,QAAM,QAAQ,MAAM,cAAc,KAAK,CAAC,WAAW,WAAW,QAAQ,SAAS,CAAC;AAChF,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,6BAA6B,UAAU,QAAQ,EAAE;AAC7E,SAAO;AACT;AAEA,SAAS,WAAW,MAAc,OAAe,QAA8C;AAC7F,SAAO;AAAA,IACL,UAAU,CAAC,EAAE,MAAM,QAAQ,OAAO,CAAC,EAAE,KAAK,CAAC,EAAE,CAAC;AAAA,IAC9C,kBAAkB;AAAA,MAChB,oBAAoB,CAAC,OAAO;AAAA,MAC5B,gBAAgB,EAAE,OAAO,EAAE,UAAU,WAAW,MAAM,EAAE,EAAE;AAAA,MAC1D,cAAc;AAAA,QACZ,aAAa,EAAE,qBAAqB,EAAE,WAAW,MAAM,EAAE;AAAA,MAC3D;AAAA,IACF;AAAA,EACF;AACF;AAEA,eAAe,SACb,MACA,QACiB;AACjB,MAAI,OAAO,SAAS,SAAU,QAAO;AACrC,MAAI,WAAW;AACf,mBAAiB,QAAQ,MAAM;AAC7B,QAAI,QAAQ,QAAS,OAAM,IAAI,aAAa,WAAW,YAAY;AACnE,gBAAY;AAAA,EACd;AACA,SAAO;AACT;AAEA,SAAS,UAAU,MAAc,QAAwB;AACvD,MAAI,SAAS,GAAI,QAAO,QAAQ,MAAM;AACtC,MAAI;AACJ,MAAI;AACF,aAAS,KAAK,MAAM,IAAI;AAAA,EAC1B,QAAQ;AACN,WAAO;AAAA,EACT;AACA,MAAI,CAAC,SAAS,MAAM,KAAK,MAAM,QAAQ,MAAM,EAAG,QAAO;AACvD,QAAM,SAAS,YAAY,QAAQ,SAAS;AAC5C,MAAI,WAAW,OAAW,QAAO;AACjC,QAAM,QAAQ,OAAO;AACrB,MAAI,OAAO,UAAU,SAAU,QAAO;AACtC,MAAI,SAAS,KAAK,KAAK,CAAC,MAAM,QAAQ,KAAK,GAAG;AAC5C,UAAM,SAAS,YAAY,OAAO,SAAS;AAC3C,QAAI,WAAW,OAAW,QAAO;AAAA,EACnC;AACA,SAAO;AACT;AAEA,SAAS,WAAW,MAAmD;AACrE,QAAM,QAAQ,KAAK;AACnB,QAAM,QAAQ,KAAK;AACnB,QAAM,SAAS,SAAS,KAAK,KAAK,CAAC,MAAM,QAAQ,KAAK,IAAI,QAAQ;AAClE,MAAI,CAAC,SAAS,MAAM,KAAK,MAAM,QAAQ,MAAM,EAAG,QAAO;AACvD,SAAO,YAAY,QAAQ,MAAM;AACnC;AAEA,SAAS,WAAW,MAAyC;AAC3D,QAAM,aAAa,KAAK;AACxB,MAAI,CAAC,MAAM,QAAQ,UAAU,EAAG,QAAO,CAAC;AACxC,QAAM,QAAQ,WAAW,CAAC;AAC1B,MAAI,CAAC,SAAS,KAAK,KAAK,MAAM,QAAQ,KAAK,EAAG,QAAO,CAAC;AACtD,QAAM,UAAU,MAAM;AACtB,MAAI,CAAC,SAAS,OAAO,KAAK,MAAM,QAAQ,OAAO,EAAG,QAAO,CAAC;AAC1D,QAAM,QAAQ,QAAQ;AACtB,MAAI,CAAC,MAAM,QAAQ,KAAK,EAAG,QAAO,CAAC;AACnC,QAAM,SAAmB,CAAC;AAC1B,aAAW,QAAQ,OAAO;AACxB,QAAI,CAAC,SAAS,IAAI,KAAK,MAAM,QAAQ,IAAI,EAAG;AAC5C,UAAM,OAAO,WAAW,IAAI;AAC5B,QAAI,SAAS,UAAa,SAAS,GAAI;AACvC,WAAO,KAAK,IAAI;AAAA,EAClB;AACA,SAAO;AACT;AAEA,SAAS,OAAO,QAAkC;AAChD,QAAM,SAAS,OAAO,OAAO,CAAC,KAAK,UAAU,MAAM,MAAM,YAAY,CAAC;AACtE,QAAM,MAAM,IAAI,WAAW,MAAM;AACjC,MAAI,SAAS;AACb,aAAW,SAAS,QAAQ;AAC1B,QAAI,IAAI,OAAO,MAAM;AACrB,cAAU,MAAM;AAAA,EAClB;AACA,SAAO;AACT;AAEA,eAAe,YACb,OACA,SACA,KACA,MACmB;AACnB,QAAM,UAAU,cAAc,OAAO,OAAO;AAC5C,UAAQ,cAAc,IAAI;AAC1B,QAAM,YAAY,SAAS,SAAS,WAAW;AAC/C,QAAM,WAAW,MAAM,UAAU,KAAK;AAAA,IACpC,QAAQ;AAAA,IACR;AAAA,IACA;AAAA,IACA,QAAQ,cAAc,OAAO;AAAA,EAC/B,CAAC;AACD,QAAM,SAAS,aAAa,EAAE,QAAQ,SAAS,QAAQ,SAAS,CAAC,EAAE,GAAG,KAAc;AACpF,MAAI,CAAC,SAAS,IAAI;AAChB,UAAM,OAAO,MAAM,SAAS,KAAK;AACjC,UAAM,IAAI,MAAM,UAAU,MAAM,SAAS,MAAM,CAAC;AAAA,EAClD;AACA,SAAO;AACT;AAEO,SAAS,YACd,OACA,OACA,SACmB;AACnB,QAAM,SAAS,IAAI,kBAAkB;AACrC,OAAK,WAAW,OAAO,OAAO,SAAS,MAAM;AAC7C,SAAO;AACT;AAEA,eAAe,WACb,OACA,OACA,SACA,QACA;AACA,MAAI;AACF,QAAI,SAAS,QAAQ,QAAS,OAAM,IAAI,aAAa,WAAW,YAAY;AAC5E,UAAM,SAAS,cAAc,OAAO,MAAM,MAAM;AAChD,UAAM,OAAO,MAAM,SAAS,MAAM,MAAM,SAAS,MAAM;AACvD,UAAM,WAAW,OAAO,MAAM,MAAM,OAAO,QAAQ,SAAS,MAAM;AAAA,EACpE,SAAS,OAAO;AACd,UAAM,QAAQ,aAAa,KAAK;AAChC,WAAO,KAAK,EAAE,MAAM,SAAS,QAAQ,MAAM,QAAQ,OAAO,MAAM,MAAM,CAAC;AAAA,EACzE;AACF;AAEA,eAAe,WACb,OACA,MACA,OACA,QACA,SACA,QACA;AACA,QAAM,UAAU,WAAW,MAAM,OAAO,MAAM;AAC9C,QAAM,OAAO,MAAM,aAAa,SAAS,OAAO,OAAO;AACvD,MAAI,CAAC,SAAS,IAAI,KAAK,MAAM,QAAQ,IAAI,EAAG,OAAM,IAAI,MAAM,6BAA6B;AACzF,QAAM,WAAW,MAAM;AAAA,IACrB;AAAA,IACA;AAAA,IACA,SAAS,MAAM,SAAS,UAAU,mBAAmB,MAAM,EAAE,CAAC,kBAAkB;AAAA,IAChF,KAAK,UAAU,IAAI;AAAA,EACrB;AACA,QAAM,SAAS,MAAM,SAAS,QAAQ;AACtC,MAAI,CAAC,SAAS,MAAM,KAAK,MAAM,QAAQ,MAAM,EAAG,OAAM,IAAI,MAAM,8BAA8B;AAC9F,QAAM,UAAU,WAAW,MAAM;AACjC,MAAI,QAAQ,WAAW,EAAG,OAAM,IAAI,MAAM,8BAA8B;AACxE,QAAM,OAAO,OAAO,QAAQ,IAAI,CAAC,SAAS,kBAAkB,IAAI,CAAC,CAAC;AAClE,QAAM,UAAU,aAAa,EAAE,MAAM,QAAQ,WAAW,EAAE,CAAC;AAC3D,QAAM,SAAwB,EAAE,YAAY,KAAK,OAAO;AACxD,MAAI,UAAU,EAAG,QAAO,gBAAgB;AACxC,SAAO,KAAK,EAAE,MAAM,SAAS,OAAO,CAAC;AACrC,SAAO,KAAK;AAAA,IACV,MAAM;AAAA,IACN,OAAO,EAAE,MAAM,QAAQ,WAAW,EAAE;AAAA,IACpC,MAAM,EAAE,QAAQ,MAAM,IAAI,OAAO,GAAG,KAAK,KAAK,OAAO;AAAA,EACvD,CAAC;AACD,SAAO,KAAK,EAAE,MAAM,QAAQ,OAAO,WAAW,MAAM,WAAW,MAAM,EAAE,CAAC;AAC1E;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
const mimes = {
|
|
2
|
+
wav: "AUDIO_WAV",
|
|
3
|
+
pcm16: "AUDIO_L16",
|
|
4
|
+
mulaw: "AUDIO_MULAW",
|
|
5
|
+
alaw: "AUDIO_ALAW"
|
|
6
|
+
};
|
|
7
|
+
function speechMime(format) {
|
|
8
|
+
const name = mimes[format.encoding];
|
|
9
|
+
if (!name) throw new Error(`Unsupported speech format ${format.encoding}`);
|
|
10
|
+
return name;
|
|
11
|
+
}
|
|
12
|
+
function sameFormat(left, right) {
|
|
13
|
+
return left.encoding === right.encoding && left.sampleRate === right.sampleRate && left.channels === right.channels;
|
|
14
|
+
}
|
|
15
|
+
function headerValue(headers, name) {
|
|
16
|
+
const expected = name.toLowerCase();
|
|
17
|
+
for (const [key, value] of Object.entries(headers)) {
|
|
18
|
+
if (key.toLowerCase() === expected && value.trim() !== "") return value;
|
|
19
|
+
}
|
|
20
|
+
return void 0;
|
|
21
|
+
}
|
|
22
|
+
function googleHeaders(model, options) {
|
|
23
|
+
const headers = { ...model.headers };
|
|
24
|
+
const extra = options?.headers;
|
|
25
|
+
if (extra) {
|
|
26
|
+
for (const [key, value] of Object.entries(extra)) {
|
|
27
|
+
if (value === null) delete headers[key];
|
|
28
|
+
else headers[key] = value;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
if (headerValue(headers, "x-goog-api-key") === void 0) {
|
|
32
|
+
const apiKey = options?.apiKey;
|
|
33
|
+
if (!apiKey) throw new Error(`No API key for provider: ${model.provider}`);
|
|
34
|
+
headers["x-goog-api-key"] = apiKey;
|
|
35
|
+
}
|
|
36
|
+
return headers;
|
|
37
|
+
}
|
|
38
|
+
export {
|
|
39
|
+
googleHeaders,
|
|
40
|
+
sameFormat,
|
|
41
|
+
speechMime
|
|
42
|
+
};
|
|
43
|
+
//# sourceMappingURL=wire.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../src/google/wire.ts"],
|
|
4
|
+
"sourcesContent": ["import type { ProviderRequestOptions } from '@earendil-works/pi-ai';\nimport type { AudioFormat } from '../audio.ts';\n\nconst mimes: Record<string, string> = {\n wav: 'AUDIO_WAV',\n pcm16: 'AUDIO_L16',\n mulaw: 'AUDIO_MULAW',\n alaw: 'AUDIO_ALAW',\n};\n\nexport function speechMime(format: AudioFormat): string {\n const name = mimes[format.encoding];\n if (!name) throw new Error(`Unsupported speech format ${format.encoding}`);\n return name;\n}\n\nexport function sameFormat(left: AudioFormat, right: AudioFormat): boolean {\n return (\n left.encoding === right.encoding &&\n left.sampleRate === right.sampleRate &&\n left.channels === right.channels\n );\n}\n\nfunction headerValue(headers: Record<string, string>, name: string): string | undefined {\n const expected = name.toLowerCase();\n for (const [key, value] of Object.entries(headers)) {\n if (key.toLowerCase() === expected && value.trim() !== '') return value;\n }\n return undefined;\n}\n\nexport function googleHeaders(\n model: { provider: string; headers?: Record<string, string> },\n options: ProviderRequestOptions | undefined\n): Record<string, string> {\n const headers: Record<string, string> = { ...model.headers };\n const extra = options?.headers;\n if (extra) {\n for (const [key, value] of Object.entries(extra)) {\n if (value === null) delete headers[key];\n else headers[key] = value;\n }\n }\n if (headerValue(headers, 'x-goog-api-key') === undefined) {\n const apiKey = options?.apiKey;\n if (!apiKey) throw new Error(`No API key for provider: ${model.provider}`);\n headers['x-goog-api-key'] = apiKey;\n }\n return headers;\n}\n"],
|
|
5
|
+
"mappings": "AAGA,MAAM,QAAgC;AAAA,EACpC,KAAK;AAAA,EACL,OAAO;AAAA,EACP,OAAO;AAAA,EACP,MAAM;AACR;AAEO,SAAS,WAAW,QAA6B;AACtD,QAAM,OAAO,MAAM,OAAO,QAAQ;AAClC,MAAI,CAAC,KAAM,OAAM,IAAI,MAAM,6BAA6B,OAAO,QAAQ,EAAE;AACzE,SAAO;AACT;AAEO,SAAS,WAAW,MAAmB,OAA6B;AACzE,SACE,KAAK,aAAa,MAAM,YACxB,KAAK,eAAe,MAAM,cAC1B,KAAK,aAAa,MAAM;AAE5B;AAEA,SAAS,YAAY,SAAiC,MAAkC;AACtF,QAAM,WAAW,KAAK,YAAY;AAClC,aAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,OAAO,GAAG;AAClD,QAAI,IAAI,YAAY,MAAM,YAAY,MAAM,KAAK,MAAM,GAAI,QAAO;AAAA,EACpE;AACA,SAAO;AACT;AAEO,SAAS,cACd,OACA,SACwB;AACxB,QAAM,UAAkC,EAAE,GAAG,MAAM,QAAQ;AAC3D,QAAM,QAAQ,SAAS;AACvB,MAAI,OAAO;AACT,eAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,KAAK,GAAG;AAChD,UAAI,UAAU,KAAM,QAAO,QAAQ,GAAG;AAAA,UACjC,SAAQ,GAAG,IAAI;AAAA,IACtB;AAAA,EACF;AACA,MAAI,YAAY,SAAS,gBAAgB,MAAM,QAAW;AACxD,UAAM,SAAS,SAAS;AACxB,QAAI,CAAC,OAAQ,OAAM,IAAI,MAAM,4BAA4B,MAAM,QAAQ,EAAE;AACzE,YAAQ,gBAAgB,IAAI;AAAA,EAC9B;AACA,SAAO;AACT;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
package/dist/soundscape.js
CHANGED
|
@@ -14,6 +14,9 @@ function followBed(bed, soundscape, wait) {
|
|
|
14
14
|
const stop = () => {
|
|
15
15
|
generation += 1;
|
|
16
16
|
clearTimer();
|
|
17
|
+
if (playing === void 0) return;
|
|
18
|
+
playing = void 0;
|
|
19
|
+
bed.stop(fadeOf(soundscape));
|
|
17
20
|
};
|
|
18
21
|
const observe = (phase) => {
|
|
19
22
|
const cues = listCues(soundscape.cue(phase, previous));
|
|
@@ -69,10 +72,19 @@ function firstLoop(cues) {
|
|
|
69
72
|
}
|
|
70
73
|
return void 0;
|
|
71
74
|
}
|
|
75
|
+
const sourceIds = /* @__PURE__ */ new WeakMap();
|
|
76
|
+
let sourceSerial = 0;
|
|
77
|
+
function sourceIdentity(value) {
|
|
78
|
+
const existing = sourceIds.get(value);
|
|
79
|
+
if (existing !== void 0) return existing;
|
|
80
|
+
sourceSerial += 1;
|
|
81
|
+
sourceIds.set(value, sourceSerial);
|
|
82
|
+
return sourceSerial;
|
|
83
|
+
}
|
|
72
84
|
function sourceKey(source) {
|
|
73
85
|
if ("url" in source) return `url:${source.url}`;
|
|
74
|
-
if ("buffer" in source) return `buffer:${source.buffer
|
|
75
|
-
return
|
|
86
|
+
if ("buffer" in source) return `buffer:${sourceIdentity(source.buffer)}`;
|
|
87
|
+
return `synth:${sourceIdentity(source.synth)}`;
|
|
76
88
|
}
|
|
77
89
|
function fadeOf(soundscape) {
|
|
78
90
|
const fade = soundscape.fadeMs;
|
package/dist/soundscape.js.map
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../src/soundscape.ts"],
|
|
4
|
-
"sourcesContent": ["import type { AudioSink, AudioSource } from './audio.ts';\nimport type { VoicePhase } from './turns.ts';\n\nexport type CueSource =\n | { url: string }\n | { buffer: AudioBuffer }\n | { synth: (ctx: BaseAudioContext) => AudioNode };\n\nexport type Cue =\n | { kind: 'loop'; source: CueSource; gain: number }\n | { kind: 'oneShot'; source: CueSource; gain: number }\n | { kind: 'keep' }\n | { kind: 'silence' };\n\nexport interface Soundscape {\n cue(next: VoicePhase, previous: VoicePhase): Cue | readonly Cue[];\n graceMs?: number;\n fadeMs?: number;\n duckUnderSpeech?: number;\n duckUnderUser?: number;\n}\n\nexport interface BedChannel {\n play(cue: Cue, fadeMs: number): void;\n duck(gain: number, rampMs: number): void;\n stop(fadeMs: number): void;\n}\n\nexport interface AudioOut {\n readonly context: BaseAudioContext;\n speech: AudioSink;\n bed: BedChannel;\n readonly reference: AudioSource;\n readonly playback: 'webaudio' | 'rtc-loopback';\n}\n\nexport interface BedFollower {\n observe(phase: VoicePhase): void;\n stop(): void;\n}\n\nexport function isAudioOut(out: AudioSink | AudioOut): out is AudioOut {\n return 'speech' in out && 'bed' in out;\n}\n\nexport function followBed(\n bed: BedChannel,\n soundscape: Soundscape,\n wait?: (run: () => void, delayMs: number) => ReturnType<typeof setTimeout>\n): BedFollower {\n const schedule =\n wait === undefined ? (run: () => void, delayMs: number) => setTimeout(run, delayMs) : wait;\n let previous: VoicePhase = { phase: 'idle' };\n let timer: ReturnType<typeof setTimeout> | undefined;\n let generation = 0;\n let playing: string | undefined;\n\n const clearTimer = () => {\n if (timer !== undefined) clearTimeout(timer);\n timer = undefined;\n };\n\n const stop = () => {\n generation += 1;\n clearTimer();\n };\n\n const observe = (phase: VoicePhase) => {\n const cues = listCues(soundscape.cue(phase, previous));\n previous = phase;\n const fade = fadeOf(soundscape);\n bed.duck(duckFor(phase, soundscape), fade);\n for (const cue of cues) {\n if (cue.kind === 'oneShot') bed.play(cue, fade);\n }\n if (cues.some((cue) => cue.kind === 'silence')) {\n generation += 1;\n clearTimer();\n playing = undefined;\n bed.stop(fade);\n return;\n }\n const loop = firstLoop(cues);\n if (!loop) {\n generation += 1;\n clearTimer();\n return;\n }\n const key = sourceKey(loop.source);\n if (playing === key) {\n generation += 1;\n clearTimer();\n return;\n }\n generation += 1;\n const token = generation;\n clearTimer();\n const start = () => {\n if (token !== generation) return;\n playing = key;\n bed.play(loop, fade);\n };\n const grace = graceOf(soundscape);\n if (grace <= 0) {\n start();\n return;\n }\n timer = schedule(start, grace);\n };\n\n return { observe, stop };\n}\n\nfunction listCues(cue: Cue | readonly Cue[]): readonly Cue[] {\n if ('kind' in cue) return [cue];\n return cue;\n}\n\nfunction firstLoop(cues: readonly Cue[]): Extract<Cue, { kind: 'loop' }> | undefined {\n for (const cue of cues) {\n if (cue.kind === 'loop') return cue;\n }\n return undefined;\n}\n\nfunction sourceKey(source: CueSource): string {\n if ('url' in source) return `url:${source.url}`;\n if ('buffer' in source) return `buffer:${source.buffer
|
|
5
|
-
"mappings": "AAyCO,SAAS,WAAW,KAA4C;AACrE,SAAO,YAAY,OAAO,SAAS;AACrC;AAEO,SAAS,UACd,KACA,YACA,MACa;AACb,QAAM,WACJ,SAAS,SAAY,CAAC,KAAiB,YAAoB,WAAW,KAAK,OAAO,IAAI;AACxF,MAAI,WAAuB,EAAE,OAAO,OAAO;AAC3C,MAAI;AACJ,MAAI,aAAa;AACjB,MAAI;AAEJ,QAAM,aAAa,MAAM;AACvB,QAAI,UAAU,OAAW,cAAa,KAAK;AAC3C,YAAQ;AAAA,EACV;AAEA,QAAM,OAAO,MAAM;AACjB,kBAAc;AACd,eAAW;AAAA,
|
|
4
|
+
"sourcesContent": ["import type { AudioSink, AudioSource } from './audio.ts';\nimport type { VoicePhase } from './turns.ts';\n\nexport type CueSource =\n | { url: string }\n | { buffer: AudioBuffer }\n | { synth: (ctx: BaseAudioContext) => AudioNode };\n\nexport type Cue =\n | { kind: 'loop'; source: CueSource; gain: number }\n | { kind: 'oneShot'; source: CueSource; gain: number }\n | { kind: 'keep' }\n | { kind: 'silence' };\n\nexport interface Soundscape {\n cue(next: VoicePhase, previous: VoicePhase): Cue | readonly Cue[];\n graceMs?: number;\n fadeMs?: number;\n duckUnderSpeech?: number;\n duckUnderUser?: number;\n}\n\nexport interface BedChannel {\n play(cue: Cue, fadeMs: number): void;\n duck(gain: number, rampMs: number): void;\n stop(fadeMs: number): void;\n}\n\nexport interface AudioOut {\n readonly context: BaseAudioContext;\n speech: AudioSink;\n bed: BedChannel;\n readonly reference: AudioSource;\n readonly playback: 'webaudio' | 'rtc-loopback';\n}\n\nexport interface BedFollower {\n observe(phase: VoicePhase): void;\n stop(): void;\n}\n\nexport function isAudioOut(out: AudioSink | AudioOut): out is AudioOut {\n return 'speech' in out && 'bed' in out;\n}\n\nexport function followBed(\n bed: BedChannel,\n soundscape: Soundscape,\n wait?: (run: () => void, delayMs: number) => ReturnType<typeof setTimeout>\n): BedFollower {\n const schedule =\n wait === undefined ? (run: () => void, delayMs: number) => setTimeout(run, delayMs) : wait;\n let previous: VoicePhase = { phase: 'idle' };\n let timer: ReturnType<typeof setTimeout> | undefined;\n let generation = 0;\n let playing: string | undefined;\n\n const clearTimer = () => {\n if (timer !== undefined) clearTimeout(timer);\n timer = undefined;\n };\n\n const stop = () => {\n generation += 1;\n clearTimer();\n if (playing === undefined) return;\n playing = undefined;\n bed.stop(fadeOf(soundscape));\n };\n\n const observe = (phase: VoicePhase) => {\n const cues = listCues(soundscape.cue(phase, previous));\n previous = phase;\n const fade = fadeOf(soundscape);\n bed.duck(duckFor(phase, soundscape), fade);\n for (const cue of cues) {\n if (cue.kind === 'oneShot') bed.play(cue, fade);\n }\n if (cues.some((cue) => cue.kind === 'silence')) {\n generation += 1;\n clearTimer();\n playing = undefined;\n bed.stop(fade);\n return;\n }\n const loop = firstLoop(cues);\n if (!loop) {\n generation += 1;\n clearTimer();\n return;\n }\n const key = sourceKey(loop.source);\n if (playing === key) {\n generation += 1;\n clearTimer();\n return;\n }\n generation += 1;\n const token = generation;\n clearTimer();\n const start = () => {\n if (token !== generation) return;\n playing = key;\n bed.play(loop, fade);\n };\n const grace = graceOf(soundscape);\n if (grace <= 0) {\n start();\n return;\n }\n timer = schedule(start, grace);\n };\n\n return { observe, stop };\n}\n\nfunction listCues(cue: Cue | readonly Cue[]): readonly Cue[] {\n if ('kind' in cue) return [cue];\n return cue;\n}\n\nfunction firstLoop(cues: readonly Cue[]): Extract<Cue, { kind: 'loop' }> | undefined {\n for (const cue of cues) {\n if (cue.kind === 'loop') return cue;\n }\n return undefined;\n}\n\nconst sourceIds = new WeakMap<object, number>();\nlet sourceSerial = 0;\n\nfunction sourceIdentity(value: object): number {\n const existing = sourceIds.get(value);\n if (existing !== undefined) return existing;\n sourceSerial += 1;\n sourceIds.set(value, sourceSerial);\n return sourceSerial;\n}\n\nfunction sourceKey(source: CueSource): string {\n if ('url' in source) return `url:${source.url}`;\n if ('buffer' in source) return `buffer:${sourceIdentity(source.buffer)}`;\n return `synth:${sourceIdentity(source.synth)}`;\n}\n\nfunction fadeOf(soundscape: Soundscape): number {\n const fade = soundscape.fadeMs;\n if (fade === undefined) return 0;\n return fade;\n}\n\nfunction graceOf(soundscape: Soundscape): number {\n const grace = soundscape.graceMs;\n if (grace === undefined) return 0;\n return grace;\n}\n\nfunction duckFor(phase: VoicePhase, soundscape: Soundscape): number {\n if (phase.phase === 'speaking') {\n const gain = soundscape.duckUnderSpeech;\n if (gain === undefined) return 1;\n return gain;\n }\n if (phase.phase === 'listening') {\n const gain = soundscape.duckUnderUser;\n if (gain === undefined) return 1;\n return gain;\n }\n return 1;\n}\n"],
|
|
5
|
+
"mappings": "AAyCO,SAAS,WAAW,KAA4C;AACrE,SAAO,YAAY,OAAO,SAAS;AACrC;AAEO,SAAS,UACd,KACA,YACA,MACa;AACb,QAAM,WACJ,SAAS,SAAY,CAAC,KAAiB,YAAoB,WAAW,KAAK,OAAO,IAAI;AACxF,MAAI,WAAuB,EAAE,OAAO,OAAO;AAC3C,MAAI;AACJ,MAAI,aAAa;AACjB,MAAI;AAEJ,QAAM,aAAa,MAAM;AACvB,QAAI,UAAU,OAAW,cAAa,KAAK;AAC3C,YAAQ;AAAA,EACV;AAEA,QAAM,OAAO,MAAM;AACjB,kBAAc;AACd,eAAW;AACX,QAAI,YAAY,OAAW;AAC3B,cAAU;AACV,QAAI,KAAK,OAAO,UAAU,CAAC;AAAA,EAC7B;AAEA,QAAM,UAAU,CAAC,UAAsB;AACrC,UAAM,OAAO,SAAS,WAAW,IAAI,OAAO,QAAQ,CAAC;AACrD,eAAW;AACX,UAAM,OAAO,OAAO,UAAU;AAC9B,QAAI,KAAK,QAAQ,OAAO,UAAU,GAAG,IAAI;AACzC,eAAW,OAAO,MAAM;AACtB,UAAI,IAAI,SAAS,UAAW,KAAI,KAAK,KAAK,IAAI;AAAA,IAChD;AACA,QAAI,KAAK,KAAK,CAAC,QAAQ,IAAI,SAAS,SAAS,GAAG;AAC9C,oBAAc;AACd,iBAAW;AACX,gBAAU;AACV,UAAI,KAAK,IAAI;AACb;AAAA,IACF;AACA,UAAM,OAAO,UAAU,IAAI;AAC3B,QAAI,CAAC,MAAM;AACT,oBAAc;AACd,iBAAW;AACX;AAAA,IACF;AACA,UAAM,MAAM,UAAU,KAAK,MAAM;AACjC,QAAI,YAAY,KAAK;AACnB,oBAAc;AACd,iBAAW;AACX;AAAA,IACF;AACA,kBAAc;AACd,UAAM,QAAQ;AACd,eAAW;AACX,UAAM,QAAQ,MAAM;AAClB,UAAI,UAAU,WAAY;AAC1B,gBAAU;AACV,UAAI,KAAK,MAAM,IAAI;AAAA,IACrB;AACA,UAAM,QAAQ,QAAQ,UAAU;AAChC,QAAI,SAAS,GAAG;AACd,YAAM;AACN;AAAA,IACF;AACA,YAAQ,SAAS,OAAO,KAAK;AAAA,EAC/B;AAEA,SAAO,EAAE,SAAS,KAAK;AACzB;AAEA,SAAS,SAAS,KAA2C;AAC3D,MAAI,UAAU,IAAK,QAAO,CAAC,GAAG;AAC9B,SAAO;AACT;AAEA,SAAS,UAAU,MAAkE;AACnF,aAAW,OAAO,MAAM;AACtB,QAAI,IAAI,SAAS,OAAQ,QAAO;AAAA,EAClC;AACA,SAAO;AACT;AAEA,MAAM,YAAY,oBAAI,QAAwB;AAC9C,IAAI,eAAe;AAEnB,SAAS,eAAe,OAAuB;AAC7C,QAAM,WAAW,UAAU,IAAI,KAAK;AACpC,MAAI,aAAa,OAAW,QAAO;AACnC,kBAAgB;AAChB,YAAU,IAAI,OAAO,YAAY;AACjC,SAAO;AACT;AAEA,SAAS,UAAU,QAA2B;AAC5C,MAAI,SAAS,OAAQ,QAAO,OAAO,OAAO,GAAG;AAC7C,MAAI,YAAY,OAAQ,QAAO,UAAU,eAAe,OAAO,MAAM,CAAC;AACtE,SAAO,SAAS,eAAe,OAAO,KAAK,CAAC;AAC9C;AAEA,SAAS,OAAO,YAAgC;AAC9C,QAAM,OAAO,WAAW;AACxB,MAAI,SAAS,OAAW,QAAO;AAC/B,SAAO;AACT;AAEA,SAAS,QAAQ,YAAgC;AAC/C,QAAM,QAAQ,WAAW;AACzB,MAAI,UAAU,OAAW,QAAO;AAChC,SAAO;AACT;AAEA,SAAS,QAAQ,OAAmB,YAAgC;AAClE,MAAI,MAAM,UAAU,YAAY;AAC9B,UAAM,OAAO,WAAW;AACxB,QAAI,SAAS,OAAW,QAAO;AAC/B,WAAO;AAAA,EACT;AACA,MAAI,MAAM,UAAU,aAAa;AAC/B,UAAM,OAAO,WAAW;AACxB,QAAI,SAAS,OAAW,QAAO;AAC/B,WAAO;AAAA,EACT;AACA,SAAO;AACT;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
package/dist/turns.js
CHANGED
|
@@ -26,6 +26,7 @@ function voiceTurns(models, conversation, mic, out, options, signal) {
|
|
|
26
26
|
phases,
|
|
27
27
|
captions,
|
|
28
28
|
maps: [],
|
|
29
|
+
clocks: /* @__PURE__ */ new WeakMap(),
|
|
29
30
|
bed: attachBed(out, options.soundscape)
|
|
30
31
|
};
|
|
31
32
|
const done = runSession(session).finally(() => {
|
|
@@ -52,12 +53,16 @@ async function runSession(session) {
|
|
|
52
53
|
notePlayback(session, active, tag, offsetMs);
|
|
53
54
|
});
|
|
54
55
|
const setPhase = phaseSetter(session.phases, session.bed);
|
|
56
|
+
const turnFailed = Promise.withResolvers();
|
|
55
57
|
try {
|
|
56
58
|
if (session.signal.aborted) return;
|
|
57
59
|
setPhase({ phase: "listening" });
|
|
58
|
-
|
|
60
|
+
const heard = session.models.listen(session.options.ears, session.mic, {
|
|
59
61
|
signal: session.signal
|
|
60
|
-
}))
|
|
62
|
+
})[Symbol.asyncIterator]();
|
|
63
|
+
while (true) {
|
|
64
|
+
const event = await nextVoiceEvent(heard, turnFailed.promise);
|
|
65
|
+
if (!event) break;
|
|
61
66
|
if (event.type === "error") {
|
|
62
67
|
if (event.reason === "aborted" || session.signal.aborted) return;
|
|
63
68
|
throw new VoiceRequestError(event.reason, event.error);
|
|
@@ -73,7 +78,7 @@ async function runSession(session) {
|
|
|
73
78
|
setPhase({ phase: "heard", text: event.text });
|
|
74
79
|
const meta = turnMeta(session.options, confidence);
|
|
75
80
|
confidence = void 0;
|
|
76
|
-
active = startTurn(session, setPhase, turnId, event, meta);
|
|
81
|
+
active = startTurn(session, setPhase, turnId, event, meta, turnFailed.reject);
|
|
77
82
|
}
|
|
78
83
|
} finally {
|
|
79
84
|
if (session.signal.aborted && active) await interrupt(session, active);
|
|
@@ -119,7 +124,12 @@ async function finishTurn(active) {
|
|
|
119
124
|
await active.done;
|
|
120
125
|
return void 0;
|
|
121
126
|
}
|
|
122
|
-
function
|
|
127
|
+
async function nextVoiceEvent(heard, failed) {
|
|
128
|
+
const step = await Promise.race([heard.next(), failed]);
|
|
129
|
+
if (step.done) return void 0;
|
|
130
|
+
return step.value;
|
|
131
|
+
}
|
|
132
|
+
function startTurn(session, setPhase, turnId, event, meta, rejectTurn) {
|
|
123
133
|
const linked = linkSignal(session.signal);
|
|
124
134
|
const active = {
|
|
125
135
|
controller: linked.controller,
|
|
@@ -127,11 +137,13 @@ function startTurn(session, setPhase, turnId, event, meta) {
|
|
|
127
137
|
turnId,
|
|
128
138
|
utteranceId: `${turnId}:agent`,
|
|
129
139
|
spokenUpTo: 0,
|
|
140
|
+
plays: [],
|
|
130
141
|
done: Promise.resolve()
|
|
131
142
|
};
|
|
132
143
|
active.done = performTurn(session, setPhase, active, event.text, event.at, meta).finally(() => {
|
|
133
144
|
linked.unlink();
|
|
134
145
|
});
|
|
146
|
+
void active.done.catch((error) => rejectTurn(error));
|
|
135
147
|
return active;
|
|
136
148
|
}
|
|
137
149
|
async function performTurn(session, setPhase, active, text, at, meta) {
|
|
@@ -180,7 +192,7 @@ async function readAgent(session, setPhase, active, message, meta, feed, maps) {
|
|
|
180
192
|
);
|
|
181
193
|
feed.push(piece.text);
|
|
182
194
|
}
|
|
183
|
-
if (state.entryId) bindEntry(maps, state.entryId);
|
|
195
|
+
if (state.entryId) bindEntry(session, active, maps, state.entryId);
|
|
184
196
|
}
|
|
185
197
|
} finally {
|
|
186
198
|
feed.end();
|
|
@@ -220,40 +232,75 @@ async function pipeSpeech(session, setPhase, active, audio, maps) {
|
|
|
220
232
|
if (event.type !== "audio") continue;
|
|
221
233
|
setPhase({ phase: "speaking" });
|
|
222
234
|
const span = translateSpan(event.span, maps);
|
|
223
|
-
const playing = {
|
|
224
|
-
start: event.span.start,
|
|
225
|
-
end: event.span.end,
|
|
226
|
-
durationMs: Math.round(chunkSeconds(event.chunk) * 1e3)
|
|
227
|
-
};
|
|
228
|
-
if (event.alignment) playing.alignment = event.alignment;
|
|
229
|
-
session.playing = playing;
|
|
230
235
|
const tag = { span };
|
|
231
236
|
if (event.alignment) tag.alignment = event.alignment;
|
|
237
|
+
session.clocks.set(tag, clockOf(event));
|
|
232
238
|
session.out.write(event.chunk, tag);
|
|
233
239
|
}
|
|
234
240
|
}
|
|
241
|
+
function clockOf(event) {
|
|
242
|
+
const clock = {
|
|
243
|
+
start: event.span.start,
|
|
244
|
+
end: event.span.end,
|
|
245
|
+
durationMs: chunkDurationMs(event)
|
|
246
|
+
};
|
|
247
|
+
if (event.alignment) clock.alignment = event.alignment;
|
|
248
|
+
return clock;
|
|
249
|
+
}
|
|
250
|
+
function chunkDurationMs(event) {
|
|
251
|
+
const measured = Math.round(chunkSeconds(event.chunk) * 1e3);
|
|
252
|
+
if (measured > 0) return measured;
|
|
253
|
+
const aligned = alignedDuration(event.alignment);
|
|
254
|
+
if (aligned > 0) return aligned;
|
|
255
|
+
const span = event.span.end - event.span.start;
|
|
256
|
+
if (span > 0) return span;
|
|
257
|
+
return 0;
|
|
258
|
+
}
|
|
259
|
+
function alignedDuration(alignment) {
|
|
260
|
+
if (!alignment) return 0;
|
|
261
|
+
let ms = 0;
|
|
262
|
+
for (const duration of alignment.durationMs) ms += duration;
|
|
263
|
+
return ms;
|
|
264
|
+
}
|
|
235
265
|
function notePlayback(session, active, tag, offsetMs) {
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
session.captions.push({
|
|
242
|
-
type: "agent_spoken",
|
|
243
|
-
turnId: active.turnId,
|
|
244
|
-
utteranceId: active.utteranceId,
|
|
245
|
-
upTo
|
|
246
|
-
});
|
|
247
|
-
}
|
|
248
|
-
}
|
|
249
|
-
if (!("entryId" in tag.span)) return;
|
|
266
|
+
rememberPlay(active, tag, offsetMs);
|
|
267
|
+
const playing = session.clocks.get(tag);
|
|
268
|
+
if (playing) revealSpoken(session, active, playing, offsetMs);
|
|
269
|
+
const span = translateSpan(tag.span, session.maps);
|
|
270
|
+
if (!("entryId" in span)) return;
|
|
250
271
|
active.cut = {
|
|
251
|
-
entryId:
|
|
252
|
-
contentIndex:
|
|
253
|
-
heardUpTo:
|
|
272
|
+
entryId: span.entryId,
|
|
273
|
+
contentIndex: span.contentIndex,
|
|
274
|
+
heardUpTo: playedSource(span, playing, offsetMs),
|
|
254
275
|
heardMs: session.out.playedMs
|
|
255
276
|
};
|
|
256
277
|
}
|
|
278
|
+
function rememberPlay(active, tag, offsetMs) {
|
|
279
|
+
const existing = active.plays.find((play) => play.tag === tag);
|
|
280
|
+
if (existing) {
|
|
281
|
+
existing.offsetMs = offsetMs;
|
|
282
|
+
return;
|
|
283
|
+
}
|
|
284
|
+
active.plays.push({ tag, offsetMs });
|
|
285
|
+
}
|
|
286
|
+
function revealSpoken(session, active, playing, offsetMs) {
|
|
287
|
+
const upTo = utteranceIndex(spokenOffset(playing, offsetMs), session.maps);
|
|
288
|
+
if (upTo <= active.spokenUpTo) return;
|
|
289
|
+
active.spokenUpTo = upTo;
|
|
290
|
+
session.captions.push({
|
|
291
|
+
type: "agent_spoken",
|
|
292
|
+
turnId: active.turnId,
|
|
293
|
+
utteranceId: active.utteranceId,
|
|
294
|
+
upTo
|
|
295
|
+
});
|
|
296
|
+
}
|
|
297
|
+
function playedSource(span, clock, offsetMs) {
|
|
298
|
+
if (!clock) return span.end;
|
|
299
|
+
const length = clock.end - clock.start;
|
|
300
|
+
if (length <= 0) return span.end;
|
|
301
|
+
const local = spokenOffset(clock, offsetMs) - clock.start;
|
|
302
|
+
return span.start + Math.round(local * (span.end - span.start) / length);
|
|
303
|
+
}
|
|
257
304
|
function translateSpan(span, maps) {
|
|
258
305
|
if ("entryId" in span) return span;
|
|
259
306
|
const map = answerMap(span, maps);
|
|
@@ -276,10 +323,11 @@ function scale(offset, spokenLen, sourceLen) {
|
|
|
276
323
|
if (spokenLen <= 0) return 0;
|
|
277
324
|
return Math.round(offset * sourceLen / spokenLen);
|
|
278
325
|
}
|
|
279
|
-
function bindEntry(maps, entryId) {
|
|
326
|
+
function bindEntry(session, active, maps, entryId) {
|
|
280
327
|
for (const map of maps) {
|
|
281
328
|
if (map.kind === "answer" && map.entryId === void 0) map.entryId = entryId;
|
|
282
329
|
}
|
|
330
|
+
for (const play of active.plays) notePlayback(session, active, play.tag, play.offsetMs);
|
|
283
331
|
}
|
|
284
332
|
function notePhase(setPhase, event) {
|
|
285
333
|
if (event.type === "run_start" || event.type === "turn_start") setPhase({ phase: "thinking" });
|
package/dist/turns.js.map
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../src/turns.ts"],
|
|
4
|
-
"sourcesContent": ["import type {\n AssistantMessageEvent,\n JsonObject,\n ProviderRequestOptions,\n ToolCall,\n UserMessage,\n} from '@earendil-works/pi-ai';\nimport {\n type AudioSink,\n type AudioSource,\n type AudioTag,\n chunkSeconds,\n type TextSpan,\n} from './audio.ts';\nimport { type SpokenClock, spokenOffset, utteranceIndex } from './captions.ts';\nimport { type EchoControl, openEchoMic } from './echo.ts';\nimport { VoiceRequestError } from './errors.ts';\nimport type { SpeechModel, TranscriptionModel } from './models.ts';\nimport { pushQueue } from './queue.ts';\nimport {\n type AudioOut,\n type BedFollower,\n followBed,\n isAudioOut,\n type Soundscape,\n} from './soundscape.ts';\nimport type { SpeechEvent, SpeechInput } from './speech.ts';\nimport {\n createSpokenState,\n type SpokenPiece,\n spokenPieces,\n type VoiceAgentEvent,\n} from './spoken.ts';\nimport type { ListenOptions, TranscriptionEvent } from './transcription.ts';\n\nexport interface VoiceTurnMeta {\n source: 'voice';\n transcriptionModel: string;\n confidence?: number;\n}\n\nexport interface VoiceCut {\n entryId: string;\n contentIndex: number;\n heardUpTo: number;\n heardMs: number;\n}\n\nexport interface VoiceConversation {\n prompt(\n message: UserMessage,\n meta: VoiceTurnMeta,\n signal: AbortSignal\n ): AsyncIterable<VoiceAgentEvent>;\n truncate(cut: VoiceCut): Promise<void>;\n}\n\nexport type BargeIn = 'interrupt' | 'onWords' | 'ignore' | 'auto';\n\nexport type EchoRecommendation = 'interrupt' | 'onWords' | 'headphones';\n\nexport interface VoiceTurnOptions {\n ears: TranscriptionModel<string>;\n mouth: SpeechModel<string>;\n voice: string;\n speakable?: (event: AssistantMessageEvent) => string | undefined;\n announce?: (call: ToolCall) => string | undefined;\n bargeIn?: BargeIn;\n echoRecommendation?: EchoRecommendation;\n echo?: EchoControl;\n soundscape?: Soundscape;\n}\n\nexport interface VoiceModels {\n listen(\n model: TranscriptionModel<string>,\n audio: AudioSource,\n options?: ListenOptions\n ): AsyncIterable<TranscriptionEvent>;\n speak(\n model: SpeechModel<string>,\n input: SpeechInput,\n options?: ProviderRequestOptions\n ): AsyncIterable<SpeechEvent>;\n}\n\nexport type VoicePhase =\n | { phase: 'listening' }\n | { phase: 'heard'; text: string }\n | { phase: 'thinking' }\n | { phase: 'tool'; toolName: string; toolCallId: string; args: JsonObject }\n | { phase: 'retrying'; attempt: number; at: number }\n | { phase: 'compacting' }\n | { phase: 'speaking' }\n | { phase: 'idle' };\n\nexport type CaptionEvent =\n | { type: 'user_partial'; turnId: string; segmentId: string; text: string }\n | { type: 'user_final'; turnId: string; segmentId: string; text: string; confidence?: number }\n | { type: 'agent_generated'; turnId: string; utteranceId: string; text: string }\n | { type: 'agent_spoken'; turnId: string; utteranceId: string; upTo: number }\n | { type: 'agent_cut'; turnId: string; utteranceId: string; heardUpTo: number }\n | { type: 'announce'; turnId: string; text: string }\n | { type: 'thinker_progress'; turnId: string; callId: string; text: string }\n | { type: 'thinker_result'; turnId: string; callId: string; summary: string; isError: boolean };\n\nexport interface VoiceSession {\n phases: AsyncIterable<VoicePhase>;\n captions: AsyncIterable<CaptionEvent>;\n done: Promise<void>;\n}\n\ninterface SpokenMap {\n spokenStart: number;\n spokenEnd: number;\n contentIndex: number;\n start: number;\n end: number;\n entryId?: string;\n kind: SpokenPiece['kind'];\n}\n\ninterface ActiveTurn {\n controller: AbortController;\n unlink: () => void;\n done: Promise<void>;\n turnId: string;\n utteranceId: string;\n spokenUpTo: number;\n cut?: VoiceCut;\n}\n\nexport function voiceTurns(\n models: VoiceModels,\n conversation: VoiceConversation,\n mic: AudioSource,\n out: AudioSink | AudioOut,\n options: VoiceTurnOptions,\n signal: AbortSignal\n): VoiceSession {\n const phases = pushQueue<VoicePhase>();\n const captions = pushQueue<CaptionEvent>();\n const session: Session = {\n models,\n conversation,\n mic: heardMic(options, mic),\n out: speechOf(out),\n options,\n signal,\n phases,\n captions,\n maps: [],\n bed: attachBed(out, options.soundscape),\n };\n const done = runSession(session).finally(() => {\n phases.end();\n captions.end();\n });\n return { phases: phases.iterable, captions: captions.iterable, done };\n}\n\ninterface Session {\n models: VoiceModels;\n conversation: VoiceConversation;\n mic: AudioSource;\n out: AudioSink;\n options: VoiceTurnOptions;\n signal: AbortSignal;\n phases: ReturnType<typeof pushQueue<VoicePhase>>;\n captions: ReturnType<typeof pushQueue<CaptionEvent>>;\n maps: SpokenMap[];\n playing?: SpokenClock;\n bed: BedFollower;\n}\n\nasync function runSession(session: Session): Promise<void> {\n const mode = bargeMode(session.options);\n let active: ActiveTurn | undefined;\n let confidence: number | undefined;\n let reserved = 0;\n let openId = '';\n const openTurn = () => {\n if (!openId) {\n reserved += 1;\n openId = String(reserved);\n }\n return openId;\n };\n const stopPlayback = session.out.onPlayback((tag, offsetMs) => {\n if (!active) return;\n notePlayback(session, active, tag, offsetMs);\n });\n const setPhase = phaseSetter(session.phases, session.bed);\n try {\n if (session.signal.aborted) return;\n setPhase({ phase: 'listening' });\n for await (const event of session.models.listen(session.options.ears, session.mic, {\n signal: session.signal,\n })) {\n if (event.type === 'error') {\n if (event.reason === 'aborted' || session.signal.aborted) return;\n throw new VoiceRequestError(event.reason, event.error);\n }\n const captionId =\n event.type === 'partial' || event.type === 'final' || event.type === 'turn_end'\n ? openTurn()\n : openId;\n confidence = noteUser(session, event, captionId, confidence);\n if (shouldBarge(mode, event)) active = await interrupt(session, active);\n if (event.type !== 'turn_end') continue;\n const turnId = openId || openTurn();\n openId = '';\n if (mode === 'ignore') active = await finishTurn(active);\n else active = await interrupt(session, active);\n setPhase({ phase: 'heard', text: event.text });\n const meta = turnMeta(session.options, confidence);\n confidence = undefined;\n active = startTurn(session, setPhase, turnId, event, meta);\n }\n } finally {\n if (session.signal.aborted && active) await interrupt(session, active);\n else if (active) await active.done;\n stopPlayback();\n setPhase({ phase: 'idle' });\n session.bed.stop();\n }\n}\n\nfunction turnMeta(options: VoiceTurnOptions, confidence: number | undefined): VoiceTurnMeta {\n return {\n source: 'voice',\n transcriptionModel: options.ears.id,\n ...(confidence === undefined ? {} : { confidence }),\n };\n}\n\nfunction noteUser(\n session: Session,\n event: TranscriptionEvent,\n turnId: string,\n confidence: number | undefined\n): number | undefined {\n if (event.type === 'partial') {\n session.captions.push({\n type: 'user_partial',\n turnId,\n segmentId: event.segmentId,\n text: event.text,\n });\n }\n if (event.type === 'final') {\n session.captions.push({\n type: 'user_final',\n turnId,\n segmentId: event.segmentId,\n text: event.text,\n ...(event.confidence === undefined ? {} : { confidence: event.confidence }),\n });\n return event.confidence ?? confidence;\n }\n if (event.type === 'turn_end') {\n session.captions.push({ type: 'user_final', turnId, segmentId: 'turn', text: event.text });\n }\n return confidence;\n}\n\nasync function finishTurn(active: ActiveTurn | undefined): Promise<undefined> {\n if (!active) return undefined;\n await active.done;\n return undefined;\n}\n\nfunction startTurn(\n session: Session,\n setPhase: (phase: VoicePhase) => void,\n turnId: string,\n event: Extract<TranscriptionEvent, { type: 'turn_end' }>,\n meta: VoiceTurnMeta\n): ActiveTurn {\n const linked = linkSignal(session.signal);\n const active: ActiveTurn = {\n controller: linked.controller,\n unlink: linked.unlink,\n turnId,\n utteranceId: `${turnId}:agent`,\n spokenUpTo: 0,\n done: Promise.resolve(),\n };\n active.done = performTurn(session, setPhase, active, event.text, event.at, meta).finally(() => {\n linked.unlink();\n });\n return active;\n}\n\nasync function performTurn(\n session: Session,\n setPhase: (phase: VoicePhase) => void,\n active: ActiveTurn,\n text: string,\n at: number,\n meta: VoiceTurnMeta\n): Promise<void> {\n const signal = active.controller.signal;\n try {\n setPhase({ phase: 'thinking' });\n const message: UserMessage = { role: 'user', content: text, timestamp: at };\n const feed = textFeed(signal);\n const maps: SpokenMap[] = [];\n session.maps = maps;\n const producing = readAgent(session, setPhase, active, message, meta, feed, maps);\n const audio = session.models.speak(\n session.options.mouth,\n { text: feed.iterable, voice: session.options.voice },\n { signal }\n );\n await Promise.all([producing, pipeSpeech(session, setPhase, active, audio, maps)]);\n if (!signal.aborted) setPhase({ phase: 'listening' });\n } catch (error) {\n if (signal.aborted || session.signal.aborted) return;\n throw error;\n }\n}\n\nasync function readAgent(\n session: Session,\n setPhase: (phase: VoicePhase) => void,\n active: ActiveTurn,\n message: UserMessage,\n meta: VoiceTurnMeta,\n feed: TextFeed,\n maps: SpokenMap[]\n): Promise<void> {\n const state = createSpokenState();\n let spokenChars = 0;\n let generated = '';\n try {\n for await (const event of session.conversation.prompt(\n message,\n meta,\n active.controller.signal\n )) {\n if (active.controller.signal.aborted) break;\n notePhase(setPhase, event);\n for (const piece of spokenPieces(event, state, session.options)) {\n spokenChars += piece.text.length;\n generated = rememberPiece(\n session,\n active,\n piece,\n maps,\n spokenChars,\n generated,\n state.entryId\n );\n feed.push(piece.text);\n }\n if (state.entryId) bindEntry(maps, state.entryId);\n }\n } finally {\n feed.end();\n }\n}\n\nfunction rememberPiece(\n session: Session,\n active: ActiveTurn,\n piece: SpokenPiece,\n maps: SpokenMap[],\n spokenEnd: number,\n generated: string,\n entryId: string | undefined\n): string {\n const bound = piece.entryId ?? entryId;\n maps.push({\n spokenStart: spokenEnd - piece.text.length,\n spokenEnd,\n contentIndex: piece.contentIndex,\n start: piece.start,\n end: piece.end,\n kind: piece.kind,\n ...(bound ? { entryId: bound } : {}),\n });\n if (piece.kind === 'announce') {\n session.captions.push({ type: 'announce', turnId: active.turnId, text: piece.text });\n return generated;\n }\n const text = generated + piece.text;\n session.captions.push({\n type: 'agent_generated',\n turnId: active.turnId,\n utteranceId: active.utteranceId,\n text,\n });\n return text;\n}\n\nasync function pipeSpeech(\n session: Session,\n setPhase: (phase: VoicePhase) => void,\n active: ActiveTurn,\n audio: AsyncIterable<SpeechEvent>,\n maps: SpokenMap[]\n): Promise<void> {\n for await (const event of audio) {\n if (active.controller.signal.aborted) return;\n if (event.type === 'error') {\n if (event.reason === 'aborted') return;\n throw new VoiceRequestError(event.reason, event.error);\n }\n if (event.type !== 'audio') continue;\n setPhase({ phase: 'speaking' });\n const span = translateSpan(event.span, maps);\n const playing: SpokenClock = {\n start: event.span.start,\n end: event.span.end,\n durationMs: Math.round(chunkSeconds(event.chunk) * 1000),\n };\n if (event.alignment) playing.alignment = event.alignment;\n session.playing = playing;\n const tag: AudioTag = { span };\n if (event.alignment) tag.alignment = event.alignment;\n session.out.write(event.chunk, tag);\n }\n}\n\nfunction notePlayback(session: Session, active: ActiveTurn, tag: AudioTag, offsetMs: number): void {\n const playing = session.playing;\n if (playing) {\n const upTo = utteranceIndex(spokenOffset(playing, offsetMs), session.maps);\n if (upTo > active.spokenUpTo) {\n active.spokenUpTo = upTo;\n session.captions.push({\n type: 'agent_spoken',\n turnId: active.turnId,\n utteranceId: active.utteranceId,\n upTo,\n });\n }\n }\n if (!('entryId' in tag.span)) return;\n active.cut = {\n entryId: tag.span.entryId,\n contentIndex: tag.span.contentIndex,\n heardUpTo: tag.span.end,\n heardMs: session.out.playedMs,\n };\n}\n\nfunction translateSpan(span: TextSpan, maps: SpokenMap[]): TextSpan {\n if ('entryId' in span) return span;\n const map = answerMap(span, maps);\n if (!map?.entryId) return span;\n const spokenLen = map.spokenEnd - map.spokenStart;\n const sourceLen = map.end - map.start;\n const start = map.start + scale(Math.max(0, span.start - map.spokenStart), spokenLen, sourceLen);\n const end =\n map.start + scale(Math.min(spokenLen, span.end - map.spokenStart), spokenLen, sourceLen);\n return { entryId: map.entryId, contentIndex: map.contentIndex, start, end };\n}\n\nfunction answerMap(span: { start: number; end: number }, maps: SpokenMap[]): SpokenMap | undefined {\n for (let index = maps.length - 1; index >= 0; index -= 1) {\n const item = maps[index];\n if (item?.kind !== 'answer' || !item.entryId) continue;\n if (item.spokenStart < span.end && item.spokenEnd > span.start) return item;\n }\n return undefined;\n}\n\nfunction scale(offset: number, spokenLen: number, sourceLen: number): number {\n if (spokenLen <= 0) return 0;\n return Math.round((offset * sourceLen) / spokenLen);\n}\n\nfunction bindEntry(maps: SpokenMap[], entryId: string): void {\n for (const map of maps) {\n if (map.kind === 'answer' && map.entryId === undefined) map.entryId = entryId;\n }\n}\n\nfunction notePhase(setPhase: (phase: VoicePhase) => void, event: VoiceAgentEvent): void {\n if (event.type === 'run_start' || event.type === 'turn_start') setPhase({ phase: 'thinking' });\n if (event.type === 'compaction_start') setPhase({ phase: 'compacting' });\n if (event.type === 'auto_retry_start' && event.attempt !== undefined && event.at !== undefined) {\n setPhase({ phase: 'retrying', attempt: event.attempt, at: event.at });\n }\n if (event.type === 'tool_execution_start' && event.toolName && event.toolCallId) {\n setPhase({\n phase: 'tool',\n toolName: event.toolName,\n toolCallId: event.toolCallId,\n args: event.args ?? {},\n });\n }\n}\n\nasync function interrupt(session: Session, active: ActiveTurn | undefined): Promise<undefined> {\n if (!active) return undefined;\n active.controller.abort();\n session.out.flush();\n await active.done;\n if (active.cut) {\n session.captions.push({\n type: 'agent_cut',\n turnId: active.turnId,\n utteranceId: active.utteranceId,\n heardUpTo: active.cut.heardUpTo,\n });\n await session.conversation.truncate(active.cut);\n }\n return undefined;\n}\n\nfunction bargeMode(options: VoiceTurnOptions): 'interrupt' | 'onWords' | 'ignore' {\n const mode = options.bargeIn ?? 'interrupt';\n if (mode !== 'auto') return mode;\n if (options.echoRecommendation === 'interrupt') return 'interrupt';\n if (options.echoRecommendation === 'headphones') return 'ignore';\n return 'onWords';\n}\n\nfunction shouldBarge(mode: 'interrupt' | 'onWords' | 'ignore', event: TranscriptionEvent): boolean {\n if (mode === 'ignore') return false;\n if (mode === 'interrupt') return event.type === 'speech_start';\n return event.type === 'partial' && event.text.trim().length > 0;\n}\n\nfunction heardMic(options: VoiceTurnOptions, mic: AudioSource): AudioSource {\n const echo = options.echo;\n if (echo === undefined) return mic;\n return openEchoMic(echo, mic);\n}\n\nfunction speechOf(out: AudioSink | AudioOut): AudioSink {\n if (isAudioOut(out)) return out.speech;\n return out;\n}\n\nfunction attachBed(out: AudioSink | AudioOut, soundscape: Soundscape | undefined): BedFollower {\n if (!soundscape) return idleBed();\n if (!isAudioOut(out)) return idleBed();\n return followBed(out.bed, soundscape);\n}\n\nfunction idleBed(): BedFollower {\n return {\n observe() {},\n stop() {},\n };\n}\n\nfunction phaseSetter(phases: Session['phases'], bed: BedFollower): (phase: VoicePhase) => void {\n let previous = '';\n return (phase) => {\n const key = JSON.stringify(phase);\n if (key === previous) return;\n previous = key;\n phases.push(phase);\n bed.observe(phase);\n };\n}\n\nfunction linkSignal(parent: AbortSignal): { controller: AbortController; unlink: () => void } {\n const controller = new AbortController();\n const abort = () => controller.abort();\n if (parent.aborted) controller.abort();\n else parent.addEventListener('abort', abort);\n return {\n controller,\n unlink: () => parent.removeEventListener('abort', abort),\n };\n}\n\ninterface TextFeed {\n push(text: string): void;\n end(): void;\n iterable: AsyncIterable<string>;\n}\n\nfunction textFeed(signal: AbortSignal): TextFeed {\n const queue: string[] = [];\n let ended = false;\n let wake: (() => void) | undefined;\n const notify = () => {\n const current = wake;\n wake = undefined;\n current?.();\n };\n const onAbort = () => notify();\n signal.addEventListener('abort', onAbort);\n return {\n push(text) {\n if (ended || signal.aborted || text.length === 0) return;\n queue.push(text);\n notify();\n },\n end() {\n ended = true;\n notify();\n },\n iterable: {\n async *[Symbol.asyncIterator]() {\n try {\n while (!signal.aborted) {\n const next = queue.shift();\n if (next !== undefined) {\n yield next;\n continue;\n }\n if (ended) return;\n await new Promise<void>((resolve) => {\n if (queue.length > 0 || ended || signal.aborted) resolve();\n else wake = resolve;\n });\n }\n } finally {\n signal.removeEventListener('abort', onAbort);\n }\n },\n },\n };\n}\n"],
|
|
5
|
-
"mappings": "AAOA;AAAA,EAIE;AAAA,OAEK;AACP,SAA2B,cAAc,sBAAsB;AAC/D,SAA2B,mBAAmB;AAC9C,SAAS,yBAAyB;AAElC,SAAS,iBAAiB;AAC1B;AAAA,EAGE;AAAA,EACA;AAAA,OAEK;AAEP;AAAA,EACE;AAAA,EAEA;AAAA,OAEK;AAoGA,SAAS,WACd,QACA,cACA,KACA,KACA,SACA,QACc;AACd,QAAM,SAAS,UAAsB;AACrC,QAAM,WAAW,UAAwB;AACzC,QAAM,UAAmB;AAAA,IACvB;AAAA,IACA;AAAA,IACA,KAAK,SAAS,SAAS,GAAG;AAAA,IAC1B,KAAK,SAAS,GAAG;AAAA,IACjB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,MAAM,CAAC;AAAA,IACP,KAAK,UAAU,KAAK,QAAQ,UAAU;AAAA,EACxC;AACA,QAAM,OAAO,WAAW,OAAO,EAAE,QAAQ,MAAM;AAC7C,WAAO,IAAI;AACX,aAAS,IAAI;AAAA,EACf,CAAC;AACD,SAAO,EAAE,QAAQ,OAAO,UAAU,UAAU,SAAS,UAAU,KAAK;AACtE;AAgBA,eAAe,WAAW,SAAiC;AACzD,QAAM,OAAO,UAAU,QAAQ,OAAO;AACtC,MAAI;AACJ,MAAI;AACJ,MAAI,WAAW;AACf,MAAI,SAAS;AACb,QAAM,WAAW,MAAM;AACrB,QAAI,CAAC,QAAQ;AACX,kBAAY;AACZ,eAAS,OAAO,QAAQ;AAAA,IAC1B;AACA,WAAO;AAAA,EACT;AACA,QAAM,eAAe,QAAQ,IAAI,WAAW,CAAC,KAAK,aAAa;AAC7D,QAAI,CAAC,OAAQ;AACb,iBAAa,SAAS,QAAQ,KAAK,QAAQ;AAAA,EAC7C,CAAC;AACD,QAAM,WAAW,YAAY,QAAQ,QAAQ,QAAQ,GAAG;AACxD,MAAI;AACF,QAAI,QAAQ,OAAO,QAAS;AAC5B,aAAS,EAAE,OAAO,YAAY,CAAC;AAC/B,qBAAiB,SAAS,QAAQ,OAAO,OAAO,QAAQ,QAAQ,MAAM,QAAQ,KAAK;AAAA,MACjF,QAAQ,QAAQ;AAAA,IAClB,CAAC,GAAG;AACF,UAAI,MAAM,SAAS,SAAS;AAC1B,YAAI,MAAM,WAAW,aAAa,QAAQ,OAAO,QAAS;AAC1D,cAAM,IAAI,kBAAkB,MAAM,QAAQ,MAAM,KAAK;AAAA,MACvD;AACA,YAAM,YACJ,MAAM,SAAS,aAAa,MAAM,SAAS,WAAW,MAAM,SAAS,aACjE,SAAS,IACT;AACN,mBAAa,SAAS,SAAS,OAAO,WAAW,UAAU;AAC3D,UAAI,YAAY,MAAM,KAAK,EAAG,UAAS,MAAM,UAAU,SAAS,MAAM;AACtE,UAAI,MAAM,SAAS,WAAY;AAC/B,YAAM,SAAS,UAAU,SAAS;AAClC,eAAS;AACT,UAAI,SAAS,SAAU,UAAS,MAAM,WAAW,MAAM;AAAA,UAClD,UAAS,MAAM,UAAU,SAAS,MAAM;AAC7C,eAAS,EAAE,OAAO,SAAS,MAAM,MAAM,KAAK,CAAC;AAC7C,YAAM,OAAO,SAAS,QAAQ,SAAS,UAAU;AACjD,mBAAa;AACb,eAAS,UAAU,SAAS,UAAU,QAAQ,OAAO,IAAI;AAAA,IAC3D;AAAA,EACF,UAAE;AACA,QAAI,QAAQ,OAAO,WAAW,OAAQ,OAAM,UAAU,SAAS,MAAM;AAAA,aAC5D,OAAQ,OAAM,OAAO;AAC9B,iBAAa;AACb,aAAS,EAAE,OAAO,OAAO,CAAC;AAC1B,YAAQ,IAAI,KAAK;AAAA,EACnB;AACF;AAEA,SAAS,SAAS,SAA2B,YAA+C;AAC1F,SAAO;AAAA,IACL,QAAQ;AAAA,IACR,oBAAoB,QAAQ,KAAK;AAAA,IACjC,GAAI,eAAe,SAAY,CAAC,IAAI,EAAE,WAAW;AAAA,EACnD;AACF;AAEA,SAAS,SACP,SACA,OACA,QACA,YACoB;AACpB,MAAI,MAAM,SAAS,WAAW;AAC5B,YAAQ,SAAS,KAAK;AAAA,MACpB,MAAM;AAAA,MACN;AAAA,MACA,WAAW,MAAM;AAAA,MACjB,MAAM,MAAM;AAAA,IACd,CAAC;AAAA,EACH;AACA,MAAI,MAAM,SAAS,SAAS;AAC1B,YAAQ,SAAS,KAAK;AAAA,MACpB,MAAM;AAAA,MACN;AAAA,MACA,WAAW,MAAM;AAAA,MACjB,MAAM,MAAM;AAAA,MACZ,GAAI,MAAM,eAAe,SAAY,CAAC,IAAI,EAAE,YAAY,MAAM,WAAW;AAAA,IAC3E,CAAC;AACD,WAAO,MAAM,cAAc;AAAA,EAC7B;AACA,MAAI,MAAM,SAAS,YAAY;AAC7B,YAAQ,SAAS,KAAK,EAAE,MAAM,cAAc,QAAQ,WAAW,QAAQ,MAAM,MAAM,KAAK,CAAC;AAAA,EAC3F;AACA,SAAO;AACT;AAEA,eAAe,WAAW,QAAoD;AAC5E,MAAI,CAAC,OAAQ,QAAO;AACpB,QAAM,OAAO;AACb,SAAO;AACT;AAEA,SAAS,UACP,SACA,UACA,QACA,OACA,MACY;AACZ,QAAM,SAAS,WAAW,QAAQ,MAAM;AACxC,QAAM,SAAqB;AAAA,IACzB,YAAY,OAAO;AAAA,IACnB,QAAQ,OAAO;AAAA,IACf;AAAA,IACA,aAAa,GAAG,MAAM;AAAA,IACtB,YAAY;AAAA,IACZ,MAAM,QAAQ,QAAQ;AAAA,EACxB;AACA,SAAO,OAAO,YAAY,SAAS,UAAU,QAAQ,MAAM,MAAM,MAAM,IAAI,IAAI,EAAE,QAAQ,MAAM;AAC7F,WAAO,OAAO;AAAA,EAChB,CAAC;AACD,SAAO;AACT;AAEA,eAAe,YACb,SACA,UACA,QACA,MACA,IACA,MACe;AACf,QAAM,SAAS,OAAO,WAAW;AACjC,MAAI;AACF,aAAS,EAAE,OAAO,WAAW,CAAC;AAC9B,UAAM,UAAuB,EAAE,MAAM,QAAQ,SAAS,MAAM,WAAW,GAAG;AAC1E,UAAM,OAAO,SAAS,MAAM;AAC5B,UAAM,OAAoB,CAAC;AAC3B,YAAQ,OAAO;AACf,UAAM,YAAY,UAAU,SAAS,UAAU,QAAQ,SAAS,MAAM,MAAM,IAAI;AAChF,UAAM,QAAQ,QAAQ,OAAO;AAAA,MAC3B,QAAQ,QAAQ;AAAA,MAChB,EAAE,MAAM,KAAK,UAAU,OAAO,QAAQ,QAAQ,MAAM;AAAA,MACpD,EAAE,OAAO;AAAA,IACX;AACA,UAAM,QAAQ,IAAI,CAAC,WAAW,WAAW,SAAS,UAAU,QAAQ,OAAO,IAAI,CAAC,CAAC;AACjF,QAAI,CAAC,OAAO,QAAS,UAAS,EAAE,OAAO,YAAY,CAAC;AAAA,EACtD,SAAS,OAAO;AACd,QAAI,OAAO,WAAW,QAAQ,OAAO,QAAS;AAC9C,UAAM;AAAA,EACR;AACF;AAEA,eAAe,UACb,SACA,UACA,QACA,SACA,MACA,MACA,MACe;AACf,QAAM,QAAQ,kBAAkB;AAChC,MAAI,cAAc;AAClB,MAAI,YAAY;AAChB,MAAI;AACF,qBAAiB,SAAS,QAAQ,aAAa;AAAA,MAC7C;AAAA,MACA;AAAA,MACA,OAAO,WAAW;AAAA,IACpB,GAAG;AACD,UAAI,OAAO,WAAW,OAAO,QAAS;AACtC,gBAAU,UAAU,KAAK;AACzB,iBAAW,SAAS,aAAa,OAAO,OAAO,QAAQ,OAAO,GAAG;AAC/D,uBAAe,MAAM,KAAK;AAC1B,oBAAY;AAAA,UACV;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA,MAAM;AAAA,QACR;AACA,aAAK,KAAK,MAAM,IAAI;AAAA,MACtB;AACA,UAAI,MAAM,QAAS,WAAU,MAAM,MAAM,OAAO;AAAA,IAClD;AAAA,EACF,UAAE;AACA,SAAK,IAAI;AAAA,EACX;AACF;AAEA,SAAS,cACP,SACA,QACA,OACA,MACA,WACA,WACA,SACQ;AACR,QAAM,QAAQ,MAAM,WAAW;AAC/B,OAAK,KAAK;AAAA,IACR,aAAa,YAAY,MAAM,KAAK;AAAA,IACpC;AAAA,IACA,cAAc,MAAM;AAAA,IACpB,OAAO,MAAM;AAAA,IACb,KAAK,MAAM;AAAA,IACX,MAAM,MAAM;AAAA,IACZ,GAAI,QAAQ,EAAE,SAAS,MAAM,IAAI,CAAC;AAAA,EACpC,CAAC;AACD,MAAI,MAAM,SAAS,YAAY;AAC7B,YAAQ,SAAS,KAAK,EAAE,MAAM,YAAY,QAAQ,OAAO,QAAQ,MAAM,MAAM,KAAK,CAAC;AACnF,WAAO;AAAA,EACT;AACA,QAAM,OAAO,YAAY,MAAM;AAC/B,UAAQ,SAAS,KAAK;AAAA,IACpB,MAAM;AAAA,IACN,QAAQ,OAAO;AAAA,IACf,aAAa,OAAO;AAAA,IACpB;AAAA,EACF,CAAC;AACD,SAAO;AACT;AAEA,eAAe,WACb,SACA,UACA,QACA,OACA,MACe;AACf,mBAAiB,SAAS,OAAO;AAC/B,QAAI,OAAO,WAAW,OAAO,QAAS;AACtC,QAAI,MAAM,SAAS,SAAS;AAC1B,UAAI,MAAM,WAAW,UAAW;AAChC,YAAM,IAAI,kBAAkB,MAAM,QAAQ,MAAM,KAAK;AAAA,IACvD;AACA,QAAI,MAAM,SAAS,QAAS;AAC5B,aAAS,EAAE,OAAO,WAAW,CAAC;AAC9B,UAAM,OAAO,cAAc,MAAM,MAAM,IAAI;AAC3C,UAAM,UAAuB;AAAA,MAC3B,OAAO,MAAM,KAAK;AAAA,MAClB,KAAK,MAAM,KAAK;AAAA,MAChB,YAAY,KAAK,MAAM,aAAa,MAAM,KAAK,IAAI,GAAI;AAAA,IACzD;AACA,QAAI,MAAM,UAAW,SAAQ,YAAY,MAAM;AAC/C,YAAQ,UAAU;AAClB,UAAM,MAAgB,EAAE,KAAK;AAC7B,QAAI,MAAM,UAAW,KAAI,YAAY,MAAM;AAC3C,YAAQ,IAAI,MAAM,MAAM,OAAO,GAAG;AAAA,EACpC;AACF;AAEA,SAAS,aAAa,SAAkB,QAAoB,KAAe,UAAwB;AACjG,QAAM,UAAU,QAAQ;AACxB,MAAI,SAAS;AACX,UAAM,OAAO,eAAe,aAAa,SAAS,QAAQ,GAAG,QAAQ,IAAI;AACzE,QAAI,OAAO,OAAO,YAAY;AAC5B,aAAO,aAAa;AACpB,cAAQ,SAAS,KAAK;AAAA,QACpB,MAAM;AAAA,QACN,QAAQ,OAAO;AAAA,QACf,aAAa,OAAO;AAAA,QACpB;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AACA,MAAI,EAAE,aAAa,IAAI,MAAO;AAC9B,SAAO,MAAM;AAAA,IACX,SAAS,IAAI,KAAK;AAAA,IAClB,cAAc,IAAI,KAAK;AAAA,IACvB,WAAW,IAAI,KAAK;AAAA,IACpB,SAAS,QAAQ,IAAI;AAAA,EACvB;AACF;AAEA,SAAS,cAAc,MAAgB,MAA6B;AAClE,MAAI,aAAa,KAAM,QAAO;AAC9B,QAAM,MAAM,UAAU,MAAM,IAAI;AAChC,MAAI,CAAC,KAAK,QAAS,QAAO;AAC1B,QAAM,YAAY,IAAI,YAAY,IAAI;AACtC,QAAM,YAAY,IAAI,MAAM,IAAI;AAChC,QAAM,QAAQ,IAAI,QAAQ,MAAM,KAAK,IAAI,GAAG,KAAK,QAAQ,IAAI,WAAW,GAAG,WAAW,SAAS;AAC/F,QAAM,MACJ,IAAI,QAAQ,MAAM,KAAK,IAAI,WAAW,KAAK,MAAM,IAAI,WAAW,GAAG,WAAW,SAAS;AACzF,SAAO,EAAE,SAAS,IAAI,SAAS,cAAc,IAAI,cAAc,OAAO,IAAI;AAC5E;AAEA,SAAS,UAAU,MAAsC,MAA0C;AACjG,WAAS,QAAQ,KAAK,SAAS,GAAG,SAAS,GAAG,SAAS,GAAG;AACxD,UAAM,OAAO,KAAK,KAAK;AACvB,QAAI,MAAM,SAAS,YAAY,CAAC,KAAK,QAAS;AAC9C,QAAI,KAAK,cAAc,KAAK,OAAO,KAAK,YAAY,KAAK,MAAO,QAAO;AAAA,EACzE;AACA,SAAO;AACT;AAEA,SAAS,MAAM,QAAgB,WAAmB,WAA2B;AAC3E,MAAI,aAAa,EAAG,QAAO;AAC3B,SAAO,KAAK,MAAO,SAAS,YAAa,SAAS;AACpD;AAEA,SAAS,UAAU,MAAmB,SAAuB;AAC3D,aAAW,OAAO,MAAM;AACtB,QAAI,IAAI,SAAS,YAAY,IAAI,YAAY,OAAW,KAAI,UAAU;AAAA,EACxE;AACF;AAEA,SAAS,UAAU,UAAuC,OAA8B;AACtF,MAAI,MAAM,SAAS,eAAe,MAAM,SAAS,aAAc,UAAS,EAAE,OAAO,WAAW,CAAC;AAC7F,MAAI,MAAM,SAAS,mBAAoB,UAAS,EAAE,OAAO,aAAa,CAAC;AACvE,MAAI,MAAM,SAAS,sBAAsB,MAAM,YAAY,UAAa,MAAM,OAAO,QAAW;AAC9F,aAAS,EAAE,OAAO,YAAY,SAAS,MAAM,SAAS,IAAI,MAAM,GAAG,CAAC;AAAA,EACtE;AACA,MAAI,MAAM,SAAS,0BAA0B,MAAM,YAAY,MAAM,YAAY;AAC/E,aAAS;AAAA,MACP,OAAO;AAAA,MACP,UAAU,MAAM;AAAA,MAChB,YAAY,MAAM;AAAA,MAClB,MAAM,MAAM,QAAQ,CAAC;AAAA,IACvB,CAAC;AAAA,EACH;AACF;AAEA,eAAe,UAAU,SAAkB,QAAoD;AAC7F,MAAI,CAAC,OAAQ,QAAO;AACpB,SAAO,WAAW,MAAM;AACxB,UAAQ,IAAI,MAAM;AAClB,QAAM,OAAO;AACb,MAAI,OAAO,KAAK;AACd,YAAQ,SAAS,KAAK;AAAA,MACpB,MAAM;AAAA,MACN,QAAQ,OAAO;AAAA,MACf,aAAa,OAAO;AAAA,MACpB,WAAW,OAAO,IAAI;AAAA,IACxB,CAAC;AACD,UAAM,QAAQ,aAAa,SAAS,OAAO,GAAG;AAAA,EAChD;AACA,SAAO;AACT;AAEA,SAAS,UAAU,SAA+D;AAChF,QAAM,OAAO,QAAQ,WAAW;AAChC,MAAI,SAAS,OAAQ,QAAO;AAC5B,MAAI,QAAQ,uBAAuB,YAAa,QAAO;AACvD,MAAI,QAAQ,uBAAuB,aAAc,QAAO;AACxD,SAAO;AACT;AAEA,SAAS,YAAY,MAA0C,OAAoC;AACjG,MAAI,SAAS,SAAU,QAAO;AAC9B,MAAI,SAAS,YAAa,QAAO,MAAM,SAAS;AAChD,SAAO,MAAM,SAAS,aAAa,MAAM,KAAK,KAAK,EAAE,SAAS;AAChE;AAEA,SAAS,SAAS,SAA2B,KAA+B;AAC1E,QAAM,OAAO,QAAQ;AACrB,MAAI,SAAS,OAAW,QAAO;AAC/B,SAAO,YAAY,MAAM,GAAG;AAC9B;AAEA,SAAS,SAAS,KAAsC;AACtD,MAAI,WAAW,GAAG,EAAG,QAAO,IAAI;AAChC,SAAO;AACT;AAEA,SAAS,UAAU,KAA2B,YAAiD;AAC7F,MAAI,CAAC,WAAY,QAAO,QAAQ;AAChC,MAAI,CAAC,WAAW,GAAG,EAAG,QAAO,QAAQ;AACrC,SAAO,UAAU,IAAI,KAAK,UAAU;AACtC;AAEA,SAAS,UAAuB;AAC9B,SAAO;AAAA,IACL,UAAU;AAAA,IAAC;AAAA,IACX,OAAO;AAAA,IAAC;AAAA,EACV;AACF;AAEA,SAAS,YAAY,QAA2B,KAA+C;AAC7F,MAAI,WAAW;AACf,SAAO,CAAC,UAAU;AAChB,UAAM,MAAM,KAAK,UAAU,KAAK;AAChC,QAAI,QAAQ,SAAU;AACtB,eAAW;AACX,WAAO,KAAK,KAAK;AACjB,QAAI,QAAQ,KAAK;AAAA,EACnB;AACF;AAEA,SAAS,WAAW,QAA0E;AAC5F,QAAM,aAAa,IAAI,gBAAgB;AACvC,QAAM,QAAQ,MAAM,WAAW,MAAM;AACrC,MAAI,OAAO,QAAS,YAAW,MAAM;AAAA,MAChC,QAAO,iBAAiB,SAAS,KAAK;AAC3C,SAAO;AAAA,IACL;AAAA,IACA,QAAQ,MAAM,OAAO,oBAAoB,SAAS,KAAK;AAAA,EACzD;AACF;AAQA,SAAS,SAAS,QAA+B;AAC/C,QAAM,QAAkB,CAAC;AACzB,MAAI,QAAQ;AACZ,MAAI;AACJ,QAAM,SAAS,MAAM;AACnB,UAAM,UAAU;AAChB,WAAO;AACP,cAAU;AAAA,EACZ;AACA,QAAM,UAAU,MAAM,OAAO;AAC7B,SAAO,iBAAiB,SAAS,OAAO;AACxC,SAAO;AAAA,IACL,KAAK,MAAM;AACT,UAAI,SAAS,OAAO,WAAW,KAAK,WAAW,EAAG;AAClD,YAAM,KAAK,IAAI;AACf,aAAO;AAAA,IACT;AAAA,IACA,MAAM;AACJ,cAAQ;AACR,aAAO;AAAA,IACT;AAAA,IACA,UAAU;AAAA,MACR,QAAQ,OAAO,aAAa,IAAI;AAC9B,YAAI;AACF,iBAAO,CAAC,OAAO,SAAS;AACtB,kBAAM,OAAO,MAAM,MAAM;AACzB,gBAAI,SAAS,QAAW;AACtB,oBAAM;AACN;AAAA,YACF;AACA,gBAAI,MAAO;AACX,kBAAM,IAAI,QAAc,CAAC,YAAY;AACnC,kBAAI,MAAM,SAAS,KAAK,SAAS,OAAO,QAAS,SAAQ;AAAA,kBACpD,QAAO;AAAA,YACd,CAAC;AAAA,UACH;AAAA,QACF,UAAE;AACA,iBAAO,oBAAoB,SAAS,OAAO;AAAA,QAC7C;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;",
|
|
4
|
+
"sourcesContent": ["import type {\n AssistantMessageEvent,\n JsonObject,\n ProviderRequestOptions,\n ToolCall,\n UserMessage,\n} from '@earendil-works/pi-ai';\nimport {\n type AudioSink,\n type AudioSource,\n type AudioTag,\n type CharAlignment,\n chunkSeconds,\n type TextSpan,\n} from './audio.ts';\nimport { type SpokenClock, spokenOffset, utteranceIndex } from './captions.ts';\nimport { type EchoControl, openEchoMic } from './echo.ts';\nimport { VoiceRequestError } from './errors.ts';\nimport type { SpeechModel, TranscriptionModel } from './models.ts';\nimport { pushQueue } from './queue.ts';\nimport {\n type AudioOut,\n type BedFollower,\n followBed,\n isAudioOut,\n type Soundscape,\n} from './soundscape.ts';\nimport type { SpeechEvent, SpeechInput } from './speech.ts';\nimport {\n createSpokenState,\n type SpokenPiece,\n spokenPieces,\n type VoiceAgentEvent,\n} from './spoken.ts';\nimport type { ListenOptions, TranscriptionEvent } from './transcription.ts';\n\nexport interface VoiceTurnMeta {\n source: 'voice';\n transcriptionModel: string;\n confidence?: number;\n}\n\nexport interface VoiceCut {\n entryId: string;\n contentIndex: number;\n heardUpTo: number;\n heardMs: number;\n}\n\nexport interface VoiceConversation {\n prompt(\n message: UserMessage,\n meta: VoiceTurnMeta,\n signal: AbortSignal\n ): AsyncIterable<VoiceAgentEvent>;\n truncate(cut: VoiceCut): Promise<void>;\n}\n\nexport type BargeIn = 'interrupt' | 'onWords' | 'ignore' | 'auto';\n\nexport type EchoRecommendation = 'interrupt' | 'onWords' | 'headphones';\n\nexport interface VoiceTurnOptions {\n ears: TranscriptionModel<string>;\n mouth: SpeechModel<string>;\n voice: string;\n speakable?: (event: AssistantMessageEvent) => string | undefined;\n announce?: (call: ToolCall) => string | undefined;\n bargeIn?: BargeIn;\n echoRecommendation?: EchoRecommendation;\n echo?: EchoControl;\n soundscape?: Soundscape;\n}\n\nexport interface VoiceModels {\n listen(\n model: TranscriptionModel<string>,\n audio: AudioSource,\n options?: ListenOptions\n ): AsyncIterable<TranscriptionEvent>;\n speak(\n model: SpeechModel<string>,\n input: SpeechInput,\n options?: ProviderRequestOptions\n ): AsyncIterable<SpeechEvent>;\n}\n\nexport type VoicePhase =\n | { phase: 'listening' }\n | { phase: 'heard'; text: string }\n | { phase: 'thinking' }\n | { phase: 'tool'; toolName: string; toolCallId: string; args: JsonObject }\n | { phase: 'retrying'; attempt: number; at: number }\n | { phase: 'compacting' }\n | { phase: 'speaking' }\n | { phase: 'idle' };\n\nexport type CaptionEvent =\n | { type: 'user_partial'; turnId: string; segmentId: string; text: string }\n | { type: 'user_final'; turnId: string; segmentId: string; text: string; confidence?: number }\n | { type: 'agent_generated'; turnId: string; utteranceId: string; text: string }\n | { type: 'agent_spoken'; turnId: string; utteranceId: string; upTo: number }\n | { type: 'agent_cut'; turnId: string; utteranceId: string; heardUpTo: number }\n | { type: 'announce'; turnId: string; text: string }\n | { type: 'thinker_progress'; turnId: string; callId: string; text: string }\n | { type: 'thinker_result'; turnId: string; callId: string; summary: string; isError: boolean };\n\nexport interface VoiceSession {\n phases: AsyncIterable<VoicePhase>;\n captions: AsyncIterable<CaptionEvent>;\n done: Promise<void>;\n}\n\ninterface SpokenMap {\n spokenStart: number;\n spokenEnd: number;\n contentIndex: number;\n start: number;\n end: number;\n entryId?: string;\n kind: SpokenPiece['kind'];\n}\n\ninterface PlayedTag {\n tag: AudioTag;\n offsetMs: number;\n}\n\ninterface ActiveTurn {\n controller: AbortController;\n unlink: () => void;\n done: Promise<void>;\n turnId: string;\n utteranceId: string;\n spokenUpTo: number;\n plays: PlayedTag[];\n cut?: VoiceCut;\n}\n\nexport function voiceTurns(\n models: VoiceModels,\n conversation: VoiceConversation,\n mic: AudioSource,\n out: AudioSink | AudioOut,\n options: VoiceTurnOptions,\n signal: AbortSignal\n): VoiceSession {\n const phases = pushQueue<VoicePhase>();\n const captions = pushQueue<CaptionEvent>();\n const session: Session = {\n models,\n conversation,\n mic: heardMic(options, mic),\n out: speechOf(out),\n options,\n signal,\n phases,\n captions,\n maps: [],\n clocks: new WeakMap(),\n bed: attachBed(out, options.soundscape),\n };\n const done = runSession(session).finally(() => {\n phases.end();\n captions.end();\n });\n return { phases: phases.iterable, captions: captions.iterable, done };\n}\n\ninterface Session {\n models: VoiceModels;\n conversation: VoiceConversation;\n mic: AudioSource;\n out: AudioSink;\n options: VoiceTurnOptions;\n signal: AbortSignal;\n phases: ReturnType<typeof pushQueue<VoicePhase>>;\n captions: ReturnType<typeof pushQueue<CaptionEvent>>;\n maps: SpokenMap[];\n clocks: WeakMap<AudioTag, SpokenClock>;\n bed: BedFollower;\n}\n\nasync function runSession(session: Session): Promise<void> {\n const mode = bargeMode(session.options);\n let active: ActiveTurn | undefined;\n let confidence: number | undefined;\n let reserved = 0;\n let openId = '';\n const openTurn = () => {\n if (!openId) {\n reserved += 1;\n openId = String(reserved);\n }\n return openId;\n };\n const stopPlayback = session.out.onPlayback((tag, offsetMs) => {\n if (!active) return;\n notePlayback(session, active, tag, offsetMs);\n });\n const setPhase = phaseSetter(session.phases, session.bed);\n const turnFailed = Promise.withResolvers<never>();\n try {\n if (session.signal.aborted) return;\n setPhase({ phase: 'listening' });\n const heard = session.models\n .listen(session.options.ears, session.mic, {\n signal: session.signal,\n })\n [Symbol.asyncIterator]();\n while (true) {\n const event = await nextVoiceEvent(heard, turnFailed.promise);\n if (!event) break;\n if (event.type === 'error') {\n if (event.reason === 'aborted' || session.signal.aborted) return;\n throw new VoiceRequestError(event.reason, event.error);\n }\n const captionId =\n event.type === 'partial' || event.type === 'final' || event.type === 'turn_end'\n ? openTurn()\n : openId;\n confidence = noteUser(session, event, captionId, confidence);\n if (shouldBarge(mode, event)) active = await interrupt(session, active);\n if (event.type !== 'turn_end') continue;\n const turnId = openId || openTurn();\n openId = '';\n if (mode === 'ignore') active = await finishTurn(active);\n else active = await interrupt(session, active);\n setPhase({ phase: 'heard', text: event.text });\n const meta = turnMeta(session.options, confidence);\n confidence = undefined;\n active = startTurn(session, setPhase, turnId, event, meta, turnFailed.reject);\n }\n } finally {\n if (session.signal.aborted && active) await interrupt(session, active);\n else if (active) await active.done;\n stopPlayback();\n setPhase({ phase: 'idle' });\n session.bed.stop();\n }\n}\n\nfunction turnMeta(options: VoiceTurnOptions, confidence: number | undefined): VoiceTurnMeta {\n return {\n source: 'voice',\n transcriptionModel: options.ears.id,\n ...(confidence === undefined ? {} : { confidence }),\n };\n}\n\nfunction noteUser(\n session: Session,\n event: TranscriptionEvent,\n turnId: string,\n confidence: number | undefined\n): number | undefined {\n if (event.type === 'partial') {\n session.captions.push({\n type: 'user_partial',\n turnId,\n segmentId: event.segmentId,\n text: event.text,\n });\n }\n if (event.type === 'final') {\n session.captions.push({\n type: 'user_final',\n turnId,\n segmentId: event.segmentId,\n text: event.text,\n ...(event.confidence === undefined ? {} : { confidence: event.confidence }),\n });\n return event.confidence ?? confidence;\n }\n if (event.type === 'turn_end') {\n session.captions.push({ type: 'user_final', turnId, segmentId: 'turn', text: event.text });\n }\n return confidence;\n}\n\nasync function finishTurn(active: ActiveTurn | undefined): Promise<undefined> {\n if (!active) return undefined;\n await active.done;\n return undefined;\n}\n\nasync function nextVoiceEvent(\n heard: AsyncIterator<TranscriptionEvent>,\n failed: Promise<never>\n): Promise<TranscriptionEvent | undefined> {\n const step = await Promise.race([heard.next(), failed]);\n if (step.done) return undefined;\n return step.value;\n}\n\nfunction startTurn(\n session: Session,\n setPhase: (phase: VoicePhase) => void,\n turnId: string,\n event: Extract<TranscriptionEvent, { type: 'turn_end' }>,\n meta: VoiceTurnMeta,\n rejectTurn: (error: unknown) => void\n): ActiveTurn {\n const linked = linkSignal(session.signal);\n const active: ActiveTurn = {\n controller: linked.controller,\n unlink: linked.unlink,\n turnId,\n utteranceId: `${turnId}:agent`,\n spokenUpTo: 0,\n plays: [],\n done: Promise.resolve(),\n };\n active.done = performTurn(session, setPhase, active, event.text, event.at, meta).finally(() => {\n linked.unlink();\n });\n void active.done.catch((error: unknown) => rejectTurn(error));\n return active;\n}\n\nasync function performTurn(\n session: Session,\n setPhase: (phase: VoicePhase) => void,\n active: ActiveTurn,\n text: string,\n at: number,\n meta: VoiceTurnMeta\n): Promise<void> {\n const signal = active.controller.signal;\n try {\n setPhase({ phase: 'thinking' });\n const message: UserMessage = { role: 'user', content: text, timestamp: at };\n const feed = textFeed(signal);\n const maps: SpokenMap[] = [];\n session.maps = maps;\n const producing = readAgent(session, setPhase, active, message, meta, feed, maps);\n const audio = session.models.speak(\n session.options.mouth,\n { text: feed.iterable, voice: session.options.voice },\n { signal }\n );\n await Promise.all([producing, pipeSpeech(session, setPhase, active, audio, maps)]);\n if (!signal.aborted) setPhase({ phase: 'listening' });\n } catch (error) {\n if (signal.aborted || session.signal.aborted) return;\n throw error;\n }\n}\n\nasync function readAgent(\n session: Session,\n setPhase: (phase: VoicePhase) => void,\n active: ActiveTurn,\n message: UserMessage,\n meta: VoiceTurnMeta,\n feed: TextFeed,\n maps: SpokenMap[]\n): Promise<void> {\n const state = createSpokenState();\n let spokenChars = 0;\n let generated = '';\n try {\n for await (const event of session.conversation.prompt(\n message,\n meta,\n active.controller.signal\n )) {\n if (active.controller.signal.aborted) break;\n notePhase(setPhase, event);\n for (const piece of spokenPieces(event, state, session.options)) {\n spokenChars += piece.text.length;\n generated = rememberPiece(\n session,\n active,\n piece,\n maps,\n spokenChars,\n generated,\n state.entryId\n );\n feed.push(piece.text);\n }\n if (state.entryId) bindEntry(session, active, maps, state.entryId);\n }\n } finally {\n feed.end();\n }\n}\n\nfunction rememberPiece(\n session: Session,\n active: ActiveTurn,\n piece: SpokenPiece,\n maps: SpokenMap[],\n spokenEnd: number,\n generated: string,\n entryId: string | undefined\n): string {\n const bound = piece.entryId ?? entryId;\n maps.push({\n spokenStart: spokenEnd - piece.text.length,\n spokenEnd,\n contentIndex: piece.contentIndex,\n start: piece.start,\n end: piece.end,\n kind: piece.kind,\n ...(bound ? { entryId: bound } : {}),\n });\n if (piece.kind === 'announce') {\n session.captions.push({ type: 'announce', turnId: active.turnId, text: piece.text });\n return generated;\n }\n const text = generated + piece.text;\n session.captions.push({\n type: 'agent_generated',\n turnId: active.turnId,\n utteranceId: active.utteranceId,\n text,\n });\n return text;\n}\n\nasync function pipeSpeech(\n session: Session,\n setPhase: (phase: VoicePhase) => void,\n active: ActiveTurn,\n audio: AsyncIterable<SpeechEvent>,\n maps: SpokenMap[]\n): Promise<void> {\n for await (const event of audio) {\n if (active.controller.signal.aborted) return;\n if (event.type === 'error') {\n if (event.reason === 'aborted') return;\n throw new VoiceRequestError(event.reason, event.error);\n }\n if (event.type !== 'audio') continue;\n setPhase({ phase: 'speaking' });\n const span = translateSpan(event.span, maps);\n const tag: AudioTag = { span };\n if (event.alignment) tag.alignment = event.alignment;\n session.clocks.set(tag, clockOf(event));\n session.out.write(event.chunk, tag);\n }\n}\n\nfunction clockOf(event: Extract<SpeechEvent, { type: 'audio' }>): SpokenClock {\n const clock: SpokenClock = {\n start: event.span.start,\n end: event.span.end,\n durationMs: chunkDurationMs(event),\n };\n if (event.alignment) clock.alignment = event.alignment;\n return clock;\n}\n\nfunction chunkDurationMs(event: Extract<SpeechEvent, { type: 'audio' }>): number {\n const measured = Math.round(chunkSeconds(event.chunk) * 1000);\n if (measured > 0) return measured;\n const aligned = alignedDuration(event.alignment);\n if (aligned > 0) return aligned;\n const span = event.span.end - event.span.start;\n if (span > 0) return span;\n return 0;\n}\n\nfunction alignedDuration(alignment: CharAlignment | undefined): number {\n if (!alignment) return 0;\n let ms = 0;\n for (const duration of alignment.durationMs) ms += duration;\n return ms;\n}\n\nfunction notePlayback(session: Session, active: ActiveTurn, tag: AudioTag, offsetMs: number): void {\n rememberPlay(active, tag, offsetMs);\n const playing = session.clocks.get(tag);\n if (playing) revealSpoken(session, active, playing, offsetMs);\n const span = translateSpan(tag.span, session.maps);\n if (!('entryId' in span)) return;\n active.cut = {\n entryId: span.entryId,\n contentIndex: span.contentIndex,\n heardUpTo: playedSource(span, playing, offsetMs),\n heardMs: session.out.playedMs,\n };\n}\n\nfunction rememberPlay(active: ActiveTurn, tag: AudioTag, offsetMs: number) {\n const existing = active.plays.find((play) => play.tag === tag);\n if (existing) {\n existing.offsetMs = offsetMs;\n return;\n }\n active.plays.push({ tag, offsetMs });\n}\n\nfunction revealSpoken(\n session: Session,\n active: ActiveTurn,\n playing: SpokenClock,\n offsetMs: number\n) {\n const upTo = utteranceIndex(spokenOffset(playing, offsetMs), session.maps);\n if (upTo <= active.spokenUpTo) return;\n active.spokenUpTo = upTo;\n session.captions.push({\n type: 'agent_spoken',\n turnId: active.turnId,\n utteranceId: active.utteranceId,\n upTo,\n });\n}\n\nfunction playedSource(\n span: { start: number; end: number },\n clock: SpokenClock | undefined,\n offsetMs: number\n): number {\n if (!clock) return span.end;\n const length = clock.end - clock.start;\n if (length <= 0) return span.end;\n const local = spokenOffset(clock, offsetMs) - clock.start;\n return span.start + Math.round((local * (span.end - span.start)) / length);\n}\n\nfunction translateSpan(span: TextSpan, maps: SpokenMap[]): TextSpan {\n if ('entryId' in span) return span;\n const map = answerMap(span, maps);\n if (!map?.entryId) return span;\n const spokenLen = map.spokenEnd - map.spokenStart;\n const sourceLen = map.end - map.start;\n const start = map.start + scale(Math.max(0, span.start - map.spokenStart), spokenLen, sourceLen);\n const end =\n map.start + scale(Math.min(spokenLen, span.end - map.spokenStart), spokenLen, sourceLen);\n return { entryId: map.entryId, contentIndex: map.contentIndex, start, end };\n}\n\nfunction answerMap(span: { start: number; end: number }, maps: SpokenMap[]): SpokenMap | undefined {\n for (let index = maps.length - 1; index >= 0; index -= 1) {\n const item = maps[index];\n if (item?.kind !== 'answer' || !item.entryId) continue;\n if (item.spokenStart < span.end && item.spokenEnd > span.start) return item;\n }\n return undefined;\n}\n\nfunction scale(offset: number, spokenLen: number, sourceLen: number): number {\n if (spokenLen <= 0) return 0;\n return Math.round((offset * sourceLen) / spokenLen);\n}\n\nfunction bindEntry(session: Session, active: ActiveTurn, maps: SpokenMap[], entryId: string): void {\n for (const map of maps) {\n if (map.kind === 'answer' && map.entryId === undefined) map.entryId = entryId;\n }\n for (const play of active.plays) notePlayback(session, active, play.tag, play.offsetMs);\n}\n\nfunction notePhase(setPhase: (phase: VoicePhase) => void, event: VoiceAgentEvent): void {\n if (event.type === 'run_start' || event.type === 'turn_start') setPhase({ phase: 'thinking' });\n if (event.type === 'compaction_start') setPhase({ phase: 'compacting' });\n if (event.type === 'auto_retry_start' && event.attempt !== undefined && event.at !== undefined) {\n setPhase({ phase: 'retrying', attempt: event.attempt, at: event.at });\n }\n if (event.type === 'tool_execution_start' && event.toolName && event.toolCallId) {\n setPhase({\n phase: 'tool',\n toolName: event.toolName,\n toolCallId: event.toolCallId,\n args: event.args ?? {},\n });\n }\n}\n\nasync function interrupt(session: Session, active: ActiveTurn | undefined): Promise<undefined> {\n if (!active) return undefined;\n active.controller.abort();\n session.out.flush();\n await active.done;\n if (active.cut) {\n session.captions.push({\n type: 'agent_cut',\n turnId: active.turnId,\n utteranceId: active.utteranceId,\n heardUpTo: active.cut.heardUpTo,\n });\n await session.conversation.truncate(active.cut);\n }\n return undefined;\n}\n\nfunction bargeMode(options: VoiceTurnOptions): 'interrupt' | 'onWords' | 'ignore' {\n const mode = options.bargeIn ?? 'interrupt';\n if (mode !== 'auto') return mode;\n if (options.echoRecommendation === 'interrupt') return 'interrupt';\n if (options.echoRecommendation === 'headphones') return 'ignore';\n return 'onWords';\n}\n\nfunction shouldBarge(mode: 'interrupt' | 'onWords' | 'ignore', event: TranscriptionEvent): boolean {\n if (mode === 'ignore') return false;\n if (mode === 'interrupt') return event.type === 'speech_start';\n return event.type === 'partial' && event.text.trim().length > 0;\n}\n\nfunction heardMic(options: VoiceTurnOptions, mic: AudioSource): AudioSource {\n const echo = options.echo;\n if (echo === undefined) return mic;\n return openEchoMic(echo, mic);\n}\n\nfunction speechOf(out: AudioSink | AudioOut): AudioSink {\n if (isAudioOut(out)) return out.speech;\n return out;\n}\n\nfunction attachBed(out: AudioSink | AudioOut, soundscape: Soundscape | undefined): BedFollower {\n if (!soundscape) return idleBed();\n if (!isAudioOut(out)) return idleBed();\n return followBed(out.bed, soundscape);\n}\n\nfunction idleBed(): BedFollower {\n return {\n observe() {},\n stop() {},\n };\n}\n\nfunction phaseSetter(phases: Session['phases'], bed: BedFollower): (phase: VoicePhase) => void {\n let previous = '';\n return (phase) => {\n const key = JSON.stringify(phase);\n if (key === previous) return;\n previous = key;\n phases.push(phase);\n bed.observe(phase);\n };\n}\n\nfunction linkSignal(parent: AbortSignal): { controller: AbortController; unlink: () => void } {\n const controller = new AbortController();\n const abort = () => controller.abort();\n if (parent.aborted) controller.abort();\n else parent.addEventListener('abort', abort);\n return {\n controller,\n unlink: () => parent.removeEventListener('abort', abort),\n };\n}\n\ninterface TextFeed {\n push(text: string): void;\n end(): void;\n iterable: AsyncIterable<string>;\n}\n\nfunction textFeed(signal: AbortSignal): TextFeed {\n const queue: string[] = [];\n let ended = false;\n let wake: (() => void) | undefined;\n const notify = () => {\n const current = wake;\n wake = undefined;\n current?.();\n };\n const onAbort = () => notify();\n signal.addEventListener('abort', onAbort);\n return {\n push(text) {\n if (ended || signal.aborted || text.length === 0) return;\n queue.push(text);\n notify();\n },\n end() {\n ended = true;\n notify();\n },\n iterable: {\n async *[Symbol.asyncIterator]() {\n try {\n while (!signal.aborted) {\n const next = queue.shift();\n if (next !== undefined) {\n yield next;\n continue;\n }\n if (ended) return;\n await new Promise<void>((resolve) => {\n if (queue.length > 0 || ended || signal.aborted) resolve();\n else wake = resolve;\n });\n }\n } finally {\n signal.removeEventListener('abort', onAbort);\n }\n },\n },\n };\n}\n"],
|
|
5
|
+
"mappings": "AAOA;AAAA,EAKE;AAAA,OAEK;AACP,SAA2B,cAAc,sBAAsB;AAC/D,SAA2B,mBAAmB;AAC9C,SAAS,yBAAyB;AAElC,SAAS,iBAAiB;AAC1B;AAAA,EAGE;AAAA,EACA;AAAA,OAEK;AAEP;AAAA,EACE;AAAA,EAEA;AAAA,OAEK;AA0GA,SAAS,WACd,QACA,cACA,KACA,KACA,SACA,QACc;AACd,QAAM,SAAS,UAAsB;AACrC,QAAM,WAAW,UAAwB;AACzC,QAAM,UAAmB;AAAA,IACvB;AAAA,IACA;AAAA,IACA,KAAK,SAAS,SAAS,GAAG;AAAA,IAC1B,KAAK,SAAS,GAAG;AAAA,IACjB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,MAAM,CAAC;AAAA,IACP,QAAQ,oBAAI,QAAQ;AAAA,IACpB,KAAK,UAAU,KAAK,QAAQ,UAAU;AAAA,EACxC;AACA,QAAM,OAAO,WAAW,OAAO,EAAE,QAAQ,MAAM;AAC7C,WAAO,IAAI;AACX,aAAS,IAAI;AAAA,EACf,CAAC;AACD,SAAO,EAAE,QAAQ,OAAO,UAAU,UAAU,SAAS,UAAU,KAAK;AACtE;AAgBA,eAAe,WAAW,SAAiC;AACzD,QAAM,OAAO,UAAU,QAAQ,OAAO;AACtC,MAAI;AACJ,MAAI;AACJ,MAAI,WAAW;AACf,MAAI,SAAS;AACb,QAAM,WAAW,MAAM;AACrB,QAAI,CAAC,QAAQ;AACX,kBAAY;AACZ,eAAS,OAAO,QAAQ;AAAA,IAC1B;AACA,WAAO;AAAA,EACT;AACA,QAAM,eAAe,QAAQ,IAAI,WAAW,CAAC,KAAK,aAAa;AAC7D,QAAI,CAAC,OAAQ;AACb,iBAAa,SAAS,QAAQ,KAAK,QAAQ;AAAA,EAC7C,CAAC;AACD,QAAM,WAAW,YAAY,QAAQ,QAAQ,QAAQ,GAAG;AACxD,QAAM,aAAa,QAAQ,cAAqB;AAChD,MAAI;AACF,QAAI,QAAQ,OAAO,QAAS;AAC5B,aAAS,EAAE,OAAO,YAAY,CAAC;AAC/B,UAAM,QAAQ,QAAQ,OACnB,OAAO,QAAQ,QAAQ,MAAM,QAAQ,KAAK;AAAA,MACzC,QAAQ,QAAQ;AAAA,IAClB,CAAC,EACA,OAAO,aAAa,EAAE;AACzB,WAAO,MAAM;AACX,YAAM,QAAQ,MAAM,eAAe,OAAO,WAAW,OAAO;AAC5D,UAAI,CAAC,MAAO;AACZ,UAAI,MAAM,SAAS,SAAS;AAC1B,YAAI,MAAM,WAAW,aAAa,QAAQ,OAAO,QAAS;AAC1D,cAAM,IAAI,kBAAkB,MAAM,QAAQ,MAAM,KAAK;AAAA,MACvD;AACA,YAAM,YACJ,MAAM,SAAS,aAAa,MAAM,SAAS,WAAW,MAAM,SAAS,aACjE,SAAS,IACT;AACN,mBAAa,SAAS,SAAS,OAAO,WAAW,UAAU;AAC3D,UAAI,YAAY,MAAM,KAAK,EAAG,UAAS,MAAM,UAAU,SAAS,MAAM;AACtE,UAAI,MAAM,SAAS,WAAY;AAC/B,YAAM,SAAS,UAAU,SAAS;AAClC,eAAS;AACT,UAAI,SAAS,SAAU,UAAS,MAAM,WAAW,MAAM;AAAA,UAClD,UAAS,MAAM,UAAU,SAAS,MAAM;AAC7C,eAAS,EAAE,OAAO,SAAS,MAAM,MAAM,KAAK,CAAC;AAC7C,YAAM,OAAO,SAAS,QAAQ,SAAS,UAAU;AACjD,mBAAa;AACb,eAAS,UAAU,SAAS,UAAU,QAAQ,OAAO,MAAM,WAAW,MAAM;AAAA,IAC9E;AAAA,EACF,UAAE;AACA,QAAI,QAAQ,OAAO,WAAW,OAAQ,OAAM,UAAU,SAAS,MAAM;AAAA,aAC5D,OAAQ,OAAM,OAAO;AAC9B,iBAAa;AACb,aAAS,EAAE,OAAO,OAAO,CAAC;AAC1B,YAAQ,IAAI,KAAK;AAAA,EACnB;AACF;AAEA,SAAS,SAAS,SAA2B,YAA+C;AAC1F,SAAO;AAAA,IACL,QAAQ;AAAA,IACR,oBAAoB,QAAQ,KAAK;AAAA,IACjC,GAAI,eAAe,SAAY,CAAC,IAAI,EAAE,WAAW;AAAA,EACnD;AACF;AAEA,SAAS,SACP,SACA,OACA,QACA,YACoB;AACpB,MAAI,MAAM,SAAS,WAAW;AAC5B,YAAQ,SAAS,KAAK;AAAA,MACpB,MAAM;AAAA,MACN;AAAA,MACA,WAAW,MAAM;AAAA,MACjB,MAAM,MAAM;AAAA,IACd,CAAC;AAAA,EACH;AACA,MAAI,MAAM,SAAS,SAAS;AAC1B,YAAQ,SAAS,KAAK;AAAA,MACpB,MAAM;AAAA,MACN;AAAA,MACA,WAAW,MAAM;AAAA,MACjB,MAAM,MAAM;AAAA,MACZ,GAAI,MAAM,eAAe,SAAY,CAAC,IAAI,EAAE,YAAY,MAAM,WAAW;AAAA,IAC3E,CAAC;AACD,WAAO,MAAM,cAAc;AAAA,EAC7B;AACA,MAAI,MAAM,SAAS,YAAY;AAC7B,YAAQ,SAAS,KAAK,EAAE,MAAM,cAAc,QAAQ,WAAW,QAAQ,MAAM,MAAM,KAAK,CAAC;AAAA,EAC3F;AACA,SAAO;AACT;AAEA,eAAe,WAAW,QAAoD;AAC5E,MAAI,CAAC,OAAQ,QAAO;AACpB,QAAM,OAAO;AACb,SAAO;AACT;AAEA,eAAe,eACb,OACA,QACyC;AACzC,QAAM,OAAO,MAAM,QAAQ,KAAK,CAAC,MAAM,KAAK,GAAG,MAAM,CAAC;AACtD,MAAI,KAAK,KAAM,QAAO;AACtB,SAAO,KAAK;AACd;AAEA,SAAS,UACP,SACA,UACA,QACA,OACA,MACA,YACY;AACZ,QAAM,SAAS,WAAW,QAAQ,MAAM;AACxC,QAAM,SAAqB;AAAA,IACzB,YAAY,OAAO;AAAA,IACnB,QAAQ,OAAO;AAAA,IACf;AAAA,IACA,aAAa,GAAG,MAAM;AAAA,IACtB,YAAY;AAAA,IACZ,OAAO,CAAC;AAAA,IACR,MAAM,QAAQ,QAAQ;AAAA,EACxB;AACA,SAAO,OAAO,YAAY,SAAS,UAAU,QAAQ,MAAM,MAAM,MAAM,IAAI,IAAI,EAAE,QAAQ,MAAM;AAC7F,WAAO,OAAO;AAAA,EAChB,CAAC;AACD,OAAK,OAAO,KAAK,MAAM,CAAC,UAAmB,WAAW,KAAK,CAAC;AAC5D,SAAO;AACT;AAEA,eAAe,YACb,SACA,UACA,QACA,MACA,IACA,MACe;AACf,QAAM,SAAS,OAAO,WAAW;AACjC,MAAI;AACF,aAAS,EAAE,OAAO,WAAW,CAAC;AAC9B,UAAM,UAAuB,EAAE,MAAM,QAAQ,SAAS,MAAM,WAAW,GAAG;AAC1E,UAAM,OAAO,SAAS,MAAM;AAC5B,UAAM,OAAoB,CAAC;AAC3B,YAAQ,OAAO;AACf,UAAM,YAAY,UAAU,SAAS,UAAU,QAAQ,SAAS,MAAM,MAAM,IAAI;AAChF,UAAM,QAAQ,QAAQ,OAAO;AAAA,MAC3B,QAAQ,QAAQ;AAAA,MAChB,EAAE,MAAM,KAAK,UAAU,OAAO,QAAQ,QAAQ,MAAM;AAAA,MACpD,EAAE,OAAO;AAAA,IACX;AACA,UAAM,QAAQ,IAAI,CAAC,WAAW,WAAW,SAAS,UAAU,QAAQ,OAAO,IAAI,CAAC,CAAC;AACjF,QAAI,CAAC,OAAO,QAAS,UAAS,EAAE,OAAO,YAAY,CAAC;AAAA,EACtD,SAAS,OAAO;AACd,QAAI,OAAO,WAAW,QAAQ,OAAO,QAAS;AAC9C,UAAM;AAAA,EACR;AACF;AAEA,eAAe,UACb,SACA,UACA,QACA,SACA,MACA,MACA,MACe;AACf,QAAM,QAAQ,kBAAkB;AAChC,MAAI,cAAc;AAClB,MAAI,YAAY;AAChB,MAAI;AACF,qBAAiB,SAAS,QAAQ,aAAa;AAAA,MAC7C;AAAA,MACA;AAAA,MACA,OAAO,WAAW;AAAA,IACpB,GAAG;AACD,UAAI,OAAO,WAAW,OAAO,QAAS;AACtC,gBAAU,UAAU,KAAK;AACzB,iBAAW,SAAS,aAAa,OAAO,OAAO,QAAQ,OAAO,GAAG;AAC/D,uBAAe,MAAM,KAAK;AAC1B,oBAAY;AAAA,UACV;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA,MAAM;AAAA,QACR;AACA,aAAK,KAAK,MAAM,IAAI;AAAA,MACtB;AACA,UAAI,MAAM,QAAS,WAAU,SAAS,QAAQ,MAAM,MAAM,OAAO;AAAA,IACnE;AAAA,EACF,UAAE;AACA,SAAK,IAAI;AAAA,EACX;AACF;AAEA,SAAS,cACP,SACA,QACA,OACA,MACA,WACA,WACA,SACQ;AACR,QAAM,QAAQ,MAAM,WAAW;AAC/B,OAAK,KAAK;AAAA,IACR,aAAa,YAAY,MAAM,KAAK;AAAA,IACpC;AAAA,IACA,cAAc,MAAM;AAAA,IACpB,OAAO,MAAM;AAAA,IACb,KAAK,MAAM;AAAA,IACX,MAAM,MAAM;AAAA,IACZ,GAAI,QAAQ,EAAE,SAAS,MAAM,IAAI,CAAC;AAAA,EACpC,CAAC;AACD,MAAI,MAAM,SAAS,YAAY;AAC7B,YAAQ,SAAS,KAAK,EAAE,MAAM,YAAY,QAAQ,OAAO,QAAQ,MAAM,MAAM,KAAK,CAAC;AACnF,WAAO;AAAA,EACT;AACA,QAAM,OAAO,YAAY,MAAM;AAC/B,UAAQ,SAAS,KAAK;AAAA,IACpB,MAAM;AAAA,IACN,QAAQ,OAAO;AAAA,IACf,aAAa,OAAO;AAAA,IACpB;AAAA,EACF,CAAC;AACD,SAAO;AACT;AAEA,eAAe,WACb,SACA,UACA,QACA,OACA,MACe;AACf,mBAAiB,SAAS,OAAO;AAC/B,QAAI,OAAO,WAAW,OAAO,QAAS;AACtC,QAAI,MAAM,SAAS,SAAS;AAC1B,UAAI,MAAM,WAAW,UAAW;AAChC,YAAM,IAAI,kBAAkB,MAAM,QAAQ,MAAM,KAAK;AAAA,IACvD;AACA,QAAI,MAAM,SAAS,QAAS;AAC5B,aAAS,EAAE,OAAO,WAAW,CAAC;AAC9B,UAAM,OAAO,cAAc,MAAM,MAAM,IAAI;AAC3C,UAAM,MAAgB,EAAE,KAAK;AAC7B,QAAI,MAAM,UAAW,KAAI,YAAY,MAAM;AAC3C,YAAQ,OAAO,IAAI,KAAK,QAAQ,KAAK,CAAC;AACtC,YAAQ,IAAI,MAAM,MAAM,OAAO,GAAG;AAAA,EACpC;AACF;AAEA,SAAS,QAAQ,OAA6D;AAC5E,QAAM,QAAqB;AAAA,IACzB,OAAO,MAAM,KAAK;AAAA,IAClB,KAAK,MAAM,KAAK;AAAA,IAChB,YAAY,gBAAgB,KAAK;AAAA,EACnC;AACA,MAAI,MAAM,UAAW,OAAM,YAAY,MAAM;AAC7C,SAAO;AACT;AAEA,SAAS,gBAAgB,OAAwD;AAC/E,QAAM,WAAW,KAAK,MAAM,aAAa,MAAM,KAAK,IAAI,GAAI;AAC5D,MAAI,WAAW,EAAG,QAAO;AACzB,QAAM,UAAU,gBAAgB,MAAM,SAAS;AAC/C,MAAI,UAAU,EAAG,QAAO;AACxB,QAAM,OAAO,MAAM,KAAK,MAAM,MAAM,KAAK;AACzC,MAAI,OAAO,EAAG,QAAO;AACrB,SAAO;AACT;AAEA,SAAS,gBAAgB,WAA8C;AACrE,MAAI,CAAC,UAAW,QAAO;AACvB,MAAI,KAAK;AACT,aAAW,YAAY,UAAU,WAAY,OAAM;AACnD,SAAO;AACT;AAEA,SAAS,aAAa,SAAkB,QAAoB,KAAe,UAAwB;AACjG,eAAa,QAAQ,KAAK,QAAQ;AAClC,QAAM,UAAU,QAAQ,OAAO,IAAI,GAAG;AACtC,MAAI,QAAS,cAAa,SAAS,QAAQ,SAAS,QAAQ;AAC5D,QAAM,OAAO,cAAc,IAAI,MAAM,QAAQ,IAAI;AACjD,MAAI,EAAE,aAAa,MAAO;AAC1B,SAAO,MAAM;AAAA,IACX,SAAS,KAAK;AAAA,IACd,cAAc,KAAK;AAAA,IACnB,WAAW,aAAa,MAAM,SAAS,QAAQ;AAAA,IAC/C,SAAS,QAAQ,IAAI;AAAA,EACvB;AACF;AAEA,SAAS,aAAa,QAAoB,KAAe,UAAkB;AACzE,QAAM,WAAW,OAAO,MAAM,KAAK,CAAC,SAAS,KAAK,QAAQ,GAAG;AAC7D,MAAI,UAAU;AACZ,aAAS,WAAW;AACpB;AAAA,EACF;AACA,SAAO,MAAM,KAAK,EAAE,KAAK,SAAS,CAAC;AACrC;AAEA,SAAS,aACP,SACA,QACA,SACA,UACA;AACA,QAAM,OAAO,eAAe,aAAa,SAAS,QAAQ,GAAG,QAAQ,IAAI;AACzE,MAAI,QAAQ,OAAO,WAAY;AAC/B,SAAO,aAAa;AACpB,UAAQ,SAAS,KAAK;AAAA,IACpB,MAAM;AAAA,IACN,QAAQ,OAAO;AAAA,IACf,aAAa,OAAO;AAAA,IACpB;AAAA,EACF,CAAC;AACH;AAEA,SAAS,aACP,MACA,OACA,UACQ;AACR,MAAI,CAAC,MAAO,QAAO,KAAK;AACxB,QAAM,SAAS,MAAM,MAAM,MAAM;AACjC,MAAI,UAAU,EAAG,QAAO,KAAK;AAC7B,QAAM,QAAQ,aAAa,OAAO,QAAQ,IAAI,MAAM;AACpD,SAAO,KAAK,QAAQ,KAAK,MAAO,SAAS,KAAK,MAAM,KAAK,SAAU,MAAM;AAC3E;AAEA,SAAS,cAAc,MAAgB,MAA6B;AAClE,MAAI,aAAa,KAAM,QAAO;AAC9B,QAAM,MAAM,UAAU,MAAM,IAAI;AAChC,MAAI,CAAC,KAAK,QAAS,QAAO;AAC1B,QAAM,YAAY,IAAI,YAAY,IAAI;AACtC,QAAM,YAAY,IAAI,MAAM,IAAI;AAChC,QAAM,QAAQ,IAAI,QAAQ,MAAM,KAAK,IAAI,GAAG,KAAK,QAAQ,IAAI,WAAW,GAAG,WAAW,SAAS;AAC/F,QAAM,MACJ,IAAI,QAAQ,MAAM,KAAK,IAAI,WAAW,KAAK,MAAM,IAAI,WAAW,GAAG,WAAW,SAAS;AACzF,SAAO,EAAE,SAAS,IAAI,SAAS,cAAc,IAAI,cAAc,OAAO,IAAI;AAC5E;AAEA,SAAS,UAAU,MAAsC,MAA0C;AACjG,WAAS,QAAQ,KAAK,SAAS,GAAG,SAAS,GAAG,SAAS,GAAG;AACxD,UAAM,OAAO,KAAK,KAAK;AACvB,QAAI,MAAM,SAAS,YAAY,CAAC,KAAK,QAAS;AAC9C,QAAI,KAAK,cAAc,KAAK,OAAO,KAAK,YAAY,KAAK,MAAO,QAAO;AAAA,EACzE;AACA,SAAO;AACT;AAEA,SAAS,MAAM,QAAgB,WAAmB,WAA2B;AAC3E,MAAI,aAAa,EAAG,QAAO;AAC3B,SAAO,KAAK,MAAO,SAAS,YAAa,SAAS;AACpD;AAEA,SAAS,UAAU,SAAkB,QAAoB,MAAmB,SAAuB;AACjG,aAAW,OAAO,MAAM;AACtB,QAAI,IAAI,SAAS,YAAY,IAAI,YAAY,OAAW,KAAI,UAAU;AAAA,EACxE;AACA,aAAW,QAAQ,OAAO,MAAO,cAAa,SAAS,QAAQ,KAAK,KAAK,KAAK,QAAQ;AACxF;AAEA,SAAS,UAAU,UAAuC,OAA8B;AACtF,MAAI,MAAM,SAAS,eAAe,MAAM,SAAS,aAAc,UAAS,EAAE,OAAO,WAAW,CAAC;AAC7F,MAAI,MAAM,SAAS,mBAAoB,UAAS,EAAE,OAAO,aAAa,CAAC;AACvE,MAAI,MAAM,SAAS,sBAAsB,MAAM,YAAY,UAAa,MAAM,OAAO,QAAW;AAC9F,aAAS,EAAE,OAAO,YAAY,SAAS,MAAM,SAAS,IAAI,MAAM,GAAG,CAAC;AAAA,EACtE;AACA,MAAI,MAAM,SAAS,0BAA0B,MAAM,YAAY,MAAM,YAAY;AAC/E,aAAS;AAAA,MACP,OAAO;AAAA,MACP,UAAU,MAAM;AAAA,MAChB,YAAY,MAAM;AAAA,MAClB,MAAM,MAAM,QAAQ,CAAC;AAAA,IACvB,CAAC;AAAA,EACH;AACF;AAEA,eAAe,UAAU,SAAkB,QAAoD;AAC7F,MAAI,CAAC,OAAQ,QAAO;AACpB,SAAO,WAAW,MAAM;AACxB,UAAQ,IAAI,MAAM;AAClB,QAAM,OAAO;AACb,MAAI,OAAO,KAAK;AACd,YAAQ,SAAS,KAAK;AAAA,MACpB,MAAM;AAAA,MACN,QAAQ,OAAO;AAAA,MACf,aAAa,OAAO;AAAA,MACpB,WAAW,OAAO,IAAI;AAAA,IACxB,CAAC;AACD,UAAM,QAAQ,aAAa,SAAS,OAAO,GAAG;AAAA,EAChD;AACA,SAAO;AACT;AAEA,SAAS,UAAU,SAA+D;AAChF,QAAM,OAAO,QAAQ,WAAW;AAChC,MAAI,SAAS,OAAQ,QAAO;AAC5B,MAAI,QAAQ,uBAAuB,YAAa,QAAO;AACvD,MAAI,QAAQ,uBAAuB,aAAc,QAAO;AACxD,SAAO;AACT;AAEA,SAAS,YAAY,MAA0C,OAAoC;AACjG,MAAI,SAAS,SAAU,QAAO;AAC9B,MAAI,SAAS,YAAa,QAAO,MAAM,SAAS;AAChD,SAAO,MAAM,SAAS,aAAa,MAAM,KAAK,KAAK,EAAE,SAAS;AAChE;AAEA,SAAS,SAAS,SAA2B,KAA+B;AAC1E,QAAM,OAAO,QAAQ;AACrB,MAAI,SAAS,OAAW,QAAO;AAC/B,SAAO,YAAY,MAAM,GAAG;AAC9B;AAEA,SAAS,SAAS,KAAsC;AACtD,MAAI,WAAW,GAAG,EAAG,QAAO,IAAI;AAChC,SAAO;AACT;AAEA,SAAS,UAAU,KAA2B,YAAiD;AAC7F,MAAI,CAAC,WAAY,QAAO,QAAQ;AAChC,MAAI,CAAC,WAAW,GAAG,EAAG,QAAO,QAAQ;AACrC,SAAO,UAAU,IAAI,KAAK,UAAU;AACtC;AAEA,SAAS,UAAuB;AAC9B,SAAO;AAAA,IACL,UAAU;AAAA,IAAC;AAAA,IACX,OAAO;AAAA,IAAC;AAAA,EACV;AACF;AAEA,SAAS,YAAY,QAA2B,KAA+C;AAC7F,MAAI,WAAW;AACf,SAAO,CAAC,UAAU;AAChB,UAAM,MAAM,KAAK,UAAU,KAAK;AAChC,QAAI,QAAQ,SAAU;AACtB,eAAW;AACX,WAAO,KAAK,KAAK;AACjB,QAAI,QAAQ,KAAK;AAAA,EACnB;AACF;AAEA,SAAS,WAAW,QAA0E;AAC5F,QAAM,aAAa,IAAI,gBAAgB;AACvC,QAAM,QAAQ,MAAM,WAAW,MAAM;AACrC,MAAI,OAAO,QAAS,YAAW,MAAM;AAAA,MAChC,QAAO,iBAAiB,SAAS,KAAK;AAC3C,SAAO;AAAA,IACL;AAAA,IACA,QAAQ,MAAM,OAAO,oBAAoB,SAAS,KAAK;AAAA,EACzD;AACF;AAQA,SAAS,SAAS,QAA+B;AAC/C,QAAM,QAAkB,CAAC;AACzB,MAAI,QAAQ;AACZ,MAAI;AACJ,QAAM,SAAS,MAAM;AACnB,UAAM,UAAU;AAChB,WAAO;AACP,cAAU;AAAA,EACZ;AACA,QAAM,UAAU,MAAM,OAAO;AAC7B,SAAO,iBAAiB,SAAS,OAAO;AACxC,SAAO;AAAA,IACL,KAAK,MAAM;AACT,UAAI,SAAS,OAAO,WAAW,KAAK,WAAW,EAAG;AAClD,YAAM,KAAK,IAAI;AACf,aAAO;AAAA,IACT;AAAA,IACA,MAAM;AACJ,cAAQ;AACR,aAAO;AAAA,IACT;AAAA,IACA,UAAU;AAAA,MACR,QAAQ,OAAO,aAAa,IAAI;AAC9B,YAAI;AACF,iBAAO,CAAC,OAAO,SAAS;AACtB,kBAAM,OAAO,MAAM,MAAM;AACzB,gBAAI,SAAS,QAAW;AACtB,oBAAM;AACN;AAAA,YACF;AACA,gBAAI,MAAO;AACX,kBAAM,IAAI,QAAc,CAAC,YAAY;AACnC,kBAAI,MAAM,SAAS,KAAK,SAAS,OAAO,QAAS,SAAQ;AAAA,kBACpD,QAAO;AAAA,YACd,CAAC;AAAA,UACH;AAAA,QACF,UAAE;AACA,iBAAO,oBAAoB,SAAS,OAAO;AAAA,QAC7C;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function registerGoogle(): () => void;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { registerGoogle } from './catalog.ts';
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { ProviderRequestOptions } from '@earendil-works/pi-ai';
|
|
2
|
+
import type { SpeechModel } from '../models.ts';
|
|
3
|
+
import { SpeechEventStream, type SpeechInput } from '../speech.ts';
|
|
4
|
+
export declare function speakGoogle(model: SpeechModel<string>, input: SpeechInput, options?: ProviderRequestOptions): SpeechEventStream;
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { ProviderRequestOptions } from '@earendil-works/pi-ai';
|
|
2
|
+
import type { AudioFormat } from '../audio.ts';
|
|
3
|
+
export declare function speechMime(format: AudioFormat): string;
|
|
4
|
+
export declare function sameFormat(left: AudioFormat, right: AudioFormat): boolean;
|
|
5
|
+
export declare function googleHeaders(model: {
|
|
6
|
+
provider: string;
|
|
7
|
+
headers?: Record<string, string>;
|
|
8
|
+
}, options: ProviderRequestOptions | undefined): Record<string, string>;
|
package/dist/vad-listen.js
CHANGED
|
@@ -71,6 +71,7 @@ function closeTurn(session, at) {
|
|
|
71
71
|
session.pendingEnd = void 0;
|
|
72
72
|
session.heldEnd = void 0;
|
|
73
73
|
session.providerTurn = void 0;
|
|
74
|
+
session.sawFinal = false;
|
|
74
75
|
session.stream.push({ type: "turn_end", text, at });
|
|
75
76
|
}
|
|
76
77
|
function openTurn(session, at) {
|
|
@@ -79,6 +80,7 @@ function openTurn(session, at) {
|
|
|
79
80
|
session.started = true;
|
|
80
81
|
session.turnOpen = true;
|
|
81
82
|
session.pendingStart = void 0;
|
|
83
|
+
session.sawFinal = false;
|
|
82
84
|
}
|
|
83
85
|
function applyEvent(session, event) {
|
|
84
86
|
if (event.type === "speech_start") {
|
|
@@ -95,6 +97,7 @@ function applyEvent(session, event) {
|
|
|
95
97
|
}
|
|
96
98
|
if (event.type === "partial" || event.type === "final") {
|
|
97
99
|
if (!session.started) openTurn(session, session.pendingStart ?? session.lastAt);
|
|
100
|
+
if (event.type === "final") session.sawFinal = true;
|
|
98
101
|
session.segments.set(event.segmentId, event.text);
|
|
99
102
|
session.stream.push(event);
|
|
100
103
|
return;
|
|
@@ -106,8 +109,21 @@ function applyEvent(session, event) {
|
|
|
106
109
|
}
|
|
107
110
|
if (event.type === "done") session.pendingDone = event;
|
|
108
111
|
}
|
|
112
|
+
function turnReady(session) {
|
|
113
|
+
if (session.sawFinal) return true;
|
|
114
|
+
if (session.providerTurn !== void 0) return true;
|
|
115
|
+
if (session.pendingDone) return true;
|
|
116
|
+
return false;
|
|
117
|
+
}
|
|
118
|
+
function holdForNextSpeech(session, events, update) {
|
|
119
|
+
if (session.heldEnd === void 0 || !session.turnOpen) return;
|
|
120
|
+
const started = update?.speechStart !== void 0 || events.some((event) => event.type === "speech_start");
|
|
121
|
+
if (!started) return;
|
|
122
|
+
closeTurn(session, session.heldEnd);
|
|
123
|
+
}
|
|
109
124
|
function absorb(session, events, update) {
|
|
110
125
|
if (session.stopped) return;
|
|
126
|
+
holdForNextSpeech(session, events, update);
|
|
111
127
|
if (update?.speechStart !== void 0) session.pendingStart = update.speechStart;
|
|
112
128
|
if (update?.turnEnd !== void 0) session.pendingEnd = update.turnEnd;
|
|
113
129
|
const textual = events.some(
|
|
@@ -124,7 +140,21 @@ function absorb(session, events, update) {
|
|
|
124
140
|
session.heldEnd = session.pendingEnd;
|
|
125
141
|
session.pendingEnd = void 0;
|
|
126
142
|
}
|
|
127
|
-
if (session.heldEnd !== void 0) closeTurn(session, session.heldEnd);
|
|
143
|
+
if (session.heldEnd !== void 0 && turnReady(session)) closeTurn(session, session.heldEnd);
|
|
144
|
+
}
|
|
145
|
+
async function finishWhenClosed(session, options, stream, closed, take) {
|
|
146
|
+
while (!closed() && !session.stopped) {
|
|
147
|
+
if (options?.signal?.aborted) {
|
|
148
|
+
stream.push({ type: "error", reason: "aborted", error: "aborted" });
|
|
149
|
+
session.stopped = true;
|
|
150
|
+
return;
|
|
151
|
+
}
|
|
152
|
+
await tick();
|
|
153
|
+
take();
|
|
154
|
+
}
|
|
155
|
+
if (session.stopped) return;
|
|
156
|
+
take();
|
|
157
|
+
finish(session);
|
|
128
158
|
}
|
|
129
159
|
function finish(session) {
|
|
130
160
|
if (session.stopped) return;
|
|
@@ -161,8 +191,14 @@ async function pump(listenModel, model, audio, options, stream, vad) {
|
|
|
161
191
|
const inbox = [];
|
|
162
192
|
let readerClosed = false;
|
|
163
193
|
const reading = (async () => {
|
|
164
|
-
|
|
165
|
-
|
|
194
|
+
try {
|
|
195
|
+
for await (const event of inner) inbox.push(event);
|
|
196
|
+
} catch (error) {
|
|
197
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
198
|
+
inbox.push({ type: "error", reason: "error", error: message });
|
|
199
|
+
} finally {
|
|
200
|
+
readerClosed = true;
|
|
201
|
+
}
|
|
166
202
|
})();
|
|
167
203
|
const session = {
|
|
168
204
|
stream,
|
|
@@ -170,6 +206,7 @@ async function pump(listenModel, model, audio, options, stream, vad) {
|
|
|
170
206
|
closed: "",
|
|
171
207
|
turnOpen: false,
|
|
172
208
|
started: false,
|
|
209
|
+
sawFinal: false,
|
|
173
210
|
stopped: false,
|
|
174
211
|
lastAt: 0
|
|
175
212
|
};
|
|
@@ -199,15 +236,9 @@ async function pump(listenModel, model, audio, options, stream, vad) {
|
|
|
199
236
|
take(update);
|
|
200
237
|
}
|
|
201
238
|
gate.push(null);
|
|
202
|
-
if (!session.stopped)
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
await tick();
|
|
206
|
-
take();
|
|
207
|
-
attempt += 1;
|
|
208
|
-
} while (!readerClosed && attempt < 50);
|
|
209
|
-
finish(session);
|
|
210
|
-
} else await tick();
|
|
239
|
+
if (!session.stopped)
|
|
240
|
+
await finishWhenClosed(session, options, stream, () => readerClosed, take);
|
|
241
|
+
else await tick();
|
|
211
242
|
} catch (error) {
|
|
212
243
|
gate.push(null);
|
|
213
244
|
if (!session.stopped) {
|
package/dist/vad-listen.js.map
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../src/vad-listen.ts"],
|
|
4
|
-
"sourcesContent": ["import type { AudioChunk, AudioSource } from './audio.ts';\nimport type { TranscriptionModel } from './models.ts';\nimport {\n type ListenOptions,\n type TranscriptionEvent,\n TranscriptionEventStream,\n} from './transcription.ts';\nimport { energyVad, type VadHook, type VadUpdate } from './vad.ts';\n\ntype ListenModel = (\n model: TranscriptionModel<string>,\n audio: AudioSource,\n options?: ListenOptions\n) => TranscriptionEventStream;\n\ninterface Gate {\n source: AudioSource;\n push(chunk: AudioChunk | null): void;\n}\n\nfunction tick(): Promise<void> {\n return new Promise((resolve) => {\n setTimeout(resolve, 0);\n });\n}\n\nfunction audioGate(): Gate {\n const queued: { chunk: AudioChunk | null }[] = [];\n let resolveWait: ((chunk: AudioChunk | null) => void) | undefined;\n const source: AudioSource = {\n async *[Symbol.asyncIterator]() {\n while (true) {\n const ready = queued.shift();\n if (ready !== undefined) {\n if (ready.chunk === null) return;\n yield ready.chunk;\n continue;\n }\n const chunk = await new Promise<AudioChunk | null>((resolve) => {\n resolveWait = resolve;\n });\n if (chunk === null) return;\n yield chunk;\n }\n },\n };\n return {\n source,\n push(chunk) {\n if (resolveWait) {\n const resolve = resolveWait;\n resolveWait = undefined;\n resolve(chunk);\n return;\n }\n queued.push({ chunk });\n },\n };\n}\n\nfunction joinSegments(parts: string[]): string {\n let text = '';\n for (const part of parts) {\n if (text.length === 0 || /\\s$/.test(text) || /^\\s/.test(part)) text += part;\n else text += ` ${part}`;\n }\n return text;\n}\n\ninterface Session {\n stream: TranscriptionEventStream;\n segments: Map<string, string>;\n closed: string;\n turnOpen: boolean;\n started: boolean;\n pendingStart?: number;\n pendingEnd?: number;\n heldEnd?: number;\n providerTurn?: string;\n pendingDone?: Extract<TranscriptionEvent, { type: 'done' }>;\n stopped: boolean;\n lastAt: number;\n}\n\nfunction liveText(session: Session): string {\n const parts = [...session.segments.values()];\n if (parts.length > 0) return joinSegments(parts);\n return session.pendingDone?.transcript.text ?? session.providerTurn ?? '';\n}\n\nfunction turnText(session: Session): string {\n const live = liveText(session);\n if (session.closed.length > 0 && live.startsWith(session.closed)) {\n return live.slice(session.closed.length).trim();\n }\n if (session.providerTurn && !live.startsWith(session.providerTurn)) return session.providerTurn;\n return live.trim();\n}\n\nfunction closeTurn(session: Session, at: number) {\n if (!session.turnOpen || session.stopped) return;\n const text = turnText(session);\n if (text.length === 0) return;\n session.closed = liveText(session);\n session.turnOpen = false;\n session.started = false;\n session.pendingEnd = undefined;\n session.heldEnd = undefined;\n session.providerTurn = undefined;\n session.stream.push({ type: 'turn_end', text, at });\n}\n\nfunction openTurn(session: Session, at: number) {\n if (session.started || session.stopped) return;\n session.stream.push({ type: 'speech_start', at });\n session.started = true;\n session.turnOpen = true;\n session.pendingStart = undefined;\n}\n\nfunction applyEvent(session: Session, event: TranscriptionEvent) {\n if (event.type === 'speech_start') {\n if (!session.started) session.stream.push(event);\n session.started = true;\n session.turnOpen = true;\n session.pendingStart = undefined;\n return;\n }\n if (event.type === 'turn_end') {\n session.providerTurn = event.text;\n session.pendingEnd = event.at;\n return;\n }\n if (event.type === 'partial' || event.type === 'final') {\n if (!session.started) openTurn(session, session.pendingStart ?? session.lastAt);\n session.segments.set(event.segmentId, event.text);\n session.stream.push(event);\n return;\n }\n if (event.type === 'error') {\n session.stream.push(event);\n session.stopped = true;\n return;\n }\n if (event.type === 'done') session.pendingDone = event;\n}\n\nfunction absorb(session: Session, events: TranscriptionEvent[], update: VadUpdate | undefined) {\n if (session.stopped) return;\n if (update?.speechStart !== undefined) session.pendingStart = update.speechStart;\n if (update?.turnEnd !== undefined) session.pendingEnd = update.turnEnd;\n const textual = events.some(\n (event) => event.type === 'partial' || event.type === 'final' || event.type === 'done'\n );\n if (session.pendingStart !== undefined && (textual || events.length === 0)) {\n openTurn(session, session.pendingStart);\n }\n for (const event of events) {\n applyEvent(session, event);\n if (session.stopped) return;\n }\n if (session.pendingEnd !== undefined) {\n session.heldEnd = session.pendingEnd;\n session.pendingEnd = undefined;\n }\n if (session.heldEnd !== undefined) closeTurn(session, session.heldEnd);\n}\n\nfunction finish(session: Session) {\n if (session.stopped) return;\n if (session.pendingStart !== undefined) openTurn(session, session.pendingStart);\n if (session.turnOpen) closeTurn(session, session.heldEnd ?? session.pendingEnd ?? session.lastAt);\n if (session.pendingDone) session.stream.push(session.pendingDone);\n else {\n session.stream.push({\n type: 'error',\n reason: 'error',\n error: 'Transcription ended without a transcript',\n });\n }\n}\n\nexport function listenWithVad(\n listenModel: ListenModel,\n model: TranscriptionModel<string>,\n audio: AudioSource,\n options?: ListenOptions,\n vad: VadHook = options?.vad ?? energyVad()\n): TranscriptionEventStream {\n const stream = new TranscriptionEventStream();\n void pump(listenModel, model, audio, options, stream, vad);\n return stream;\n}\n\nasync function pump(\n listenModel: ListenModel,\n model: TranscriptionModel<string>,\n audio: AudioSource,\n options: ListenOptions | undefined,\n stream: TranscriptionEventStream,\n vad: VadHook\n) {\n if (options?.signal?.aborted) {\n stream.push({ type: 'error', reason: 'aborted', error: 'aborted' });\n return;\n }\n const gate = audioGate();\n let inner: TranscriptionEventStream;\n try {\n inner = listenModel(model, gate.source, options);\n } catch (error) {\n const message = error instanceof Error ? error.message : String(error);\n stream.push({ type: 'error', reason: 'error', error: message });\n return;\n }\n const inbox: TranscriptionEvent[] = [];\n let readerClosed = false;\n const reading = (async () => {\n for await (const event of inner) inbox.push(event);\n readerClosed = true;\n })();\n const session: Session = {\n stream,\n segments: new Map(),\n closed: '',\n turnOpen: false,\n started: false,\n stopped: false,\n lastAt: 0,\n };\n const take = (update?: VadUpdate) => {\n const events = inbox.splice(0, inbox.length);\n absorb(session, events, update);\n };\n try {\n await tick();\n take();\n for await (const chunk of audio) {\n if (session.stopped) break;\n if (options?.signal?.aborted) {\n gate.push(null);\n await tick();\n take();\n if (!session.stopped) {\n stream.push({ type: 'error', reason: 'aborted', error: 'aborted' });\n session.stopped = true;\n }\n break;\n }\n session.lastAt = chunk.timestamp;\n const update = vad.push(chunk);\n gate.push(chunk);\n await tick();\n take(update);\n }\n gate.push(null);\n if (!session.stopped)
|
|
5
|
-
"mappings": "AAEA;AAAA,EAGE;AAAA,OACK;AACP,SAAS,iBAA+C;AAaxD,SAAS,OAAsB;AAC7B,SAAO,IAAI,QAAQ,CAAC,YAAY;AAC9B,eAAW,SAAS,CAAC;AAAA,EACvB,CAAC;AACH;AAEA,SAAS,YAAkB;AACzB,QAAM,SAAyC,CAAC;AAChD,MAAI;AACJ,QAAM,SAAsB;AAAA,IAC1B,QAAQ,OAAO,aAAa,IAAI;AAC9B,aAAO,MAAM;AACX,cAAM,QAAQ,OAAO,MAAM;AAC3B,YAAI,UAAU,QAAW;AACvB,cAAI,MAAM,UAAU,KAAM;AAC1B,gBAAM,MAAM;AACZ;AAAA,QACF;AACA,cAAM,QAAQ,MAAM,IAAI,QAA2B,CAAC,YAAY;AAC9D,wBAAc;AAAA,QAChB,CAAC;AACD,YAAI,UAAU,KAAM;AACpB,cAAM;AAAA,MACR;AAAA,IACF;AAAA,EACF;AACA,SAAO;AAAA,IACL;AAAA,IACA,KAAK,OAAO;AACV,UAAI,aAAa;AACf,cAAM,UAAU;AAChB,sBAAc;AACd,gBAAQ,KAAK;AACb;AAAA,MACF;AACA,aAAO,KAAK,EAAE,MAAM,CAAC;AAAA,IACvB;AAAA,EACF;AACF;AAEA,SAAS,aAAa,OAAyB;AAC7C,MAAI,OAAO;AACX,aAAW,QAAQ,OAAO;AACxB,QAAI,KAAK,WAAW,KAAK,MAAM,KAAK,IAAI,KAAK,MAAM,KAAK,IAAI,EAAG,SAAQ;AAAA,QAClE,SAAQ,IAAI,IAAI;AAAA,EACvB;AACA,SAAO;AACT;
|
|
4
|
+
"sourcesContent": ["import type { AudioChunk, AudioSource } from './audio.ts';\nimport type { TranscriptionModel } from './models.ts';\nimport {\n type ListenOptions,\n type TranscriptionEvent,\n TranscriptionEventStream,\n} from './transcription.ts';\nimport { energyVad, type VadHook, type VadUpdate } from './vad.ts';\n\ntype ListenModel = (\n model: TranscriptionModel<string>,\n audio: AudioSource,\n options?: ListenOptions\n) => TranscriptionEventStream;\n\ninterface Gate {\n source: AudioSource;\n push(chunk: AudioChunk | null): void;\n}\n\nfunction tick(): Promise<void> {\n return new Promise((resolve) => {\n setTimeout(resolve, 0);\n });\n}\n\nfunction audioGate(): Gate {\n const queued: { chunk: AudioChunk | null }[] = [];\n let resolveWait: ((chunk: AudioChunk | null) => void) | undefined;\n const source: AudioSource = {\n async *[Symbol.asyncIterator]() {\n while (true) {\n const ready = queued.shift();\n if (ready !== undefined) {\n if (ready.chunk === null) return;\n yield ready.chunk;\n continue;\n }\n const chunk = await new Promise<AudioChunk | null>((resolve) => {\n resolveWait = resolve;\n });\n if (chunk === null) return;\n yield chunk;\n }\n },\n };\n return {\n source,\n push(chunk) {\n if (resolveWait) {\n const resolve = resolveWait;\n resolveWait = undefined;\n resolve(chunk);\n return;\n }\n queued.push({ chunk });\n },\n };\n}\n\nfunction joinSegments(parts: string[]): string {\n let text = '';\n for (const part of parts) {\n if (text.length === 0 || /\\s$/.test(text) || /^\\s/.test(part)) text += part;\n else text += ` ${part}`;\n }\n return text;\n}\n\ninterface Session {\n stream: TranscriptionEventStream;\n segments: Map<string, string>;\n closed: string;\n turnOpen: boolean;\n started: boolean;\n pendingStart?: number;\n pendingEnd?: number;\n heldEnd?: number;\n providerTurn?: string;\n pendingDone?: Extract<TranscriptionEvent, { type: 'done' }>;\n sawFinal: boolean;\n stopped: boolean;\n lastAt: number;\n}\n\nfunction liveText(session: Session): string {\n const parts = [...session.segments.values()];\n if (parts.length > 0) return joinSegments(parts);\n return session.pendingDone?.transcript.text ?? session.providerTurn ?? '';\n}\n\nfunction turnText(session: Session): string {\n const live = liveText(session);\n if (session.closed.length > 0 && live.startsWith(session.closed)) {\n return live.slice(session.closed.length).trim();\n }\n if (session.providerTurn && !live.startsWith(session.providerTurn)) return session.providerTurn;\n return live.trim();\n}\n\nfunction closeTurn(session: Session, at: number) {\n if (!session.turnOpen || session.stopped) return;\n const text = turnText(session);\n if (text.length === 0) return;\n session.closed = liveText(session);\n session.turnOpen = false;\n session.started = false;\n session.pendingEnd = undefined;\n session.heldEnd = undefined;\n session.providerTurn = undefined;\n session.sawFinal = false;\n session.stream.push({ type: 'turn_end', text, at });\n}\n\nfunction openTurn(session: Session, at: number) {\n if (session.started || session.stopped) return;\n session.stream.push({ type: 'speech_start', at });\n session.started = true;\n session.turnOpen = true;\n session.pendingStart = undefined;\n session.sawFinal = false;\n}\n\nfunction applyEvent(session: Session, event: TranscriptionEvent) {\n if (event.type === 'speech_start') {\n if (!session.started) session.stream.push(event);\n session.started = true;\n session.turnOpen = true;\n session.pendingStart = undefined;\n return;\n }\n if (event.type === 'turn_end') {\n session.providerTurn = event.text;\n session.pendingEnd = event.at;\n return;\n }\n if (event.type === 'partial' || event.type === 'final') {\n if (!session.started) openTurn(session, session.pendingStart ?? session.lastAt);\n if (event.type === 'final') session.sawFinal = true;\n session.segments.set(event.segmentId, event.text);\n session.stream.push(event);\n return;\n }\n if (event.type === 'error') {\n session.stream.push(event);\n session.stopped = true;\n return;\n }\n if (event.type === 'done') session.pendingDone = event;\n}\n\nfunction turnReady(session: Session): boolean {\n if (session.sawFinal) return true;\n if (session.providerTurn !== undefined) return true;\n if (session.pendingDone) return true;\n return false;\n}\n\nfunction holdForNextSpeech(\n session: Session,\n events: TranscriptionEvent[],\n update: VadUpdate | undefined\n) {\n if (session.heldEnd === undefined || !session.turnOpen) return;\n const started =\n update?.speechStart !== undefined || events.some((event) => event.type === 'speech_start');\n if (!started) return;\n closeTurn(session, session.heldEnd);\n}\n\nfunction absorb(session: Session, events: TranscriptionEvent[], update: VadUpdate | undefined) {\n if (session.stopped) return;\n holdForNextSpeech(session, events, update);\n if (update?.speechStart !== undefined) session.pendingStart = update.speechStart;\n if (update?.turnEnd !== undefined) session.pendingEnd = update.turnEnd;\n const textual = events.some(\n (event) => event.type === 'partial' || event.type === 'final' || event.type === 'done'\n );\n if (session.pendingStart !== undefined && (textual || events.length === 0)) {\n openTurn(session, session.pendingStart);\n }\n for (const event of events) {\n applyEvent(session, event);\n if (session.stopped) return;\n }\n if (session.pendingEnd !== undefined) {\n session.heldEnd = session.pendingEnd;\n session.pendingEnd = undefined;\n }\n if (session.heldEnd !== undefined && turnReady(session)) closeTurn(session, session.heldEnd);\n}\n\nasync function finishWhenClosed(\n session: Session,\n options: ListenOptions | undefined,\n stream: TranscriptionEventStream,\n closed: () => boolean,\n take: () => void\n) {\n while (!closed() && !session.stopped) {\n if (options?.signal?.aborted) {\n stream.push({ type: 'error', reason: 'aborted', error: 'aborted' });\n session.stopped = true;\n return;\n }\n await tick();\n take();\n }\n if (session.stopped) return;\n take();\n finish(session);\n}\n\nfunction finish(session: Session) {\n if (session.stopped) return;\n if (session.pendingStart !== undefined) openTurn(session, session.pendingStart);\n if (session.turnOpen) closeTurn(session, session.heldEnd ?? session.pendingEnd ?? session.lastAt);\n if (session.pendingDone) session.stream.push(session.pendingDone);\n else {\n session.stream.push({\n type: 'error',\n reason: 'error',\n error: 'Transcription ended without a transcript',\n });\n }\n}\n\nexport function listenWithVad(\n listenModel: ListenModel,\n model: TranscriptionModel<string>,\n audio: AudioSource,\n options?: ListenOptions,\n vad: VadHook = options?.vad ?? energyVad()\n): TranscriptionEventStream {\n const stream = new TranscriptionEventStream();\n void pump(listenModel, model, audio, options, stream, vad);\n return stream;\n}\n\nasync function pump(\n listenModel: ListenModel,\n model: TranscriptionModel<string>,\n audio: AudioSource,\n options: ListenOptions | undefined,\n stream: TranscriptionEventStream,\n vad: VadHook\n) {\n if (options?.signal?.aborted) {\n stream.push({ type: 'error', reason: 'aborted', error: 'aborted' });\n return;\n }\n const gate = audioGate();\n let inner: TranscriptionEventStream;\n try {\n inner = listenModel(model, gate.source, options);\n } catch (error) {\n const message = error instanceof Error ? error.message : String(error);\n stream.push({ type: 'error', reason: 'error', error: message });\n return;\n }\n const inbox: TranscriptionEvent[] = [];\n let readerClosed = false;\n const reading = (async () => {\n try {\n for await (const event of inner) inbox.push(event);\n } catch (error) {\n const message = error instanceof Error ? error.message : String(error);\n inbox.push({ type: 'error', reason: 'error', error: message });\n } finally {\n readerClosed = true;\n }\n })();\n const session: Session = {\n stream,\n segments: new Map(),\n closed: '',\n turnOpen: false,\n started: false,\n sawFinal: false,\n stopped: false,\n lastAt: 0,\n };\n const take = (update?: VadUpdate) => {\n const events = inbox.splice(0, inbox.length);\n absorb(session, events, update);\n };\n try {\n await tick();\n take();\n for await (const chunk of audio) {\n if (session.stopped) break;\n if (options?.signal?.aborted) {\n gate.push(null);\n await tick();\n take();\n if (!session.stopped) {\n stream.push({ type: 'error', reason: 'aborted', error: 'aborted' });\n session.stopped = true;\n }\n break;\n }\n session.lastAt = chunk.timestamp;\n const update = vad.push(chunk);\n gate.push(chunk);\n await tick();\n take(update);\n }\n gate.push(null);\n if (!session.stopped)\n await finishWhenClosed(session, options, stream, () => readerClosed, take);\n else await tick();\n } catch (error) {\n gate.push(null);\n if (!session.stopped) {\n const message = error instanceof Error ? error.message : String(error);\n stream.push({ type: 'error', reason: 'error', error: message });\n }\n }\n await reading;\n}\n"],
|
|
5
|
+
"mappings": "AAEA;AAAA,EAGE;AAAA,OACK;AACP,SAAS,iBAA+C;AAaxD,SAAS,OAAsB;AAC7B,SAAO,IAAI,QAAQ,CAAC,YAAY;AAC9B,eAAW,SAAS,CAAC;AAAA,EACvB,CAAC;AACH;AAEA,SAAS,YAAkB;AACzB,QAAM,SAAyC,CAAC;AAChD,MAAI;AACJ,QAAM,SAAsB;AAAA,IAC1B,QAAQ,OAAO,aAAa,IAAI;AAC9B,aAAO,MAAM;AACX,cAAM,QAAQ,OAAO,MAAM;AAC3B,YAAI,UAAU,QAAW;AACvB,cAAI,MAAM,UAAU,KAAM;AAC1B,gBAAM,MAAM;AACZ;AAAA,QACF;AACA,cAAM,QAAQ,MAAM,IAAI,QAA2B,CAAC,YAAY;AAC9D,wBAAc;AAAA,QAChB,CAAC;AACD,YAAI,UAAU,KAAM;AACpB,cAAM;AAAA,MACR;AAAA,IACF;AAAA,EACF;AACA,SAAO;AAAA,IACL;AAAA,IACA,KAAK,OAAO;AACV,UAAI,aAAa;AACf,cAAM,UAAU;AAChB,sBAAc;AACd,gBAAQ,KAAK;AACb;AAAA,MACF;AACA,aAAO,KAAK,EAAE,MAAM,CAAC;AAAA,IACvB;AAAA,EACF;AACF;AAEA,SAAS,aAAa,OAAyB;AAC7C,MAAI,OAAO;AACX,aAAW,QAAQ,OAAO;AACxB,QAAI,KAAK,WAAW,KAAK,MAAM,KAAK,IAAI,KAAK,MAAM,KAAK,IAAI,EAAG,SAAQ;AAAA,QAClE,SAAQ,IAAI,IAAI;AAAA,EACvB;AACA,SAAO;AACT;AAkBA,SAAS,SAAS,SAA0B;AAC1C,QAAM,QAAQ,CAAC,GAAG,QAAQ,SAAS,OAAO,CAAC;AAC3C,MAAI,MAAM,SAAS,EAAG,QAAO,aAAa,KAAK;AAC/C,SAAO,QAAQ,aAAa,WAAW,QAAQ,QAAQ,gBAAgB;AACzE;AAEA,SAAS,SAAS,SAA0B;AAC1C,QAAM,OAAO,SAAS,OAAO;AAC7B,MAAI,QAAQ,OAAO,SAAS,KAAK,KAAK,WAAW,QAAQ,MAAM,GAAG;AAChE,WAAO,KAAK,MAAM,QAAQ,OAAO,MAAM,EAAE,KAAK;AAAA,EAChD;AACA,MAAI,QAAQ,gBAAgB,CAAC,KAAK,WAAW,QAAQ,YAAY,EAAG,QAAO,QAAQ;AACnF,SAAO,KAAK,KAAK;AACnB;AAEA,SAAS,UAAU,SAAkB,IAAY;AAC/C,MAAI,CAAC,QAAQ,YAAY,QAAQ,QAAS;AAC1C,QAAM,OAAO,SAAS,OAAO;AAC7B,MAAI,KAAK,WAAW,EAAG;AACvB,UAAQ,SAAS,SAAS,OAAO;AACjC,UAAQ,WAAW;AACnB,UAAQ,UAAU;AAClB,UAAQ,aAAa;AACrB,UAAQ,UAAU;AAClB,UAAQ,eAAe;AACvB,UAAQ,WAAW;AACnB,UAAQ,OAAO,KAAK,EAAE,MAAM,YAAY,MAAM,GAAG,CAAC;AACpD;AAEA,SAAS,SAAS,SAAkB,IAAY;AAC9C,MAAI,QAAQ,WAAW,QAAQ,QAAS;AACxC,UAAQ,OAAO,KAAK,EAAE,MAAM,gBAAgB,GAAG,CAAC;AAChD,UAAQ,UAAU;AAClB,UAAQ,WAAW;AACnB,UAAQ,eAAe;AACvB,UAAQ,WAAW;AACrB;AAEA,SAAS,WAAW,SAAkB,OAA2B;AAC/D,MAAI,MAAM,SAAS,gBAAgB;AACjC,QAAI,CAAC,QAAQ,QAAS,SAAQ,OAAO,KAAK,KAAK;AAC/C,YAAQ,UAAU;AAClB,YAAQ,WAAW;AACnB,YAAQ,eAAe;AACvB;AAAA,EACF;AACA,MAAI,MAAM,SAAS,YAAY;AAC7B,YAAQ,eAAe,MAAM;AAC7B,YAAQ,aAAa,MAAM;AAC3B;AAAA,EACF;AACA,MAAI,MAAM,SAAS,aAAa,MAAM,SAAS,SAAS;AACtD,QAAI,CAAC,QAAQ,QAAS,UAAS,SAAS,QAAQ,gBAAgB,QAAQ,MAAM;AAC9E,QAAI,MAAM,SAAS,QAAS,SAAQ,WAAW;AAC/C,YAAQ,SAAS,IAAI,MAAM,WAAW,MAAM,IAAI;AAChD,YAAQ,OAAO,KAAK,KAAK;AACzB;AAAA,EACF;AACA,MAAI,MAAM,SAAS,SAAS;AAC1B,YAAQ,OAAO,KAAK,KAAK;AACzB,YAAQ,UAAU;AAClB;AAAA,EACF;AACA,MAAI,MAAM,SAAS,OAAQ,SAAQ,cAAc;AACnD;AAEA,SAAS,UAAU,SAA2B;AAC5C,MAAI,QAAQ,SAAU,QAAO;AAC7B,MAAI,QAAQ,iBAAiB,OAAW,QAAO;AAC/C,MAAI,QAAQ,YAAa,QAAO;AAChC,SAAO;AACT;AAEA,SAAS,kBACP,SACA,QACA,QACA;AACA,MAAI,QAAQ,YAAY,UAAa,CAAC,QAAQ,SAAU;AACxD,QAAM,UACJ,QAAQ,gBAAgB,UAAa,OAAO,KAAK,CAAC,UAAU,MAAM,SAAS,cAAc;AAC3F,MAAI,CAAC,QAAS;AACd,YAAU,SAAS,QAAQ,OAAO;AACpC;AAEA,SAAS,OAAO,SAAkB,QAA8B,QAA+B;AAC7F,MAAI,QAAQ,QAAS;AACrB,oBAAkB,SAAS,QAAQ,MAAM;AACzC,MAAI,QAAQ,gBAAgB,OAAW,SAAQ,eAAe,OAAO;AACrE,MAAI,QAAQ,YAAY,OAAW,SAAQ,aAAa,OAAO;AAC/D,QAAM,UAAU,OAAO;AAAA,IACrB,CAAC,UAAU,MAAM,SAAS,aAAa,MAAM,SAAS,WAAW,MAAM,SAAS;AAAA,EAClF;AACA,MAAI,QAAQ,iBAAiB,WAAc,WAAW,OAAO,WAAW,IAAI;AAC1E,aAAS,SAAS,QAAQ,YAAY;AAAA,EACxC;AACA,aAAW,SAAS,QAAQ;AAC1B,eAAW,SAAS,KAAK;AACzB,QAAI,QAAQ,QAAS;AAAA,EACvB;AACA,MAAI,QAAQ,eAAe,QAAW;AACpC,YAAQ,UAAU,QAAQ;AAC1B,YAAQ,aAAa;AAAA,EACvB;AACA,MAAI,QAAQ,YAAY,UAAa,UAAU,OAAO,EAAG,WAAU,SAAS,QAAQ,OAAO;AAC7F;AAEA,eAAe,iBACb,SACA,SACA,QACA,QACA,MACA;AACA,SAAO,CAAC,OAAO,KAAK,CAAC,QAAQ,SAAS;AACpC,QAAI,SAAS,QAAQ,SAAS;AAC5B,aAAO,KAAK,EAAE,MAAM,SAAS,QAAQ,WAAW,OAAO,UAAU,CAAC;AAClE,cAAQ,UAAU;AAClB;AAAA,IACF;AACA,UAAM,KAAK;AACX,SAAK;AAAA,EACP;AACA,MAAI,QAAQ,QAAS;AACrB,OAAK;AACL,SAAO,OAAO;AAChB;AAEA,SAAS,OAAO,SAAkB;AAChC,MAAI,QAAQ,QAAS;AACrB,MAAI,QAAQ,iBAAiB,OAAW,UAAS,SAAS,QAAQ,YAAY;AAC9E,MAAI,QAAQ,SAAU,WAAU,SAAS,QAAQ,WAAW,QAAQ,cAAc,QAAQ,MAAM;AAChG,MAAI,QAAQ,YAAa,SAAQ,OAAO,KAAK,QAAQ,WAAW;AAAA,OAC3D;AACH,YAAQ,OAAO,KAAK;AAAA,MAClB,MAAM;AAAA,MACN,QAAQ;AAAA,MACR,OAAO;AAAA,IACT,CAAC;AAAA,EACH;AACF;AAEO,SAAS,cACd,aACA,OACA,OACA,SACA,MAAe,SAAS,OAAO,UAAU,GACf;AAC1B,QAAM,SAAS,IAAI,yBAAyB;AAC5C,OAAK,KAAK,aAAa,OAAO,OAAO,SAAS,QAAQ,GAAG;AACzD,SAAO;AACT;AAEA,eAAe,KACb,aACA,OACA,OACA,SACA,QACA,KACA;AACA,MAAI,SAAS,QAAQ,SAAS;AAC5B,WAAO,KAAK,EAAE,MAAM,SAAS,QAAQ,WAAW,OAAO,UAAU,CAAC;AAClE;AAAA,EACF;AACA,QAAM,OAAO,UAAU;AACvB,MAAI;AACJ,MAAI;AACF,YAAQ,YAAY,OAAO,KAAK,QAAQ,OAAO;AAAA,EACjD,SAAS,OAAO;AACd,UAAM,UAAU,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AACrE,WAAO,KAAK,EAAE,MAAM,SAAS,QAAQ,SAAS,OAAO,QAAQ,CAAC;AAC9D;AAAA,EACF;AACA,QAAM,QAA8B,CAAC;AACrC,MAAI,eAAe;AACnB,QAAM,WAAW,YAAY;AAC3B,QAAI;AACF,uBAAiB,SAAS,MAAO,OAAM,KAAK,KAAK;AAAA,IACnD,SAAS,OAAO;AACd,YAAM,UAAU,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AACrE,YAAM,KAAK,EAAE,MAAM,SAAS,QAAQ,SAAS,OAAO,QAAQ,CAAC;AAAA,IAC/D,UAAE;AACA,qBAAe;AAAA,IACjB;AAAA,EACF,GAAG;AACH,QAAM,UAAmB;AAAA,IACvB;AAAA,IACA,UAAU,oBAAI,IAAI;AAAA,IAClB,QAAQ;AAAA,IACR,UAAU;AAAA,IACV,SAAS;AAAA,IACT,UAAU;AAAA,IACV,SAAS;AAAA,IACT,QAAQ;AAAA,EACV;AACA,QAAM,OAAO,CAAC,WAAuB;AACnC,UAAM,SAAS,MAAM,OAAO,GAAG,MAAM,MAAM;AAC3C,WAAO,SAAS,QAAQ,MAAM;AAAA,EAChC;AACA,MAAI;AACF,UAAM,KAAK;AACX,SAAK;AACL,qBAAiB,SAAS,OAAO;AAC/B,UAAI,QAAQ,QAAS;AACrB,UAAI,SAAS,QAAQ,SAAS;AAC5B,aAAK,KAAK,IAAI;AACd,cAAM,KAAK;AACX,aAAK;AACL,YAAI,CAAC,QAAQ,SAAS;AACpB,iBAAO,KAAK,EAAE,MAAM,SAAS,QAAQ,WAAW,OAAO,UAAU,CAAC;AAClE,kBAAQ,UAAU;AAAA,QACpB;AACA;AAAA,MACF;AACA,cAAQ,SAAS,MAAM;AACvB,YAAM,SAAS,IAAI,KAAK,KAAK;AAC7B,WAAK,KAAK,KAAK;AACf,YAAM,KAAK;AACX,WAAK,MAAM;AAAA,IACb;AACA,SAAK,KAAK,IAAI;AACd,QAAI,CAAC,QAAQ;AACX,YAAM,iBAAiB,SAAS,SAAS,QAAQ,MAAM,cAAc,IAAI;AAAA,QACtE,OAAM,KAAK;AAAA,EAClB,SAAS,OAAO;AACd,SAAK,KAAK,IAAI;AACd,QAAI,CAAC,QAAQ,SAAS;AACpB,YAAM,UAAU,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AACrE,aAAO,KAAK,EAAE,MAAM,SAAS,QAAQ,SAAS,OAAO,QAAQ,CAAC;AAAA,IAChE;AAAA,EACF;AACA,QAAM;AACR;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
package/dist/vad.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { convertAudio } from "./convert.js";
|
|
1
2
|
function width(encoding) {
|
|
2
3
|
if (encoding === "pcm16") return 2;
|
|
3
4
|
if (encoding === "pcmf32") return 4;
|
|
@@ -22,6 +23,15 @@ function rms(chunk) {
|
|
|
22
23
|
}
|
|
23
24
|
return Math.sqrt(sum / samples);
|
|
24
25
|
}
|
|
26
|
+
const compressed = /* @__PURE__ */ new Set(["opus", "mp3", "flac"]);
|
|
27
|
+
function pcmEnergy(chunk) {
|
|
28
|
+
const encoding = chunk.format.encoding;
|
|
29
|
+
if (encoding === "pcm16" || encoding === "pcmf32") return chunk;
|
|
30
|
+
if (compressed.has(encoding)) {
|
|
31
|
+
throw new Error(`energyVad requires PCM audio, received ${encoding}`);
|
|
32
|
+
}
|
|
33
|
+
return convertAudio(chunk, { ...chunk.format, encoding: "pcm16" });
|
|
34
|
+
}
|
|
25
35
|
function energyVad(options = {}) {
|
|
26
36
|
const threshold = options.threshold ?? 0.02;
|
|
27
37
|
const silenceMs = options.silenceMs ?? 500;
|
|
@@ -29,13 +39,11 @@ function energyVad(options = {}) {
|
|
|
29
39
|
let silence = 0;
|
|
30
40
|
return {
|
|
31
41
|
push(chunk) {
|
|
32
|
-
const
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
}
|
|
38
|
-
const duration = durationMs(chunk);
|
|
42
|
+
const sample = pcmEnergy(chunk);
|
|
43
|
+
const measured = rms(sample);
|
|
44
|
+
let level = 0;
|
|
45
|
+
if (measured !== void 0) level = measured;
|
|
46
|
+
const duration = durationMs(sample);
|
|
39
47
|
if (level >= threshold) {
|
|
40
48
|
silence = 0;
|
|
41
49
|
if (speaking) return void 0;
|
package/dist/vad.js.map
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../src/vad.ts"],
|
|
4
|
-
"sourcesContent": ["import type { AudioChunk } from './audio.ts';\n\nexport interface VadUpdate {\n speechStart?: number;\n turnEnd?: number;\n}\n\nexport interface VadHook {\n push(chunk: AudioChunk): VadUpdate | undefined;\n}\n\nexport interface EnergyVadOptions {\n threshold?: number;\n silenceMs?: number;\n}\n\nfunction width(encoding: AudioChunk['format']['encoding']): number | undefined {\n if (encoding === 'pcm16') return 2;\n if (encoding === 'pcmf32') return 4;\n return undefined;\n}\n\nfunction durationMs(chunk: AudioChunk): number {\n const bytes = width(chunk.format.encoding);\n if (bytes === undefined || chunk.format.sampleRate <= 0 || chunk.format.channels <= 0) return 0;\n const samples = chunk.data.byteLength / bytes / chunk.format.channels;\n return (samples / chunk.format.sampleRate) * 1000;\n}\n\nfunction rms(chunk: AudioChunk): number | undefined {\n const bytes = width(chunk.format.encoding);\n if (bytes === undefined) return undefined;\n const samples = Math.floor(chunk.data.byteLength / bytes);\n if (samples === 0) return 0;\n const view = new DataView(chunk.data.buffer, chunk.data.byteOffset, chunk.data.byteLength);\n let sum = 0;\n for (let index = 0; index < samples; index += 1) {\n const sample =\n bytes === 2 ? view.getInt16(index * 2, true) / 32768 : view.getFloat32(index * 4, true);\n sum += sample * sample;\n }\n return Math.sqrt(sum / samples);\n}\n\nexport function energyVad(options: EnergyVadOptions = {}): VadHook {\n const threshold = options.threshold ?? 0.02;\n const silenceMs = options.silenceMs ?? 500;\n let speaking = false;\n let silence = 0;\n return {\n push(chunk) {\n const
|
|
5
|
-
"mappings": "
|
|
4
|
+
"sourcesContent": ["import type { AudioChunk, AudioEncoding } from './audio.ts';\nimport { convertAudio } from './convert.ts';\n\nexport interface VadUpdate {\n speechStart?: number;\n turnEnd?: number;\n}\n\nexport interface VadHook {\n push(chunk: AudioChunk): VadUpdate | undefined;\n}\n\nexport interface EnergyVadOptions {\n threshold?: number;\n silenceMs?: number;\n}\n\nfunction width(encoding: AudioChunk['format']['encoding']): number | undefined {\n if (encoding === 'pcm16') return 2;\n if (encoding === 'pcmf32') return 4;\n return undefined;\n}\n\nfunction durationMs(chunk: AudioChunk): number {\n const bytes = width(chunk.format.encoding);\n if (bytes === undefined || chunk.format.sampleRate <= 0 || chunk.format.channels <= 0) return 0;\n const samples = chunk.data.byteLength / bytes / chunk.format.channels;\n return (samples / chunk.format.sampleRate) * 1000;\n}\n\nfunction rms(chunk: AudioChunk): number | undefined {\n const bytes = width(chunk.format.encoding);\n if (bytes === undefined) return undefined;\n const samples = Math.floor(chunk.data.byteLength / bytes);\n if (samples === 0) return 0;\n const view = new DataView(chunk.data.buffer, chunk.data.byteOffset, chunk.data.byteLength);\n let sum = 0;\n for (let index = 0; index < samples; index += 1) {\n const sample =\n bytes === 2 ? view.getInt16(index * 2, true) / 32768 : view.getFloat32(index * 4, true);\n sum += sample * sample;\n }\n return Math.sqrt(sum / samples);\n}\n\nconst compressed = new Set<AudioEncoding>(['opus', 'mp3', 'flac']);\n\nfunction pcmEnergy(chunk: AudioChunk): AudioChunk {\n const encoding = chunk.format.encoding;\n if (encoding === 'pcm16' || encoding === 'pcmf32') return chunk;\n if (compressed.has(encoding)) {\n throw new Error(`energyVad requires PCM audio, received ${encoding}`);\n }\n return convertAudio(chunk, { ...chunk.format, encoding: 'pcm16' });\n}\n\nexport function energyVad(options: EnergyVadOptions = {}): VadHook {\n const threshold = options.threshold ?? 0.02;\n const silenceMs = options.silenceMs ?? 500;\n let speaking = false;\n let silence = 0;\n return {\n push(chunk) {\n const sample = pcmEnergy(chunk);\n const measured = rms(sample);\n let level = 0;\n if (measured !== undefined) level = measured;\n const duration = durationMs(sample);\n if (level >= threshold) {\n silence = 0;\n if (speaking) return undefined;\n speaking = true;\n return { speechStart: chunk.timestamp };\n }\n if (!speaking) return undefined;\n silence += duration;\n if (silence < silenceMs) return undefined;\n speaking = false;\n silence = 0;\n return { turnEnd: chunk.timestamp + duration };\n },\n };\n}\n"],
|
|
5
|
+
"mappings": "AACA,SAAS,oBAAoB;AAgB7B,SAAS,MAAM,UAAgE;AAC7E,MAAI,aAAa,QAAS,QAAO;AACjC,MAAI,aAAa,SAAU,QAAO;AAClC,SAAO;AACT;AAEA,SAAS,WAAW,OAA2B;AAC7C,QAAM,QAAQ,MAAM,MAAM,OAAO,QAAQ;AACzC,MAAI,UAAU,UAAa,MAAM,OAAO,cAAc,KAAK,MAAM,OAAO,YAAY,EAAG,QAAO;AAC9F,QAAM,UAAU,MAAM,KAAK,aAAa,QAAQ,MAAM,OAAO;AAC7D,SAAQ,UAAU,MAAM,OAAO,aAAc;AAC/C;AAEA,SAAS,IAAI,OAAuC;AAClD,QAAM,QAAQ,MAAM,MAAM,OAAO,QAAQ;AACzC,MAAI,UAAU,OAAW,QAAO;AAChC,QAAM,UAAU,KAAK,MAAM,MAAM,KAAK,aAAa,KAAK;AACxD,MAAI,YAAY,EAAG,QAAO;AAC1B,QAAM,OAAO,IAAI,SAAS,MAAM,KAAK,QAAQ,MAAM,KAAK,YAAY,MAAM,KAAK,UAAU;AACzF,MAAI,MAAM;AACV,WAAS,QAAQ,GAAG,QAAQ,SAAS,SAAS,GAAG;AAC/C,UAAM,SACJ,UAAU,IAAI,KAAK,SAAS,QAAQ,GAAG,IAAI,IAAI,QAAQ,KAAK,WAAW,QAAQ,GAAG,IAAI;AACxF,WAAO,SAAS;AAAA,EAClB;AACA,SAAO,KAAK,KAAK,MAAM,OAAO;AAChC;AAEA,MAAM,aAAa,oBAAI,IAAmB,CAAC,QAAQ,OAAO,MAAM,CAAC;AAEjE,SAAS,UAAU,OAA+B;AAChD,QAAM,WAAW,MAAM,OAAO;AAC9B,MAAI,aAAa,WAAW,aAAa,SAAU,QAAO;AAC1D,MAAI,WAAW,IAAI,QAAQ,GAAG;AAC5B,UAAM,IAAI,MAAM,0CAA0C,QAAQ,EAAE;AAAA,EACtE;AACA,SAAO,aAAa,OAAO,EAAE,GAAG,MAAM,QAAQ,UAAU,QAAQ,CAAC;AACnE;AAEO,SAAS,UAAU,UAA4B,CAAC,GAAY;AACjE,QAAM,YAAY,QAAQ,aAAa;AACvC,QAAM,YAAY,QAAQ,aAAa;AACvC,MAAI,WAAW;AACf,MAAI,UAAU;AACd,SAAO;AAAA,IACL,KAAK,OAAO;AACV,YAAM,SAAS,UAAU,KAAK;AAC9B,YAAM,WAAW,IAAI,MAAM;AAC3B,UAAI,QAAQ;AACZ,UAAI,aAAa,OAAW,SAAQ;AACpC,YAAM,WAAW,WAAW,MAAM;AAClC,UAAI,SAAS,WAAW;AACtB,kBAAU;AACV,YAAI,SAAU,QAAO;AACrB,mBAAW;AACX,eAAO,EAAE,aAAa,MAAM,UAAU;AAAA,MACxC;AACA,UAAI,CAAC,SAAU,QAAO;AACtB,iBAAW;AACX,UAAI,UAAU,UAAW,QAAO;AAChC,iBAAW;AACX,gBAAU;AACV,aAAO,EAAE,SAAS,MAAM,YAAY,SAAS;AAAA,IAC/C;AAAA,EACF;AACF;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ai-ecoverse/pi-audio",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.20.1",
|
|
4
4
|
"description": "Voice bindings for pi: speech-to-text, text-to-speech and realtime speech for pi-coding-agent plugins and slicc-agent",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -58,6 +58,10 @@
|
|
|
58
58
|
"types": "./dist/types/fish/index.d.ts",
|
|
59
59
|
"default": "./dist/fish/index.js"
|
|
60
60
|
},
|
|
61
|
+
"./google": {
|
|
62
|
+
"types": "./dist/types/google/index.d.ts",
|
|
63
|
+
"default": "./dist/google/index.js"
|
|
64
|
+
},
|
|
61
65
|
"./durable": {
|
|
62
66
|
"types": "./dist/types/durable/index.d.ts",
|
|
63
67
|
"default": "./dist/durable/index.js"
|