@liforma/client 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/README.md +14 -1
- package/dist/byo/audioCapture.d.ts +24 -0
- package/dist/byo/audioCapture.d.ts.map +1 -0
- package/dist/byo/index.d.ts +8 -0
- package/dist/byo/index.d.ts.map +1 -0
- package/dist/byo/index.js +307 -0
- package/dist/byo/index.js.map +1 -0
- package/dist/byo/mic.d.ts +27 -0
- package/dist/byo/mic.d.ts.map +1 -0
- package/dist/byo/pcm.d.ts +33 -0
- package/dist/byo/pcm.d.ts.map +1 -0
- package/dist/byo/pcmCarry.d.ts +17 -0
- package/dist/byo/pcmCarry.d.ts.map +1 -0
- package/dist/byo/resample.d.ts +18 -0
- package/dist/byo/resample.d.ts.map +1 -0
- package/dist/byo/turn.d.ts +39 -0
- package/dist/byo/turn.d.ts.map +1 -0
- package/dist/byo/types.d.ts +74 -0
- package/dist/byo/types.d.ts.map +1 -0
- package/dist/deepgram/connectDeepgramAgent.d.ts +51 -0
- package/dist/deepgram/connectDeepgramAgent.d.ts.map +1 -0
- package/dist/deepgram/deepgramAgentMessages.d.ts +31 -0
- package/dist/deepgram/deepgramAgentMessages.d.ts.map +1 -0
- package/dist/deepgram/index.d.ts +4 -0
- package/dist/deepgram/index.d.ts.map +1 -0
- package/dist/deepgram/index.js +142 -0
- package/dist/deepgram/index.js.map +1 -0
- package/dist/elevenlabs/connectElevenLabsAgent.d.ts +23 -0
- package/dist/elevenlabs/connectElevenLabsAgent.d.ts.map +1 -0
- package/dist/elevenlabs/index.d.ts +4 -0
- package/dist/elevenlabs/index.d.ts.map +1 -0
- package/dist/elevenlabs/index.js +89 -0
- package/dist/elevenlabs/index.js.map +1 -0
- package/dist/google/connectGeminiLive.d.ts +34 -0
- package/dist/google/connectGeminiLive.d.ts.map +1 -0
- package/dist/google/index.d.ts +4 -0
- package/dist/google/index.d.ts.map +1 -0
- package/dist/google/index.js +98 -0
- package/dist/google/index.js.map +1 -0
- package/dist/livekit/connectLiveKitAgent.d.ts +46 -0
- package/dist/livekit/connectLiveKitAgent.d.ts.map +1 -0
- package/dist/livekit/index.d.ts +3 -0
- package/dist/livekit/index.d.ts.map +1 -0
- package/dist/livekit/index.js +113 -0
- package/dist/livekit/index.js.map +1 -0
- package/dist/next/byo/audioCapture.d.ts +24 -0
- package/dist/next/byo/audioCapture.d.ts.map +1 -0
- package/dist/next/byo/pcm.d.ts +33 -0
- package/dist/next/byo/pcm.d.ts.map +1 -0
- package/dist/next/byo/resample.d.ts +18 -0
- package/dist/next/byo/resample.d.ts.map +1 -0
- package/dist/next/index.js +675 -474
- package/dist/next/index.js.map +1 -1
- package/dist/next/v2/embed/iframeHost.d.ts +16 -1
- package/dist/next/v2/embed/iframeHost.d.ts.map +1 -1
- package/dist/next/v2/embed/protocol.d.ts +41 -1
- package/dist/next/v2/embed/protocol.d.ts.map +1 -1
- package/dist/next/v2/experience.d.ts +14 -2
- package/dist/next/v2/experience.d.ts.map +1 -1
- package/dist/next/v2/speech/captureMediaStreamTrack.d.ts +1 -1
- package/dist/next/v2/speech/captureMediaStreamTrack.d.ts.map +1 -1
- package/dist/next/v2/speech/speechController.d.ts.map +1 -1
- package/dist/next/v2/speech/types.d.ts +13 -0
- package/dist/next/v2/speech/types.d.ts.map +1 -1
- package/dist/next/widget/mountExperienceWidget.d.ts.map +1 -1
- package/dist/next/widget/styles.d.ts +1 -1
- package/dist/next/widget/styles.d.ts.map +1 -1
- package/dist/npm/byo/audioCapture.d.ts +24 -0
- package/dist/npm/byo/audioCapture.d.ts.map +1 -0
- package/dist/npm/byo/pcm.d.ts +33 -0
- package/dist/npm/byo/pcm.d.ts.map +1 -0
- package/dist/npm/byo/resample.d.ts +18 -0
- package/dist/npm/byo/resample.d.ts.map +1 -0
- package/dist/npm/index.js +700 -499
- package/dist/npm/index.js.map +1 -1
- package/dist/npm/v2/embed/iframeHost.d.ts +16 -1
- package/dist/npm/v2/embed/iframeHost.d.ts.map +1 -1
- package/dist/npm/v2/embed/protocol.d.ts +41 -1
- package/dist/npm/v2/embed/protocol.d.ts.map +1 -1
- package/dist/npm/v2/experience.d.ts +14 -2
- package/dist/npm/v2/experience.d.ts.map +1 -1
- package/dist/npm/v2/speech/captureMediaStreamTrack.d.ts +1 -1
- package/dist/npm/v2/speech/captureMediaStreamTrack.d.ts.map +1 -1
- package/dist/npm/v2/speech/speechController.d.ts.map +1 -1
- package/dist/npm/v2/speech/types.d.ts +13 -0
- package/dist/npm/v2/speech/types.d.ts.map +1 -1
- package/dist/npm/widget/mountExperienceWidget.d.ts.map +1 -1
- package/dist/npm/widget/styles.d.ts +1 -1
- package/dist/npm/widget/styles.d.ts.map +1 -1
- package/dist/openai/connectOpenAiRealtime.d.ts +41 -0
- package/dist/openai/connectOpenAiRealtime.d.ts.map +1 -0
- package/dist/openai/connectOpenAiRealtimeWebRtc.d.ts +37 -0
- package/dist/openai/connectOpenAiRealtimeWebRtc.d.ts.map +1 -0
- package/dist/openai/index.d.ts +5 -0
- package/dist/openai/index.d.ts.map +1 -0
- package/dist/openai/index.js +282 -0
- package/dist/openai/index.js.map +1 -0
- package/dist/react/byo/audioCapture.d.ts +24 -0
- package/dist/react/byo/audioCapture.d.ts.map +1 -0
- package/dist/react/byo/pcm.d.ts +33 -0
- package/dist/react/byo/pcm.d.ts.map +1 -0
- package/dist/react/byo/resample.d.ts +18 -0
- package/dist/react/byo/resample.d.ts.map +1 -0
- package/dist/react/index.js +696 -495
- package/dist/react/index.js.map +1 -1
- package/dist/react/v2/embed/iframeHost.d.ts +16 -1
- package/dist/react/v2/embed/iframeHost.d.ts.map +1 -1
- package/dist/react/v2/embed/protocol.d.ts +41 -1
- package/dist/react/v2/embed/protocol.d.ts.map +1 -1
- package/dist/react/v2/experience.d.ts +14 -2
- package/dist/react/v2/experience.d.ts.map +1 -1
- package/dist/react/v2/speech/captureMediaStreamTrack.d.ts +1 -1
- package/dist/react/v2/speech/captureMediaStreamTrack.d.ts.map +1 -1
- package/dist/react/v2/speech/speechController.d.ts.map +1 -1
- package/dist/react/v2/speech/types.d.ts +13 -0
- package/dist/react/v2/speech/types.d.ts.map +1 -1
- package/dist/react/widget/mountExperienceWidget.d.ts.map +1 -1
- package/dist/react/widget/styles.d.ts +1 -1
- package/dist/react/widget/styles.d.ts.map +1 -1
- package/dist/sdk/v2/client.js +37 -10
- package/dist/sdk/v2/client.js.map +1 -1
- package/package.json +55 -7
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import { MAX_PCM_CHUNK_BYTES as e, base64ToPcmBytes as t, createTurnSession as n, floatToPcm16Le as r, pcmBytesToBase64 as i, pcmBytesToBase64 as a, resampleFloat32 as o, shouldCaptureMic as s, startPcmMic as c, writePcmInChunks as l } from "@liforma/client/byo";
|
|
2
|
+
//#region src/google/connectGeminiLive.ts
|
|
3
|
+
var u = 24e3, d = 16e3;
|
|
4
|
+
async function f(e, t) {
|
|
5
|
+
let { proxyUrl: r, inputSampleRate: i = d, outputSampleRate: o = u, captureMic: l = !0, mediaStream: f, onLog: p, onDisconnect: m, onError: h } = t, g = r.trim();
|
|
6
|
+
if (!g) throw Error("Pass proxyUrl from your Gemini Live proxy");
|
|
7
|
+
let _ = (e, t = "info") => p?.(e, t), v = null, y = !1, b = null, x = n(e, {
|
|
8
|
+
sampleRate: o,
|
|
9
|
+
isEnded: () => y,
|
|
10
|
+
onWriteError: (e) => h?.(e instanceof Error ? e.message : String(e))
|
|
11
|
+
});
|
|
12
|
+
function S(e) {
|
|
13
|
+
!v || v.readyState !== WebSocket.OPEN || v.send(JSON.stringify(e));
|
|
14
|
+
}
|
|
15
|
+
function C(e) {
|
|
16
|
+
let t;
|
|
17
|
+
try {
|
|
18
|
+
t = JSON.parse(e);
|
|
19
|
+
} catch {
|
|
20
|
+
return;
|
|
21
|
+
}
|
|
22
|
+
if (t.error?.message) {
|
|
23
|
+
h?.(t.error.message), _(t.error.message, "warn");
|
|
24
|
+
return;
|
|
25
|
+
}
|
|
26
|
+
let n = t.serverContent;
|
|
27
|
+
if (n) {
|
|
28
|
+
if (n.interrupted) {
|
|
29
|
+
x.bargeIn();
|
|
30
|
+
return;
|
|
31
|
+
}
|
|
32
|
+
n.outputTranscription?.text && x.setTranscript(n.outputTranscription.text, "append");
|
|
33
|
+
for (let e of n.modelTurn?.parts ?? []) {
|
|
34
|
+
let t = e.inlineData?.data;
|
|
35
|
+
typeof t == "string" && t && x.writePcmBase64(t);
|
|
36
|
+
}
|
|
37
|
+
(n.generationComplete || n.turnComplete) && x.closeActiveTurn();
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
if (v = new WebSocket(g), await new Promise((e, t) => {
|
|
41
|
+
if (!v) {
|
|
42
|
+
t(/* @__PURE__ */ Error("WebSocket failed to construct"));
|
|
43
|
+
return;
|
|
44
|
+
}
|
|
45
|
+
let n = v, r = () => {
|
|
46
|
+
a(), e();
|
|
47
|
+
}, i = () => {
|
|
48
|
+
a(), t(/* @__PURE__ */ Error("Gemini Live WebSocket failed to open"));
|
|
49
|
+
}, a = () => {
|
|
50
|
+
n.removeEventListener("open", r), n.removeEventListener("error", i);
|
|
51
|
+
};
|
|
52
|
+
n.addEventListener("open", r), n.addEventListener("error", i);
|
|
53
|
+
}), v.addEventListener("message", (e) => {
|
|
54
|
+
typeof e.data == "string" && C(e.data);
|
|
55
|
+
}), v.addEventListener("close", () => {
|
|
56
|
+
b?.stop(), b = null, v = null, !y && (x.abandonActiveTurn(), _("Gemini Live disconnected"), m?.());
|
|
57
|
+
}), s({
|
|
58
|
+
captureMic: l,
|
|
59
|
+
mediaStream: f
|
|
60
|
+
})) try {
|
|
61
|
+
b = await c({
|
|
62
|
+
targetSampleRate: i,
|
|
63
|
+
mediaStream: f,
|
|
64
|
+
onLog: _,
|
|
65
|
+
shouldSend: () => !y && v != null && v.readyState === WebSocket.OPEN,
|
|
66
|
+
onPcm: (e) => {
|
|
67
|
+
S({ realtimeInput: { mediaChunks: [{
|
|
68
|
+
mimeType: `audio/pcm;rate=${i}`,
|
|
69
|
+
data: a(e)
|
|
70
|
+
}] } });
|
|
71
|
+
}
|
|
72
|
+
});
|
|
73
|
+
} catch (e) {
|
|
74
|
+
v.close();
|
|
75
|
+
let t = e instanceof Error ? e.message : String(e);
|
|
76
|
+
throw Error(`Microphone access failed: ${t}`);
|
|
77
|
+
}
|
|
78
|
+
return _("Gemini Live connected. Speak into your mic — the avatar should lip-sync."), {
|
|
79
|
+
isConnected: () => {
|
|
80
|
+
let e = v;
|
|
81
|
+
return e != null && e.readyState === WebSocket.OPEN && !y;
|
|
82
|
+
},
|
|
83
|
+
async end() {
|
|
84
|
+
if (y) return;
|
|
85
|
+
y = !0, await x.closeActiveTurn(), b?.stop(), b = null;
|
|
86
|
+
let e = v;
|
|
87
|
+
if (v = null, e && e.readyState === WebSocket.OPEN) try {
|
|
88
|
+
e.close();
|
|
89
|
+
} catch (e) {
|
|
90
|
+
console.error(e);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
//#endregion
|
|
96
|
+
export { d as GEMINI_LIVE_INPUT_RATE, u as GEMINI_LIVE_OUTPUT_RATE, e as MAX_PCM_CHUNK_BYTES, t as base64ToPcmBytes, f as connectGeminiLive, r as floatToPcm16Le, i as pcmBytesToBase64, o as resampleFloat32, l as writePcmInChunks };
|
|
97
|
+
|
|
98
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","names":[],"sources":["../../src/google/connectGeminiLive.ts"],"sourcesContent":["/**\n * Gemini Live (BidiGenerateContent) → Liforma BYO voice helper.\n *\n * Mount a Liforma Experience (typically `mode=\"presenter\"`, `speechInputMode=\"off\"`),\n * wait until the player has started (audio unlocked), then call this helper with a\n * same-origin WebSocket proxy that terminates Gemini Live (never ship Google API keys).\n *\n * The helper expects proxy messages shaped like Gemini Live `serverContent` JSON, and\n * (when captureMic is enabled) sends `realtimeInput.mediaChunks` PCM @ 16 kHz.\n *\n * Docs: https://docs.liforma.ai/avatar-experiences/bring-your-own-voice/google\n */\n\nimport {\n\tcreateTurnSession,\n\tpcmBytesToBase64,\n\tshouldCaptureMic,\n\tstartPcmMic,\n\ttype ByoBridge,\n\ttype ByoConnectCallbacks,\n\ttype ByoMicCaptureOptions,\n\ttype ByoPcmExperience,\n\ttype PcmMicSession\n} from '@liforma/client/byo';\n\nexport type GeminiLiveExperience = ByoPcmExperience;\n\n/** Gemini Live output PCM rate expected by this helper. */\nexport const GEMINI_LIVE_OUTPUT_RATE = 24_000;\n\n/** Mic PCM rate streamed to Gemini Live input. */\nexport const GEMINI_LIVE_INPUT_RATE = 16_000;\n\nexport type ConnectGeminiLiveOptions = ByoConnectCallbacks &\n\tByoMicCaptureOptions & {\n\t\t/**\n\t\t * Same-origin (or CORS-friendly) WebSocket URL that proxies Gemini Live\n\t\t * BidiGenerateContent. Keep Google credentials on the server.\n\t\t */\n\t\treadonly proxyUrl: string;\n\t\treadonly inputSampleRate?: number;\n\t\treadonly outputSampleRate?: number;\n\t};\n\nexport type GeminiLiveBridge = ByoBridge;\n\n/**\n * Connect to a Gemini Live proxy WebSocket and pipe agent PCM + transcript into\n * `experience.speech.createUtterance` so the Liforma avatar talks.\n */\nexport async function connectGeminiLive(\n\texperience: GeminiLiveExperience,\n\toptions: ConnectGeminiLiveOptions\n): Promise<GeminiLiveBridge> {\n\tconst {\n\t\tproxyUrl,\n\t\tinputSampleRate = GEMINI_LIVE_INPUT_RATE,\n\t\toutputSampleRate = GEMINI_LIVE_OUTPUT_RATE,\n\t\tcaptureMic = true,\n\t\tmediaStream,\n\t\tonLog,\n\t\tonDisconnect,\n\t\tonError\n\t} = options;\n\n\tconst url = proxyUrl.trim();\n\tif (!url) throw new Error('Pass proxyUrl from your Gemini Live proxy');\n\n\tconst log = (line: string, kind: 'info' | 'warn' = 'info') => onLog?.(line, kind);\n\n\tlet ws: WebSocket | null = null;\n\tlet ended = false;\n\tlet mic: PcmMicSession | null = null;\n\n\tconst turns = createTurnSession(experience, {\n\t\tsampleRate: outputSampleRate,\n\t\tisEnded: () => ended,\n\t\tonWriteError: (err) => onError?.(err instanceof Error ? err.message : String(err))\n\t});\n\n\tfunction send(event: Record<string, unknown>): void {\n\t\tif (!ws || ws.readyState !== WebSocket.OPEN) return;\n\t\tws.send(JSON.stringify(event));\n\t}\n\n\tfunction handleServerEvent(raw: string): void {\n\t\tlet msg: {\n\t\t\tserverContent?: {\n\t\t\t\tinterrupted?: boolean;\n\t\t\t\tgenerationComplete?: boolean;\n\t\t\t\tturnComplete?: boolean;\n\t\t\t\toutputTranscription?: { text?: string };\n\t\t\t\tmodelTurn?: {\n\t\t\t\t\tparts?: Array<{ inlineData?: { data?: string; mimeType?: string } }>;\n\t\t\t\t};\n\t\t\t};\n\t\t\terror?: { message?: string };\n\t\t};\n\t\ttry {\n\t\t\tmsg = JSON.parse(raw) as typeof msg;\n\t\t} catch {\n\t\t\treturn;\n\t\t}\n\n\t\tif (msg.error?.message) {\n\t\t\tonError?.(msg.error.message);\n\t\t\tlog(msg.error.message, 'warn');\n\t\t\treturn;\n\t\t}\n\n\t\tconst content = msg.serverContent;\n\t\tif (!content) return;\n\n\t\tif (content.interrupted) {\n\t\t\tturns.bargeIn();\n\t\t\treturn;\n\t\t}\n\n\t\tif (content.outputTranscription?.text) {\n\t\t\tturns.setTranscript(content.outputTranscription.text, 'append');\n\t\t}\n\n\t\tfor (const part of content.modelTurn?.parts ?? []) {\n\t\t\tconst b64 = part.inlineData?.data;\n\t\t\tif (typeof b64 === 'string' && b64) turns.writePcmBase64(b64);\n\t\t}\n\n\t\t// Prefer generationComplete — turnComplete can lag while Gemini assumes client playback.\n\t\tif (content.generationComplete || content.turnComplete) {\n\t\t\tvoid turns.closeActiveTurn();\n\t\t}\n\t}\n\n\tws = new WebSocket(url);\n\n\tawait new Promise<void>((resolve, reject) => {\n\t\tif (!ws) {\n\t\t\treject(new Error('WebSocket failed to construct'));\n\t\t\treturn;\n\t\t}\n\t\tconst socket = ws;\n\t\tconst onOpen = () => {\n\t\t\tcleanup();\n\t\t\tresolve();\n\t\t};\n\t\tconst onErrorEvt = () => {\n\t\t\tcleanup();\n\t\t\treject(new Error('Gemini Live WebSocket failed to open'));\n\t\t};\n\t\tconst cleanup = () => {\n\t\t\tsocket.removeEventListener('open', onOpen);\n\t\t\tsocket.removeEventListener('error', onErrorEvt);\n\t\t};\n\t\tsocket.addEventListener('open', onOpen);\n\t\tsocket.addEventListener('error', onErrorEvt);\n\t});\n\n\tws.addEventListener('message', (ev) => {\n\t\tif (typeof ev.data === 'string') handleServerEvent(ev.data);\n\t});\n\n\tws.addEventListener('close', () => {\n\t\tmic?.stop();\n\t\tmic = null;\n\t\tws = null;\n\t\t// Skip after intentional end() — turn was already closed cleanly.\n\t\tif (ended) return;\n\t\tturns.abandonActiveTurn();\n\t\tlog('Gemini Live disconnected');\n\t\tonDisconnect?.();\n\t});\n\n\tif (shouldCaptureMic({ captureMic, mediaStream })) {\n\t\ttry {\n\t\t\tmic = await startPcmMic({\n\t\t\t\ttargetSampleRate: inputSampleRate,\n\t\t\t\tmediaStream,\n\t\t\t\tonLog: log,\n\t\t\t\tshouldSend: () => !ended && ws != null && ws.readyState === WebSocket.OPEN,\n\t\t\t\tonPcm: (pcm) => {\n\t\t\t\t\tsend({\n\t\t\t\t\t\trealtimeInput: {\n\t\t\t\t\t\t\tmediaChunks: [\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\tmimeType: `audio/pcm;rate=${inputSampleRate}`,\n\t\t\t\t\t\t\t\t\tdata: pcmBytesToBase64(pcm)\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t]\n\t\t\t\t\t\t}\n\t\t\t\t\t});\n\t\t\t\t}\n\t\t\t});\n\t\t} catch (err) {\n\t\t\tws.close();\n\t\t\tconst message = err instanceof Error ? err.message : String(err);\n\t\t\tthrow new Error(`Microphone access failed: ${message}`);\n\t\t}\n\t}\n\n\tlog('Gemini Live connected. Speak into your mic — the avatar should lip-sync.');\n\n\treturn {\n\t\tisConnected: () => {\n\t\t\tconst socket = ws;\n\t\t\treturn socket != null && socket.readyState === WebSocket.OPEN && !ended;\n\t\t},\n\t\tasync end() {\n\t\t\tif (ended) return;\n\t\t\tended = true;\n\t\t\tawait turns.closeActiveTurn();\n\t\t\tmic?.stop();\n\t\t\tmic = null;\n\t\t\tconst active = ws;\n\t\t\tws = null;\n\t\t\tif (active && active.readyState === WebSocket.OPEN) {\n\t\t\t\ttry {\n\t\t\t\t\tactive.close();\n\t\t\t\t} catch (err) {\n\t\t\t\t\tconsole.error(err);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t};\n}\n"],"mappings":";;AA4BA,IAAa,IAA0B,MAG1B,IAAyB;AAmBtC,eAAsB,EACrB,GACA,GAC4B;CAC5B,IAAM,EACL,aACA,qBAAkB,GAClB,sBAAmB,GACnB,gBAAa,IACb,gBACA,UACA,iBACA,eACG,GAEE,IAAM,EAAS,KAAK;CAC1B,IAAI,CAAC,GAAK,MAAU,MAAM,2CAA2C;CAErE,IAAM,KAAO,GAAc,IAAwB,WAAW,IAAQ,GAAM,CAAI,GAE5E,IAAuB,MACvB,IAAQ,IACR,IAA4B,MAE1B,IAAQ,EAAkB,GAAY;EAC3C,YAAY;EACZ,eAAe;EACf,eAAe,MAAQ,IAAU,aAAe,QAAQ,EAAI,UAAU,OAAO,CAAG,CAAC;CAClF,CAAC;CAED,SAAS,EAAK,GAAsC;EAC/C,CAAC,KAAM,EAAG,eAAe,UAAU,QACvC,EAAG,KAAK,KAAK,UAAU,CAAK,CAAC;CAC9B;CAEA,SAAS,EAAkB,GAAmB;EAC7C,IAAI;EAYJ,IAAI;GACH,IAAM,KAAK,MAAM,CAAG;EACrB,QAAQ;GACP;EACD;EAEA,IAAI,EAAI,OAAO,SAAS;GAEvB,AADA,IAAU,EAAI,MAAM,OAAO,GAC3B,EAAI,EAAI,MAAM,SAAS,MAAM;GAC7B;EACD;EAEA,IAAM,IAAU,EAAI;EACf,OAEL;OAAI,EAAQ,aAAa;IACxB,EAAM,QAAQ;IACd;GACD;GAEA,AAAI,EAAQ,qBAAqB,QAChC,EAAM,cAAc,EAAQ,oBAAoB,MAAM,QAAQ;GAG/D,KAAK,IAAM,KAAQ,EAAQ,WAAW,SAAS,CAAC,GAAG;IAClD,IAAM,IAAM,EAAK,YAAY;IAC7B,AAAI,OAAO,KAAQ,YAAY,KAAK,EAAM,eAAe,CAAG;GAC7D;GAGA,CAAI,EAAQ,sBAAsB,EAAQ,iBACzC,EAAW,gBAAgB;EAb5B;CAeD;CAyCA,IAvCA,IAAK,IAAI,UAAU,CAAG,GAEtB,MAAM,IAAI,SAAe,GAAS,MAAW;EAC5C,IAAI,CAAC,GAAI;GACR,EAAO,gBAAI,MAAM,+BAA+B,CAAC;GACjD;EACD;EACA,IAAM,IAAS,GACT,UAAe;GAEpB,AADA,EAAQ,GACR,EAAQ;EACT,GACM,UAAmB;GAExB,AADA,EAAQ,GACR,EAAO,gBAAI,MAAM,sCAAsC,CAAC;EACzD,GACM,UAAgB;GAErB,AADA,EAAO,oBAAoB,QAAQ,CAAM,GACzC,EAAO,oBAAoB,SAAS,CAAU;EAC/C;EAEA,AADA,EAAO,iBAAiB,QAAQ,CAAM,GACtC,EAAO,iBAAiB,SAAS,CAAU;CAC5C,CAAC,GAED,EAAG,iBAAiB,YAAY,MAAO;EACtC,AAAI,OAAO,EAAG,QAAS,YAAU,EAAkB,EAAG,IAAI;CAC3D,CAAC,GAED,EAAG,iBAAiB,eAAe;EAClC,GAAK,KAAK,GACV,IAAM,MACN,IAAK,MAED,OACJ,EAAM,kBAAkB,GACxB,EAAI,0BAA0B,GAC9B,IAAe;CAChB,CAAC,GAEG,EAAiB;EAAE;EAAY;CAAY,CAAC,GAC/C,IAAI;EACH,IAAM,MAAM,EAAY;GACvB,kBAAkB;GAClB;GACA,OAAO;GACP,kBAAkB,CAAC,KAAS,KAAM,QAAQ,EAAG,eAAe,UAAU;GACtE,QAAQ,MAAQ;IACf,EAAK,EACJ,eAAe,EACd,aAAa,CACZ;KACC,UAAU,kBAAkB;KAC5B,MAAM,EAAiB,CAAG;IAC3B,CACD,EACD,EACD,CAAC;GACF;EACD,CAAC;CACF,SAAS,GAAK;EACb,EAAG,MAAM;EACT,IAAM,IAAU,aAAe,QAAQ,EAAI,UAAU,OAAO,CAAG;EAC/D,MAAU,MAAM,6BAA6B,GAAS;CACvD;CAKD,OAFA,EAAI,0EAA0E,GAEvE;EACN,mBAAmB;GAClB,IAAM,IAAS;GACf,OAAO,KAAU,QAAQ,EAAO,eAAe,UAAU,QAAQ,CAAC;EACnE;EACA,MAAM,MAAM;GACX,IAAI,GAAO;GAIX,AAHA,IAAQ,IACR,MAAM,EAAM,gBAAgB,GAC5B,GAAK,KAAK,GACV,IAAM;GACN,IAAM,IAAS;GAEf,IADA,IAAK,MACD,KAAU,EAAO,eAAe,UAAU,MAC7C,IAAI;IACH,EAAO,MAAM;GACd,SAAS,GAAK;IACb,QAAQ,MAAM,CAAG;GAClB;EAEF;CACD;AACD"}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LiveKit remote audio → Liforma BYO voice helper.
|
|
3
|
+
*
|
|
4
|
+
* Mount a Liforma Experience (typically `mode="presenter"`, `speechInputMode="off"`),
|
|
5
|
+
* wait until the player has started (audio unlocked), mint a LiveKit token on your
|
|
6
|
+
* server, then call this helper.
|
|
7
|
+
*
|
|
8
|
+
* Bridges remote agent audio via `createUtterance({ track })` and optionally applies
|
|
9
|
+
* LiveKit agent transcriptions (`lk.transcription` text streams) for force-align lipsync.
|
|
10
|
+
* Do not also attach the remote track to an `<audio>` element.
|
|
11
|
+
*
|
|
12
|
+
* Docs: https://docs.liforma.ai/avatar-experiences/bring-your-own-voice/livekit
|
|
13
|
+
*/
|
|
14
|
+
import { Room, type RemoteParticipant } from 'livekit-client';
|
|
15
|
+
import type { ByoBridge, ByoConnectCallbacks, ByoPcmExperience } from '@liforma/client/byo';
|
|
16
|
+
export type LiveKitAgentExperience = ByoPcmExperience;
|
|
17
|
+
export type ConnectLiveKitAgentOptions = ByoConnectCallbacks & {
|
|
18
|
+
/** LiveKit server URL (wss://…). */
|
|
19
|
+
readonly url: string;
|
|
20
|
+
/** Participant token from your backend (never mint long-lived tokens in the browser). */
|
|
21
|
+
readonly token: string;
|
|
22
|
+
/**
|
|
23
|
+
* Only bridge remote audio from participants matching this predicate.
|
|
24
|
+
* Defaults to identities starting with `agent`.
|
|
25
|
+
*/
|
|
26
|
+
readonly shouldBridgeParticipant?: (participant: RemoteParticipant) => boolean;
|
|
27
|
+
/** Sample rate hint for track capture (LiveKit often delivers 48 kHz). */
|
|
28
|
+
readonly sampleRate?: number;
|
|
29
|
+
/**
|
|
30
|
+
* When true (default), register a text-stream handler on `transcriptionTopic`
|
|
31
|
+
* and forward agent text into `utterance.setTranscript` for force-align lipsync.
|
|
32
|
+
*/
|
|
33
|
+
readonly enableTranscript?: boolean;
|
|
34
|
+
/** LiveKit text-stream topic for transcriptions (default `lk.transcription`). */
|
|
35
|
+
readonly transcriptionTopic?: string;
|
|
36
|
+
};
|
|
37
|
+
export type LiveKitAgentBridge = ByoBridge & {
|
|
38
|
+
/** Underlying LiveKit room (for advanced host control). */
|
|
39
|
+
readonly room: Room;
|
|
40
|
+
};
|
|
41
|
+
/**
|
|
42
|
+
* Connect a LiveKit room and pipe remote agent audio tracks into
|
|
43
|
+
* `experience.speech.createUtterance({ track })` so the Liforma avatar talks.
|
|
44
|
+
*/
|
|
45
|
+
export declare function connectLiveKitAgent(experience: LiveKitAgentExperience, options: ConnectLiveKitAgentOptions): Promise<LiveKitAgentBridge>;
|
|
46
|
+
//# sourceMappingURL=connectLiveKitAgent.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"connectLiveKitAgent.d.ts","sourceRoot":"","sources":["../../src/livekit/connectLiveKitAgent.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAEN,IAAI,EAGJ,KAAK,iBAAiB,EAEtB,MAAM,gBAAgB,CAAC;AACxB,OAAO,KAAK,EACX,SAAS,EACT,mBAAmB,EACnB,gBAAgB,EAEhB,MAAM,qBAAqB,CAAC;AAE7B,MAAM,MAAM,sBAAsB,GAAG,gBAAgB,CAAC;AAEtD,MAAM,MAAM,0BAA0B,GAAG,mBAAmB,GAAG;IAC9D,oCAAoC;IACpC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,yFAAyF;IACzF,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;;;OAGG;IACH,QAAQ,CAAC,uBAAuB,CAAC,EAAE,CAAC,WAAW,EAAE,iBAAiB,KAAK,OAAO,CAAC;IAC/E,0EAA0E;IAC1E,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B;;;OAGG;IACH,QAAQ,CAAC,gBAAgB,CAAC,EAAE,OAAO,CAAC;IACpC,iFAAiF;IACjF,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC;CACrC,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG,SAAS,GAAG;IAC5C,2DAA2D;IAC3D,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;CACpB,CAAC;AASF;;;GAGG;AACH,wBAAsB,mBAAmB,CACxC,UAAU,EAAE,sBAAsB,EAClC,OAAO,EAAE,0BAA0B,GACjC,OAAO,CAAC,kBAAkB,CAAC,CA8L7B"}
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
export { connectLiveKitAgent, type ConnectLiveKitAgentOptions, type LiveKitAgentBridge, type LiveKitAgentExperience } from './connectLiveKitAgent';
|
|
2
|
+
export type { ByoPcmSpeechController as LiveKitSpeechController } from '@liforma/client/byo';
|
|
3
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/livekit/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACN,mBAAmB,EACnB,KAAK,0BAA0B,EAC/B,KAAK,kBAAkB,EACvB,KAAK,sBAAsB,EAC3B,MAAM,uBAAuB,CAAC;AAE/B,YAAY,EAAE,sBAAsB,IAAI,uBAAuB,EAAE,MAAM,qBAAqB,CAAC"}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import { ConnectionState as e, Room as t, RoomEvent as n, Track as r } from "livekit-client";
|
|
2
|
+
//#region src/livekit/connectLiveKitAgent.ts
|
|
3
|
+
var i = 48e3, a = "lk.transcription";
|
|
4
|
+
function o(e) {
|
|
5
|
+
return e.identity.startsWith("agent");
|
|
6
|
+
}
|
|
7
|
+
async function s(s, c) {
|
|
8
|
+
let { url: l, token: u, shouldBridgeParticipant: d = o, sampleRate: f = i, enableTranscript: p = !0, transcriptionTopic: m = a, onLog: h, onDisconnect: g, onError: _ } = c, v = l.trim(), y = u.trim();
|
|
9
|
+
if (!v) throw Error("Pass LiveKit url");
|
|
10
|
+
if (!y) throw Error("Pass LiveKit token from your server mint route");
|
|
11
|
+
let b = (e, t = "info") => h?.(e, t), x = new t(), S = !1, C = !1, w = 0, T = null, E = "";
|
|
12
|
+
function D() {
|
|
13
|
+
w <= 0 || (T = null, E = "", s.speech.interrupt({ scope: "active" }).catch(() => void 0));
|
|
14
|
+
}
|
|
15
|
+
function O(e, t) {
|
|
16
|
+
let n = t === "replace" ? e.trim() : `${E}${e}`.trim();
|
|
17
|
+
n && (E = n, T && T.setTranscript(n).catch(() => void 0));
|
|
18
|
+
}
|
|
19
|
+
async function k(e, t) {
|
|
20
|
+
if (S || e.kind !== r.Kind.Audio) return;
|
|
21
|
+
if (!d(t)) {
|
|
22
|
+
b(`Skipping audio from ${t.identity}`);
|
|
23
|
+
return;
|
|
24
|
+
}
|
|
25
|
+
let n = e.mediaStreamTrack;
|
|
26
|
+
if (!n) {
|
|
27
|
+
_?.("LiveKit audio track missing mediaStreamTrack");
|
|
28
|
+
return;
|
|
29
|
+
}
|
|
30
|
+
b(`Bridging LiveKit audio from ${t.identity}`), w += 1, E = "";
|
|
31
|
+
let i = s.speech.createUtterance({
|
|
32
|
+
track: n,
|
|
33
|
+
sampleRate: f,
|
|
34
|
+
queue: "replace-active"
|
|
35
|
+
});
|
|
36
|
+
T = i;
|
|
37
|
+
try {
|
|
38
|
+
i.result ? await i.result : await new Promise((e) => {
|
|
39
|
+
if (n.readyState === "ended") {
|
|
40
|
+
e();
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
n.addEventListener("ended", () => e(), { once: !0 });
|
|
44
|
+
});
|
|
45
|
+
} catch (e) {
|
|
46
|
+
let t = e instanceof Error ? e.message : String(e);
|
|
47
|
+
_?.(t), b(t, "warn");
|
|
48
|
+
} finally {
|
|
49
|
+
T === i && (T = null, E = ""), w = Math.max(0, w - 1);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
if (p) try {
|
|
53
|
+
x.registerTextStreamHandler(m, (e, t) => {
|
|
54
|
+
if (S) return;
|
|
55
|
+
let n = t.identity, r = x.remoteParticipants.get(n);
|
|
56
|
+
(r ? d(r) : n.startsWith("agent")) && (async () => {
|
|
57
|
+
try {
|
|
58
|
+
let t = await e.readAll();
|
|
59
|
+
if (!t.trim()) return;
|
|
60
|
+
let n = (e.info?.attributes ?? {})["lk.transcription_final"] !== "false";
|
|
61
|
+
O(t, "replace"), n && b(`Agent transcript: ${t.trim().slice(0, 80)}`);
|
|
62
|
+
} catch (e) {
|
|
63
|
+
b(`Transcription stream error: ${e instanceof Error ? e.message : String(e)}`, "warn");
|
|
64
|
+
}
|
|
65
|
+
})();
|
|
66
|
+
}), b(`Listening for transcripts on text stream "${m}"`);
|
|
67
|
+
} catch (e) {
|
|
68
|
+
b(`Could not register transcription handler: ${e instanceof Error ? e.message : String(e)}`, "warn");
|
|
69
|
+
}
|
|
70
|
+
x.on(n.TrackSubscribed, (e, t, n) => {
|
|
71
|
+
k(e, n);
|
|
72
|
+
}), x.on(n.TrackUnsubscribed, (e) => {
|
|
73
|
+
e.kind === r.Kind.Audio && D();
|
|
74
|
+
}), x.on(n.Disconnected, () => {
|
|
75
|
+
C = !1, !S && (D(), b("LiveKit room disconnected"), g?.());
|
|
76
|
+
});
|
|
77
|
+
try {
|
|
78
|
+
await x.connect(v, y), C = !0, b("LiveKit room connected — waiting for agent audio tracks");
|
|
79
|
+
} catch (e) {
|
|
80
|
+
let t = e instanceof Error ? e.message : String(e);
|
|
81
|
+
throw Error(`LiveKit connect failed: ${t}`);
|
|
82
|
+
}
|
|
83
|
+
for (let e of x.remoteParticipants.values()) for (let t of e.audioTrackPublications.values()) {
|
|
84
|
+
let n = t.track;
|
|
85
|
+
n && t.isSubscribed && k(n, e);
|
|
86
|
+
}
|
|
87
|
+
return {
|
|
88
|
+
room: x,
|
|
89
|
+
isConnected: () => C && !S && x.state === e.Connected,
|
|
90
|
+
async end() {
|
|
91
|
+
if (!S) {
|
|
92
|
+
S = !0;
|
|
93
|
+
try {
|
|
94
|
+
x.unregisterTextStreamHandler(m);
|
|
95
|
+
} catch {}
|
|
96
|
+
if (w > 0) try {
|
|
97
|
+
await s.speech.interrupt({ scope: "active" });
|
|
98
|
+
} catch {}
|
|
99
|
+
T = null, E = "";
|
|
100
|
+
try {
|
|
101
|
+
await x.disconnect();
|
|
102
|
+
} catch (e) {
|
|
103
|
+
console.error(e);
|
|
104
|
+
}
|
|
105
|
+
C = !1;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
//#endregion
|
|
111
|
+
export { s as connectLiveKitAgent };
|
|
112
|
+
|
|
113
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","names":[],"sources":["../../src/livekit/connectLiveKitAgent.ts"],"sourcesContent":["/**\n * LiveKit remote audio → Liforma BYO voice helper.\n *\n * Mount a Liforma Experience (typically `mode=\"presenter\"`, `speechInputMode=\"off\"`),\n * wait until the player has started (audio unlocked), mint a LiveKit token on your\n * server, then call this helper.\n *\n * Bridges remote agent audio via `createUtterance({ track })` and optionally applies\n * LiveKit agent transcriptions (`lk.transcription` text streams) for force-align lipsync.\n * Do not also attach the remote track to an `<audio>` element.\n *\n * Docs: https://docs.liforma.ai/avatar-experiences/bring-your-own-voice/livekit\n */\n\nimport {\n\tConnectionState,\n\tRoom,\n\tRoomEvent,\n\tTrack,\n\ttype RemoteParticipant,\n\ttype RemoteTrack\n} from 'livekit-client';\nimport type {\n\tByoBridge,\n\tByoConnectCallbacks,\n\tByoPcmExperience,\n\tByoSpeechUtterance\n} from '@liforma/client/byo';\n\nexport type LiveKitAgentExperience = ByoPcmExperience;\n\nexport type ConnectLiveKitAgentOptions = ByoConnectCallbacks & {\n\t/** LiveKit server URL (wss://…). */\n\treadonly url: string;\n\t/** Participant token from your backend (never mint long-lived tokens in the browser). */\n\treadonly token: string;\n\t/**\n\t * Only bridge remote audio from participants matching this predicate.\n\t * Defaults to identities starting with `agent`.\n\t */\n\treadonly shouldBridgeParticipant?: (participant: RemoteParticipant) => boolean;\n\t/** Sample rate hint for track capture (LiveKit often delivers 48 kHz). */\n\treadonly sampleRate?: number;\n\t/**\n\t * When true (default), register a text-stream handler on `transcriptionTopic`\n\t * and forward agent text into `utterance.setTranscript` for force-align lipsync.\n\t */\n\treadonly enableTranscript?: boolean;\n\t/** LiveKit text-stream topic for transcriptions (default `lk.transcription`). */\n\treadonly transcriptionTopic?: string;\n};\n\nexport type LiveKitAgentBridge = ByoBridge & {\n\t/** Underlying LiveKit room (for advanced host control). */\n\treadonly room: Room;\n};\n\nconst DEFAULT_SAMPLE_RATE = 48_000;\nconst DEFAULT_TRANSCRIPTION_TOPIC = 'lk.transcription';\n\nfunction defaultShouldBridge(participant: RemoteParticipant): boolean {\n\treturn participant.identity.startsWith('agent');\n}\n\n/**\n * Connect a LiveKit room and pipe remote agent audio tracks into\n * `experience.speech.createUtterance({ track })` so the Liforma avatar talks.\n */\nexport async function connectLiveKitAgent(\n\texperience: LiveKitAgentExperience,\n\toptions: ConnectLiveKitAgentOptions\n): Promise<LiveKitAgentBridge> {\n\tconst {\n\t\turl,\n\t\ttoken,\n\t\tshouldBridgeParticipant = defaultShouldBridge,\n\t\tsampleRate = DEFAULT_SAMPLE_RATE,\n\t\tenableTranscript = true,\n\t\ttranscriptionTopic = DEFAULT_TRANSCRIPTION_TOPIC,\n\t\tonLog,\n\t\tonDisconnect,\n\t\tonError\n\t} = options;\n\n\tconst livekitUrl = url.trim();\n\tconst livekitToken = token.trim();\n\tif (!livekitUrl) throw new Error('Pass LiveKit url');\n\tif (!livekitToken) throw new Error('Pass LiveKit token from your server mint route');\n\n\tconst log = (line: string, kind: 'info' | 'warn' = 'info') => onLog?.(line, kind);\n\n\tconst room = new Room();\n\tlet ended = false;\n\tlet connected = false;\n\t/** Outstanding createUtterance bridges — interrupt only when > 0 (abandon-safe). */\n\tlet outstandingPlays = 0;\n\tlet activeUtterance: ByoSpeechUtterance | null = null;\n\tlet activeTranscript = '';\n\n\tfunction interruptIfPlaying(): void {\n\t\tif (outstandingPlays <= 0) return;\n\t\tactiveUtterance = null;\n\t\tactiveTranscript = '';\n\t\tvoid experience.speech.interrupt({ scope: 'active' }).catch(() => undefined);\n\t}\n\n\tfunction applyTranscript(text: string, mode: 'replace' | 'append'): void {\n\t\tconst next =\n\t\t\tmode === 'replace' ? text.trim() : `${activeTranscript}${text}`.trim();\n\t\tif (!next) return;\n\t\tactiveTranscript = next;\n\t\tif (!activeUtterance) return;\n\t\tvoid activeUtterance.setTranscript(next).catch(() => undefined);\n\t}\n\n\tasync function bridgeTrack(track: RemoteTrack, participant: RemoteParticipant): Promise<void> {\n\t\tif (ended || track.kind !== Track.Kind.Audio) return;\n\t\tif (!shouldBridgeParticipant(participant)) {\n\t\t\tlog(`Skipping audio from ${participant.identity}`);\n\t\t\treturn;\n\t\t}\n\n\t\tconst mediaTrack = track.mediaStreamTrack;\n\t\tif (!mediaTrack) {\n\t\t\tonError?.('LiveKit audio track missing mediaStreamTrack');\n\t\t\treturn;\n\t\t}\n\n\t\tlog(`Bridging LiveKit audio from ${participant.identity}`);\n\t\toutstandingPlays += 1;\n\t\tactiveTranscript = '';\n\t\tconst utterance = experience.speech.createUtterance({\n\t\t\ttrack: mediaTrack,\n\t\t\tsampleRate,\n\t\t\tqueue: 'replace-active'\n\t\t});\n\t\tactiveUtterance = utterance;\n\t\ttry {\n\t\t\tif (utterance.result) {\n\t\t\t\tawait utterance.result;\n\t\t\t} else {\n\t\t\t\t// Duck-type hosts without `result` — wait for the media track to end.\n\t\t\t\tawait new Promise<void>((resolve) => {\n\t\t\t\t\tif (mediaTrack.readyState === 'ended') {\n\t\t\t\t\t\tresolve();\n\t\t\t\t\t\treturn;\n\t\t\t\t\t}\n\t\t\t\t\tmediaTrack.addEventListener('ended', () => resolve(), { once: true });\n\t\t\t\t});\n\t\t\t}\n\t\t} catch (err) {\n\t\t\tconst message = err instanceof Error ? err.message : String(err);\n\t\t\tonError?.(message);\n\t\t\tlog(message, 'warn');\n\t\t} finally {\n\t\t\tif (activeUtterance === utterance) {\n\t\t\t\tactiveUtterance = null;\n\t\t\t\tactiveTranscript = '';\n\t\t\t}\n\t\t\toutstandingPlays = Math.max(0, outstandingPlays - 1);\n\t\t}\n\t}\n\n\tif (enableTranscript) {\n\t\ttry {\n\t\t\troom.registerTextStreamHandler(transcriptionTopic, (reader, participantInfo) => {\n\t\t\t\tif (ended) return;\n\t\t\t\tconst identity = participantInfo.identity;\n\t\t\t\tconst participant = room.remoteParticipants.get(identity);\n\t\t\t\tconst accepted = participant\n\t\t\t\t\t? shouldBridgeParticipant(participant)\n\t\t\t\t\t: identity.startsWith('agent');\n\t\t\t\tif (!accepted) return;\n\n\t\t\t\tvoid (async () => {\n\t\t\t\t\ttry {\n\t\t\t\t\t\tconst text = await reader.readAll();\n\t\t\t\t\t\tif (!text.trim()) return;\n\t\t\t\t\t\tconst attrs = reader.info?.attributes ?? {};\n\t\t\t\t\t\tconst isFinal = attrs['lk.transcription_final'] !== 'false';\n\t\t\t\t\t\t// Interim and final both replace — force-align wants the best current text.\n\t\t\t\t\t\tapplyTranscript(text, 'replace');\n\t\t\t\t\t\tif (isFinal) {\n\t\t\t\t\t\t\tlog(`Agent transcript: ${text.trim().slice(0, 80)}`);\n\t\t\t\t\t\t}\n\t\t\t\t\t} catch (err) {\n\t\t\t\t\t\tconst message = err instanceof Error ? err.message : String(err);\n\t\t\t\t\t\tlog(`Transcription stream error: ${message}`, 'warn');\n\t\t\t\t\t}\n\t\t\t\t})();\n\t\t\t});\n\t\t\tlog(`Listening for transcripts on text stream \"${transcriptionTopic}\"`);\n\t\t} catch (err) {\n\t\t\tconst message = err instanceof Error ? err.message : String(err);\n\t\t\tlog(`Could not register transcription handler: ${message}`, 'warn');\n\t\t}\n\t}\n\n\troom.on(RoomEvent.TrackSubscribed, (track, _publication, participant) => {\n\t\tvoid bridgeTrack(track as RemoteTrack, participant);\n\t});\n\n\troom.on(RoomEvent.TrackUnsubscribed, (track) => {\n\t\tif (track.kind !== Track.Kind.Audio) return;\n\t\tinterruptIfPlaying();\n\t});\n\n\troom.on(RoomEvent.Disconnected, () => {\n\t\tconnected = false;\n\t\tif (ended) return;\n\t\tinterruptIfPlaying();\n\t\tlog('LiveKit room disconnected');\n\t\tonDisconnect?.();\n\t});\n\n\ttry {\n\t\tawait room.connect(livekitUrl, livekitToken);\n\t\tconnected = true;\n\t\tlog('LiveKit room connected — waiting for agent audio tracks');\n\t} catch (err) {\n\t\tconst message = err instanceof Error ? err.message : String(err);\n\t\tthrow new Error(`LiveKit connect failed: ${message}`);\n\t}\n\n\tfor (const participant of room.remoteParticipants.values()) {\n\t\tfor (const publication of participant.audioTrackPublications.values()) {\n\t\t\tconst track = publication.track;\n\t\t\tif (track && publication.isSubscribed) {\n\t\t\t\tvoid bridgeTrack(track as RemoteTrack, participant);\n\t\t\t}\n\t\t}\n\t}\n\n\treturn {\n\t\troom,\n\t\tisConnected: () => connected && !ended && room.state === ConnectionState.Connected,\n\t\tasync end() {\n\t\t\tif (ended) return;\n\t\t\tended = true;\n\t\t\ttry {\n\t\t\t\troom.unregisterTextStreamHandler(transcriptionTopic);\n\t\t\t} catch {\n\t\t\t\t/* ignore */\n\t\t\t}\n\t\t\tif (outstandingPlays > 0) {\n\t\t\t\ttry {\n\t\t\t\t\tawait experience.speech.interrupt({ scope: 'active' });\n\t\t\t\t} catch {\n\t\t\t\t\t/* ignore */\n\t\t\t\t}\n\t\t\t}\n\t\t\tactiveUtterance = null;\n\t\t\tactiveTranscript = '';\n\t\t\ttry {\n\t\t\t\tawait room.disconnect();\n\t\t\t} catch (err) {\n\t\t\t\tconsole.error(err);\n\t\t\t}\n\t\t\tconnected = false;\n\t\t}\n\t};\n}\n"],"mappings":";;AAyDA,IAAM,IAAsB,MACtB,IAA8B;AAEpC,SAAS,EAAoB,GAAyC;CACrE,OAAO,EAAY,SAAS,WAAW,OAAO;AAC/C;AAMA,eAAsB,EACrB,GACA,GAC8B;CAC9B,IAAM,EACL,QACA,UACA,6BAA0B,GAC1B,gBAAa,GACb,sBAAmB,IACnB,wBAAqB,GACrB,UACA,iBACA,eACG,GAEE,IAAa,EAAI,KAAK,GACtB,IAAe,EAAM,KAAK;CAChC,IAAI,CAAC,GAAY,MAAU,MAAM,kBAAkB;CACnD,IAAI,CAAC,GAAc,MAAU,MAAM,gDAAgD;CAEnF,IAAM,KAAO,GAAc,IAAwB,WAAW,IAAQ,GAAM,CAAI,GAE1E,IAAO,IAAI,EAAK,GAClB,IAAQ,IACR,IAAY,IAEZ,IAAmB,GACnB,IAA6C,MAC7C,IAAmB;CAEvB,SAAS,IAA2B;EAC/B,KAAoB,MACxB,IAAkB,MAClB,IAAmB,IACnB,EAAgB,OAAO,UAAU,EAAE,OAAO,SAAS,CAAC,EAAE,YAAY,KAAA,CAAS;CAC5E;CAEA,SAAS,EAAgB,GAAc,GAAkC;EACxE,IAAM,IACL,MAAS,YAAY,EAAK,KAAK,IAAI,GAAG,IAAmB,IAAO,KAAK;EACjE,MACL,IAAmB,GACd,KACL,EAAqB,cAAc,CAAI,EAAE,YAAY,KAAA,CAAS;CAC/D;CAEA,eAAe,EAAY,GAAoB,GAA+C;EAC7F,IAAI,KAAS,EAAM,SAAS,EAAM,KAAK,OAAO;EAC9C,IAAI,CAAC,EAAwB,CAAW,GAAG;GAC1C,EAAI,uBAAuB,EAAY,UAAU;GACjD;EACD;EAEA,IAAM,IAAa,EAAM;EACzB,IAAI,CAAC,GAAY;GAChB,IAAU,8CAA8C;GACxD;EACD;EAIA,AAFA,EAAI,+BAA+B,EAAY,UAAU,GACzD,KAAoB,GACpB,IAAmB;EACnB,IAAM,IAAY,EAAW,OAAO,gBAAgB;GACnD,OAAO;GACP;GACA,OAAO;EACR,CAAC;EACD,IAAkB;EAClB,IAAI;GACH,AAAI,EAAU,SACb,MAAM,EAAU,SAGhB,MAAM,IAAI,SAAe,MAAY;IACpC,IAAI,EAAW,eAAe,SAAS;KACtC,EAAQ;KACR;IACD;IACA,EAAW,iBAAiB,eAAe,EAAQ,GAAG,EAAE,MAAM,GAAK,CAAC;GACrE,CAAC;EAEH,SAAS,GAAK;GACb,IAAM,IAAU,aAAe,QAAQ,EAAI,UAAU,OAAO,CAAG;GAE/D,AADA,IAAU,CAAO,GACjB,EAAI,GAAS,MAAM;EACpB,UAAU;GAKT,AAJI,MAAoB,MACvB,IAAkB,MAClB,IAAmB,KAEpB,IAAmB,KAAK,IAAI,GAAG,IAAmB,CAAC;EACpD;CACD;CAEA,IAAI,GACH,IAAI;EA2BH,AA1BA,EAAK,0BAA0B,IAAqB,GAAQ,MAAoB;GAC/E,IAAI,GAAO;GACX,IAAM,IAAW,EAAgB,UAC3B,IAAc,EAAK,mBAAmB,IAAI,CAAQ;GACvC,KACd,EAAwB,CAAW,IACnC,EAAS,WAAW,OAAO,OAGxB,YAAY;IACjB,IAAI;KACH,IAAM,IAAO,MAAM,EAAO,QAAQ;KAClC,IAAI,CAAC,EAAK,KAAK,GAAG;KAElB,IAAM,KADQ,EAAO,MAAM,cAAc,CAAC,GACpB,8BAA8B;KAGpD,AADA,EAAgB,GAAM,SAAS,GAC3B,KACH,EAAI,qBAAqB,EAAK,KAAK,EAAE,MAAM,GAAG,EAAE,GAAG;IAErD,SAAS,GAAK;KAEb,EAAI,+BADY,aAAe,QAAQ,EAAI,UAAU,OAAO,CAAG,KACjB,MAAM;IACrD;GACD,GAAG;EACJ,CAAC,GACD,EAAI,6CAA6C,EAAmB,EAAE;CACvE,SAAS,GAAK;EAEb,EAAI,6CADY,aAAe,QAAQ,EAAI,UAAU,OAAO,CAAG,KACH,MAAM;CACnE;CAYD,AATA,EAAK,GAAG,EAAU,kBAAkB,GAAO,GAAc,MAAgB;EACxE,EAAiB,GAAsB,CAAW;CACnD,CAAC,GAED,EAAK,GAAG,EAAU,oBAAoB,MAAU;EAC3C,EAAM,SAAS,EAAM,KAAK,SAC9B,EAAmB;CACpB,CAAC,GAED,EAAK,GAAG,EAAU,oBAAoB;EACrC,IAAY,IACR,OACJ,EAAmB,GACnB,EAAI,2BAA2B,GAC/B,IAAe;CAChB,CAAC;CAED,IAAI;EAGH,AAFA,MAAM,EAAK,QAAQ,GAAY,CAAY,GAC3C,IAAY,IACZ,EAAI,yDAAyD;CAC9D,SAAS,GAAK;EACb,IAAM,IAAU,aAAe,QAAQ,EAAI,UAAU,OAAO,CAAG;EAC/D,MAAU,MAAM,2BAA2B,GAAS;CACrD;CAEA,KAAK,IAAM,KAAe,EAAK,mBAAmB,OAAO,GACxD,KAAK,IAAM,KAAe,EAAY,uBAAuB,OAAO,GAAG;EACtE,IAAM,IAAQ,EAAY;EAC1B,AAAI,KAAS,EAAY,gBACxB,EAAiB,GAAsB,CAAW;CAEpD;CAGD,OAAO;EACN;EACA,mBAAmB,KAAa,CAAC,KAAS,EAAK,UAAU,EAAgB;EACzE,MAAM,MAAM;GACP,QACJ;QAAQ;IACR,IAAI;KACH,EAAK,4BAA4B,CAAkB;IACpD,QAAQ,CAER;IACA,IAAI,IAAmB,GACtB,IAAI;KACH,MAAM,EAAW,OAAO,UAAU,EAAE,OAAO,SAAS,CAAC;IACtD,QAAQ,CAER;IAGD,AADA,IAAkB,MAClB,IAAmB;IACnB,IAAI;KACH,MAAM,EAAK,WAAW;IACvB,SAAS,GAAK;KACb,QAAQ,MAAM,CAAG;IAClB;IACA,IAAY;GApBJ;EAqBT;CACD;AACD"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared Web Audio capture: AudioWorklet when available, ScriptProcessor fallback.
|
|
3
|
+
* Used by BYO mic uplink and MediaStreamTrack → PCM piping.
|
|
4
|
+
*/
|
|
5
|
+
export type AudioCaptureSession = {
|
|
6
|
+
stop(): void;
|
|
7
|
+
readonly fromSampleRate: number;
|
|
8
|
+
readonly usingWorklet: boolean;
|
|
9
|
+
};
|
|
10
|
+
export type StartAudioCaptureOptions = {
|
|
11
|
+
readonly mediaStream: MediaStream;
|
|
12
|
+
readonly targetSampleRate: number;
|
|
13
|
+
/** Called with pcm_s16le mono at targetSampleRate. */
|
|
14
|
+
readonly onPcm: (bytes: Uint8Array) => void;
|
|
15
|
+
/** When false, frames are dropped. */
|
|
16
|
+
readonly shouldSend?: () => boolean;
|
|
17
|
+
readonly onLog?: (line: string, kind?: 'info' | 'warn') => void;
|
|
18
|
+
};
|
|
19
|
+
/**
|
|
20
|
+
* Capture a MediaStream as pcm_s16le at targetSampleRate.
|
|
21
|
+
* Prefers AudioWorklet; falls back to ScriptProcessorNode.
|
|
22
|
+
*/
|
|
23
|
+
export declare function startAudioCapture(options: StartAudioCaptureOptions): Promise<AudioCaptureSession>;
|
|
24
|
+
//# sourceMappingURL=audioCapture.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"audioCapture.d.ts","sourceRoot":"","sources":["../../../src/byo/audioCapture.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,MAAM,MAAM,mBAAmB,GAAG;IACjC,IAAI,IAAI,IAAI,CAAC;IACb,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC;CAC/B,CAAC;AAEF,MAAM,MAAM,wBAAwB,GAAG;IACtC,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;IAClC,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,sDAAsD;IACtD,QAAQ,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,UAAU,KAAK,IAAI,CAAC;IAC5C,sCAAsC;IACtC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,OAAO,CAAC;IACpC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,KAAK,IAAI,CAAC;CAChE,CAAC;AAsCF;;;GAGG;AACH,wBAAsB,iBAAiB,CACtC,OAAO,EAAE,wBAAwB,GAC/B,OAAO,CAAC,mBAAmB,CAAC,CAoI9B"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/** Liforma `speech.write` rejects chunks larger than 64 KiB. */
|
|
2
|
+
export declare const MAX_PCM_CHUNK_BYTES: number;
|
|
3
|
+
/**
|
|
4
|
+
* Playback / chunk guidance for custom BYO bridges (mirrors speech controller limits).
|
|
5
|
+
* Prefer ~20–200 ms PCM frames; the player starts after an adaptive ~80–250 ms jitter
|
|
6
|
+
* buffer (default 150 ms) based on chunk arrival variance.
|
|
7
|
+
*/
|
|
8
|
+
export declare const BYO_PLAYBACK_GUIDANCE: {
|
|
9
|
+
readonly RECOMMENDED_CHUNK_MS_MIN: 20;
|
|
10
|
+
readonly RECOMMENDED_CHUNK_MS_MAX: 200;
|
|
11
|
+
readonly PLAYBACK_JITTER_BUFFER_MS: 150;
|
|
12
|
+
readonly PLAYBACK_JITTER_BUFFER_MS_MIN: 80;
|
|
13
|
+
readonly PLAYBACK_JITTER_BUFFER_MS_MAX: 250;
|
|
14
|
+
};
|
|
15
|
+
export declare function base64ToPcmBytes(b64: string): Uint8Array;
|
|
16
|
+
export declare function pcmBytesToBase64(bytes: Uint8Array): string;
|
|
17
|
+
export declare function floatToPcm16Le(input: Float32Array): Uint8Array;
|
|
18
|
+
/** Naive linear resample Float32 → target rate. */
|
|
19
|
+
export declare function resampleFloat32(input: Float32Array, fromRate: number, toRate: number): Float32Array;
|
|
20
|
+
/**
|
|
21
|
+
* Parse vendor format strings like `pcm_16000` / `pcm_24000`.
|
|
22
|
+
* Returns null when the format is missing or not PCM.
|
|
23
|
+
*/
|
|
24
|
+
export declare function parsePcmSampleRate(format: string | null | undefined): number | null;
|
|
25
|
+
/**
|
|
26
|
+
* Write even-length PCM, splitting at MAX_PCM_CHUNK_BYTES on even boundaries.
|
|
27
|
+
* Callers that may receive odd-length frames should run bytes through
|
|
28
|
+
* {@link takeEvenPcmBytes} first so samples are not dropped across chunks.
|
|
29
|
+
*/
|
|
30
|
+
export declare function writePcmInChunks(utterance: {
|
|
31
|
+
write: (chunk: ArrayBuffer | ArrayBufferView) => Promise<unknown>;
|
|
32
|
+
}, bytes: Uint8Array): Promise<void>;
|
|
33
|
+
//# sourceMappingURL=pcm.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pcm.d.ts","sourceRoot":"","sources":["../../../src/byo/pcm.ts"],"names":[],"mappings":"AAAA,gEAAgE;AAChE,eAAO,MAAM,mBAAmB,QAAY,CAAC;AAE7C;;;;GAIG;AACH,eAAO,MAAM,qBAAqB;;;;;;CAMxB,CAAC;AAEX,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,UAAU,CAKxD;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,CAO1D;AAED,wBAAgB,cAAc,CAAC,KAAK,EAAE,YAAY,GAAG,UAAU,CAQ9D;AAED,mDAAmD;AACnD,wBAAgB,eAAe,CAAC,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,YAAY,CAanG;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,MAAM,GAAG,IAAI,CAKnF;AAED;;;;GAIG;AACH,wBAAsB,gBAAgB,CACrC,SAAS,EAAE;IAAE,KAAK,EAAE,CAAC,KAAK,EAAE,WAAW,GAAG,eAAe,KAAK,OAAO,CAAC,OAAO,CAAC,CAAA;CAAE,EAChF,KAAK,EAAE,UAAU,GACf,OAAO,CAAC,IAAI,CAAC,CAmBf"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Stateful cubic (Catmull-Rom) resampler for continuous PCM capture streams.
|
|
3
|
+
* Better than naive linear for uplink mic / track capture.
|
|
4
|
+
*/
|
|
5
|
+
export type FloatResampler = {
|
|
6
|
+
/** Push a mono float buffer; returns samples at the target rate (may be empty). */
|
|
7
|
+
push(input: Float32Array): Float32Array;
|
|
8
|
+
/** Flush any remaining fractional sample (usually silence). */
|
|
9
|
+
flush(): Float32Array;
|
|
10
|
+
readonly fromRate: number;
|
|
11
|
+
readonly toRate: number;
|
|
12
|
+
};
|
|
13
|
+
/**
|
|
14
|
+
* Create a continuous resampler. When rates match, push() returns the input as-is
|
|
15
|
+
* (no copy) when possible.
|
|
16
|
+
*/
|
|
17
|
+
export declare function createCubicResampler(fromRate: number, toRate: number): FloatResampler;
|
|
18
|
+
//# sourceMappingURL=resample.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"resample.d.ts","sourceRoot":"","sources":["../../../src/byo/resample.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,MAAM,MAAM,cAAc,GAAG;IAC5B,mFAAmF;IACnF,IAAI,CAAC,KAAK,EAAE,YAAY,GAAG,YAAY,CAAC;IACxC,+DAA+D;IAC/D,KAAK,IAAI,YAAY,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACxB,CAAC;AAUF;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,cAAc,CAkErF"}
|