@inkibra/voice-sdk 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +75 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +673 -0
- package/dist/public.d.ts +109 -0
- package/dist/server.d.ts +120 -0
- package/dist/server.js +392 -0
- package/package.json +35 -0
package/dist/index.js
ADDED
|
@@ -0,0 +1,673 @@
|
|
|
1
|
+
// src/client.ts
|
|
2
|
+
function sessionStartMessage(options) {
|
|
3
|
+
return {
|
|
4
|
+
type: "session.start",
|
|
5
|
+
developer: options.developer
|
|
6
|
+
};
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
class LiveVoiceClient {
|
|
10
|
+
endpoint;
|
|
11
|
+
callbacks;
|
|
12
|
+
socket;
|
|
13
|
+
microphone;
|
|
14
|
+
output;
|
|
15
|
+
preparation;
|
|
16
|
+
resolvePrepared;
|
|
17
|
+
rejectPrepared;
|
|
18
|
+
prepared = false;
|
|
19
|
+
sessionToken;
|
|
20
|
+
starting;
|
|
21
|
+
resolveStarted;
|
|
22
|
+
rejectStarted;
|
|
23
|
+
constructor(endpoint, callbacks) {
|
|
24
|
+
this.endpoint = endpoint;
|
|
25
|
+
this.callbacks = callbacks;
|
|
26
|
+
}
|
|
27
|
+
async prepare(token) {
|
|
28
|
+
if (this.prepared && this.sessionToken === token && this.socket?.readyState === WebSocket.OPEN)
|
|
29
|
+
return;
|
|
30
|
+
if (this.preparation && this.sessionToken === token)
|
|
31
|
+
return this.preparation;
|
|
32
|
+
if (this.socket)
|
|
33
|
+
this.stop();
|
|
34
|
+
this.sessionToken = token;
|
|
35
|
+
this.callbacks.onConnection("connecting");
|
|
36
|
+
const endpoint = this.endpoint.replace(/\/$/u, "");
|
|
37
|
+
const socket = new WebSocket(withSessionToken(`${endpoint}/${crypto.randomUUID()}`, token));
|
|
38
|
+
this.socket = socket;
|
|
39
|
+
socket.binaryType = "arraybuffer";
|
|
40
|
+
socket.addEventListener("message", (event) => this.onMessage(event.data));
|
|
41
|
+
socket.addEventListener("close", (event) => {
|
|
42
|
+
if (this.socket !== socket)
|
|
43
|
+
return;
|
|
44
|
+
const wasPrepared = this.prepared;
|
|
45
|
+
if (!wasPrepared)
|
|
46
|
+
this.rejectPrepared?.(new Error("Session connection closed before preparation completed."));
|
|
47
|
+
this.microphone?.stop();
|
|
48
|
+
this.microphone = undefined;
|
|
49
|
+
this.output?.clear();
|
|
50
|
+
this.output = undefined;
|
|
51
|
+
this.socket = undefined;
|
|
52
|
+
this.prepared = false;
|
|
53
|
+
this.preparation = undefined;
|
|
54
|
+
this.resolvePrepared = undefined;
|
|
55
|
+
this.rejectPrepared = undefined;
|
|
56
|
+
this.sessionToken = undefined;
|
|
57
|
+
this.rejectStarted?.(new Error("Session connection closed before startup completed."));
|
|
58
|
+
this.clearStarting();
|
|
59
|
+
this.callbacks.onConnection("closed", event.reason || undefined);
|
|
60
|
+
});
|
|
61
|
+
socket.addEventListener("error", () => {
|
|
62
|
+
if (this.socket !== socket)
|
|
63
|
+
return;
|
|
64
|
+
const message = "Durable Object WebSocket failed.";
|
|
65
|
+
this.failPreparation(message);
|
|
66
|
+
this.rejectStarted?.(new Error(message));
|
|
67
|
+
this.clearStarting();
|
|
68
|
+
this.callbacks.onConnection("error", message);
|
|
69
|
+
});
|
|
70
|
+
this.preparation = new Promise((resolve, reject) => {
|
|
71
|
+
this.resolvePrepared = resolve;
|
|
72
|
+
this.rejectPrepared = reject;
|
|
73
|
+
});
|
|
74
|
+
await waitForOpen(socket);
|
|
75
|
+
socket.send(JSON.stringify({ type: "session.prepare", protocolVersion: 1 }));
|
|
76
|
+
return this.preparation;
|
|
77
|
+
}
|
|
78
|
+
async start(options) {
|
|
79
|
+
await this.prepare(options.token);
|
|
80
|
+
this.output = new OutputAudio((receipt) => {
|
|
81
|
+
if (this.socket?.readyState === WebSocket.OPEN)
|
|
82
|
+
this.socket.send(JSON.stringify({ type: "output.receipt", ...receipt }));
|
|
83
|
+
});
|
|
84
|
+
await this.output.resume();
|
|
85
|
+
this.starting = new Promise((resolve, reject) => {
|
|
86
|
+
this.resolveStarted = resolve;
|
|
87
|
+
this.rejectStarted = reject;
|
|
88
|
+
});
|
|
89
|
+
this.socket?.send(JSON.stringify(sessionStartMessage(options)));
|
|
90
|
+
await this.starting;
|
|
91
|
+
this.microphone = new Microphone((buffer) => {
|
|
92
|
+
if (this.socket?.readyState === WebSocket.OPEN)
|
|
93
|
+
this.socket.send(buffer);
|
|
94
|
+
}, (level) => {
|
|
95
|
+
this.output?.sidechain(level);
|
|
96
|
+
this.callbacks.onLevel(level);
|
|
97
|
+
});
|
|
98
|
+
await this.microphone.start();
|
|
99
|
+
this.callbacks.onConnection("live");
|
|
100
|
+
}
|
|
101
|
+
stop() {
|
|
102
|
+
this.microphone?.stop();
|
|
103
|
+
this.microphone = undefined;
|
|
104
|
+
this.output?.clear();
|
|
105
|
+
this.output = undefined;
|
|
106
|
+
if (this.socket?.readyState === WebSocket.OPEN)
|
|
107
|
+
this.socket.send(JSON.stringify({ type: "session.stop" }));
|
|
108
|
+
this.socket?.close(1000, "stopped");
|
|
109
|
+
this.socket = undefined;
|
|
110
|
+
this.prepared = false;
|
|
111
|
+
this.preparation = undefined;
|
|
112
|
+
this.resolvePrepared = undefined;
|
|
113
|
+
this.rejectPrepared = undefined;
|
|
114
|
+
this.sessionToken = undefined;
|
|
115
|
+
this.rejectStarted?.(new Error("Session stopped before startup completed."));
|
|
116
|
+
this.clearStarting();
|
|
117
|
+
this.callbacks.onConnection("closed");
|
|
118
|
+
}
|
|
119
|
+
dispose() {
|
|
120
|
+
this.stop();
|
|
121
|
+
}
|
|
122
|
+
playUiCue(cue) {
|
|
123
|
+
if (this.socket?.readyState === WebSocket.OPEN)
|
|
124
|
+
this.socket.send(JSON.stringify({ type: "cue.request", cue }));
|
|
125
|
+
}
|
|
126
|
+
onMessage(raw) {
|
|
127
|
+
if (typeof raw !== "string")
|
|
128
|
+
return;
|
|
129
|
+
let message;
|
|
130
|
+
try {
|
|
131
|
+
message = JSON.parse(raw);
|
|
132
|
+
} catch {
|
|
133
|
+
return;
|
|
134
|
+
}
|
|
135
|
+
const type = String(message.type ?? "");
|
|
136
|
+
if (type === "session.prepared") {
|
|
137
|
+
this.prepared = true;
|
|
138
|
+
this.resolvePrepared?.();
|
|
139
|
+
this.resolvePrepared = undefined;
|
|
140
|
+
this.rejectPrepared = undefined;
|
|
141
|
+
return;
|
|
142
|
+
}
|
|
143
|
+
if (type === "session.prepare_failed") {
|
|
144
|
+
this.failPreparation(String(message.message ?? "Session preparation failed."));
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
if (type === "session.started") {
|
|
148
|
+
this.resolveStarted?.();
|
|
149
|
+
this.clearStarting();
|
|
150
|
+
return;
|
|
151
|
+
}
|
|
152
|
+
if (type === "public.event") {
|
|
153
|
+
this.callbacks.onPublicEvent({
|
|
154
|
+
name: String(message.name ?? "unknown"),
|
|
155
|
+
payload: asRecord(message.payload),
|
|
156
|
+
...Number.isFinite(Number(message.atMs)) ? { atMs: Number(message.atMs) } : {}
|
|
157
|
+
});
|
|
158
|
+
return;
|
|
159
|
+
}
|
|
160
|
+
if (type === "tool.call") {
|
|
161
|
+
this.handleToolCall(message);
|
|
162
|
+
return;
|
|
163
|
+
}
|
|
164
|
+
if (type === "session.limit") {
|
|
165
|
+
this.callbacks.onSessionLimit?.(String(message.reason ?? "SESSION_LIMIT"), Number.isFinite(Number(message.durationSeconds)) ? Number(message.durationSeconds) : undefined);
|
|
166
|
+
return;
|
|
167
|
+
}
|
|
168
|
+
if (type === "output.block") {
|
|
169
|
+
this.output?.blockForeground(String(message.generationId ?? ""));
|
|
170
|
+
return;
|
|
171
|
+
}
|
|
172
|
+
if (type === "output.audio" && typeof message.delta === "string") {
|
|
173
|
+
this.output?.enqueue({
|
|
174
|
+
encoded: message.delta,
|
|
175
|
+
lane: message.queue === "reaction" ? "REACTION" : "FOREGROUND",
|
|
176
|
+
generationId: String(message.generationId ?? ""),
|
|
177
|
+
mode: message.mode === "play" ? "play" : "buffer",
|
|
178
|
+
gain: clamp01(Number(message.gain ?? 1))
|
|
179
|
+
});
|
|
180
|
+
return;
|
|
181
|
+
}
|
|
182
|
+
if (type === "output.release") {
|
|
183
|
+
this.output?.release(String(message.generationId ?? ""), clamp01(Number(message.gain ?? 1)));
|
|
184
|
+
return;
|
|
185
|
+
}
|
|
186
|
+
if (type === "output.cancel") {
|
|
187
|
+
this.output?.cancel(String(message.generationId ?? ""));
|
|
188
|
+
return;
|
|
189
|
+
}
|
|
190
|
+
if (type === "output.done") {
|
|
191
|
+
this.output?.done(message.queue === "reaction" ? "REACTION" : "FOREGROUND", String(message.generationId ?? ""));
|
|
192
|
+
return;
|
|
193
|
+
}
|
|
194
|
+
this.callbacks.onUnknown?.(message);
|
|
195
|
+
}
|
|
196
|
+
async handleToolCall(message) {
|
|
197
|
+
const callId = String(message.callId ?? "");
|
|
198
|
+
const name = String(message.name ?? "");
|
|
199
|
+
if (!callId || !name || this.socket?.readyState !== WebSocket.OPEN)
|
|
200
|
+
return;
|
|
201
|
+
try {
|
|
202
|
+
if (!this.callbacks.onToolCall)
|
|
203
|
+
throw new Error(`No browser handler is registered for tool ${name}`);
|
|
204
|
+
const output = await this.callbacks.onToolCall({ callId, name, input: asRecord(message.input) });
|
|
205
|
+
this.socket?.send(JSON.stringify({ type: "tool.result", callId, ok: true, output: output ?? null }));
|
|
206
|
+
} catch (error) {
|
|
207
|
+
this.socket?.send(JSON.stringify({
|
|
208
|
+
type: "tool.result",
|
|
209
|
+
callId,
|
|
210
|
+
ok: false,
|
|
211
|
+
error: error instanceof Error ? error.message : String(error)
|
|
212
|
+
}));
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
failPreparation(message) {
|
|
216
|
+
if (!this.prepared)
|
|
217
|
+
this.rejectPrepared?.(new Error(message));
|
|
218
|
+
this.preparation = undefined;
|
|
219
|
+
this.resolvePrepared = undefined;
|
|
220
|
+
this.rejectPrepared = undefined;
|
|
221
|
+
}
|
|
222
|
+
clearStarting() {
|
|
223
|
+
this.starting = undefined;
|
|
224
|
+
this.resolveStarted = undefined;
|
|
225
|
+
this.rejectStarted = undefined;
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
function withSessionToken(value, token) {
|
|
229
|
+
if (!token)
|
|
230
|
+
return value;
|
|
231
|
+
const url = new URL(value);
|
|
232
|
+
url.searchParams.set("token", token);
|
|
233
|
+
return url.toString();
|
|
234
|
+
}
|
|
235
|
+
var OPTICAL_SIDECHAIN = {
|
|
236
|
+
thresholdDb: -42,
|
|
237
|
+
ratio: 4,
|
|
238
|
+
maxReductionDb: 12,
|
|
239
|
+
attackSeconds: 0.12,
|
|
240
|
+
releaseSeconds: 0.36
|
|
241
|
+
};
|
|
242
|
+
|
|
243
|
+
class OutputAudio {
|
|
244
|
+
onReceipt;
|
|
245
|
+
context = new AudioContext({ sampleRate: 24000, latencyHint: "interactive" });
|
|
246
|
+
foregroundGain = this.context.createGain();
|
|
247
|
+
reactionGain = this.context.createGain();
|
|
248
|
+
outputBus = this.context.createGain();
|
|
249
|
+
nextStart = 0;
|
|
250
|
+
sources = new Map;
|
|
251
|
+
buffered = new Map;
|
|
252
|
+
released = new Set;
|
|
253
|
+
blockers = new Set;
|
|
254
|
+
started = new Set;
|
|
255
|
+
startedTimers = new Map;
|
|
256
|
+
completed = new Set;
|
|
257
|
+
doneGenerations = new Map;
|
|
258
|
+
playedSamples = new Map;
|
|
259
|
+
constructor(onReceipt) {
|
|
260
|
+
this.onReceipt = onReceipt;
|
|
261
|
+
this.foregroundGain.connect(this.outputBus);
|
|
262
|
+
this.reactionGain.connect(this.outputBus);
|
|
263
|
+
this.outputBus.connect(this.context.destination);
|
|
264
|
+
}
|
|
265
|
+
async resume() {
|
|
266
|
+
await this.context.resume();
|
|
267
|
+
}
|
|
268
|
+
sidechain(level) {
|
|
269
|
+
const gain = opticalSidechainGain(level);
|
|
270
|
+
const now = this.context.currentTime;
|
|
271
|
+
const current = this.foregroundGain.gain.value;
|
|
272
|
+
this.foregroundGain.gain.cancelAndHoldAtTime(now);
|
|
273
|
+
this.foregroundGain.gain.setTargetAtTime(gain, now, gain < current ? OPTICAL_SIDECHAIN.attackSeconds : OPTICAL_SIDECHAIN.releaseSeconds);
|
|
274
|
+
}
|
|
275
|
+
blockForeground(generationId) {
|
|
276
|
+
if (generationId)
|
|
277
|
+
this.blockers.add(generationId);
|
|
278
|
+
}
|
|
279
|
+
enqueue(chunk) {
|
|
280
|
+
if (!chunk.generationId)
|
|
281
|
+
return;
|
|
282
|
+
if (chunk.lane === "REACTION")
|
|
283
|
+
this.blockers.add(chunk.generationId);
|
|
284
|
+
const blocked = chunk.lane === "FOREGROUND" && this.blockers.size > 0;
|
|
285
|
+
if (blocked || chunk.mode === "buffer" && !this.released.has(chunk.generationId)) {
|
|
286
|
+
const queue = this.buffered.get(chunk.generationId) ?? [];
|
|
287
|
+
queue.push({ encoded: chunk.encoded, lane: chunk.lane, gain: chunk.gain });
|
|
288
|
+
this.buffered.set(chunk.generationId, queue);
|
|
289
|
+
return;
|
|
290
|
+
}
|
|
291
|
+
this.schedule(chunk.encoded, chunk.lane, chunk.generationId, chunk.gain);
|
|
292
|
+
}
|
|
293
|
+
release(generationId, gain = 1) {
|
|
294
|
+
if (!generationId)
|
|
295
|
+
return;
|
|
296
|
+
const chunks = this.buffered.get(generationId);
|
|
297
|
+
if (chunks?.[0]?.lane === "REACTION") {
|
|
298
|
+
for (const chunk of chunks)
|
|
299
|
+
chunk.gain = gain;
|
|
300
|
+
}
|
|
301
|
+
this.released.add(generationId);
|
|
302
|
+
const lane = this.buffered.get(generationId)?.[0]?.lane ?? this.doneGenerations.get(generationId);
|
|
303
|
+
if (lane === "REACTION" || !this.blockers.size)
|
|
304
|
+
this.flush(generationId);
|
|
305
|
+
}
|
|
306
|
+
done(lane, generationId) {
|
|
307
|
+
this.doneGenerations.set(generationId, lane);
|
|
308
|
+
this.finishIfComplete(generationId);
|
|
309
|
+
}
|
|
310
|
+
cancel(generationId) {
|
|
311
|
+
if (!generationId)
|
|
312
|
+
return;
|
|
313
|
+
const bufferedLane = this.buffered.get(generationId)?.[0]?.lane;
|
|
314
|
+
const sourceLane = [...this.sources.values()].find((value) => value.generationId === generationId)?.lane;
|
|
315
|
+
const lane = sourceLane ?? bufferedLane ?? this.doneGenerations.get(generationId) ?? "FOREGROUND";
|
|
316
|
+
this.buffered.delete(generationId);
|
|
317
|
+
this.released.delete(generationId);
|
|
318
|
+
this.blockers.delete(generationId);
|
|
319
|
+
this.doneGenerations.delete(generationId);
|
|
320
|
+
for (const [source, metadata] of this.sources) {
|
|
321
|
+
if (metadata.generationId !== generationId)
|
|
322
|
+
continue;
|
|
323
|
+
const elapsed = Math.max(0, this.context.currentTime - metadata.startAt);
|
|
324
|
+
const played = Math.min(metadata.samples, Math.floor(elapsed * 24000));
|
|
325
|
+
this.playedSamples.set(generationId, (this.playedSamples.get(generationId) ?? 0) + played);
|
|
326
|
+
source.onended = null;
|
|
327
|
+
this.sources.delete(source);
|
|
328
|
+
try {
|
|
329
|
+
source.stop();
|
|
330
|
+
} catch {}
|
|
331
|
+
}
|
|
332
|
+
this.recomputeNextStart();
|
|
333
|
+
const timer = this.startedTimers.get(generationId);
|
|
334
|
+
if (timer !== undefined)
|
|
335
|
+
window.clearTimeout(timer);
|
|
336
|
+
this.startedTimers.delete(generationId);
|
|
337
|
+
this.flushReleasedForeground();
|
|
338
|
+
this.onReceipt({ event: "cancelled", lane, generationId, playedSamples: this.playedSamples.get(generationId) ?? 0 });
|
|
339
|
+
}
|
|
340
|
+
clear() {
|
|
341
|
+
for (const source of this.sources.keys())
|
|
342
|
+
try {
|
|
343
|
+
source.stop();
|
|
344
|
+
} catch {}
|
|
345
|
+
for (const timer of this.startedTimers.values())
|
|
346
|
+
window.clearTimeout(timer);
|
|
347
|
+
this.sources.clear();
|
|
348
|
+
this.buffered.clear();
|
|
349
|
+
this.released.clear();
|
|
350
|
+
this.blockers.clear();
|
|
351
|
+
this.started.clear();
|
|
352
|
+
this.startedTimers.clear();
|
|
353
|
+
this.completed.clear();
|
|
354
|
+
this.doneGenerations.clear();
|
|
355
|
+
this.playedSamples.clear();
|
|
356
|
+
this.nextStart = 0;
|
|
357
|
+
this.context.close();
|
|
358
|
+
}
|
|
359
|
+
flush(generationId) {
|
|
360
|
+
const chunks = this.buffered.get(generationId);
|
|
361
|
+
if (!chunks)
|
|
362
|
+
return;
|
|
363
|
+
this.buffered.delete(generationId);
|
|
364
|
+
for (const chunk of chunks)
|
|
365
|
+
this.schedule(chunk.encoded, chunk.lane, generationId, chunk.gain);
|
|
366
|
+
}
|
|
367
|
+
flushReleasedForeground() {
|
|
368
|
+
if (this.blockers.size)
|
|
369
|
+
return;
|
|
370
|
+
for (const generationId of this.released)
|
|
371
|
+
this.flush(generationId);
|
|
372
|
+
}
|
|
373
|
+
schedule(encoded, lane, generationId, gain) {
|
|
374
|
+
const binary = atob(encoded);
|
|
375
|
+
const bytes = Uint8Array.from(binary, (character) => character.charCodeAt(0));
|
|
376
|
+
const pcm = new Int16Array(bytes.buffer);
|
|
377
|
+
const audio = this.context.createBuffer(1, pcm.length, 24000);
|
|
378
|
+
const samples = audio.getChannelData(0);
|
|
379
|
+
for (let index = 0;index < pcm.length; index += 1)
|
|
380
|
+
samples[index] = pcm[index] / 32768;
|
|
381
|
+
const source = this.context.createBufferSource();
|
|
382
|
+
const chunkGain = this.context.createGain();
|
|
383
|
+
chunkGain.gain.value = lane === "REACTION" ? gain : 1;
|
|
384
|
+
source.buffer = audio;
|
|
385
|
+
source.connect(chunkGain).connect(lane === "REACTION" ? this.reactionGain : this.foregroundGain);
|
|
386
|
+
const startAt = Math.max(this.context.currentTime + 0.008, this.nextStart);
|
|
387
|
+
this.nextStart = startAt + audio.duration;
|
|
388
|
+
this.sources.set(source, { generationId, lane, samples: pcm.length, startAt });
|
|
389
|
+
source.onended = () => {
|
|
390
|
+
const metadata = this.sources.get(source);
|
|
391
|
+
this.sources.delete(source);
|
|
392
|
+
this.recomputeNextStart();
|
|
393
|
+
if (!metadata)
|
|
394
|
+
return;
|
|
395
|
+
this.playedSamples.set(generationId, (this.playedSamples.get(generationId) ?? 0) + metadata.samples);
|
|
396
|
+
this.finishIfComplete(generationId);
|
|
397
|
+
};
|
|
398
|
+
source.start(startAt);
|
|
399
|
+
if (!this.started.has(generationId)) {
|
|
400
|
+
this.started.add(generationId);
|
|
401
|
+
const timer = window.setTimeout(() => {
|
|
402
|
+
this.startedTimers.delete(generationId);
|
|
403
|
+
if ([...this.sources.values()].some((value) => value.generationId === generationId)) {
|
|
404
|
+
this.onReceipt({ event: "started", lane, generationId, playedSamples: 0 });
|
|
405
|
+
}
|
|
406
|
+
}, Math.max(0, (startAt - this.context.currentTime) * 1000));
|
|
407
|
+
this.startedTimers.set(generationId, timer);
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
recomputeNextStart() {
|
|
411
|
+
this.nextStart = Math.max(this.context.currentTime, ...[...this.sources.values()].map((source) => source.startAt + source.samples / 24000));
|
|
412
|
+
}
|
|
413
|
+
finishIfComplete(generationId) {
|
|
414
|
+
const lane = this.doneGenerations.get(generationId);
|
|
415
|
+
if (!lane || this.completed.has(generationId) || this.buffered.has(generationId))
|
|
416
|
+
return;
|
|
417
|
+
if ([...this.sources.values()].some((value) => value.generationId === generationId))
|
|
418
|
+
return;
|
|
419
|
+
this.completed.add(generationId);
|
|
420
|
+
if (lane === "REACTION") {
|
|
421
|
+
this.blockers.delete(generationId);
|
|
422
|
+
this.flushReleasedForeground();
|
|
423
|
+
}
|
|
424
|
+
this.onReceipt({ event: "completed", lane, generationId, playedSamples: this.playedSamples.get(generationId) ?? 0 });
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
function opticalSidechainGain(level) {
|
|
428
|
+
const safeLevel = Math.max(Number.EPSILON, level);
|
|
429
|
+
const inputDb = 20 * Math.log10(safeLevel);
|
|
430
|
+
if (inputDb <= OPTICAL_SIDECHAIN.thresholdDb)
|
|
431
|
+
return 1;
|
|
432
|
+
const compressedDb = Math.min(OPTICAL_SIDECHAIN.maxReductionDb, (inputDb - OPTICAL_SIDECHAIN.thresholdDb) * (1 - 1 / OPTICAL_SIDECHAIN.ratio));
|
|
433
|
+
return 10 ** (-compressedDb / 20);
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
class Microphone {
|
|
437
|
+
onFrame;
|
|
438
|
+
onLevel;
|
|
439
|
+
context;
|
|
440
|
+
stream;
|
|
441
|
+
node;
|
|
442
|
+
samples = [];
|
|
443
|
+
constructor(onFrame, onLevel) {
|
|
444
|
+
this.onFrame = onFrame;
|
|
445
|
+
this.onLevel = onLevel;
|
|
446
|
+
}
|
|
447
|
+
async start() {
|
|
448
|
+
this.stream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true, channelCount: 1 } });
|
|
449
|
+
this.context = new AudioContext({ sampleRate: 24000, latencyHint: "interactive" });
|
|
450
|
+
const sourceCode = `class LiveVoiceMic extends AudioWorkletProcessor { process(inputs) { const input=inputs[0]?.[0]; if(input) this.port.postMessage(input.slice()); return true; } } registerProcessor('live-voice-mic', LiveVoiceMic);`;
|
|
451
|
+
const moduleUrl = URL.createObjectURL(new Blob([sourceCode], { type: "text/javascript" }));
|
|
452
|
+
await this.context.audioWorklet.addModule(moduleUrl);
|
|
453
|
+
URL.revokeObjectURL(moduleUrl);
|
|
454
|
+
const source = this.context.createMediaStreamSource(this.stream);
|
|
455
|
+
this.node = new AudioWorkletNode(this.context, "live-voice-mic");
|
|
456
|
+
const silent = this.context.createGain();
|
|
457
|
+
silent.gain.value = 0;
|
|
458
|
+
source.connect(this.node).connect(silent).connect(this.context.destination);
|
|
459
|
+
this.node.port.onmessage = (event) => this.accept(event.data);
|
|
460
|
+
await this.context.resume();
|
|
461
|
+
}
|
|
462
|
+
stop() {
|
|
463
|
+
this.node?.disconnect();
|
|
464
|
+
this.stream?.getTracks().forEach((track) => track.stop());
|
|
465
|
+
this.context?.close();
|
|
466
|
+
}
|
|
467
|
+
accept(input) {
|
|
468
|
+
for (const value of input)
|
|
469
|
+
this.samples.push(value);
|
|
470
|
+
while (this.samples.length >= 480) {
|
|
471
|
+
const frame = this.samples.splice(0, 480);
|
|
472
|
+
const pcm = new Int16Array(480);
|
|
473
|
+
let energy = 0;
|
|
474
|
+
for (let index = 0;index < frame.length; index += 1) {
|
|
475
|
+
const value = Math.max(-1, Math.min(1, frame[index]));
|
|
476
|
+
energy += value * value;
|
|
477
|
+
pcm[index] = value < 0 ? value * 32768 : value * 32767;
|
|
478
|
+
}
|
|
479
|
+
this.onLevel(Math.sqrt(energy / frame.length));
|
|
480
|
+
this.onFrame(pcm.buffer);
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
function waitForOpen(socket) {
|
|
485
|
+
return new Promise((resolve, reject) => {
|
|
486
|
+
const timeout = window.setTimeout(() => reject(new Error("Durable Object connection timed out.")), 1e4);
|
|
487
|
+
socket.addEventListener("open", () => {
|
|
488
|
+
window.clearTimeout(timeout);
|
|
489
|
+
resolve();
|
|
490
|
+
}, { once: true });
|
|
491
|
+
socket.addEventListener("error", () => {
|
|
492
|
+
window.clearTimeout(timeout);
|
|
493
|
+
reject(new Error("Could not connect to the Durable Object."));
|
|
494
|
+
}, { once: true });
|
|
495
|
+
});
|
|
496
|
+
}
|
|
497
|
+
function clamp01(value) {
|
|
498
|
+
return Number.isFinite(value) ? Math.max(0, Math.min(1, value)) : 1;
|
|
499
|
+
}
|
|
500
|
+
function asRecord(value) {
|
|
501
|
+
return value && typeof value === "object" ? value : {};
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
// src/public.ts
|
|
505
|
+
var defaultEndpoint = "wss://inkibra-voice.inkibra.workers.dev/v1/live";
|
|
506
|
+
|
|
507
|
+
class LiveVoiceSession {
|
|
508
|
+
client;
|
|
509
|
+
tools;
|
|
510
|
+
listeners = new Map;
|
|
511
|
+
currentPhase;
|
|
512
|
+
constructor(client, tools) {
|
|
513
|
+
this.client = client;
|
|
514
|
+
this.tools = tools;
|
|
515
|
+
}
|
|
516
|
+
static async connect(options) {
|
|
517
|
+
if (!options.token)
|
|
518
|
+
throw new Error("A short-lived Live Voice session token is required.");
|
|
519
|
+
let session;
|
|
520
|
+
const client = new LiveVoiceClient(options.endpoint ?? defaultEndpoint, {
|
|
521
|
+
onConnection: (state, detail) => {
|
|
522
|
+
session?.emit("connection", { state, detail });
|
|
523
|
+
if (state === "error")
|
|
524
|
+
session?.emit("error", { code: "CONNECTION_ERROR", message: detail ?? "Live Voice connection failed." });
|
|
525
|
+
},
|
|
526
|
+
onPublicEvent: (event) => session?.acceptPublicEvent(event.name, event.payload, event.atMs),
|
|
527
|
+
onLevel: () => {},
|
|
528
|
+
onUnknown: (message) => session?.emit("unknown", { message }),
|
|
529
|
+
onSessionLimit: (reason, durationSeconds) => session?.emit("limit", { reason, durationSeconds }),
|
|
530
|
+
onToolCall: async (call) => session.executeTool(call)
|
|
531
|
+
});
|
|
532
|
+
session = new LiveVoiceSession(client, options.tools ?? {});
|
|
533
|
+
await client.start({
|
|
534
|
+
token: options.token,
|
|
535
|
+
developer: {
|
|
536
|
+
prompt: options.prompt ?? "",
|
|
537
|
+
conversation: options.conversation,
|
|
538
|
+
tools: Object.entries(options.tools ?? {}).map(([name, tool]) => ({
|
|
539
|
+
type: "function",
|
|
540
|
+
name,
|
|
541
|
+
description: tool.description,
|
|
542
|
+
parameters: tool.parameters
|
|
543
|
+
}))
|
|
544
|
+
}
|
|
545
|
+
});
|
|
546
|
+
return session;
|
|
547
|
+
}
|
|
548
|
+
on(event, handler) {
|
|
549
|
+
const handlers = this.listeners.get(event) ?? new Set;
|
|
550
|
+
handlers.add(handler);
|
|
551
|
+
this.listeners.set(event, handlers);
|
|
552
|
+
return () => handlers.delete(handler);
|
|
553
|
+
}
|
|
554
|
+
get phase() {
|
|
555
|
+
return this.currentPhase;
|
|
556
|
+
}
|
|
557
|
+
stop() {
|
|
558
|
+
this.client.stop();
|
|
559
|
+
}
|
|
560
|
+
dispose() {
|
|
561
|
+
this.client.dispose();
|
|
562
|
+
this.listeners.clear();
|
|
563
|
+
}
|
|
564
|
+
emit(event, value) {
|
|
565
|
+
for (const handler of this.listeners.get(event) ?? [])
|
|
566
|
+
handler(value);
|
|
567
|
+
}
|
|
568
|
+
acceptPublicEvent(name, payload, atMs) {
|
|
569
|
+
if (name === "transcript.instantaneous" && typeof payload.text === "string") {
|
|
570
|
+
this.emit("transcript.instantaneous", { text: payload.text });
|
|
571
|
+
return;
|
|
572
|
+
}
|
|
573
|
+
if (name === "transcript.canonical" && Array.isArray(payload.turns)) {
|
|
574
|
+
const turns = [];
|
|
575
|
+
for (const value of payload.turns) {
|
|
576
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
577
|
+
continue;
|
|
578
|
+
const turn = value;
|
|
579
|
+
const role = turn.role;
|
|
580
|
+
if (typeof turn.id !== "string" || role !== "user" && role !== "assistant" || typeof turn.text !== "string")
|
|
581
|
+
continue;
|
|
582
|
+
turns.push({
|
|
583
|
+
id: turn.id,
|
|
584
|
+
role,
|
|
585
|
+
text: turn.text,
|
|
586
|
+
...typeof turn.truncated === "boolean" ? { truncated: turn.truncated } : {}
|
|
587
|
+
});
|
|
588
|
+
}
|
|
589
|
+
this.emit("transcript.canonical", { turns });
|
|
590
|
+
return;
|
|
591
|
+
}
|
|
592
|
+
if (name === "floor" && (payload.owner === "USER" || payload.owner === "ASSISTANT") && typeof payload.assistantAudible === "boolean") {
|
|
593
|
+
this.emit("floor", { owner: payload.owner, assistantAudible: payload.assistantAudible });
|
|
594
|
+
return;
|
|
595
|
+
}
|
|
596
|
+
if (name === "phase" && typeof payload.phase === "string") {
|
|
597
|
+
const event = {
|
|
598
|
+
phase: payload.phase,
|
|
599
|
+
source: payload.source === "floor" ? "floor" : "initial",
|
|
600
|
+
...typeof payload.previousPhase === "string" ? { previousPhase: payload.previousPhase } : {}
|
|
601
|
+
};
|
|
602
|
+
this.currentPhase = event.phase;
|
|
603
|
+
this.emit("phase", event);
|
|
604
|
+
return;
|
|
605
|
+
}
|
|
606
|
+
if (name === "quick-reaction" && typeof payload.reaction === "string" && (payload.status === "prepared" || payload.status === "playing" || payload.status === "completed" || payload.status === "cancelled")) {
|
|
607
|
+
this.emit("quick-reaction", { reaction: payload.reaction, status: payload.status });
|
|
608
|
+
return;
|
|
609
|
+
}
|
|
610
|
+
if (name === "assistant.energy" && Number.isFinite(Number(payload.current)) && Array.isArray(payload.predictions)) {
|
|
611
|
+
const predictions = [];
|
|
612
|
+
let valid = true;
|
|
613
|
+
for (const value of payload.predictions) {
|
|
614
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
615
|
+
valid = false;
|
|
616
|
+
break;
|
|
617
|
+
}
|
|
618
|
+
const prediction = value;
|
|
619
|
+
const startMs = Number(prediction.startMs);
|
|
620
|
+
const endMs = Number(prediction.endMs);
|
|
621
|
+
const energy = Number(prediction.energy);
|
|
622
|
+
if (!Number.isFinite(startMs) || startMs < 0 || !Number.isFinite(endMs) || endMs <= startMs || !Number.isFinite(energy)) {
|
|
623
|
+
valid = false;
|
|
624
|
+
break;
|
|
625
|
+
}
|
|
626
|
+
predictions.push({ startMs, endMs, energy: Math.max(0, Math.min(1, energy)) });
|
|
627
|
+
}
|
|
628
|
+
if (valid) {
|
|
629
|
+
this.emit("assistant.energy", {
|
|
630
|
+
current: Math.max(0, Math.min(1, Number(payload.current))),
|
|
631
|
+
predictions
|
|
632
|
+
});
|
|
633
|
+
return;
|
|
634
|
+
}
|
|
635
|
+
}
|
|
636
|
+
if (name === "error" && typeof payload.code === "string" && typeof payload.message === "string") {
|
|
637
|
+
this.emit("error", { code: payload.code, message: payload.message });
|
|
638
|
+
return;
|
|
639
|
+
}
|
|
640
|
+
this.emit("unknown", { message: { type: "public.event", name, payload, ...atMs === undefined ? {} : { atMs } } });
|
|
641
|
+
}
|
|
642
|
+
async executeTool(call) {
|
|
643
|
+
this.emit("tool.call", call);
|
|
644
|
+
const tool = this.tools[call.name];
|
|
645
|
+
if (!tool) {
|
|
646
|
+
const message = `No handler is registered for tool ${call.name}`;
|
|
647
|
+
this.emit("tool.result", { ...call, ok: false, error: message });
|
|
648
|
+
throw new Error(message);
|
|
649
|
+
}
|
|
650
|
+
try {
|
|
651
|
+
const output = await tool.execute(call.input);
|
|
652
|
+
this.emit("tool.result", { ...call, ok: true, output });
|
|
653
|
+
return output;
|
|
654
|
+
} catch (error) {
|
|
655
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
656
|
+
this.emit("tool.result", { ...call, ok: false, error: message });
|
|
657
|
+
throw error;
|
|
658
|
+
}
|
|
659
|
+
}
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
class LiveVoice {
|
|
663
|
+
static connect(options) {
|
|
664
|
+
return LiveVoiceSession.connect(options);
|
|
665
|
+
}
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
// src/index.ts
|
|
669
|
+
var src_default = LiveVoice;
|
|
670
|
+
export {
|
|
671
|
+
LiveVoiceSession,
|
|
672
|
+
src_default as default
|
|
673
|
+
};
|