@livekit/agents 1.2.0 → 1.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_exceptions.cjs.map +1 -1
- package/dist/_exceptions.d.ts.map +1 -1
- package/dist/_exceptions.js.map +1 -1
- package/dist/audio.cjs +10 -0
- package/dist/audio.cjs.map +1 -1
- package/dist/audio.d.cts +1 -1
- package/dist/audio.d.ts +1 -1
- package/dist/audio.d.ts.map +1 -1
- package/dist/audio.js +10 -0
- package/dist/audio.js.map +1 -1
- package/dist/beta/workflows/task_group.cjs +7 -4
- package/dist/beta/workflows/task_group.cjs.map +1 -1
- package/dist/beta/workflows/task_group.d.ts.map +1 -1
- package/dist/beta/workflows/task_group.js +7 -4
- package/dist/beta/workflows/task_group.js.map +1 -1
- package/dist/inference/api_protos.d.cts +26 -26
- package/dist/inference/api_protos.d.ts +26 -26
- package/dist/inference/interruption/http_transport.cjs.map +1 -1
- package/dist/inference/interruption/http_transport.d.cts +3 -1
- package/dist/inference/interruption/http_transport.d.ts +3 -1
- package/dist/inference/interruption/http_transport.d.ts.map +1 -1
- package/dist/inference/interruption/http_transport.js.map +1 -1
- package/dist/inference/interruption/ws_transport.cjs +37 -32
- package/dist/inference/interruption/ws_transport.cjs.map +1 -1
- package/dist/inference/interruption/ws_transport.d.ts.map +1 -1
- package/dist/inference/interruption/ws_transport.js +37 -32
- package/dist/inference/interruption/ws_transport.js.map +1 -1
- package/dist/inference/tts.cjs +14 -1
- package/dist/inference/tts.cjs.map +1 -1
- package/dist/inference/tts.d.cts +42 -4
- package/dist/inference/tts.d.ts +42 -4
- package/dist/inference/tts.d.ts.map +1 -1
- package/dist/inference/tts.js +24 -3
- package/dist/inference/tts.js.map +1 -1
- package/dist/inference/tts.test.cjs +72 -0
- package/dist/inference/tts.test.cjs.map +1 -1
- package/dist/inference/tts.test.js +72 -0
- package/dist/inference/tts.test.js.map +1 -1
- package/dist/ipc/job_proc_lazy_main.cjs +7 -2
- package/dist/ipc/job_proc_lazy_main.cjs.map +1 -1
- package/dist/ipc/job_proc_lazy_main.js +7 -2
- package/dist/ipc/job_proc_lazy_main.js.map +1 -1
- package/dist/ipc/supervised_proc.cjs +4 -1
- package/dist/ipc/supervised_proc.cjs.map +1 -1
- package/dist/ipc/supervised_proc.d.ts.map +1 -1
- package/dist/ipc/supervised_proc.js +4 -1
- package/dist/ipc/supervised_proc.js.map +1 -1
- package/dist/ipc/supervised_proc.test.cjs +82 -0
- package/dist/ipc/supervised_proc.test.cjs.map +1 -1
- package/dist/ipc/supervised_proc.test.js +82 -0
- package/dist/ipc/supervised_proc.test.js.map +1 -1
- package/dist/job.cjs +2 -1
- package/dist/job.cjs.map +1 -1
- package/dist/job.d.ts.map +1 -1
- package/dist/job.js +2 -1
- package/dist/job.js.map +1 -1
- package/dist/llm/chat_context.cjs +102 -31
- package/dist/llm/chat_context.cjs.map +1 -1
- package/dist/llm/chat_context.d.ts.map +1 -1
- package/dist/llm/chat_context.js +102 -31
- package/dist/llm/chat_context.js.map +1 -1
- package/dist/llm/chat_context.test.cjs +123 -5
- package/dist/llm/chat_context.test.cjs.map +1 -1
- package/dist/llm/chat_context.test.js +123 -5
- package/dist/llm/chat_context.test.js.map +1 -1
- package/dist/llm/fallback_adapter.cjs +2 -0
- package/dist/llm/fallback_adapter.cjs.map +1 -1
- package/dist/llm/fallback_adapter.d.ts.map +1 -1
- package/dist/llm/fallback_adapter.js +2 -0
- package/dist/llm/fallback_adapter.js.map +1 -1
- package/dist/llm/index.cjs +2 -0
- package/dist/llm/index.cjs.map +1 -1
- package/dist/llm/index.d.cts +1 -1
- package/dist/llm/index.d.ts +1 -1
- package/dist/llm/index.d.ts.map +1 -1
- package/dist/llm/index.js +2 -0
- package/dist/llm/index.js.map +1 -1
- package/dist/llm/utils.cjs +89 -0
- package/dist/llm/utils.cjs.map +1 -1
- package/dist/llm/utils.d.cts +8 -0
- package/dist/llm/utils.d.ts +8 -0
- package/dist/llm/utils.d.ts.map +1 -1
- package/dist/llm/utils.js +88 -0
- package/dist/llm/utils.js.map +1 -1
- package/dist/llm/utils.test.cjs +90 -0
- package/dist/llm/utils.test.cjs.map +1 -1
- package/dist/llm/utils.test.js +98 -2
- package/dist/llm/utils.test.js.map +1 -1
- package/dist/stt/stt.cjs +8 -0
- package/dist/stt/stt.cjs.map +1 -1
- package/dist/stt/stt.d.cts +8 -0
- package/dist/stt/stt.d.ts +8 -0
- package/dist/stt/stt.d.ts.map +1 -1
- package/dist/stt/stt.js +8 -0
- package/dist/stt/stt.js.map +1 -1
- package/dist/tts/fallback_adapter.cjs +6 -0
- package/dist/tts/fallback_adapter.cjs.map +1 -1
- package/dist/tts/fallback_adapter.d.ts.map +1 -1
- package/dist/tts/fallback_adapter.js +6 -0
- package/dist/tts/fallback_adapter.js.map +1 -1
- package/dist/typed_promise.cjs +48 -0
- package/dist/typed_promise.cjs.map +1 -0
- package/dist/typed_promise.d.cts +24 -0
- package/dist/typed_promise.d.ts +24 -0
- package/dist/typed_promise.d.ts.map +1 -0
- package/dist/typed_promise.js +28 -0
- package/dist/typed_promise.js.map +1 -0
- package/dist/utils.cjs +30 -2
- package/dist/utils.cjs.map +1 -1
- package/dist/utils.d.cts +18 -0
- package/dist/utils.d.ts +18 -0
- package/dist/utils.d.ts.map +1 -1
- package/dist/utils.js +27 -2
- package/dist/utils.js.map +1 -1
- package/dist/version.cjs +1 -1
- package/dist/version.js +1 -1
- package/dist/voice/agent_activity.cjs +10 -0
- package/dist/voice/agent_activity.cjs.map +1 -1
- package/dist/voice/agent_activity.d.ts.map +1 -1
- package/dist/voice/agent_activity.js +11 -0
- package/dist/voice/agent_activity.js.map +1 -1
- package/dist/voice/agent_session.cjs +1 -1
- package/dist/voice/agent_session.cjs.map +1 -1
- package/dist/voice/agent_session.d.cts +4 -2
- package/dist/voice/agent_session.d.ts +4 -2
- package/dist/voice/agent_session.d.ts.map +1 -1
- package/dist/voice/agent_session.js +1 -1
- package/dist/voice/agent_session.js.map +1 -1
- package/dist/voice/events.cjs +11 -0
- package/dist/voice/events.cjs.map +1 -1
- package/dist/voice/events.d.cts +12 -1
- package/dist/voice/events.d.ts +12 -1
- package/dist/voice/events.d.ts.map +1 -1
- package/dist/voice/events.js +10 -0
- package/dist/voice/events.js.map +1 -1
- package/dist/voice/generation.cjs +23 -4
- package/dist/voice/generation.cjs.map +1 -1
- package/dist/voice/generation.d.ts.map +1 -1
- package/dist/voice/generation.js +32 -5
- package/dist/voice/generation.js.map +1 -1
- package/dist/voice/generation_tts_timeout.test.cjs +85 -0
- package/dist/voice/generation_tts_timeout.test.cjs.map +1 -0
- package/dist/voice/generation_tts_timeout.test.js +84 -0
- package/dist/voice/generation_tts_timeout.test.js.map +1 -0
- package/dist/voice/index.cjs.map +1 -1
- package/dist/voice/index.d.cts +1 -1
- package/dist/voice/index.d.ts +1 -1
- package/dist/voice/index.d.ts.map +1 -1
- package/dist/voice/index.js +3 -1
- package/dist/voice/index.js.map +1 -1
- package/dist/voice/recorder_io/recorder_io.cjs +1 -2
- package/dist/voice/recorder_io/recorder_io.cjs.map +1 -1
- package/dist/voice/recorder_io/recorder_io.d.ts.map +1 -1
- package/dist/voice/recorder_io/recorder_io.js +2 -3
- package/dist/voice/recorder_io/recorder_io.js.map +1 -1
- package/dist/voice/report.cjs +1 -1
- package/dist/voice/report.cjs.map +1 -1
- package/dist/voice/report.js +1 -1
- package/dist/voice/report.js.map +1 -1
- package/dist/voice/report.test.cjs +70 -0
- package/dist/voice/report.test.cjs.map +1 -1
- package/dist/voice/report.test.js +70 -0
- package/dist/voice/report.test.js.map +1 -1
- package/dist/voice/room_io/room_io.cjs +5 -1
- package/dist/voice/room_io/room_io.cjs.map +1 -1
- package/dist/voice/room_io/room_io.d.ts.map +1 -1
- package/dist/voice/room_io/room_io.js +5 -1
- package/dist/voice/room_io/room_io.js.map +1 -1
- package/dist/voice/room_io/room_io.test.cjs +18 -0
- package/dist/voice/room_io/room_io.test.cjs.map +1 -0
- package/dist/voice/room_io/room_io.test.js +17 -0
- package/dist/voice/room_io/room_io.test.js.map +1 -0
- package/package.json +4 -2
- package/src/_exceptions.ts +5 -0
- package/src/audio.ts +12 -1
- package/src/beta/workflows/task_group.ts +14 -5
- package/src/inference/interruption/http_transport.ts +2 -1
- package/src/inference/interruption/ws_transport.ts +44 -34
- package/src/inference/tts.test.ts +87 -0
- package/src/inference/tts.ts +71 -9
- package/src/ipc/job_proc_lazy_main.ts +7 -2
- package/src/ipc/supervised_proc.test.ts +96 -0
- package/src/ipc/supervised_proc.ts +8 -1
- package/src/job.ts +1 -0
- package/src/llm/chat_context.test.ts +137 -5
- package/src/llm/chat_context.ts +119 -38
- package/src/llm/fallback_adapter.ts +5 -2
- package/src/llm/index.ts +2 -0
- package/src/llm/utils.test.ts +103 -2
- package/src/llm/utils.ts +128 -0
- package/src/stt/stt.ts +9 -1
- package/src/tts/fallback_adapter.ts +9 -2
- package/src/typed_promise.ts +67 -0
- package/src/utils.ts +45 -2
- package/src/voice/agent_activity.ts +11 -0
- package/src/voice/agent_session.ts +13 -7
- package/src/voice/events.ts +21 -0
- package/src/voice/generation.ts +35 -8
- package/src/voice/generation_tts_timeout.test.ts +112 -0
- package/src/voice/index.ts +6 -1
- package/src/voice/recorder_io/recorder_io.ts +2 -7
- package/src/voice/report.test.ts +78 -0
- package/src/voice/report.ts +1 -1
- package/src/voice/room_io/room_io.test.ts +38 -0
- package/src/voice/room_io/room_io.ts +7 -2
|
@@ -227,15 +227,15 @@ export declare const sttWordSchema: z.ZodObject<{
|
|
|
227
227
|
extra: z.ZodOptional<z.ZodNullable<z.ZodUnknown>>;
|
|
228
228
|
}, "strip", z.ZodTypeAny, {
|
|
229
229
|
end: number;
|
|
230
|
-
confidence: number;
|
|
231
230
|
start: number;
|
|
231
|
+
confidence: number;
|
|
232
232
|
word: string;
|
|
233
233
|
extra?: unknown;
|
|
234
234
|
}, {
|
|
235
235
|
end?: number | undefined;
|
|
236
|
+
start?: number | undefined;
|
|
236
237
|
extra?: unknown;
|
|
237
238
|
confidence?: number | undefined;
|
|
238
|
-
start?: number | undefined;
|
|
239
239
|
word?: string | undefined;
|
|
240
240
|
}>;
|
|
241
241
|
export declare const sttInterimTranscriptEventSchema: z.ZodObject<{
|
|
@@ -254,29 +254,29 @@ export declare const sttInterimTranscriptEventSchema: z.ZodObject<{
|
|
|
254
254
|
extra: z.ZodOptional<z.ZodNullable<z.ZodUnknown>>;
|
|
255
255
|
}, "strip", z.ZodTypeAny, {
|
|
256
256
|
end: number;
|
|
257
|
-
confidence: number;
|
|
258
257
|
start: number;
|
|
258
|
+
confidence: number;
|
|
259
259
|
word: string;
|
|
260
260
|
extra?: unknown;
|
|
261
261
|
}, {
|
|
262
262
|
end?: number | undefined;
|
|
263
|
+
start?: number | undefined;
|
|
263
264
|
extra?: unknown;
|
|
264
265
|
confidence?: number | undefined;
|
|
265
|
-
start?: number | undefined;
|
|
266
266
|
word?: string | undefined;
|
|
267
267
|
}>, "many">>>;
|
|
268
268
|
extra: z.ZodOptional<z.ZodNullable<z.ZodUnknown>>;
|
|
269
269
|
}, "strip", z.ZodTypeAny, {
|
|
270
|
+
start: number;
|
|
270
271
|
type: "interim_transcript";
|
|
271
272
|
transcript: string;
|
|
272
273
|
language: string;
|
|
273
274
|
confidence: number;
|
|
274
|
-
start: number;
|
|
275
275
|
duration: number;
|
|
276
276
|
words: {
|
|
277
277
|
end: number;
|
|
278
|
-
confidence: number;
|
|
279
278
|
start: number;
|
|
279
|
+
confidence: number;
|
|
280
280
|
word: string;
|
|
281
281
|
extra?: unknown;
|
|
282
282
|
}[];
|
|
@@ -284,18 +284,18 @@ export declare const sttInterimTranscriptEventSchema: z.ZodObject<{
|
|
|
284
284
|
session_id?: string | undefined;
|
|
285
285
|
}, {
|
|
286
286
|
type: "interim_transcript";
|
|
287
|
+
start?: number | undefined;
|
|
287
288
|
extra?: unknown;
|
|
288
289
|
transcript?: string | undefined;
|
|
289
290
|
session_id?: string | undefined;
|
|
290
291
|
language?: string | undefined;
|
|
291
292
|
confidence?: number | undefined;
|
|
292
|
-
start?: number | undefined;
|
|
293
293
|
duration?: number | undefined;
|
|
294
294
|
words?: {
|
|
295
295
|
end?: number | undefined;
|
|
296
|
+
start?: number | undefined;
|
|
296
297
|
extra?: unknown;
|
|
297
298
|
confidence?: number | undefined;
|
|
298
|
-
start?: number | undefined;
|
|
299
299
|
word?: string | undefined;
|
|
300
300
|
}[] | undefined;
|
|
301
301
|
}>;
|
|
@@ -315,29 +315,29 @@ export declare const sttFinalTranscriptEventSchema: z.ZodObject<{
|
|
|
315
315
|
extra: z.ZodOptional<z.ZodNullable<z.ZodUnknown>>;
|
|
316
316
|
}, "strip", z.ZodTypeAny, {
|
|
317
317
|
end: number;
|
|
318
|
-
confidence: number;
|
|
319
318
|
start: number;
|
|
319
|
+
confidence: number;
|
|
320
320
|
word: string;
|
|
321
321
|
extra?: unknown;
|
|
322
322
|
}, {
|
|
323
323
|
end?: number | undefined;
|
|
324
|
+
start?: number | undefined;
|
|
324
325
|
extra?: unknown;
|
|
325
326
|
confidence?: number | undefined;
|
|
326
|
-
start?: number | undefined;
|
|
327
327
|
word?: string | undefined;
|
|
328
328
|
}>, "many">>>;
|
|
329
329
|
extra: z.ZodOptional<z.ZodNullable<z.ZodUnknown>>;
|
|
330
330
|
}, "strip", z.ZodTypeAny, {
|
|
331
|
+
start: number;
|
|
331
332
|
type: "final_transcript";
|
|
332
333
|
transcript: string;
|
|
333
334
|
language: string;
|
|
334
335
|
confidence: number;
|
|
335
|
-
start: number;
|
|
336
336
|
duration: number;
|
|
337
337
|
words: {
|
|
338
338
|
end: number;
|
|
339
|
-
confidence: number;
|
|
340
339
|
start: number;
|
|
340
|
+
confidence: number;
|
|
341
341
|
word: string;
|
|
342
342
|
extra?: unknown;
|
|
343
343
|
}[];
|
|
@@ -345,18 +345,18 @@ export declare const sttFinalTranscriptEventSchema: z.ZodObject<{
|
|
|
345
345
|
session_id?: string | undefined;
|
|
346
346
|
}, {
|
|
347
347
|
type: "final_transcript";
|
|
348
|
+
start?: number | undefined;
|
|
348
349
|
extra?: unknown;
|
|
349
350
|
transcript?: string | undefined;
|
|
350
351
|
session_id?: string | undefined;
|
|
351
352
|
language?: string | undefined;
|
|
352
353
|
confidence?: number | undefined;
|
|
353
|
-
start?: number | undefined;
|
|
354
354
|
duration?: number | undefined;
|
|
355
355
|
words?: {
|
|
356
356
|
end?: number | undefined;
|
|
357
|
+
start?: number | undefined;
|
|
357
358
|
extra?: unknown;
|
|
358
359
|
confidence?: number | undefined;
|
|
359
|
-
start?: number | undefined;
|
|
360
360
|
word?: string | undefined;
|
|
361
361
|
}[] | undefined;
|
|
362
362
|
}>;
|
|
@@ -434,29 +434,29 @@ export declare const sttServerEventSchema: z.ZodDiscriminatedUnion<"type", [z.Zo
|
|
|
434
434
|
extra: z.ZodOptional<z.ZodNullable<z.ZodUnknown>>;
|
|
435
435
|
}, "strip", z.ZodTypeAny, {
|
|
436
436
|
end: number;
|
|
437
|
-
confidence: number;
|
|
438
437
|
start: number;
|
|
438
|
+
confidence: number;
|
|
439
439
|
word: string;
|
|
440
440
|
extra?: unknown;
|
|
441
441
|
}, {
|
|
442
442
|
end?: number | undefined;
|
|
443
|
+
start?: number | undefined;
|
|
443
444
|
extra?: unknown;
|
|
444
445
|
confidence?: number | undefined;
|
|
445
|
-
start?: number | undefined;
|
|
446
446
|
word?: string | undefined;
|
|
447
447
|
}>, "many">>>;
|
|
448
448
|
extra: z.ZodOptional<z.ZodNullable<z.ZodUnknown>>;
|
|
449
449
|
}, "strip", z.ZodTypeAny, {
|
|
450
|
+
start: number;
|
|
450
451
|
type: "interim_transcript";
|
|
451
452
|
transcript: string;
|
|
452
453
|
language: string;
|
|
453
454
|
confidence: number;
|
|
454
|
-
start: number;
|
|
455
455
|
duration: number;
|
|
456
456
|
words: {
|
|
457
457
|
end: number;
|
|
458
|
-
confidence: number;
|
|
459
458
|
start: number;
|
|
459
|
+
confidence: number;
|
|
460
460
|
word: string;
|
|
461
461
|
extra?: unknown;
|
|
462
462
|
}[];
|
|
@@ -464,18 +464,18 @@ export declare const sttServerEventSchema: z.ZodDiscriminatedUnion<"type", [z.Zo
|
|
|
464
464
|
session_id?: string | undefined;
|
|
465
465
|
}, {
|
|
466
466
|
type: "interim_transcript";
|
|
467
|
+
start?: number | undefined;
|
|
467
468
|
extra?: unknown;
|
|
468
469
|
transcript?: string | undefined;
|
|
469
470
|
session_id?: string | undefined;
|
|
470
471
|
language?: string | undefined;
|
|
471
472
|
confidence?: number | undefined;
|
|
472
|
-
start?: number | undefined;
|
|
473
473
|
duration?: number | undefined;
|
|
474
474
|
words?: {
|
|
475
475
|
end?: number | undefined;
|
|
476
|
+
start?: number | undefined;
|
|
476
477
|
extra?: unknown;
|
|
477
478
|
confidence?: number | undefined;
|
|
478
|
-
start?: number | undefined;
|
|
479
479
|
word?: string | undefined;
|
|
480
480
|
}[] | undefined;
|
|
481
481
|
}>, z.ZodObject<{
|
|
@@ -494,29 +494,29 @@ export declare const sttServerEventSchema: z.ZodDiscriminatedUnion<"type", [z.Zo
|
|
|
494
494
|
extra: z.ZodOptional<z.ZodNullable<z.ZodUnknown>>;
|
|
495
495
|
}, "strip", z.ZodTypeAny, {
|
|
496
496
|
end: number;
|
|
497
|
-
confidence: number;
|
|
498
497
|
start: number;
|
|
498
|
+
confidence: number;
|
|
499
499
|
word: string;
|
|
500
500
|
extra?: unknown;
|
|
501
501
|
}, {
|
|
502
502
|
end?: number | undefined;
|
|
503
|
+
start?: number | undefined;
|
|
503
504
|
extra?: unknown;
|
|
504
505
|
confidence?: number | undefined;
|
|
505
|
-
start?: number | undefined;
|
|
506
506
|
word?: string | undefined;
|
|
507
507
|
}>, "many">>>;
|
|
508
508
|
extra: z.ZodOptional<z.ZodNullable<z.ZodUnknown>>;
|
|
509
509
|
}, "strip", z.ZodTypeAny, {
|
|
510
|
+
start: number;
|
|
510
511
|
type: "final_transcript";
|
|
511
512
|
transcript: string;
|
|
512
513
|
language: string;
|
|
513
514
|
confidence: number;
|
|
514
|
-
start: number;
|
|
515
515
|
duration: number;
|
|
516
516
|
words: {
|
|
517
517
|
end: number;
|
|
518
|
-
confidence: number;
|
|
519
518
|
start: number;
|
|
519
|
+
confidence: number;
|
|
520
520
|
word: string;
|
|
521
521
|
extra?: unknown;
|
|
522
522
|
}[];
|
|
@@ -524,18 +524,18 @@ export declare const sttServerEventSchema: z.ZodDiscriminatedUnion<"type", [z.Zo
|
|
|
524
524
|
session_id?: string | undefined;
|
|
525
525
|
}, {
|
|
526
526
|
type: "final_transcript";
|
|
527
|
+
start?: number | undefined;
|
|
527
528
|
extra?: unknown;
|
|
528
529
|
transcript?: string | undefined;
|
|
529
530
|
session_id?: string | undefined;
|
|
530
531
|
language?: string | undefined;
|
|
531
532
|
confidence?: number | undefined;
|
|
532
|
-
start?: number | undefined;
|
|
533
533
|
duration?: number | undefined;
|
|
534
534
|
words?: {
|
|
535
535
|
end?: number | undefined;
|
|
536
|
+
start?: number | undefined;
|
|
536
537
|
extra?: unknown;
|
|
537
538
|
confidence?: number | undefined;
|
|
538
|
-
start?: number | undefined;
|
|
539
539
|
word?: string | undefined;
|
|
540
540
|
}[] | undefined;
|
|
541
541
|
}>, z.ZodObject<{
|
|
@@ -227,15 +227,15 @@ export declare const sttWordSchema: z.ZodObject<{
|
|
|
227
227
|
extra: z.ZodOptional<z.ZodNullable<z.ZodUnknown>>;
|
|
228
228
|
}, "strip", z.ZodTypeAny, {
|
|
229
229
|
end: number;
|
|
230
|
-
confidence: number;
|
|
231
230
|
start: number;
|
|
231
|
+
confidence: number;
|
|
232
232
|
word: string;
|
|
233
233
|
extra?: unknown;
|
|
234
234
|
}, {
|
|
235
235
|
end?: number | undefined;
|
|
236
|
+
start?: number | undefined;
|
|
236
237
|
extra?: unknown;
|
|
237
238
|
confidence?: number | undefined;
|
|
238
|
-
start?: number | undefined;
|
|
239
239
|
word?: string | undefined;
|
|
240
240
|
}>;
|
|
241
241
|
export declare const sttInterimTranscriptEventSchema: z.ZodObject<{
|
|
@@ -254,29 +254,29 @@ export declare const sttInterimTranscriptEventSchema: z.ZodObject<{
|
|
|
254
254
|
extra: z.ZodOptional<z.ZodNullable<z.ZodUnknown>>;
|
|
255
255
|
}, "strip", z.ZodTypeAny, {
|
|
256
256
|
end: number;
|
|
257
|
-
confidence: number;
|
|
258
257
|
start: number;
|
|
258
|
+
confidence: number;
|
|
259
259
|
word: string;
|
|
260
260
|
extra?: unknown;
|
|
261
261
|
}, {
|
|
262
262
|
end?: number | undefined;
|
|
263
|
+
start?: number | undefined;
|
|
263
264
|
extra?: unknown;
|
|
264
265
|
confidence?: number | undefined;
|
|
265
|
-
start?: number | undefined;
|
|
266
266
|
word?: string | undefined;
|
|
267
267
|
}>, "many">>>;
|
|
268
268
|
extra: z.ZodOptional<z.ZodNullable<z.ZodUnknown>>;
|
|
269
269
|
}, "strip", z.ZodTypeAny, {
|
|
270
|
+
start: number;
|
|
270
271
|
type: "interim_transcript";
|
|
271
272
|
transcript: string;
|
|
272
273
|
language: string;
|
|
273
274
|
confidence: number;
|
|
274
|
-
start: number;
|
|
275
275
|
duration: number;
|
|
276
276
|
words: {
|
|
277
277
|
end: number;
|
|
278
|
-
confidence: number;
|
|
279
278
|
start: number;
|
|
279
|
+
confidence: number;
|
|
280
280
|
word: string;
|
|
281
281
|
extra?: unknown;
|
|
282
282
|
}[];
|
|
@@ -284,18 +284,18 @@ export declare const sttInterimTranscriptEventSchema: z.ZodObject<{
|
|
|
284
284
|
session_id?: string | undefined;
|
|
285
285
|
}, {
|
|
286
286
|
type: "interim_transcript";
|
|
287
|
+
start?: number | undefined;
|
|
287
288
|
extra?: unknown;
|
|
288
289
|
transcript?: string | undefined;
|
|
289
290
|
session_id?: string | undefined;
|
|
290
291
|
language?: string | undefined;
|
|
291
292
|
confidence?: number | undefined;
|
|
292
|
-
start?: number | undefined;
|
|
293
293
|
duration?: number | undefined;
|
|
294
294
|
words?: {
|
|
295
295
|
end?: number | undefined;
|
|
296
|
+
start?: number | undefined;
|
|
296
297
|
extra?: unknown;
|
|
297
298
|
confidence?: number | undefined;
|
|
298
|
-
start?: number | undefined;
|
|
299
299
|
word?: string | undefined;
|
|
300
300
|
}[] | undefined;
|
|
301
301
|
}>;
|
|
@@ -315,29 +315,29 @@ export declare const sttFinalTranscriptEventSchema: z.ZodObject<{
|
|
|
315
315
|
extra: z.ZodOptional<z.ZodNullable<z.ZodUnknown>>;
|
|
316
316
|
}, "strip", z.ZodTypeAny, {
|
|
317
317
|
end: number;
|
|
318
|
-
confidence: number;
|
|
319
318
|
start: number;
|
|
319
|
+
confidence: number;
|
|
320
320
|
word: string;
|
|
321
321
|
extra?: unknown;
|
|
322
322
|
}, {
|
|
323
323
|
end?: number | undefined;
|
|
324
|
+
start?: number | undefined;
|
|
324
325
|
extra?: unknown;
|
|
325
326
|
confidence?: number | undefined;
|
|
326
|
-
start?: number | undefined;
|
|
327
327
|
word?: string | undefined;
|
|
328
328
|
}>, "many">>>;
|
|
329
329
|
extra: z.ZodOptional<z.ZodNullable<z.ZodUnknown>>;
|
|
330
330
|
}, "strip", z.ZodTypeAny, {
|
|
331
|
+
start: number;
|
|
331
332
|
type: "final_transcript";
|
|
332
333
|
transcript: string;
|
|
333
334
|
language: string;
|
|
334
335
|
confidence: number;
|
|
335
|
-
start: number;
|
|
336
336
|
duration: number;
|
|
337
337
|
words: {
|
|
338
338
|
end: number;
|
|
339
|
-
confidence: number;
|
|
340
339
|
start: number;
|
|
340
|
+
confidence: number;
|
|
341
341
|
word: string;
|
|
342
342
|
extra?: unknown;
|
|
343
343
|
}[];
|
|
@@ -345,18 +345,18 @@ export declare const sttFinalTranscriptEventSchema: z.ZodObject<{
|
|
|
345
345
|
session_id?: string | undefined;
|
|
346
346
|
}, {
|
|
347
347
|
type: "final_transcript";
|
|
348
|
+
start?: number | undefined;
|
|
348
349
|
extra?: unknown;
|
|
349
350
|
transcript?: string | undefined;
|
|
350
351
|
session_id?: string | undefined;
|
|
351
352
|
language?: string | undefined;
|
|
352
353
|
confidence?: number | undefined;
|
|
353
|
-
start?: number | undefined;
|
|
354
354
|
duration?: number | undefined;
|
|
355
355
|
words?: {
|
|
356
356
|
end?: number | undefined;
|
|
357
|
+
start?: number | undefined;
|
|
357
358
|
extra?: unknown;
|
|
358
359
|
confidence?: number | undefined;
|
|
359
|
-
start?: number | undefined;
|
|
360
360
|
word?: string | undefined;
|
|
361
361
|
}[] | undefined;
|
|
362
362
|
}>;
|
|
@@ -434,29 +434,29 @@ export declare const sttServerEventSchema: z.ZodDiscriminatedUnion<"type", [z.Zo
|
|
|
434
434
|
extra: z.ZodOptional<z.ZodNullable<z.ZodUnknown>>;
|
|
435
435
|
}, "strip", z.ZodTypeAny, {
|
|
436
436
|
end: number;
|
|
437
|
-
confidence: number;
|
|
438
437
|
start: number;
|
|
438
|
+
confidence: number;
|
|
439
439
|
word: string;
|
|
440
440
|
extra?: unknown;
|
|
441
441
|
}, {
|
|
442
442
|
end?: number | undefined;
|
|
443
|
+
start?: number | undefined;
|
|
443
444
|
extra?: unknown;
|
|
444
445
|
confidence?: number | undefined;
|
|
445
|
-
start?: number | undefined;
|
|
446
446
|
word?: string | undefined;
|
|
447
447
|
}>, "many">>>;
|
|
448
448
|
extra: z.ZodOptional<z.ZodNullable<z.ZodUnknown>>;
|
|
449
449
|
}, "strip", z.ZodTypeAny, {
|
|
450
|
+
start: number;
|
|
450
451
|
type: "interim_transcript";
|
|
451
452
|
transcript: string;
|
|
452
453
|
language: string;
|
|
453
454
|
confidence: number;
|
|
454
|
-
start: number;
|
|
455
455
|
duration: number;
|
|
456
456
|
words: {
|
|
457
457
|
end: number;
|
|
458
|
-
confidence: number;
|
|
459
458
|
start: number;
|
|
459
|
+
confidence: number;
|
|
460
460
|
word: string;
|
|
461
461
|
extra?: unknown;
|
|
462
462
|
}[];
|
|
@@ -464,18 +464,18 @@ export declare const sttServerEventSchema: z.ZodDiscriminatedUnion<"type", [z.Zo
|
|
|
464
464
|
session_id?: string | undefined;
|
|
465
465
|
}, {
|
|
466
466
|
type: "interim_transcript";
|
|
467
|
+
start?: number | undefined;
|
|
467
468
|
extra?: unknown;
|
|
468
469
|
transcript?: string | undefined;
|
|
469
470
|
session_id?: string | undefined;
|
|
470
471
|
language?: string | undefined;
|
|
471
472
|
confidence?: number | undefined;
|
|
472
|
-
start?: number | undefined;
|
|
473
473
|
duration?: number | undefined;
|
|
474
474
|
words?: {
|
|
475
475
|
end?: number | undefined;
|
|
476
|
+
start?: number | undefined;
|
|
476
477
|
extra?: unknown;
|
|
477
478
|
confidence?: number | undefined;
|
|
478
|
-
start?: number | undefined;
|
|
479
479
|
word?: string | undefined;
|
|
480
480
|
}[] | undefined;
|
|
481
481
|
}>, z.ZodObject<{
|
|
@@ -494,29 +494,29 @@ export declare const sttServerEventSchema: z.ZodDiscriminatedUnion<"type", [z.Zo
|
|
|
494
494
|
extra: z.ZodOptional<z.ZodNullable<z.ZodUnknown>>;
|
|
495
495
|
}, "strip", z.ZodTypeAny, {
|
|
496
496
|
end: number;
|
|
497
|
-
confidence: number;
|
|
498
497
|
start: number;
|
|
498
|
+
confidence: number;
|
|
499
499
|
word: string;
|
|
500
500
|
extra?: unknown;
|
|
501
501
|
}, {
|
|
502
502
|
end?: number | undefined;
|
|
503
|
+
start?: number | undefined;
|
|
503
504
|
extra?: unknown;
|
|
504
505
|
confidence?: number | undefined;
|
|
505
|
-
start?: number | undefined;
|
|
506
506
|
word?: string | undefined;
|
|
507
507
|
}>, "many">>>;
|
|
508
508
|
extra: z.ZodOptional<z.ZodNullable<z.ZodUnknown>>;
|
|
509
509
|
}, "strip", z.ZodTypeAny, {
|
|
510
|
+
start: number;
|
|
510
511
|
type: "final_transcript";
|
|
511
512
|
transcript: string;
|
|
512
513
|
language: string;
|
|
513
514
|
confidence: number;
|
|
514
|
-
start: number;
|
|
515
515
|
duration: number;
|
|
516
516
|
words: {
|
|
517
517
|
end: number;
|
|
518
|
-
confidence: number;
|
|
519
518
|
start: number;
|
|
519
|
+
confidence: number;
|
|
520
520
|
word: string;
|
|
521
521
|
extra?: unknown;
|
|
522
522
|
}[];
|
|
@@ -524,18 +524,18 @@ export declare const sttServerEventSchema: z.ZodDiscriminatedUnion<"type", [z.Zo
|
|
|
524
524
|
session_id?: string | undefined;
|
|
525
525
|
}, {
|
|
526
526
|
type: "final_transcript";
|
|
527
|
+
start?: number | undefined;
|
|
527
528
|
extra?: unknown;
|
|
528
529
|
transcript?: string | undefined;
|
|
529
530
|
session_id?: string | undefined;
|
|
530
531
|
language?: string | undefined;
|
|
531
532
|
confidence?: number | undefined;
|
|
532
|
-
start?: number | undefined;
|
|
533
533
|
duration?: number | undefined;
|
|
534
534
|
words?: {
|
|
535
535
|
end?: number | undefined;
|
|
536
|
+
start?: number | undefined;
|
|
536
537
|
extra?: unknown;
|
|
537
538
|
confidence?: number | undefined;
|
|
538
|
-
start?: number | undefined;
|
|
539
539
|
word?: string | undefined;
|
|
540
540
|
}[] | undefined;
|
|
541
541
|
}>, z.ZodObject<{
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../src/inference/interruption/http_transport.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2026 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { FetchError, ofetch } from 'ofetch';\nimport { TransformStream } from 'stream/web';\nimport { z } from 'zod';\nimport { APIConnectionError, APIError, APIStatusError, isAPIError } from '../../_exceptions.js';\nimport { log } from '../../log.js';\nimport { createAccessToken } from '../utils.js';\nimport { InterruptionCacheEntry } from './interruption_cache_entry.js';\nimport type { OverlappingSpeechEvent } from './types.js';\nimport type { BoundedCache } from './utils.js';\n\nexport interface PostOptions {\n baseUrl: string;\n token: string;\n signal?: AbortSignal;\n timeout?: number;\n maxRetries?: number;\n}\n\nexport interface PredictOptions {\n threshold: number;\n minFrames: number;\n}\n\nexport const predictEndpointResponseSchema = z.object({\n created_at: z.number(),\n is_bargein: z.boolean(),\n probabilities: z.array(z.number()),\n});\n\nexport type PredictEndpointResponse = z.infer<typeof predictEndpointResponseSchema>;\n\nexport interface PredictResponse {\n createdAt: number;\n isBargein: boolean;\n probabilities: number[];\n predictionDurationInS: number;\n}\n\nexport async function predictHTTP(\n data: Int16Array,\n predictOptions: PredictOptions,\n options: PostOptions,\n): Promise<PredictResponse> {\n const createdAt = performance.now();\n const url = new URL(`/bargein`, options.baseUrl);\n url.searchParams.append('threshold', predictOptions.threshold.toString());\n url.searchParams.append('min_frames', predictOptions.minFrames.toFixed());\n url.searchParams.append('created_at', createdAt.toFixed());\n\n try {\n const response = await ofetch(url.toString(), {\n retry: 0,\n headers: {\n 'Content-Type': 'application/octet-stream',\n Authorization: `Bearer ${options.token}`,\n },\n signal: options.signal,\n timeout: options.timeout,\n method: 'POST',\n body: data,\n });\n const { created_at, is_bargein, probabilities } = predictEndpointResponseSchema.parse(response);\n\n return {\n createdAt: created_at,\n isBargein: is_bargein,\n probabilities,\n predictionDurationInS: (performance.now() - createdAt) / 1000,\n };\n } catch (err) {\n if (isAPIError(err)) throw err;\n if (err instanceof FetchError) {\n if (err.statusCode) {\n throw new APIStatusError({\n message: `error during interruption prediction: ${err.message}`,\n options: { statusCode: err.statusCode, body: err.data },\n });\n }\n if (\n err.cause instanceof Error &&\n (err.cause.name === 'TimeoutError' || err.cause.name === 'AbortError')\n ) {\n throw new APIStatusError({\n message: `interruption inference timeout: ${err.message}`,\n options: { statusCode: 408, retryable: false },\n });\n }\n throw new APIConnectionError({\n message: `interruption inference connection error: ${err.message}`,\n });\n }\n throw new APIError(`error during interruption prediction: ${err}`);\n }\n}\n\nexport interface HttpTransportOptions {\n baseUrl: string;\n apiKey: string;\n apiSecret: string;\n threshold: number;\n minFrames: number;\n timeout: number;\n maxRetries?: number;\n}\n\nexport interface HttpTransportState {\n overlapSpeechStarted: boolean;\n overlapSpeechStartedAt: number | undefined;\n cache: BoundedCache<number, InterruptionCacheEntry>;\n}\n\n/**\n * Creates an HTTP transport TransformStream for interruption detection.\n *\n * This transport receives Int16Array audio slices and outputs InterruptionEvents.\n * Each audio slice triggers an HTTP POST request.\n *\n * @param options - Transport options object. This is read on each request, so mutations\n * to threshold/minFrames will be picked up dynamically.\n */\nexport function createHttpTransport(\n options: HttpTransportOptions,\n getState: () => HttpTransportState,\n setState: (partial: Partial<HttpTransportState>) => void,\n updateUserSpeakingSpan?: (entry: InterruptionCacheEntry) => void,\n getAndResetNumRequests?: () => number,\n): TransformStream<Int16Array | OverlappingSpeechEvent, OverlappingSpeechEvent> {\n const logger = log();\n\n return new TransformStream<Int16Array | OverlappingSpeechEvent, OverlappingSpeechEvent>(\n {\n async transform(chunk, controller) {\n if (!(chunk instanceof Int16Array)) {\n controller.enqueue(chunk);\n return;\n }\n\n const state = getState();\n const overlapSpeechStartedAt = state.overlapSpeechStartedAt;\n if (overlapSpeechStartedAt === undefined || !state.overlapSpeechStarted) return;\n\n try {\n const resp = await predictHTTP(\n chunk,\n { threshold: options.threshold, minFrames: options.minFrames },\n {\n baseUrl: options.baseUrl,\n timeout: options.timeout,\n maxRetries: options.maxRetries,\n token: await createAccessToken(options.apiKey, options.apiSecret),\n },\n );\n\n const { createdAt, isBargein, probabilities, predictionDurationInS } = resp;\n const entry = state.cache.setOrUpdate(\n createdAt,\n () => new InterruptionCacheEntry({ createdAt }),\n {\n probabilities,\n isInterruption: isBargein,\n speechInput: chunk,\n totalDurationInS: (performance.now() - createdAt) / 1000,\n detectionDelayInS: (Date.now() - overlapSpeechStartedAt) / 1000,\n predictionDurationInS,\n },\n );\n\n if (state.overlapSpeechStarted && entry.isInterruption) {\n if (updateUserSpeakingSpan) {\n updateUserSpeakingSpan(entry);\n }\n const event: OverlappingSpeechEvent = {\n type: 'overlapping_speech',\n detectedAt: Date.now(),\n overlapStartedAt: overlapSpeechStartedAt,\n isInterruption: entry.isInterruption,\n speechInput: entry.speechInput,\n probabilities: entry.probabilities,\n totalDurationInS: entry.totalDurationInS,\n predictionDurationInS: entry.predictionDurationInS,\n detectionDelayInS: entry.detectionDelayInS,\n probability: entry.probability,\n numRequests: getAndResetNumRequests?.() ?? 0,\n };\n logger.debug(\n {\n detectionDelayInS: entry.detectionDelayInS,\n totalDurationInS: entry.totalDurationInS,\n },\n 'interruption detected',\n );\n setState({ overlapSpeechStarted: false });\n controller.enqueue(event);\n }\n } catch (err) {\n controller.error(err);\n }\n },\n },\n { highWaterMark: 2 },\n { highWaterMark: 2 },\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,oBAAmC;AACnC,iBAAgC;AAChC,iBAAkB;AAClB,wBAAyE;AACzE,iBAAoB;AACpB,mBAAkC;AAClC,sCAAuC;AAiBhC,MAAM,gCAAgC,aAAE,OAAO;AAAA,EACpD,YAAY,aAAE,OAAO;AAAA,EACrB,YAAY,aAAE,QAAQ;AAAA,EACtB,eAAe,aAAE,MAAM,aAAE,OAAO,CAAC;AACnC,CAAC;AAWD,eAAsB,YACpB,MACA,gBACA,SAC0B;AAC1B,QAAM,YAAY,YAAY,IAAI;AAClC,QAAM,MAAM,IAAI,IAAI,YAAY,QAAQ,OAAO;AAC/C,MAAI,aAAa,OAAO,aAAa,eAAe,UAAU,SAAS,CAAC;AACxE,MAAI,aAAa,OAAO,cAAc,eAAe,UAAU,QAAQ,CAAC;AACxE,MAAI,aAAa,OAAO,cAAc,UAAU,QAAQ,CAAC;AAEzD,MAAI;AACF,UAAM,WAAW,UAAM,sBAAO,IAAI,SAAS,GAAG;AAAA,MAC5C,OAAO;AAAA,MACP,SAAS;AAAA,QACP,gBAAgB;AAAA,QAChB,eAAe,UAAU,QAAQ,KAAK;AAAA,MACxC;AAAA,MACA,QAAQ,QAAQ;AAAA,MAChB,SAAS,QAAQ;AAAA,MACjB,QAAQ;AAAA,MACR,MAAM;AAAA,IACR,CAAC;AACD,UAAM,EAAE,YAAY,YAAY,cAAc,IAAI,8BAA8B,MAAM,QAAQ;AAE9F,WAAO;AAAA,MACL,WAAW;AAAA,MACX,WAAW;AAAA,MACX;AAAA,MACA,wBAAwB,YAAY,IAAI,IAAI,aAAa;AAAA,IAC3D;AAAA,EACF,SAAS,KAAK;AACZ,YAAI,8BAAW,GAAG,EAAG,OAAM;AAC3B,QAAI,eAAe,0BAAY;AAC7B,UAAI,IAAI,YAAY;AAClB,cAAM,IAAI,iCAAe;AAAA,UACvB,SAAS,yCAAyC,IAAI,OAAO;AAAA,UAC7D,SAAS,EAAE,YAAY,IAAI,YAAY,MAAM,IAAI,KAAK;AAAA,QACxD,CAAC;AAAA,MACH;AACA,UACE,IAAI,iBAAiB,UACpB,IAAI,MAAM,SAAS,kBAAkB,IAAI,MAAM,SAAS,eACzD;AACA,cAAM,IAAI,iCAAe;AAAA,UACvB,SAAS,mCAAmC,IAAI,OAAO;AAAA,UACvD,SAAS,EAAE,YAAY,KAAK,WAAW,MAAM;AAAA,QAC/C,CAAC;AAAA,MACH;AACA,YAAM,IAAI,qCAAmB;AAAA,QAC3B,SAAS,4CAA4C,IAAI,OAAO;AAAA,MAClE,CAAC;AAAA,IACH;AACA,UAAM,IAAI,2BAAS,yCAAyC,GAAG,EAAE;AAAA,EACnE;AACF;AA2BO,SAAS,oBACd,SACA,UACA,UACA,wBACA,wBAC8E;AAC9E,QAAM,aAAS,gBAAI;AAEnB,SAAO,IAAI;AAAA,IACT;AAAA,MACE,MAAM,UAAU,OAAO,YAAY;AACjC,YAAI,EAAE,iBAAiB,aAAa;AAClC,qBAAW,QAAQ,KAAK;AACxB;AAAA,QACF;AAEA,cAAM,QAAQ,SAAS;AACvB,cAAM,yBAAyB,MAAM;AACrC,YAAI,2BAA2B,UAAa,CAAC,MAAM,qBAAsB;AAEzE,YAAI;AACF,gBAAM,OAAO,MAAM;AAAA,YACjB;AAAA,YACA,EAAE,WAAW,QAAQ,WAAW,WAAW,QAAQ,UAAU;AAAA,YAC7D;AAAA,cACE,SAAS,QAAQ;AAAA,cACjB,SAAS,QAAQ;AAAA,cACjB,YAAY,QAAQ;AAAA,cACpB,OAAO,UAAM,gCAAkB,QAAQ,QAAQ,QAAQ,SAAS;AAAA,YAClE;AAAA,UACF;AAEA,gBAAM,EAAE,WAAW,WAAW,eAAe,sBAAsB,IAAI;AACvE,gBAAM,QAAQ,MAAM,MAAM;AAAA,YACxB;AAAA,YACA,MAAM,IAAI,uDAAuB,EAAE,UAAU,CAAC;AAAA,YAC9C;AAAA,cACE;AAAA,cACA,gBAAgB;AAAA,cAChB,aAAa;AAAA,cACb,mBAAmB,YAAY,IAAI,IAAI,aAAa;AAAA,cACpD,oBAAoB,KAAK,IAAI,IAAI,0BAA0B;AAAA,cAC3D;AAAA,YACF;AAAA,UACF;AAEA,cAAI,MAAM,wBAAwB,MAAM,gBAAgB;AACtD,gBAAI,wBAAwB;AAC1B,qCAAuB,KAAK;AAAA,YAC9B;AACA,kBAAM,QAAgC;AAAA,cACpC,MAAM;AAAA,cACN,YAAY,KAAK,IAAI;AAAA,cACrB,kBAAkB;AAAA,cAClB,gBAAgB,MAAM;AAAA,cACtB,aAAa,MAAM;AAAA,cACnB,eAAe,MAAM;AAAA,cACrB,kBAAkB,MAAM;AAAA,cACxB,uBAAuB,MAAM;AAAA,cAC7B,mBAAmB,MAAM;AAAA,cACzB,aAAa,MAAM;AAAA,cACnB,cAAa,uEAA8B;AAAA,YAC7C;AACA,mBAAO;AAAA,cACL;AAAA,gBACE,mBAAmB,MAAM;AAAA,gBACzB,kBAAkB,MAAM;AAAA,cAC1B;AAAA,cACA;AAAA,YACF;AACA,qBAAS,EAAE,sBAAsB,MAAM,CAAC;AACxC,uBAAW,QAAQ,KAAK;AAAA,UAC1B;AAAA,QACF,SAAS,KAAK;AACZ,qBAAW,MAAM,GAAG;AAAA,QACtB;AAAA,MACF;AAAA,IACF;AAAA,IACA,EAAE,eAAe,EAAE;AAAA,IACnB,EAAE,eAAe,EAAE;AAAA,EACrB;AACF;","names":[]}
|
|
1
|
+
{"version":3,"sources":["../../../src/inference/interruption/http_transport.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2026 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { Throws } from '@livekit/throws-transformer/throws';\nimport { FetchError, ofetch } from 'ofetch';\nimport { TransformStream } from 'stream/web';\nimport { z } from 'zod';\nimport { APIConnectionError, APIError, APIStatusError, isAPIError } from '../../_exceptions.js';\nimport { log } from '../../log.js';\nimport { createAccessToken } from '../utils.js';\nimport { InterruptionCacheEntry } from './interruption_cache_entry.js';\nimport type { OverlappingSpeechEvent } from './types.js';\nimport type { BoundedCache } from './utils.js';\n\nexport interface PostOptions {\n baseUrl: string;\n token: string;\n signal?: AbortSignal;\n timeout?: number;\n maxRetries?: number;\n}\n\nexport interface PredictOptions {\n threshold: number;\n minFrames: number;\n}\n\nexport const predictEndpointResponseSchema = z.object({\n created_at: z.number(),\n is_bargein: z.boolean(),\n probabilities: z.array(z.number()),\n});\n\nexport type PredictEndpointResponse = z.infer<typeof predictEndpointResponseSchema>;\n\nexport interface PredictResponse {\n createdAt: number;\n isBargein: boolean;\n probabilities: number[];\n predictionDurationInS: number;\n}\n\nexport async function predictHTTP(\n data: Int16Array,\n predictOptions: PredictOptions,\n options: PostOptions,\n): Promise<Throws<PredictResponse, APIConnectionError | APIStatusError | APIError>> {\n const createdAt = performance.now();\n const url = new URL(`/bargein`, options.baseUrl);\n url.searchParams.append('threshold', predictOptions.threshold.toString());\n url.searchParams.append('min_frames', predictOptions.minFrames.toFixed());\n url.searchParams.append('created_at', createdAt.toFixed());\n\n try {\n const response = await ofetch(url.toString(), {\n retry: 0,\n headers: {\n 'Content-Type': 'application/octet-stream',\n Authorization: `Bearer ${options.token}`,\n },\n signal: options.signal,\n timeout: options.timeout,\n method: 'POST',\n body: data,\n });\n const { created_at, is_bargein, probabilities } = predictEndpointResponseSchema.parse(response);\n\n return {\n createdAt: created_at,\n isBargein: is_bargein,\n probabilities,\n predictionDurationInS: (performance.now() - createdAt) / 1000,\n };\n } catch (err) {\n if (isAPIError(err)) throw err;\n if (err instanceof FetchError) {\n if (err.statusCode) {\n throw new APIStatusError({\n message: `error during interruption prediction: ${err.message}`,\n options: { statusCode: err.statusCode, body: err.data },\n });\n }\n if (\n err.cause instanceof Error &&\n (err.cause.name === 'TimeoutError' || err.cause.name === 'AbortError')\n ) {\n throw new APIStatusError({\n message: `interruption inference timeout: ${err.message}`,\n options: { statusCode: 408, retryable: false },\n });\n }\n throw new APIConnectionError({\n message: `interruption inference connection error: ${err.message}`,\n });\n }\n throw new APIError(`error during interruption prediction: ${err}`);\n }\n}\n\nexport interface HttpTransportOptions {\n baseUrl: string;\n apiKey: string;\n apiSecret: string;\n threshold: number;\n minFrames: number;\n timeout: number;\n maxRetries?: number;\n}\n\nexport interface HttpTransportState {\n overlapSpeechStarted: boolean;\n overlapSpeechStartedAt: number | undefined;\n cache: BoundedCache<number, InterruptionCacheEntry>;\n}\n\n/**\n * Creates an HTTP transport TransformStream for interruption detection.\n *\n * This transport receives Int16Array audio slices and outputs InterruptionEvents.\n * Each audio slice triggers an HTTP POST request.\n *\n * @param options - Transport options object. This is read on each request, so mutations\n * to threshold/minFrames will be picked up dynamically.\n */\nexport function createHttpTransport(\n options: HttpTransportOptions,\n getState: () => HttpTransportState,\n setState: (partial: Partial<HttpTransportState>) => void,\n updateUserSpeakingSpan?: (entry: InterruptionCacheEntry) => void,\n getAndResetNumRequests?: () => number,\n): TransformStream<Int16Array | OverlappingSpeechEvent, OverlappingSpeechEvent> {\n const logger = log();\n\n return new TransformStream<Int16Array | OverlappingSpeechEvent, OverlappingSpeechEvent>(\n {\n async transform(chunk, controller) {\n if (!(chunk instanceof Int16Array)) {\n controller.enqueue(chunk);\n return;\n }\n\n const state = getState();\n const overlapSpeechStartedAt = state.overlapSpeechStartedAt;\n if (overlapSpeechStartedAt === undefined || !state.overlapSpeechStarted) return;\n\n try {\n const resp = await predictHTTP(\n chunk,\n { threshold: options.threshold, minFrames: options.minFrames },\n {\n baseUrl: options.baseUrl,\n timeout: options.timeout,\n maxRetries: options.maxRetries,\n token: await createAccessToken(options.apiKey, options.apiSecret),\n },\n );\n\n const { createdAt, isBargein, probabilities, predictionDurationInS } = resp;\n const entry = state.cache.setOrUpdate(\n createdAt,\n () => new InterruptionCacheEntry({ createdAt }),\n {\n probabilities,\n isInterruption: isBargein,\n speechInput: chunk,\n totalDurationInS: (performance.now() - createdAt) / 1000,\n detectionDelayInS: (Date.now() - overlapSpeechStartedAt) / 1000,\n predictionDurationInS,\n },\n );\n\n if (state.overlapSpeechStarted && entry.isInterruption) {\n if (updateUserSpeakingSpan) {\n updateUserSpeakingSpan(entry);\n }\n const event: OverlappingSpeechEvent = {\n type: 'overlapping_speech',\n detectedAt: Date.now(),\n overlapStartedAt: overlapSpeechStartedAt,\n isInterruption: entry.isInterruption,\n speechInput: entry.speechInput,\n probabilities: entry.probabilities,\n totalDurationInS: entry.totalDurationInS,\n predictionDurationInS: entry.predictionDurationInS,\n detectionDelayInS: entry.detectionDelayInS,\n probability: entry.probability,\n numRequests: getAndResetNumRequests?.() ?? 0,\n };\n logger.debug(\n {\n detectionDelayInS: entry.detectionDelayInS,\n totalDurationInS: entry.totalDurationInS,\n },\n 'interruption detected',\n );\n setState({ overlapSpeechStarted: false });\n controller.enqueue(event);\n }\n } catch (err) {\n controller.error(err);\n }\n },\n },\n { highWaterMark: 2 },\n { highWaterMark: 2 },\n );\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,oBAAmC;AACnC,iBAAgC;AAChC,iBAAkB;AAClB,wBAAyE;AACzE,iBAAoB;AACpB,mBAAkC;AAClC,sCAAuC;AAiBhC,MAAM,gCAAgC,aAAE,OAAO;AAAA,EACpD,YAAY,aAAE,OAAO;AAAA,EACrB,YAAY,aAAE,QAAQ;AAAA,EACtB,eAAe,aAAE,MAAM,aAAE,OAAO,CAAC;AACnC,CAAC;AAWD,eAAsB,YACpB,MACA,gBACA,SACkF;AAClF,QAAM,YAAY,YAAY,IAAI;AAClC,QAAM,MAAM,IAAI,IAAI,YAAY,QAAQ,OAAO;AAC/C,MAAI,aAAa,OAAO,aAAa,eAAe,UAAU,SAAS,CAAC;AACxE,MAAI,aAAa,OAAO,cAAc,eAAe,UAAU,QAAQ,CAAC;AACxE,MAAI,aAAa,OAAO,cAAc,UAAU,QAAQ,CAAC;AAEzD,MAAI;AACF,UAAM,WAAW,UAAM,sBAAO,IAAI,SAAS,GAAG;AAAA,MAC5C,OAAO;AAAA,MACP,SAAS;AAAA,QACP,gBAAgB;AAAA,QAChB,eAAe,UAAU,QAAQ,KAAK;AAAA,MACxC;AAAA,MACA,QAAQ,QAAQ;AAAA,MAChB,SAAS,QAAQ;AAAA,MACjB,QAAQ;AAAA,MACR,MAAM;AAAA,IACR,CAAC;AACD,UAAM,EAAE,YAAY,YAAY,cAAc,IAAI,8BAA8B,MAAM,QAAQ;AAE9F,WAAO;AAAA,MACL,WAAW;AAAA,MACX,WAAW;AAAA,MACX;AAAA,MACA,wBAAwB,YAAY,IAAI,IAAI,aAAa;AAAA,IAC3D;AAAA,EACF,SAAS,KAAK;AACZ,YAAI,8BAAW,GAAG,EAAG,OAAM;AAC3B,QAAI,eAAe,0BAAY;AAC7B,UAAI,IAAI,YAAY;AAClB,cAAM,IAAI,iCAAe;AAAA,UACvB,SAAS,yCAAyC,IAAI,OAAO;AAAA,UAC7D,SAAS,EAAE,YAAY,IAAI,YAAY,MAAM,IAAI,KAAK;AAAA,QACxD,CAAC;AAAA,MACH;AACA,UACE,IAAI,iBAAiB,UACpB,IAAI,MAAM,SAAS,kBAAkB,IAAI,MAAM,SAAS,eACzD;AACA,cAAM,IAAI,iCAAe;AAAA,UACvB,SAAS,mCAAmC,IAAI,OAAO;AAAA,UACvD,SAAS,EAAE,YAAY,KAAK,WAAW,MAAM;AAAA,QAC/C,CAAC;AAAA,MACH;AACA,YAAM,IAAI,qCAAmB;AAAA,QAC3B,SAAS,4CAA4C,IAAI,OAAO;AAAA,MAClE,CAAC;AAAA,IACH;AACA,UAAM,IAAI,2BAAS,yCAAyC,GAAG,EAAE;AAAA,EACnE;AACF;AA2BO,SAAS,oBACd,SACA,UACA,UACA,wBACA,wBAC8E;AAC9E,QAAM,aAAS,gBAAI;AAEnB,SAAO,IAAI;AAAA,IACT;AAAA,MACE,MAAM,UAAU,OAAO,YAAY;AACjC,YAAI,EAAE,iBAAiB,aAAa;AAClC,qBAAW,QAAQ,KAAK;AACxB;AAAA,QACF;AAEA,cAAM,QAAQ,SAAS;AACvB,cAAM,yBAAyB,MAAM;AACrC,YAAI,2BAA2B,UAAa,CAAC,MAAM,qBAAsB;AAEzE,YAAI;AACF,gBAAM,OAAO,MAAM;AAAA,YACjB;AAAA,YACA,EAAE,WAAW,QAAQ,WAAW,WAAW,QAAQ,UAAU;AAAA,YAC7D;AAAA,cACE,SAAS,QAAQ;AAAA,cACjB,SAAS,QAAQ;AAAA,cACjB,YAAY,QAAQ;AAAA,cACpB,OAAO,UAAM,gCAAkB,QAAQ,QAAQ,QAAQ,SAAS;AAAA,YAClE;AAAA,UACF;AAEA,gBAAM,EAAE,WAAW,WAAW,eAAe,sBAAsB,IAAI;AACvE,gBAAM,QAAQ,MAAM,MAAM;AAAA,YACxB;AAAA,YACA,MAAM,IAAI,uDAAuB,EAAE,UAAU,CAAC;AAAA,YAC9C;AAAA,cACE;AAAA,cACA,gBAAgB;AAAA,cAChB,aAAa;AAAA,cACb,mBAAmB,YAAY,IAAI,IAAI,aAAa;AAAA,cACpD,oBAAoB,KAAK,IAAI,IAAI,0BAA0B;AAAA,cAC3D;AAAA,YACF;AAAA,UACF;AAEA,cAAI,MAAM,wBAAwB,MAAM,gBAAgB;AACtD,gBAAI,wBAAwB;AAC1B,qCAAuB,KAAK;AAAA,YAC9B;AACA,kBAAM,QAAgC;AAAA,cACpC,MAAM;AAAA,cACN,YAAY,KAAK,IAAI;AAAA,cACrB,kBAAkB;AAAA,cAClB,gBAAgB,MAAM;AAAA,cACtB,aAAa,MAAM;AAAA,cACnB,eAAe,MAAM;AAAA,cACrB,kBAAkB,MAAM;AAAA,cACxB,uBAAuB,MAAM;AAAA,cAC7B,mBAAmB,MAAM;AAAA,cACzB,aAAa,MAAM;AAAA,cACnB,cAAa,uEAA8B;AAAA,YAC7C;AACA,mBAAO;AAAA,cACL;AAAA,gBACE,mBAAmB,MAAM;AAAA,gBACzB,kBAAkB,MAAM;AAAA,cAC1B;AAAA,cACA;AAAA,YACF;AACA,qBAAS,EAAE,sBAAsB,MAAM,CAAC;AACxC,uBAAW,QAAQ,KAAK;AAAA,UAC1B;AAAA,QACF,SAAS,KAAK;AACZ,qBAAW,MAAM,GAAG;AAAA,QACtB;AAAA,MACF;AAAA,IACF;AAAA,IACA,EAAE,eAAe,EAAE;AAAA,IACnB,EAAE,eAAe,EAAE;AAAA,EACrB;AACF;","names":[]}
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
/// <reference types="node" resolution-mode="require"/>
|
|
2
|
+
import type { Throws } from '@livekit/throws-transformer/throws';
|
|
2
3
|
import { TransformStream } from 'stream/web';
|
|
3
4
|
import { z } from 'zod';
|
|
5
|
+
import { APIConnectionError, APIError, APIStatusError } from '../../_exceptions.js';
|
|
4
6
|
import { InterruptionCacheEntry } from './interruption_cache_entry.js';
|
|
5
7
|
import type { OverlappingSpeechEvent } from './types.js';
|
|
6
8
|
import type { BoundedCache } from './utils.js';
|
|
@@ -35,7 +37,7 @@ export interface PredictResponse {
|
|
|
35
37
|
probabilities: number[];
|
|
36
38
|
predictionDurationInS: number;
|
|
37
39
|
}
|
|
38
|
-
export declare function predictHTTP(data: Int16Array, predictOptions: PredictOptions, options: PostOptions): Promise<PredictResponse
|
|
40
|
+
export declare function predictHTTP(data: Int16Array, predictOptions: PredictOptions, options: PostOptions): Promise<Throws<PredictResponse, APIConnectionError | APIStatusError | APIError>>;
|
|
39
41
|
export interface HttpTransportOptions {
|
|
40
42
|
baseUrl: string;
|
|
41
43
|
apiKey: string;
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
/// <reference types="node" resolution-mode="require"/>
|
|
2
|
+
import type { Throws } from '@livekit/throws-transformer/throws';
|
|
2
3
|
import { TransformStream } from 'stream/web';
|
|
3
4
|
import { z } from 'zod';
|
|
5
|
+
import { APIConnectionError, APIError, APIStatusError } from '../../_exceptions.js';
|
|
4
6
|
import { InterruptionCacheEntry } from './interruption_cache_entry.js';
|
|
5
7
|
import type { OverlappingSpeechEvent } from './types.js';
|
|
6
8
|
import type { BoundedCache } from './utils.js';
|
|
@@ -35,7 +37,7 @@ export interface PredictResponse {
|
|
|
35
37
|
probabilities: number[];
|
|
36
38
|
predictionDurationInS: number;
|
|
37
39
|
}
|
|
38
|
-
export declare function predictHTTP(data: Int16Array, predictOptions: PredictOptions, options: PostOptions): Promise<PredictResponse
|
|
40
|
+
export declare function predictHTTP(data: Int16Array, predictOptions: PredictOptions, options: PostOptions): Promise<Throws<PredictResponse, APIConnectionError | APIStatusError | APIError>>;
|
|
39
41
|
export interface HttpTransportOptions {
|
|
40
42
|
baseUrl: string;
|
|
41
43
|
apiKey: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http_transport.d.ts","sourceRoot":"","sources":["../../../src/inference/interruption/http_transport.ts"],"names":[],"mappings":";
|
|
1
|
+
{"version":3,"file":"http_transport.d.ts","sourceRoot":"","sources":["../../../src/inference/interruption/http_transport.ts"],"names":[],"mappings":";AAGA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oCAAoC,CAAC;AAEjE,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,kBAAkB,EAAE,QAAQ,EAAE,cAAc,EAAc,MAAM,sBAAsB,CAAC;AAGhG,OAAO,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AACvE,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AACzD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE/C,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,eAAO,MAAM,6BAA6B;;;;;;;;;;;;EAIxC,CAAC;AAEH,MAAM,MAAM,uBAAuB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,6BAA6B,CAAC,CAAC;AAEpF,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,OAAO,CAAC;IACnB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,qBAAqB,EAAE,MAAM,CAAC;CAC/B;AAED,wBAAsB,WAAW,CAC/B,IAAI,EAAE,UAAU,EAChB,cAAc,EAAE,cAAc,EAC9B,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC,MAAM,CAAC,eAAe,EAAE,kBAAkB,GAAG,cAAc,GAAG,QAAQ,CAAC,CAAC,CAmDlF;AAED,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,kBAAkB;IACjC,oBAAoB,EAAE,OAAO,CAAC;IAC9B,sBAAsB,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3C,KAAK,EAAE,YAAY,CAAC,MAAM,EAAE,sBAAsB,CAAC,CAAC;CACrD;AAED;;;;;;;;GAQG;AACH,wBAAgB,mBAAmB,CACjC,OAAO,EAAE,oBAAoB,EAC7B,QAAQ,EAAE,MAAM,kBAAkB,EAClC,QAAQ,EAAE,CAAC,OAAO,EAAE,OAAO,CAAC,kBAAkB,CAAC,KAAK,IAAI,EACxD,sBAAsB,CAAC,EAAE,CAAC,KAAK,EAAE,sBAAsB,KAAK,IAAI,EAChE,sBAAsB,CAAC,EAAE,MAAM,MAAM,GACpC,eAAe,CAAC,UAAU,GAAG,sBAAsB,EAAE,sBAAsB,CAAC,CA4E9E"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../src/inference/interruption/http_transport.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2026 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { FetchError, ofetch } from 'ofetch';\nimport { TransformStream } from 'stream/web';\nimport { z } from 'zod';\nimport { APIConnectionError, APIError, APIStatusError, isAPIError } from '../../_exceptions.js';\nimport { log } from '../../log.js';\nimport { createAccessToken } from '../utils.js';\nimport { InterruptionCacheEntry } from './interruption_cache_entry.js';\nimport type { OverlappingSpeechEvent } from './types.js';\nimport type { BoundedCache } from './utils.js';\n\nexport interface PostOptions {\n baseUrl: string;\n token: string;\n signal?: AbortSignal;\n timeout?: number;\n maxRetries?: number;\n}\n\nexport interface PredictOptions {\n threshold: number;\n minFrames: number;\n}\n\nexport const predictEndpointResponseSchema = z.object({\n created_at: z.number(),\n is_bargein: z.boolean(),\n probabilities: z.array(z.number()),\n});\n\nexport type PredictEndpointResponse = z.infer<typeof predictEndpointResponseSchema>;\n\nexport interface PredictResponse {\n createdAt: number;\n isBargein: boolean;\n probabilities: number[];\n predictionDurationInS: number;\n}\n\nexport async function predictHTTP(\n data: Int16Array,\n predictOptions: PredictOptions,\n options: PostOptions,\n): Promise<PredictResponse> {\n const createdAt = performance.now();\n const url = new URL(`/bargein`, options.baseUrl);\n url.searchParams.append('threshold', predictOptions.threshold.toString());\n url.searchParams.append('min_frames', predictOptions.minFrames.toFixed());\n url.searchParams.append('created_at', createdAt.toFixed());\n\n try {\n const response = await ofetch(url.toString(), {\n retry: 0,\n headers: {\n 'Content-Type': 'application/octet-stream',\n Authorization: `Bearer ${options.token}`,\n },\n signal: options.signal,\n timeout: options.timeout,\n method: 'POST',\n body: data,\n });\n const { created_at, is_bargein, probabilities } = predictEndpointResponseSchema.parse(response);\n\n return {\n createdAt: created_at,\n isBargein: is_bargein,\n probabilities,\n predictionDurationInS: (performance.now() - createdAt) / 1000,\n };\n } catch (err) {\n if (isAPIError(err)) throw err;\n if (err instanceof FetchError) {\n if (err.statusCode) {\n throw new APIStatusError({\n message: `error during interruption prediction: ${err.message}`,\n options: { statusCode: err.statusCode, body: err.data },\n });\n }\n if (\n err.cause instanceof Error &&\n (err.cause.name === 'TimeoutError' || err.cause.name === 'AbortError')\n ) {\n throw new APIStatusError({\n message: `interruption inference timeout: ${err.message}`,\n options: { statusCode: 408, retryable: false },\n });\n }\n throw new APIConnectionError({\n message: `interruption inference connection error: ${err.message}`,\n });\n }\n throw new APIError(`error during interruption prediction: ${err}`);\n }\n}\n\nexport interface HttpTransportOptions {\n baseUrl: string;\n apiKey: string;\n apiSecret: string;\n threshold: number;\n minFrames: number;\n timeout: number;\n maxRetries?: number;\n}\n\nexport interface HttpTransportState {\n overlapSpeechStarted: boolean;\n overlapSpeechStartedAt: number | undefined;\n cache: BoundedCache<number, InterruptionCacheEntry>;\n}\n\n/**\n * Creates an HTTP transport TransformStream for interruption detection.\n *\n * This transport receives Int16Array audio slices and outputs InterruptionEvents.\n * Each audio slice triggers an HTTP POST request.\n *\n * @param options - Transport options object. This is read on each request, so mutations\n * to threshold/minFrames will be picked up dynamically.\n */\nexport function createHttpTransport(\n options: HttpTransportOptions,\n getState: () => HttpTransportState,\n setState: (partial: Partial<HttpTransportState>) => void,\n updateUserSpeakingSpan?: (entry: InterruptionCacheEntry) => void,\n getAndResetNumRequests?: () => number,\n): TransformStream<Int16Array | OverlappingSpeechEvent, OverlappingSpeechEvent> {\n const logger = log();\n\n return new TransformStream<Int16Array | OverlappingSpeechEvent, OverlappingSpeechEvent>(\n {\n async transform(chunk, controller) {\n if (!(chunk instanceof Int16Array)) {\n controller.enqueue(chunk);\n return;\n }\n\n const state = getState();\n const overlapSpeechStartedAt = state.overlapSpeechStartedAt;\n if (overlapSpeechStartedAt === undefined || !state.overlapSpeechStarted) return;\n\n try {\n const resp = await predictHTTP(\n chunk,\n { threshold: options.threshold, minFrames: options.minFrames },\n {\n baseUrl: options.baseUrl,\n timeout: options.timeout,\n maxRetries: options.maxRetries,\n token: await createAccessToken(options.apiKey, options.apiSecret),\n },\n );\n\n const { createdAt, isBargein, probabilities, predictionDurationInS } = resp;\n const entry = state.cache.setOrUpdate(\n createdAt,\n () => new InterruptionCacheEntry({ createdAt }),\n {\n probabilities,\n isInterruption: isBargein,\n speechInput: chunk,\n totalDurationInS: (performance.now() - createdAt) / 1000,\n detectionDelayInS: (Date.now() - overlapSpeechStartedAt) / 1000,\n predictionDurationInS,\n },\n );\n\n if (state.overlapSpeechStarted && entry.isInterruption) {\n if (updateUserSpeakingSpan) {\n updateUserSpeakingSpan(entry);\n }\n const event: OverlappingSpeechEvent = {\n type: 'overlapping_speech',\n detectedAt: Date.now(),\n overlapStartedAt: overlapSpeechStartedAt,\n isInterruption: entry.isInterruption,\n speechInput: entry.speechInput,\n probabilities: entry.probabilities,\n totalDurationInS: entry.totalDurationInS,\n predictionDurationInS: entry.predictionDurationInS,\n detectionDelayInS: entry.detectionDelayInS,\n probability: entry.probability,\n numRequests: getAndResetNumRequests?.() ?? 0,\n };\n logger.debug(\n {\n detectionDelayInS: entry.detectionDelayInS,\n totalDurationInS: entry.totalDurationInS,\n },\n 'interruption detected',\n );\n setState({ overlapSpeechStarted: false });\n controller.enqueue(event);\n }\n } catch (err) {\n controller.error(err);\n }\n },\n },\n { highWaterMark: 2 },\n { highWaterMark: 2 },\n );\n}\n"],"mappings":"AAGA,SAAS,YAAY,cAAc;AACnC,SAAS,uBAAuB;AAChC,SAAS,SAAS;AAClB,SAAS,oBAAoB,UAAU,gBAAgB,kBAAkB;AACzE,SAAS,WAAW;AACpB,SAAS,yBAAyB;AAClC,SAAS,8BAA8B;AAiBhC,MAAM,gCAAgC,EAAE,OAAO;AAAA,EACpD,YAAY,EAAE,OAAO;AAAA,EACrB,YAAY,EAAE,QAAQ;AAAA,EACtB,eAAe,EAAE,MAAM,EAAE,OAAO,CAAC;AACnC,CAAC;AAWD,eAAsB,YACpB,MACA,gBACA,SAC0B;AAC1B,QAAM,YAAY,YAAY,IAAI;AAClC,QAAM,MAAM,IAAI,IAAI,YAAY,QAAQ,OAAO;AAC/C,MAAI,aAAa,OAAO,aAAa,eAAe,UAAU,SAAS,CAAC;AACxE,MAAI,aAAa,OAAO,cAAc,eAAe,UAAU,QAAQ,CAAC;AACxE,MAAI,aAAa,OAAO,cAAc,UAAU,QAAQ,CAAC;AAEzD,MAAI;AACF,UAAM,WAAW,MAAM,OAAO,IAAI,SAAS,GAAG;AAAA,MAC5C,OAAO;AAAA,MACP,SAAS;AAAA,QACP,gBAAgB;AAAA,QAChB,eAAe,UAAU,QAAQ,KAAK;AAAA,MACxC;AAAA,MACA,QAAQ,QAAQ;AAAA,MAChB,SAAS,QAAQ;AAAA,MACjB,QAAQ;AAAA,MACR,MAAM;AAAA,IACR,CAAC;AACD,UAAM,EAAE,YAAY,YAAY,cAAc,IAAI,8BAA8B,MAAM,QAAQ;AAE9F,WAAO;AAAA,MACL,WAAW;AAAA,MACX,WAAW;AAAA,MACX;AAAA,MACA,wBAAwB,YAAY,IAAI,IAAI,aAAa;AAAA,IAC3D;AAAA,EACF,SAAS,KAAK;AACZ,QAAI,WAAW,GAAG,EAAG,OAAM;AAC3B,QAAI,eAAe,YAAY;AAC7B,UAAI,IAAI,YAAY;AAClB,cAAM,IAAI,eAAe;AAAA,UACvB,SAAS,yCAAyC,IAAI,OAAO;AAAA,UAC7D,SAAS,EAAE,YAAY,IAAI,YAAY,MAAM,IAAI,KAAK;AAAA,QACxD,CAAC;AAAA,MACH;AACA,UACE,IAAI,iBAAiB,UACpB,IAAI,MAAM,SAAS,kBAAkB,IAAI,MAAM,SAAS,eACzD;AACA,cAAM,IAAI,eAAe;AAAA,UACvB,SAAS,mCAAmC,IAAI,OAAO;AAAA,UACvD,SAAS,EAAE,YAAY,KAAK,WAAW,MAAM;AAAA,QAC/C,CAAC;AAAA,MACH;AACA,YAAM,IAAI,mBAAmB;AAAA,QAC3B,SAAS,4CAA4C,IAAI,OAAO;AAAA,MAClE,CAAC;AAAA,IACH;AACA,UAAM,IAAI,SAAS,yCAAyC,GAAG,EAAE;AAAA,EACnE;AACF;AA2BO,SAAS,oBACd,SACA,UACA,UACA,wBACA,wBAC8E;AAC9E,QAAM,SAAS,IAAI;AAEnB,SAAO,IAAI;AAAA,IACT;AAAA,MACE,MAAM,UAAU,OAAO,YAAY;AACjC,YAAI,EAAE,iBAAiB,aAAa;AAClC,qBAAW,QAAQ,KAAK;AACxB;AAAA,QACF;AAEA,cAAM,QAAQ,SAAS;AACvB,cAAM,yBAAyB,MAAM;AACrC,YAAI,2BAA2B,UAAa,CAAC,MAAM,qBAAsB;AAEzE,YAAI;AACF,gBAAM,OAAO,MAAM;AAAA,YACjB;AAAA,YACA,EAAE,WAAW,QAAQ,WAAW,WAAW,QAAQ,UAAU;AAAA,YAC7D;AAAA,cACE,SAAS,QAAQ;AAAA,cACjB,SAAS,QAAQ;AAAA,cACjB,YAAY,QAAQ;AAAA,cACpB,OAAO,MAAM,kBAAkB,QAAQ,QAAQ,QAAQ,SAAS;AAAA,YAClE;AAAA,UACF;AAEA,gBAAM,EAAE,WAAW,WAAW,eAAe,sBAAsB,IAAI;AACvE,gBAAM,QAAQ,MAAM,MAAM;AAAA,YACxB;AAAA,YACA,MAAM,IAAI,uBAAuB,EAAE,UAAU,CAAC;AAAA,YAC9C;AAAA,cACE;AAAA,cACA,gBAAgB;AAAA,cAChB,aAAa;AAAA,cACb,mBAAmB,YAAY,IAAI,IAAI,aAAa;AAAA,cACpD,oBAAoB,KAAK,IAAI,IAAI,0BAA0B;AAAA,cAC3D;AAAA,YACF;AAAA,UACF;AAEA,cAAI,MAAM,wBAAwB,MAAM,gBAAgB;AACtD,gBAAI,wBAAwB;AAC1B,qCAAuB,KAAK;AAAA,YAC9B;AACA,kBAAM,QAAgC;AAAA,cACpC,MAAM;AAAA,cACN,YAAY,KAAK,IAAI;AAAA,cACrB,kBAAkB;AAAA,cAClB,gBAAgB,MAAM;AAAA,cACtB,aAAa,MAAM;AAAA,cACnB,eAAe,MAAM;AAAA,cACrB,kBAAkB,MAAM;AAAA,cACxB,uBAAuB,MAAM;AAAA,cAC7B,mBAAmB,MAAM;AAAA,cACzB,aAAa,MAAM;AAAA,cACnB,cAAa,uEAA8B;AAAA,YAC7C;AACA,mBAAO;AAAA,cACL;AAAA,gBACE,mBAAmB,MAAM;AAAA,gBACzB,kBAAkB,MAAM;AAAA,cAC1B;AAAA,cACA;AAAA,YACF;AACA,qBAAS,EAAE,sBAAsB,MAAM,CAAC;AACxC,uBAAW,QAAQ,KAAK;AAAA,UAC1B;AAAA,QACF,SAAS,KAAK;AACZ,qBAAW,MAAM,GAAG;AAAA,QACtB;AAAA,MACF;AAAA,IACF;AAAA,IACA,EAAE,eAAe,EAAE;AAAA,IACnB,EAAE,eAAe,EAAE;AAAA,EACrB;AACF;","names":[]}
|
|
1
|
+
{"version":3,"sources":["../../../src/inference/interruption/http_transport.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2026 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { Throws } from '@livekit/throws-transformer/throws';\nimport { FetchError, ofetch } from 'ofetch';\nimport { TransformStream } from 'stream/web';\nimport { z } from 'zod';\nimport { APIConnectionError, APIError, APIStatusError, isAPIError } from '../../_exceptions.js';\nimport { log } from '../../log.js';\nimport { createAccessToken } from '../utils.js';\nimport { InterruptionCacheEntry } from './interruption_cache_entry.js';\nimport type { OverlappingSpeechEvent } from './types.js';\nimport type { BoundedCache } from './utils.js';\n\nexport interface PostOptions {\n baseUrl: string;\n token: string;\n signal?: AbortSignal;\n timeout?: number;\n maxRetries?: number;\n}\n\nexport interface PredictOptions {\n threshold: number;\n minFrames: number;\n}\n\nexport const predictEndpointResponseSchema = z.object({\n created_at: z.number(),\n is_bargein: z.boolean(),\n probabilities: z.array(z.number()),\n});\n\nexport type PredictEndpointResponse = z.infer<typeof predictEndpointResponseSchema>;\n\nexport interface PredictResponse {\n createdAt: number;\n isBargein: boolean;\n probabilities: number[];\n predictionDurationInS: number;\n}\n\nexport async function predictHTTP(\n data: Int16Array,\n predictOptions: PredictOptions,\n options: PostOptions,\n): Promise<Throws<PredictResponse, APIConnectionError | APIStatusError | APIError>> {\n const createdAt = performance.now();\n const url = new URL(`/bargein`, options.baseUrl);\n url.searchParams.append('threshold', predictOptions.threshold.toString());\n url.searchParams.append('min_frames', predictOptions.minFrames.toFixed());\n url.searchParams.append('created_at', createdAt.toFixed());\n\n try {\n const response = await ofetch(url.toString(), {\n retry: 0,\n headers: {\n 'Content-Type': 'application/octet-stream',\n Authorization: `Bearer ${options.token}`,\n },\n signal: options.signal,\n timeout: options.timeout,\n method: 'POST',\n body: data,\n });\n const { created_at, is_bargein, probabilities } = predictEndpointResponseSchema.parse(response);\n\n return {\n createdAt: created_at,\n isBargein: is_bargein,\n probabilities,\n predictionDurationInS: (performance.now() - createdAt) / 1000,\n };\n } catch (err) {\n if (isAPIError(err)) throw err;\n if (err instanceof FetchError) {\n if (err.statusCode) {\n throw new APIStatusError({\n message: `error during interruption prediction: ${err.message}`,\n options: { statusCode: err.statusCode, body: err.data },\n });\n }\n if (\n err.cause instanceof Error &&\n (err.cause.name === 'TimeoutError' || err.cause.name === 'AbortError')\n ) {\n throw new APIStatusError({\n message: `interruption inference timeout: ${err.message}`,\n options: { statusCode: 408, retryable: false },\n });\n }\n throw new APIConnectionError({\n message: `interruption inference connection error: ${err.message}`,\n });\n }\n throw new APIError(`error during interruption prediction: ${err}`);\n }\n}\n\nexport interface HttpTransportOptions {\n baseUrl: string;\n apiKey: string;\n apiSecret: string;\n threshold: number;\n minFrames: number;\n timeout: number;\n maxRetries?: number;\n}\n\nexport interface HttpTransportState {\n overlapSpeechStarted: boolean;\n overlapSpeechStartedAt: number | undefined;\n cache: BoundedCache<number, InterruptionCacheEntry>;\n}\n\n/**\n * Creates an HTTP transport TransformStream for interruption detection.\n *\n * This transport receives Int16Array audio slices and outputs InterruptionEvents.\n * Each audio slice triggers an HTTP POST request.\n *\n * @param options - Transport options object. This is read on each request, so mutations\n * to threshold/minFrames will be picked up dynamically.\n */\nexport function createHttpTransport(\n options: HttpTransportOptions,\n getState: () => HttpTransportState,\n setState: (partial: Partial<HttpTransportState>) => void,\n updateUserSpeakingSpan?: (entry: InterruptionCacheEntry) => void,\n getAndResetNumRequests?: () => number,\n): TransformStream<Int16Array | OverlappingSpeechEvent, OverlappingSpeechEvent> {\n const logger = log();\n\n return new TransformStream<Int16Array | OverlappingSpeechEvent, OverlappingSpeechEvent>(\n {\n async transform(chunk, controller) {\n if (!(chunk instanceof Int16Array)) {\n controller.enqueue(chunk);\n return;\n }\n\n const state = getState();\n const overlapSpeechStartedAt = state.overlapSpeechStartedAt;\n if (overlapSpeechStartedAt === undefined || !state.overlapSpeechStarted) return;\n\n try {\n const resp = await predictHTTP(\n chunk,\n { threshold: options.threshold, minFrames: options.minFrames },\n {\n baseUrl: options.baseUrl,\n timeout: options.timeout,\n maxRetries: options.maxRetries,\n token: await createAccessToken(options.apiKey, options.apiSecret),\n },\n );\n\n const { createdAt, isBargein, probabilities, predictionDurationInS } = resp;\n const entry = state.cache.setOrUpdate(\n createdAt,\n () => new InterruptionCacheEntry({ createdAt }),\n {\n probabilities,\n isInterruption: isBargein,\n speechInput: chunk,\n totalDurationInS: (performance.now() - createdAt) / 1000,\n detectionDelayInS: (Date.now() - overlapSpeechStartedAt) / 1000,\n predictionDurationInS,\n },\n );\n\n if (state.overlapSpeechStarted && entry.isInterruption) {\n if (updateUserSpeakingSpan) {\n updateUserSpeakingSpan(entry);\n }\n const event: OverlappingSpeechEvent = {\n type: 'overlapping_speech',\n detectedAt: Date.now(),\n overlapStartedAt: overlapSpeechStartedAt,\n isInterruption: entry.isInterruption,\n speechInput: entry.speechInput,\n probabilities: entry.probabilities,\n totalDurationInS: entry.totalDurationInS,\n predictionDurationInS: entry.predictionDurationInS,\n detectionDelayInS: entry.detectionDelayInS,\n probability: entry.probability,\n numRequests: getAndResetNumRequests?.() ?? 0,\n };\n logger.debug(\n {\n detectionDelayInS: entry.detectionDelayInS,\n totalDurationInS: entry.totalDurationInS,\n },\n 'interruption detected',\n );\n setState({ overlapSpeechStarted: false });\n controller.enqueue(event);\n }\n } catch (err) {\n controller.error(err);\n }\n },\n },\n { highWaterMark: 2 },\n { highWaterMark: 2 },\n );\n}\n"],"mappings":"AAIA,SAAS,YAAY,cAAc;AACnC,SAAS,uBAAuB;AAChC,SAAS,SAAS;AAClB,SAAS,oBAAoB,UAAU,gBAAgB,kBAAkB;AACzE,SAAS,WAAW;AACpB,SAAS,yBAAyB;AAClC,SAAS,8BAA8B;AAiBhC,MAAM,gCAAgC,EAAE,OAAO;AAAA,EACpD,YAAY,EAAE,OAAO;AAAA,EACrB,YAAY,EAAE,QAAQ;AAAA,EACtB,eAAe,EAAE,MAAM,EAAE,OAAO,CAAC;AACnC,CAAC;AAWD,eAAsB,YACpB,MACA,gBACA,SACkF;AAClF,QAAM,YAAY,YAAY,IAAI;AAClC,QAAM,MAAM,IAAI,IAAI,YAAY,QAAQ,OAAO;AAC/C,MAAI,aAAa,OAAO,aAAa,eAAe,UAAU,SAAS,CAAC;AACxE,MAAI,aAAa,OAAO,cAAc,eAAe,UAAU,QAAQ,CAAC;AACxE,MAAI,aAAa,OAAO,cAAc,UAAU,QAAQ,CAAC;AAEzD,MAAI;AACF,UAAM,WAAW,MAAM,OAAO,IAAI,SAAS,GAAG;AAAA,MAC5C,OAAO;AAAA,MACP,SAAS;AAAA,QACP,gBAAgB;AAAA,QAChB,eAAe,UAAU,QAAQ,KAAK;AAAA,MACxC;AAAA,MACA,QAAQ,QAAQ;AAAA,MAChB,SAAS,QAAQ;AAAA,MACjB,QAAQ;AAAA,MACR,MAAM;AAAA,IACR,CAAC;AACD,UAAM,EAAE,YAAY,YAAY,cAAc,IAAI,8BAA8B,MAAM,QAAQ;AAE9F,WAAO;AAAA,MACL,WAAW;AAAA,MACX,WAAW;AAAA,MACX;AAAA,MACA,wBAAwB,YAAY,IAAI,IAAI,aAAa;AAAA,IAC3D;AAAA,EACF,SAAS,KAAK;AACZ,QAAI,WAAW,GAAG,EAAG,OAAM;AAC3B,QAAI,eAAe,YAAY;AAC7B,UAAI,IAAI,YAAY;AAClB,cAAM,IAAI,eAAe;AAAA,UACvB,SAAS,yCAAyC,IAAI,OAAO;AAAA,UAC7D,SAAS,EAAE,YAAY,IAAI,YAAY,MAAM,IAAI,KAAK;AAAA,QACxD,CAAC;AAAA,MACH;AACA,UACE,IAAI,iBAAiB,UACpB,IAAI,MAAM,SAAS,kBAAkB,IAAI,MAAM,SAAS,eACzD;AACA,cAAM,IAAI,eAAe;AAAA,UACvB,SAAS,mCAAmC,IAAI,OAAO;AAAA,UACvD,SAAS,EAAE,YAAY,KAAK,WAAW,MAAM;AAAA,QAC/C,CAAC;AAAA,MACH;AACA,YAAM,IAAI,mBAAmB;AAAA,QAC3B,SAAS,4CAA4C,IAAI,OAAO;AAAA,MAClE,CAAC;AAAA,IACH;AACA,UAAM,IAAI,SAAS,yCAAyC,GAAG,EAAE;AAAA,EACnE;AACF;AA2BO,SAAS,oBACd,SACA,UACA,UACA,wBACA,wBAC8E;AAC9E,QAAM,SAAS,IAAI;AAEnB,SAAO,IAAI;AAAA,IACT;AAAA,MACE,MAAM,UAAU,OAAO,YAAY;AACjC,YAAI,EAAE,iBAAiB,aAAa;AAClC,qBAAW,QAAQ,KAAK;AACxB;AAAA,QACF;AAEA,cAAM,QAAQ,SAAS;AACvB,cAAM,yBAAyB,MAAM;AACrC,YAAI,2BAA2B,UAAa,CAAC,MAAM,qBAAsB;AAEzE,YAAI;AACF,gBAAM,OAAO,MAAM;AAAA,YACjB;AAAA,YACA,EAAE,WAAW,QAAQ,WAAW,WAAW,QAAQ,UAAU;AAAA,YAC7D;AAAA,cACE,SAAS,QAAQ;AAAA,cACjB,SAAS,QAAQ;AAAA,cACjB,YAAY,QAAQ;AAAA,cACpB,OAAO,MAAM,kBAAkB,QAAQ,QAAQ,QAAQ,SAAS;AAAA,YAClE;AAAA,UACF;AAEA,gBAAM,EAAE,WAAW,WAAW,eAAe,sBAAsB,IAAI;AACvE,gBAAM,QAAQ,MAAM,MAAM;AAAA,YACxB;AAAA,YACA,MAAM,IAAI,uBAAuB,EAAE,UAAU,CAAC;AAAA,YAC9C;AAAA,cACE;AAAA,cACA,gBAAgB;AAAA,cAChB,aAAa;AAAA,cACb,mBAAmB,YAAY,IAAI,IAAI,aAAa;AAAA,cACpD,oBAAoB,KAAK,IAAI,IAAI,0BAA0B;AAAA,cAC3D;AAAA,YACF;AAAA,UACF;AAEA,cAAI,MAAM,wBAAwB,MAAM,gBAAgB;AACtD,gBAAI,wBAAwB;AAC1B,qCAAuB,KAAK;AAAA,YAC9B;AACA,kBAAM,QAAgC;AAAA,cACpC,MAAM;AAAA,cACN,YAAY,KAAK,IAAI;AAAA,cACrB,kBAAkB;AAAA,cAClB,gBAAgB,MAAM;AAAA,cACtB,aAAa,MAAM;AAAA,cACnB,eAAe,MAAM;AAAA,cACrB,kBAAkB,MAAM;AAAA,cACxB,uBAAuB,MAAM;AAAA,cAC7B,mBAAmB,MAAM;AAAA,cACzB,aAAa,MAAM;AAAA,cACnB,cAAa,uEAA8B;AAAA,YAC7C;AACA,mBAAO;AAAA,cACL;AAAA,gBACE,mBAAmB,MAAM;AAAA,gBACzB,kBAAkB,MAAM;AAAA,cAC1B;AAAA,cACA;AAAA,YACF;AACA,qBAAS,EAAE,sBAAsB,MAAM,CAAC;AACxC,uBAAW,QAAQ,KAAK;AAAA,UAC1B;AAAA,QACF,SAAS,KAAK;AACZ,qBAAW,MAAM,GAAG;AAAA,QACtB;AAAA,MACF;AAAA,IACF;AAAA,IACA,EAAE,eAAe,EAAE;AAAA,IACnB,EAAE,eAAe,EAAE;AAAA,EACrB;AACF;","names":[]}
|