@livekit/agents 1.0.19 → 1.0.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/inference/stt.cjs +171 -125
- package/dist/inference/stt.cjs.map +1 -1
- package/dist/inference/stt.d.cts +4 -0
- package/dist/inference/stt.d.ts +4 -0
- package/dist/inference/stt.d.ts.map +1 -1
- package/dist/inference/stt.js +171 -125
- package/dist/inference/stt.js.map +1 -1
- package/dist/stt/stt.cjs +3 -0
- package/dist/stt/stt.cjs.map +1 -1
- package/dist/stt/stt.d.cts +1 -0
- package/dist/stt/stt.d.ts +1 -0
- package/dist/stt/stt.d.ts.map +1 -1
- package/dist/stt/stt.js +3 -0
- package/dist/stt/stt.js.map +1 -1
- package/dist/tts/tts.cjs +13 -1
- package/dist/tts/tts.cjs.map +1 -1
- package/dist/tts/tts.d.cts +1 -0
- package/dist/tts/tts.d.ts +1 -0
- package/dist/tts/tts.d.ts.map +1 -1
- package/dist/tts/tts.js +13 -1
- package/dist/tts/tts.js.map +1 -1
- package/dist/utils.cjs +15 -22
- package/dist/utils.cjs.map +1 -1
- package/dist/utils.d.cts +0 -1
- package/dist/utils.d.ts +0 -1
- package/dist/utils.d.ts.map +1 -1
- package/dist/utils.js +15 -21
- package/dist/utils.js.map +1 -1
- package/dist/utils.test.cjs +2 -1
- package/dist/utils.test.cjs.map +1 -1
- package/dist/utils.test.js +3 -10
- package/dist/utils.test.js.map +1 -1
- package/dist/vad.cjs +3 -0
- package/dist/vad.cjs.map +1 -1
- package/dist/vad.d.cts +1 -0
- package/dist/vad.d.ts +1 -0
- package/dist/vad.d.ts.map +1 -1
- package/dist/vad.js +3 -0
- package/dist/vad.js.map +1 -1
- package/dist/voice/agent_activity.cjs +3 -0
- package/dist/voice/agent_activity.cjs.map +1 -1
- package/dist/voice/agent_activity.d.ts.map +1 -1
- package/dist/voice/agent_activity.js +3 -0
- package/dist/voice/agent_activity.js.map +1 -1
- package/dist/voice/audio_recognition.cjs +1 -1
- package/dist/voice/audio_recognition.cjs.map +1 -1
- package/dist/voice/audio_recognition.js +1 -1
- package/dist/voice/audio_recognition.js.map +1 -1
- package/package.json +1 -1
- package/src/inference/stt.ts +201 -136
- package/src/stt/stt.ts +4 -0
- package/src/tts/tts.ts +14 -1
- package/src/utils.test.ts +3 -10
- package/src/utils.ts +21 -27
- package/src/vad.ts +4 -0
- package/src/voice/agent_activity.ts +3 -0
- package/src/voice/audio_recognition.ts +1 -1
package/dist/utils.test.js
CHANGED
|
@@ -2,15 +2,7 @@ import { AudioFrame } from "@livekit/rtc-node";
|
|
|
2
2
|
import { ReadableStream } from "node:stream/web";
|
|
3
3
|
import { describe, expect, it } from "vitest";
|
|
4
4
|
import { initializeLogger } from "../src/log.js";
|
|
5
|
-
import {
|
|
6
|
-
Event,
|
|
7
|
-
TASK_TIMEOUT_ERROR,
|
|
8
|
-
Task,
|
|
9
|
-
TaskResult,
|
|
10
|
-
delay,
|
|
11
|
-
isPending,
|
|
12
|
-
resampleStream
|
|
13
|
-
} from "../src/utils.js";
|
|
5
|
+
import { Event, Task, TaskResult, delay, isPending, resampleStream } from "../src/utils.js";
|
|
14
6
|
describe("utils", () => {
|
|
15
7
|
initializeLogger({ pretty: true, level: "debug" });
|
|
16
8
|
describe("Task", () => {
|
|
@@ -328,7 +320,8 @@ describe("utils", () => {
|
|
|
328
320
|
await task.cancelAndWait(200);
|
|
329
321
|
expect.fail("Task should have timed out");
|
|
330
322
|
} catch (error) {
|
|
331
|
-
expect(error).
|
|
323
|
+
expect(error).instanceof(Error);
|
|
324
|
+
expect(error.message).toBe("Task cancellation timed out");
|
|
332
325
|
}
|
|
333
326
|
});
|
|
334
327
|
it("should handle task that completes before timeout", async () => {
|
package/dist/utils.test.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/utils.test.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { AudioFrame } from '@livekit/rtc-node';\nimport { ReadableStream } from 'node:stream/web';\nimport { describe, expect, it } from 'vitest';\nimport { initializeLogger } from '../src/log.js';\nimport {\n Event,\n TASK_TIMEOUT_ERROR,\n Task,\n TaskResult,\n delay,\n isPending,\n resampleStream,\n} from '../src/utils.js';\n\ndescribe('utils', () => {\n // initialize logger\n initializeLogger({ pretty: true, level: 'debug' });\n\n describe('Task', () => {\n it('should execute task successfully and return result', async () => {\n const expectedResult = 'task completed';\n const task = Task.from(async () => {\n await delay(10);\n return expectedResult;\n });\n\n expect(task.done).toBe(false);\n const result = await task.result;\n expect(result).toBe(expectedResult);\n expect(task.done).toBe(true);\n });\n\n it('should handle task errors properly', async () => {\n const expectedError = new Error('Task failed');\n const task = Task.from(async () => {\n await delay(10);\n throw expectedError;\n });\n\n expect(task.done).toBe(false);\n await expect(task.result).rejects.toThrow(expectedError);\n expect(task.done).toBe(true);\n });\n\n it('should cancel task when cancel is called', async () => {\n let taskStarted = false;\n let taskCompleted = false;\n\n const task = Task.from(async (controller) => {\n taskStarted = true;\n await delay(100, { signal: controller.signal });\n taskCompleted = true;\n return 'should not complete';\n });\n\n // Wait a bit to ensure task starts\n await delay(10);\n expect(taskStarted).toBe(true);\n expect(task.done).toBe(false);\n\n // Cancel the task\n task.cancel();\n\n // The task should reject with AbortError\n try {\n await task.result;\n } catch (error: unknown) {\n expect((error as Error).name).toBe('AbortError');\n }\n\n expect(taskCompleted).toBe(false);\n expect(task.done).toBe(true);\n });\n\n it('should use provided AbortController', async () => {\n const controller = new AbortController();\n const task = Task.from(async (ctrl) => {\n expect(ctrl).toBe(controller);\n await delay(100, { signal: ctrl.signal });\n return 'completed';\n }, controller);\n\n await delay(10);\n controller.abort();\n\n try {\n await task.result;\n } catch (error: unknown) {\n expect((error as Error).name).toBe('AbortError');\n }\n\n expect(task.done).toBe(true);\n });\n\n it('should handle immediate resolution', async () => {\n const task = Task.from(async () => {\n return 'immediate';\n });\n\n const result = await task.result;\n expect(result).toBe('immediate');\n expect(task.done).toBe(true);\n });\n\n it('should handle immediate rejection', async () => {\n const expectedError = new Error('Immediate error');\n const task = Task.from(async () => {\n throw expectedError;\n });\n\n try {\n await task.result;\n } catch (error: unknown) {\n expect(error).toBe(expectedError);\n }\n\n expect(task.done).toBe(true);\n });\n\n it('should handle multiple calls to cancel', async () => {\n const task = Task.from(async (controller) => {\n await delay(100, { signal: controller.signal });\n return 'should not complete';\n });\n\n await delay(10);\n\n // Multiple cancellations should not cause issues\n task.cancel();\n task.cancel();\n task.cancel();\n\n try {\n await task.result;\n } catch (error: unknown) {\n expect((error as Error).name).toBe('AbortError');\n }\n\n expect(task.done).toBe(true);\n });\n\n it('should handle task that checks abort signal manually', async () => {\n const arr: number[] = [];\n const task = Task.from(async (controller) => {\n for (let i = 0; i < 10; i++) {\n if (controller.signal.aborted) {\n throw new Error('Task was aborted');\n }\n await delay(10);\n arr.push(i);\n }\n return 'completed';\n });\n\n await delay(39);\n task.cancel();\n\n expect(arr).toEqual([0, 1, 2]);\n try {\n await task.result;\n } catch (error: unknown) {\n expect((error as Error).message).toBe('Task was aborted');\n }\n\n expect(task.done).toBe(true);\n });\n\n it('should handle cleanup in finally block', async () => {\n let cleanupExecuted = false;\n\n const task = Task.from(async (controller) => {\n try {\n await delay(100, { signal: controller.signal });\n return 'completed';\n } finally {\n cleanupExecuted = true;\n }\n });\n\n await delay(10);\n task.cancel();\n\n try {\n await task.result;\n } catch {\n // Ignore the abort error\n }\n\n // Cleanup should still execute even when cancelled\n expect(cleanupExecuted).toBe(true);\n });\n\n it('should handle accessing result multiple times', async () => {\n const task = Task.from(async () => {\n await delay(10);\n return 'result';\n });\n\n const result1 = await task.result;\n const result2 = await task.result;\n const result3 = await task.result;\n\n expect(result1).toBe('result');\n expect(result2).toBe('result');\n expect(result3).toBe('result');\n expect(task.done).toBe(true);\n });\n\n it('should handle accessing result promise before completion', async () => {\n const task = Task.from(async () => {\n await delay(50);\n return 'delayed result';\n });\n\n // Get references to result promise before completion\n const resultPromise1 = task.result;\n const resultPromise2 = task.result;\n\n expect(task.done).toBe(false);\n\n // Both promises should resolve to the same value\n const [result1, result2] = await Promise.all([resultPromise1, resultPromise2]);\n\n expect(result1).toBe('delayed result');\n expect(result2).toBe('delayed result');\n expect(task.done).toBe(true);\n });\n\n it('should cancel child tasks when parent task is canceled', async () => {\n let parentStarted = false;\n let child1Started = false;\n let child2Started = false;\n let parentCompleted = false;\n let child1Completed = false;\n let child2Completed = false;\n\n let child1Task: Task<string> | undefined = undefined;\n let child2Task: Task<string> | undefined = undefined;\n\n const parentTask = Task.from(async (controller) => {\n parentStarted = true;\n\n // Create two child tasks using the parent's controller\n child1Task = Task.from(async (childController) => {\n child1Started = true;\n await delay(100, { signal: childController.signal });\n child1Completed = true;\n return 'child1';\n }, controller);\n\n child2Task = Task.from(async (childController) => {\n child2Started = true;\n await delay(100, { signal: childController.signal });\n child2Completed = true;\n return 'child2';\n }, controller);\n\n // Wait for both child tasks\n const results = await Promise.all([child1Task.result, child2Task.result]);\n parentCompleted = true;\n return results;\n });\n\n // Let tasks start\n await delay(20);\n\n // Verify tasks have started\n expect(parentStarted).toBe(true);\n expect(child1Started).toBe(true);\n expect(child2Started).toBe(true);\n\n // Cancel parent task\n parentTask.cancel();\n\n // Use Promise.allSettled to handle all promise settlements\n const [parentResult, child1Result, child2Result] = await Promise.allSettled([\n parentTask.result,\n child1Task!.result,\n child2Task!.result,\n ]);\n\n // Verify all tasks were rejected with AbortError\n expect(parentResult.status).toBe('rejected');\n expect((parentResult as PromiseRejectedResult).reason.name).toBe('AbortError');\n\n expect(child1Result.status).toBe('rejected');\n expect((child1Result as PromiseRejectedResult).reason.name).toBe('AbortError');\n\n expect(child2Result.status).toBe('rejected');\n expect((child2Result as PromiseRejectedResult).reason.name).toBe('AbortError');\n\n // Verify none of the tasks completed\n expect(parentCompleted).toBe(false);\n expect(child1Completed).toBe(false);\n expect(child2Completed).toBe(false);\n expect(parentTask.done).toBe(true);\n expect(child1Task!.done).toBe(true);\n expect(child2Task!.done).toBe(true);\n });\n\n it('should handle nested tasks that complete successfully', async () => {\n const results: string[] = [];\n\n const parentTask = Task.from(async (controller) => {\n results.push('parent-start');\n\n // Create first child task\n const child1Task = Task.from(async () => {\n results.push('child1-start');\n await delay(25);\n results.push('child1-end');\n return 'child1-result';\n }, controller);\n\n // Create second child task that depends on first\n const child2Task = Task.from(async (childController) => {\n results.push('child2-start');\n\n // Create a grandchild task\n const grandchildTask = Task.from(async () => {\n results.push('grandchild-start');\n await delay(10);\n results.push('grandchild-end');\n return 'grandchild-result';\n }, childController);\n\n const grandchildResult = await grandchildTask.result;\n await delay(10);\n results.push('child2-end');\n return `child2-result-with-${grandchildResult}`;\n }, controller);\n\n // Wait for all tasks\n const [child1Result, child2Result] = await Promise.all([\n child1Task.result,\n child2Task.result,\n ]);\n\n results.push('parent-end');\n return {\n parent: 'parent-result',\n child1: child1Result,\n child2: child2Result,\n };\n });\n\n // Wait for everything to complete\n const finalResult = await parentTask.result;\n\n // Verify results\n expect(finalResult).toEqual({\n parent: 'parent-result',\n child1: 'child1-result',\n child2: 'child2-result-with-grandchild-result',\n });\n\n // Verify execution order\n // Check important ordering constraints without being strict about parallel task ordering\n expect(results).toEqual([\n 'parent-start',\n 'child1-start',\n 'child2-start',\n 'grandchild-start',\n 'grandchild-end',\n 'child2-end',\n 'child1-end',\n 'parent-end',\n ]);\n\n // All tasks should be done\n expect(parentTask.done).toBe(true);\n });\n\n it('should propagate errors from nested tasks', async () => {\n let parentError: Error | null = null;\n let child1Completed = false;\n let child2Started = false;\n\n const parentTask = Task.from(async (controller) => {\n const child1Task = Task.from(async () => {\n await delay(20);\n throw new Error('child1 error');\n }, controller);\n\n const child2Task = Task.from(async () => {\n child2Started = true;\n await delay(30);\n child1Completed = true;\n return 'child2-result';\n }, controller);\n\n // This will throw when child1 fails\n const results = await Promise.all([child1Task.result, child2Task.result]);\n return results;\n });\n\n // Wait for the parent task to fail\n try {\n await parentTask.result;\n expect.fail('Parent task should have thrown');\n } catch (error: unknown) {\n parentError = error as Error;\n }\n\n // Verify the error propagated correctly\n expect(parentError?.message).toBe('child1 error');\n expect(child1Completed).toBe(false);\n expect(child2Started).toBe(true);\n expect(parentTask.done).toBe(true);\n });\n\n it('should cancel and wait for task completion', async () => {\n let taskCompleted = false;\n\n const task = Task.from(async (controller) => {\n await delay(5000, { signal: controller.signal });\n taskCompleted = true;\n return 'should not complete';\n });\n\n // Cancel and wait should complete quickly when task is aborted\n const start = Date.now();\n const result = await task.cancelAndWait(1000);\n const duration = Date.now() - start;\n\n expect(result).toBe(TaskResult.Aborted);\n expect(duration).toBeLessThan(100); // Should not wait for full timeout\n expect(taskCompleted).toBe(false);\n expect(task.done).toBe(true);\n });\n\n it('should timeout if task does not respond to cancellation', async () => {\n const task = Task.from(async () => {\n await delay(1000);\n });\n\n // This should timeout because the task ignores cancellation\n try {\n await task.cancelAndWait(200);\n expect.fail('Task should have timed out');\n } catch (error: unknown) {\n expect(error).toBe(TASK_TIMEOUT_ERROR);\n }\n });\n\n it('should handle task that completes before timeout', async () => {\n const task = Task.from(async () => {\n await delay(50);\n });\n\n // Start the task\n await delay(10);\n\n // Cancel and wait - but task will complete normally before being canceled\n const result = await task.cancelAndWait(1000);\n\n // Task should have completed normally\n expect(result).toBe(TaskResult.Completed);\n expect(task.done).toBe(true);\n });\n\n it('should propagate non-abort errors from cancelAndWait', async () => {\n const task = Task.from(async () => {\n await delay(10);\n throw new TypeError('Custom error');\n });\n\n try {\n await task.cancelAndWait(1000);\n expect.fail('Task should have thrown');\n } catch (error: unknown) {\n expect((error as Error).message).toBe('Custom error');\n expect((error as Error).name).toBe('TypeError');\n }\n });\n });\n\n describe('Event', () => {\n it('wait resolves immediately when the event is already set', async () => {\n const event = new Event();\n event.set();\n\n const result = await event.wait();\n expect(result).toBe(true);\n });\n\n it('wait resolves after set is called', async () => {\n // check promise is pending\n const event = new Event();\n const waiterPromise = event.wait();\n\n await delay(10);\n expect(await isPending(waiterPromise)).toBe(true);\n\n // check promise is resolved after set is called\n event.set();\n const result = await waiterPromise;\n expect(result).toBe(true);\n });\n\n it('all waiters resolve once set is called', async () => {\n const event = new Event();\n const waiters = [event.wait(), event.wait(), event.wait()];\n\n await delay(10);\n const pendings = await Promise.all(waiters.map((w) => isPending(w)));\n expect(pendings).toEqual([true, true, true]);\n\n event.set();\n const results = await Promise.all(waiters);\n expect(results).toEqual([true, true, true]);\n });\n\n it('wait after 2 seconds is still pending before set', async () => {\n const event = new Event();\n const waiter = event.wait();\n\n await delay(2000);\n expect(await isPending(waiter)).toBe(true);\n\n event.set();\n const result = await waiter;\n expect(result).toBe(true);\n });\n\n it('wait after set and clear should be pending', async () => {\n const event = new Event();\n const waiterBeforeSet = event.wait();\n event.set();\n event.clear();\n\n const waiterAfterSet = event.wait();\n\n const result = await Promise.race([\n waiterBeforeSet.then(() => 'before'),\n waiterAfterSet.then(() => 'after'),\n ]);\n\n expect(result).toBe('before');\n expect(await isPending(waiterBeforeSet)).toBe(false);\n expect(await isPending(waiterAfterSet)).toBe(true);\n\n event.set();\n expect(await waiterAfterSet).toBe(true);\n });\n });\n\n describe('resampleStream', () => {\n const createAudioFrame = (sampleRate: number, samples: number, channels = 1): AudioFrame => {\n const data = new Int16Array(samples * channels);\n for (let i = 0; i < data.length; i++) {\n data[i] = Math.sin((i / samples) * Math.PI * 2) * 16000;\n }\n return new AudioFrame(data, sampleRate, channels, samples);\n };\n\n const streamToArray = async (stream: ReadableStream<AudioFrame>): Promise<AudioFrame[]> => {\n const reader = stream.getReader();\n const chunks: AudioFrame[] = [];\n try {\n while (true) {\n const { done, value } = await reader.read();\n if (done) break;\n chunks.push(value);\n }\n } finally {\n reader.releaseLock();\n }\n return chunks;\n };\n\n it('should resample audio frames to target sample rate', async () => {\n const inputRate = 48000;\n const outputRate = 16000;\n const inputFrame = createAudioFrame(inputRate, 960); // 20ms at 48kHz\n\n const inputStream = new ReadableStream<AudioFrame>({\n start(controller) {\n controller.enqueue(inputFrame);\n controller.close();\n },\n });\n\n const outputStream = resampleStream({ stream: inputStream, outputRate });\n const outputFrames = await streamToArray(outputStream);\n\n expect(outputFrames.length).toBeGreaterThan(0);\n\n for (const frame of outputFrames) {\n expect(frame.sampleRate).toBe(outputRate);\n expect(frame.channels).toBe(inputFrame.channels);\n }\n });\n\n it('should handle same input and output rate', async () => {\n const sampleRate = 44100;\n const inputFrame = createAudioFrame(sampleRate, 1024);\n\n const inputStream = new ReadableStream<AudioFrame>({\n start(controller) {\n controller.enqueue(inputFrame);\n controller.close();\n },\n });\n\n const outputStream = resampleStream({ stream: inputStream, outputRate: sampleRate });\n const outputFrames = await streamToArray(outputStream);\n\n expect(outputFrames.length).toBeGreaterThan(0);\n\n for (const frame of outputFrames) {\n expect(frame.sampleRate).toBe(sampleRate);\n expect(frame.channels).toBe(inputFrame.channels);\n }\n });\n\n it('should handle multiple input frames', async () => {\n const inputRate = 32000;\n const outputRate = 48000;\n const frame1 = createAudioFrame(inputRate, 640);\n const frame2 = createAudioFrame(inputRate, 640);\n\n const inputStream = new ReadableStream<AudioFrame>({\n start(controller) {\n controller.enqueue(frame1);\n controller.enqueue(frame2);\n controller.close();\n },\n });\n\n const outputStream = resampleStream({ stream: inputStream, outputRate });\n const outputFrames = await streamToArray(outputStream);\n\n expect(outputFrames.length).toBeGreaterThan(0);\n\n for (const frame of outputFrames) {\n expect(frame.sampleRate).toBe(outputRate);\n expect(frame.channels).toBe(frame1.channels);\n }\n });\n\n it('should handle empty stream', async () => {\n const inputStream = new ReadableStream<AudioFrame>({\n start(controller) {\n controller.close();\n },\n });\n\n const outputStream = resampleStream({ stream: inputStream, outputRate: 44100 });\n const outputFrames = await streamToArray(outputStream);\n\n expect(outputFrames).toEqual([]);\n });\n });\n});\n"],"mappings":"AAGA,SAAS,kBAAkB;AAC3B,SAAS,sBAAsB;AAC/B,SAAS,UAAU,QAAQ,UAAU;AACrC,SAAS,wBAAwB;AACjC;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AAEP,SAAS,SAAS,MAAM;AAEtB,mBAAiB,EAAE,QAAQ,MAAM,OAAO,QAAQ,CAAC;AAEjD,WAAS,QAAQ,MAAM;AACrB,OAAG,sDAAsD,YAAY;AACnE,YAAM,iBAAiB;AACvB,YAAM,OAAO,KAAK,KAAK,YAAY;AACjC,cAAM,MAAM,EAAE;AACd,eAAO;AAAA,MACT,CAAC;AAED,aAAO,KAAK,IAAI,EAAE,KAAK,KAAK;AAC5B,YAAM,SAAS,MAAM,KAAK;AAC1B,aAAO,MAAM,EAAE,KAAK,cAAc;AAClC,aAAO,KAAK,IAAI,EAAE,KAAK,IAAI;AAAA,IAC7B,CAAC;AAED,OAAG,sCAAsC,YAAY;AACnD,YAAM,gBAAgB,IAAI,MAAM,aAAa;AAC7C,YAAM,OAAO,KAAK,KAAK,YAAY;AACjC,cAAM,MAAM,EAAE;AACd,cAAM;AAAA,MACR,CAAC;AAED,aAAO,KAAK,IAAI,EAAE,KAAK,KAAK;AAC5B,YAAM,OAAO,KAAK,MAAM,EAAE,QAAQ,QAAQ,aAAa;AACvD,aAAO,KAAK,IAAI,EAAE,KAAK,IAAI;AAAA,IAC7B,CAAC;AAED,OAAG,4CAA4C,YAAY;AACzD,UAAI,cAAc;AAClB,UAAI,gBAAgB;AAEpB,YAAM,OAAO,KAAK,KAAK,OAAO,eAAe;AAC3C,sBAAc;AACd,cAAM,MAAM,KAAK,EAAE,QAAQ,WAAW,OAAO,CAAC;AAC9C,wBAAgB;AAChB,eAAO;AAAA,MACT,CAAC;AAGD,YAAM,MAAM,EAAE;AACd,aAAO,WAAW,EAAE,KAAK,IAAI;AAC7B,aAAO,KAAK,IAAI,EAAE,KAAK,KAAK;AAG5B,WAAK,OAAO;AAGZ,UAAI;AACF,cAAM,KAAK;AAAA,MACb,SAAS,OAAgB;AACvB,eAAQ,MAAgB,IAAI,EAAE,KAAK,YAAY;AAAA,MACjD;AAEA,aAAO,aAAa,EAAE,KAAK,KAAK;AAChC,aAAO,KAAK,IAAI,EAAE,KAAK,IAAI;AAAA,IAC7B,CAAC;AAED,OAAG,uCAAuC,YAAY;AACpD,YAAM,aAAa,IAAI,gBAAgB;AACvC,YAAM,OAAO,KAAK,KAAK,OAAO,SAAS;AACrC,eAAO,IAAI,EAAE,KAAK,UAAU;AAC5B,cAAM,MAAM,KAAK,EAAE,QAAQ,KAAK,OAAO,CAAC;AACxC,eAAO;AAAA,MACT,GAAG,UAAU;AAEb,YAAM,MAAM,EAAE;AACd,iBAAW,MAAM;AAEjB,UAAI;AACF,cAAM,KAAK;AAAA,MACb,SAAS,OAAgB;AACvB,eAAQ,MAAgB,IAAI,EAAE,KAAK,YAAY;AAAA,MACjD;AAEA,aAAO,KAAK,IAAI,EAAE,KAAK,IAAI;AAAA,IAC7B,CAAC;AAED,OAAG,sCAAsC,YAAY;AACnD,YAAM,OAAO,KAAK,KAAK,YAAY;AACjC,eAAO;AAAA,MACT,CAAC;AAED,YAAM,SAAS,MAAM,KAAK;AAC1B,aAAO,MAAM,EAAE,KAAK,WAAW;AAC/B,aAAO,KAAK,IAAI,EAAE,KAAK,IAAI;AAAA,IAC7B,CAAC;AAED,OAAG,qCAAqC,YAAY;AAClD,YAAM,gBAAgB,IAAI,MAAM,iBAAiB;AACjD,YAAM,OAAO,KAAK,KAAK,YAAY;AACjC,cAAM;AAAA,MACR,CAAC;AAED,UAAI;AACF,cAAM,KAAK;AAAA,MACb,SAAS,OAAgB;AACvB,eAAO,KAAK,EAAE,KAAK,aAAa;AAAA,MAClC;AAEA,aAAO,KAAK,IAAI,EAAE,KAAK,IAAI;AAAA,IAC7B,CAAC;AAED,OAAG,0CAA0C,YAAY;AACvD,YAAM,OAAO,KAAK,KAAK,OAAO,eAAe;AAC3C,cAAM,MAAM,KAAK,EAAE,QAAQ,WAAW,OAAO,CAAC;AAC9C,eAAO;AAAA,MACT,CAAC;AAED,YAAM,MAAM,EAAE;AAGd,WAAK,OAAO;AACZ,WAAK,OAAO;AACZ,WAAK,OAAO;AAEZ,UAAI;AACF,cAAM,KAAK;AAAA,MACb,SAAS,OAAgB;AACvB,eAAQ,MAAgB,IAAI,EAAE,KAAK,YAAY;AAAA,MACjD;AAEA,aAAO,KAAK,IAAI,EAAE,KAAK,IAAI;AAAA,IAC7B,CAAC;AAED,OAAG,wDAAwD,YAAY;AACrE,YAAM,MAAgB,CAAC;AACvB,YAAM,OAAO,KAAK,KAAK,OAAO,eAAe;AAC3C,iBAAS,IAAI,GAAG,IAAI,IAAI,KAAK;AAC3B,cAAI,WAAW,OAAO,SAAS;AAC7B,kBAAM,IAAI,MAAM,kBAAkB;AAAA,UACpC;AACA,gBAAM,MAAM,EAAE;AACd,cAAI,KAAK,CAAC;AAAA,QACZ;AACA,eAAO;AAAA,MACT,CAAC;AAED,YAAM,MAAM,EAAE;AACd,WAAK,OAAO;AAEZ,aAAO,GAAG,EAAE,QAAQ,CAAC,GAAG,GAAG,CAAC,CAAC;AAC7B,UAAI;AACF,cAAM,KAAK;AAAA,MACb,SAAS,OAAgB;AACvB,eAAQ,MAAgB,OAAO,EAAE,KAAK,kBAAkB;AAAA,MAC1D;AAEA,aAAO,KAAK,IAAI,EAAE,KAAK,IAAI;AAAA,IAC7B,CAAC;AAED,OAAG,0CAA0C,YAAY;AACvD,UAAI,kBAAkB;AAEtB,YAAM,OAAO,KAAK,KAAK,OAAO,eAAe;AAC3C,YAAI;AACF,gBAAM,MAAM,KAAK,EAAE,QAAQ,WAAW,OAAO,CAAC;AAC9C,iBAAO;AAAA,QACT,UAAE;AACA,4BAAkB;AAAA,QACpB;AAAA,MACF,CAAC;AAED,YAAM,MAAM,EAAE;AACd,WAAK,OAAO;AAEZ,UAAI;AACF,cAAM,KAAK;AAAA,MACb,QAAQ;AAAA,MAER;AAGA,aAAO,eAAe,EAAE,KAAK,IAAI;AAAA,IACnC,CAAC;AAED,OAAG,iDAAiD,YAAY;AAC9D,YAAM,OAAO,KAAK,KAAK,YAAY;AACjC,cAAM,MAAM,EAAE;AACd,eAAO;AAAA,MACT,CAAC;AAED,YAAM,UAAU,MAAM,KAAK;AAC3B,YAAM,UAAU,MAAM,KAAK;AAC3B,YAAM,UAAU,MAAM,KAAK;AAE3B,aAAO,OAAO,EAAE,KAAK,QAAQ;AAC7B,aAAO,OAAO,EAAE,KAAK,QAAQ;AAC7B,aAAO,OAAO,EAAE,KAAK,QAAQ;AAC7B,aAAO,KAAK,IAAI,EAAE,KAAK,IAAI;AAAA,IAC7B,CAAC;AAED,OAAG,4DAA4D,YAAY;AACzE,YAAM,OAAO,KAAK,KAAK,YAAY;AACjC,cAAM,MAAM,EAAE;AACd,eAAO;AAAA,MACT,CAAC;AAGD,YAAM,iBAAiB,KAAK;AAC5B,YAAM,iBAAiB,KAAK;AAE5B,aAAO,KAAK,IAAI,EAAE,KAAK,KAAK;AAG5B,YAAM,CAAC,SAAS,OAAO,IAAI,MAAM,QAAQ,IAAI,CAAC,gBAAgB,cAAc,CAAC;AAE7E,aAAO,OAAO,EAAE,KAAK,gBAAgB;AACrC,aAAO,OAAO,EAAE,KAAK,gBAAgB;AACrC,aAAO,KAAK,IAAI,EAAE,KAAK,IAAI;AAAA,IAC7B,CAAC;AAED,OAAG,0DAA0D,YAAY;AACvE,UAAI,gBAAgB;AACpB,UAAI,gBAAgB;AACpB,UAAI,gBAAgB;AACpB,UAAI,kBAAkB;AACtB,UAAI,kBAAkB;AACtB,UAAI,kBAAkB;AAEtB,UAAI,aAAuC;AAC3C,UAAI,aAAuC;AAE3C,YAAM,aAAa,KAAK,KAAK,OAAO,eAAe;AACjD,wBAAgB;AAGhB,qBAAa,KAAK,KAAK,OAAO,oBAAoB;AAChD,0BAAgB;AAChB,gBAAM,MAAM,KAAK,EAAE,QAAQ,gBAAgB,OAAO,CAAC;AACnD,4BAAkB;AAClB,iBAAO;AAAA,QACT,GAAG,UAAU;AAEb,qBAAa,KAAK,KAAK,OAAO,oBAAoB;AAChD,0BAAgB;AAChB,gBAAM,MAAM,KAAK,EAAE,QAAQ,gBAAgB,OAAO,CAAC;AACnD,4BAAkB;AAClB,iBAAO;AAAA,QACT,GAAG,UAAU;AAGb,cAAM,UAAU,MAAM,QAAQ,IAAI,CAAC,WAAW,QAAQ,WAAW,MAAM,CAAC;AACxE,0BAAkB;AAClB,eAAO;AAAA,MACT,CAAC;AAGD,YAAM,MAAM,EAAE;AAGd,aAAO,aAAa,EAAE,KAAK,IAAI;AAC/B,aAAO,aAAa,EAAE,KAAK,IAAI;AAC/B,aAAO,aAAa,EAAE,KAAK,IAAI;AAG/B,iBAAW,OAAO;AAGlB,YAAM,CAAC,cAAc,cAAc,YAAY,IAAI,MAAM,QAAQ,WAAW;AAAA,QAC1E,WAAW;AAAA,QACX,WAAY;AAAA,QACZ,WAAY;AAAA,MACd,CAAC;AAGD,aAAO,aAAa,MAAM,EAAE,KAAK,UAAU;AAC3C,aAAQ,aAAuC,OAAO,IAAI,EAAE,KAAK,YAAY;AAE7E,aAAO,aAAa,MAAM,EAAE,KAAK,UAAU;AAC3C,aAAQ,aAAuC,OAAO,IAAI,EAAE,KAAK,YAAY;AAE7E,aAAO,aAAa,MAAM,EAAE,KAAK,UAAU;AAC3C,aAAQ,aAAuC,OAAO,IAAI,EAAE,KAAK,YAAY;AAG7E,aAAO,eAAe,EAAE,KAAK,KAAK;AAClC,aAAO,eAAe,EAAE,KAAK,KAAK;AAClC,aAAO,eAAe,EAAE,KAAK,KAAK;AAClC,aAAO,WAAW,IAAI,EAAE,KAAK,IAAI;AACjC,aAAO,WAAY,IAAI,EAAE,KAAK,IAAI;AAClC,aAAO,WAAY,IAAI,EAAE,KAAK,IAAI;AAAA,IACpC,CAAC;AAED,OAAG,yDAAyD,YAAY;AACtE,YAAM,UAAoB,CAAC;AAE3B,YAAM,aAAa,KAAK,KAAK,OAAO,eAAe;AACjD,gBAAQ,KAAK,cAAc;AAG3B,cAAM,aAAa,KAAK,KAAK,YAAY;AACvC,kBAAQ,KAAK,cAAc;AAC3B,gBAAM,MAAM,EAAE;AACd,kBAAQ,KAAK,YAAY;AACzB,iBAAO;AAAA,QACT,GAAG,UAAU;AAGb,cAAM,aAAa,KAAK,KAAK,OAAO,oBAAoB;AACtD,kBAAQ,KAAK,cAAc;AAG3B,gBAAM,iBAAiB,KAAK,KAAK,YAAY;AAC3C,oBAAQ,KAAK,kBAAkB;AAC/B,kBAAM,MAAM,EAAE;AACd,oBAAQ,KAAK,gBAAgB;AAC7B,mBAAO;AAAA,UACT,GAAG,eAAe;AAElB,gBAAM,mBAAmB,MAAM,eAAe;AAC9C,gBAAM,MAAM,EAAE;AACd,kBAAQ,KAAK,YAAY;AACzB,iBAAO,sBAAsB,gBAAgB;AAAA,QAC/C,GAAG,UAAU;AAGb,cAAM,CAAC,cAAc,YAAY,IAAI,MAAM,QAAQ,IAAI;AAAA,UACrD,WAAW;AAAA,UACX,WAAW;AAAA,QACb,CAAC;AAED,gBAAQ,KAAK,YAAY;AACzB,eAAO;AAAA,UACL,QAAQ;AAAA,UACR,QAAQ;AAAA,UACR,QAAQ;AAAA,QACV;AAAA,MACF,CAAC;AAGD,YAAM,cAAc,MAAM,WAAW;AAGrC,aAAO,WAAW,EAAE,QAAQ;AAAA,QAC1B,QAAQ;AAAA,QACR,QAAQ;AAAA,QACR,QAAQ;AAAA,MACV,CAAC;AAID,aAAO,OAAO,EAAE,QAAQ;AAAA,QACtB;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF,CAAC;AAGD,aAAO,WAAW,IAAI,EAAE,KAAK,IAAI;AAAA,IACnC,CAAC;AAED,OAAG,6CAA6C,YAAY;AAC1D,UAAI,cAA4B;AAChC,UAAI,kBAAkB;AACtB,UAAI,gBAAgB;AAEpB,YAAM,aAAa,KAAK,KAAK,OAAO,eAAe;AACjD,cAAM,aAAa,KAAK,KAAK,YAAY;AACvC,gBAAM,MAAM,EAAE;AACd,gBAAM,IAAI,MAAM,cAAc;AAAA,QAChC,GAAG,UAAU;AAEb,cAAM,aAAa,KAAK,KAAK,YAAY;AACvC,0BAAgB;AAChB,gBAAM,MAAM,EAAE;AACd,4BAAkB;AAClB,iBAAO;AAAA,QACT,GAAG,UAAU;AAGb,cAAM,UAAU,MAAM,QAAQ,IAAI,CAAC,WAAW,QAAQ,WAAW,MAAM,CAAC;AACxE,eAAO;AAAA,MACT,CAAC;AAGD,UAAI;AACF,cAAM,WAAW;AACjB,eAAO,KAAK,gCAAgC;AAAA,MAC9C,SAAS,OAAgB;AACvB,sBAAc;AAAA,MAChB;AAGA,aAAO,2CAAa,OAAO,EAAE,KAAK,cAAc;AAChD,aAAO,eAAe,EAAE,KAAK,KAAK;AAClC,aAAO,aAAa,EAAE,KAAK,IAAI;AAC/B,aAAO,WAAW,IAAI,EAAE,KAAK,IAAI;AAAA,IACnC,CAAC;AAED,OAAG,8CAA8C,YAAY;AAC3D,UAAI,gBAAgB;AAEpB,YAAM,OAAO,KAAK,KAAK,OAAO,eAAe;AAC3C,cAAM,MAAM,KAAM,EAAE,QAAQ,WAAW,OAAO,CAAC;AAC/C,wBAAgB;AAChB,eAAO;AAAA,MACT,CAAC;AAGD,YAAM,QAAQ,KAAK,IAAI;AACvB,YAAM,SAAS,MAAM,KAAK,cAAc,GAAI;AAC5C,YAAM,WAAW,KAAK,IAAI,IAAI;AAE9B,aAAO,MAAM,EAAE,KAAK,WAAW,OAAO;AACtC,aAAO,QAAQ,EAAE,aAAa,GAAG;AACjC,aAAO,aAAa,EAAE,KAAK,KAAK;AAChC,aAAO,KAAK,IAAI,EAAE,KAAK,IAAI;AAAA,IAC7B,CAAC;AAED,OAAG,2DAA2D,YAAY;AACxE,YAAM,OAAO,KAAK,KAAK,YAAY;AACjC,cAAM,MAAM,GAAI;AAAA,MAClB,CAAC;AAGD,UAAI;AACF,cAAM,KAAK,cAAc,GAAG;AAC5B,eAAO,KAAK,4BAA4B;AAAA,MAC1C,SAAS,OAAgB;AACvB,eAAO,KAAK,EAAE,KAAK,kBAAkB;AAAA,MACvC;AAAA,IACF,CAAC;AAED,OAAG,oDAAoD,YAAY;AACjE,YAAM,OAAO,KAAK,KAAK,YAAY;AACjC,cAAM,MAAM,EAAE;AAAA,MAChB,CAAC;AAGD,YAAM,MAAM,EAAE;AAGd,YAAM,SAAS,MAAM,KAAK,cAAc,GAAI;AAG5C,aAAO,MAAM,EAAE,KAAK,WAAW,SAAS;AACxC,aAAO,KAAK,IAAI,EAAE,KAAK,IAAI;AAAA,IAC7B,CAAC;AAED,OAAG,wDAAwD,YAAY;AACrE,YAAM,OAAO,KAAK,KAAK,YAAY;AACjC,cAAM,MAAM,EAAE;AACd,cAAM,IAAI,UAAU,cAAc;AAAA,MACpC,CAAC;AAED,UAAI;AACF,cAAM,KAAK,cAAc,GAAI;AAC7B,eAAO,KAAK,yBAAyB;AAAA,MACvC,SAAS,OAAgB;AACvB,eAAQ,MAAgB,OAAO,EAAE,KAAK,cAAc;AACpD,eAAQ,MAAgB,IAAI,EAAE,KAAK,WAAW;AAAA,MAChD;AAAA,IACF,CAAC;AAAA,EACH,CAAC;AAED,WAAS,SAAS,MAAM;AACtB,OAAG,2DAA2D,YAAY;AACxE,YAAM,QAAQ,IAAI,MAAM;AACxB,YAAM,IAAI;AAEV,YAAM,SAAS,MAAM,MAAM,KAAK;AAChC,aAAO,MAAM,EAAE,KAAK,IAAI;AAAA,IAC1B,CAAC;AAED,OAAG,qCAAqC,YAAY;AAElD,YAAM,QAAQ,IAAI,MAAM;AACxB,YAAM,gBAAgB,MAAM,KAAK;AAEjC,YAAM,MAAM,EAAE;AACd,aAAO,MAAM,UAAU,aAAa,CAAC,EAAE,KAAK,IAAI;AAGhD,YAAM,IAAI;AACV,YAAM,SAAS,MAAM;AACrB,aAAO,MAAM,EAAE,KAAK,IAAI;AAAA,IAC1B,CAAC;AAED,OAAG,0CAA0C,YAAY;AACvD,YAAM,QAAQ,IAAI,MAAM;AACxB,YAAM,UAAU,CAAC,MAAM,KAAK,GAAG,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC;AAEzD,YAAM,MAAM,EAAE;AACd,YAAM,WAAW,MAAM,QAAQ,IAAI,QAAQ,IAAI,CAAC,MAAM,UAAU,CAAC,CAAC,CAAC;AACnE,aAAO,QAAQ,EAAE,QAAQ,CAAC,MAAM,MAAM,IAAI,CAAC;AAE3C,YAAM,IAAI;AACV,YAAM,UAAU,MAAM,QAAQ,IAAI,OAAO;AACzC,aAAO,OAAO,EAAE,QAAQ,CAAC,MAAM,MAAM,IAAI,CAAC;AAAA,IAC5C,CAAC;AAED,OAAG,oDAAoD,YAAY;AACjE,YAAM,QAAQ,IAAI,MAAM;AACxB,YAAM,SAAS,MAAM,KAAK;AAE1B,YAAM,MAAM,GAAI;AAChB,aAAO,MAAM,UAAU,MAAM,CAAC,EAAE,KAAK,IAAI;AAEzC,YAAM,IAAI;AACV,YAAM,SAAS,MAAM;AACrB,aAAO,MAAM,EAAE,KAAK,IAAI;AAAA,IAC1B,CAAC;AAED,OAAG,8CAA8C,YAAY;AAC3D,YAAM,QAAQ,IAAI,MAAM;AACxB,YAAM,kBAAkB,MAAM,KAAK;AACnC,YAAM,IAAI;AACV,YAAM,MAAM;AAEZ,YAAM,iBAAiB,MAAM,KAAK;AAElC,YAAM,SAAS,MAAM,QAAQ,KAAK;AAAA,QAChC,gBAAgB,KAAK,MAAM,QAAQ;AAAA,QACnC,eAAe,KAAK,MAAM,OAAO;AAAA,MACnC,CAAC;AAED,aAAO,MAAM,EAAE,KAAK,QAAQ;AAC5B,aAAO,MAAM,UAAU,eAAe,CAAC,EAAE,KAAK,KAAK;AACnD,aAAO,MAAM,UAAU,cAAc,CAAC,EAAE,KAAK,IAAI;AAEjD,YAAM,IAAI;AACV,aAAO,MAAM,cAAc,EAAE,KAAK,IAAI;AAAA,IACxC,CAAC;AAAA,EACH,CAAC;AAED,WAAS,kBAAkB,MAAM;AAC/B,UAAM,mBAAmB,CAAC,YAAoB,SAAiB,WAAW,MAAkB;AAC1F,YAAM,OAAO,IAAI,WAAW,UAAU,QAAQ;AAC9C,eAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,aAAK,CAAC,IAAI,KAAK,IAAK,IAAI,UAAW,KAAK,KAAK,CAAC,IAAI;AAAA,MACpD;AACA,aAAO,IAAI,WAAW,MAAM,YAAY,UAAU,OAAO;AAAA,IAC3D;AAEA,UAAM,gBAAgB,OAAO,WAA8D;AACzF,YAAM,SAAS,OAAO,UAAU;AAChC,YAAM,SAAuB,CAAC;AAC9B,UAAI;AACF,eAAO,MAAM;AACX,gBAAM,EAAE,MAAM,MAAM,IAAI,MAAM,OAAO,KAAK;AAC1C,cAAI,KAAM;AACV,iBAAO,KAAK,KAAK;AAAA,QACnB;AAAA,MACF,UAAE;AACA,eAAO,YAAY;AAAA,MACrB;AACA,aAAO;AAAA,IACT;AAEA,OAAG,sDAAsD,YAAY;AACnE,YAAM,YAAY;AAClB,YAAM,aAAa;AACnB,YAAM,aAAa,iBAAiB,WAAW,GAAG;AAElD,YAAM,cAAc,IAAI,eAA2B;AAAA,QACjD,MAAM,YAAY;AAChB,qBAAW,QAAQ,UAAU;AAC7B,qBAAW,MAAM;AAAA,QACnB;AAAA,MACF,CAAC;AAED,YAAM,eAAe,eAAe,EAAE,QAAQ,aAAa,WAAW,CAAC;AACvE,YAAM,eAAe,MAAM,cAAc,YAAY;AAErD,aAAO,aAAa,MAAM,EAAE,gBAAgB,CAAC;AAE7C,iBAAW,SAAS,cAAc;AAChC,eAAO,MAAM,UAAU,EAAE,KAAK,UAAU;AACxC,eAAO,MAAM,QAAQ,EAAE,KAAK,WAAW,QAAQ;AAAA,MACjD;AAAA,IACF,CAAC;AAED,OAAG,4CAA4C,YAAY;AACzD,YAAM,aAAa;AACnB,YAAM,aAAa,iBAAiB,YAAY,IAAI;AAEpD,YAAM,cAAc,IAAI,eAA2B;AAAA,QACjD,MAAM,YAAY;AAChB,qBAAW,QAAQ,UAAU;AAC7B,qBAAW,MAAM;AAAA,QACnB;AAAA,MACF,CAAC;AAED,YAAM,eAAe,eAAe,EAAE,QAAQ,aAAa,YAAY,WAAW,CAAC;AACnF,YAAM,eAAe,MAAM,cAAc,YAAY;AAErD,aAAO,aAAa,MAAM,EAAE,gBAAgB,CAAC;AAE7C,iBAAW,SAAS,cAAc;AAChC,eAAO,MAAM,UAAU,EAAE,KAAK,UAAU;AACxC,eAAO,MAAM,QAAQ,EAAE,KAAK,WAAW,QAAQ;AAAA,MACjD;AAAA,IACF,CAAC;AAED,OAAG,uCAAuC,YAAY;AACpD,YAAM,YAAY;AAClB,YAAM,aAAa;AACnB,YAAM,SAAS,iBAAiB,WAAW,GAAG;AAC9C,YAAM,SAAS,iBAAiB,WAAW,GAAG;AAE9C,YAAM,cAAc,IAAI,eAA2B;AAAA,QACjD,MAAM,YAAY;AAChB,qBAAW,QAAQ,MAAM;AACzB,qBAAW,QAAQ,MAAM;AACzB,qBAAW,MAAM;AAAA,QACnB;AAAA,MACF,CAAC;AAED,YAAM,eAAe,eAAe,EAAE,QAAQ,aAAa,WAAW,CAAC;AACvE,YAAM,eAAe,MAAM,cAAc,YAAY;AAErD,aAAO,aAAa,MAAM,EAAE,gBAAgB,CAAC;AAE7C,iBAAW,SAAS,cAAc;AAChC,eAAO,MAAM,UAAU,EAAE,KAAK,UAAU;AACxC,eAAO,MAAM,QAAQ,EAAE,KAAK,OAAO,QAAQ;AAAA,MAC7C;AAAA,IACF,CAAC;AAED,OAAG,8BAA8B,YAAY;AAC3C,YAAM,cAAc,IAAI,eAA2B;AAAA,QACjD,MAAM,YAAY;AAChB,qBAAW,MAAM;AAAA,QACnB;AAAA,MACF,CAAC;AAED,YAAM,eAAe,eAAe,EAAE,QAAQ,aAAa,YAAY,MAAM,CAAC;AAC9E,YAAM,eAAe,MAAM,cAAc,YAAY;AAErD,aAAO,YAAY,EAAE,QAAQ,CAAC,CAAC;AAAA,IACjC,CAAC;AAAA,EACH,CAAC;AACH,CAAC;","names":[]}
|
|
1
|
+
{"version":3,"sources":["../src/utils.test.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { AudioFrame } from '@livekit/rtc-node';\nimport { ReadableStream } from 'node:stream/web';\nimport { describe, expect, it } from 'vitest';\nimport { initializeLogger } from '../src/log.js';\nimport { Event, Task, TaskResult, delay, isPending, resampleStream } from '../src/utils.js';\n\ndescribe('utils', () => {\n // initialize logger\n initializeLogger({ pretty: true, level: 'debug' });\n\n describe('Task', () => {\n it('should execute task successfully and return result', async () => {\n const expectedResult = 'task completed';\n const task = Task.from(async () => {\n await delay(10);\n return expectedResult;\n });\n\n expect(task.done).toBe(false);\n const result = await task.result;\n expect(result).toBe(expectedResult);\n expect(task.done).toBe(true);\n });\n\n it('should handle task errors properly', async () => {\n const expectedError = new Error('Task failed');\n const task = Task.from(async () => {\n await delay(10);\n throw expectedError;\n });\n\n expect(task.done).toBe(false);\n await expect(task.result).rejects.toThrow(expectedError);\n expect(task.done).toBe(true);\n });\n\n it('should cancel task when cancel is called', async () => {\n let taskStarted = false;\n let taskCompleted = false;\n\n const task = Task.from(async (controller) => {\n taskStarted = true;\n await delay(100, { signal: controller.signal });\n taskCompleted = true;\n return 'should not complete';\n });\n\n // Wait a bit to ensure task starts\n await delay(10);\n expect(taskStarted).toBe(true);\n expect(task.done).toBe(false);\n\n // Cancel the task\n task.cancel();\n\n // The task should reject with AbortError\n try {\n await task.result;\n } catch (error: unknown) {\n expect((error as Error).name).toBe('AbortError');\n }\n\n expect(taskCompleted).toBe(false);\n expect(task.done).toBe(true);\n });\n\n it('should use provided AbortController', async () => {\n const controller = new AbortController();\n const task = Task.from(async (ctrl) => {\n expect(ctrl).toBe(controller);\n await delay(100, { signal: ctrl.signal });\n return 'completed';\n }, controller);\n\n await delay(10);\n controller.abort();\n\n try {\n await task.result;\n } catch (error: unknown) {\n expect((error as Error).name).toBe('AbortError');\n }\n\n expect(task.done).toBe(true);\n });\n\n it('should handle immediate resolution', async () => {\n const task = Task.from(async () => {\n return 'immediate';\n });\n\n const result = await task.result;\n expect(result).toBe('immediate');\n expect(task.done).toBe(true);\n });\n\n it('should handle immediate rejection', async () => {\n const expectedError = new Error('Immediate error');\n const task = Task.from(async () => {\n throw expectedError;\n });\n\n try {\n await task.result;\n } catch (error: unknown) {\n expect(error).toBe(expectedError);\n }\n\n expect(task.done).toBe(true);\n });\n\n it('should handle multiple calls to cancel', async () => {\n const task = Task.from(async (controller) => {\n await delay(100, { signal: controller.signal });\n return 'should not complete';\n });\n\n await delay(10);\n\n // Multiple cancellations should not cause issues\n task.cancel();\n task.cancel();\n task.cancel();\n\n try {\n await task.result;\n } catch (error: unknown) {\n expect((error as Error).name).toBe('AbortError');\n }\n\n expect(task.done).toBe(true);\n });\n\n it('should handle task that checks abort signal manually', async () => {\n const arr: number[] = [];\n const task = Task.from(async (controller) => {\n for (let i = 0; i < 10; i++) {\n if (controller.signal.aborted) {\n throw new Error('Task was aborted');\n }\n await delay(10);\n arr.push(i);\n }\n return 'completed';\n });\n\n await delay(39);\n task.cancel();\n\n expect(arr).toEqual([0, 1, 2]);\n try {\n await task.result;\n } catch (error: unknown) {\n expect((error as Error).message).toBe('Task was aborted');\n }\n\n expect(task.done).toBe(true);\n });\n\n it('should handle cleanup in finally block', async () => {\n let cleanupExecuted = false;\n\n const task = Task.from(async (controller) => {\n try {\n await delay(100, { signal: controller.signal });\n return 'completed';\n } finally {\n cleanupExecuted = true;\n }\n });\n\n await delay(10);\n task.cancel();\n\n try {\n await task.result;\n } catch {\n // Ignore the abort error\n }\n\n // Cleanup should still execute even when cancelled\n expect(cleanupExecuted).toBe(true);\n });\n\n it('should handle accessing result multiple times', async () => {\n const task = Task.from(async () => {\n await delay(10);\n return 'result';\n });\n\n const result1 = await task.result;\n const result2 = await task.result;\n const result3 = await task.result;\n\n expect(result1).toBe('result');\n expect(result2).toBe('result');\n expect(result3).toBe('result');\n expect(task.done).toBe(true);\n });\n\n it('should handle accessing result promise before completion', async () => {\n const task = Task.from(async () => {\n await delay(50);\n return 'delayed result';\n });\n\n // Get references to result promise before completion\n const resultPromise1 = task.result;\n const resultPromise2 = task.result;\n\n expect(task.done).toBe(false);\n\n // Both promises should resolve to the same value\n const [result1, result2] = await Promise.all([resultPromise1, resultPromise2]);\n\n expect(result1).toBe('delayed result');\n expect(result2).toBe('delayed result');\n expect(task.done).toBe(true);\n });\n\n it('should cancel child tasks when parent task is canceled', async () => {\n let parentStarted = false;\n let child1Started = false;\n let child2Started = false;\n let parentCompleted = false;\n let child1Completed = false;\n let child2Completed = false;\n\n let child1Task: Task<string> | undefined = undefined;\n let child2Task: Task<string> | undefined = undefined;\n\n const parentTask = Task.from(async (controller) => {\n parentStarted = true;\n\n // Create two child tasks using the parent's controller\n child1Task = Task.from(async (childController) => {\n child1Started = true;\n await delay(100, { signal: childController.signal });\n child1Completed = true;\n return 'child1';\n }, controller);\n\n child2Task = Task.from(async (childController) => {\n child2Started = true;\n await delay(100, { signal: childController.signal });\n child2Completed = true;\n return 'child2';\n }, controller);\n\n // Wait for both child tasks\n const results = await Promise.all([child1Task.result, child2Task.result]);\n parentCompleted = true;\n return results;\n });\n\n // Let tasks start\n await delay(20);\n\n // Verify tasks have started\n expect(parentStarted).toBe(true);\n expect(child1Started).toBe(true);\n expect(child2Started).toBe(true);\n\n // Cancel parent task\n parentTask.cancel();\n\n // Use Promise.allSettled to handle all promise settlements\n const [parentResult, child1Result, child2Result] = await Promise.allSettled([\n parentTask.result,\n child1Task!.result,\n child2Task!.result,\n ]);\n\n // Verify all tasks were rejected with AbortError\n expect(parentResult.status).toBe('rejected');\n expect((parentResult as PromiseRejectedResult).reason.name).toBe('AbortError');\n\n expect(child1Result.status).toBe('rejected');\n expect((child1Result as PromiseRejectedResult).reason.name).toBe('AbortError');\n\n expect(child2Result.status).toBe('rejected');\n expect((child2Result as PromiseRejectedResult).reason.name).toBe('AbortError');\n\n // Verify none of the tasks completed\n expect(parentCompleted).toBe(false);\n expect(child1Completed).toBe(false);\n expect(child2Completed).toBe(false);\n expect(parentTask.done).toBe(true);\n expect(child1Task!.done).toBe(true);\n expect(child2Task!.done).toBe(true);\n });\n\n it('should handle nested tasks that complete successfully', async () => {\n const results: string[] = [];\n\n const parentTask = Task.from(async (controller) => {\n results.push('parent-start');\n\n // Create first child task\n const child1Task = Task.from(async () => {\n results.push('child1-start');\n await delay(25);\n results.push('child1-end');\n return 'child1-result';\n }, controller);\n\n // Create second child task that depends on first\n const child2Task = Task.from(async (childController) => {\n results.push('child2-start');\n\n // Create a grandchild task\n const grandchildTask = Task.from(async () => {\n results.push('grandchild-start');\n await delay(10);\n results.push('grandchild-end');\n return 'grandchild-result';\n }, childController);\n\n const grandchildResult = await grandchildTask.result;\n await delay(10);\n results.push('child2-end');\n return `child2-result-with-${grandchildResult}`;\n }, controller);\n\n // Wait for all tasks\n const [child1Result, child2Result] = await Promise.all([\n child1Task.result,\n child2Task.result,\n ]);\n\n results.push('parent-end');\n return {\n parent: 'parent-result',\n child1: child1Result,\n child2: child2Result,\n };\n });\n\n // Wait for everything to complete\n const finalResult = await parentTask.result;\n\n // Verify results\n expect(finalResult).toEqual({\n parent: 'parent-result',\n child1: 'child1-result',\n child2: 'child2-result-with-grandchild-result',\n });\n\n // Verify execution order\n // Check important ordering constraints without being strict about parallel task ordering\n expect(results).toEqual([\n 'parent-start',\n 'child1-start',\n 'child2-start',\n 'grandchild-start',\n 'grandchild-end',\n 'child2-end',\n 'child1-end',\n 'parent-end',\n ]);\n\n // All tasks should be done\n expect(parentTask.done).toBe(true);\n });\n\n it('should propagate errors from nested tasks', async () => {\n let parentError: Error | null = null;\n let child1Completed = false;\n let child2Started = false;\n\n const parentTask = Task.from(async (controller) => {\n const child1Task = Task.from(async () => {\n await delay(20);\n throw new Error('child1 error');\n }, controller);\n\n const child2Task = Task.from(async () => {\n child2Started = true;\n await delay(30);\n child1Completed = true;\n return 'child2-result';\n }, controller);\n\n // This will throw when child1 fails\n const results = await Promise.all([child1Task.result, child2Task.result]);\n return results;\n });\n\n // Wait for the parent task to fail\n try {\n await parentTask.result;\n expect.fail('Parent task should have thrown');\n } catch (error: unknown) {\n parentError = error as Error;\n }\n\n // Verify the error propagated correctly\n expect(parentError?.message).toBe('child1 error');\n expect(child1Completed).toBe(false);\n expect(child2Started).toBe(true);\n expect(parentTask.done).toBe(true);\n });\n\n it('should cancel and wait for task completion', async () => {\n let taskCompleted = false;\n\n const task = Task.from(async (controller) => {\n await delay(5000, { signal: controller.signal });\n taskCompleted = true;\n return 'should not complete';\n });\n\n // Cancel and wait should complete quickly when task is aborted\n const start = Date.now();\n const result = await task.cancelAndWait(1000);\n const duration = Date.now() - start;\n\n expect(result).toBe(TaskResult.Aborted);\n expect(duration).toBeLessThan(100); // Should not wait for full timeout\n expect(taskCompleted).toBe(false);\n expect(task.done).toBe(true);\n });\n\n it('should timeout if task does not respond to cancellation', async () => {\n const task = Task.from(async () => {\n await delay(1000);\n });\n\n // This should timeout because the task ignores cancellation\n try {\n await task.cancelAndWait(200);\n expect.fail('Task should have timed out');\n } catch (error: unknown) {\n expect(error).instanceof(Error);\n expect((error as Error).message).toBe('Task cancellation timed out');\n }\n });\n\n it('should handle task that completes before timeout', async () => {\n const task = Task.from(async () => {\n await delay(50);\n });\n\n // Start the task\n await delay(10);\n\n // Cancel and wait - but task will complete normally before being canceled\n const result = await task.cancelAndWait(1000);\n\n // Task should have completed normally\n expect(result).toBe(TaskResult.Completed);\n expect(task.done).toBe(true);\n });\n\n it('should propagate non-abort errors from cancelAndWait', async () => {\n const task = Task.from(async () => {\n await delay(10);\n throw new TypeError('Custom error');\n });\n\n try {\n await task.cancelAndWait(1000);\n expect.fail('Task should have thrown');\n } catch (error: unknown) {\n expect((error as Error).message).toBe('Custom error');\n expect((error as Error).name).toBe('TypeError');\n }\n });\n });\n\n describe('Event', () => {\n it('wait resolves immediately when the event is already set', async () => {\n const event = new Event();\n event.set();\n\n const result = await event.wait();\n expect(result).toBe(true);\n });\n\n it('wait resolves after set is called', async () => {\n // check promise is pending\n const event = new Event();\n const waiterPromise = event.wait();\n\n await delay(10);\n expect(await isPending(waiterPromise)).toBe(true);\n\n // check promise is resolved after set is called\n event.set();\n const result = await waiterPromise;\n expect(result).toBe(true);\n });\n\n it('all waiters resolve once set is called', async () => {\n const event = new Event();\n const waiters = [event.wait(), event.wait(), event.wait()];\n\n await delay(10);\n const pendings = await Promise.all(waiters.map((w) => isPending(w)));\n expect(pendings).toEqual([true, true, true]);\n\n event.set();\n const results = await Promise.all(waiters);\n expect(results).toEqual([true, true, true]);\n });\n\n it('wait after 2 seconds is still pending before set', async () => {\n const event = new Event();\n const waiter = event.wait();\n\n await delay(2000);\n expect(await isPending(waiter)).toBe(true);\n\n event.set();\n const result = await waiter;\n expect(result).toBe(true);\n });\n\n it('wait after set and clear should be pending', async () => {\n const event = new Event();\n const waiterBeforeSet = event.wait();\n event.set();\n event.clear();\n\n const waiterAfterSet = event.wait();\n\n const result = await Promise.race([\n waiterBeforeSet.then(() => 'before'),\n waiterAfterSet.then(() => 'after'),\n ]);\n\n expect(result).toBe('before');\n expect(await isPending(waiterBeforeSet)).toBe(false);\n expect(await isPending(waiterAfterSet)).toBe(true);\n\n event.set();\n expect(await waiterAfterSet).toBe(true);\n });\n });\n\n describe('resampleStream', () => {\n const createAudioFrame = (sampleRate: number, samples: number, channels = 1): AudioFrame => {\n const data = new Int16Array(samples * channels);\n for (let i = 0; i < data.length; i++) {\n data[i] = Math.sin((i / samples) * Math.PI * 2) * 16000;\n }\n return new AudioFrame(data, sampleRate, channels, samples);\n };\n\n const streamToArray = async (stream: ReadableStream<AudioFrame>): Promise<AudioFrame[]> => {\n const reader = stream.getReader();\n const chunks: AudioFrame[] = [];\n try {\n while (true) {\n const { done, value } = await reader.read();\n if (done) break;\n chunks.push(value);\n }\n } finally {\n reader.releaseLock();\n }\n return chunks;\n };\n\n it('should resample audio frames to target sample rate', async () => {\n const inputRate = 48000;\n const outputRate = 16000;\n const inputFrame = createAudioFrame(inputRate, 960); // 20ms at 48kHz\n\n const inputStream = new ReadableStream<AudioFrame>({\n start(controller) {\n controller.enqueue(inputFrame);\n controller.close();\n },\n });\n\n const outputStream = resampleStream({ stream: inputStream, outputRate });\n const outputFrames = await streamToArray(outputStream);\n\n expect(outputFrames.length).toBeGreaterThan(0);\n\n for (const frame of outputFrames) {\n expect(frame.sampleRate).toBe(outputRate);\n expect(frame.channels).toBe(inputFrame.channels);\n }\n });\n\n it('should handle same input and output rate', async () => {\n const sampleRate = 44100;\n const inputFrame = createAudioFrame(sampleRate, 1024);\n\n const inputStream = new ReadableStream<AudioFrame>({\n start(controller) {\n controller.enqueue(inputFrame);\n controller.close();\n },\n });\n\n const outputStream = resampleStream({ stream: inputStream, outputRate: sampleRate });\n const outputFrames = await streamToArray(outputStream);\n\n expect(outputFrames.length).toBeGreaterThan(0);\n\n for (const frame of outputFrames) {\n expect(frame.sampleRate).toBe(sampleRate);\n expect(frame.channels).toBe(inputFrame.channels);\n }\n });\n\n it('should handle multiple input frames', async () => {\n const inputRate = 32000;\n const outputRate = 48000;\n const frame1 = createAudioFrame(inputRate, 640);\n const frame2 = createAudioFrame(inputRate, 640);\n\n const inputStream = new ReadableStream<AudioFrame>({\n start(controller) {\n controller.enqueue(frame1);\n controller.enqueue(frame2);\n controller.close();\n },\n });\n\n const outputStream = resampleStream({ stream: inputStream, outputRate });\n const outputFrames = await streamToArray(outputStream);\n\n expect(outputFrames.length).toBeGreaterThan(0);\n\n for (const frame of outputFrames) {\n expect(frame.sampleRate).toBe(outputRate);\n expect(frame.channels).toBe(frame1.channels);\n }\n });\n\n it('should handle empty stream', async () => {\n const inputStream = new ReadableStream<AudioFrame>({\n start(controller) {\n controller.close();\n },\n });\n\n const outputStream = resampleStream({ stream: inputStream, outputRate: 44100 });\n const outputFrames = await streamToArray(outputStream);\n\n expect(outputFrames).toEqual([]);\n });\n });\n});\n"],"mappings":"AAGA,SAAS,kBAAkB;AAC3B,SAAS,sBAAsB;AAC/B,SAAS,UAAU,QAAQ,UAAU;AACrC,SAAS,wBAAwB;AACjC,SAAS,OAAO,MAAM,YAAY,OAAO,WAAW,sBAAsB;AAE1E,SAAS,SAAS,MAAM;AAEtB,mBAAiB,EAAE,QAAQ,MAAM,OAAO,QAAQ,CAAC;AAEjD,WAAS,QAAQ,MAAM;AACrB,OAAG,sDAAsD,YAAY;AACnE,YAAM,iBAAiB;AACvB,YAAM,OAAO,KAAK,KAAK,YAAY;AACjC,cAAM,MAAM,EAAE;AACd,eAAO;AAAA,MACT,CAAC;AAED,aAAO,KAAK,IAAI,EAAE,KAAK,KAAK;AAC5B,YAAM,SAAS,MAAM,KAAK;AAC1B,aAAO,MAAM,EAAE,KAAK,cAAc;AAClC,aAAO,KAAK,IAAI,EAAE,KAAK,IAAI;AAAA,IAC7B,CAAC;AAED,OAAG,sCAAsC,YAAY;AACnD,YAAM,gBAAgB,IAAI,MAAM,aAAa;AAC7C,YAAM,OAAO,KAAK,KAAK,YAAY;AACjC,cAAM,MAAM,EAAE;AACd,cAAM;AAAA,MACR,CAAC;AAED,aAAO,KAAK,IAAI,EAAE,KAAK,KAAK;AAC5B,YAAM,OAAO,KAAK,MAAM,EAAE,QAAQ,QAAQ,aAAa;AACvD,aAAO,KAAK,IAAI,EAAE,KAAK,IAAI;AAAA,IAC7B,CAAC;AAED,OAAG,4CAA4C,YAAY;AACzD,UAAI,cAAc;AAClB,UAAI,gBAAgB;AAEpB,YAAM,OAAO,KAAK,KAAK,OAAO,eAAe;AAC3C,sBAAc;AACd,cAAM,MAAM,KAAK,EAAE,QAAQ,WAAW,OAAO,CAAC;AAC9C,wBAAgB;AAChB,eAAO;AAAA,MACT,CAAC;AAGD,YAAM,MAAM,EAAE;AACd,aAAO,WAAW,EAAE,KAAK,IAAI;AAC7B,aAAO,KAAK,IAAI,EAAE,KAAK,KAAK;AAG5B,WAAK,OAAO;AAGZ,UAAI;AACF,cAAM,KAAK;AAAA,MACb,SAAS,OAAgB;AACvB,eAAQ,MAAgB,IAAI,EAAE,KAAK,YAAY;AAAA,MACjD;AAEA,aAAO,aAAa,EAAE,KAAK,KAAK;AAChC,aAAO,KAAK,IAAI,EAAE,KAAK,IAAI;AAAA,IAC7B,CAAC;AAED,OAAG,uCAAuC,YAAY;AACpD,YAAM,aAAa,IAAI,gBAAgB;AACvC,YAAM,OAAO,KAAK,KAAK,OAAO,SAAS;AACrC,eAAO,IAAI,EAAE,KAAK,UAAU;AAC5B,cAAM,MAAM,KAAK,EAAE,QAAQ,KAAK,OAAO,CAAC;AACxC,eAAO;AAAA,MACT,GAAG,UAAU;AAEb,YAAM,MAAM,EAAE;AACd,iBAAW,MAAM;AAEjB,UAAI;AACF,cAAM,KAAK;AAAA,MACb,SAAS,OAAgB;AACvB,eAAQ,MAAgB,IAAI,EAAE,KAAK,YAAY;AAAA,MACjD;AAEA,aAAO,KAAK,IAAI,EAAE,KAAK,IAAI;AAAA,IAC7B,CAAC;AAED,OAAG,sCAAsC,YAAY;AACnD,YAAM,OAAO,KAAK,KAAK,YAAY;AACjC,eAAO;AAAA,MACT,CAAC;AAED,YAAM,SAAS,MAAM,KAAK;AAC1B,aAAO,MAAM,EAAE,KAAK,WAAW;AAC/B,aAAO,KAAK,IAAI,EAAE,KAAK,IAAI;AAAA,IAC7B,CAAC;AAED,OAAG,qCAAqC,YAAY;AAClD,YAAM,gBAAgB,IAAI,MAAM,iBAAiB;AACjD,YAAM,OAAO,KAAK,KAAK,YAAY;AACjC,cAAM;AAAA,MACR,CAAC;AAED,UAAI;AACF,cAAM,KAAK;AAAA,MACb,SAAS,OAAgB;AACvB,eAAO,KAAK,EAAE,KAAK,aAAa;AAAA,MAClC;AAEA,aAAO,KAAK,IAAI,EAAE,KAAK,IAAI;AAAA,IAC7B,CAAC;AAED,OAAG,0CAA0C,YAAY;AACvD,YAAM,OAAO,KAAK,KAAK,OAAO,eAAe;AAC3C,cAAM,MAAM,KAAK,EAAE,QAAQ,WAAW,OAAO,CAAC;AAC9C,eAAO;AAAA,MACT,CAAC;AAED,YAAM,MAAM,EAAE;AAGd,WAAK,OAAO;AACZ,WAAK,OAAO;AACZ,WAAK,OAAO;AAEZ,UAAI;AACF,cAAM,KAAK;AAAA,MACb,SAAS,OAAgB;AACvB,eAAQ,MAAgB,IAAI,EAAE,KAAK,YAAY;AAAA,MACjD;AAEA,aAAO,KAAK,IAAI,EAAE,KAAK,IAAI;AAAA,IAC7B,CAAC;AAED,OAAG,wDAAwD,YAAY;AACrE,YAAM,MAAgB,CAAC;AACvB,YAAM,OAAO,KAAK,KAAK,OAAO,eAAe;AAC3C,iBAAS,IAAI,GAAG,IAAI,IAAI,KAAK;AAC3B,cAAI,WAAW,OAAO,SAAS;AAC7B,kBAAM,IAAI,MAAM,kBAAkB;AAAA,UACpC;AACA,gBAAM,MAAM,EAAE;AACd,cAAI,KAAK,CAAC;AAAA,QACZ;AACA,eAAO;AAAA,MACT,CAAC;AAED,YAAM,MAAM,EAAE;AACd,WAAK,OAAO;AAEZ,aAAO,GAAG,EAAE,QAAQ,CAAC,GAAG,GAAG,CAAC,CAAC;AAC7B,UAAI;AACF,cAAM,KAAK;AAAA,MACb,SAAS,OAAgB;AACvB,eAAQ,MAAgB,OAAO,EAAE,KAAK,kBAAkB;AAAA,MAC1D;AAEA,aAAO,KAAK,IAAI,EAAE,KAAK,IAAI;AAAA,IAC7B,CAAC;AAED,OAAG,0CAA0C,YAAY;AACvD,UAAI,kBAAkB;AAEtB,YAAM,OAAO,KAAK,KAAK,OAAO,eAAe;AAC3C,YAAI;AACF,gBAAM,MAAM,KAAK,EAAE,QAAQ,WAAW,OAAO,CAAC;AAC9C,iBAAO;AAAA,QACT,UAAE;AACA,4BAAkB;AAAA,QACpB;AAAA,MACF,CAAC;AAED,YAAM,MAAM,EAAE;AACd,WAAK,OAAO;AAEZ,UAAI;AACF,cAAM,KAAK;AAAA,MACb,QAAQ;AAAA,MAER;AAGA,aAAO,eAAe,EAAE,KAAK,IAAI;AAAA,IACnC,CAAC;AAED,OAAG,iDAAiD,YAAY;AAC9D,YAAM,OAAO,KAAK,KAAK,YAAY;AACjC,cAAM,MAAM,EAAE;AACd,eAAO;AAAA,MACT,CAAC;AAED,YAAM,UAAU,MAAM,KAAK;AAC3B,YAAM,UAAU,MAAM,KAAK;AAC3B,YAAM,UAAU,MAAM,KAAK;AAE3B,aAAO,OAAO,EAAE,KAAK,QAAQ;AAC7B,aAAO,OAAO,EAAE,KAAK,QAAQ;AAC7B,aAAO,OAAO,EAAE,KAAK,QAAQ;AAC7B,aAAO,KAAK,IAAI,EAAE,KAAK,IAAI;AAAA,IAC7B,CAAC;AAED,OAAG,4DAA4D,YAAY;AACzE,YAAM,OAAO,KAAK,KAAK,YAAY;AACjC,cAAM,MAAM,EAAE;AACd,eAAO;AAAA,MACT,CAAC;AAGD,YAAM,iBAAiB,KAAK;AAC5B,YAAM,iBAAiB,KAAK;AAE5B,aAAO,KAAK,IAAI,EAAE,KAAK,KAAK;AAG5B,YAAM,CAAC,SAAS,OAAO,IAAI,MAAM,QAAQ,IAAI,CAAC,gBAAgB,cAAc,CAAC;AAE7E,aAAO,OAAO,EAAE,KAAK,gBAAgB;AACrC,aAAO,OAAO,EAAE,KAAK,gBAAgB;AACrC,aAAO,KAAK,IAAI,EAAE,KAAK,IAAI;AAAA,IAC7B,CAAC;AAED,OAAG,0DAA0D,YAAY;AACvE,UAAI,gBAAgB;AACpB,UAAI,gBAAgB;AACpB,UAAI,gBAAgB;AACpB,UAAI,kBAAkB;AACtB,UAAI,kBAAkB;AACtB,UAAI,kBAAkB;AAEtB,UAAI,aAAuC;AAC3C,UAAI,aAAuC;AAE3C,YAAM,aAAa,KAAK,KAAK,OAAO,eAAe;AACjD,wBAAgB;AAGhB,qBAAa,KAAK,KAAK,OAAO,oBAAoB;AAChD,0BAAgB;AAChB,gBAAM,MAAM,KAAK,EAAE,QAAQ,gBAAgB,OAAO,CAAC;AACnD,4BAAkB;AAClB,iBAAO;AAAA,QACT,GAAG,UAAU;AAEb,qBAAa,KAAK,KAAK,OAAO,oBAAoB;AAChD,0BAAgB;AAChB,gBAAM,MAAM,KAAK,EAAE,QAAQ,gBAAgB,OAAO,CAAC;AACnD,4BAAkB;AAClB,iBAAO;AAAA,QACT,GAAG,UAAU;AAGb,cAAM,UAAU,MAAM,QAAQ,IAAI,CAAC,WAAW,QAAQ,WAAW,MAAM,CAAC;AACxE,0BAAkB;AAClB,eAAO;AAAA,MACT,CAAC;AAGD,YAAM,MAAM,EAAE;AAGd,aAAO,aAAa,EAAE,KAAK,IAAI;AAC/B,aAAO,aAAa,EAAE,KAAK,IAAI;AAC/B,aAAO,aAAa,EAAE,KAAK,IAAI;AAG/B,iBAAW,OAAO;AAGlB,YAAM,CAAC,cAAc,cAAc,YAAY,IAAI,MAAM,QAAQ,WAAW;AAAA,QAC1E,WAAW;AAAA,QACX,WAAY;AAAA,QACZ,WAAY;AAAA,MACd,CAAC;AAGD,aAAO,aAAa,MAAM,EAAE,KAAK,UAAU;AAC3C,aAAQ,aAAuC,OAAO,IAAI,EAAE,KAAK,YAAY;AAE7E,aAAO,aAAa,MAAM,EAAE,KAAK,UAAU;AAC3C,aAAQ,aAAuC,OAAO,IAAI,EAAE,KAAK,YAAY;AAE7E,aAAO,aAAa,MAAM,EAAE,KAAK,UAAU;AAC3C,aAAQ,aAAuC,OAAO,IAAI,EAAE,KAAK,YAAY;AAG7E,aAAO,eAAe,EAAE,KAAK,KAAK;AAClC,aAAO,eAAe,EAAE,KAAK,KAAK;AAClC,aAAO,eAAe,EAAE,KAAK,KAAK;AAClC,aAAO,WAAW,IAAI,EAAE,KAAK,IAAI;AACjC,aAAO,WAAY,IAAI,EAAE,KAAK,IAAI;AAClC,aAAO,WAAY,IAAI,EAAE,KAAK,IAAI;AAAA,IACpC,CAAC;AAED,OAAG,yDAAyD,YAAY;AACtE,YAAM,UAAoB,CAAC;AAE3B,YAAM,aAAa,KAAK,KAAK,OAAO,eAAe;AACjD,gBAAQ,KAAK,cAAc;AAG3B,cAAM,aAAa,KAAK,KAAK,YAAY;AACvC,kBAAQ,KAAK,cAAc;AAC3B,gBAAM,MAAM,EAAE;AACd,kBAAQ,KAAK,YAAY;AACzB,iBAAO;AAAA,QACT,GAAG,UAAU;AAGb,cAAM,aAAa,KAAK,KAAK,OAAO,oBAAoB;AACtD,kBAAQ,KAAK,cAAc;AAG3B,gBAAM,iBAAiB,KAAK,KAAK,YAAY;AAC3C,oBAAQ,KAAK,kBAAkB;AAC/B,kBAAM,MAAM,EAAE;AACd,oBAAQ,KAAK,gBAAgB;AAC7B,mBAAO;AAAA,UACT,GAAG,eAAe;AAElB,gBAAM,mBAAmB,MAAM,eAAe;AAC9C,gBAAM,MAAM,EAAE;AACd,kBAAQ,KAAK,YAAY;AACzB,iBAAO,sBAAsB,gBAAgB;AAAA,QAC/C,GAAG,UAAU;AAGb,cAAM,CAAC,cAAc,YAAY,IAAI,MAAM,QAAQ,IAAI;AAAA,UACrD,WAAW;AAAA,UACX,WAAW;AAAA,QACb,CAAC;AAED,gBAAQ,KAAK,YAAY;AACzB,eAAO;AAAA,UACL,QAAQ;AAAA,UACR,QAAQ;AAAA,UACR,QAAQ;AAAA,QACV;AAAA,MACF,CAAC;AAGD,YAAM,cAAc,MAAM,WAAW;AAGrC,aAAO,WAAW,EAAE,QAAQ;AAAA,QAC1B,QAAQ;AAAA,QACR,QAAQ;AAAA,QACR,QAAQ;AAAA,MACV,CAAC;AAID,aAAO,OAAO,EAAE,QAAQ;AAAA,QACtB;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF,CAAC;AAGD,aAAO,WAAW,IAAI,EAAE,KAAK,IAAI;AAAA,IACnC,CAAC;AAED,OAAG,6CAA6C,YAAY;AAC1D,UAAI,cAA4B;AAChC,UAAI,kBAAkB;AACtB,UAAI,gBAAgB;AAEpB,YAAM,aAAa,KAAK,KAAK,OAAO,eAAe;AACjD,cAAM,aAAa,KAAK,KAAK,YAAY;AACvC,gBAAM,MAAM,EAAE;AACd,gBAAM,IAAI,MAAM,cAAc;AAAA,QAChC,GAAG,UAAU;AAEb,cAAM,aAAa,KAAK,KAAK,YAAY;AACvC,0BAAgB;AAChB,gBAAM,MAAM,EAAE;AACd,4BAAkB;AAClB,iBAAO;AAAA,QACT,GAAG,UAAU;AAGb,cAAM,UAAU,MAAM,QAAQ,IAAI,CAAC,WAAW,QAAQ,WAAW,MAAM,CAAC;AACxE,eAAO;AAAA,MACT,CAAC;AAGD,UAAI;AACF,cAAM,WAAW;AACjB,eAAO,KAAK,gCAAgC;AAAA,MAC9C,SAAS,OAAgB;AACvB,sBAAc;AAAA,MAChB;AAGA,aAAO,2CAAa,OAAO,EAAE,KAAK,cAAc;AAChD,aAAO,eAAe,EAAE,KAAK,KAAK;AAClC,aAAO,aAAa,EAAE,KAAK,IAAI;AAC/B,aAAO,WAAW,IAAI,EAAE,KAAK,IAAI;AAAA,IACnC,CAAC;AAED,OAAG,8CAA8C,YAAY;AAC3D,UAAI,gBAAgB;AAEpB,YAAM,OAAO,KAAK,KAAK,OAAO,eAAe;AAC3C,cAAM,MAAM,KAAM,EAAE,QAAQ,WAAW,OAAO,CAAC;AAC/C,wBAAgB;AAChB,eAAO;AAAA,MACT,CAAC;AAGD,YAAM,QAAQ,KAAK,IAAI;AACvB,YAAM,SAAS,MAAM,KAAK,cAAc,GAAI;AAC5C,YAAM,WAAW,KAAK,IAAI,IAAI;AAE9B,aAAO,MAAM,EAAE,KAAK,WAAW,OAAO;AACtC,aAAO,QAAQ,EAAE,aAAa,GAAG;AACjC,aAAO,aAAa,EAAE,KAAK,KAAK;AAChC,aAAO,KAAK,IAAI,EAAE,KAAK,IAAI;AAAA,IAC7B,CAAC;AAED,OAAG,2DAA2D,YAAY;AACxE,YAAM,OAAO,KAAK,KAAK,YAAY;AACjC,cAAM,MAAM,GAAI;AAAA,MAClB,CAAC;AAGD,UAAI;AACF,cAAM,KAAK,cAAc,GAAG;AAC5B,eAAO,KAAK,4BAA4B;AAAA,MAC1C,SAAS,OAAgB;AACvB,eAAO,KAAK,EAAE,WAAW,KAAK;AAC9B,eAAQ,MAAgB,OAAO,EAAE,KAAK,6BAA6B;AAAA,MACrE;AAAA,IACF,CAAC;AAED,OAAG,oDAAoD,YAAY;AACjE,YAAM,OAAO,KAAK,KAAK,YAAY;AACjC,cAAM,MAAM,EAAE;AAAA,MAChB,CAAC;AAGD,YAAM,MAAM,EAAE;AAGd,YAAM,SAAS,MAAM,KAAK,cAAc,GAAI;AAG5C,aAAO,MAAM,EAAE,KAAK,WAAW,SAAS;AACxC,aAAO,KAAK,IAAI,EAAE,KAAK,IAAI;AAAA,IAC7B,CAAC;AAED,OAAG,wDAAwD,YAAY;AACrE,YAAM,OAAO,KAAK,KAAK,YAAY;AACjC,cAAM,MAAM,EAAE;AACd,cAAM,IAAI,UAAU,cAAc;AAAA,MACpC,CAAC;AAED,UAAI;AACF,cAAM,KAAK,cAAc,GAAI;AAC7B,eAAO,KAAK,yBAAyB;AAAA,MACvC,SAAS,OAAgB;AACvB,eAAQ,MAAgB,OAAO,EAAE,KAAK,cAAc;AACpD,eAAQ,MAAgB,IAAI,EAAE,KAAK,WAAW;AAAA,MAChD;AAAA,IACF,CAAC;AAAA,EACH,CAAC;AAED,WAAS,SAAS,MAAM;AACtB,OAAG,2DAA2D,YAAY;AACxE,YAAM,QAAQ,IAAI,MAAM;AACxB,YAAM,IAAI;AAEV,YAAM,SAAS,MAAM,MAAM,KAAK;AAChC,aAAO,MAAM,EAAE,KAAK,IAAI;AAAA,IAC1B,CAAC;AAED,OAAG,qCAAqC,YAAY;AAElD,YAAM,QAAQ,IAAI,MAAM;AACxB,YAAM,gBAAgB,MAAM,KAAK;AAEjC,YAAM,MAAM,EAAE;AACd,aAAO,MAAM,UAAU,aAAa,CAAC,EAAE,KAAK,IAAI;AAGhD,YAAM,IAAI;AACV,YAAM,SAAS,MAAM;AACrB,aAAO,MAAM,EAAE,KAAK,IAAI;AAAA,IAC1B,CAAC;AAED,OAAG,0CAA0C,YAAY;AACvD,YAAM,QAAQ,IAAI,MAAM;AACxB,YAAM,UAAU,CAAC,MAAM,KAAK,GAAG,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC;AAEzD,YAAM,MAAM,EAAE;AACd,YAAM,WAAW,MAAM,QAAQ,IAAI,QAAQ,IAAI,CAAC,MAAM,UAAU,CAAC,CAAC,CAAC;AACnE,aAAO,QAAQ,EAAE,QAAQ,CAAC,MAAM,MAAM,IAAI,CAAC;AAE3C,YAAM,IAAI;AACV,YAAM,UAAU,MAAM,QAAQ,IAAI,OAAO;AACzC,aAAO,OAAO,EAAE,QAAQ,CAAC,MAAM,MAAM,IAAI,CAAC;AAAA,IAC5C,CAAC;AAED,OAAG,oDAAoD,YAAY;AACjE,YAAM,QAAQ,IAAI,MAAM;AACxB,YAAM,SAAS,MAAM,KAAK;AAE1B,YAAM,MAAM,GAAI;AAChB,aAAO,MAAM,UAAU,MAAM,CAAC,EAAE,KAAK,IAAI;AAEzC,YAAM,IAAI;AACV,YAAM,SAAS,MAAM;AACrB,aAAO,MAAM,EAAE,KAAK,IAAI;AAAA,IAC1B,CAAC;AAED,OAAG,8CAA8C,YAAY;AAC3D,YAAM,QAAQ,IAAI,MAAM;AACxB,YAAM,kBAAkB,MAAM,KAAK;AACnC,YAAM,IAAI;AACV,YAAM,MAAM;AAEZ,YAAM,iBAAiB,MAAM,KAAK;AAElC,YAAM,SAAS,MAAM,QAAQ,KAAK;AAAA,QAChC,gBAAgB,KAAK,MAAM,QAAQ;AAAA,QACnC,eAAe,KAAK,MAAM,OAAO;AAAA,MACnC,CAAC;AAED,aAAO,MAAM,EAAE,KAAK,QAAQ;AAC5B,aAAO,MAAM,UAAU,eAAe,CAAC,EAAE,KAAK,KAAK;AACnD,aAAO,MAAM,UAAU,cAAc,CAAC,EAAE,KAAK,IAAI;AAEjD,YAAM,IAAI;AACV,aAAO,MAAM,cAAc,EAAE,KAAK,IAAI;AAAA,IACxC,CAAC;AAAA,EACH,CAAC;AAED,WAAS,kBAAkB,MAAM;AAC/B,UAAM,mBAAmB,CAAC,YAAoB,SAAiB,WAAW,MAAkB;AAC1F,YAAM,OAAO,IAAI,WAAW,UAAU,QAAQ;AAC9C,eAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,aAAK,CAAC,IAAI,KAAK,IAAK,IAAI,UAAW,KAAK,KAAK,CAAC,IAAI;AAAA,MACpD;AACA,aAAO,IAAI,WAAW,MAAM,YAAY,UAAU,OAAO;AAAA,IAC3D;AAEA,UAAM,gBAAgB,OAAO,WAA8D;AACzF,YAAM,SAAS,OAAO,UAAU;AAChC,YAAM,SAAuB,CAAC;AAC9B,UAAI;AACF,eAAO,MAAM;AACX,gBAAM,EAAE,MAAM,MAAM,IAAI,MAAM,OAAO,KAAK;AAC1C,cAAI,KAAM;AACV,iBAAO,KAAK,KAAK;AAAA,QACnB;AAAA,MACF,UAAE;AACA,eAAO,YAAY;AAAA,MACrB;AACA,aAAO;AAAA,IACT;AAEA,OAAG,sDAAsD,YAAY;AACnE,YAAM,YAAY;AAClB,YAAM,aAAa;AACnB,YAAM,aAAa,iBAAiB,WAAW,GAAG;AAElD,YAAM,cAAc,IAAI,eAA2B;AAAA,QACjD,MAAM,YAAY;AAChB,qBAAW,QAAQ,UAAU;AAC7B,qBAAW,MAAM;AAAA,QACnB;AAAA,MACF,CAAC;AAED,YAAM,eAAe,eAAe,EAAE,QAAQ,aAAa,WAAW,CAAC;AACvE,YAAM,eAAe,MAAM,cAAc,YAAY;AAErD,aAAO,aAAa,MAAM,EAAE,gBAAgB,CAAC;AAE7C,iBAAW,SAAS,cAAc;AAChC,eAAO,MAAM,UAAU,EAAE,KAAK,UAAU;AACxC,eAAO,MAAM,QAAQ,EAAE,KAAK,WAAW,QAAQ;AAAA,MACjD;AAAA,IACF,CAAC;AAED,OAAG,4CAA4C,YAAY;AACzD,YAAM,aAAa;AACnB,YAAM,aAAa,iBAAiB,YAAY,IAAI;AAEpD,YAAM,cAAc,IAAI,eAA2B;AAAA,QACjD,MAAM,YAAY;AAChB,qBAAW,QAAQ,UAAU;AAC7B,qBAAW,MAAM;AAAA,QACnB;AAAA,MACF,CAAC;AAED,YAAM,eAAe,eAAe,EAAE,QAAQ,aAAa,YAAY,WAAW,CAAC;AACnF,YAAM,eAAe,MAAM,cAAc,YAAY;AAErD,aAAO,aAAa,MAAM,EAAE,gBAAgB,CAAC;AAE7C,iBAAW,SAAS,cAAc;AAChC,eAAO,MAAM,UAAU,EAAE,KAAK,UAAU;AACxC,eAAO,MAAM,QAAQ,EAAE,KAAK,WAAW,QAAQ;AAAA,MACjD;AAAA,IACF,CAAC;AAED,OAAG,uCAAuC,YAAY;AACpD,YAAM,YAAY;AAClB,YAAM,aAAa;AACnB,YAAM,SAAS,iBAAiB,WAAW,GAAG;AAC9C,YAAM,SAAS,iBAAiB,WAAW,GAAG;AAE9C,YAAM,cAAc,IAAI,eAA2B;AAAA,QACjD,MAAM,YAAY;AAChB,qBAAW,QAAQ,MAAM;AACzB,qBAAW,QAAQ,MAAM;AACzB,qBAAW,MAAM;AAAA,QACnB;AAAA,MACF,CAAC;AAED,YAAM,eAAe,eAAe,EAAE,QAAQ,aAAa,WAAW,CAAC;AACvE,YAAM,eAAe,MAAM,cAAc,YAAY;AAErD,aAAO,aAAa,MAAM,EAAE,gBAAgB,CAAC;AAE7C,iBAAW,SAAS,cAAc;AAChC,eAAO,MAAM,UAAU,EAAE,KAAK,UAAU;AACxC,eAAO,MAAM,QAAQ,EAAE,KAAK,OAAO,QAAQ;AAAA,MAC7C;AAAA,IACF,CAAC;AAED,OAAG,8BAA8B,YAAY;AAC3C,YAAM,cAAc,IAAI,eAA2B;AAAA,QACjD,MAAM,YAAY;AAChB,qBAAW,MAAM;AAAA,QACnB;AAAA,MACF,CAAC;AAED,YAAM,eAAe,eAAe,EAAE,QAAQ,aAAa,YAAY,MAAM,CAAC;AAC9E,YAAM,eAAe,MAAM,cAAc,YAAY;AAErD,aAAO,YAAY,EAAE,QAAQ,CAAC,CAAC;AAAA,IACjC,CAAC;AAAA,EACH,CAAC;AACH,CAAC;","names":[]}
|
package/dist/vad.cjs
CHANGED
package/dist/vad.cjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/vad.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport type {\n ReadableStream,\n ReadableStreamDefaultReader,\n WritableStreamDefaultWriter,\n} from 'node:stream/web';\nimport { log } from './log.js';\nimport type { VADMetrics } from './metrics/base.js';\nimport { DeferredReadableStream } from './stream/deferred_stream.js';\nimport { IdentityTransform } from './stream/identity_transform.js';\n\nexport enum VADEventType {\n START_OF_SPEECH,\n INFERENCE_DONE,\n END_OF_SPEECH,\n METRICS_COLLECTED,\n}\n\nexport interface VADEvent {\n /** Type of the VAD event (e.g., start of speech, end of speech, inference done). */\n type: VADEventType;\n /**\n * Index of the audio sample where the event occurred, relative to the inference sample rate.\n */\n samplesIndex: number;\n /** Timestamp when the event was fired. */\n timestamp: number;\n /** Duration of the speech segment. */\n speechDuration: number;\n /** Duration of the silence segment. */\n silenceDuration: number;\n /**\n * List of audio frames associated with the speech.\n *\n * @remarks\n * - For `start_of_speech` events, this contains the audio chunks that triggered the detection.\n * - For `inference_done` events, this contains the audio chunks that were processed.\n * - For `end_of_speech` events, this contains the complete user speech.\n */\n frames: AudioFrame[];\n /** Probability that speech is present (only for `INFERENCE_DONE` events). */\n probability: number;\n /** Time taken to perform the inference, in seconds (only for `INFERENCE_DONE` events). */\n inferenceDuration: number;\n /** Indicates whether speech was detected in the frames. */\n speaking: boolean;\n /** Threshold used to detect silence. */\n rawAccumulatedSilence: number;\n /** Threshold used to detect speech. */\n rawAccumulatedSpeech: number;\n}\n\nexport interface VADCapabilities {\n updateInterval: number;\n}\n\nexport type VADCallbacks = {\n ['metrics_collected']: (metrics: VADMetrics) => void;\n};\n\nexport abstract class VAD extends (EventEmitter as new () => TypedEmitter<VADCallbacks>) {\n #capabilities: VADCapabilities;\n abstract label: string;\n\n constructor(capabilities: VADCapabilities) {\n super();\n this.#capabilities = capabilities;\n }\n\n get capabilities(): VADCapabilities {\n return this.#capabilities;\n }\n\n /**\n * Returns a {@link VADStream} that can be used to push audio frames and receive VAD events.\n */\n abstract stream(): VADStream;\n}\n\nexport abstract class VADStream implements AsyncIterableIterator<VADEvent> {\n protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');\n protected input = new IdentityTransform<AudioFrame | typeof VADStream.FLUSH_SENTINEL>();\n protected output = new IdentityTransform<VADEvent>();\n protected inputWriter: WritableStreamDefaultWriter<AudioFrame | typeof VADStream.FLUSH_SENTINEL>;\n protected inputReader: ReadableStreamDefaultReader<AudioFrame | typeof VADStream.FLUSH_SENTINEL>;\n protected outputWriter: WritableStreamDefaultWriter<VADEvent>;\n protected outputReader: ReadableStreamDefaultReader<VADEvent>;\n protected closed = false;\n protected inputClosed = false;\n\n #vad: VAD;\n #lastActivityTime = BigInt(0);\n private logger = log();\n private deferredInputStream: DeferredReadableStream<AudioFrame>;\n\n private metricsStream: ReadableStream<VADEvent>;\n constructor(vad: VAD) {\n this.#vad = vad;\n this.deferredInputStream = new DeferredReadableStream<AudioFrame>();\n\n this.inputWriter = this.input.writable.getWriter();\n this.inputReader = this.input.readable.getReader();\n this.outputWriter = this.output.writable.getWriter();\n\n const [outputStream, metricsStream] = this.output.readable.tee();\n this.metricsStream = metricsStream;\n this.outputReader = outputStream.getReader();\n\n this.pumpDeferredStream();\n this.monitorMetrics();\n }\n\n /**\n * Reads from the deferred input stream and forwards chunks to the input writer.\n *\n * Note: we can't just do this.deferredInputStream.stream.pipeTo(this.input.writable)\n * because the inputWriter locks the this.input.writable stream. All writes must go through\n * the inputWriter.\n */\n private async pumpDeferredStream() {\n const reader = this.deferredInputStream.stream.getReader();\n try {\n while (true) {\n const { done, value } = await reader.read();\n if (done) break;\n await this.inputWriter.write(value);\n }\n } catch (e) {\n this.logger.error(`Error pumping deferred stream: ${e}`);\n throw e;\n } finally {\n reader.releaseLock();\n }\n }\n\n protected async monitorMetrics() {\n let inferenceDurationTotalMs = 0;\n let inferenceCount = 0;\n const metricsReader = this.metricsStream.getReader();\n while (true) {\n const { done, value } = await metricsReader.read();\n if (done) {\n break;\n }\n switch (value.type) {\n case VADEventType.START_OF_SPEECH:\n inferenceCount++;\n if (inferenceCount >= 1 / this.#vad.capabilities.updateInterval) {\n this.#vad.emit('metrics_collected', {\n type: 'vad_metrics',\n timestamp: Date.now(),\n idleTimeMs: Math.trunc(\n Number((process.hrtime.bigint() - this.#lastActivityTime) / BigInt(1000000)),\n ),\n inferenceDurationTotalMs,\n inferenceCount,\n label: this.#vad.label,\n });\n\n inferenceCount = 0;\n inferenceDurationTotalMs = 0;\n }\n break;\n case VADEventType.INFERENCE_DONE:\n inferenceDurationTotalMs += Math.round(value.inferenceDuration);\n this.#lastActivityTime = process.hrtime.bigint();\n break;\n case VADEventType.END_OF_SPEECH:\n this.#lastActivityTime = process.hrtime.bigint();\n break;\n }\n }\n }\n\n /**\n * Safely send a VAD event to the output stream, handling writer release errors during shutdown.\n * @returns true if the event was sent, false if the stream is closing\n * @throws Error if an unexpected error occurs\n */\n protected sendVADEvent(event: VADEvent): boolean {\n if (this.closed) {\n return false;\n }\n\n try {\n this.outputWriter.write(event);\n return true;\n } catch (e) {\n throw e;\n }\n }\n\n updateInputStream(audioStream: ReadableStream<AudioFrame>) {\n this.deferredInputStream.setSource(audioStream);\n }\n\n detachInputStream() {\n this.deferredInputStream.detachSource();\n }\n\n /** @deprecated Use `updateInputStream` instead */\n pushFrame(frame: AudioFrame) {\n // TODO(AJS-395): remove this method\n if (this.inputClosed) {\n throw new Error('Input is closed');\n }\n if (this.closed) {\n throw new Error('Stream is closed');\n }\n this.inputWriter.write(frame);\n }\n\n flush() {\n if (this.inputClosed) {\n throw new Error('Input is closed');\n }\n if (this.closed) {\n throw new Error('Stream is closed');\n }\n this.inputWriter.write(VADStream.FLUSH_SENTINEL);\n }\n\n endInput() {\n if (this.inputClosed) {\n throw new Error('Input is closed');\n }\n if (this.closed) {\n throw new Error('Stream is closed');\n }\n this.inputClosed = true;\n this.input.writable.close();\n }\n\n async next(): Promise<IteratorResult<VADEvent>> {\n return this.outputReader.read().then(({ done, value }) => {\n if (done) {\n return { done: true, value: undefined };\n }\n return { done: false, value };\n });\n }\n\n close() {\n this.outputWriter.releaseLock();\n this.outputReader.cancel();\n this.output.writable.close();\n this.closed = true;\n }\n\n [Symbol.asyncIterator](): VADStream {\n return this;\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAKA,yBAA6B;AAM7B,iBAAoB;AAEpB,6BAAuC;AACvC,gCAAkC;AAE3B,IAAK,eAAL,kBAAKA,kBAAL;AACL,EAAAA,4BAAA;AACA,EAAAA,4BAAA;AACA,EAAAA,4BAAA;AACA,EAAAA,4BAAA;AAJU,SAAAA;AAAA,GAAA;AAiDL,MAAe,YAAa,gCAAsD;AAAA,EACvF;AAAA,EAGA,YAAY,cAA+B;AACzC,UAAM;AACN,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,IAAI,eAAgC;AAClC,WAAO,KAAK;AAAA,EACd;AAMF;AAEO,MAAe,UAAqD;AAAA,EACzE,OAA0B,iBAAiB,OAAO,gBAAgB;AAAA,EACxD,QAAQ,IAAI,4CAAgE;AAAA,EAC5E,SAAS,IAAI,4CAA4B;AAAA,EACzC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,SAAS;AAAA,EACT,cAAc;AAAA,EAExB;AAAA,EACA,oBAAoB,OAAO,CAAC;AAAA,EACpB,aAAS,gBAAI;AAAA,EACb;AAAA,EAEA;AAAA,EACR,YAAY,KAAU;AACpB,SAAK,OAAO;AACZ,SAAK,sBAAsB,IAAI,8CAAmC;AAElE,SAAK,cAAc,KAAK,MAAM,SAAS,UAAU;AACjD,SAAK,cAAc,KAAK,MAAM,SAAS,UAAU;AACjD,SAAK,eAAe,KAAK,OAAO,SAAS,UAAU;AAEnD,UAAM,CAAC,cAAc,aAAa,IAAI,KAAK,OAAO,SAAS,IAAI;AAC/D,SAAK,gBAAgB;AACrB,SAAK,eAAe,aAAa,UAAU;AAE3C,SAAK,mBAAmB;AACxB,SAAK,eAAe;AAAA,EACtB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAc,qBAAqB;AACjC,UAAM,SAAS,KAAK,oBAAoB,OAAO,UAAU;AACzD,QAAI;AACF,aAAO,MAAM;AACX,cAAM,EAAE,MAAM,MAAM,IAAI,MAAM,OAAO,KAAK;AAC1C,YAAI,KAAM;AACV,cAAM,KAAK,YAAY,MAAM,KAAK;AAAA,MACpC;AAAA,IACF,SAAS,GAAG;AACV,WAAK,OAAO,MAAM,kCAAkC,CAAC,EAAE;AACvD,YAAM;AAAA,IACR,UAAE;AACA,aAAO,YAAY;AAAA,IACrB;AAAA,EACF;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,QAAI,2BAA2B;AAC/B,QAAI,iBAAiB;AACrB,UAAM,gBAAgB,KAAK,cAAc,UAAU;AACnD,WAAO,MAAM;AACX,YAAM,EAAE,MAAM,MAAM,IAAI,MAAM,cAAc,KAAK;AACjD,UAAI,MAAM;AACR;AAAA,MACF;AACA,cAAQ,MAAM,MAAM;AAAA,QAClB,KAAK;AACH;AACA,cAAI,kBAAkB,IAAI,KAAK,KAAK,aAAa,gBAAgB;AAC/D,iBAAK,KAAK,KAAK,qBAAqB;AAAA,cAClC,MAAM;AAAA,cACN,WAAW,KAAK,IAAI;AAAA,cACpB,YAAY,KAAK;AAAA,gBACf,QAAQ,QAAQ,OAAO,OAAO,IAAI,KAAK,qBAAqB,OAAO,GAAO,CAAC;AAAA,cAC7E;AAAA,cACA;AAAA,cACA;AAAA,cACA,OAAO,KAAK,KAAK;AAAA,YACnB,CAAC;AAED,6BAAiB;AACjB,uCAA2B;AAAA,UAC7B;AACA;AAAA,QACF,KAAK;AACH,sCAA4B,KAAK,MAAM,MAAM,iBAAiB;AAC9D,eAAK,oBAAoB,QAAQ,OAAO,OAAO;AAC/C;AAAA,QACF,KAAK;AACH,eAAK,oBAAoB,QAAQ,OAAO,OAAO;AAC/C;AAAA,MACJ;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOU,aAAa,OAA0B;AAC/C,QAAI,KAAK,QAAQ;AACf,aAAO;AAAA,IACT;AAEA,QAAI;AACF,WAAK,aAAa,MAAM,KAAK;AAC7B,aAAO;AAAA,IACT,SAAS,GAAG;AACV,YAAM;AAAA,IACR;AAAA,EACF;AAAA,EAEA,kBAAkB,aAAyC;AACzD,SAAK,oBAAoB,UAAU,WAAW;AAAA,EAChD;AAAA,EAEA,oBAAoB;AAClB,SAAK,oBAAoB,aAAa;AAAA,EACxC;AAAA;AAAA,EAGA,UAAU,OAAmB;AAE3B,QAAI,KAAK,aAAa;AACpB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,YAAY,MAAM,KAAK;AAAA,EAC9B;AAAA,EAEA,QAAQ;AACN,QAAI,KAAK,aAAa;AACpB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,YAAY,MAAM,UAAU,cAAc;AAAA,EACjD;AAAA,EAEA,WAAW;AACT,QAAI,KAAK,aAAa;AACpB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,cAAc;AACnB,SAAK,MAAM,SAAS,MAAM;AAAA,EAC5B;AAAA,EAEA,MAAM,OAA0C;AAC9C,WAAO,KAAK,aAAa,KAAK,EAAE,KAAK,CAAC,EAAE,MAAM,MAAM,MAAM;AACxD,UAAI,MAAM;AACR,eAAO,EAAE,MAAM,MAAM,OAAO,OAAU;AAAA,MACxC;AACA,aAAO,EAAE,MAAM,OAAO,MAAM;AAAA,IAC9B,CAAC;AAAA,EACH;AAAA,EAEA,QAAQ;AACN,SAAK,aAAa,YAAY;AAC9B,SAAK,aAAa,OAAO;AACzB,SAAK,OAAO,SAAS,MAAM;AAC3B,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAe;AAClC,WAAO;AAAA,EACT;AACF;","names":["VADEventType"]}
|
|
1
|
+
{"version":3,"sources":["../src/vad.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport type {\n ReadableStream,\n ReadableStreamDefaultReader,\n WritableStreamDefaultWriter,\n} from 'node:stream/web';\nimport { log } from './log.js';\nimport type { VADMetrics } from './metrics/base.js';\nimport { DeferredReadableStream } from './stream/deferred_stream.js';\nimport { IdentityTransform } from './stream/identity_transform.js';\n\nexport enum VADEventType {\n START_OF_SPEECH,\n INFERENCE_DONE,\n END_OF_SPEECH,\n METRICS_COLLECTED,\n}\n\nexport interface VADEvent {\n /** Type of the VAD event (e.g., start of speech, end of speech, inference done). */\n type: VADEventType;\n /**\n * Index of the audio sample where the event occurred, relative to the inference sample rate.\n */\n samplesIndex: number;\n /** Timestamp when the event was fired. */\n timestamp: number;\n /** Duration of the speech segment. */\n speechDuration: number;\n /** Duration of the silence segment. */\n silenceDuration: number;\n /**\n * List of audio frames associated with the speech.\n *\n * @remarks\n * - For `start_of_speech` events, this contains the audio chunks that triggered the detection.\n * - For `inference_done` events, this contains the audio chunks that were processed.\n * - For `end_of_speech` events, this contains the complete user speech.\n */\n frames: AudioFrame[];\n /** Probability that speech is present (only for `INFERENCE_DONE` events). */\n probability: number;\n /** Time taken to perform the inference, in seconds (only for `INFERENCE_DONE` events). */\n inferenceDuration: number;\n /** Indicates whether speech was detected in the frames. */\n speaking: boolean;\n /** Threshold used to detect silence. */\n rawAccumulatedSilence: number;\n /** Threshold used to detect speech. */\n rawAccumulatedSpeech: number;\n}\n\nexport interface VADCapabilities {\n updateInterval: number;\n}\n\nexport type VADCallbacks = {\n ['metrics_collected']: (metrics: VADMetrics) => void;\n};\n\nexport abstract class VAD extends (EventEmitter as new () => TypedEmitter<VADCallbacks>) {\n #capabilities: VADCapabilities;\n abstract label: string;\n\n constructor(capabilities: VADCapabilities) {\n super();\n this.#capabilities = capabilities;\n }\n\n get capabilities(): VADCapabilities {\n return this.#capabilities;\n }\n\n /**\n * Returns a {@link VADStream} that can be used to push audio frames and receive VAD events.\n */\n abstract stream(): VADStream;\n\n async close(): Promise<void> {\n return;\n }\n}\n\nexport abstract class VADStream implements AsyncIterableIterator<VADEvent> {\n protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');\n protected input = new IdentityTransform<AudioFrame | typeof VADStream.FLUSH_SENTINEL>();\n protected output = new IdentityTransform<VADEvent>();\n protected inputWriter: WritableStreamDefaultWriter<AudioFrame | typeof VADStream.FLUSH_SENTINEL>;\n protected inputReader: ReadableStreamDefaultReader<AudioFrame | typeof VADStream.FLUSH_SENTINEL>;\n protected outputWriter: WritableStreamDefaultWriter<VADEvent>;\n protected outputReader: ReadableStreamDefaultReader<VADEvent>;\n protected closed = false;\n protected inputClosed = false;\n\n #vad: VAD;\n #lastActivityTime = BigInt(0);\n private logger = log();\n private deferredInputStream: DeferredReadableStream<AudioFrame>;\n\n private metricsStream: ReadableStream<VADEvent>;\n constructor(vad: VAD) {\n this.#vad = vad;\n this.deferredInputStream = new DeferredReadableStream<AudioFrame>();\n\n this.inputWriter = this.input.writable.getWriter();\n this.inputReader = this.input.readable.getReader();\n this.outputWriter = this.output.writable.getWriter();\n\n const [outputStream, metricsStream] = this.output.readable.tee();\n this.metricsStream = metricsStream;\n this.outputReader = outputStream.getReader();\n\n this.pumpDeferredStream();\n this.monitorMetrics();\n }\n\n /**\n * Reads from the deferred input stream and forwards chunks to the input writer.\n *\n * Note: we can't just do this.deferredInputStream.stream.pipeTo(this.input.writable)\n * because the inputWriter locks the this.input.writable stream. All writes must go through\n * the inputWriter.\n */\n private async pumpDeferredStream() {\n const reader = this.deferredInputStream.stream.getReader();\n try {\n while (true) {\n const { done, value } = await reader.read();\n if (done) break;\n await this.inputWriter.write(value);\n }\n } catch (e) {\n this.logger.error(`Error pumping deferred stream: ${e}`);\n throw e;\n } finally {\n reader.releaseLock();\n }\n }\n\n protected async monitorMetrics() {\n let inferenceDurationTotalMs = 0;\n let inferenceCount = 0;\n const metricsReader = this.metricsStream.getReader();\n while (true) {\n const { done, value } = await metricsReader.read();\n if (done) {\n break;\n }\n switch (value.type) {\n case VADEventType.START_OF_SPEECH:\n inferenceCount++;\n if (inferenceCount >= 1 / this.#vad.capabilities.updateInterval) {\n this.#vad.emit('metrics_collected', {\n type: 'vad_metrics',\n timestamp: Date.now(),\n idleTimeMs: Math.trunc(\n Number((process.hrtime.bigint() - this.#lastActivityTime) / BigInt(1000000)),\n ),\n inferenceDurationTotalMs,\n inferenceCount,\n label: this.#vad.label,\n });\n\n inferenceCount = 0;\n inferenceDurationTotalMs = 0;\n }\n break;\n case VADEventType.INFERENCE_DONE:\n inferenceDurationTotalMs += Math.round(value.inferenceDuration);\n this.#lastActivityTime = process.hrtime.bigint();\n break;\n case VADEventType.END_OF_SPEECH:\n this.#lastActivityTime = process.hrtime.bigint();\n break;\n }\n }\n }\n\n /**\n * Safely send a VAD event to the output stream, handling writer release errors during shutdown.\n * @returns true if the event was sent, false if the stream is closing\n * @throws Error if an unexpected error occurs\n */\n protected sendVADEvent(event: VADEvent): boolean {\n if (this.closed) {\n return false;\n }\n\n try {\n this.outputWriter.write(event);\n return true;\n } catch (e) {\n throw e;\n }\n }\n\n updateInputStream(audioStream: ReadableStream<AudioFrame>) {\n this.deferredInputStream.setSource(audioStream);\n }\n\n detachInputStream() {\n this.deferredInputStream.detachSource();\n }\n\n /** @deprecated Use `updateInputStream` instead */\n pushFrame(frame: AudioFrame) {\n // TODO(AJS-395): remove this method\n if (this.inputClosed) {\n throw new Error('Input is closed');\n }\n if (this.closed) {\n throw new Error('Stream is closed');\n }\n this.inputWriter.write(frame);\n }\n\n flush() {\n if (this.inputClosed) {\n throw new Error('Input is closed');\n }\n if (this.closed) {\n throw new Error('Stream is closed');\n }\n this.inputWriter.write(VADStream.FLUSH_SENTINEL);\n }\n\n endInput() {\n if (this.inputClosed) {\n throw new Error('Input is closed');\n }\n if (this.closed) {\n throw new Error('Stream is closed');\n }\n this.inputClosed = true;\n this.input.writable.close();\n }\n\n async next(): Promise<IteratorResult<VADEvent>> {\n return this.outputReader.read().then(({ done, value }) => {\n if (done) {\n return { done: true, value: undefined };\n }\n return { done: false, value };\n });\n }\n\n close() {\n this.outputWriter.releaseLock();\n this.outputReader.cancel();\n this.output.writable.close();\n this.closed = true;\n }\n\n [Symbol.asyncIterator](): VADStream {\n return this;\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAKA,yBAA6B;AAM7B,iBAAoB;AAEpB,6BAAuC;AACvC,gCAAkC;AAE3B,IAAK,eAAL,kBAAKA,kBAAL;AACL,EAAAA,4BAAA;AACA,EAAAA,4BAAA;AACA,EAAAA,4BAAA;AACA,EAAAA,4BAAA;AAJU,SAAAA;AAAA,GAAA;AAiDL,MAAe,YAAa,gCAAsD;AAAA,EACvF;AAAA,EAGA,YAAY,cAA+B;AACzC,UAAM;AACN,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,IAAI,eAAgC;AAClC,WAAO,KAAK;AAAA,EACd;AAAA,EAOA,MAAM,QAAuB;AAC3B;AAAA,EACF;AACF;AAEO,MAAe,UAAqD;AAAA,EACzE,OAA0B,iBAAiB,OAAO,gBAAgB;AAAA,EACxD,QAAQ,IAAI,4CAAgE;AAAA,EAC5E,SAAS,IAAI,4CAA4B;AAAA,EACzC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,SAAS;AAAA,EACT,cAAc;AAAA,EAExB;AAAA,EACA,oBAAoB,OAAO,CAAC;AAAA,EACpB,aAAS,gBAAI;AAAA,EACb;AAAA,EAEA;AAAA,EACR,YAAY,KAAU;AACpB,SAAK,OAAO;AACZ,SAAK,sBAAsB,IAAI,8CAAmC;AAElE,SAAK,cAAc,KAAK,MAAM,SAAS,UAAU;AACjD,SAAK,cAAc,KAAK,MAAM,SAAS,UAAU;AACjD,SAAK,eAAe,KAAK,OAAO,SAAS,UAAU;AAEnD,UAAM,CAAC,cAAc,aAAa,IAAI,KAAK,OAAO,SAAS,IAAI;AAC/D,SAAK,gBAAgB;AACrB,SAAK,eAAe,aAAa,UAAU;AAE3C,SAAK,mBAAmB;AACxB,SAAK,eAAe;AAAA,EACtB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAc,qBAAqB;AACjC,UAAM,SAAS,KAAK,oBAAoB,OAAO,UAAU;AACzD,QAAI;AACF,aAAO,MAAM;AACX,cAAM,EAAE,MAAM,MAAM,IAAI,MAAM,OAAO,KAAK;AAC1C,YAAI,KAAM;AACV,cAAM,KAAK,YAAY,MAAM,KAAK;AAAA,MACpC;AAAA,IACF,SAAS,GAAG;AACV,WAAK,OAAO,MAAM,kCAAkC,CAAC,EAAE;AACvD,YAAM;AAAA,IACR,UAAE;AACA,aAAO,YAAY;AAAA,IACrB;AAAA,EACF;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,QAAI,2BAA2B;AAC/B,QAAI,iBAAiB;AACrB,UAAM,gBAAgB,KAAK,cAAc,UAAU;AACnD,WAAO,MAAM;AACX,YAAM,EAAE,MAAM,MAAM,IAAI,MAAM,cAAc,KAAK;AACjD,UAAI,MAAM;AACR;AAAA,MACF;AACA,cAAQ,MAAM,MAAM;AAAA,QAClB,KAAK;AACH;AACA,cAAI,kBAAkB,IAAI,KAAK,KAAK,aAAa,gBAAgB;AAC/D,iBAAK,KAAK,KAAK,qBAAqB;AAAA,cAClC,MAAM;AAAA,cACN,WAAW,KAAK,IAAI;AAAA,cACpB,YAAY,KAAK;AAAA,gBACf,QAAQ,QAAQ,OAAO,OAAO,IAAI,KAAK,qBAAqB,OAAO,GAAO,CAAC;AAAA,cAC7E;AAAA,cACA;AAAA,cACA;AAAA,cACA,OAAO,KAAK,KAAK;AAAA,YACnB,CAAC;AAED,6BAAiB;AACjB,uCAA2B;AAAA,UAC7B;AACA;AAAA,QACF,KAAK;AACH,sCAA4B,KAAK,MAAM,MAAM,iBAAiB;AAC9D,eAAK,oBAAoB,QAAQ,OAAO,OAAO;AAC/C;AAAA,QACF,KAAK;AACH,eAAK,oBAAoB,QAAQ,OAAO,OAAO;AAC/C;AAAA,MACJ;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOU,aAAa,OAA0B;AAC/C,QAAI,KAAK,QAAQ;AACf,aAAO;AAAA,IACT;AAEA,QAAI;AACF,WAAK,aAAa,MAAM,KAAK;AAC7B,aAAO;AAAA,IACT,SAAS,GAAG;AACV,YAAM;AAAA,IACR;AAAA,EACF;AAAA,EAEA,kBAAkB,aAAyC;AACzD,SAAK,oBAAoB,UAAU,WAAW;AAAA,EAChD;AAAA,EAEA,oBAAoB;AAClB,SAAK,oBAAoB,aAAa;AAAA,EACxC;AAAA;AAAA,EAGA,UAAU,OAAmB;AAE3B,QAAI,KAAK,aAAa;AACpB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,YAAY,MAAM,KAAK;AAAA,EAC9B;AAAA,EAEA,QAAQ;AACN,QAAI,KAAK,aAAa;AACpB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,YAAY,MAAM,UAAU,cAAc;AAAA,EACjD;AAAA,EAEA,WAAW;AACT,QAAI,KAAK,aAAa;AACpB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,cAAc;AACnB,SAAK,MAAM,SAAS,MAAM;AAAA,EAC5B;AAAA,EAEA,MAAM,OAA0C;AAC9C,WAAO,KAAK,aAAa,KAAK,EAAE,KAAK,CAAC,EAAE,MAAM,MAAM,MAAM;AACxD,UAAI,MAAM;AACR,eAAO,EAAE,MAAM,MAAM,OAAO,OAAU;AAAA,MACxC;AACA,aAAO,EAAE,MAAM,OAAO,MAAM;AAAA,IAC9B,CAAC;AAAA,EACH;AAAA,EAEA,QAAQ;AACN,SAAK,aAAa,YAAY;AAC9B,SAAK,aAAa,OAAO;AACzB,SAAK,OAAO,SAAS,MAAM;AAC3B,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAe;AAClC,WAAO;AAAA,EACT;AACF;","names":["VADEventType"]}
|
package/dist/vad.d.cts
CHANGED
|
@@ -59,6 +59,7 @@ export declare abstract class VAD extends VAD_base {
|
|
|
59
59
|
* Returns a {@link VADStream} that can be used to push audio frames and receive VAD events.
|
|
60
60
|
*/
|
|
61
61
|
abstract stream(): VADStream;
|
|
62
|
+
close(): Promise<void>;
|
|
62
63
|
}
|
|
63
64
|
export declare abstract class VADStream implements AsyncIterableIterator<VADEvent> {
|
|
64
65
|
#private;
|
package/dist/vad.d.ts
CHANGED
|
@@ -59,6 +59,7 @@ export declare abstract class VAD extends VAD_base {
|
|
|
59
59
|
* Returns a {@link VADStream} that can be used to push audio frames and receive VAD events.
|
|
60
60
|
*/
|
|
61
61
|
abstract stream(): VADStream;
|
|
62
|
+
close(): Promise<void>;
|
|
62
63
|
}
|
|
63
64
|
export declare abstract class VADStream implements AsyncIterableIterator<VADEvent> {
|
|
64
65
|
#private;
|
package/dist/vad.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vad.d.ts","sourceRoot":"","sources":["../src/vad.ts"],"names":[],"mappings":";AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,KAAK,EAAE,iBAAiB,IAAI,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAEhF,OAAO,KAAK,EACV,cAAc,EACd,2BAA2B,EAC3B,2BAA2B,EAC5B,MAAM,iBAAiB,CAAC;AAEzB,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAEpD,OAAO,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAC;AAEnE,oBAAY,YAAY;IACtB,eAAe,IAAA;IACf,cAAc,IAAA;IACd,aAAa,IAAA;IACb,iBAAiB,IAAA;CAClB;AAED,MAAM,WAAW,QAAQ;IACvB,oFAAoF;IACpF,IAAI,EAAE,YAAY,CAAC;IACnB;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB,0CAA0C;IAC1C,SAAS,EAAE,MAAM,CAAC;IAClB,sCAAsC;IACtC,cAAc,EAAE,MAAM,CAAC;IACvB,uCAAuC;IACvC,eAAe,EAAE,MAAM,CAAC;IACxB;;;;;;;OAOG;IACH,MAAM,EAAE,UAAU,EAAE,CAAC;IACrB,6EAA6E;IAC7E,WAAW,EAAE,MAAM,CAAC;IACpB,0FAA0F;IAC1F,iBAAiB,EAAE,MAAM,CAAC;IAC1B,2DAA2D;IAC3D,QAAQ,EAAE,OAAO,CAAC;IAClB,wCAAwC;IACxC,qBAAqB,EAAE,MAAM,CAAC;IAC9B,uCAAuC;IACvC,oBAAoB,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,eAAe;IAC9B,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,MAAM,YAAY,GAAG;IACzB,CAAC,mBAAmB,CAAC,EAAE,CAAC,OAAO,EAAE,UAAU,KAAK,IAAI,CAAC;CACtD,CAAC;kCAE2D,aAAa,YAAY,CAAC;AAAvF,8BAAsB,GAAI,SAAQ,QAAsD;;IAEtF,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;gBAEX,YAAY,EAAE,eAAe;IAKzC,IAAI,YAAY,IAAI,eAAe,CAElC;IAED;;OAEG;IACH,QAAQ,CAAC,MAAM,IAAI,SAAS;
|
|
1
|
+
{"version":3,"file":"vad.d.ts","sourceRoot":"","sources":["../src/vad.ts"],"names":[],"mappings":";AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,KAAK,EAAE,iBAAiB,IAAI,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAEhF,OAAO,KAAK,EACV,cAAc,EACd,2BAA2B,EAC3B,2BAA2B,EAC5B,MAAM,iBAAiB,CAAC;AAEzB,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAEpD,OAAO,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAC;AAEnE,oBAAY,YAAY;IACtB,eAAe,IAAA;IACf,cAAc,IAAA;IACd,aAAa,IAAA;IACb,iBAAiB,IAAA;CAClB;AAED,MAAM,WAAW,QAAQ;IACvB,oFAAoF;IACpF,IAAI,EAAE,YAAY,CAAC;IACnB;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB,0CAA0C;IAC1C,SAAS,EAAE,MAAM,CAAC;IAClB,sCAAsC;IACtC,cAAc,EAAE,MAAM,CAAC;IACvB,uCAAuC;IACvC,eAAe,EAAE,MAAM,CAAC;IACxB;;;;;;;OAOG;IACH,MAAM,EAAE,UAAU,EAAE,CAAC;IACrB,6EAA6E;IAC7E,WAAW,EAAE,MAAM,CAAC;IACpB,0FAA0F;IAC1F,iBAAiB,EAAE,MAAM,CAAC;IAC1B,2DAA2D;IAC3D,QAAQ,EAAE,OAAO,CAAC;IAClB,wCAAwC;IACxC,qBAAqB,EAAE,MAAM,CAAC;IAC9B,uCAAuC;IACvC,oBAAoB,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,eAAe;IAC9B,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,MAAM,YAAY,GAAG;IACzB,CAAC,mBAAmB,CAAC,EAAE,CAAC,OAAO,EAAE,UAAU,KAAK,IAAI,CAAC;CACtD,CAAC;kCAE2D,aAAa,YAAY,CAAC;AAAvF,8BAAsB,GAAI,SAAQ,QAAsD;;IAEtF,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;gBAEX,YAAY,EAAE,eAAe;IAKzC,IAAI,YAAY,IAAI,eAAe,CAElC;IAED;;OAEG;IACH,QAAQ,CAAC,MAAM,IAAI,SAAS;IAEtB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B;AAED,8BAAsB,SAAU,YAAW,qBAAqB,CAAC,QAAQ,CAAC;;IACxE,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,gBAA4B;IACpE,SAAS,CAAC,KAAK,kEAAyE;IACxF,SAAS,CAAC,MAAM,8BAAqC;IACrD,SAAS,CAAC,WAAW,EAAE,2BAA2B,CAAC,UAAU,GAAG,OAAO,SAAS,CAAC,cAAc,CAAC,CAAC;IACjG,SAAS,CAAC,WAAW,EAAE,2BAA2B,CAAC,UAAU,GAAG,OAAO,SAAS,CAAC,cAAc,CAAC,CAAC;IACjG,SAAS,CAAC,YAAY,EAAE,2BAA2B,CAAC,QAAQ,CAAC,CAAC;IAC9D,SAAS,CAAC,YAAY,EAAE,2BAA2B,CAAC,QAAQ,CAAC,CAAC;IAC9D,SAAS,CAAC,MAAM,UAAS;IACzB,SAAS,CAAC,WAAW,UAAS;IAI9B,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,mBAAmB,CAAqC;IAEhE,OAAO,CAAC,aAAa,CAA2B;gBACpC,GAAG,EAAE,GAAG;IAgBpB;;;;;;OAMG;YACW,kBAAkB;cAgBhB,cAAc;IAuC9B;;;;OAIG;IACH,SAAS,CAAC,YAAY,CAAC,KAAK,EAAE,QAAQ,GAAG,OAAO;IAahD,iBAAiB,CAAC,WAAW,EAAE,cAAc,CAAC,UAAU,CAAC;IAIzD,iBAAiB;IAIjB,kDAAkD;IAClD,SAAS,CAAC,KAAK,EAAE,UAAU;IAW3B,KAAK;IAUL,QAAQ;IAWF,IAAI,IAAI,OAAO,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;IAS/C,KAAK;IAOL,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,SAAS;CAGpC"}
|
package/dist/vad.js
CHANGED
package/dist/vad.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/vad.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport type {\n ReadableStream,\n ReadableStreamDefaultReader,\n WritableStreamDefaultWriter,\n} from 'node:stream/web';\nimport { log } from './log.js';\nimport type { VADMetrics } from './metrics/base.js';\nimport { DeferredReadableStream } from './stream/deferred_stream.js';\nimport { IdentityTransform } from './stream/identity_transform.js';\n\nexport enum VADEventType {\n START_OF_SPEECH,\n INFERENCE_DONE,\n END_OF_SPEECH,\n METRICS_COLLECTED,\n}\n\nexport interface VADEvent {\n /** Type of the VAD event (e.g., start of speech, end of speech, inference done). */\n type: VADEventType;\n /**\n * Index of the audio sample where the event occurred, relative to the inference sample rate.\n */\n samplesIndex: number;\n /** Timestamp when the event was fired. */\n timestamp: number;\n /** Duration of the speech segment. */\n speechDuration: number;\n /** Duration of the silence segment. */\n silenceDuration: number;\n /**\n * List of audio frames associated with the speech.\n *\n * @remarks\n * - For `start_of_speech` events, this contains the audio chunks that triggered the detection.\n * - For `inference_done` events, this contains the audio chunks that were processed.\n * - For `end_of_speech` events, this contains the complete user speech.\n */\n frames: AudioFrame[];\n /** Probability that speech is present (only for `INFERENCE_DONE` events). */\n probability: number;\n /** Time taken to perform the inference, in seconds (only for `INFERENCE_DONE` events). */\n inferenceDuration: number;\n /** Indicates whether speech was detected in the frames. */\n speaking: boolean;\n /** Threshold used to detect silence. */\n rawAccumulatedSilence: number;\n /** Threshold used to detect speech. */\n rawAccumulatedSpeech: number;\n}\n\nexport interface VADCapabilities {\n updateInterval: number;\n}\n\nexport type VADCallbacks = {\n ['metrics_collected']: (metrics: VADMetrics) => void;\n};\n\nexport abstract class VAD extends (EventEmitter as new () => TypedEmitter<VADCallbacks>) {\n #capabilities: VADCapabilities;\n abstract label: string;\n\n constructor(capabilities: VADCapabilities) {\n super();\n this.#capabilities = capabilities;\n }\n\n get capabilities(): VADCapabilities {\n return this.#capabilities;\n }\n\n /**\n * Returns a {@link VADStream} that can be used to push audio frames and receive VAD events.\n */\n abstract stream(): VADStream;\n}\n\nexport abstract class VADStream implements AsyncIterableIterator<VADEvent> {\n protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');\n protected input = new IdentityTransform<AudioFrame | typeof VADStream.FLUSH_SENTINEL>();\n protected output = new IdentityTransform<VADEvent>();\n protected inputWriter: WritableStreamDefaultWriter<AudioFrame | typeof VADStream.FLUSH_SENTINEL>;\n protected inputReader: ReadableStreamDefaultReader<AudioFrame | typeof VADStream.FLUSH_SENTINEL>;\n protected outputWriter: WritableStreamDefaultWriter<VADEvent>;\n protected outputReader: ReadableStreamDefaultReader<VADEvent>;\n protected closed = false;\n protected inputClosed = false;\n\n #vad: VAD;\n #lastActivityTime = BigInt(0);\n private logger = log();\n private deferredInputStream: DeferredReadableStream<AudioFrame>;\n\n private metricsStream: ReadableStream<VADEvent>;\n constructor(vad: VAD) {\n this.#vad = vad;\n this.deferredInputStream = new DeferredReadableStream<AudioFrame>();\n\n this.inputWriter = this.input.writable.getWriter();\n this.inputReader = this.input.readable.getReader();\n this.outputWriter = this.output.writable.getWriter();\n\n const [outputStream, metricsStream] = this.output.readable.tee();\n this.metricsStream = metricsStream;\n this.outputReader = outputStream.getReader();\n\n this.pumpDeferredStream();\n this.monitorMetrics();\n }\n\n /**\n * Reads from the deferred input stream and forwards chunks to the input writer.\n *\n * Note: we can't just do this.deferredInputStream.stream.pipeTo(this.input.writable)\n * because the inputWriter locks the this.input.writable stream. All writes must go through\n * the inputWriter.\n */\n private async pumpDeferredStream() {\n const reader = this.deferredInputStream.stream.getReader();\n try {\n while (true) {\n const { done, value } = await reader.read();\n if (done) break;\n await this.inputWriter.write(value);\n }\n } catch (e) {\n this.logger.error(`Error pumping deferred stream: ${e}`);\n throw e;\n } finally {\n reader.releaseLock();\n }\n }\n\n protected async monitorMetrics() {\n let inferenceDurationTotalMs = 0;\n let inferenceCount = 0;\n const metricsReader = this.metricsStream.getReader();\n while (true) {\n const { done, value } = await metricsReader.read();\n if (done) {\n break;\n }\n switch (value.type) {\n case VADEventType.START_OF_SPEECH:\n inferenceCount++;\n if (inferenceCount >= 1 / this.#vad.capabilities.updateInterval) {\n this.#vad.emit('metrics_collected', {\n type: 'vad_metrics',\n timestamp: Date.now(),\n idleTimeMs: Math.trunc(\n Number((process.hrtime.bigint() - this.#lastActivityTime) / BigInt(1000000)),\n ),\n inferenceDurationTotalMs,\n inferenceCount,\n label: this.#vad.label,\n });\n\n inferenceCount = 0;\n inferenceDurationTotalMs = 0;\n }\n break;\n case VADEventType.INFERENCE_DONE:\n inferenceDurationTotalMs += Math.round(value.inferenceDuration);\n this.#lastActivityTime = process.hrtime.bigint();\n break;\n case VADEventType.END_OF_SPEECH:\n this.#lastActivityTime = process.hrtime.bigint();\n break;\n }\n }\n }\n\n /**\n * Safely send a VAD event to the output stream, handling writer release errors during shutdown.\n * @returns true if the event was sent, false if the stream is closing\n * @throws Error if an unexpected error occurs\n */\n protected sendVADEvent(event: VADEvent): boolean {\n if (this.closed) {\n return false;\n }\n\n try {\n this.outputWriter.write(event);\n return true;\n } catch (e) {\n throw e;\n }\n }\n\n updateInputStream(audioStream: ReadableStream<AudioFrame>) {\n this.deferredInputStream.setSource(audioStream);\n }\n\n detachInputStream() {\n this.deferredInputStream.detachSource();\n }\n\n /** @deprecated Use `updateInputStream` instead */\n pushFrame(frame: AudioFrame) {\n // TODO(AJS-395): remove this method\n if (this.inputClosed) {\n throw new Error('Input is closed');\n }\n if (this.closed) {\n throw new Error('Stream is closed');\n }\n this.inputWriter.write(frame);\n }\n\n flush() {\n if (this.inputClosed) {\n throw new Error('Input is closed');\n }\n if (this.closed) {\n throw new Error('Stream is closed');\n }\n this.inputWriter.write(VADStream.FLUSH_SENTINEL);\n }\n\n endInput() {\n if (this.inputClosed) {\n throw new Error('Input is closed');\n }\n if (this.closed) {\n throw new Error('Stream is closed');\n }\n this.inputClosed = true;\n this.input.writable.close();\n }\n\n async next(): Promise<IteratorResult<VADEvent>> {\n return this.outputReader.read().then(({ done, value }) => {\n if (done) {\n return { done: true, value: undefined };\n }\n return { done: false, value };\n });\n }\n\n close() {\n this.outputWriter.releaseLock();\n this.outputReader.cancel();\n this.output.writable.close();\n this.closed = true;\n }\n\n [Symbol.asyncIterator](): VADStream {\n return this;\n }\n}\n"],"mappings":"AAKA,SAAS,oBAAoB;AAM7B,SAAS,WAAW;AAEpB,SAAS,8BAA8B;AACvC,SAAS,yBAAyB;AAE3B,IAAK,eAAL,kBAAKA,kBAAL;AACL,EAAAA,4BAAA;AACA,EAAAA,4BAAA;AACA,EAAAA,4BAAA;AACA,EAAAA,4BAAA;AAJU,SAAAA;AAAA,GAAA;AAiDL,MAAe,YAAa,aAAsD;AAAA,EACvF;AAAA,EAGA,YAAY,cAA+B;AACzC,UAAM;AACN,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,IAAI,eAAgC;AAClC,WAAO,KAAK;AAAA,EACd;AAMF;AAEO,MAAe,UAAqD;AAAA,EACzE,OAA0B,iBAAiB,OAAO,gBAAgB;AAAA,EACxD,QAAQ,IAAI,kBAAgE;AAAA,EAC5E,SAAS,IAAI,kBAA4B;AAAA,EACzC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,SAAS;AAAA,EACT,cAAc;AAAA,EAExB;AAAA,EACA,oBAAoB,OAAO,CAAC;AAAA,EACpB,SAAS,IAAI;AAAA,EACb;AAAA,EAEA;AAAA,EACR,YAAY,KAAU;AACpB,SAAK,OAAO;AACZ,SAAK,sBAAsB,IAAI,uBAAmC;AAElE,SAAK,cAAc,KAAK,MAAM,SAAS,UAAU;AACjD,SAAK,cAAc,KAAK,MAAM,SAAS,UAAU;AACjD,SAAK,eAAe,KAAK,OAAO,SAAS,UAAU;AAEnD,UAAM,CAAC,cAAc,aAAa,IAAI,KAAK,OAAO,SAAS,IAAI;AAC/D,SAAK,gBAAgB;AACrB,SAAK,eAAe,aAAa,UAAU;AAE3C,SAAK,mBAAmB;AACxB,SAAK,eAAe;AAAA,EACtB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAc,qBAAqB;AACjC,UAAM,SAAS,KAAK,oBAAoB,OAAO,UAAU;AACzD,QAAI;AACF,aAAO,MAAM;AACX,cAAM,EAAE,MAAM,MAAM,IAAI,MAAM,OAAO,KAAK;AAC1C,YAAI,KAAM;AACV,cAAM,KAAK,YAAY,MAAM,KAAK;AAAA,MACpC;AAAA,IACF,SAAS,GAAG;AACV,WAAK,OAAO,MAAM,kCAAkC,CAAC,EAAE;AACvD,YAAM;AAAA,IACR,UAAE;AACA,aAAO,YAAY;AAAA,IACrB;AAAA,EACF;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,QAAI,2BAA2B;AAC/B,QAAI,iBAAiB;AACrB,UAAM,gBAAgB,KAAK,cAAc,UAAU;AACnD,WAAO,MAAM;AACX,YAAM,EAAE,MAAM,MAAM,IAAI,MAAM,cAAc,KAAK;AACjD,UAAI,MAAM;AACR;AAAA,MACF;AACA,cAAQ,MAAM,MAAM;AAAA,QAClB,KAAK;AACH;AACA,cAAI,kBAAkB,IAAI,KAAK,KAAK,aAAa,gBAAgB;AAC/D,iBAAK,KAAK,KAAK,qBAAqB;AAAA,cAClC,MAAM;AAAA,cACN,WAAW,KAAK,IAAI;AAAA,cACpB,YAAY,KAAK;AAAA,gBACf,QAAQ,QAAQ,OAAO,OAAO,IAAI,KAAK,qBAAqB,OAAO,GAAO,CAAC;AAAA,cAC7E;AAAA,cACA;AAAA,cACA;AAAA,cACA,OAAO,KAAK,KAAK;AAAA,YACnB,CAAC;AAED,6BAAiB;AACjB,uCAA2B;AAAA,UAC7B;AACA;AAAA,QACF,KAAK;AACH,sCAA4B,KAAK,MAAM,MAAM,iBAAiB;AAC9D,eAAK,oBAAoB,QAAQ,OAAO,OAAO;AAC/C;AAAA,QACF,KAAK;AACH,eAAK,oBAAoB,QAAQ,OAAO,OAAO;AAC/C;AAAA,MACJ;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOU,aAAa,OAA0B;AAC/C,QAAI,KAAK,QAAQ;AACf,aAAO;AAAA,IACT;AAEA,QAAI;AACF,WAAK,aAAa,MAAM,KAAK;AAC7B,aAAO;AAAA,IACT,SAAS,GAAG;AACV,YAAM;AAAA,IACR;AAAA,EACF;AAAA,EAEA,kBAAkB,aAAyC;AACzD,SAAK,oBAAoB,UAAU,WAAW;AAAA,EAChD;AAAA,EAEA,oBAAoB;AAClB,SAAK,oBAAoB,aAAa;AAAA,EACxC;AAAA;AAAA,EAGA,UAAU,OAAmB;AAE3B,QAAI,KAAK,aAAa;AACpB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,YAAY,MAAM,KAAK;AAAA,EAC9B;AAAA,EAEA,QAAQ;AACN,QAAI,KAAK,aAAa;AACpB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,YAAY,MAAM,UAAU,cAAc;AAAA,EACjD;AAAA,EAEA,WAAW;AACT,QAAI,KAAK,aAAa;AACpB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,cAAc;AACnB,SAAK,MAAM,SAAS,MAAM;AAAA,EAC5B;AAAA,EAEA,MAAM,OAA0C;AAC9C,WAAO,KAAK,aAAa,KAAK,EAAE,KAAK,CAAC,EAAE,MAAM,MAAM,MAAM;AACxD,UAAI,MAAM;AACR,eAAO,EAAE,MAAM,MAAM,OAAO,OAAU;AAAA,MACxC;AACA,aAAO,EAAE,MAAM,OAAO,MAAM;AAAA,IAC9B,CAAC;AAAA,EACH;AAAA,EAEA,QAAQ;AACN,SAAK,aAAa,YAAY;AAC9B,SAAK,aAAa,OAAO;AACzB,SAAK,OAAO,SAAS,MAAM;AAC3B,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAe;AAClC,WAAO;AAAA,EACT;AACF;","names":["VADEventType"]}
|
|
1
|
+
{"version":3,"sources":["../src/vad.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport type {\n ReadableStream,\n ReadableStreamDefaultReader,\n WritableStreamDefaultWriter,\n} from 'node:stream/web';\nimport { log } from './log.js';\nimport type { VADMetrics } from './metrics/base.js';\nimport { DeferredReadableStream } from './stream/deferred_stream.js';\nimport { IdentityTransform } from './stream/identity_transform.js';\n\nexport enum VADEventType {\n START_OF_SPEECH,\n INFERENCE_DONE,\n END_OF_SPEECH,\n METRICS_COLLECTED,\n}\n\nexport interface VADEvent {\n /** Type of the VAD event (e.g., start of speech, end of speech, inference done). */\n type: VADEventType;\n /**\n * Index of the audio sample where the event occurred, relative to the inference sample rate.\n */\n samplesIndex: number;\n /** Timestamp when the event was fired. */\n timestamp: number;\n /** Duration of the speech segment. */\n speechDuration: number;\n /** Duration of the silence segment. */\n silenceDuration: number;\n /**\n * List of audio frames associated with the speech.\n *\n * @remarks\n * - For `start_of_speech` events, this contains the audio chunks that triggered the detection.\n * - For `inference_done` events, this contains the audio chunks that were processed.\n * - For `end_of_speech` events, this contains the complete user speech.\n */\n frames: AudioFrame[];\n /** Probability that speech is present (only for `INFERENCE_DONE` events). */\n probability: number;\n /** Time taken to perform the inference, in seconds (only for `INFERENCE_DONE` events). */\n inferenceDuration: number;\n /** Indicates whether speech was detected in the frames. */\n speaking: boolean;\n /** Threshold used to detect silence. */\n rawAccumulatedSilence: number;\n /** Threshold used to detect speech. */\n rawAccumulatedSpeech: number;\n}\n\nexport interface VADCapabilities {\n updateInterval: number;\n}\n\nexport type VADCallbacks = {\n ['metrics_collected']: (metrics: VADMetrics) => void;\n};\n\nexport abstract class VAD extends (EventEmitter as new () => TypedEmitter<VADCallbacks>) {\n #capabilities: VADCapabilities;\n abstract label: string;\n\n constructor(capabilities: VADCapabilities) {\n super();\n this.#capabilities = capabilities;\n }\n\n get capabilities(): VADCapabilities {\n return this.#capabilities;\n }\n\n /**\n * Returns a {@link VADStream} that can be used to push audio frames and receive VAD events.\n */\n abstract stream(): VADStream;\n\n async close(): Promise<void> {\n return;\n }\n}\n\nexport abstract class VADStream implements AsyncIterableIterator<VADEvent> {\n protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');\n protected input = new IdentityTransform<AudioFrame | typeof VADStream.FLUSH_SENTINEL>();\n protected output = new IdentityTransform<VADEvent>();\n protected inputWriter: WritableStreamDefaultWriter<AudioFrame | typeof VADStream.FLUSH_SENTINEL>;\n protected inputReader: ReadableStreamDefaultReader<AudioFrame | typeof VADStream.FLUSH_SENTINEL>;\n protected outputWriter: WritableStreamDefaultWriter<VADEvent>;\n protected outputReader: ReadableStreamDefaultReader<VADEvent>;\n protected closed = false;\n protected inputClosed = false;\n\n #vad: VAD;\n #lastActivityTime = BigInt(0);\n private logger = log();\n private deferredInputStream: DeferredReadableStream<AudioFrame>;\n\n private metricsStream: ReadableStream<VADEvent>;\n constructor(vad: VAD) {\n this.#vad = vad;\n this.deferredInputStream = new DeferredReadableStream<AudioFrame>();\n\n this.inputWriter = this.input.writable.getWriter();\n this.inputReader = this.input.readable.getReader();\n this.outputWriter = this.output.writable.getWriter();\n\n const [outputStream, metricsStream] = this.output.readable.tee();\n this.metricsStream = metricsStream;\n this.outputReader = outputStream.getReader();\n\n this.pumpDeferredStream();\n this.monitorMetrics();\n }\n\n /**\n * Reads from the deferred input stream and forwards chunks to the input writer.\n *\n * Note: we can't just do this.deferredInputStream.stream.pipeTo(this.input.writable)\n * because the inputWriter locks the this.input.writable stream. All writes must go through\n * the inputWriter.\n */\n private async pumpDeferredStream() {\n const reader = this.deferredInputStream.stream.getReader();\n try {\n while (true) {\n const { done, value } = await reader.read();\n if (done) break;\n await this.inputWriter.write(value);\n }\n } catch (e) {\n this.logger.error(`Error pumping deferred stream: ${e}`);\n throw e;\n } finally {\n reader.releaseLock();\n }\n }\n\n protected async monitorMetrics() {\n let inferenceDurationTotalMs = 0;\n let inferenceCount = 0;\n const metricsReader = this.metricsStream.getReader();\n while (true) {\n const { done, value } = await metricsReader.read();\n if (done) {\n break;\n }\n switch (value.type) {\n case VADEventType.START_OF_SPEECH:\n inferenceCount++;\n if (inferenceCount >= 1 / this.#vad.capabilities.updateInterval) {\n this.#vad.emit('metrics_collected', {\n type: 'vad_metrics',\n timestamp: Date.now(),\n idleTimeMs: Math.trunc(\n Number((process.hrtime.bigint() - this.#lastActivityTime) / BigInt(1000000)),\n ),\n inferenceDurationTotalMs,\n inferenceCount,\n label: this.#vad.label,\n });\n\n inferenceCount = 0;\n inferenceDurationTotalMs = 0;\n }\n break;\n case VADEventType.INFERENCE_DONE:\n inferenceDurationTotalMs += Math.round(value.inferenceDuration);\n this.#lastActivityTime = process.hrtime.bigint();\n break;\n case VADEventType.END_OF_SPEECH:\n this.#lastActivityTime = process.hrtime.bigint();\n break;\n }\n }\n }\n\n /**\n * Safely send a VAD event to the output stream, handling writer release errors during shutdown.\n * @returns true if the event was sent, false if the stream is closing\n * @throws Error if an unexpected error occurs\n */\n protected sendVADEvent(event: VADEvent): boolean {\n if (this.closed) {\n return false;\n }\n\n try {\n this.outputWriter.write(event);\n return true;\n } catch (e) {\n throw e;\n }\n }\n\n updateInputStream(audioStream: ReadableStream<AudioFrame>) {\n this.deferredInputStream.setSource(audioStream);\n }\n\n detachInputStream() {\n this.deferredInputStream.detachSource();\n }\n\n /** @deprecated Use `updateInputStream` instead */\n pushFrame(frame: AudioFrame) {\n // TODO(AJS-395): remove this method\n if (this.inputClosed) {\n throw new Error('Input is closed');\n }\n if (this.closed) {\n throw new Error('Stream is closed');\n }\n this.inputWriter.write(frame);\n }\n\n flush() {\n if (this.inputClosed) {\n throw new Error('Input is closed');\n }\n if (this.closed) {\n throw new Error('Stream is closed');\n }\n this.inputWriter.write(VADStream.FLUSH_SENTINEL);\n }\n\n endInput() {\n if (this.inputClosed) {\n throw new Error('Input is closed');\n }\n if (this.closed) {\n throw new Error('Stream is closed');\n }\n this.inputClosed = true;\n this.input.writable.close();\n }\n\n async next(): Promise<IteratorResult<VADEvent>> {\n return this.outputReader.read().then(({ done, value }) => {\n if (done) {\n return { done: true, value: undefined };\n }\n return { done: false, value };\n });\n }\n\n close() {\n this.outputWriter.releaseLock();\n this.outputReader.cancel();\n this.output.writable.close();\n this.closed = true;\n }\n\n [Symbol.asyncIterator](): VADStream {\n return this;\n }\n}\n"],"mappings":"AAKA,SAAS,oBAAoB;AAM7B,SAAS,WAAW;AAEpB,SAAS,8BAA8B;AACvC,SAAS,yBAAyB;AAE3B,IAAK,eAAL,kBAAKA,kBAAL;AACL,EAAAA,4BAAA;AACA,EAAAA,4BAAA;AACA,EAAAA,4BAAA;AACA,EAAAA,4BAAA;AAJU,SAAAA;AAAA,GAAA;AAiDL,MAAe,YAAa,aAAsD;AAAA,EACvF;AAAA,EAGA,YAAY,cAA+B;AACzC,UAAM;AACN,SAAK,gBAAgB;AAAA,EACvB;AAAA,EAEA,IAAI,eAAgC;AAClC,WAAO,KAAK;AAAA,EACd;AAAA,EAOA,MAAM,QAAuB;AAC3B;AAAA,EACF;AACF;AAEO,MAAe,UAAqD;AAAA,EACzE,OAA0B,iBAAiB,OAAO,gBAAgB;AAAA,EACxD,QAAQ,IAAI,kBAAgE;AAAA,EAC5E,SAAS,IAAI,kBAA4B;AAAA,EACzC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,SAAS;AAAA,EACT,cAAc;AAAA,EAExB;AAAA,EACA,oBAAoB,OAAO,CAAC;AAAA,EACpB,SAAS,IAAI;AAAA,EACb;AAAA,EAEA;AAAA,EACR,YAAY,KAAU;AACpB,SAAK,OAAO;AACZ,SAAK,sBAAsB,IAAI,uBAAmC;AAElE,SAAK,cAAc,KAAK,MAAM,SAAS,UAAU;AACjD,SAAK,cAAc,KAAK,MAAM,SAAS,UAAU;AACjD,SAAK,eAAe,KAAK,OAAO,SAAS,UAAU;AAEnD,UAAM,CAAC,cAAc,aAAa,IAAI,KAAK,OAAO,SAAS,IAAI;AAC/D,SAAK,gBAAgB;AACrB,SAAK,eAAe,aAAa,UAAU;AAE3C,SAAK,mBAAmB;AACxB,SAAK,eAAe;AAAA,EACtB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA,MAAc,qBAAqB;AACjC,UAAM,SAAS,KAAK,oBAAoB,OAAO,UAAU;AACzD,QAAI;AACF,aAAO,MAAM;AACX,cAAM,EAAE,MAAM,MAAM,IAAI,MAAM,OAAO,KAAK;AAC1C,YAAI,KAAM;AACV,cAAM,KAAK,YAAY,MAAM,KAAK;AAAA,MACpC;AAAA,IACF,SAAS,GAAG;AACV,WAAK,OAAO,MAAM,kCAAkC,CAAC,EAAE;AACvD,YAAM;AAAA,IACR,UAAE;AACA,aAAO,YAAY;AAAA,IACrB;AAAA,EACF;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,QAAI,2BAA2B;AAC/B,QAAI,iBAAiB;AACrB,UAAM,gBAAgB,KAAK,cAAc,UAAU;AACnD,WAAO,MAAM;AACX,YAAM,EAAE,MAAM,MAAM,IAAI,MAAM,cAAc,KAAK;AACjD,UAAI,MAAM;AACR;AAAA,MACF;AACA,cAAQ,MAAM,MAAM;AAAA,QAClB,KAAK;AACH;AACA,cAAI,kBAAkB,IAAI,KAAK,KAAK,aAAa,gBAAgB;AAC/D,iBAAK,KAAK,KAAK,qBAAqB;AAAA,cAClC,MAAM;AAAA,cACN,WAAW,KAAK,IAAI;AAAA,cACpB,YAAY,KAAK;AAAA,gBACf,QAAQ,QAAQ,OAAO,OAAO,IAAI,KAAK,qBAAqB,OAAO,GAAO,CAAC;AAAA,cAC7E;AAAA,cACA;AAAA,cACA;AAAA,cACA,OAAO,KAAK,KAAK;AAAA,YACnB,CAAC;AAED,6BAAiB;AACjB,uCAA2B;AAAA,UAC7B;AACA;AAAA,QACF,KAAK;AACH,sCAA4B,KAAK,MAAM,MAAM,iBAAiB;AAC9D,eAAK,oBAAoB,QAAQ,OAAO,OAAO;AAC/C;AAAA,QACF,KAAK;AACH,eAAK,oBAAoB,QAAQ,OAAO,OAAO;AAC/C;AAAA,MACJ;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOU,aAAa,OAA0B;AAC/C,QAAI,KAAK,QAAQ;AACf,aAAO;AAAA,IACT;AAEA,QAAI;AACF,WAAK,aAAa,MAAM,KAAK;AAC7B,aAAO;AAAA,IACT,SAAS,GAAG;AACV,YAAM;AAAA,IACR;AAAA,EACF;AAAA,EAEA,kBAAkB,aAAyC;AACzD,SAAK,oBAAoB,UAAU,WAAW;AAAA,EAChD;AAAA,EAEA,oBAAoB;AAClB,SAAK,oBAAoB,aAAa;AAAA,EACxC;AAAA;AAAA,EAGA,UAAU,OAAmB;AAE3B,QAAI,KAAK,aAAa;AACpB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,YAAY,MAAM,KAAK;AAAA,EAC9B;AAAA,EAEA,QAAQ;AACN,QAAI,KAAK,aAAa;AACpB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,YAAY,MAAM,UAAU,cAAc;AAAA,EACjD;AAAA,EAEA,WAAW;AACT,QAAI,KAAK,aAAa;AACpB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,cAAc;AACnB,SAAK,MAAM,SAAS,MAAM;AAAA,EAC5B;AAAA,EAEA,MAAM,OAA0C;AAC9C,WAAO,KAAK,aAAa,KAAK,EAAE,KAAK,CAAC,EAAE,MAAM,MAAM,MAAM;AACxD,UAAI,MAAM;AACR,eAAO,EAAE,MAAM,MAAM,OAAO,OAAU;AAAA,MACxC;AACA,aAAO,EAAE,MAAM,OAAO,MAAM;AAAA,IAC9B,CAAC;AAAA,EACH;AAAA,EAEA,QAAQ;AACN,SAAK,aAAa,YAAY;AAC9B,SAAK,aAAa,OAAO;AACzB,SAAK,OAAO,SAAS,MAAM;AAC3B,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAe;AAClC,WAAO;AAAA,EACT;AACF;","names":["VADEventType"]}
|
|
@@ -1601,12 +1601,15 @@ ${instructions}` : instructions,
|
|
|
1601
1601
|
}
|
|
1602
1602
|
if (this.stt instanceof import_stt.STT) {
|
|
1603
1603
|
this.stt.off("metrics_collected", this.onMetricsCollected);
|
|
1604
|
+
await this.stt.close();
|
|
1604
1605
|
}
|
|
1605
1606
|
if (this.tts instanceof import_tts.TTS) {
|
|
1606
1607
|
this.tts.off("metrics_collected", this.onMetricsCollected);
|
|
1608
|
+
await this.tts.close();
|
|
1607
1609
|
}
|
|
1608
1610
|
if (this.vad instanceof import_vad.VAD) {
|
|
1609
1611
|
this.vad.off("metrics_collected", this.onMetricsCollected);
|
|
1612
|
+
await this.vad.close();
|
|
1610
1613
|
}
|
|
1611
1614
|
this.detachAudioInput();
|
|
1612
1615
|
await ((_a = this.realtimeSession) == null ? void 0 : _a.close());
|