@livekit/agents 1.0.40 → 1.0.41

package/dist/voice/transcription/synchronizer.test.cjs

@@ -0,0 +1,151 @@
+"use strict";
+var import_vitest = require("vitest");
+var import_synchronizer = require("./synchronizer.cjs");
+(0, import_vitest.describe)("SpeakingRateData", () => {
+  (0, import_vitest.describe)("constructor", () => {
+    (0, import_vitest.it)("should initialize with empty arrays", () => {
+      const data = new import_synchronizer.SpeakingRateData();
+      (0, import_vitest.expect)(data.timestamps).toEqual([]);
+      (0, import_vitest.expect)(data.speakingRate).toEqual([]);
+      (0, import_vitest.expect)(data.speakIntegrals).toEqual([]);
+      (0, import_vitest.expect)(data.pushedDuration).toBe(0);
+    });
+  });
+  (0, import_vitest.describe)("addByRate", () => {
+    (0, import_vitest.it)("should add a single rate entry", () => {
+      const data = new import_synchronizer.SpeakingRateData();
+      data.addByRate(1, 5);
+      (0, import_vitest.expect)(data.timestamps).toEqual([1]);
+      (0, import_vitest.expect)(data.speakingRate).toEqual([5]);
+      (0, import_vitest.expect)(data.speakIntegrals).toEqual([5]);
+      (0, import_vitest.expect)(data.pushedDuration).toBe(1);
+    });
+    (0, import_vitest.it)("should accumulate integrals across multiple entries", () => {
+      const data = new import_synchronizer.SpeakingRateData();
+      data.addByRate(1, 4);
+      data.addByRate(2, 6);
+      data.addByRate(3.5, 2);
+      (0, import_vitest.expect)(data.timestamps).toEqual([1, 2, 3.5]);
+      (0, import_vitest.expect)(data.speakingRate).toEqual([4, 6, 2]);
+      (0, import_vitest.expect)(data.speakIntegrals).toEqual([4, 10, 13]);
+      (0, import_vitest.expect)(data.pushedDuration).toBe(3.5);
+    });
+    (0, import_vitest.it)("should handle zero rate", () => {
+      const data = new import_synchronizer.SpeakingRateData();
+      data.addByRate(1, 0);
+      (0, import_vitest.expect)(data.timestamps).toEqual([1]);
+      (0, import_vitest.expect)(data.speakingRate).toEqual([0]);
+      (0, import_vitest.expect)(data.speakIntegrals).toEqual([0]);
+    });
+  });
+  (0, import_vitest.describe)("addByAnnotation", () => {
+    (0, import_vitest.it)("should buffer text without startTime", () => {
+      const data = new import_synchronizer.SpeakingRateData();
+      data.addByAnnotation("hello", void 0, void 0);
+      (0, import_vitest.expect)(data.timestamps).toEqual([]);
+      (0, import_vitest.expect)(data.pushedDuration).toBe(0);
+    });
+    (0, import_vitest.it)("should add entry when startTime is provided", () => {
+      const data = new import_synchronizer.SpeakingRateData();
+      data.addByAnnotation("hello", void 0, void 0);
+      data.addByAnnotation("world", 1, void 0);
+      (0, import_vitest.expect)(data.timestamps).toEqual([1]);
+      (0, import_vitest.expect)(data.speakingRate).toEqual([5]);
+      (0, import_vitest.expect)(data.speakIntegrals).toEqual([5]);
+    });
+    (0, import_vitest.it)("should handle startTime and endTime together", () => {
+      const data = new import_synchronizer.SpeakingRateData();
+      data.addByAnnotation("hello ", 0, 0.5);
+      data.addByAnnotation("world", 0.5, 1);
+      (0, import_vitest.expect)(data.timestamps.length).toBeGreaterThanOrEqual(2);
+      (0, import_vitest.expect)(data.pushedDuration).toBe(1);
+    });
+    (0, import_vitest.it)("should calculate rate based on buffered text length", () => {
+      const data = new import_synchronizer.SpeakingRateData();
+      data.addByAnnotation("ab", void 0, void 0);
+      data.addByAnnotation("cde", void 0, void 0);
+      data.addByAnnotation("", 2, void 0);
+      (0, import_vitest.expect)(data.timestamps).toEqual([2]);
+      (0, import_vitest.expect)(data.speakingRate).toEqual([2.5]);
+      (0, import_vitest.expect)(data.speakIntegrals).toEqual([5]);
+    });
+    (0, import_vitest.it)("should handle zero time delta gracefully", () => {
+      const data = new import_synchronizer.SpeakingRateData();
+      data.addByAnnotation("hello", 0, void 0);
+      (0, import_vitest.expect)(data.timestamps).toEqual([0]);
+      (0, import_vitest.expect)(data.speakingRate).toEqual([0]);
+      (0, import_vitest.expect)(data.speakIntegrals).toEqual([0]);
+    });
+  });
+  (0, import_vitest.describe)("accumulateTo", () => {
+    (0, import_vitest.it)("should return 0 for empty data", () => {
+      const data = new import_synchronizer.SpeakingRateData();
+      (0, import_vitest.expect)(data.accumulateTo(1)).toBe(0);
+    });
+    (0, import_vitest.it)("should return 0 for timestamp before first entry", () => {
+      const data = new import_synchronizer.SpeakingRateData();
+      data.addByRate(1, 5);
+      (0, import_vitest.expect)(data.accumulateTo(0.5)).toBe(0);
+    });
+    (0, import_vitest.it)("should return exact integral at timestamp", () => {
+      const data = new import_synchronizer.SpeakingRateData();
+      data.addByRate(1, 4);
+      data.addByRate(2, 6);
+      (0, import_vitest.expect)(data.accumulateTo(1)).toBe(4);
+      (0, import_vitest.expect)(data.accumulateTo(2)).toBe(10);
+    });
+    (0, import_vitest.it)("should interpolate between timestamps", () => {
+      const data = new import_synchronizer.SpeakingRateData();
+      data.addByRate(1, 4);
+      data.addByRate(2, 6);
+      (0, import_vitest.expect)(data.accumulateTo(1.5)).toBe(7);
+    });
+    (0, import_vitest.it)("should extrapolate beyond last timestamp", () => {
+      const data = new import_synchronizer.SpeakingRateData();
+      data.addByRate(1, 4);
+      data.addByRate(2, 6);
+      (0, import_vitest.expect)(data.accumulateTo(3)).toBe(16);
+    });
+    (0, import_vitest.it)("should not exceed next integral when interpolating", () => {
+      const data = new import_synchronizer.SpeakingRateData();
+      data.addByRate(1, 100);
+      data.addByRate(2, 1);
+      (0, import_vitest.expect)(data.accumulateTo(1.5)).toBe(100.5);
+    });
+  });
+  (0, import_vitest.describe)("pushedDuration", () => {
+    (0, import_vitest.it)("should return 0 when empty", () => {
+      const data = new import_synchronizer.SpeakingRateData();
+      (0, import_vitest.expect)(data.pushedDuration).toBe(0);
+    });
+    (0, import_vitest.it)("should return last timestamp", () => {
+      const data = new import_synchronizer.SpeakingRateData();
+      data.addByRate(1, 5);
+      data.addByRate(2.5, 3);
+      data.addByRate(4, 7);
+      (0, import_vitest.expect)(data.pushedDuration).toBe(4);
+    });
+  });
+  (0, import_vitest.describe)("integration scenarios", () => {
+    (0, import_vitest.it)("should handle typical TTS word timing scenario", () => {
+      const data = new import_synchronizer.SpeakingRateData();
+      data.addByAnnotation("Hello ", 0, 0.3);
+      data.addByAnnotation("world", 0.3, 0.6);
+      (0, import_vitest.expect)(data.pushedDuration).toBe(0.6);
+      const mid1 = data.accumulateTo(0.15);
+      (0, import_vitest.expect)(mid1).toBeGreaterThan(0);
+      (0, import_vitest.expect)(mid1).toBeLessThan(6);
+      const mid2 = data.accumulateTo(0.45);
+      (0, import_vitest.expect)(mid2).toBeGreaterThan(6);
+    });
+    (0, import_vitest.it)("should handle mixed rate and annotation data", () => {
+      const data = new import_synchronizer.SpeakingRateData();
+      data.addByRate(0.5, 4);
+      data.addByAnnotation("test", void 0, void 0);
+      data.addByAnnotation("", 1, void 0);
+      (0, import_vitest.expect)(data.timestamps).toEqual([0.5, 1]);
+      (0, import_vitest.expect)(data.speakIntegrals).toEqual([2, 6]);
+    });
+  });
+});
+//# sourceMappingURL=synchronizer.test.cjs.map

package/dist/voice/transcription/synchronizer.test.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/voice/transcription/synchronizer.test.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { describe, expect, it } from 'vitest';\nimport { SpeakingRateData } from './synchronizer.js';\n\ndescribe('SpeakingRateData', () => {\n  describe('constructor', () => {\n    it('should initialize with empty arrays', () => {\n      const data = new SpeakingRateData();\n      expect(data.timestamps).toEqual([]);\n      expect(data.speakingRate).toEqual([]);\n      expect(data.speakIntegrals).toEqual([]);\n      expect(data.pushedDuration).toBe(0);\n    });\n  });\n\n  describe('addByRate', () => {\n    it('should add a single rate entry', () => {\n      const data = new SpeakingRateData();\n      data.addByRate(1.0, 5.0);\n\n      expect(data.timestamps).toEqual([1.0]);\n      expect(data.speakingRate).toEqual([5.0]);\n      // integral = 0 + 5.0 * (1.0 - 0) = 5.0\n      expect(data.speakIntegrals).toEqual([5.0]);\n      expect(data.pushedDuration).toBe(1.0);\n    });\n\n    it('should accumulate integrals across multiple entries', () => {\n      const data = new SpeakingRateData();\n      data.addByRate(1.0, 4.0); // integral = 0 + 4.0 * 1.0 = 4.0\n      data.addByRate(2.0, 6.0); // integral = 4.0 + 6.0 * 1.0 = 10.0\n      data.addByRate(3.5, 2.0); // integral = 10.0 + 2.0 * 1.5 = 13.0\n\n      expect(data.timestamps).toEqual([1.0, 2.0, 3.5]);\n      expect(data.speakingRate).toEqual([4.0, 6.0, 2.0]);\n      expect(data.speakIntegrals).toEqual([4.0, 10.0, 13.0]);\n      expect(data.pushedDuration).toBe(3.5);\n    });\n\n    it('should handle zero rate', () => {\n      const data = new SpeakingRateData();\n      data.addByRate(1.0, 0.0);\n\n      expect(data.timestamps).toEqual([1.0]);\n      expect(data.speakingRate).toEqual([0.0]);\n      expect(data.speakIntegrals).toEqual([0.0]);\n    });\n  });\n\n  describe('addByAnnotation', () => {\n    it('should buffer text without startTime', () => {\n      const data = new SpeakingRateData();\n      data.addByAnnotation('hello', undefined, undefined);\n\n      // Text is buffered, no timestamp entry yet\n      expect(data.timestamps).toEqual([]);\n      expect(data.pushedDuration).toBe(0);\n    });\n\n    it('should add entry when startTime is provided', () => {\n      const data = new SpeakingRateData();\n      data.addByAnnotation('hello', undefined, undefined); // buffer \"hello\"\n      data.addByAnnotation('world', 1.0, undefined); // flush with startTime\n\n      expect(data.timestamps).toEqual([1.0]);\n      // textLen = 5 (hello), dt = 1.0, rate = 5/1 = 5.0\n      expect(data.speakingRate).toEqual([5.0]);\n      expect(data.speakIntegrals).toEqual([5.0]);\n    });\n\n    it('should handle startTime and endTime together', () => {\n      const data = new SpeakingRateData();\n      data.addByAnnotation('hello ', 0.0, 0.5);\n      data.addByAnnotation('world', 0.5, 1.0);\n\n      // First annotation: startTime=0.0, text=\"hello \", then recursively calls with endTime=0.5\n      // Second annotation: startTime=0.5, text=\"world\", then recursively calls with endTime=1.0\n      expect(data.timestamps.length).toBeGreaterThanOrEqual(2);\n      expect(data.pushedDuration).toBe(1.0);\n    });\n\n    it('should calculate rate based on buffered text length', () => {\n      const data = new SpeakingRateData();\n      data.addByAnnotation('ab', undefined, undefined); // buffer 2 chars\n      data.addByAnnotation('cde', undefined, undefined); // buffer 3 more chars\n      data.addByAnnotation('', 2.0, undefined); // flush: textLen=5, dt=2.0, rate=2.5\n\n      expect(data.timestamps).toEqual([2.0]);\n      expect(data.speakingRate).toEqual([2.5]);\n      expect(data.speakIntegrals).toEqual([5.0]);\n    });\n\n    it('should handle zero time delta gracefully', () => {\n      const data = new SpeakingRateData();\n      data.addByAnnotation('hello', 0.0, undefined); // dt=0, rate should be 0\n\n      expect(data.timestamps).toEqual([0.0]);\n      expect(data.speakingRate).toEqual([0.0]);\n      expect(data.speakIntegrals).toEqual([0.0]);\n    });\n  });\n\n  describe('accumulateTo', () => {\n    it('should return 0 for empty data', () => {\n      const data = new SpeakingRateData();\n      expect(data.accumulateTo(1.0)).toBe(0);\n    });\n\n    it('should return 0 for timestamp before first entry', () => {\n      const data = new SpeakingRateData();\n      data.addByRate(1.0, 5.0);\n      expect(data.accumulateTo(0.5)).toBe(0);\n    });\n\n    it('should return exact integral at timestamp', () => {\n      const data = new SpeakingRateData();\n      data.addByRate(1.0, 4.0); // integral = 4.0\n      data.addByRate(2.0, 6.0); // integral = 10.0\n\n      expect(data.accumulateTo(1.0)).toBe(4.0);\n      expect(data.accumulateTo(2.0)).toBe(10.0);\n    });\n\n    it('should interpolate between timestamps', () => {\n      const data = new SpeakingRateData();\n      data.addByRate(1.0, 4.0); // integral = 4.0\n      data.addByRate(2.0, 6.0); // integral = 10.0\n\n      // At 1.5: integral = 4.0 + 6.0 * 0.5 = 7.0\n      expect(data.accumulateTo(1.5)).toBe(7.0);\n    });\n\n    it('should extrapolate beyond last timestamp', () => {\n      const data = new SpeakingRateData();\n      data.addByRate(1.0, 4.0); // integral = 4.0\n      data.addByRate(2.0, 6.0); // integral = 10.0\n\n      // At 3.0: integral = 10.0 + 6.0 * 1.0 = 16.0\n      expect(data.accumulateTo(3.0)).toBe(16.0);\n    });\n\n    it('should not exceed next integral when interpolating', () => {\n      const data = new SpeakingRateData();\n      data.addByRate(1.0, 100.0); // integral = 100.0 (very high rate)\n      data.addByRate(2.0, 1.0); // integral = 101.0\n\n      // At 1.5 with rate 1.0: would be 100.0 + 1.0 * 0.5 = 100.5\n      // But capped at next integral 101.0, so result is min(100.5, 101.0) = 100.5\n      expect(data.accumulateTo(1.5)).toBe(100.5);\n    });\n  });\n\n  describe('pushedDuration', () => {\n    it('should return 0 when empty', () => {\n      const data = new SpeakingRateData();\n      expect(data.pushedDuration).toBe(0);\n    });\n\n    it('should return last timestamp', () => {\n      const data = new SpeakingRateData();\n      data.addByRate(1.0, 5.0);\n      data.addByRate(2.5, 3.0);\n      data.addByRate(4.0, 7.0);\n\n      expect(data.pushedDuration).toBe(4.0);\n    });\n  });\n\n  describe('integration scenarios', () => {\n    it('should handle typical TTS word timing scenario', () => {\n      const data = new SpeakingRateData();\n\n      // Simulating words with timing: \"Hello \" at 0-0.3s, \"world\" at 0.3-0.6s\n      data.addByAnnotation('Hello ', 0.0, 0.3);\n      data.addByAnnotation('world', 0.3, 0.6);\n\n      // Should have accumulated text lengths at each timestamp\n      expect(data.pushedDuration).toBe(0.6);\n\n      // At 0.15s (middle of first word), should be partway through\n      const mid1 = data.accumulateTo(0.15);\n      expect(mid1).toBeGreaterThan(0);\n      expect(mid1).toBeLessThan(6); // \"Hello \" is 6 chars\n\n      // At 0.45s (middle of second word), should be past first word\n      const mid2 = data.accumulateTo(0.45);\n      expect(mid2).toBeGreaterThan(6);\n    });\n\n    it('should handle mixed rate and annotation data', () => {\n      const data = new SpeakingRateData();\n\n      // Start with rate-based data\n      data.addByRate(0.5, 4.0); // integral = 2.0\n\n      // Then add annotation\n      data.addByAnnotation('test', undefined, undefined);\n      data.addByAnnotation('', 1.0, undefined); // textLen=4, dt=0.5, rate=8.0, integral = 2.0 + 4.0 = 6.0\n\n      expect(data.timestamps).toEqual([0.5, 1.0]);\n      expect(data.speakIntegrals).toEqual([2.0, 6.0]);\n    });\n  });\n});\n"],"mappings":";AAGA,oBAAqC;AACrC,0BAAiC;AAAA,IAEjC,wBAAS,oBAAoB,MAAM;AACjC,8BAAS,eAAe,MAAM;AAC5B,0BAAG,uCAAuC,MAAM;AAC9C,YAAM,OAAO,IAAI,qCAAiB;AAClC,gCAAO,KAAK,UAAU,EAAE,QAAQ,CAAC,CAAC;AAClC,gCAAO,KAAK,YAAY,EAAE,QAAQ,CAAC,CAAC;AACpC,gCAAO,KAAK,cAAc,EAAE,QAAQ,CAAC,CAAC;AACtC,gCAAO,KAAK,cAAc,EAAE,KAAK,CAAC;AAAA,IACpC,CAAC;AAAA,EACH,CAAC;AAED,8BAAS,aAAa,MAAM;AAC1B,0BAAG,kCAAkC,MAAM;AACzC,YAAM,OAAO,IAAI,qCAAiB;AAClC,WAAK,UAAU,GAAK,CAAG;AAEvB,gCAAO,KAAK,UAAU,EAAE,QAAQ,CAAC,CAAG,CAAC;AACrC,gCAAO,KAAK,YAAY,EAAE,QAAQ,CAAC,CAAG,CAAC;AAEvC,gCAAO,KAAK,cAAc,EAAE,QAAQ,CAAC,CAAG,CAAC;AACzC,gCAAO,KAAK,cAAc,EAAE,KAAK,CAAG;AAAA,IACtC,CAAC;AAED,0BAAG,uDAAuD,MAAM;AAC9D,YAAM,OAAO,IAAI,qCAAiB;AAClC,WAAK,UAAU,GAAK,CAAG;AACvB,WAAK,UAAU,GAAK,CAAG;AACvB,WAAK,UAAU,KAAK,CAAG;AAEvB,gCAAO,KAAK,UAAU,EAAE,QAAQ,CAAC,GAAK,GAAK,GAAG,CAAC;AAC/C,gCAAO,KAAK,YAAY,EAAE,QAAQ,CAAC,GAAK,GAAK,CAAG,CAAC;AACjD,gCAAO,KAAK,cAAc,EAAE,QAAQ,CAAC,GAAK,IAAM,EAAI,CAAC;AACrD,gCAAO,KAAK,cAAc,EAAE,KAAK,GAAG;AAAA,IACtC,CAAC;AAED,0BAAG,2BAA2B,MAAM;AAClC,YAAM,OAAO,IAAI,qCAAiB;AAClC,WAAK,UAAU,GAAK,CAAG;AAEvB,gCAAO,KAAK,UAAU,EAAE,QAAQ,CAAC,CAAG,CAAC;AACrC,gCAAO,KAAK,YAAY,EAAE,QAAQ,CAAC,CAAG,CAAC;AACvC,gCAAO,KAAK,cAAc,EAAE,QAAQ,CAAC,CAAG,CAAC;AAAA,IAC3C,CAAC;AAAA,EACH,CAAC;AAED,8BAAS,mBAAmB,MAAM;AAChC,0BAAG,wCAAwC,MAAM;AAC/C,YAAM,OAAO,IAAI,qCAAiB;AAClC,WAAK,gBAAgB,SAAS,QAAW,MAAS;AAGlD,gCAAO,KAAK,UAAU,EAAE,QAAQ,CAAC,CAAC;AAClC,gCAAO,KAAK,cAAc,EAAE,KAAK,CAAC;AAAA,IACpC,CAAC;AAED,0BAAG,+CAA+C,MAAM;AACtD,YAAM,OAAO,IAAI,qCAAiB;AAClC,WAAK,gBAAgB,SAAS,QAAW,MAAS;AAClD,WAAK,gBAAgB,SAAS,GAAK,MAAS;AAE5C,gCAAO,KAAK,UAAU,EAAE,QAAQ,CAAC,CAAG,CAAC;AAErC,gCAAO,KAAK,YAAY,EAAE,QAAQ,CAAC,CAAG,CAAC;AACvC,gCAAO,KAAK,cAAc,EAAE,QAAQ,CAAC,CAAG,CAAC;AAAA,IAC3C,CAAC;AAED,0BAAG,gDAAgD,MAAM;AACvD,YAAM,OAAO,IAAI,qCAAiB;AAClC,WAAK,gBAAgB,UAAU,GAAK,GAAG;AACvC,WAAK,gBAAgB,SAAS,KAAK,CAAG;AAItC,gCAAO,KAAK,WAAW,MAAM,EAAE,uBAAuB,CAAC;AACvD,gCAAO,KAAK,cAAc,EAAE,KAAK,CAAG;AAAA,IACtC,CAAC;AAED,0BAAG,uDAAuD,MAAM;AAC9D,YAAM,OAAO,IAAI,qCAAiB;AAClC,WAAK,gBAAgB,MAAM,QAAW,MAAS;AAC/C,WAAK,gBAAgB,OAAO,QAAW,MAAS;AAChD,WAAK,gBAAgB,IAAI,GAAK,MAAS;AAEvC,gCAAO,KAAK,UAAU,EAAE,QAAQ,CAAC,CAAG,CAAC;AACrC,gCAAO,KAAK,YAAY,EAAE,QAAQ,CAAC,GAAG,CAAC;AACvC,gCAAO,KAAK,cAAc,EAAE,QAAQ,CAAC,CAAG,CAAC;AAAA,IAC3C,CAAC;AAED,0BAAG,4CAA4C,MAAM;AACnD,YAAM,OAAO,IAAI,qCAAiB;AAClC,WAAK,gBAAgB,SAAS,GAAK,MAAS;AAE5C,gCAAO,KAAK,UAAU,EAAE,QAAQ,CAAC,CAAG,CAAC;AACrC,gCAAO,KAAK,YAAY,EAAE,QAAQ,CAAC,CAAG,CAAC;AACvC,gCAAO,KAAK,cAAc,EAAE,QAAQ,CAAC,CAAG,CAAC;AAAA,IAC3C,CAAC;AAAA,EACH,CAAC;AAED,8BAAS,gBAAgB,MAAM;AAC7B,0BAAG,kCAAkC,MAAM;AACzC,YAAM,OAAO,IAAI,qCAAiB;AAClC,gCAAO,KAAK,aAAa,CAAG,CAAC,EAAE,KAAK,CAAC;AAAA,IACvC,CAAC;AAED,0BAAG,oDAAoD,MAAM;AAC3D,YAAM,OAAO,IAAI,qCAAiB;AAClC,WAAK,UAAU,GAAK,CAAG;AACvB,gCAAO,KAAK,aAAa,GAAG,CAAC,EAAE,KAAK,CAAC;AAAA,IACvC,CAAC;AAED,0BAAG,6CAA6C,MAAM;AACpD,YAAM,OAAO,IAAI,qCAAiB;AAClC,WAAK,UAAU,GAAK,CAAG;AACvB,WAAK,UAAU,GAAK,CAAG;AAEvB,gCAAO,KAAK,aAAa,CAAG,CAAC,EAAE,KAAK,CAAG;AACvC,gCAAO,KAAK,aAAa,CAAG,CAAC,EAAE,KAAK,EAAI;AAAA,IAC1C,CAAC;AAED,0BAAG,yCAAyC,MAAM;AAChD,YAAM,OAAO,IAAI,qCAAiB;AAClC,WAAK,UAAU,GAAK,CAAG;AACvB,WAAK,UAAU,GAAK,CAAG;AAGvB,gCAAO,KAAK,aAAa,GAAG,CAAC,EAAE,KAAK,CAAG;AAAA,IACzC,CAAC;AAED,0BAAG,4CAA4C,MAAM;AACnD,YAAM,OAAO,IAAI,qCAAiB;AAClC,WAAK,UAAU,GAAK,CAAG;AACvB,WAAK,UAAU,GAAK,CAAG;AAGvB,gCAAO,KAAK,aAAa,CAAG,CAAC,EAAE,KAAK,EAAI;AAAA,IAC1C,CAAC;AAED,0BAAG,sDAAsD,MAAM;AAC7D,YAAM,OAAO,IAAI,qCAAiB;AAClC,WAAK,UAAU,GAAK,GAAK;AACzB,WAAK,UAAU,GAAK,CAAG;AAIvB,gCAAO,KAAK,aAAa,GAAG,CAAC,EAAE,KAAK,KAAK;AAAA,IAC3C,CAAC;AAAA,EACH,CAAC;AAED,8BAAS,kBAAkB,MAAM;AAC/B,0BAAG,8BAA8B,MAAM;AACrC,YAAM,OAAO,IAAI,qCAAiB;AAClC,gCAAO,KAAK,cAAc,EAAE,KAAK,CAAC;AAAA,IACpC,CAAC;AAED,0BAAG,gCAAgC,MAAM;AACvC,YAAM,OAAO,IAAI,qCAAiB;AAClC,WAAK,UAAU,GAAK,CAAG;AACvB,WAAK,UAAU,KAAK,CAAG;AACvB,WAAK,UAAU,GAAK,CAAG;AAEvB,gCAAO,KAAK,cAAc,EAAE,KAAK,CAAG;AAAA,IACtC,CAAC;AAAA,EACH,CAAC;AAED,8BAAS,yBAAyB,MAAM;AACtC,0BAAG,kDAAkD,MAAM;AACzD,YAAM,OAAO,IAAI,qCAAiB;AAGlC,WAAK,gBAAgB,UAAU,GAAK,GAAG;AACvC,WAAK,gBAAgB,SAAS,KAAK,GAAG;AAGtC,gCAAO,KAAK,cAAc,EAAE,KAAK,GAAG;AAGpC,YAAM,OAAO,KAAK,aAAa,IAAI;AACnC,gCAAO,IAAI,EAAE,gBAAgB,CAAC;AAC9B,gCAAO,IAAI,EAAE,aAAa,CAAC;AAG3B,YAAM,OAAO,KAAK,aAAa,IAAI;AACnC,gCAAO,IAAI,EAAE,gBAAgB,CAAC;AAAA,IAChC,CAAC;AAED,0BAAG,gDAAgD,MAAM;AACvD,YAAM,OAAO,IAAI,qCAAiB;AAGlC,WAAK,UAAU,KAAK,CAAG;AAGvB,WAAK,gBAAgB,QAAQ,QAAW,MAAS;AACjD,WAAK,gBAAgB,IAAI,GAAK,MAAS;AAEvC,gCAAO,KAAK,UAAU,EAAE,QAAQ,CAAC,KAAK,CAAG,CAAC;AAC1C,gCAAO,KAAK,cAAc,EAAE,QAAQ,CAAC,GAAK,CAAG,CAAC;AAAA,IAChD,CAAC;AAAA,EACH,CAAC;AACH,CAAC;","names":[]}

package/dist/voice/transcription/synchronizer.test.js

@@ -0,0 +1,150 @@
+import { describe, expect, it } from "vitest";
+import { SpeakingRateData } from "./synchronizer.js";
+describe("SpeakingRateData", () => {
+  describe("constructor", () => {
+    it("should initialize with empty arrays", () => {
+      const data = new SpeakingRateData();
+      expect(data.timestamps).toEqual([]);
+      expect(data.speakingRate).toEqual([]);
+      expect(data.speakIntegrals).toEqual([]);
+      expect(data.pushedDuration).toBe(0);
+    });
+  });
+  describe("addByRate", () => {
+    it("should add a single rate entry", () => {
+      const data = new SpeakingRateData();
+      data.addByRate(1, 5);
+      expect(data.timestamps).toEqual([1]);
+      expect(data.speakingRate).toEqual([5]);
+      expect(data.speakIntegrals).toEqual([5]);
+      expect(data.pushedDuration).toBe(1);
+    });
+    it("should accumulate integrals across multiple entries", () => {
+      const data = new SpeakingRateData();
+      data.addByRate(1, 4);
+      data.addByRate(2, 6);
+      data.addByRate(3.5, 2);
+      expect(data.timestamps).toEqual([1, 2, 3.5]);
+      expect(data.speakingRate).toEqual([4, 6, 2]);
+      expect(data.speakIntegrals).toEqual([4, 10, 13]);
+      expect(data.pushedDuration).toBe(3.5);
+    });
+    it("should handle zero rate", () => {
+      const data = new SpeakingRateData();
+      data.addByRate(1, 0);
+      expect(data.timestamps).toEqual([1]);
+      expect(data.speakingRate).toEqual([0]);
+      expect(data.speakIntegrals).toEqual([0]);
+    });
+  });
+  describe("addByAnnotation", () => {
+    it("should buffer text without startTime", () => {
+      const data = new SpeakingRateData();
+      data.addByAnnotation("hello", void 0, void 0);
+      expect(data.timestamps).toEqual([]);
+      expect(data.pushedDuration).toBe(0);
+    });
+    it("should add entry when startTime is provided", () => {
+      const data = new SpeakingRateData();
+      data.addByAnnotation("hello", void 0, void 0);
+      data.addByAnnotation("world", 1, void 0);
+      expect(data.timestamps).toEqual([1]);
+      expect(data.speakingRate).toEqual([5]);
+      expect(data.speakIntegrals).toEqual([5]);
+    });
+    it("should handle startTime and endTime together", () => {
+      const data = new SpeakingRateData();
+      data.addByAnnotation("hello ", 0, 0.5);
+      data.addByAnnotation("world", 0.5, 1);
+      expect(data.timestamps.length).toBeGreaterThanOrEqual(2);
+      expect(data.pushedDuration).toBe(1);
+    });
+    it("should calculate rate based on buffered text length", () => {
+      const data = new SpeakingRateData();
+      data.addByAnnotation("ab", void 0, void 0);
+      data.addByAnnotation("cde", void 0, void 0);
+      data.addByAnnotation("", 2, void 0);
+      expect(data.timestamps).toEqual([2]);
+      expect(data.speakingRate).toEqual([2.5]);
+      expect(data.speakIntegrals).toEqual([5]);
+    });
+    it("should handle zero time delta gracefully", () => {
+      const data = new SpeakingRateData();
+      data.addByAnnotation("hello", 0, void 0);
+      expect(data.timestamps).toEqual([0]);
+      expect(data.speakingRate).toEqual([0]);
+      expect(data.speakIntegrals).toEqual([0]);
+    });
+  });
+  describe("accumulateTo", () => {
+    it("should return 0 for empty data", () => {
+      const data = new SpeakingRateData();
+      expect(data.accumulateTo(1)).toBe(0);
+    });
+    it("should return 0 for timestamp before first entry", () => {
+      const data = new SpeakingRateData();
+      data.addByRate(1, 5);
+      expect(data.accumulateTo(0.5)).toBe(0);
+    });
+    it("should return exact integral at timestamp", () => {
+      const data = new SpeakingRateData();
+      data.addByRate(1, 4);
+      data.addByRate(2, 6);
+      expect(data.accumulateTo(1)).toBe(4);
+      expect(data.accumulateTo(2)).toBe(10);
+    });
+    it("should interpolate between timestamps", () => {
+      const data = new SpeakingRateData();
+      data.addByRate(1, 4);
+      data.addByRate(2, 6);
+      expect(data.accumulateTo(1.5)).toBe(7);
+    });
+    it("should extrapolate beyond last timestamp", () => {
+      const data = new SpeakingRateData();
+      data.addByRate(1, 4);
+      data.addByRate(2, 6);
+      expect(data.accumulateTo(3)).toBe(16);
+    });
+    it("should not exceed next integral when interpolating", () => {
+      const data = new SpeakingRateData();
+      data.addByRate(1, 100);
+      data.addByRate(2, 1);
+      expect(data.accumulateTo(1.5)).toBe(100.5);
+    });
+  });
+  describe("pushedDuration", () => {
+    it("should return 0 when empty", () => {
+      const data = new SpeakingRateData();
+      expect(data.pushedDuration).toBe(0);
+    });
+    it("should return last timestamp", () => {
+      const data = new SpeakingRateData();
+      data.addByRate(1, 5);
+      data.addByRate(2.5, 3);
+      data.addByRate(4, 7);
+      expect(data.pushedDuration).toBe(4);
+    });
+  });
+  describe("integration scenarios", () => {
+    it("should handle typical TTS word timing scenario", () => {
+      const data = new SpeakingRateData();
+      data.addByAnnotation("Hello ", 0, 0.3);
+      data.addByAnnotation("world", 0.3, 0.6);
+      expect(data.pushedDuration).toBe(0.6);
+      const mid1 = data.accumulateTo(0.15);
+      expect(mid1).toBeGreaterThan(0);
+      expect(mid1).toBeLessThan(6);
+      const mid2 = data.accumulateTo(0.45);
+      expect(mid2).toBeGreaterThan(6);
+    });
+    it("should handle mixed rate and annotation data", () => {
+      const data = new SpeakingRateData();
+      data.addByRate(0.5, 4);
+      data.addByAnnotation("test", void 0, void 0);
+      data.addByAnnotation("", 1, void 0);
+      expect(data.timestamps).toEqual([0.5, 1]);
+      expect(data.speakIntegrals).toEqual([2, 6]);
+    });
+  });
+});
+//# sourceMappingURL=synchronizer.test.js.map

package/dist/voice/transcription/synchronizer.test.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/voice/transcription/synchronizer.test.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { describe, expect, it } from 'vitest';\nimport { SpeakingRateData } from './synchronizer.js';\n\ndescribe('SpeakingRateData', () => {\n  describe('constructor', () => {\n    it('should initialize with empty arrays', () => {\n      const data = new SpeakingRateData();\n      expect(data.timestamps).toEqual([]);\n      expect(data.speakingRate).toEqual([]);\n      expect(data.speakIntegrals).toEqual([]);\n      expect(data.pushedDuration).toBe(0);\n    });\n  });\n\n  describe('addByRate', () => {\n    it('should add a single rate entry', () => {\n      const data = new SpeakingRateData();\n      data.addByRate(1.0, 5.0);\n\n      expect(data.timestamps).toEqual([1.0]);\n      expect(data.speakingRate).toEqual([5.0]);\n      // integral = 0 + 5.0 * (1.0 - 0) = 5.0\n      expect(data.speakIntegrals).toEqual([5.0]);\n      expect(data.pushedDuration).toBe(1.0);\n    });\n\n    it('should accumulate integrals across multiple entries', () => {\n      const data = new SpeakingRateData();\n      data.addByRate(1.0, 4.0); // integral = 0 + 4.0 * 1.0 = 4.0\n      data.addByRate(2.0, 6.0); // integral = 4.0 + 6.0 * 1.0 = 10.0\n      data.addByRate(3.5, 2.0); // integral = 10.0 + 2.0 * 1.5 = 13.0\n\n      expect(data.timestamps).toEqual([1.0, 2.0, 3.5]);\n      expect(data.speakingRate).toEqual([4.0, 6.0, 2.0]);\n      expect(data.speakIntegrals).toEqual([4.0, 10.0, 13.0]);\n      expect(data.pushedDuration).toBe(3.5);\n    });\n\n    it('should handle zero rate', () => {\n      const data = new SpeakingRateData();\n      data.addByRate(1.0, 0.0);\n\n      expect(data.timestamps).toEqual([1.0]);\n      expect(data.speakingRate).toEqual([0.0]);\n      expect(data.speakIntegrals).toEqual([0.0]);\n    });\n  });\n\n  describe('addByAnnotation', () => {\n    it('should buffer text without startTime', () => {\n      const data = new SpeakingRateData();\n      data.addByAnnotation('hello', undefined, undefined);\n\n      // Text is buffered, no timestamp entry yet\n      expect(data.timestamps).toEqual([]);\n      expect(data.pushedDuration).toBe(0);\n    });\n\n    it('should add entry when startTime is provided', () => {\n      const data = new SpeakingRateData();\n      data.addByAnnotation('hello', undefined, undefined); // buffer \"hello\"\n      data.addByAnnotation('world', 1.0, undefined); // flush with startTime\n\n      expect(data.timestamps).toEqual([1.0]);\n      // textLen = 5 (hello), dt = 1.0, rate = 5/1 = 5.0\n      expect(data.speakingRate).toEqual([5.0]);\n      expect(data.speakIntegrals).toEqual([5.0]);\n    });\n\n    it('should handle startTime and endTime together', () => {\n      const data = new SpeakingRateData();\n      data.addByAnnotation('hello ', 0.0, 0.5);\n      data.addByAnnotation('world', 0.5, 1.0);\n\n      // First annotation: startTime=0.0, text=\"hello \", then recursively calls with endTime=0.5\n      // Second annotation: startTime=0.5, text=\"world\", then recursively calls with endTime=1.0\n      expect(data.timestamps.length).toBeGreaterThanOrEqual(2);\n      expect(data.pushedDuration).toBe(1.0);\n    });\n\n    it('should calculate rate based on buffered text length', () => {\n      const data = new SpeakingRateData();\n      data.addByAnnotation('ab', undefined, undefined); // buffer 2 chars\n      data.addByAnnotation('cde', undefined, undefined); // buffer 3 more chars\n      data.addByAnnotation('', 2.0, undefined); // flush: textLen=5, dt=2.0, rate=2.5\n\n      expect(data.timestamps).toEqual([2.0]);\n      expect(data.speakingRate).toEqual([2.5]);\n      expect(data.speakIntegrals).toEqual([5.0]);\n    });\n\n    it('should handle zero time delta gracefully', () => {\n      const data = new SpeakingRateData();\n      data.addByAnnotation('hello', 0.0, undefined); // dt=0, rate should be 0\n\n      expect(data.timestamps).toEqual([0.0]);\n      expect(data.speakingRate).toEqual([0.0]);\n      expect(data.speakIntegrals).toEqual([0.0]);\n    });\n  });\n\n  describe('accumulateTo', () => {\n    it('should return 0 for empty data', () => {\n      const data = new SpeakingRateData();\n      expect(data.accumulateTo(1.0)).toBe(0);\n    });\n\n    it('should return 0 for timestamp before first entry', () => {\n      const data = new SpeakingRateData();\n      data.addByRate(1.0, 5.0);\n      expect(data.accumulateTo(0.5)).toBe(0);\n    });\n\n    it('should return exact integral at timestamp', () => {\n      const data = new SpeakingRateData();\n      data.addByRate(1.0, 4.0); // integral = 4.0\n      data.addByRate(2.0, 6.0); // integral = 10.0\n\n      expect(data.accumulateTo(1.0)).toBe(4.0);\n      expect(data.accumulateTo(2.0)).toBe(10.0);\n    });\n\n    it('should interpolate between timestamps', () => {\n      const data = new SpeakingRateData();\n      data.addByRate(1.0, 4.0); // integral = 4.0\n      data.addByRate(2.0, 6.0); // integral = 10.0\n\n      // At 1.5: integral = 4.0 + 6.0 * 0.5 = 7.0\n      expect(data.accumulateTo(1.5)).toBe(7.0);\n    });\n\n    it('should extrapolate beyond last timestamp', () => {\n      const data = new SpeakingRateData();\n      data.addByRate(1.0, 4.0); // integral = 4.0\n      data.addByRate(2.0, 6.0); // integral = 10.0\n\n      // At 3.0: integral = 10.0 + 6.0 * 1.0 = 16.0\n      expect(data.accumulateTo(3.0)).toBe(16.0);\n    });\n\n    it('should not exceed next integral when interpolating', () => {\n      const data = new SpeakingRateData();\n      data.addByRate(1.0, 100.0); // integral = 100.0 (very high rate)\n      data.addByRate(2.0, 1.0); // integral = 101.0\n\n      // At 1.5 with rate 1.0: would be 100.0 + 1.0 * 0.5 = 100.5\n      // But capped at next integral 101.0, so result is min(100.5, 101.0) = 100.5\n      expect(data.accumulateTo(1.5)).toBe(100.5);\n    });\n  });\n\n  describe('pushedDuration', () => {\n    it('should return 0 when empty', () => {\n      const data = new SpeakingRateData();\n      expect(data.pushedDuration).toBe(0);\n    });\n\n    it('should return last timestamp', () => {\n      const data = new SpeakingRateData();\n      data.addByRate(1.0, 5.0);\n      data.addByRate(2.5, 3.0);\n      data.addByRate(4.0, 7.0);\n\n      expect(data.pushedDuration).toBe(4.0);\n    });\n  });\n\n  describe('integration scenarios', () => {\n    it('should handle typical TTS word timing scenario', () => {\n      const data = new SpeakingRateData();\n\n      // Simulating words with timing: \"Hello \" at 0-0.3s, \"world\" at 0.3-0.6s\n      data.addByAnnotation('Hello ', 0.0, 0.3);\n      data.addByAnnotation('world', 0.3, 0.6);\n\n      // Should have accumulated text lengths at each timestamp\n      expect(data.pushedDuration).toBe(0.6);\n\n      // At 0.15s (middle of first word), should be partway through\n      const mid1 = data.accumulateTo(0.15);\n      expect(mid1).toBeGreaterThan(0);\n      expect(mid1).toBeLessThan(6); // \"Hello \" is 6 chars\n\n      // At 0.45s (middle of second word), should be past first word\n      const mid2 = data.accumulateTo(0.45);\n      expect(mid2).toBeGreaterThan(6);\n    });\n\n    it('should handle mixed rate and annotation data', () => {\n      const data = new SpeakingRateData();\n\n      // Start with rate-based data\n      data.addByRate(0.5, 4.0); // integral = 2.0\n\n      // Then add annotation\n      data.addByAnnotation('test', undefined, undefined);\n      data.addByAnnotation('', 1.0, undefined); // textLen=4, dt=0.5, rate=8.0, integral = 2.0 + 4.0 = 6.0\n\n      expect(data.timestamps).toEqual([0.5, 1.0]);\n      expect(data.speakIntegrals).toEqual([2.0, 6.0]);\n    });\n  });\n});\n"],"mappings":"AAGA,SAAS,UAAU,QAAQ,UAAU;AACrC,SAAS,wBAAwB;AAEjC,SAAS,oBAAoB,MAAM;AACjC,WAAS,eAAe,MAAM;AAC5B,OAAG,uCAAuC,MAAM;AAC9C,YAAM,OAAO,IAAI,iBAAiB;AAClC,aAAO,KAAK,UAAU,EAAE,QAAQ,CAAC,CAAC;AAClC,aAAO,KAAK,YAAY,EAAE,QAAQ,CAAC,CAAC;AACpC,aAAO,KAAK,cAAc,EAAE,QAAQ,CAAC,CAAC;AACtC,aAAO,KAAK,cAAc,EAAE,KAAK,CAAC;AAAA,IACpC,CAAC;AAAA,EACH,CAAC;AAED,WAAS,aAAa,MAAM;AAC1B,OAAG,kCAAkC,MAAM;AACzC,YAAM,OAAO,IAAI,iBAAiB;AAClC,WAAK,UAAU,GAAK,CAAG;AAEvB,aAAO,KAAK,UAAU,EAAE,QAAQ,CAAC,CAAG,CAAC;AACrC,aAAO,KAAK,YAAY,EAAE,QAAQ,CAAC,CAAG,CAAC;AAEvC,aAAO,KAAK,cAAc,EAAE,QAAQ,CAAC,CAAG,CAAC;AACzC,aAAO,KAAK,cAAc,EAAE,KAAK,CAAG;AAAA,IACtC,CAAC;AAED,OAAG,uDAAuD,MAAM;AAC9D,YAAM,OAAO,IAAI,iBAAiB;AAClC,WAAK,UAAU,GAAK,CAAG;AACvB,WAAK,UAAU,GAAK,CAAG;AACvB,WAAK,UAAU,KAAK,CAAG;AAEvB,aAAO,KAAK,UAAU,EAAE,QAAQ,CAAC,GAAK,GAAK,GAAG,CAAC;AAC/C,aAAO,KAAK,YAAY,EAAE,QAAQ,CAAC,GAAK,GAAK,CAAG,CAAC;AACjD,aAAO,KAAK,cAAc,EAAE,QAAQ,CAAC,GAAK,IAAM,EAAI,CAAC;AACrD,aAAO,KAAK,cAAc,EAAE,KAAK,GAAG;AAAA,IACtC,CAAC;AAED,OAAG,2BAA2B,MAAM;AAClC,YAAM,OAAO,IAAI,iBAAiB;AAClC,WAAK,UAAU,GAAK,CAAG;AAEvB,aAAO,KAAK,UAAU,EAAE,QAAQ,CAAC,CAAG,CAAC;AACrC,aAAO,KAAK,YAAY,EAAE,QAAQ,CAAC,CAAG,CAAC;AACvC,aAAO,KAAK,cAAc,EAAE,QAAQ,CAAC,CAAG,CAAC;AAAA,IAC3C,CAAC;AAAA,EACH,CAAC;AAED,WAAS,mBAAmB,MAAM;AAChC,OAAG,wCAAwC,MAAM;AAC/C,YAAM,OAAO,IAAI,iBAAiB;AAClC,WAAK,gBAAgB,SAAS,QAAW,MAAS;AAGlD,aAAO,KAAK,UAAU,EAAE,QAAQ,CAAC,CAAC;AAClC,aAAO,KAAK,cAAc,EAAE,KAAK,CAAC;AAAA,IACpC,CAAC;AAED,OAAG,+CAA+C,MAAM;AACtD,YAAM,OAAO,IAAI,iBAAiB;AAClC,WAAK,gBAAgB,SAAS,QAAW,MAAS;AAClD,WAAK,gBAAgB,SAAS,GAAK,MAAS;AAE5C,aAAO,KAAK,UAAU,EAAE,QAAQ,CAAC,CAAG,CAAC;AAErC,aAAO,KAAK,YAAY,EAAE,QAAQ,CAAC,CAAG,CAAC;AACvC,aAAO,KAAK,cAAc,EAAE,QAAQ,CAAC,CAAG,CAAC;AAAA,IAC3C,CAAC;AAED,OAAG,gDAAgD,MAAM;AACvD,YAAM,OAAO,IAAI,iBAAiB;AAClC,WAAK,gBAAgB,UAAU,GAAK,GAAG;AACvC,WAAK,gBAAgB,SAAS,KAAK,CAAG;AAItC,aAAO,KAAK,WAAW,MAAM,EAAE,uBAAuB,CAAC;AACvD,aAAO,KAAK,cAAc,EAAE,KAAK,CAAG;AAAA,IACtC,CAAC;AAED,OAAG,uDAAuD,MAAM;AAC9D,YAAM,OAAO,IAAI,iBAAiB;AAClC,WAAK,gBAAgB,MAAM,QAAW,MAAS;AAC/C,WAAK,gBAAgB,OAAO,QAAW,MAAS;AAChD,WAAK,gBAAgB,IAAI,GAAK,MAAS;AAEvC,aAAO,KAAK,UAAU,EAAE,QAAQ,CAAC,CAAG,CAAC;AACrC,aAAO,KAAK,YAAY,EAAE,QAAQ,CAAC,GAAG,CAAC;AACvC,aAAO,KAAK,cAAc,EAAE,QAAQ,CAAC,CAAG,CAAC;AAAA,IAC3C,CAAC;AAED,OAAG,4CAA4C,MAAM;AACnD,YAAM,OAAO,IAAI,iBAAiB;AAClC,WAAK,gBAAgB,SAAS,GAAK,MAAS;AAE5C,aAAO,KAAK,UAAU,EAAE,QAAQ,CAAC,CAAG,CAAC;AACrC,aAAO,KAAK,YAAY,EAAE,QAAQ,CAAC,CAAG,CAAC;AACvC,aAAO,KAAK,cAAc,EAAE,QAAQ,CAAC,CAAG,CAAC;AAAA,IAC3C,CAAC;AAAA,EACH,CAAC;AAED,WAAS,gBAAgB,MAAM;AAC7B,OAAG,kCAAkC,MAAM;AACzC,YAAM,OAAO,IAAI,iBAAiB;AAClC,aAAO,KAAK,aAAa,CAAG,CAAC,EAAE,KAAK,CAAC;AAAA,IACvC,CAAC;AAED,OAAG,oDAAoD,MAAM;AAC3D,YAAM,OAAO,IAAI,iBAAiB;AAClC,WAAK,UAAU,GAAK,CAAG;AACvB,aAAO,KAAK,aAAa,GAAG,CAAC,EAAE,KAAK,CAAC;AAAA,IACvC,CAAC;AAED,OAAG,6CAA6C,MAAM;AACpD,YAAM,OAAO,IAAI,iBAAiB;AAClC,WAAK,UAAU,GAAK,CAAG;AACvB,WAAK,UAAU,GAAK,CAAG;AAEvB,aAAO,KAAK,aAAa,CAAG,CAAC,EAAE,KAAK,CAAG;AACvC,aAAO,KAAK,aAAa,CAAG,CAAC,EAAE,KAAK,EAAI;AAAA,IAC1C,CAAC;AAED,OAAG,yCAAyC,MAAM;AAChD,YAAM,OAAO,IAAI,iBAAiB;AAClC,WAAK,UAAU,GAAK,CAAG;AACvB,WAAK,UAAU,GAAK,CAAG;AAGvB,aAAO,KAAK,aAAa,GAAG,CAAC,EAAE,KAAK,CAAG;AAAA,IACzC,CAAC;AAED,OAAG,4CAA4C,MAAM;AACnD,YAAM,OAAO,IAAI,iBAAiB;AAClC,WAAK,UAAU,GAAK,CAAG;AACvB,WAAK,UAAU,GAAK,CAAG;AAGvB,aAAO,KAAK,aAAa,CAAG,CAAC,EAAE,KAAK,EAAI;AAAA,IAC1C,CAAC;AAED,OAAG,sDAAsD,MAAM;AAC7D,YAAM,OAAO,IAAI,iBAAiB;AAClC,WAAK,UAAU,GAAK,GAAK;AACzB,WAAK,UAAU,GAAK,CAAG;AAIvB,aAAO,KAAK,aAAa,GAAG,CAAC,EAAE,KAAK,KAAK;AAAA,IAC3C,CAAC;AAAA,EACH,CAAC;AAED,WAAS,kBAAkB,MAAM;AAC/B,OAAG,8BAA8B,MAAM;AACrC,YAAM,OAAO,IAAI,iBAAiB;AAClC,aAAO,KAAK,cAAc,EAAE,KAAK,CAAC;AAAA,IACpC,CAAC;AAED,OAAG,gCAAgC,MAAM;AACvC,YAAM,OAAO,IAAI,iBAAiB;AAClC,WAAK,UAAU,GAAK,CAAG;AACvB,WAAK,UAAU,KAAK,CAAG;AACvB,WAAK,UAAU,GAAK,CAAG;AAEvB,aAAO,KAAK,cAAc,EAAE,KAAK,CAAG;AAAA,IACtC,CAAC;AAAA,EACH,CAAC;AAED,WAAS,yBAAyB,MAAM;AACtC,OAAG,kDAAkD,MAAM;AACzD,YAAM,OAAO,IAAI,iBAAiB;AAGlC,WAAK,gBAAgB,UAAU,GAAK,GAAG;AACvC,WAAK,gBAAgB,SAAS,KAAK,GAAG;AAGtC,aAAO,KAAK,cAAc,EAAE,KAAK,GAAG;AAGpC,YAAM,OAAO,KAAK,aAAa,IAAI;AACnC,aAAO,IAAI,EAAE,gBAAgB,CAAC;AAC9B,aAAO,IAAI,EAAE,aAAa,CAAC;AAG3B,YAAM,OAAO,KAAK,aAAa,IAAI;AACnC,aAAO,IAAI,EAAE,gBAAgB,CAAC;AAAA,IAChC,CAAC;AAED,OAAG,gDAAgD,MAAM;AACvD,YAAM,OAAO,IAAI,iBAAiB;AAGlC,WAAK,UAAU,KAAK,CAAG;AAGvB,WAAK,gBAAgB,QAAQ,QAAW,MAAS;AACjD,WAAK,gBAAgB,IAAI,GAAK,MAAS;AAEvC,aAAO,KAAK,UAAU,EAAE,QAAQ,CAAC,KAAK,CAAG,CAAC;AAC1C,aAAO,KAAK,cAAc,EAAE,QAAQ,CAAC,GAAK,CAAG,CAAC;AAAA,IAChD,CAAC;AAAA,EACH,CAAC;AACH,CAAC;","names":[]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@livekit/agents",
-  "version": "1.0.40",
+  "version": "1.0.41",
   "description": "LiveKit Agents - Node.js",
   "main": "dist/index.js",
   "require": "dist/index.cjs",

package/src/cli.ts

@@ -143,13 +143,14 @@ export const runApp = (opts: ServerOptions) => {
         .default('debug')
         .env('LOG_LEVEL'),
     )
-    .action(() => {
-      const options = program.optsWithGlobals();
-      opts.wsURL = options.url || opts.wsURL;
-      opts.apiKey = options.apiKey || opts.apiKey;
-      opts.apiSecret = options.apiSecret || opts.apiSecret;
-      opts.logLevel = options.logLevel || opts.logLevel;
-      opts.workerToken = options.workerToken || opts.workerToken;
+    .action((...[, command]) => {
+      const globalOptions = program.optsWithGlobals();
+      const commandOptions = command.opts();
+      opts.wsURL = globalOptions.url || opts.wsURL;
+      opts.apiKey = globalOptions.apiKey || opts.apiKey;
+      opts.apiSecret = globalOptions.apiSecret || opts.apiSecret;
+      opts.logLevel = commandOptions.logLevel || globalOptions.logLevel || opts.logLevel;
+      opts.workerToken = globalOptions.workerToken || opts.workerToken;
       runServer({
         opts,
         production: false,
@@ -169,18 +170,19 @@ export const runApp = (opts: ServerOptions) => {
         .env('LOG_LEVEL'),
     )
     .action((...[, command]) => {
-      const options = command.optsWithGlobals();
-      opts.wsURL = options.url || opts.wsURL;
-      opts.apiKey = options.apiKey || opts.apiKey;
-      opts.apiSecret = options.apiSecret || opts.apiSecret;
-      opts.logLevel = options.logLevel || opts.logLevel;
-      opts.workerToken = options.workerToken || opts.workerToken;
+      const globalOptions = program.optsWithGlobals();
+      const commandOptions = command.opts();
+      opts.wsURL = globalOptions.url || opts.wsURL;
+      opts.apiKey = globalOptions.apiKey || opts.apiKey;
+      opts.apiSecret = globalOptions.apiSecret || opts.apiSecret;
+      opts.logLevel = commandOptions.logLevel || globalOptions.logLevel || opts.logLevel;
+      opts.workerToken = globalOptions.workerToken || opts.workerToken;
       runServer({
         opts,
         production: false,
         watch: false,
-        room: options.room,
-        participantIdentity: options.participantIdentity,
+        room: commandOptions.room,
+        participantIdentity: commandOptions.participantIdentity,
       });
     });
 
@@ -193,9 +195,9 @@ export const runApp = (opts: ServerOptions) => {
         .default('debug')
         .env('LOG_LEVEL'),
     )
-    .action(() => {
-      const options = program.optsWithGlobals();
-      initializeLogger({ pretty: true, level: options.logLevel });
+    .action((...[, command]) => {
+      const commandOptions = command.opts();
+      initializeLogger({ pretty: true, level: commandOptions.logLevel });
       const logger = log();
 
       const downloadFiles = async () => {

package/src/index.ts

@@ -34,6 +34,7 @@ export * from './types.js';
 export * from './utils.js';
 export * from './vad.js';
 export * from './version.js';
+export { createTimedString, isTimedString, type TimedString } from './voice/io.js';
 export * from './worker.js';
 
 export { cli, inference, ipc, llm, metrics, stream, stt, telemetry, tokenize, tts, voice };

package/src/inference/stt.ts

@@ -16,7 +16,7 @@ import {
 } from '../stt/index.js';
 import { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js';
 import { type AudioBuffer, Event, Task, cancelAndWait, shortuuid, waitForAbort } from '../utils.js';
-import type { TimedString } from '../voice/io.js';
+import { type TimedString, createTimedString } from '../voice/io.js';
 import {
   type SttServerEvent,
   type SttTranscriptEvent,
@@ -491,13 +491,14 @@ export class SpeechStream<TModel extends STTModels> extends BaseSpeechStream {
         confidence: data.confidence,
         text,
         words: data.words.map(
-          (word): TimedString => ({
-            text: word.word,
-            startTime: word.start + this.startTimeOffset,
-            endTime: word.end + this.startTimeOffset,
-            startTimeOffset: this.startTimeOffset,
-            confidence: word.confidence,
-          }),
+          (word): TimedString =>
+            createTimedString({
+              text: word.word,
+              startTime: word.start + this.startTimeOffset,
+              endTime: word.end + this.startTimeOffset,
+              startTimeOffset: this.startTimeOffset,
+              confidence: word.confidence,
+            }),
         ),
       };
 

package/src/llm/realtime.ts

@@ -6,6 +6,7 @@ import { EventEmitter } from 'events';
 import type { ReadableStream } from 'node:stream/web';
 import { DeferredReadableStream } from '../stream/deferred_stream.js';
 import { Task } from '../utils.js';
+import type { TimedString } from '../voice/io.js';
 import type { ChatContext, FunctionCall } from './chat_context.js';
 import type { ToolChoice, ToolContext } from './tool_context.js';
 
@@ -17,7 +18,10 @@ export interface InputSpeechStoppedEvent {
 
 export interface MessageGeneration {
   messageId: string;
-  textStream: ReadableStream<string>;
+  /**
+   * Text stream that may contain plain strings or TimedString objects with timestamps.
+   */
+  textStream: ReadableStream<string | TimedString>;
   audioStream: ReadableStream<AudioFrame>;
   modalities?: Promise<('text' | 'audio')[]>;
 }

package/src/tts/stream_adapter.ts

@@ -3,7 +3,9 @@
 // SPDX-License-Identifier: Apache-2.0
 import type { SentenceStream, SentenceTokenizer } from '../tokenize/index.js';
 import type { APIConnectOptions } from '../types.js';
+import { USERDATA_TIMED_TRANSCRIPT } from '../types.js';
 import { Task } from '../utils.js';
+import { createTimedString } from '../voice/io.js';
 import type { ChunkedStream } from './tts.js';
 import { SynthesizeStream, TTS } from './tts.js';
 
@@ -13,7 +15,7 @@ export class StreamAdapter extends TTS {
   label: string;
 
   constructor(tts: TTS, sentenceTokenizer: SentenceTokenizer) {
-    super(tts.sampleRate, tts.numChannels, { streaming: true });
+    super(tts.sampleRate, tts.numChannels, { streaming: true, alignedTranscript: true });
     this.#tts = tts;
     this.#sentenceTokenizer = sentenceTokenizer;
     this.label = this.#tts.label;
@@ -53,6 +55,8 @@ export class StreamAdapterWrapper extends SynthesizeStream {
   }
 
   protected async run() {
+    let cumulativeDuration = 0;
+
     const forwardInput = async () => {
       for await (const input of this.input) {
         if (this.abortController.signal.aborted) break;
@@ -99,8 +103,26 @@ export class StreamAdapterWrapper extends SynthesizeStream {
       await prevTask?.result;
       if (controller.signal.aborted) return;
 
+      // Create a TimedString with the sentence text and current cumulative duration
+      const timedString = createTimedString({
+        text: token,
+        startTime: cumulativeDuration,
+      });
+
+      let isFirstFrame = true;
       for await (const audio of audioStream) {
         if (controller.signal.aborted) break;
+
+        // Attach the TimedString to the first frame of this sentence
+        if (isFirstFrame) {
+          audio.frame.userdata[USERDATA_TIMED_TRANSCRIPT] = [timedString];
+          isFirstFrame = false;
+        }
+
+        // Track cumulative duration
+        const frameDuration = audio.frame.samplesPerChannel / audio.frame.sampleRate;
+        cumulativeDuration += frameDuration;
+
         this.queue.put(audio);
       }
     };

package/src/tts/tts.ts

@@ -13,8 +13,11 @@ import { DeferredReadableStream } from '../stream/deferred_stream.js';
 import { recordException, traceTypes, tracer } from '../telemetry/index.js';
 import { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS, intervalForRetry } from '../types.js';
 import { AsyncIterableQueue, delay, mergeFrames, startSoon, toError } from '../utils.js';
+import type { TimedString } from '../voice/io.js';
 
-/** SynthesizedAudio is a packet of speech synthesis as returned by the TTS. */
+/**
+ * SynthesizedAudio is a packet of speech synthesis as returned by the TTS.
+ */
 export interface SynthesizedAudio {
   /** Request ID (one segment could be made up of multiple requests) */
   requestId: string;
@@ -26,6 +29,10 @@ export interface SynthesizedAudio {
   deltaText?: string;
   /** Whether this is the last frame of the segment (streaming only) */
   final: boolean;
+  /**
+   * Timed transcripts associated with this audio packet (word-level timestamps).
+   */
+  timedTranscripts?: TimedString[];
 }
 
 /**
@@ -37,6 +44,8 @@ export interface SynthesizedAudio {
  */
 export interface TTSCapabilities {
   streaming: boolean;
+  // Whether this TTS supports aligned transcripts (word-level timestamps).
+  alignedTranscript?: boolean;
 }
 
 export interface TTSError {

package/src/types.ts

@@ -2,6 +2,11 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 
+/**
+ * Key used to store timed transcripts in AudioFrame.userdata.
+ */
+export const USERDATA_TIMED_TRANSCRIPT = 'lk.timed_transcripts';
+
 /**
  * Connection options for API calls, controlling retry and timeout behavior.
  */