@livekit/agents 0.6.3 → 0.7.0

package/dist/stt/stream_adapter.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/stt/stream_adapter.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport type { VAD, VADStream } from '../vad.js';\nimport { VADEventType } from '../vad.js';\nimport type { SpeechEvent } from './stt.js';\nimport { STT, SpeechEventType, SpeechStream } from './stt.js';\n\nexport class StreamAdapter extends STT {\n  #stt: STT;\n  #vad: VAD;\n  label: string;\n\n  constructor(stt: STT, vad: VAD) {\n    super({ streaming: true, interimResults: false });\n    this.#stt = stt;\n    this.#vad = vad;\n    this.label = `stt.StreamAdapter<${this.#stt.label}>`;\n\n    this.#stt.on(SpeechEventType.METRICS_COLLECTED, (metrics) => {\n      this.emit(SpeechEventType.METRICS_COLLECTED, metrics);\n    });\n  }\n\n  _recognize(frame: AudioFrame): Promise<SpeechEvent> {\n    return this.#stt.recognize(frame);\n  }\n\n  stream(): StreamAdapterWrapper {\n    return new StreamAdapterWrapper(this.#stt, this.#vad);\n  }\n}\n\nexport class StreamAdapterWrapper extends SpeechStream {\n  #stt: STT;\n  #vadStream: VADStream;\n  label: string;\n\n  constructor(stt: STT, vad: VAD) {\n    super(stt);\n    this.#stt = stt;\n    this.#vadStream = vad.stream();\n    this.label = `stt.StreamAdapterWrapper<${this.#stt.label}>`;\n\n    this.#run();\n  }\n\n  async monitorMetrics() {\n    return; // do nothing\n  }\n\n  async #run() {\n    const forwardInput = async () => {\n      for await (const input of this.input) {\n        if (input === SpeechStream.FLUSH_SENTINEL) {\n          this.#vadStream.flush();\n        } else {\n          this.#vadStream.pushFrame(input);\n        }\n      }\n      this.#vadStream.endInput();\n    };\n\n    const recognize = async () => {\n      for await (const ev of this.#vadStream) {\n        switch (ev.type) {\n          case VADEventType.START_OF_SPEECH:\n            this.output.put({ type: SpeechEventType.START_OF_SPEECH });\n            break;\n          case VADEventType.END_OF_SPEECH:\n            this.output.put({ type: SpeechEventType.END_OF_SPEECH });\n\n            const event = await this.#stt.recognize(ev.frames);\n            if (!event.alternatives![0].text) {\n              continue;\n            }\n\n            this.output.put(event);\n            break;\n        }\n      }\n    };\n\n    Promise.all([forwardInput(), recognize()]);\n  }\n}\n"],"mappings":"AAKA,SAAS,oBAAoB;AAE7B,SAAS,KAAK,iBAAiB,oBAAoB;AAE5C,MAAM,sBAAsB,IAAI;AAAA,EACrC;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YAAY,KAAU,KAAU;AAC9B,UAAM,EAAE,WAAW,MAAM,gBAAgB,MAAM,CAAC;AAChD,SAAK,OAAO;AACZ,SAAK,OAAO;AACZ,SAAK,QAAQ,qBAAqB,KAAK,KAAK,KAAK;AAEjD,SAAK,KAAK,GAAG,gBAAgB,mBAAmB,CAAC,YAAY;AAC3D,WAAK,KAAK,gBAAgB,mBAAmB,OAAO;AAAA,IACtD,CAAC;AAAA,EACH;AAAA,EAEA,WAAW,OAAyC;AAClD,WAAO,KAAK,KAAK,UAAU,KAAK;AAAA,EAClC;AAAA,EAEA,SAA+B;AAC7B,WAAO,IAAI,qBAAqB,KAAK,MAAM,KAAK,IAAI;AAAA,EACtD;AACF;AAEO,MAAM,6BAA6B,aAAa;AAAA,EACrD;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YAAY,KAAU,KAAU;AAC9B,UAAM,GAAG;AACT,SAAK,OAAO;AACZ,SAAK,aAAa,IAAI,OAAO;AAC7B,SAAK,QAAQ,4BAA4B,KAAK,KAAK,KAAK;AAExD,SAAK,KAAK;AAAA,EACZ;AAAA,EAEA,MAAM,iBAAiB;AACrB;AAAA,EACF;AAAA,EAEA,MAAM,OAAO;AACX,UAAM,eAAe,YAAY;AAC/B,uBAAiB,SAAS,KAAK,OAAO;AACpC,YAAI,UAAU,aAAa,gBAAgB;AACzC,eAAK,WAAW,MAAM;AAAA,QACxB,OAAO;AACL,eAAK,WAAW,UAAU,KAAK;AAAA,QACjC;AAAA,MACF;AACA,WAAK,WAAW,SAAS;AAAA,IAC3B;AAEA,UAAM,YAAY,YAAY;AAC5B,uBAAiB,MAAM,KAAK,YAAY;AACtC,gBAAQ,GAAG,MAAM;AAAA,UACf,KAAK,aAAa;AAChB,iBAAK,OAAO,IAAI,EAAE,MAAM,gBAAgB,gBAAgB,CAAC;AACzD;AAAA,UACF,KAAK,aAAa;AAChB,iBAAK,OAAO,IAAI,EAAE,MAAM,gBAAgB,cAAc,CAAC;AAEvD,kBAAM,QAAQ,MAAM,KAAK,KAAK,UAAU,GAAG,MAAM;AACjD,gBAAI,CAAC,MAAM,aAAc,CAAC,EAAE,MAAM;AAChC;AAAA,YACF;AAEA,iBAAK,OAAO,IAAI,KAAK;AACrB;AAAA,QACJ;AAAA,MACF;AAAA,IACF;AAEA,YAAQ,IAAI,CAAC,aAAa,GAAG,UAAU,CAAC,CAAC;AAAA,EAC3C;AACF;","names":[]}
+{"version":3,"sources":["../../src/stt/stream_adapter.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport { log } from '../log.js';\nimport type { VAD, VADStream } from '../vad.js';\nimport { VADEventType } from '../vad.js';\nimport type { SpeechEvent } from './stt.js';\nimport { STT, SpeechEventType, SpeechStream } from './stt.js';\n\nexport class StreamAdapter extends STT {\n  #stt: STT;\n  #vad: VAD;\n  label: string;\n\n  constructor(stt: STT, vad: VAD) {\n    super({ streaming: true, interimResults: false });\n    this.#stt = stt;\n    this.#vad = vad;\n    this.label = `stt.StreamAdapter<${this.#stt.label}>`;\n\n    this.#stt.on(SpeechEventType.METRICS_COLLECTED, (metrics) => {\n      this.emit(SpeechEventType.METRICS_COLLECTED, metrics);\n    });\n  }\n\n  _recognize(frame: AudioFrame): Promise<SpeechEvent> {\n    return this.#stt.recognize(frame);\n  }\n\n  stream(): StreamAdapterWrapper {\n    return new StreamAdapterWrapper(this.#stt, this.#vad);\n  }\n}\n\nexport class StreamAdapterWrapper extends SpeechStream {\n  #stt: STT;\n  #vadStream: VADStream;\n  label: string;\n\n  constructor(stt: STT, vad: VAD) {\n    super(stt);\n    this.#stt = stt;\n    this.#vadStream = vad.stream();\n    this.label = `stt.StreamAdapterWrapper<${this.#stt.label}>`;\n\n    this.#run();\n  }\n\n  async monitorMetrics() {\n    return; // do nothing\n  }\n\n  async #run() {\n    const forwardInput = async () => {\n      for await (const input of this.input) {\n        if (input === SpeechStream.FLUSH_SENTINEL) {\n          this.#vadStream.flush();\n        } else {\n          this.#vadStream.pushFrame(input);\n        }\n      }\n      this.#vadStream.endInput();\n    };\n\n    const recognize = async () => {\n      for await (const ev of this.#vadStream) {\n        switch (ev.type) {\n          case VADEventType.START_OF_SPEECH:\n            this.output.put({ type: SpeechEventType.START_OF_SPEECH });\n            break;\n          case VADEventType.END_OF_SPEECH:\n            this.output.put({ type: SpeechEventType.END_OF_SPEECH });\n\n            try {\n              const event = await this.#stt.recognize(ev.frames);\n              if (!event.alternatives![0].text) {\n                continue;\n              }\n\n              this.output.put(event);\n              break;\n            } catch (error) {\n              let logger = log();\n              if (error instanceof Error) {\n                logger = logger.child({ error: error.message });\n              } else {\n                logger = logger.child({ error });\n              }\n              logger.error(`${this.label}: provider recognize task failed`);\n              continue;\n            }\n        }\n      }\n    };\n\n    Promise.all([forwardInput(), recognize()]);\n  }\n}\n"],"mappings":"AAIA,SAAS,WAAW;AAEpB,SAAS,oBAAoB;AAE7B,SAAS,KAAK,iBAAiB,oBAAoB;AAE5C,MAAM,sBAAsB,IAAI;AAAA,EACrC;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YAAY,KAAU,KAAU;AAC9B,UAAM,EAAE,WAAW,MAAM,gBAAgB,MAAM,CAAC;AAChD,SAAK,OAAO;AACZ,SAAK,OAAO;AACZ,SAAK,QAAQ,qBAAqB,KAAK,KAAK,KAAK;AAEjD,SAAK,KAAK,GAAG,gBAAgB,mBAAmB,CAAC,YAAY;AAC3D,WAAK,KAAK,gBAAgB,mBAAmB,OAAO;AAAA,IACtD,CAAC;AAAA,EACH;AAAA,EAEA,WAAW,OAAyC;AAClD,WAAO,KAAK,KAAK,UAAU,KAAK;AAAA,EAClC;AAAA,EAEA,SAA+B;AAC7B,WAAO,IAAI,qBAAqB,KAAK,MAAM,KAAK,IAAI;AAAA,EACtD;AACF;AAEO,MAAM,6BAA6B,aAAa;AAAA,EACrD;AAAA,EACA;AAAA,EACA;AAAA,EAEA,YAAY,KAAU,KAAU;AAC9B,UAAM,GAAG;AACT,SAAK,OAAO;AACZ,SAAK,aAAa,IAAI,OAAO;AAC7B,SAAK,QAAQ,4BAA4B,KAAK,KAAK,KAAK;AAExD,SAAK,KAAK;AAAA,EACZ;AAAA,EAEA,MAAM,iBAAiB;AACrB;AAAA,EACF;AAAA,EAEA,MAAM,OAAO;AACX,UAAM,eAAe,YAAY;AAC/B,uBAAiB,SAAS,KAAK,OAAO;AACpC,YAAI,UAAU,aAAa,gBAAgB;AACzC,eAAK,WAAW,MAAM;AAAA,QACxB,OAAO;AACL,eAAK,WAAW,UAAU,KAAK;AAAA,QACjC;AAAA,MACF;AACA,WAAK,WAAW,SAAS;AAAA,IAC3B;AAEA,UAAM,YAAY,YAAY;AAC5B,uBAAiB,MAAM,KAAK,YAAY;AACtC,gBAAQ,GAAG,MAAM;AAAA,UACf,KAAK,aAAa;AAChB,iBAAK,OAAO,IAAI,EAAE,MAAM,gBAAgB,gBAAgB,CAAC;AACzD;AAAA,UACF,KAAK,aAAa;AAChB,iBAAK,OAAO,IAAI,EAAE,MAAM,gBAAgB,cAAc,CAAC;AAEvD,gBAAI;AACF,oBAAM,QAAQ,MAAM,KAAK,KAAK,UAAU,GAAG,MAAM;AACjD,kBAAI,CAAC,MAAM,aAAc,CAAC,EAAE,MAAM;AAChC;AAAA,cACF;AAEA,mBAAK,OAAO,IAAI,KAAK;AACrB;AAAA,YACF,SAAS,OAAO;AACd,kBAAI,SAAS,IAAI;AACjB,kBAAI,iBAAiB,OAAO;AAC1B,yBAAS,OAAO,MAAM,EAAE,OAAO,MAAM,QAAQ,CAAC;AAAA,cAChD,OAAO;AACL,yBAAS,OAAO,MAAM,EAAE,MAAM,CAAC;AAAA,cACjC;AACA,qBAAO,MAAM,GAAG,KAAK,KAAK,kCAAkC;AAC5D;AAAA,YACF;AAAA,QACJ;AAAA,MACF;AAAA,IACF;AAEA,YAAQ,IAAI,CAAC,aAAa,GAAG,UAAU,CAAC,CAAC;AAAA,EAC3C;AACF;","names":[]}

package/dist/tokenize/basic/basic.cjs

@@ -31,6 +31,7 @@ __export(basic_exports, {
   SentenceTokenizer: () => SentenceTokenizer,
   WordTokenizer: () => WordTokenizer,
   hyphenateWord: () => hyphenateWord,
+  splitWords: () => import_word.splitWords,
   tokenizeParagraphs: () => tokenizeParagraphs
 });
 module.exports = __toCommonJS(basic_exports);
@@ -93,6 +94,7 @@ const tokenizeParagraphs = (text) => {
   SentenceTokenizer,
   WordTokenizer,
   hyphenateWord,
+  splitWords,
   tokenizeParagraphs
 });
 //# sourceMappingURL=basic.cjs.map

package/dist/tokenize/basic/basic.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/tokenize/basic/basic.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { BufferedSentenceStream, BufferedWordStream } from '../token_stream.js';\nimport * as tokenizer from '../tokenizer.js';\nimport { hyphenator } from './hyphenator.js';\nimport { splitParagraphs } from './paragraph.js';\nimport { splitSentences } from './sentence.js';\nimport { splitWords } from './word.js';\n\ninterface TokenizerOptions {\n  language: string;\n  minSentenceLength: number;\n  streamContextLength: number;\n}\n\nexport class SentenceTokenizer extends tokenizer.SentenceTokenizer {\n  #config: TokenizerOptions;\n\n  constructor(language = 'en-US', minSentenceLength = 20, streamContextLength = 10) {\n    super();\n    this.#config = {\n      language,\n      minSentenceLength,\n      streamContextLength,\n    };\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  tokenize(text: string, language?: string): string[] {\n    return splitSentences(text, this.#config.minSentenceLength).map((tok) => tok[0]);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  stream(language?: string): tokenizer.SentenceStream {\n    return new BufferedSentenceStream(\n      (text: string) => splitSentences(text, this.#config.minSentenceLength),\n      this.#config.minSentenceLength,\n      this.#config.streamContextLength,\n    );\n  }\n}\n\nexport class WordTokenizer extends tokenizer.WordTokenizer {\n  #ignorePunctuation: boolean;\n\n  constructor(ignorePunctuation = true) {\n    super();\n    this.#ignorePunctuation = ignorePunctuation;\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  tokenize(text: string, language?: string): string[] {\n    return splitWords(text, this.#ignorePunctuation).map((tok) => tok[0]);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  stream(language?: string): tokenizer.WordStream {\n    return new BufferedWordStream(\n      (text: string) => splitWords(text, this.#ignorePunctuation),\n      1,\n      1,\n    );\n  }\n}\n\nexport const hyphenateWord = (word: string): string[] => {\n  return hyphenator.hyphenateWord(word);\n};\n\nexport const tokenizeParagraphs = (text: string): string[] => {\n  return splitParagraphs(text).map((tok) => tok[0]);\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,0BAA2D;AAC3D,gBAA2B;AAC3B,wBAA2B;AAC3B,uBAAgC;AAChC,sBAA+B;AAC/B,kBAA2B;AAQpB,MAAM,0BAA0B,UAAU,kBAAkB;AAAA,EACjE;AAAA,EAEA,YAAY,WAAW,SAAS,oBAAoB,IAAI,sBAAsB,IAAI;AAChF,UAAM;AACN,SAAK,UAAU;AAAA,MACb;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAGA,SAAS,MAAc,UAA6B;AAClD,eAAO,gCAAe,MAAM,KAAK,QAAQ,iBAAiB,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;AAAA,EACjF;AAAA;AAAA,EAGA,OAAO,UAA6C;AAClD,WAAO,IAAI;AAAA,MACT,CAAC,aAAiB,gCAAe,MAAM,KAAK,QAAQ,iBAAiB;AAAA,MACrE,KAAK,QAAQ;AAAA,MACb,KAAK,QAAQ;AAAA,IACf;AAAA,EACF;AACF;AAEO,MAAM,sBAAsB,UAAU,cAAc;AAAA,EACzD;AAAA,EAEA,YAAY,oBAAoB,MAAM;AACpC,UAAM;AACN,SAAK,qBAAqB;AAAA,EAC5B;AAAA;AAAA,EAGA,SAAS,MAAc,UAA6B;AAClD,eAAO,wBAAW,MAAM,KAAK,kBAAkB,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;AAAA,EACtE;AAAA;AAAA,EAGA,OAAO,UAAyC;AAC9C,WAAO,IAAI;AAAA,MACT,CAAC,aAAiB,wBAAW,MAAM,KAAK,kBAAkB;AAAA,MAC1D;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;AAEO,MAAM,gBAAgB,CAAC,SAA2B;AACvD,SAAO,6BAAW,cAAc,IAAI;AACtC;AAEO,MAAM,qBAAqB,CAAC,SAA2B;AAC5D,aAAO,kCAAgB,IAAI,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;AAClD;","names":[]}
+{"version":3,"sources":["../../../src/tokenize/basic/basic.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { BufferedSentenceStream, BufferedWordStream } from '../token_stream.js';\nimport * as tokenizer from '../tokenizer.js';\nimport { hyphenator } from './hyphenator.js';\nimport { splitParagraphs } from './paragraph.js';\nimport { splitSentences } from './sentence.js';\nimport { splitWords } from './word.js';\n\ninterface TokenizerOptions {\n  language: string;\n  minSentenceLength: number;\n  streamContextLength: number;\n}\n\nexport class SentenceTokenizer extends tokenizer.SentenceTokenizer {\n  #config: TokenizerOptions;\n\n  constructor(language = 'en-US', minSentenceLength = 20, streamContextLength = 10) {\n    super();\n    this.#config = {\n      language,\n      minSentenceLength,\n      streamContextLength,\n    };\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  tokenize(text: string, language?: string): string[] {\n    return splitSentences(text, this.#config.minSentenceLength).map((tok) => tok[0]);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  stream(language?: string): tokenizer.SentenceStream {\n    return new BufferedSentenceStream(\n      (text: string) => splitSentences(text, this.#config.minSentenceLength),\n      this.#config.minSentenceLength,\n      this.#config.streamContextLength,\n    );\n  }\n}\n\nexport class WordTokenizer extends tokenizer.WordTokenizer {\n  #ignorePunctuation: boolean;\n\n  constructor(ignorePunctuation = true) {\n    super();\n    this.#ignorePunctuation = ignorePunctuation;\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  tokenize(text: string, language?: string): string[] {\n    return splitWords(text, this.#ignorePunctuation).map((tok) => tok[0]);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  stream(language?: string): tokenizer.WordStream {\n    return new BufferedWordStream(\n      (text: string) => splitWords(text, this.#ignorePunctuation),\n      1,\n      1,\n    );\n  }\n}\n\nexport const hyphenateWord = (word: string): string[] => {\n  return hyphenator.hyphenateWord(word);\n};\n\nexport { splitWords };\n\nexport const tokenizeParagraphs = (text: string): string[] => {\n  return splitParagraphs(text).map((tok) => tok[0]);\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,0BAA2D;AAC3D,gBAA2B;AAC3B,wBAA2B;AAC3B,uBAAgC;AAChC,sBAA+B;AAC/B,kBAA2B;AAQpB,MAAM,0BAA0B,UAAU,kBAAkB;AAAA,EACjE;AAAA,EAEA,YAAY,WAAW,SAAS,oBAAoB,IAAI,sBAAsB,IAAI;AAChF,UAAM;AACN,SAAK,UAAU;AAAA,MACb;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAGA,SAAS,MAAc,UAA6B;AAClD,eAAO,gCAAe,MAAM,KAAK,QAAQ,iBAAiB,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;AAAA,EACjF;AAAA;AAAA,EAGA,OAAO,UAA6C;AAClD,WAAO,IAAI;AAAA,MACT,CAAC,aAAiB,gCAAe,MAAM,KAAK,QAAQ,iBAAiB;AAAA,MACrE,KAAK,QAAQ;AAAA,MACb,KAAK,QAAQ;AAAA,IACf;AAAA,EACF;AACF;AAEO,MAAM,sBAAsB,UAAU,cAAc;AAAA,EACzD;AAAA,EAEA,YAAY,oBAAoB,MAAM;AACpC,UAAM;AACN,SAAK,qBAAqB;AAAA,EAC5B;AAAA;AAAA,EAGA,SAAS,MAAc,UAA6B;AAClD,eAAO,wBAAW,MAAM,KAAK,kBAAkB,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;AAAA,EACtE;AAAA;AAAA,EAGA,OAAO,UAAyC;AAC9C,WAAO,IAAI;AAAA,MACT,CAAC,aAAiB,wBAAW,MAAM,KAAK,kBAAkB;AAAA,MAC1D;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;AAEO,MAAM,gBAAgB,CAAC,SAA2B;AACvD,SAAO,6BAAW,cAAc,IAAI;AACtC;AAIO,MAAM,qBAAqB,CAAC,SAA2B;AAC5D,aAAO,kCAAgB,IAAI,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;AAClD;","names":[]}

package/dist/tokenize/basic/basic.d.ts

@@ -1,4 +1,5 @@
 import * as tokenizer from '../tokenizer.js';
+import { splitWords } from './word.js';
 export declare class SentenceTokenizer extends tokenizer.SentenceTokenizer {
     #private;
     constructor(language?: string, minSentenceLength?: number, streamContextLength?: number);
@@ -12,5 +13,6 @@ export declare class WordTokenizer extends tokenizer.WordTokenizer {
     stream(language?: string): tokenizer.WordStream;
 }
 export declare const hyphenateWord: (word: string) => string[];
+export { splitWords };
 export declare const tokenizeParagraphs: (text: string) => string[];
 //# sourceMappingURL=basic.d.ts.map

package/dist/tokenize/basic/basic.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"basic.d.ts","sourceRoot":"","sources":["../../../src/tokenize/basic/basic.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,SAAS,MAAM,iBAAiB,CAAC;AAY7C,qBAAa,iBAAkB,SAAQ,SAAS,CAAC,iBAAiB;;gBAGpD,QAAQ,SAAU,EAAE,iBAAiB,SAAK,EAAE,mBAAmB,SAAK;IAUhF,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE;IAKnD,MAAM,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC,cAAc;CAOpD;AAED,qBAAa,aAAc,SAAQ,SAAS,CAAC,aAAa;;gBAG5C,iBAAiB,UAAO;IAMpC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE;IAKnD,MAAM,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC,UAAU;CAOhD;AAED,eAAO,MAAM,aAAa,SAAU,MAAM,KAAG,MAAM,EAElD,CAAC;AAEF,eAAO,MAAM,kBAAkB,SAAU,MAAM,KAAG,MAAM,EAEvD,CAAC"}
+{"version":3,"file":"basic.d.ts","sourceRoot":"","sources":["../../../src/tokenize/basic/basic.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,SAAS,MAAM,iBAAiB,CAAC;AAI7C,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAQvC,qBAAa,iBAAkB,SAAQ,SAAS,CAAC,iBAAiB;;gBAGpD,QAAQ,SAAU,EAAE,iBAAiB,SAAK,EAAE,mBAAmB,SAAK;IAUhF,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE;IAKnD,MAAM,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC,cAAc;CAOpD;AAED,qBAAa,aAAc,SAAQ,SAAS,CAAC,aAAa;;gBAG5C,iBAAiB,UAAO;IAMpC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE;IAKnD,MAAM,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC,UAAU;CAOhD;AAED,eAAO,MAAM,aAAa,SAAU,MAAM,KAAG,MAAM,EAElD,CAAC;AAEF,OAAO,EAAE,UAAU,EAAE,CAAC;AAEtB,eAAO,MAAM,kBAAkB,SAAU,MAAM,KAAG,MAAM,EAEvD,CAAC"}

package/dist/tokenize/basic/basic.js

@@ -56,6 +56,7 @@ export {
   SentenceTokenizer,
   WordTokenizer,
   hyphenateWord,
+  splitWords,
   tokenizeParagraphs
 };
 //# sourceMappingURL=basic.js.map

package/dist/tokenize/basic/basic.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/tokenize/basic/basic.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { BufferedSentenceStream, BufferedWordStream } from '../token_stream.js';\nimport * as tokenizer from '../tokenizer.js';\nimport { hyphenator } from './hyphenator.js';\nimport { splitParagraphs } from './paragraph.js';\nimport { splitSentences } from './sentence.js';\nimport { splitWords } from './word.js';\n\ninterface TokenizerOptions {\n  language: string;\n  minSentenceLength: number;\n  streamContextLength: number;\n}\n\nexport class SentenceTokenizer extends tokenizer.SentenceTokenizer {\n  #config: TokenizerOptions;\n\n  constructor(language = 'en-US', minSentenceLength = 20, streamContextLength = 10) {\n    super();\n    this.#config = {\n      language,\n      minSentenceLength,\n      streamContextLength,\n    };\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  tokenize(text: string, language?: string): string[] {\n    return splitSentences(text, this.#config.minSentenceLength).map((tok) => tok[0]);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  stream(language?: string): tokenizer.SentenceStream {\n    return new BufferedSentenceStream(\n      (text: string) => splitSentences(text, this.#config.minSentenceLength),\n      this.#config.minSentenceLength,\n      this.#config.streamContextLength,\n    );\n  }\n}\n\nexport class WordTokenizer extends tokenizer.WordTokenizer {\n  #ignorePunctuation: boolean;\n\n  constructor(ignorePunctuation = true) {\n    super();\n    this.#ignorePunctuation = ignorePunctuation;\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  tokenize(text: string, language?: string): string[] {\n    return splitWords(text, this.#ignorePunctuation).map((tok) => tok[0]);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  stream(language?: string): tokenizer.WordStream {\n    return new BufferedWordStream(\n      (text: string) => splitWords(text, this.#ignorePunctuation),\n      1,\n      1,\n    );\n  }\n}\n\nexport const hyphenateWord = (word: string): string[] => {\n  return hyphenator.hyphenateWord(word);\n};\n\nexport const tokenizeParagraphs = (text: string): string[] => {\n  return splitParagraphs(text).map((tok) => tok[0]);\n};\n"],"mappings":"AAGA,SAAS,wBAAwB,0BAA0B;AAC3D,YAAY,eAAe;AAC3B,SAAS,kBAAkB;AAC3B,SAAS,uBAAuB;AAChC,SAAS,sBAAsB;AAC/B,SAAS,kBAAkB;AAQpB,MAAM,0BAA0B,UAAU,kBAAkB;AAAA,EACjE;AAAA,EAEA,YAAY,WAAW,SAAS,oBAAoB,IAAI,sBAAsB,IAAI;AAChF,UAAM;AACN,SAAK,UAAU;AAAA,MACb;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAGA,SAAS,MAAc,UAA6B;AAClD,WAAO,eAAe,MAAM,KAAK,QAAQ,iBAAiB,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;AAAA,EACjF;AAAA;AAAA,EAGA,OAAO,UAA6C;AAClD,WAAO,IAAI;AAAA,MACT,CAAC,SAAiB,eAAe,MAAM,KAAK,QAAQ,iBAAiB;AAAA,MACrE,KAAK,QAAQ;AAAA,MACb,KAAK,QAAQ;AAAA,IACf;AAAA,EACF;AACF;AAEO,MAAM,sBAAsB,UAAU,cAAc;AAAA,EACzD;AAAA,EAEA,YAAY,oBAAoB,MAAM;AACpC,UAAM;AACN,SAAK,qBAAqB;AAAA,EAC5B;AAAA;AAAA,EAGA,SAAS,MAAc,UAA6B;AAClD,WAAO,WAAW,MAAM,KAAK,kBAAkB,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;AAAA,EACtE;AAAA;AAAA,EAGA,OAAO,UAAyC;AAC9C,WAAO,IAAI;AAAA,MACT,CAAC,SAAiB,WAAW,MAAM,KAAK,kBAAkB;AAAA,MAC1D;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;AAEO,MAAM,gBAAgB,CAAC,SAA2B;AACvD,SAAO,WAAW,cAAc,IAAI;AACtC;AAEO,MAAM,qBAAqB,CAAC,SAA2B;AAC5D,SAAO,gBAAgB,IAAI,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;AAClD;","names":[]}
+{"version":3,"sources":["../../../src/tokenize/basic/basic.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { BufferedSentenceStream, BufferedWordStream } from '../token_stream.js';\nimport * as tokenizer from '../tokenizer.js';\nimport { hyphenator } from './hyphenator.js';\nimport { splitParagraphs } from './paragraph.js';\nimport { splitSentences } from './sentence.js';\nimport { splitWords } from './word.js';\n\ninterface TokenizerOptions {\n  language: string;\n  minSentenceLength: number;\n  streamContextLength: number;\n}\n\nexport class SentenceTokenizer extends tokenizer.SentenceTokenizer {\n  #config: TokenizerOptions;\n\n  constructor(language = 'en-US', minSentenceLength = 20, streamContextLength = 10) {\n    super();\n    this.#config = {\n      language,\n      minSentenceLength,\n      streamContextLength,\n    };\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  tokenize(text: string, language?: string): string[] {\n    return splitSentences(text, this.#config.minSentenceLength).map((tok) => tok[0]);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  stream(language?: string): tokenizer.SentenceStream {\n    return new BufferedSentenceStream(\n      (text: string) => splitSentences(text, this.#config.minSentenceLength),\n      this.#config.minSentenceLength,\n      this.#config.streamContextLength,\n    );\n  }\n}\n\nexport class WordTokenizer extends tokenizer.WordTokenizer {\n  #ignorePunctuation: boolean;\n\n  constructor(ignorePunctuation = true) {\n    super();\n    this.#ignorePunctuation = ignorePunctuation;\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  tokenize(text: string, language?: string): string[] {\n    return splitWords(text, this.#ignorePunctuation).map((tok) => tok[0]);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  stream(language?: string): tokenizer.WordStream {\n    return new BufferedWordStream(\n      (text: string) => splitWords(text, this.#ignorePunctuation),\n      1,\n      1,\n    );\n  }\n}\n\nexport const hyphenateWord = (word: string): string[] => {\n  return hyphenator.hyphenateWord(word);\n};\n\nexport { splitWords };\n\nexport const tokenizeParagraphs = (text: string): string[] => {\n  return splitParagraphs(text).map((tok) => tok[0]);\n};\n"],"mappings":"AAGA,SAAS,wBAAwB,0BAA0B;AAC3D,YAAY,eAAe;AAC3B,SAAS,kBAAkB;AAC3B,SAAS,uBAAuB;AAChC,SAAS,sBAAsB;AAC/B,SAAS,kBAAkB;AAQpB,MAAM,0BAA0B,UAAU,kBAAkB;AAAA,EACjE;AAAA,EAEA,YAAY,WAAW,SAAS,oBAAoB,IAAI,sBAAsB,IAAI;AAChF,UAAM;AACN,SAAK,UAAU;AAAA,MACb;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAGA,SAAS,MAAc,UAA6B;AAClD,WAAO,eAAe,MAAM,KAAK,QAAQ,iBAAiB,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;AAAA,EACjF;AAAA;AAAA,EAGA,OAAO,UAA6C;AAClD,WAAO,IAAI;AAAA,MACT,CAAC,SAAiB,eAAe,MAAM,KAAK,QAAQ,iBAAiB;AAAA,MACrE,KAAK,QAAQ;AAAA,MACb,KAAK,QAAQ;AAAA,IACf;AAAA,EACF;AACF;AAEO,MAAM,sBAAsB,UAAU,cAAc;AAAA,EACzD;AAAA,EAEA,YAAY,oBAAoB,MAAM;AACpC,UAAM;AACN,SAAK,qBAAqB;AAAA,EAC5B;AAAA;AAAA,EAGA,SAAS,MAAc,UAA6B;AAClD,WAAO,WAAW,MAAM,KAAK,kBAAkB,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;AAAA,EACtE;AAAA;AAAA,EAGA,OAAO,UAAyC;AAC9C,WAAO,IAAI;AAAA,MACT,CAAC,SAAiB,WAAW,MAAM,KAAK,kBAAkB;AAAA,MAC1D;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;AAEO,MAAM,gBAAgB,CAAC,SAA2B;AACvD,SAAO,WAAW,cAAc,IAAI;AACtC;AAIO,MAAM,qBAAqB,CAAC,SAA2B;AAC5D,SAAO,gBAAgB,IAAI,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;AAClD;","names":[]}

package/dist/tokenize/basic/index.cjs

@@ -21,6 +21,7 @@ __export(basic_exports, {
   SentenceTokenizer: () => import_basic.SentenceTokenizer,
   WordTokenizer: () => import_basic.WordTokenizer,
   hyphenateWord: () => import_basic.hyphenateWord,
+  splitWords: () => import_basic.splitWords,
   tokenizeParagraphs: () => import_basic.tokenizeParagraphs
 });
 module.exports = __toCommonJS(basic_exports);
@@ -30,6 +31,7 @@ var import_basic = require("./basic.cjs");
   SentenceTokenizer,
   WordTokenizer,
   hyphenateWord,
+  splitWords,
   tokenizeParagraphs
 });
 //# sourceMappingURL=index.cjs.map

package/dist/tokenize/basic/index.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/tokenize/basic/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\nexport { SentenceTokenizer, WordTokenizer, tokenizeParagraphs, hyphenateWord } from './basic.js';\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,mBAAoF;","names":[]}
+{"version":3,"sources":["../../../src/tokenize/basic/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\nexport {\n  SentenceTokenizer,\n  WordTokenizer,\n  tokenizeParagraphs,\n  hyphenateWord,\n  splitWords,\n} from './basic.js';\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,mBAMO;","names":[]}

package/dist/tokenize/basic/index.d.ts

@@ -1,2 +1,2 @@
-export { SentenceTokenizer, WordTokenizer, tokenizeParagraphs, hyphenateWord } from './basic.js';
+export { SentenceTokenizer, WordTokenizer, tokenizeParagraphs, hyphenateWord, splitWords, } from './basic.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/tokenize/basic/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/tokenize/basic/index.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/tokenize/basic/index.ts"],"names":[],"mappings":"AAIA,OAAO,EACL,iBAAiB,EACjB,aAAa,EACb,kBAAkB,EAClB,aAAa,EACb,UAAU,GACX,MAAM,YAAY,CAAC"}

package/dist/tokenize/basic/index.js

@@ -1,8 +1,15 @@
-import { SentenceTokenizer, WordTokenizer, tokenizeParagraphs, hyphenateWord } from "./basic.js";
+import {
+  SentenceTokenizer,
+  WordTokenizer,
+  tokenizeParagraphs,
+  hyphenateWord,
+  splitWords
+} from "./basic.js";
 export {
   SentenceTokenizer,
   WordTokenizer,
   hyphenateWord,
+  splitWords,
   tokenizeParagraphs
 };
 //# sourceMappingURL=index.js.map

package/dist/tokenize/basic/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/tokenize/basic/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\nexport { SentenceTokenizer, WordTokenizer, tokenizeParagraphs, hyphenateWord } from './basic.js';\n"],"mappings":"AAIA,SAAS,mBAAmB,eAAe,oBAAoB,qBAAqB;","names":[]}
+{"version":3,"sources":["../../../src/tokenize/basic/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\nexport {\n  SentenceTokenizer,\n  WordTokenizer,\n  tokenizeParagraphs,\n  hyphenateWord,\n  splitWords,\n} from './basic.js';\n"],"mappings":"AAIA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;","names":[]}

package/dist/tokenize/token_stream.cjs

@@ -54,9 +54,11 @@ class BufferedTokenStream {
       if (tokens.length <= 1) break;
       if (this.#outBuf) this.#outBuf += " ";
       const tok = tokens.shift();
-      let tokText = tok;
-      if (tok.length > 1 && typeof tok[1] === "number") {
+      let tokText;
+      if (Array.isArray(tok)) {
         tokText = tok[0];
+      } else {
+        tokText = tok;
       }
       this.#outBuf += tokText;
       if (this.#outBuf.length >= this.#minTokenLength) {
@@ -79,7 +81,7 @@ class BufferedTokenStream {
       const tokens = this.#func(this.#inBuf);
       if (tokens) {
         if (this.#outBuf) this.#outBuf += " ";
-        if (typeof tokens[0] !== "string") {
+        if (Array.isArray(tokens[0])) {
           this.#outBuf += tokens.map((tok) => tok[0]).join(" ");
         } else {
           this.#outBuf += tokens.join(" ");

package/dist/tokenize/token_stream.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/tokenize/token_stream.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { randomUUID } from 'node:crypto';\nimport { AsyncIterableQueue } from '../utils.js';\nimport type { TokenData } from './tokenizer.js';\nimport { SentenceStream, WordStream } from './tokenizer.js';\n\ntype TokenizeFunc = (x: string) => string[] | [string, number, number][];\n\nexport class BufferedTokenStream implements AsyncIterableIterator<TokenData> {\n  protected queue = new AsyncIterableQueue<TokenData>();\n  protected closed = false;\n\n  #func: TokenizeFunc;\n  #minTokenLength: number;\n  #minContextLength: number;\n  #bufTokens: string[] = [];\n  #inBuf = '';\n  #outBuf = '';\n  #currentSegmentId: string;\n\n  constructor(func: TokenizeFunc, minTokenLength: number, minContextLength: number) {\n    this.#func = func;\n    this.#minTokenLength = minTokenLength;\n    this.#minContextLength = minContextLength;\n\n    this.#currentSegmentId = randomUUID();\n  }\n\n  /** Push a string of text into the token stream */\n  pushText(text: string) {\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n\n    this.#inBuf += text;\n    if (this.#inBuf.length < this.#minContextLength) return;\n\n    while (true) {\n      const tokens = this.#func(this.#inBuf);\n      if (tokens.length <= 1) break;\n\n      if (this.#outBuf) this.#outBuf += ' ';\n\n      const tok = tokens.shift()!;\n      let tokText = tok as string;\n      if (tok.length > 1 && typeof tok[1] === 'number') {\n        tokText = tok[0];\n      }\n\n      this.#outBuf += tokText;\n      if (this.#outBuf.length >= this.#minTokenLength) {\n        this.queue.put({ token: this.#outBuf, segmentId: this.#currentSegmentId });\n        this.#outBuf = '';\n      }\n\n      if (typeof tok! !== 'string') {\n        this.#inBuf = this.#inBuf.slice(tok![2]);\n      } else {\n        this.#inBuf = this.#inBuf\n          .slice(Math.max(0, this.#inBuf.indexOf(tok)) + tok.length)\n          .trimStart();\n      }\n    }\n  }\n\n  /** Flush the stream, causing it to process all pending text */\n  flush() {\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n\n    if (this.#inBuf || this.#outBuf) {\n      const tokens = this.#func(this.#inBuf);\n      if (tokens) {\n        if (this.#outBuf) this.#outBuf += ' ';\n\n        if (typeof tokens[0] !== 'string') {\n          this.#outBuf += tokens.map((tok) => tok[0]).join(' ');\n        } else {\n          this.#outBuf += tokens.join(' ');\n        }\n      }\n\n      if (this.#outBuf) {\n        this.queue.put({ token: this.#outBuf, segmentId: this.#currentSegmentId });\n      }\n\n      this.#currentSegmentId = randomUUID();\n    }\n\n    this.#inBuf = '';\n    this.#outBuf = '';\n  }\n\n  /** Mark the input as ended and forbid additional pushes */\n  endInput() {\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.flush();\n    this.close();\n  }\n\n  next(): Promise<IteratorResult<TokenData>> {\n    return this.queue.next();\n  }\n\n  /** Close both the input and output of the token stream */\n  close() {\n    this.queue.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): BufferedTokenStream {\n    return this;\n  }\n}\n\nexport class BufferedSentenceStream extends SentenceStream {\n  #stream: BufferedTokenStream;\n\n  constructor(func: TokenizeFunc, minTokenLength: number, minContextLength: number) {\n    super();\n    this.#stream = new BufferedTokenStream(func, minTokenLength, minContextLength);\n  }\n\n  pushText(text: string) {\n    this.#stream.pushText(text);\n  }\n\n  flush() {\n    this.#stream.flush();\n  }\n\n  close() {\n    super.close();\n    this.#stream.close();\n  }\n\n  next(): Promise<IteratorResult<TokenData>> {\n    return this.#stream.next();\n  }\n}\n\nexport class BufferedWordStream extends WordStream {\n  #stream: BufferedTokenStream;\n\n  constructor(func: TokenizeFunc, minTokenLength: number, minContextLength: number) {\n    super();\n    this.#stream = new BufferedTokenStream(func, minTokenLength, minContextLength);\n  }\n\n  pushText(text: string) {\n    this.#stream.pushText(text);\n  }\n\n  flush() {\n    this.#stream.flush();\n  }\n\n  endInput() {\n    this.#stream.endInput();\n  }\n\n  close() {\n    this.#stream.close();\n  }\n\n  next(): Promise<IteratorResult<TokenData>> {\n    return this.#stream.next();\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,yBAA2B;AAC3B,mBAAmC;AAEnC,uBAA2C;AAIpC,MAAM,oBAAgE;AAAA,EACjE,QAAQ,IAAI,gCAA8B;AAAA,EAC1C,SAAS;AAAA,EAEnB;AAAA,EACA;AAAA,EACA;AAAA,EACA,aAAuB,CAAC;AAAA,EACxB,SAAS;AAAA,EACT,UAAU;AAAA,EACV;AAAA,EAEA,YAAY,MAAoB,gBAAwB,kBAA0B;AAChF,SAAK,QAAQ;AACb,SAAK,kBAAkB;AACvB,SAAK,oBAAoB;AAEzB,SAAK,wBAAoB,+BAAW;AAAA,EACtC;AAAA;AAAA,EAGA,SAAS,MAAc;AACrB,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AAEA,SAAK,UAAU;AACf,QAAI,KAAK,OAAO,SAAS,KAAK,kBAAmB;AAEjD,WAAO,MAAM;AACX,YAAM,SAAS,KAAK,MAAM,KAAK,MAAM;AACrC,UAAI,OAAO,UAAU,EAAG;AAExB,UAAI,KAAK,QAAS,MAAK,WAAW;AAElC,YAAM,MAAM,OAAO,MAAM;AACzB,UAAI,UAAU;AACd,UAAI,IAAI,SAAS,KAAK,OAAO,IAAI,CAAC,MAAM,UAAU;AAChD,kBAAU,IAAI,CAAC;AAAA,MACjB;AAEA,WAAK,WAAW;AAChB,UAAI,KAAK,QAAQ,UAAU,KAAK,iBAAiB;AAC/C,aAAK,MAAM,IAAI,EAAE,OAAO,KAAK,SAAS,WAAW,KAAK,kBAAkB,CAAC;AACzE,aAAK,UAAU;AAAA,MACjB;AAEA,UAAI,OAAO,QAAS,UAAU;AAC5B,aAAK,SAAS,KAAK,OAAO,MAAM,IAAK,CAAC,CAAC;AAAA,MACzC,OAAO;AACL,aAAK,SAAS,KAAK,OAChB,MAAM,KAAK,IAAI,GAAG,KAAK,OAAO,QAAQ,GAAG,CAAC,IAAI,IAAI,MAAM,EACxD,UAAU;AAAA,MACf;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAGA,QAAQ;AACN,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AAEA,QAAI,KAAK,UAAU,KAAK,SAAS;AAC/B,YAAM,SAAS,KAAK,MAAM,KAAK,MAAM;AACrC,UAAI,QAAQ;AACV,YAAI,KAAK,QAAS,MAAK,WAAW;AAElC,YAAI,OAAO,OAAO,CAAC,MAAM,UAAU;AACjC,eAAK,WAAW,OAAO,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC,EAAE,KAAK,GAAG;AAAA,QACtD,OAAO;AACL,eAAK,WAAW,OAAO,KAAK,GAAG;AAAA,QACjC;AAAA,MACF;AAEA,UAAI,KAAK,SAAS;AAChB,aAAK,MAAM,IAAI,EAAE,OAAO,KAAK,SAAS,WAAW,KAAK,kBAAkB,CAAC;AAAA,MAC3E;AAEA,WAAK,wBAAoB,+BAAW;AAAA,IACtC;AAEA,SAAK,SAAS;AACd,SAAK,UAAU;AAAA,EACjB;AAAA;AAAA,EAGA,WAAW;AACT,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM;AACX,SAAK,MAAM;AAAA,EACb;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,MAAM,KAAK;AAAA,EACzB;AAAA;AAAA,EAGA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAyB;AAC5C,WAAO;AAAA,EACT;AACF;AAEO,MAAM,+BAA+B,gCAAe;AAAA,EACzD;AAAA,EAEA,YAAY,MAAoB,gBAAwB,kBAA0B;AAChF,UAAM;AACN,SAAK,UAAU,IAAI,oBAAoB,MAAM,gBAAgB,gBAAgB;AAAA,EAC/E;AAAA,EAEA,SAAS,MAAc;AACrB,SAAK,QAAQ,SAAS,IAAI;AAAA,EAC5B;AAAA,EAEA,QAAQ;AACN,SAAK,QAAQ,MAAM;AAAA,EACrB;AAAA,EAEA,QAAQ;AACN,UAAM,MAAM;AACZ,SAAK,QAAQ,MAAM;AAAA,EACrB;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,QAAQ,KAAK;AAAA,EAC3B;AACF;AAEO,MAAM,2BAA2B,4BAAW;AAAA,EACjD;AAAA,EAEA,YAAY,MAAoB,gBAAwB,kBAA0B;AAChF,UAAM;AACN,SAAK,UAAU,IAAI,oBAAoB,MAAM,gBAAgB,gBAAgB;AAAA,EAC/E;AAAA,EAEA,SAAS,MAAc;AACrB,SAAK,QAAQ,SAAS,IAAI;AAAA,EAC5B;AAAA,EAEA,QAAQ;AACN,SAAK,QAAQ,MAAM;AAAA,EACrB;AAAA,EAEA,WAAW;AACT,SAAK,QAAQ,SAAS;AAAA,EACxB;AAAA,EAEA,QAAQ;AACN,SAAK,QAAQ,MAAM;AAAA,EACrB;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,QAAQ,KAAK;AAAA,EAC3B;AACF;","names":[]}
+{"version":3,"sources":["../../src/tokenize/token_stream.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { randomUUID } from 'node:crypto';\nimport { AsyncIterableQueue } from '../utils.js';\nimport type { TokenData } from './tokenizer.js';\nimport { SentenceStream, WordStream } from './tokenizer.js';\n\ntype TokenizeFunc = (x: string) => string[] | [string, number, number][];\n\nexport class BufferedTokenStream implements AsyncIterableIterator<TokenData> {\n  protected queue = new AsyncIterableQueue<TokenData>();\n  protected closed = false;\n\n  #func: TokenizeFunc;\n  #minTokenLength: number;\n  #minContextLength: number;\n  #bufTokens: string[] = [];\n  #inBuf = '';\n  #outBuf = '';\n  #currentSegmentId: string;\n\n  constructor(func: TokenizeFunc, minTokenLength: number, minContextLength: number) {\n    this.#func = func;\n    this.#minTokenLength = minTokenLength;\n    this.#minContextLength = minContextLength;\n\n    this.#currentSegmentId = randomUUID();\n  }\n\n  /** Push a string of text into the token stream */\n  pushText(text: string) {\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n\n    this.#inBuf += text;\n    if (this.#inBuf.length < this.#minContextLength) return;\n\n    while (true) {\n      const tokens = this.#func(this.#inBuf);\n      if (tokens.length <= 1) break;\n\n      if (this.#outBuf) this.#outBuf += ' ';\n\n      const tok = tokens.shift()!;\n      let tokText: string;\n      if (Array.isArray(tok)) {\n        tokText = tok[0];\n      } else {\n        tokText = tok;\n      }\n\n      this.#outBuf += tokText;\n\n      if (this.#outBuf.length >= this.#minTokenLength) {\n        this.queue.put({ token: this.#outBuf, segmentId: this.#currentSegmentId });\n        this.#outBuf = '';\n      }\n\n      if (typeof tok! !== 'string') {\n        this.#inBuf = this.#inBuf.slice(tok![2]);\n      } else {\n        this.#inBuf = this.#inBuf\n          .slice(Math.max(0, this.#inBuf.indexOf(tok)) + tok.length)\n          .trimStart();\n      }\n    }\n  }\n\n  /** Flush the stream, causing it to process all pending text */\n  flush() {\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n\n    if (this.#inBuf || this.#outBuf) {\n      const tokens = this.#func(this.#inBuf);\n      if (tokens) {\n        if (this.#outBuf) this.#outBuf += ' ';\n\n        if (Array.isArray(tokens[0])) {\n          this.#outBuf += tokens.map((tok) => tok[0]).join(' ');\n        } else {\n          this.#outBuf += tokens.join(' ');\n        }\n      }\n\n      if (this.#outBuf) {\n        this.queue.put({ token: this.#outBuf, segmentId: this.#currentSegmentId });\n      }\n\n      this.#currentSegmentId = randomUUID();\n    }\n\n    this.#inBuf = '';\n    this.#outBuf = '';\n  }\n\n  /** Mark the input as ended and forbid additional pushes */\n  endInput() {\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.flush();\n    this.close();\n  }\n\n  next(): Promise<IteratorResult<TokenData>> {\n    return this.queue.next();\n  }\n\n  /** Close both the input and output of the token stream */\n  close() {\n    this.queue.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): BufferedTokenStream {\n    return this;\n  }\n}\n\nexport class BufferedSentenceStream extends SentenceStream {\n  #stream: BufferedTokenStream;\n\n  constructor(func: TokenizeFunc, minTokenLength: number, minContextLength: number) {\n    super();\n    this.#stream = new BufferedTokenStream(func, minTokenLength, minContextLength);\n  }\n\n  pushText(text: string) {\n    this.#stream.pushText(text);\n  }\n\n  flush() {\n    this.#stream.flush();\n  }\n\n  close() {\n    super.close();\n    this.#stream.close();\n  }\n\n  next(): Promise<IteratorResult<TokenData>> {\n    return this.#stream.next();\n  }\n}\n\nexport class BufferedWordStream extends WordStream {\n  #stream: BufferedTokenStream;\n\n  constructor(func: TokenizeFunc, minTokenLength: number, minContextLength: number) {\n    super();\n    this.#stream = new BufferedTokenStream(func, minTokenLength, minContextLength);\n  }\n\n  pushText(text: string) {\n    this.#stream.pushText(text);\n  }\n\n  flush() {\n    this.#stream.flush();\n  }\n\n  endInput() {\n    this.#stream.endInput();\n  }\n\n  close() {\n    this.#stream.close();\n  }\n\n  next(): Promise<IteratorResult<TokenData>> {\n    return this.#stream.next();\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,yBAA2B;AAC3B,mBAAmC;AAEnC,uBAA2C;AAIpC,MAAM,oBAAgE;AAAA,EACjE,QAAQ,IAAI,gCAA8B;AAAA,EAC1C,SAAS;AAAA,EAEnB;AAAA,EACA;AAAA,EACA;AAAA,EACA,aAAuB,CAAC;AAAA,EACxB,SAAS;AAAA,EACT,UAAU;AAAA,EACV;AAAA,EAEA,YAAY,MAAoB,gBAAwB,kBAA0B;AAChF,SAAK,QAAQ;AACb,SAAK,kBAAkB;AACvB,SAAK,oBAAoB;AAEzB,SAAK,wBAAoB,+BAAW;AAAA,EACtC;AAAA;AAAA,EAGA,SAAS,MAAc;AACrB,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AAEA,SAAK,UAAU;AACf,QAAI,KAAK,OAAO,SAAS,KAAK,kBAAmB;AAEjD,WAAO,MAAM;AACX,YAAM,SAAS,KAAK,MAAM,KAAK,MAAM;AACrC,UAAI,OAAO,UAAU,EAAG;AAExB,UAAI,KAAK,QAAS,MAAK,WAAW;AAElC,YAAM,MAAM,OAAO,MAAM;AACzB,UAAI;AACJ,UAAI,MAAM,QAAQ,GAAG,GAAG;AACtB,kBAAU,IAAI,CAAC;AAAA,MACjB,OAAO;AACL,kBAAU;AAAA,MACZ;AAEA,WAAK,WAAW;AAEhB,UAAI,KAAK,QAAQ,UAAU,KAAK,iBAAiB;AAC/C,aAAK,MAAM,IAAI,EAAE,OAAO,KAAK,SAAS,WAAW,KAAK,kBAAkB,CAAC;AACzE,aAAK,UAAU;AAAA,MACjB;AAEA,UAAI,OAAO,QAAS,UAAU;AAC5B,aAAK,SAAS,KAAK,OAAO,MAAM,IAAK,CAAC,CAAC;AAAA,MACzC,OAAO;AACL,aAAK,SAAS,KAAK,OAChB,MAAM,KAAK,IAAI,GAAG,KAAK,OAAO,QAAQ,GAAG,CAAC,IAAI,IAAI,MAAM,EACxD,UAAU;AAAA,MACf;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAGA,QAAQ;AACN,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AAEA,QAAI,KAAK,UAAU,KAAK,SAAS;AAC/B,YAAM,SAAS,KAAK,MAAM,KAAK,MAAM;AACrC,UAAI,QAAQ;AACV,YAAI,KAAK,QAAS,MAAK,WAAW;AAElC,YAAI,MAAM,QAAQ,OAAO,CAAC,CAAC,GAAG;AAC5B,eAAK,WAAW,OAAO,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC,EAAE,KAAK,GAAG;AAAA,QACtD,OAAO;AACL,eAAK,WAAW,OAAO,KAAK,GAAG;AAAA,QACjC;AAAA,MACF;AAEA,UAAI,KAAK,SAAS;AAChB,aAAK,MAAM,IAAI,EAAE,OAAO,KAAK,SAAS,WAAW,KAAK,kBAAkB,CAAC;AAAA,MAC3E;AAEA,WAAK,wBAAoB,+BAAW;AAAA,IACtC;AAEA,SAAK,SAAS;AACd,SAAK,UAAU;AAAA,EACjB;AAAA;AAAA,EAGA,WAAW;AACT,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM;AACX,SAAK,MAAM;AAAA,EACb;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,MAAM,KAAK;AAAA,EACzB;AAAA;AAAA,EAGA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAyB;AAC5C,WAAO;AAAA,EACT;AACF;AAEO,MAAM,+BAA+B,gCAAe;AAAA,EACzD;AAAA,EAEA,YAAY,MAAoB,gBAAwB,kBAA0B;AAChF,UAAM;AACN,SAAK,UAAU,IAAI,oBAAoB,MAAM,gBAAgB,gBAAgB;AAAA,EAC/E;AAAA,EAEA,SAAS,MAAc;AACrB,SAAK,QAAQ,SAAS,IAAI;AAAA,EAC5B;AAAA,EAEA,QAAQ;AACN,SAAK,QAAQ,MAAM;AAAA,EACrB;AAAA,EAEA,QAAQ;AACN,UAAM,MAAM;AACZ,SAAK,QAAQ,MAAM;AAAA,EACrB;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,QAAQ,KAAK;AAAA,EAC3B;AACF;AAEO,MAAM,2BAA2B,4BAAW;AAAA,EACjD;AAAA,EAEA,YAAY,MAAoB,gBAAwB,kBAA0B;AAChF,UAAM;AACN,SAAK,UAAU,IAAI,oBAAoB,MAAM,gBAAgB,gBAAgB;AAAA,EAC/E;AAAA,EAEA,SAAS,MAAc;AACrB,SAAK,QAAQ,SAAS,IAAI;AAAA,EAC5B;AAAA,EAEA,QAAQ;AACN,SAAK,QAAQ,MAAM;AAAA,EACrB;AAAA,EAEA,WAAW;AACT,SAAK,QAAQ,SAAS;AAAA,EACxB;AAAA,EAEA,QAAQ;AACN,SAAK,QAAQ,MAAM;AAAA,EACrB;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,QAAQ,KAAK;AAAA,EAC3B;AACF;","names":[]}

package/dist/tokenize/token_stream.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"token_stream.d.ts","sourceRoot":"","sources":["../../src/tokenize/token_stream.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAE5D,KAAK,YAAY,GAAG,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;AAEzE,qBAAa,mBAAoB,YAAW,qBAAqB,CAAC,SAAS,CAAC;;IAC1E,SAAS,CAAC,KAAK,gCAAuC;IACtD,SAAS,CAAC,MAAM,UAAS;gBAUb,IAAI,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM;IAQhF,kDAAkD;IAClD,QAAQ,CAAC,IAAI,EAAE,MAAM;IAoCrB,+DAA+D;IAC/D,KAAK;IA4BL,2DAA2D;IAC3D,QAAQ;IAQR,IAAI,IAAI,OAAO,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;IAI1C,0DAA0D;IAC1D,KAAK;IAKL,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,mBAAmB;CAG9C;AAED,qBAAa,sBAAuB,SAAQ,cAAc;;gBAG5C,IAAI,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM;IAKhF,QAAQ,CAAC,IAAI,EAAE,MAAM;IAIrB,KAAK;IAIL,KAAK;IAKL,IAAI,IAAI,OAAO,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;CAG3C;AAED,qBAAa,kBAAmB,SAAQ,UAAU;;gBAGpC,IAAI,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM;IAKhF,QAAQ,CAAC,IAAI,EAAE,MAAM;IAIrB,KAAK;IAIL,QAAQ;IAIR,KAAK;IAIL,IAAI,IAAI,OAAO,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;CAG3C"}
+{"version":3,"file":"token_stream.d.ts","sourceRoot":"","sources":["../../src/tokenize/token_stream.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAE5D,KAAK,YAAY,GAAG,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;AAEzE,qBAAa,mBAAoB,YAAW,qBAAqB,CAAC,SAAS,CAAC;;IAC1E,SAAS,CAAC,KAAK,gCAAuC;IACtD,SAAS,CAAC,MAAM,UAAS;gBAUb,IAAI,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM;IAQhF,kDAAkD;IAClD,QAAQ,CAAC,IAAI,EAAE,MAAM;IAuCrB,+DAA+D;IAC/D,KAAK;IA4BL,2DAA2D;IAC3D,QAAQ;IAQR,IAAI,IAAI,OAAO,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;IAI1C,0DAA0D;IAC1D,KAAK;IAKL,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,mBAAmB;CAG9C;AAED,qBAAa,sBAAuB,SAAQ,cAAc;;gBAG5C,IAAI,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM;IAKhF,QAAQ,CAAC,IAAI,EAAE,MAAM;IAIrB,KAAK;IAIL,KAAK;IAKL,IAAI,IAAI,OAAO,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;CAG3C;AAED,qBAAa,kBAAmB,SAAQ,UAAU;;gBAGpC,IAAI,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM;IAKhF,QAAQ,CAAC,IAAI,EAAE,MAAM;IAIrB,KAAK;IAIL,QAAQ;IAIR,KAAK;IAIL,IAAI,IAAI,OAAO,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;CAG3C"}

package/dist/tokenize/token_stream.js

@@ -29,9 +29,11 @@ class BufferedTokenStream {
       if (tokens.length <= 1) break;
       if (this.#outBuf) this.#outBuf += " ";
       const tok = tokens.shift();
-      let tokText = tok;
-      if (tok.length > 1 && typeof tok[1] === "number") {
+      let tokText;
+      if (Array.isArray(tok)) {
         tokText = tok[0];
+      } else {
+        tokText = tok;
       }
       this.#outBuf += tokText;
       if (this.#outBuf.length >= this.#minTokenLength) {
@@ -54,7 +56,7 @@ class BufferedTokenStream {
       const tokens = this.#func(this.#inBuf);
       if (tokens) {
         if (this.#outBuf) this.#outBuf += " ";
-        if (typeof tokens[0] !== "string") {
+        if (Array.isArray(tokens[0])) {
           this.#outBuf += tokens.map((tok) => tok[0]).join(" ");
         } else {
           this.#outBuf += tokens.join(" ");

package/dist/tokenize/token_stream.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/tokenize/token_stream.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { randomUUID } from 'node:crypto';\nimport { AsyncIterableQueue } from '../utils.js';\nimport type { TokenData } from './tokenizer.js';\nimport { SentenceStream, WordStream } from './tokenizer.js';\n\ntype TokenizeFunc = (x: string) => string[] | [string, number, number][];\n\nexport class BufferedTokenStream implements AsyncIterableIterator<TokenData> {\n  protected queue = new AsyncIterableQueue<TokenData>();\n  protected closed = false;\n\n  #func: TokenizeFunc;\n  #minTokenLength: number;\n  #minContextLength: number;\n  #bufTokens: string[] = [];\n  #inBuf = '';\n  #outBuf = '';\n  #currentSegmentId: string;\n\n  constructor(func: TokenizeFunc, minTokenLength: number, minContextLength: number) {\n    this.#func = func;\n    this.#minTokenLength = minTokenLength;\n    this.#minContextLength = minContextLength;\n\n    this.#currentSegmentId = randomUUID();\n  }\n\n  /** Push a string of text into the token stream */\n  pushText(text: string) {\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n\n    this.#inBuf += text;\n    if (this.#inBuf.length < this.#minContextLength) return;\n\n    while (true) {\n      const tokens = this.#func(this.#inBuf);\n      if (tokens.length <= 1) break;\n\n      if (this.#outBuf) this.#outBuf += ' ';\n\n      const tok = tokens.shift()!;\n      let tokText = tok as string;\n      if (tok.length > 1 && typeof tok[1] === 'number') {\n        tokText = tok[0];\n      }\n\n      this.#outBuf += tokText;\n      if (this.#outBuf.length >= this.#minTokenLength) {\n        this.queue.put({ token: this.#outBuf, segmentId: this.#currentSegmentId });\n        this.#outBuf = '';\n      }\n\n      if (typeof tok! !== 'string') {\n        this.#inBuf = this.#inBuf.slice(tok![2]);\n      } else {\n        this.#inBuf = this.#inBuf\n          .slice(Math.max(0, this.#inBuf.indexOf(tok)) + tok.length)\n          .trimStart();\n      }\n    }\n  }\n\n  /** Flush the stream, causing it to process all pending text */\n  flush() {\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n\n    if (this.#inBuf || this.#outBuf) {\n      const tokens = this.#func(this.#inBuf);\n      if (tokens) {\n        if (this.#outBuf) this.#outBuf += ' ';\n\n        if (typeof tokens[0] !== 'string') {\n          this.#outBuf += tokens.map((tok) => tok[0]).join(' ');\n        } else {\n          this.#outBuf += tokens.join(' ');\n        }\n      }\n\n      if (this.#outBuf) {\n        this.queue.put({ token: this.#outBuf, segmentId: this.#currentSegmentId });\n      }\n\n      this.#currentSegmentId = randomUUID();\n    }\n\n    this.#inBuf = '';\n    this.#outBuf = '';\n  }\n\n  /** Mark the input as ended and forbid additional pushes */\n  endInput() {\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.flush();\n    this.close();\n  }\n\n  next(): Promise<IteratorResult<TokenData>> {\n    return this.queue.next();\n  }\n\n  /** Close both the input and output of the token stream */\n  close() {\n    this.queue.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): BufferedTokenStream {\n    return this;\n  }\n}\n\nexport class BufferedSentenceStream extends SentenceStream {\n  #stream: BufferedTokenStream;\n\n  constructor(func: TokenizeFunc, minTokenLength: number, minContextLength: number) {\n    super();\n    this.#stream = new BufferedTokenStream(func, minTokenLength, minContextLength);\n  }\n\n  pushText(text: string) {\n    this.#stream.pushText(text);\n  }\n\n  flush() {\n    this.#stream.flush();\n  }\n\n  close() {\n    super.close();\n    this.#stream.close();\n  }\n\n  next(): Promise<IteratorResult<TokenData>> {\n    return this.#stream.next();\n  }\n}\n\nexport class BufferedWordStream extends WordStream {\n  #stream: BufferedTokenStream;\n\n  constructor(func: TokenizeFunc, minTokenLength: number, minContextLength: number) {\n    super();\n    this.#stream = new BufferedTokenStream(func, minTokenLength, minContextLength);\n  }\n\n  pushText(text: string) {\n    this.#stream.pushText(text);\n  }\n\n  flush() {\n    this.#stream.flush();\n  }\n\n  endInput() {\n    this.#stream.endInput();\n  }\n\n  close() {\n    this.#stream.close();\n  }\n\n  next(): Promise<IteratorResult<TokenData>> {\n    return this.#stream.next();\n  }\n}\n"],"mappings":"AAGA,SAAS,kBAAkB;AAC3B,SAAS,0BAA0B;AAEnC,SAAS,gBAAgB,kBAAkB;AAIpC,MAAM,oBAAgE;AAAA,EACjE,QAAQ,IAAI,mBAA8B;AAAA,EAC1C,SAAS;AAAA,EAEnB;AAAA,EACA;AAAA,EACA;AAAA,EACA,aAAuB,CAAC;AAAA,EACxB,SAAS;AAAA,EACT,UAAU;AAAA,EACV;AAAA,EAEA,YAAY,MAAoB,gBAAwB,kBAA0B;AAChF,SAAK,QAAQ;AACb,SAAK,kBAAkB;AACvB,SAAK,oBAAoB;AAEzB,SAAK,oBAAoB,WAAW;AAAA,EACtC;AAAA;AAAA,EAGA,SAAS,MAAc;AACrB,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AAEA,SAAK,UAAU;AACf,QAAI,KAAK,OAAO,SAAS,KAAK,kBAAmB;AAEjD,WAAO,MAAM;AACX,YAAM,SAAS,KAAK,MAAM,KAAK,MAAM;AACrC,UAAI,OAAO,UAAU,EAAG;AAExB,UAAI,KAAK,QAAS,MAAK,WAAW;AAElC,YAAM,MAAM,OAAO,MAAM;AACzB,UAAI,UAAU;AACd,UAAI,IAAI,SAAS,KAAK,OAAO,IAAI,CAAC,MAAM,UAAU;AAChD,kBAAU,IAAI,CAAC;AAAA,MACjB;AAEA,WAAK,WAAW;AAChB,UAAI,KAAK,QAAQ,UAAU,KAAK,iBAAiB;AAC/C,aAAK,MAAM,IAAI,EAAE,OAAO,KAAK,SAAS,WAAW,KAAK,kBAAkB,CAAC;AACzE,aAAK,UAAU;AAAA,MACjB;AAEA,UAAI,OAAO,QAAS,UAAU;AAC5B,aAAK,SAAS,KAAK,OAAO,MAAM,IAAK,CAAC,CAAC;AAAA,MACzC,OAAO;AACL,aAAK,SAAS,KAAK,OAChB,MAAM,KAAK,IAAI,GAAG,KAAK,OAAO,QAAQ,GAAG,CAAC,IAAI,IAAI,MAAM,EACxD,UAAU;AAAA,MACf;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAGA,QAAQ;AACN,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AAEA,QAAI,KAAK,UAAU,KAAK,SAAS;AAC/B,YAAM,SAAS,KAAK,MAAM,KAAK,MAAM;AACrC,UAAI,QAAQ;AACV,YAAI,KAAK,QAAS,MAAK,WAAW;AAElC,YAAI,OAAO,OAAO,CAAC,MAAM,UAAU;AACjC,eAAK,WAAW,OAAO,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC,EAAE,KAAK,GAAG;AAAA,QACtD,OAAO;AACL,eAAK,WAAW,OAAO,KAAK,GAAG;AAAA,QACjC;AAAA,MACF;AAEA,UAAI,KAAK,SAAS;AAChB,aAAK,MAAM,IAAI,EAAE,OAAO,KAAK,SAAS,WAAW,KAAK,kBAAkB,CAAC;AAAA,MAC3E;AAEA,WAAK,oBAAoB,WAAW;AAAA,IACtC;AAEA,SAAK,SAAS;AACd,SAAK,UAAU;AAAA,EACjB;AAAA;AAAA,EAGA,WAAW;AACT,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM;AACX,SAAK,MAAM;AAAA,EACb;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,MAAM,KAAK;AAAA,EACzB;AAAA;AAAA,EAGA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAyB;AAC5C,WAAO;AAAA,EACT;AACF;AAEO,MAAM,+BAA+B,eAAe;AAAA,EACzD;AAAA,EAEA,YAAY,MAAoB,gBAAwB,kBAA0B;AAChF,UAAM;AACN,SAAK,UAAU,IAAI,oBAAoB,MAAM,gBAAgB,gBAAgB;AAAA,EAC/E;AAAA,EAEA,SAAS,MAAc;AACrB,SAAK,QAAQ,SAAS,IAAI;AAAA,EAC5B;AAAA,EAEA,QAAQ;AACN,SAAK,QAAQ,MAAM;AAAA,EACrB;AAAA,EAEA,QAAQ;AACN,UAAM,MAAM;AACZ,SAAK,QAAQ,MAAM;AAAA,EACrB;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,QAAQ,KAAK;AAAA,EAC3B;AACF;AAEO,MAAM,2BAA2B,WAAW;AAAA,EACjD;AAAA,EAEA,YAAY,MAAoB,gBAAwB,kBAA0B;AAChF,UAAM;AACN,SAAK,UAAU,IAAI,oBAAoB,MAAM,gBAAgB,gBAAgB;AAAA,EAC/E;AAAA,EAEA,SAAS,MAAc;AACrB,SAAK,QAAQ,SAAS,IAAI;AAAA,EAC5B;AAAA,EAEA,QAAQ;AACN,SAAK,QAAQ,MAAM;AAAA,EACrB;AAAA,EAEA,WAAW;AACT,SAAK,QAAQ,SAAS;AAAA,EACxB;AAAA,EAEA,QAAQ;AACN,SAAK,QAAQ,MAAM;AAAA,EACrB;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,QAAQ,KAAK;AAAA,EAC3B;AACF;","names":[]}
+{"version":3,"sources":["../../src/tokenize/token_stream.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { randomUUID } from 'node:crypto';\nimport { AsyncIterableQueue } from '../utils.js';\nimport type { TokenData } from './tokenizer.js';\nimport { SentenceStream, WordStream } from './tokenizer.js';\n\ntype TokenizeFunc = (x: string) => string[] | [string, number, number][];\n\nexport class BufferedTokenStream implements AsyncIterableIterator<TokenData> {\n  protected queue = new AsyncIterableQueue<TokenData>();\n  protected closed = false;\n\n  #func: TokenizeFunc;\n  #minTokenLength: number;\n  #minContextLength: number;\n  #bufTokens: string[] = [];\n  #inBuf = '';\n  #outBuf = '';\n  #currentSegmentId: string;\n\n  constructor(func: TokenizeFunc, minTokenLength: number, minContextLength: number) {\n    this.#func = func;\n    this.#minTokenLength = minTokenLength;\n    this.#minContextLength = minContextLength;\n\n    this.#currentSegmentId = randomUUID();\n  }\n\n  /** Push a string of text into the token stream */\n  pushText(text: string) {\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n\n    this.#inBuf += text;\n    if (this.#inBuf.length < this.#minContextLength) return;\n\n    while (true) {\n      const tokens = this.#func(this.#inBuf);\n      if (tokens.length <= 1) break;\n\n      if (this.#outBuf) this.#outBuf += ' ';\n\n      const tok = tokens.shift()!;\n      let tokText: string;\n      if (Array.isArray(tok)) {\n        tokText = tok[0];\n      } else {\n        tokText = tok;\n      }\n\n      this.#outBuf += tokText;\n\n      if (this.#outBuf.length >= this.#minTokenLength) {\n        this.queue.put({ token: this.#outBuf, segmentId: this.#currentSegmentId });\n        this.#outBuf = '';\n      }\n\n      if (typeof tok! !== 'string') {\n        this.#inBuf = this.#inBuf.slice(tok![2]);\n      } else {\n        this.#inBuf = this.#inBuf\n          .slice(Math.max(0, this.#inBuf.indexOf(tok)) + tok.length)\n          .trimStart();\n      }\n    }\n  }\n\n  /** Flush the stream, causing it to process all pending text */\n  flush() {\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n\n    if (this.#inBuf || this.#outBuf) {\n      const tokens = this.#func(this.#inBuf);\n      if (tokens) {\n        if (this.#outBuf) this.#outBuf += ' ';\n\n        if (Array.isArray(tokens[0])) {\n          this.#outBuf += tokens.map((tok) => tok[0]).join(' ');\n        } else {\n          this.#outBuf += tokens.join(' ');\n        }\n      }\n\n      if (this.#outBuf) {\n        this.queue.put({ token: this.#outBuf, segmentId: this.#currentSegmentId });\n      }\n\n      this.#currentSegmentId = randomUUID();\n    }\n\n    this.#inBuf = '';\n    this.#outBuf = '';\n  }\n\n  /** Mark the input as ended and forbid additional pushes */\n  endInput() {\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.flush();\n    this.close();\n  }\n\n  next(): Promise<IteratorResult<TokenData>> {\n    return this.queue.next();\n  }\n\n  /** Close both the input and output of the token stream */\n  close() {\n    this.queue.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): BufferedTokenStream {\n    return this;\n  }\n}\n\nexport class BufferedSentenceStream extends SentenceStream {\n  #stream: BufferedTokenStream;\n\n  constructor(func: TokenizeFunc, minTokenLength: number, minContextLength: number) {\n    super();\n    this.#stream = new BufferedTokenStream(func, minTokenLength, minContextLength);\n  }\n\n  pushText(text: string) {\n    this.#stream.pushText(text);\n  }\n\n  flush() {\n    this.#stream.flush();\n  }\n\n  close() {\n    super.close();\n    this.#stream.close();\n  }\n\n  next(): Promise<IteratorResult<TokenData>> {\n    return this.#stream.next();\n  }\n}\n\nexport class BufferedWordStream extends WordStream {\n  #stream: BufferedTokenStream;\n\n  constructor(func: TokenizeFunc, minTokenLength: number, minContextLength: number) {\n    super();\n    this.#stream = new BufferedTokenStream(func, minTokenLength, minContextLength);\n  }\n\n  pushText(text: string) {\n    this.#stream.pushText(text);\n  }\n\n  flush() {\n    this.#stream.flush();\n  }\n\n  endInput() {\n    this.#stream.endInput();\n  }\n\n  close() {\n    this.#stream.close();\n  }\n\n  next(): Promise<IteratorResult<TokenData>> {\n    return this.#stream.next();\n  }\n}\n"],"mappings":"AAGA,SAAS,kBAAkB;AAC3B,SAAS,0BAA0B;AAEnC,SAAS,gBAAgB,kBAAkB;AAIpC,MAAM,oBAAgE;AAAA,EACjE,QAAQ,IAAI,mBAA8B;AAAA,EAC1C,SAAS;AAAA,EAEnB;AAAA,EACA;AAAA,EACA;AAAA,EACA,aAAuB,CAAC;AAAA,EACxB,SAAS;AAAA,EACT,UAAU;AAAA,EACV;AAAA,EAEA,YAAY,MAAoB,gBAAwB,kBAA0B;AAChF,SAAK,QAAQ;AACb,SAAK,kBAAkB;AACvB,SAAK,oBAAoB;AAEzB,SAAK,oBAAoB,WAAW;AAAA,EACtC;AAAA;AAAA,EAGA,SAAS,MAAc;AACrB,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AAEA,SAAK,UAAU;AACf,QAAI,KAAK,OAAO,SAAS,KAAK,kBAAmB;AAEjD,WAAO,MAAM;AACX,YAAM,SAAS,KAAK,MAAM,KAAK,MAAM;AACrC,UAAI,OAAO,UAAU,EAAG;AAExB,UAAI,KAAK,QAAS,MAAK,WAAW;AAElC,YAAM,MAAM,OAAO,MAAM;AACzB,UAAI;AACJ,UAAI,MAAM,QAAQ,GAAG,GAAG;AACtB,kBAAU,IAAI,CAAC;AAAA,MACjB,OAAO;AACL,kBAAU;AAAA,MACZ;AAEA,WAAK,WAAW;AAEhB,UAAI,KAAK,QAAQ,UAAU,KAAK,iBAAiB;AAC/C,aAAK,MAAM,IAAI,EAAE,OAAO,KAAK,SAAS,WAAW,KAAK,kBAAkB,CAAC;AACzE,aAAK,UAAU;AAAA,MACjB;AAEA,UAAI,OAAO,QAAS,UAAU;AAC5B,aAAK,SAAS,KAAK,OAAO,MAAM,IAAK,CAAC,CAAC;AAAA,MACzC,OAAO;AACL,aAAK,SAAS,KAAK,OAChB,MAAM,KAAK,IAAI,GAAG,KAAK,OAAO,QAAQ,GAAG,CAAC,IAAI,IAAI,MAAM,EACxD,UAAU;AAAA,MACf;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAGA,QAAQ;AACN,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AAEA,QAAI,KAAK,UAAU,KAAK,SAAS;AAC/B,YAAM,SAAS,KAAK,MAAM,KAAK,MAAM;AACrC,UAAI,QAAQ;AACV,YAAI,KAAK,QAAS,MAAK,WAAW;AAElC,YAAI,MAAM,QAAQ,OAAO,CAAC,CAAC,GAAG;AAC5B,eAAK,WAAW,OAAO,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC,EAAE,KAAK,GAAG;AAAA,QACtD,OAAO;AACL,eAAK,WAAW,OAAO,KAAK,GAAG;AAAA,QACjC;AAAA,MACF;AAEA,UAAI,KAAK,SAAS;AAChB,aAAK,MAAM,IAAI,EAAE,OAAO,KAAK,SAAS,WAAW,KAAK,kBAAkB,CAAC;AAAA,MAC3E;AAEA,WAAK,oBAAoB,WAAW;AAAA,IACtC;AAEA,SAAK,SAAS;AACd,SAAK,UAAU;AAAA,EACjB;AAAA;AAAA,EAGA,WAAW;AACT,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM;AACX,SAAK,MAAM;AAAA,EACb;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,MAAM,KAAK;AAAA,EACzB;AAAA;AAAA,EAGA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAyB;AAC5C,WAAO;AAAA,EACT;AACF;AAEO,MAAM,+BAA+B,eAAe;AAAA,EACzD;AAAA,EAEA,YAAY,MAAoB,gBAAwB,kBAA0B;AAChF,UAAM;AACN,SAAK,UAAU,IAAI,oBAAoB,MAAM,gBAAgB,gBAAgB;AAAA,EAC/E;AAAA,EAEA,SAAS,MAAc;AACrB,SAAK,QAAQ,SAAS,IAAI;AAAA,EAC5B;AAAA,EAEA,QAAQ;AACN,SAAK,QAAQ,MAAM;AAAA,EACrB;AAAA,EAEA,QAAQ;AACN,UAAM,MAAM;AACZ,SAAK,QAAQ,MAAM;AAAA,EACrB;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,QAAQ,KAAK;AAAA,EAC3B;AACF;AAEO,MAAM,2BAA2B,WAAW;AAAA,EACjD;AAAA,EAEA,YAAY,MAAoB,gBAAwB,kBAA0B;AAChF,UAAM;AACN,SAAK,UAAU,IAAI,oBAAoB,MAAM,gBAAgB,gBAAgB;AAAA,EAC/E;AAAA,EAEA,SAAS,MAAc;AACrB,SAAK,QAAQ,SAAS,IAAI;AAAA,EAC5B;AAAA,EAEA,QAAQ;AACN,SAAK,QAAQ,MAAM;AAAA,EACrB;AAAA,EAEA,WAAW;AACT,SAAK,QAAQ,SAAS;AAAA,EACxB;AAAA,EAEA,QAAQ;AACN,SAAK,QAAQ,MAAM;AAAA,EACrB;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,QAAQ,KAAK;AAAA,EAC3B;AACF;","names":[]}

package/dist/transcription.cjs

@@ -18,114 +18,231 @@ var __copyProps = (to, from, except, desc) => {
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 var transcription_exports = {};
 __export(transcription_exports, {
-  BasicTranscriptionForwarder: () => BasicTranscriptionForwarder
+  TextAudioSynchronizer: () => TextAudioSynchronizer,
+  defaultTextSyncOptions: () => defaultTextSyncOptions
 });
 module.exports = __toCommonJS(transcription_exports);
-var import_log = require("./log.cjs");
-class BasicTranscriptionForwarder {
-  #room;
-  #participantIdentity;
-  #trackSid;
-  #currentText = "";
-  #totalAudioDuration = 0;
-  #currentPlayoutTime = 0;
-  #DEFAULT_CHARS_PER_SECOND = 16;
-  #charsPerSecond = this.#DEFAULT_CHARS_PER_SECOND;
-  #messageId;
-  #isRunning = false;
-  #logger = (0, import_log.log)();
-  currentCharacterIndex = 0;
-  constructor(room, participantIdentity, trackSid, messageId) {
-    this.#room = room;
-    this.#participantIdentity = participantIdentity;
-    this.#trackSid = trackSid;
-    this.#messageId = messageId;
-  }
-  get text() {
-    return this.#currentText;
-  }
-  start() {
-    if (!this.#isRunning) {
-      this.#isRunning = true;
-      this.#startPublishingLoop().catch((error) => {
-        this.#logger.error("Error in publishing loop:", error);
-        this.#isRunning = false;
-      });
-    }
+var import_protocol = require("@livekit/protocol");
+var import_rtc_node = require("@livekit/rtc-node");
+var import_node_crypto = require("node:crypto");
+var import_node_events = require("node:events");
+var import_tokenize = require("./tokenize/index.cjs");
+var import_utils = require("./utils.cjs");
+const STANDARD_SPEECH_RATE = 3830;
+const defaultTextSyncOptions = {
+  language: "",
+  speed: 1,
+  newSentenceDelay: 400,
+  sentenceTokenizer: new import_tokenize.basic.SentenceTokenizer(),
+  hyphenateWord: import_tokenize.basic.hyphenateWord,
+  splitWords: import_tokenize.basic.splitWords
+};
+class TextAudioSynchronizer extends import_node_events.EventEmitter {
+  #opts;
+  #speed;
+  #closed = false;
+  #interrupted = false;
+  #closeFut = new import_utils.Future();
+  #playingSegIndex = -1;
+  #finishedSegIndex = -1;
+  #textQChanged = new import_utils.AsyncIterableQueue();
+  #textQ = [];
+  #audioQChanged = new import_utils.AsyncIterableQueue();
+  #audioQ = [];
+  #playedText = "";
+  #task;
+  #audioData;
+  #textData;
+  constructor(opts) {
+    super();
+    this.#opts = opts;
+    this.#speed = opts.speed * STANDARD_SPEECH_RATE;
   }
   pushAudio(frame) {
-    this.#totalAudioDuration += frame.samplesPerChannel / frame.sampleRate;
+    this.#checkNotClosed();
+    if (!this.#audioData) {
+      this.#audioData = { pushedDuration: 0, done: false };
+      this.#audioQ.push(this.#audioData);
+      this.#audioQChanged.put(1);
+    }
+    this.#audioData.pushedDuration += frame.samplesPerChannel / frame.sampleRate;
   }
   pushText(text) {
-    this.#currentText += text;
+    this.#checkNotClosed();
+    if (!this.#textData) {
+      this.#textData = {
+        sentenceStream: this.#opts.sentenceTokenizer.stream(),
+        pushedText: "",
+        done: false,
+        forwardedHyphens: 0,
+        forwardedSentences: 0
+      };
+      this.#textQ.push(this.#textData);
+      this.#textQChanged.put(1);
+    }
+    this.#textData.pushedText += text;
+    this.#textData.sentenceStream.pushText(text);
   }
-  #textIsComplete = false;
-  #audioIsComplete = false;
-  markTextComplete() {
-    this.#textIsComplete = true;
-    this.#adjustTimingIfBothFinished();
+  markAudioSegmentEnd() {
+    this.#checkNotClosed();
+    if (!this.#audioData) {
+      this.pushAudio(new import_rtc_node.AudioFrame(new Int16Array(), 24e3, 1, 0));
+    }
+    this.#audioData.done = true;
+    this.#audioData = void 0;
   }
-  markAudioComplete() {
-    this.#audioIsComplete = true;
-    this.#adjustTimingIfBothFinished();
+  markTextSegmentEnd() {
+    var _a, _b;
+    this.#checkNotClosed();
+    if (!this.#textData) {
+      this.pushText("");
+    }
+    this.#textData.done = true;
+    (_a = this.#textData) == null ? void 0 : _a.sentenceStream.flush();
+    (_b = this.#textData) == null ? void 0 : _b.sentenceStream.close();
+    this.#textData = void 0;
   }
-  #adjustTimingIfBothFinished() {
-    if (this.#textIsComplete && this.#audioIsComplete) {
-      const actualDuration = this.#totalAudioDuration;
-      if (actualDuration > 0 && this.#currentText.length > 0) {
-        this.#charsPerSecond = this.#currentText.length / actualDuration;
-      }
+  segmentPlayoutStarted() {
+    this.#checkNotClosed();
+    this.#playingSegIndex++;
+    if (!this.#task) {
+      this.#task = this.#mainLoop();
     }
   }
-  #computeSleepInterval() {
-    return Math.min(Math.max(1 / this.#charsPerSecond, 0.0625), 0.5);
+  segmentPlayoutFinished() {
+    this.#checkNotClosed();
+    this.#finishedSegIndex++;
   }
-  async #startPublishingLoop() {
-    this.#isRunning = true;
-    let sleepInterval = this.#computeSleepInterval();
-    let isComplete = false;
-    while (this.#isRunning && !isComplete) {
-      this.#currentPlayoutTime += sleepInterval;
-      this.currentCharacterIndex = Math.floor(this.#currentPlayoutTime * this.#charsPerSecond);
-      isComplete = this.#textIsComplete && this.currentCharacterIndex >= this.#currentText.length;
-      await this.#publishTranscription(false);
-      if (this.#isRunning && !isComplete) {
-        sleepInterval = this.#computeSleepInterval();
-        await new Promise((resolve) => setTimeout(resolve, sleepInterval * 1e3));
-      }
+  get playedText() {
+    return this.#playedText;
+  }
+  async close(interrupt) {
+    if (this.#closed) {
+      return;
+    }
+    this.#closed = true;
+    this.#interrupted = interrupt;
+    this.#closeFut.resolve();
+    for (const textData of this.#textQ) {
+      textData == null ? void 0 : textData.sentenceStream.close();
     }
-    if (this.#isRunning) {
-      this.close(false);
+    this.#textQ.push(void 0);
+    this.#audioQ.push(void 0);
+    this.#textQChanged.put(1);
+    this.#audioQChanged.put(1);
+    await this.#task;
+  }
+  async #mainLoop() {
+    let segIndex = 0;
+    let qDone = false;
+    while (!qDone) {
+      await this.#textQChanged.next();
+      await this.#audioQChanged.next();
+      while (this.#textQ.length && this.#audioQ.length) {
+        const textData = this.#textQ.pop();
+        const audioData = this.#audioQ.pop();
+        if (!(textData && audioData)) {
+          qDone = true;
+          break;
+        }
+        while (!this.#closed) {
+          if (this.#playingSegIndex >= segIndex) break;
+          await this.#sleepIfNotClosed(125);
+        }
+        const sentenceStream = textData.sentenceStream;
+        const forwardStartTime = Date.now();
+        for await (const ev of sentenceStream) {
+          await this.#syncSentence(segIndex, forwardStartTime, textData, audioData, ev.token);
+        }
+        segIndex++;
+      }
     }
   }
-  async #publishTranscription(final) {
-    var _a;
-    const textToPublish = this.#currentText.slice(0, this.currentCharacterIndex);
-    await ((_a = this.#room.localParticipant) == null ? void 0 : _a.publishTranscription({
-      participantIdentity: this.#participantIdentity,
-      trackSid: this.#trackSid,
-      segments: [
-        {
-          text: textToPublish,
-          final,
-          id: this.#messageId,
+  async #syncSentence(segIndex, segStartTime, textData, audioData, sentence) {
+    let realSpeed;
+    if (audioData.pushedDuration > 0 && audioData.done) {
+      realSpeed = this.#calcHyphens(textData.pushedText).length / audioData.pushedDuration;
+    }
+    const segId = "SG_" + (0, import_node_crypto.randomUUID)();
+    const words = this.#opts.splitWords(sentence);
+    const processedWords = [];
+    const ogText = this.#playedText;
+    for (const [word, _, end] of words) {
+      if (segIndex <= this.#finishedSegIndex) break;
+      if (this.#interrupted) return;
+      const wordHyphens = this.#opts.hyphenateWord(word).length;
+      processedWords.push(word);
+      const elapsed = Date.now() - segStartTime;
+      const text = sentence.slice(0, end);
+      let speed = this.#speed;
+      let delay;
+      if (realSpeed) {
+        speed = realSpeed;
+        const estimatedPausesMs = textData.forwardedSentences * this.#opts.newSentenceDelay;
+        const hyphPauses = estimatedPausesMs * speed;
+        const targetHyphens = Math.round(speed * elapsed);
+        const dt = targetHyphens - textData.forwardedHyphens - hyphPauses;
+        const toWaitHyphens = Math.max(0, wordHyphens - dt);
+        delay = toWaitHyphens / speed;
+      } else {
+        delay = wordHyphens / speed;
+      }
+      const firstDelay = Math.min(delay / 2, 2 / speed);
+      await this.#sleepIfNotClosed(firstDelay * 1e6);
+      this.emit(
+        "textUpdated",
+        new import_protocol.TranscriptionSegment({
+          id: segId,
+          text,
           startTime: BigInt(0),
           endTime: BigInt(0),
-          language: ""
-        }
-      ]
-    }));
+          final: false,
+          language: this.#opts.language
+        })
+      );
+      this.#playedText = `${ogText} ${text}`;
+      await this.#sleepIfNotClosed((delay - firstDelay) * 1e6);
+      textData.forwardedHyphens += wordHyphens;
+    }
+    this.emit(
+      "textUpdated",
+      new import_protocol.TranscriptionSegment({
+        id: segId,
+        text: sentence,
+        startTime: BigInt(0),
+        endTime: BigInt(0),
+        final: true,
+        language: this.#opts.language
+      })
+    );
+    this.#playedText = `${ogText} ${sentence}`;
+    await this.#sleepIfNotClosed(this.#opts.newSentenceDelay);
+    textData.forwardedSentences++;
   }
-  async close(interrupt) {
-    this.#isRunning = false;
-    if (!interrupt) {
-      this.currentCharacterIndex = this.#currentText.length;
+  async #sleepIfNotClosed(delay) {
+    await Promise.race([
+      this.#closeFut.await,
+      new Promise((resolve) => setTimeout(resolve, delay))
+    ]);
+  }
+  #calcHyphens(text) {
+    const hyphens = [];
+    const words = this.#opts.splitWords(text);
+    for (const word of words) {
+      const n = this.#opts.hyphenateWord(word[0]);
+      hyphens.push(...n);
+    }
+    return hyphens;
+  }
+  #checkNotClosed() {
+    if (this.#closed) {
+      throw new Error("TextAudioSynchronizer is closed");
     }
-    await this.#publishTranscription(true);
   }
 }
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
-  BasicTranscriptionForwarder
+  TextAudioSynchronizer,
+  defaultTextSyncOptions
 });
 //# sourceMappingURL=transcription.cjs.map