@livekit/agents 0.7.9 → 1.0.0-next.0

package/dist/stt/stt.d.ts

@@ -1,6 +1,9 @@
-import type { AudioFrame } from '@livekit/rtc-node';
+/// <reference types="node" resolution-mode="require"/>
+import { type AudioFrame, AudioResampler } from '@livekit/rtc-node';
 import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
+import type { ReadableStream } from 'node:stream/web';
 import type { STTMetrics } from '../metrics/base.js';
+import { type APIConnectOptions } from '../types.js';
 import type { AudioBuffer } from '../utils.js';
 import { AsyncIterableQueue } from '../utils.js';
 /** Indicates start/middle/end of speech */
@@ -26,8 +29,7 @@ export declare enum SpeechEventType {
      */
     END_OF_SPEECH = 3,
     /** Usage event, emitted periodically to indicate usage metrics. */
-    RECOGNITION_USAGE = 4,
-    METRICS_COLLECTED = 5
+    RECOGNITION_USAGE = 4
 }
 /** SpeechData contains metadata about this {@link SpeechEvent}. */
 export interface SpeechData {
@@ -57,8 +59,16 @@ export interface STTCapabilities {
     streaming: boolean;
     interimResults: boolean;
 }
+export interface STTError {
+    type: 'stt_error';
+    timestamp: number;
+    label: string;
+    error: Error;
+    recoverable: boolean;
+}
 export type STTCallbacks = {
-    [SpeechEventType.METRICS_COLLECTED]: (metrics: STTMetrics) => void;
+    ['metrics_collected']: (metrics: STTMetrics) => void;
+    ['error']: (error: STTError) => void;
 };
 declare const STT_base: new () => TypedEmitter<STTCallbacks>;
 /**
@@ -105,10 +115,21 @@ export declare abstract class SpeechStream implements AsyncIterableIterator<Spee
     protected input: AsyncIterableQueue<AudioFrame | typeof SpeechStream.FLUSH_SENTINEL>;
     protected output: AsyncIterableQueue<SpeechEvent>;
     protected queue: AsyncIterableQueue<SpeechEvent>;
+    protected neededSampleRate?: number;
+    protected resampler?: AudioResampler;
     abstract label: string;
     protected closed: boolean;
-    constructor(stt: STT);
+    private deferredInputStream;
+    private logger;
+    private _connOptions;
+    constructor(stt: STT, sampleRate?: number, connectionOptions?: APIConnectOptions);
+    private mainTask;
+    private emitError;
+    protected pumpInput(): Promise<void>;
     protected monitorMetrics(): Promise<void>;
+    protected abstract run(): Promise<void>;
+    updateInputStream(audioStream: ReadableStream<AudioFrame>): void;
+    detachInputStream(): void;
     /** Push an audio frame to the STT */
     pushFrame(frame: AudioFrame): void;
     /** Flush the STT, causing it to process all pending text */

package/dist/stt/stt.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"stt.d.ts","sourceRoot":"","sources":["../../src/stt/stt.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,KAAK,EAAE,iBAAiB,IAAI,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAEhF,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAEjD,2CAA2C;AAC3C,oBAAY,eAAe;IACzB;;;;OAIG;IACH,eAAe,IAAI;IACnB;;OAEG;IACH,kBAAkB,IAAI;IACtB;;;OAGG;IACH,gBAAgB,IAAI;IACpB;;;OAGG;IACH,aAAa,IAAI;IACjB,mEAAmE;IACnE,iBAAiB,IAAI;IACrB,iBAAiB,IAAI;CACtB;AAED,mEAAmE;AACnE,MAAM,WAAW,UAAU;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,sDAAsD;AACtD,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,eAAe,CAAC;IACtB,YAAY,CAAC,EAAE,CAAC,UAAU,EAAE,GAAG,UAAU,EAAE,CAAC,CAAC;IAC7C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;CACrC;AAED;;;;;GAKG;AACH,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,OAAO,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;CACzB;AAED,MAAM,MAAM,YAAY,GAAG;IACzB,CAAC,eAAe,CAAC,iBAAiB,CAAC,EAAE,CAAC,OAAO,EAAE,UAAU,KAAK,IAAI,CAAC;CACpE,CAAC;kCAS2D,aAAa,YAAY,CAAC;AAPvF;;;;;;GAMG;AACH,8BAAsB,GAAI,SAAQ,QAAsD;;IACtF,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;gBAGX,YAAY,EAAE,eAAe;IAKzC,sCAAsC;IACtC,IAAI,YAAY,IAAI,eAAe,CAElC;IAED,8FAA8F;IACxF,SAAS,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IAiBzD,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IAEvE;;;OAGG;IACH,QAAQ,CAAC,MAAM,IAAI,YAAY;CAChC;AAED;;;;;;;;;;;;;;;GAeG;AACH,8BAAsB,YAAa,YAAW,qBAAqB,CAAC,WAAW,CAAC;;IAC9E,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,gBAA4B;IACpE,SAAS,CAAC,KAAK,sEAA6E;IAC5F,SAAS,CAAC,MAAM,kCAAyC;IACzD,SAAS,CAAC,KAAK,kCAAyC;IACxD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,SAAS,CAAC,MAAM,UAAS;gBAGb,GAAG,EAAE,GAAG;cAKJ,cAAc;IAoB9B,qCAAqC;IACrC,SAAS,CAAC,KAAK,EAAE,UAAU;IAU3B,4DAA4D;IAC5D,KAAK;IAUL,2DAA2D;IAC3D,QAAQ;IAUR,IAAI,IAAI,OAAO,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;IAI5C,wDAAwD;IACxD,KAAK;IAOL,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,YAAY;CAGvC"}
+{"version":3,"file":"stt.d.ts","sourceRoot":"","sources":["../../src/stt/stt.ts"],"names":[],"mappings":";AAGA,OAAO,EAAE,KAAK,UAAU,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,KAAK,EAAE,iBAAiB,IAAI,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAGhF,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAItD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAErD,OAAO,EAAE,KAAK,iBAAiB,EAA+B,MAAM,aAAa,CAAC;AAClF,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,EAAE,kBAAkB,EAAsB,MAAM,aAAa,CAAC;AAErE,2CAA2C;AAC3C,oBAAY,eAAe;IACzB;;;;OAIG;IACH,eAAe,IAAI;IACnB;;OAEG;IACH,kBAAkB,IAAI;IACtB;;;OAGG;IACH,gBAAgB,IAAI;IACpB;;;OAGG;IACH,aAAa,IAAI;IACjB,mEAAmE;IACnE,iBAAiB,IAAI;CACtB;AAED,mEAAmE;AACnE,MAAM,WAAW,UAAU;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,sDAAsD;AACtD,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,eAAe,CAAC;IACtB,YAAY,CAAC,EAAE,CAAC,UAAU,EAAE,GAAG,UAAU,EAAE,CAAC,CAAC;IAC7C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;CACrC;AAED;;;;;GAKG;AACH,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,OAAO,CAAC;IACnB,cAAc,EAAE,OAAO,CAAC;CACzB;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,WAAW,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,KAAK,CAAC;IACb,WAAW,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,MAAM,YAAY,GAAG;IACzB,CAAC,mBAAmB,CAAC,EAAE,CAAC,OAAO,EAAE,UAAU,KAAK,IAAI,CAAC;IACrD,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,KAAK,IAAI,CAAC;CACtC,CAAC;kCAS2D,aAAa,YAAY,CAAC;AAPvF;;;;;;GAMG;AACH,8BAAsB,GAAI,SAAQ,QAAsD;;IACtF,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;gBAGX,YAAY,EAAE,eAAe;IAKzC,sCAAsC;IACtC,IAAI,YAAY,IAAI,eAAe,CAElC;IAED,8FAA8F;IACxF,SAAS,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IAezD,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IAEvE;;;OAGG;IACH,QAAQ,CAAC,MAAM,IAAI,YAAY;CAChC;AAED;;;;;;;;;;;;;;;GAeG;AACH,8BAAsB,YAAa,YAAW,qBAAqB,CAAC,WAAW,CAAC;;IAC9E,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,gBAA4B;IACpE,SAAS,CAAC,KAAK,sEAA6E;IAC5F,SAAS,CAAC,MAAM,kCAAyC;IACzD,SAAS,CAAC,KAAK,kCAAyC;IACxD,SAAS,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACpC,SAAS,CAAC,SAAS,CAAC,EAAE,cAAc,CAAC;IACrC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,SAAS,CAAC,MAAM,UAAS;IAEzB,OAAO,CAAC,mBAAmB,CAAqC;IAChE,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,YAAY,CAAoB;gBAGtC,GAAG,EAAE,GAAG,EACR,UAAU,CAAC,EAAE,MAAM,EACnB,iBAAiB,GAAE,iBAA+C;YAgBtD,QAAQ;IAoCtB,OAAO,CAAC,SAAS;cAUD,SAAS;cAkBT,cAAc;IAkB9B,SAAS,CAAC,QAAQ,CAAC,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;IAEvC,iBAAiB,CAAC,WAAW,EAAE,cAAc,CAAC,UAAU,CAAC;IAIzD,iBAAiB;IAIjB,qCAAqC;IACrC,SAAS,CAAC,KAAK,EAAE,UAAU;IAwB3B,4DAA4D;IAC5D,KAAK;IAUL,2DAA2D;IAC3D,QAAQ;IAUR,IAAI,IAAI,OAAO,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;IAI5C,wDAAwD;IACxD,KAAK;IAOL,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,YAAY;CAGvC"}

package/dist/stt/stt.js

@@ -1,12 +1,18 @@
+import { AudioResampler } from "@livekit/rtc-node";
+import { delay } from "@std/async/delay";
 import { EventEmitter } from "node:events";
-import { AsyncIterableQueue } from "../utils.js";
+import { APIConnectionError, APIError } from "../_exceptions.js";
+import { calculateAudioDuration } from "../audio.js";
+import { log } from "../log.js";
+import { DeferredReadableStream } from "../stream/deferred_stream.js";
+import { DEFAULT_API_CONNECT_OPTIONS } from "../types.js";
+import { AsyncIterableQueue, startSoon, toError } from "../utils.js";
 var SpeechEventType = /* @__PURE__ */ ((SpeechEventType2) => {
   SpeechEventType2[SpeechEventType2["START_OF_SPEECH"] = 0] = "START_OF_SPEECH";
   SpeechEventType2[SpeechEventType2["INTERIM_TRANSCRIPT"] = 1] = "INTERIM_TRANSCRIPT";
   SpeechEventType2[SpeechEventType2["FINAL_TRANSCRIPT"] = 2] = "FINAL_TRANSCRIPT";
   SpeechEventType2[SpeechEventType2["END_OF_SPEECH"] = 3] = "END_OF_SPEECH";
   SpeechEventType2[SpeechEventType2["RECOGNITION_USAGE"] = 4] = "RECOGNITION_USAGE";
-  SpeechEventType2[SpeechEventType2["METRICS_COLLECTED"] = 5] = "METRICS_COLLECTED";
   return SpeechEventType2;
 })(SpeechEventType || {});
 class STT extends EventEmitter {
@@ -24,12 +30,13 @@ class STT extends EventEmitter {
     const startTime = process.hrtime.bigint();
     const event = await this._recognize(frame);
     const duration = Number((process.hrtime.bigint() - startTime) / BigInt(1e6));
-    this.emit(5 /* METRICS_COLLECTED */, {
+    this.emit("metrics_collected", {
+      type: "stt_metrics",
       requestId: event.requestId ?? "",
       timestamp: Date.now(),
       duration,
       label: this.label,
-      audioDuration: Array.isArray(frame) ? frame.reduce((sum, a) => sum + a.samplesPerChannel / a.sampleRate, 0) : frame.samplesPerChannel / frame.sampleRate,
+      audioDuration: calculateAudioDuration(frame),
       streamed: false
     });
     return event;
@@ -40,30 +47,102 @@ class SpeechStream {
   input = new AsyncIterableQueue();
   output = new AsyncIterableQueue();
   queue = new AsyncIterableQueue();
+  neededSampleRate;
+  resampler;
   closed = false;
   #stt;
-  constructor(stt) {
+  deferredInputStream;
+  logger = log();
+  _connOptions;
+  constructor(stt, sampleRate, connectionOptions = DEFAULT_API_CONNECT_OPTIONS) {
     this.#stt = stt;
+    this._connOptions = connectionOptions;
+    this.deferredInputStream = new DeferredReadableStream();
+    this.neededSampleRate = sampleRate;
     this.monitorMetrics();
+    this.pumpInput();
+    startSoon(() => this.mainTask().then(() => this.queue.close()));
+  }
+  async mainTask() {
+    for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {
+      try {
+        return await this.run();
+      } catch (error) {
+        if (error instanceof APIError) {
+          const retryInterval = this._connOptions._intervalForRetry(i);
+          if (this._connOptions.maxRetry === 0 || !error.retryable) {
+            this.emitError({ error, recoverable: false });
+            throw error;
+          } else if (i === this._connOptions.maxRetry) {
+            this.emitError({ error, recoverable: false });
+            throw new APIConnectionError({
+              message: `failed to recognize speech after ${this._connOptions.maxRetry + 1} attempts`,
+              options: { retryable: false }
+            });
+          } else {
+            this.emitError({ error, recoverable: true });
+            this.logger.warn(
+              { tts: this.#stt.label, attempt: i + 1, error },
+              `failed to recognize speech, retrying in ${retryInterval}s`
+            );
+          }
+          if (retryInterval > 0) {
+            await delay(retryInterval);
+          }
+        } else {
+          this.emitError({ error: toError(error), recoverable: false });
+          throw error;
+        }
+      }
+    }
+  }
+  emitError({ error, recoverable }) {
+    this.#stt.emit("error", {
+      type: "stt_error",
+      timestamp: Date.now(),
+      label: this.#stt.label,
+      error,
+      recoverable
+    });
+  }
+  async pumpInput() {
+    const inputStream = this.deferredInputStream.stream;
+    const reader = inputStream.getReader();
+    try {
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        this.pushFrame(value);
+      }
+    } catch (error) {
+      this.logger.error("Error in STTStream mainTask:", error);
+    } finally {
+      reader.releaseLock();
+    }
   }
   async monitorMetrics() {
-    const startTime = process.hrtime.bigint();
     for await (const event of this.queue) {
       this.output.put(event);
       if (event.type !== 4 /* RECOGNITION_USAGE */) continue;
-      const duration = process.hrtime.bigint() - startTime;
       const metrics = {
+        type: "stt_metrics",
         timestamp: Date.now(),
         requestId: event.requestId,
-        duration: Math.trunc(Number(duration / BigInt(1e6))),
-        label: this.label,
+        duration: 0,
+        label: this.#stt.label,
         audioDuration: event.recognitionUsage.audioDuration,
         streamed: true
       };
-      this.#stt.emit(5 /* METRICS_COLLECTED */, metrics);
+      this.#stt.emit("metrics_collected", metrics);
     }
     this.output.close();
   }
+  updateInputStream(audioStream) {
+    this.deferredInputStream.setSource(audioStream);
+  }
+  detachInputStream() {
+    this.deferredInputStream.detachSource();
+  }
   /** Push an audio frame to the STT */
   pushFrame(frame) {
     if (this.input.closed) {
@@ -72,7 +151,19 @@ class SpeechStream {
     if (this.closed) {
       throw new Error("Stream is closed");
     }
-    this.input.put(frame);
+    if (this.neededSampleRate && frame.sampleRate !== this.neededSampleRate) {
+      if (!this.resampler) {
+        this.resampler = new AudioResampler(frame.sampleRate, this.neededSampleRate);
+      }
+    }
+    if (this.resampler) {
+      const frames = this.resampler.push(frame);
+      for (const frame2 of frames) {
+        this.input.put(frame2);
+      }
+    } else {
+      this.input.put(frame);
+    }
   }
   /** Flush the STT, causing it to process all pending text */
   flush() {

package/dist/stt/stt.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/stt/stt.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { AudioFrame } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { EventEmitter } from 'node:events';\nimport type { STTMetrics } from '../metrics/base.js';\nimport type { AudioBuffer } from '../utils.js';\nimport { AsyncIterableQueue } from '../utils.js';\n\n/** Indicates start/middle/end of speech */\nexport enum SpeechEventType {\n  /**\n   * Indicate the start of speech.\n   * If the STT doesn't support this event, this will be emitted at the same time\n   * as the first INTERIM_TRANSCRIPT.\n   */\n  START_OF_SPEECH = 0,\n  /**\n   * Interim transcript, useful for real-time transcription.\n   */\n  INTERIM_TRANSCRIPT = 1,\n  /**\n   * Final transcript, emitted when the STT is confident enough that a certain\n   * portion of the speech will not change.\n   */\n  FINAL_TRANSCRIPT = 2,\n  /**\n   * Indicate the end of speech, emitted when the user stops speaking.\n   * The first alternative is a combination of all the previous FINAL_TRANSCRIPT events.\n   */\n  END_OF_SPEECH = 3,\n  /** Usage event, emitted periodically to indicate usage metrics. */\n  RECOGNITION_USAGE = 4,\n  METRICS_COLLECTED = 5,\n}\n\n/** SpeechData contains metadata about this {@link SpeechEvent}. */\nexport interface SpeechData {\n  language: string;\n  text: string;\n  startTime: number;\n  endTime: number;\n  confidence: number;\n}\n\nexport interface RecognitionUsage {\n  audioDuration: number;\n}\n\n/** SpeechEvent is a packet of speech-to-text data. */\nexport interface SpeechEvent {\n  type: SpeechEventType;\n  alternatives?: [SpeechData, ...SpeechData[]];\n  requestId?: string;\n  recognitionUsage?: RecognitionUsage;\n}\n\n/**\n * Describes the capabilities of the STT provider.\n *\n * @remarks\n * At present, the framework only supports providers that have a streaming endpoint.\n */\nexport interface STTCapabilities {\n  streaming: boolean;\n  interimResults: boolean;\n}\n\nexport type STTCallbacks = {\n  [SpeechEventType.METRICS_COLLECTED]: (metrics: STTMetrics) => void;\n};\n\n/**\n * An instance of a speech-to-text adapter.\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child STT class, which inherits this class's methods.\n */\nexport abstract class STT extends (EventEmitter as new () => TypedEmitter<STTCallbacks>) {\n  abstract label: string;\n  #capabilities: STTCapabilities;\n\n  constructor(capabilities: STTCapabilities) {\n    super();\n    this.#capabilities = capabilities;\n  }\n\n  /** Returns this STT's capabilities */\n  get capabilities(): STTCapabilities {\n    return this.#capabilities;\n  }\n\n  /** Receives an audio buffer and returns transcription in the form of a {@link SpeechEvent} */\n  async recognize(frame: AudioBuffer): Promise<SpeechEvent> {\n    const startTime = process.hrtime.bigint();\n    const event = await this._recognize(frame);\n    const duration = Number((process.hrtime.bigint() - startTime) / BigInt(1000000));\n    this.emit(SpeechEventType.METRICS_COLLECTED, {\n      requestId: event.requestId ?? '',\n      timestamp: Date.now(),\n      duration,\n      label: this.label,\n      audioDuration: Array.isArray(frame)\n        ? frame.reduce((sum, a) => sum + a.samplesPerChannel / a.sampleRate, 0)\n        : frame.samplesPerChannel / frame.sampleRate,\n      streamed: false,\n    });\n    return event;\n  }\n\n  protected abstract _recognize(frame: AudioBuffer): Promise<SpeechEvent>;\n\n  /**\n   * Returns a {@link SpeechStream} that can be used to push audio frames and receive\n   * transcriptions\n   */\n  abstract stream(): SpeechStream;\n}\n\n/**\n * An instance of a speech-to-text stream, as an asynchronous iterable iterator.\n *\n * @example Looping through frames\n * ```ts\n * for await (const event of stream) {\n *   if (event.type === SpeechEventType.FINAL_TRANSCRIPT) {\n *     console.log(event.alternatives[0].text)\n *   }\n * }\n * ```\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child SpeechStream class, which inherits this class's methods.\n */\nexport abstract class SpeechStream implements AsyncIterableIterator<SpeechEvent> {\n  protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');\n  protected input = new AsyncIterableQueue<AudioFrame | typeof SpeechStream.FLUSH_SENTINEL>();\n  protected output = new AsyncIterableQueue<SpeechEvent>();\n  protected queue = new AsyncIterableQueue<SpeechEvent>();\n  abstract label: string;\n  protected closed = false;\n  #stt: STT;\n\n  constructor(stt: STT) {\n    this.#stt = stt;\n    this.monitorMetrics();\n  }\n\n  protected async monitorMetrics() {\n    const startTime = process.hrtime.bigint();\n\n    for await (const event of this.queue) {\n      this.output.put(event);\n      if (event.type !== SpeechEventType.RECOGNITION_USAGE) continue;\n      const duration = process.hrtime.bigint() - startTime;\n      const metrics: STTMetrics = {\n        timestamp: Date.now(),\n        requestId: event.requestId!,\n        duration: Math.trunc(Number(duration / BigInt(1000000))),\n        label: this.label,\n        audioDuration: event.recognitionUsage!.audioDuration,\n        streamed: true,\n      };\n      this.#stt.emit(SpeechEventType.METRICS_COLLECTED, metrics);\n    }\n    this.output.close();\n  }\n\n  /** Push an audio frame to the STT */\n  pushFrame(frame: AudioFrame) {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(frame);\n  }\n\n  /** Flush the STT, causing it to process all pending text */\n  flush() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(SpeechStream.FLUSH_SENTINEL);\n  }\n\n  /** Mark the input as ended and forbid additional pushes */\n  endInput() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.close();\n  }\n\n  next(): Promise<IteratorResult<SpeechEvent>> {\n    return this.output.next();\n  }\n\n  /** Close both the input and output of the STT stream */\n  close() {\n    this.input.close();\n    this.queue.close();\n    this.output.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): SpeechStream {\n    return this;\n  }\n}\n"],"mappings":"AAKA,SAAS,oBAAoB;AAG7B,SAAS,0BAA0B;AAG5B,IAAK,kBAAL,kBAAKA,qBAAL;AAML,EAAAA,kCAAA,qBAAkB,KAAlB;AAIA,EAAAA,kCAAA,wBAAqB,KAArB;AAKA,EAAAA,kCAAA,sBAAmB,KAAnB;AAKA,EAAAA,kCAAA,mBAAgB,KAAhB;AAEA,EAAAA,kCAAA,uBAAoB,KAApB;AACA,EAAAA,kCAAA,uBAAoB,KAApB;AAvBU,SAAAA;AAAA,GAAA;AAqEL,MAAe,YAAa,aAAsD;AAAA,EAEvF;AAAA,EAEA,YAAY,cAA+B;AACzC,UAAM;AACN,SAAK,gBAAgB;AAAA,EACvB;AAAA;AAAA,EAGA,IAAI,eAAgC;AAClC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,MAAM,UAAU,OAA0C;AACxD,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,UAAM,QAAQ,MAAM,KAAK,WAAW,KAAK;AACzC,UAAM,WAAW,QAAQ,QAAQ,OAAO,OAAO,IAAI,aAAa,OAAO,GAAO,CAAC;AAC/E,SAAK,KAAK,2BAAmC;AAAA,MAC3C,WAAW,MAAM,aAAa;AAAA,MAC9B,WAAW,KAAK,IAAI;AAAA,MACpB;AAAA,MACA,OAAO,KAAK;AAAA,MACZ,eAAe,MAAM,QAAQ,KAAK,IAC9B,MAAM,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,oBAAoB,EAAE,YAAY,CAAC,IACpE,MAAM,oBAAoB,MAAM;AAAA,MACpC,UAAU;AAAA,IACZ,CAAC;AACD,WAAO;AAAA,EACT;AASF;AAkBO,MAAe,aAA2D;AAAA,EAC/E,OAA0B,iBAAiB,OAAO,gBAAgB;AAAA,EACxD,QAAQ,IAAI,mBAAoE;AAAA,EAChF,SAAS,IAAI,mBAAgC;AAAA,EAC7C,QAAQ,IAAI,mBAAgC;AAAA,EAE5C,SAAS;AAAA,EACnB;AAAA,EAEA,YAAY,KAAU;AACpB,SAAK,OAAO;AACZ,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,UAAM,YAAY,QAAQ,OAAO,OAAO;AAExC,qBAAiB,SAAS,KAAK,OAAO;AACpC,WAAK,OAAO,IAAI,KAAK;AACrB,UAAI,MAAM,SAAS,0BAAmC;AACtD,YAAM,WAAW,QAAQ,OAAO,OAAO,IAAI;AAC3C,YAAM,UAAsB;AAAA,QAC1B,WAAW,KAAK,IAAI;AAAA,QACpB,WAAW,MAAM;AAAA,QACjB,UAAU,KAAK,MAAM,OAAO,WAAW,OAAO,GAAO,CAAC,CAAC;AAAA,QACvD,OAAO,KAAK;AAAA,QACZ,eAAe,MAAM,iBAAkB;AAAA,QACvC,UAAU;AAAA,MACZ;AACA,WAAK,KAAK,KAAK,2BAAmC,OAAO;AAAA,IAC3D;AACA,SAAK,OAAO,MAAM;AAAA,EACpB;AAAA;AAAA,EAGA,UAAU,OAAmB;AAC3B,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,KAAK;AAAA,EACtB;AAAA;AAAA,EAGA,QAAQ;AACN,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,aAAa,cAAc;AAAA,EAC5C;AAAA;AAAA,EAGA,WAAW;AACT,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,MAAM;AAAA,EACnB;AAAA,EAEA,OAA6C;AAC3C,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA;AAAA,EAGA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,MAAM,MAAM;AACjB,SAAK,OAAO,MAAM;AAClB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAkB;AACrC,WAAO;AAAA,EACT;AACF;","names":["SpeechEventType"]}
+{"version":3,"sources":["../../src/stt/stt.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { type AudioFrame, AudioResampler } from '@livekit/rtc-node';\nimport type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';\nimport { delay } from '@std/async/delay';\nimport { EventEmitter } from 'node:events';\nimport type { ReadableStream } from 'node:stream/web';\nimport { APIConnectionError, APIError } from '../_exceptions.js';\nimport { calculateAudioDuration } from '../audio.js';\nimport { log } from '../log.js';\nimport type { STTMetrics } from '../metrics/base.js';\nimport { DeferredReadableStream } from '../stream/deferred_stream.js';\nimport { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js';\nimport type { AudioBuffer } from '../utils.js';\nimport { AsyncIterableQueue, startSoon, toError } from '../utils.js';\n\n/** Indicates start/middle/end of speech */\nexport enum SpeechEventType {\n  /**\n   * Indicate the start of speech.\n   * If the STT doesn't support this event, this will be emitted at the same time\n   * as the first INTERIM_TRANSCRIPT.\n   */\n  START_OF_SPEECH = 0,\n  /**\n   * Interim transcript, useful for real-time transcription.\n   */\n  INTERIM_TRANSCRIPT = 1,\n  /**\n   * Final transcript, emitted when the STT is confident enough that a certain\n   * portion of the speech will not change.\n   */\n  FINAL_TRANSCRIPT = 2,\n  /**\n   * Indicate the end of speech, emitted when the user stops speaking.\n   * The first alternative is a combination of all the previous FINAL_TRANSCRIPT events.\n   */\n  END_OF_SPEECH = 3,\n  /** Usage event, emitted periodically to indicate usage metrics. */\n  RECOGNITION_USAGE = 4,\n}\n\n/** SpeechData contains metadata about this {@link SpeechEvent}. */\nexport interface SpeechData {\n  language: string;\n  text: string;\n  startTime: number;\n  endTime: number;\n  confidence: number;\n}\n\nexport interface RecognitionUsage {\n  audioDuration: number;\n}\n\n/** SpeechEvent is a packet of speech-to-text data. */\nexport interface SpeechEvent {\n  type: SpeechEventType;\n  alternatives?: [SpeechData, ...SpeechData[]];\n  requestId?: string;\n  recognitionUsage?: RecognitionUsage;\n}\n\n/**\n * Describes the capabilities of the STT provider.\n *\n * @remarks\n * At present, the framework only supports providers that have a streaming endpoint.\n */\nexport interface STTCapabilities {\n  streaming: boolean;\n  interimResults: boolean;\n}\n\nexport interface STTError {\n  type: 'stt_error';\n  timestamp: number;\n  label: string;\n  error: Error;\n  recoverable: boolean;\n}\n\nexport type STTCallbacks = {\n  ['metrics_collected']: (metrics: STTMetrics) => void;\n  ['error']: (error: STTError) => void;\n};\n\n/**\n * An instance of a speech-to-text adapter.\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child STT class, which inherits this class's methods.\n */\nexport abstract class STT extends (EventEmitter as new () => TypedEmitter<STTCallbacks>) {\n  abstract label: string;\n  #capabilities: STTCapabilities;\n\n  constructor(capabilities: STTCapabilities) {\n    super();\n    this.#capabilities = capabilities;\n  }\n\n  /** Returns this STT's capabilities */\n  get capabilities(): STTCapabilities {\n    return this.#capabilities;\n  }\n\n  /** Receives an audio buffer and returns transcription in the form of a {@link SpeechEvent} */\n  async recognize(frame: AudioBuffer): Promise<SpeechEvent> {\n    const startTime = process.hrtime.bigint();\n    const event = await this._recognize(frame);\n    const duration = Number((process.hrtime.bigint() - startTime) / BigInt(1000000));\n    this.emit('metrics_collected', {\n      type: 'stt_metrics',\n      requestId: event.requestId ?? '',\n      timestamp: Date.now(),\n      duration,\n      label: this.label,\n      audioDuration: calculateAudioDuration(frame),\n      streamed: false,\n    });\n    return event;\n  }\n  protected abstract _recognize(frame: AudioBuffer): Promise<SpeechEvent>;\n\n  /**\n   * Returns a {@link SpeechStream} that can be used to push audio frames and receive\n   * transcriptions\n   */\n  abstract stream(): SpeechStream;\n}\n\n/**\n * An instance of a speech-to-text stream, as an asynchronous iterable iterator.\n *\n * @example Looping through frames\n * ```ts\n * for await (const event of stream) {\n *   if (event.type === SpeechEventType.FINAL_TRANSCRIPT) {\n *     console.log(event.alternatives[0].text)\n *   }\n * }\n * ```\n *\n * @remarks\n * This class is abstract, and as such cannot be used directly. Instead, use a provider plugin that\n * exports its own child SpeechStream class, which inherits this class's methods.\n */\nexport abstract class SpeechStream implements AsyncIterableIterator<SpeechEvent> {\n  protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');\n  protected input = new AsyncIterableQueue<AudioFrame | typeof SpeechStream.FLUSH_SENTINEL>();\n  protected output = new AsyncIterableQueue<SpeechEvent>();\n  protected queue = new AsyncIterableQueue<SpeechEvent>();\n  protected neededSampleRate?: number;\n  protected resampler?: AudioResampler;\n  abstract label: string;\n  protected closed = false;\n  #stt: STT;\n  private deferredInputStream: DeferredReadableStream<AudioFrame>;\n  private logger = log();\n  private _connOptions: APIConnectOptions;\n\n  constructor(\n    stt: STT,\n    sampleRate?: number,\n    connectionOptions: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,\n  ) {\n    this.#stt = stt;\n    this._connOptions = connectionOptions;\n    this.deferredInputStream = new DeferredReadableStream<AudioFrame>();\n    this.neededSampleRate = sampleRate;\n    this.monitorMetrics();\n    this.pumpInput();\n\n    // this is a hack to immitate asyncio.create_task so that mainTask\n    // is run **after** the constructor has finished. Otherwise we get\n    // runtime error when trying to access class variables in the\n    // `run` method.\n    startSoon(() => this.mainTask().then(() => this.queue.close()));\n  }\n\n  private async mainTask() {\n    for (let i = 0; i < this._connOptions.maxRetry + 1; i++) {\n      try {\n        return await this.run();\n      } catch (error) {\n        if (error instanceof APIError) {\n          const retryInterval = this._connOptions._intervalForRetry(i);\n\n          if (this._connOptions.maxRetry === 0 || !error.retryable) {\n            this.emitError({ error, recoverable: false });\n            throw error;\n          } else if (i === this._connOptions.maxRetry) {\n            this.emitError({ error, recoverable: false });\n            throw new APIConnectionError({\n              message: `failed to recognize speech after ${this._connOptions.maxRetry + 1} attempts`,\n              options: { retryable: false },\n            });\n          } else {\n            this.emitError({ error, recoverable: true });\n            this.logger.warn(\n              { tts: this.#stt.label, attempt: i + 1, error },\n              `failed to recognize speech, retrying in ${retryInterval}s`,\n            );\n          }\n\n          if (retryInterval > 0) {\n            await delay(retryInterval);\n          }\n        } else {\n          this.emitError({ error: toError(error), recoverable: false });\n          throw error;\n        }\n      }\n    }\n  }\n\n  private emitError({ error, recoverable }: { error: Error; recoverable: boolean }) {\n    this.#stt.emit('error', {\n      type: 'stt_error',\n      timestamp: Date.now(),\n      label: this.#stt.label,\n      error,\n      recoverable,\n    });\n  }\n\n  protected async pumpInput() {\n    // TODO(AJS-35): Implement STT with webstreams API\n    const inputStream = this.deferredInputStream.stream;\n    const reader = inputStream.getReader();\n\n    try {\n      while (true) {\n        const { done, value } = await reader.read();\n        if (done) break;\n        this.pushFrame(value);\n      }\n    } catch (error) {\n      this.logger.error('Error in STTStream mainTask:', error);\n    } finally {\n      reader.releaseLock();\n    }\n  }\n\n  protected async monitorMetrics() {\n    for await (const event of this.queue) {\n      this.output.put(event);\n      if (event.type !== SpeechEventType.RECOGNITION_USAGE) continue;\n      const metrics: STTMetrics = {\n        type: 'stt_metrics',\n        timestamp: Date.now(),\n        requestId: event.requestId!,\n        duration: 0,\n        label: this.#stt.label,\n        audioDuration: event.recognitionUsage!.audioDuration,\n        streamed: true,\n      };\n      this.#stt.emit('metrics_collected', metrics);\n    }\n    this.output.close();\n  }\n\n  protected abstract run(): Promise<void>;\n\n  updateInputStream(audioStream: ReadableStream<AudioFrame>) {\n    this.deferredInputStream.setSource(audioStream);\n  }\n\n  detachInputStream() {\n    this.deferredInputStream.detachSource();\n  }\n\n  /** Push an audio frame to the STT */\n  pushFrame(frame: AudioFrame) {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n\n    if (this.neededSampleRate && frame.sampleRate !== this.neededSampleRate) {\n      if (!this.resampler) {\n        this.resampler = new AudioResampler(frame.sampleRate, this.neededSampleRate);\n      }\n    }\n\n    if (this.resampler) {\n      const frames = this.resampler.push(frame);\n      for (const frame of frames) {\n        this.input.put(frame);\n      }\n    } else {\n      this.input.put(frame);\n    }\n  }\n\n  /** Flush the STT, causing it to process all pending text */\n  flush() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(SpeechStream.FLUSH_SENTINEL);\n  }\n\n  /** Mark the input as ended and forbid additional pushes */\n  endInput() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.close();\n  }\n\n  next(): Promise<IteratorResult<SpeechEvent>> {\n    return this.output.next();\n  }\n\n  /** Close both the input and output of the STT stream */\n  close() {\n    this.input.close();\n    this.queue.close();\n    this.output.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): SpeechStream {\n    return this;\n  }\n}\n"],"mappings":"AAGA,SAA0B,sBAAsB;AAEhD,SAAS,aAAa;AACtB,SAAS,oBAAoB;AAE7B,SAAS,oBAAoB,gBAAgB;AAC7C,SAAS,8BAA8B;AACvC,SAAS,WAAW;AAEpB,SAAS,8BAA8B;AACvC,SAAiC,mCAAmC;AAEpE,SAAS,oBAAoB,WAAW,eAAe;AAGhD,IAAK,kBAAL,kBAAKA,qBAAL;AAML,EAAAA,kCAAA,qBAAkB,KAAlB;AAIA,EAAAA,kCAAA,wBAAqB,KAArB;AAKA,EAAAA,kCAAA,sBAAmB,KAAnB;AAKA,EAAAA,kCAAA,mBAAgB,KAAhB;AAEA,EAAAA,kCAAA,uBAAoB,KAApB;AAtBU,SAAAA;AAAA,GAAA;AA6EL,MAAe,YAAa,aAAsD;AAAA,EAEvF;AAAA,EAEA,YAAY,cAA+B;AACzC,UAAM;AACN,SAAK,gBAAgB;AAAA,EACvB;AAAA;AAAA,EAGA,IAAI,eAAgC;AAClC,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,MAAM,UAAU,OAA0C;AACxD,UAAM,YAAY,QAAQ,OAAO,OAAO;AACxC,UAAM,QAAQ,MAAM,KAAK,WAAW,KAAK;AACzC,UAAM,WAAW,QAAQ,QAAQ,OAAO,OAAO,IAAI,aAAa,OAAO,GAAO,CAAC;AAC/E,SAAK,KAAK,qBAAqB;AAAA,MAC7B,MAAM;AAAA,MACN,WAAW,MAAM,aAAa;AAAA,MAC9B,WAAW,KAAK,IAAI;AAAA,MACpB;AAAA,MACA,OAAO,KAAK;AAAA,MACZ,eAAe,uBAAuB,KAAK;AAAA,MAC3C,UAAU;AAAA,IACZ,CAAC;AACD,WAAO;AAAA,EACT;AAQF;AAkBO,MAAe,aAA2D;AAAA,EAC/E,OAA0B,iBAAiB,OAAO,gBAAgB;AAAA,EACxD,QAAQ,IAAI,mBAAoE;AAAA,EAChF,SAAS,IAAI,mBAAgC;AAAA,EAC7C,QAAQ,IAAI,mBAAgC;AAAA,EAC5C;AAAA,EACA;AAAA,EAEA,SAAS;AAAA,EACnB;AAAA,EACQ;AAAA,EACA,SAAS,IAAI;AAAA,EACb;AAAA,EAER,YACE,KACA,YACA,oBAAuC,6BACvC;AACA,SAAK,OAAO;AACZ,SAAK,eAAe;AACpB,SAAK,sBAAsB,IAAI,uBAAmC;AAClE,SAAK,mBAAmB;AACxB,SAAK,eAAe;AACpB,SAAK,UAAU;AAMf,cAAU,MAAM,KAAK,SAAS,EAAE,KAAK,MAAM,KAAK,MAAM,MAAM,CAAC,CAAC;AAAA,EAChE;AAAA,EAEA,MAAc,WAAW;AACvB,aAAS,IAAI,GAAG,IAAI,KAAK,aAAa,WAAW,GAAG,KAAK;AACvD,UAAI;AACF,eAAO,MAAM,KAAK,IAAI;AAAA,MACxB,SAAS,OAAO;AACd,YAAI,iBAAiB,UAAU;AAC7B,gBAAM,gBAAgB,KAAK,aAAa,kBAAkB,CAAC;AAE3D,cAAI,KAAK,aAAa,aAAa,KAAK,CAAC,MAAM,WAAW;AACxD,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM;AAAA,UACR,WAAW,MAAM,KAAK,aAAa,UAAU;AAC3C,iBAAK,UAAU,EAAE,OAAO,aAAa,MAAM,CAAC;AAC5C,kBAAM,IAAI,mBAAmB;AAAA,cAC3B,SAAS,oCAAoC,KAAK,aAAa,WAAW,CAAC;AAAA,cAC3E,SAAS,EAAE,WAAW,MAAM;AAAA,YAC9B,CAAC;AAAA,UACH,OAAO;AACL,iBAAK,UAAU,EAAE,OAAO,aAAa,KAAK,CAAC;AAC3C,iBAAK,OAAO;AAAA,cACV,EAAE,KAAK,KAAK,KAAK,OAAO,SAAS,IAAI,GAAG,MAAM;AAAA,cAC9C,2CAA2C,aAAa;AAAA,YAC1D;AAAA,UACF;AAEA,cAAI,gBAAgB,GAAG;AACrB,kBAAM,MAAM,aAAa;AAAA,UAC3B;AAAA,QACF,OAAO;AACL,eAAK,UAAU,EAAE,OAAO,QAAQ,KAAK,GAAG,aAAa,MAAM,CAAC;AAC5D,gBAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,UAAU,EAAE,OAAO,YAAY,GAA2C;AAChF,SAAK,KAAK,KAAK,SAAS;AAAA,MACtB,MAAM;AAAA,MACN,WAAW,KAAK,IAAI;AAAA,MACpB,OAAO,KAAK,KAAK;AAAA,MACjB;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAgB,YAAY;AAE1B,UAAM,cAAc,KAAK,oBAAoB;AAC7C,UAAM,SAAS,YAAY,UAAU;AAErC,QAAI;AACF,aAAO,MAAM;AACX,cAAM,EAAE,MAAM,MAAM,IAAI,MAAM,OAAO,KAAK;AAC1C,YAAI,KAAM;AACV,aAAK,UAAU,KAAK;AAAA,MACtB;AAAA,IACF,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,gCAAgC,KAAK;AAAA,IACzD,UAAE;AACA,aAAO,YAAY;AAAA,IACrB;AAAA,EACF;AAAA,EAEA,MAAgB,iBAAiB;AAC/B,qBAAiB,SAAS,KAAK,OAAO;AACpC,WAAK,OAAO,IAAI,KAAK;AACrB,UAAI,MAAM,SAAS,0BAAmC;AACtD,YAAM,UAAsB;AAAA,QAC1B,MAAM;AAAA,QACN,WAAW,KAAK,IAAI;AAAA,QACpB,WAAW,MAAM;AAAA,QACjB,UAAU;AAAA,QACV,OAAO,KAAK,KAAK;AAAA,QACjB,eAAe,MAAM,iBAAkB;AAAA,QACvC,UAAU;AAAA,MACZ;AACA,WAAK,KAAK,KAAK,qBAAqB,OAAO;AAAA,IAC7C;AACA,SAAK,OAAO,MAAM;AAAA,EACpB;AAAA,EAIA,kBAAkB,aAAyC;AACzD,SAAK,oBAAoB,UAAU,WAAW;AAAA,EAChD;AAAA,EAEA,oBAAoB;AAClB,SAAK,oBAAoB,aAAa;AAAA,EACxC;AAAA;AAAA,EAGA,UAAU,OAAmB;AAC3B,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AAEA,QAAI,KAAK,oBAAoB,MAAM,eAAe,KAAK,kBAAkB;AACvE,UAAI,CAAC,KAAK,WAAW;AACnB,aAAK,YAAY,IAAI,eAAe,MAAM,YAAY,KAAK,gBAAgB;AAAA,MAC7E;AAAA,IACF;AAEA,QAAI,KAAK,WAAW;AAClB,YAAM,SAAS,KAAK,UAAU,KAAK,KAAK;AACxC,iBAAWC,UAAS,QAAQ;AAC1B,aAAK,MAAM,IAAIA,MAAK;AAAA,MACtB;AAAA,IACF,OAAO;AACL,WAAK,MAAM,IAAI,KAAK;AAAA,IACtB;AAAA,EACF;AAAA;AAAA,EAGA,QAAQ;AACN,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,aAAa,cAAc;AAAA,EAC5C;AAAA;AAAA,EAGA,WAAW;AACT,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,MAAM;AAAA,EACnB;AAAA,EAEA,OAA6C;AAC3C,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA;AAAA,EAGA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,MAAM,MAAM;AACjB,SAAK,OAAO,MAAM;AAClB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAkB;AACrC,WAAO;AAAA,EACT;AACF;","names":["SpeechEventType","frame"]}

package/dist/tokenize/basic/basic.cjs

@@ -41,14 +41,19 @@ var import_hyphenator = require("./hyphenator.cjs");
 var import_paragraph = require("./paragraph.cjs");
 var import_sentence = require("./sentence.cjs");
 var import_word = require("./word.cjs");
+const defaultTokenizerOptions = {
+  language: "en-US",
+  minSentenceLength: 20,
+  streamContextLength: 10,
+  retainFormat: false
+};
 class SentenceTokenizer extends tokenizer.SentenceTokenizer {
   #config;
-  constructor(language = "en-US", minSentenceLength = 20, streamContextLength = 10) {
+  constructor(options) {
     super();
     this.#config = {
-      language,
-      minSentenceLength,
-      streamContextLength
+      ...defaultTokenizerOptions,
+      ...options
     };
   }
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
@@ -58,7 +63,7 @@ class SentenceTokenizer extends tokenizer.SentenceTokenizer {
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
   stream(language) {
     return new import_token_stream.BufferedSentenceStream(
-      (text) => (0, import_sentence.splitSentences)(text, this.#config.minSentenceLength),
+      (text) => (0, import_sentence.splitSentences)(text, this.#config.minSentenceLength, this.#config.retainFormat),
       this.#config.minSentenceLength,
       this.#config.streamContextLength
     );

package/dist/tokenize/basic/basic.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/tokenize/basic/basic.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { BufferedSentenceStream, BufferedWordStream } from '../token_stream.js';\nimport * as tokenizer from '../tokenizer.js';\nimport { hyphenator } from './hyphenator.js';\nimport { splitParagraphs } from './paragraph.js';\nimport { splitSentences } from './sentence.js';\nimport { splitWords } from './word.js';\n\ninterface TokenizerOptions {\n  language: string;\n  minSentenceLength: number;\n  streamContextLength: number;\n}\n\nexport class SentenceTokenizer extends tokenizer.SentenceTokenizer {\n  #config: TokenizerOptions;\n\n  constructor(language = 'en-US', minSentenceLength = 20, streamContextLength = 10) {\n    super();\n    this.#config = {\n      language,\n      minSentenceLength,\n      streamContextLength,\n    };\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  tokenize(text: string, language?: string): string[] {\n    return splitSentences(text, this.#config.minSentenceLength).map((tok) => tok[0]);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  stream(language?: string): tokenizer.SentenceStream {\n    return new BufferedSentenceStream(\n      (text: string) => splitSentences(text, this.#config.minSentenceLength),\n      this.#config.minSentenceLength,\n      this.#config.streamContextLength,\n    );\n  }\n}\n\nexport class WordTokenizer extends tokenizer.WordTokenizer {\n  #ignorePunctuation: boolean;\n\n  constructor(ignorePunctuation = true) {\n    super();\n    this.#ignorePunctuation = ignorePunctuation;\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  tokenize(text: string, language?: string): string[] {\n    return splitWords(text, this.#ignorePunctuation).map((tok) => tok[0]);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  stream(language?: string): tokenizer.WordStream {\n    return new BufferedWordStream(\n      (text: string) => splitWords(text, this.#ignorePunctuation),\n      1,\n      1,\n    );\n  }\n}\n\nexport const hyphenateWord = (word: string): string[] => {\n  return hyphenator.hyphenateWord(word);\n};\n\nexport { splitWords };\n\nexport const tokenizeParagraphs = (text: string): string[] => {\n  return splitParagraphs(text).map((tok) => tok[0]);\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,0BAA2D;AAC3D,gBAA2B;AAC3B,wBAA2B;AAC3B,uBAAgC;AAChC,sBAA+B;AAC/B,kBAA2B;AAQpB,MAAM,0BAA0B,UAAU,kBAAkB;AAAA,EACjE;AAAA,EAEA,YAAY,WAAW,SAAS,oBAAoB,IAAI,sBAAsB,IAAI;AAChF,UAAM;AACN,SAAK,UAAU;AAAA,MACb;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAGA,SAAS,MAAc,UAA6B;AAClD,eAAO,gCAAe,MAAM,KAAK,QAAQ,iBAAiB,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;AAAA,EACjF;AAAA;AAAA,EAGA,OAAO,UAA6C;AAClD,WAAO,IAAI;AAAA,MACT,CAAC,aAAiB,gCAAe,MAAM,KAAK,QAAQ,iBAAiB;AAAA,MACrE,KAAK,QAAQ;AAAA,MACb,KAAK,QAAQ;AAAA,IACf;AAAA,EACF;AACF;AAEO,MAAM,sBAAsB,UAAU,cAAc;AAAA,EACzD;AAAA,EAEA,YAAY,oBAAoB,MAAM;AACpC,UAAM;AACN,SAAK,qBAAqB;AAAA,EAC5B;AAAA;AAAA,EAGA,SAAS,MAAc,UAA6B;AAClD,eAAO,wBAAW,MAAM,KAAK,kBAAkB,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;AAAA,EACtE;AAAA;AAAA,EAGA,OAAO,UAAyC;AAC9C,WAAO,IAAI;AAAA,MACT,CAAC,aAAiB,wBAAW,MAAM,KAAK,kBAAkB;AAAA,MAC1D;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;AAEO,MAAM,gBAAgB,CAAC,SAA2B;AACvD,SAAO,6BAAW,cAAc,IAAI;AACtC;AAIO,MAAM,qBAAqB,CAAC,SAA2B;AAC5D,aAAO,kCAAgB,IAAI,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;AAClD;","names":[]}
+{"version":3,"sources":["../../../src/tokenize/basic/basic.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { BufferedSentenceStream, BufferedWordStream } from '../token_stream.js';\nimport * as tokenizer from '../tokenizer.js';\nimport { hyphenator } from './hyphenator.js';\nimport { splitParagraphs } from './paragraph.js';\nimport { splitSentences } from './sentence.js';\nimport { splitWords } from './word.js';\n\ninterface TokenizerOptions {\n  language: string;\n  minSentenceLength: number;\n  streamContextLength: number;\n  retainFormat: boolean;\n}\n\nconst defaultTokenizerOptions: TokenizerOptions = {\n  language: 'en-US',\n  minSentenceLength: 20,\n  streamContextLength: 10,\n  retainFormat: false,\n};\n\nexport class SentenceTokenizer extends tokenizer.SentenceTokenizer {\n  #config: TokenizerOptions;\n\n  constructor(options?: Partial<TokenizerOptions>) {\n    super();\n    this.#config = {\n      ...defaultTokenizerOptions,\n      ...options,\n    };\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  tokenize(text: string, language?: string): string[] {\n    return splitSentences(text, this.#config.minSentenceLength).map((tok) => tok[0]);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  stream(language?: string): tokenizer.SentenceStream {\n    return new BufferedSentenceStream(\n      (text: string) =>\n        splitSentences(text, this.#config.minSentenceLength, this.#config.retainFormat),\n      this.#config.minSentenceLength,\n      this.#config.streamContextLength,\n    );\n  }\n}\n\nexport class WordTokenizer extends tokenizer.WordTokenizer {\n  #ignorePunctuation: boolean;\n\n  constructor(ignorePunctuation = true) {\n    super();\n    this.#ignorePunctuation = ignorePunctuation;\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  tokenize(text: string, language?: string): string[] {\n    return splitWords(text, this.#ignorePunctuation).map((tok) => tok[0]);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  stream(language?: string): tokenizer.WordStream {\n    return new BufferedWordStream(\n      (text: string) => splitWords(text, this.#ignorePunctuation),\n      1,\n      1,\n    );\n  }\n}\n\nexport const hyphenateWord = (word: string): string[] => {\n  return hyphenator.hyphenateWord(word);\n};\n\nexport { splitWords };\n\nexport const tokenizeParagraphs = (text: string): string[] => {\n  return splitParagraphs(text).map((tok) => tok[0]);\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,0BAA2D;AAC3D,gBAA2B;AAC3B,wBAA2B;AAC3B,uBAAgC;AAChC,sBAA+B;AAC/B,kBAA2B;AAS3B,MAAM,0BAA4C;AAAA,EAChD,UAAU;AAAA,EACV,mBAAmB;AAAA,EACnB,qBAAqB;AAAA,EACrB,cAAc;AAChB;AAEO,MAAM,0BAA0B,UAAU,kBAAkB;AAAA,EACjE;AAAA,EAEA,YAAY,SAAqC;AAC/C,UAAM;AACN,SAAK,UAAU;AAAA,MACb,GAAG;AAAA,MACH,GAAG;AAAA,IACL;AAAA,EACF;AAAA;AAAA,EAGA,SAAS,MAAc,UAA6B;AAClD,eAAO,gCAAe,MAAM,KAAK,QAAQ,iBAAiB,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;AAAA,EACjF;AAAA;AAAA,EAGA,OAAO,UAA6C;AAClD,WAAO,IAAI;AAAA,MACT,CAAC,aACC,gCAAe,MAAM,KAAK,QAAQ,mBAAmB,KAAK,QAAQ,YAAY;AAAA,MAChF,KAAK,QAAQ;AAAA,MACb,KAAK,QAAQ;AAAA,IACf;AAAA,EACF;AACF;AAEO,MAAM,sBAAsB,UAAU,cAAc;AAAA,EACzD;AAAA,EAEA,YAAY,oBAAoB,MAAM;AACpC,UAAM;AACN,SAAK,qBAAqB;AAAA,EAC5B;AAAA;AAAA,EAGA,SAAS,MAAc,UAA6B;AAClD,eAAO,wBAAW,MAAM,KAAK,kBAAkB,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;AAAA,EACtE;AAAA;AAAA,EAGA,OAAO,UAAyC;AAC9C,WAAO,IAAI;AAAA,MACT,CAAC,aAAiB,wBAAW,MAAM,KAAK,kBAAkB;AAAA,MAC1D;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;AAEO,MAAM,gBAAgB,CAAC,SAA2B;AACvD,SAAO,6BAAW,cAAc,IAAI;AACtC;AAIO,MAAM,qBAAqB,CAAC,SAA2B;AAC5D,aAAO,kCAAgB,IAAI,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;AAClD;","names":[]}

package/dist/tokenize/basic/basic.d.cts

@@ -1,8 +1,14 @@
 import * as tokenizer from '../tokenizer.js';
 import { splitWords } from './word.js';
+interface TokenizerOptions {
+    language: string;
+    minSentenceLength: number;
+    streamContextLength: number;
+    retainFormat: boolean;
+}
 export declare class SentenceTokenizer extends tokenizer.SentenceTokenizer {
     #private;
-    constructor(language?: string, minSentenceLength?: number, streamContextLength?: number);
+    constructor(options?: Partial<TokenizerOptions>);
     tokenize(text: string, language?: string): string[];
     stream(language?: string): tokenizer.SentenceStream;
 }

package/dist/tokenize/basic/basic.d.ts

@@ -1,8 +1,14 @@
 import * as tokenizer from '../tokenizer.js';
 import { splitWords } from './word.js';
+interface TokenizerOptions {
+    language: string;
+    minSentenceLength: number;
+    streamContextLength: number;
+    retainFormat: boolean;
+}
 export declare class SentenceTokenizer extends tokenizer.SentenceTokenizer {
     #private;
-    constructor(language?: string, minSentenceLength?: number, streamContextLength?: number);
+    constructor(options?: Partial<TokenizerOptions>);
     tokenize(text: string, language?: string): string[];
     stream(language?: string): tokenizer.SentenceStream;
 }

package/dist/tokenize/basic/basic.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"basic.d.ts","sourceRoot":"","sources":["../../../src/tokenize/basic/basic.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,SAAS,MAAM,iBAAiB,CAAC;AAI7C,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAQvC,qBAAa,iBAAkB,SAAQ,SAAS,CAAC,iBAAiB;;gBAGpD,QAAQ,SAAU,EAAE,iBAAiB,SAAK,EAAE,mBAAmB,SAAK;IAUhF,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE;IAKnD,MAAM,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC,cAAc;CAOpD;AAED,qBAAa,aAAc,SAAQ,SAAS,CAAC,aAAa;;gBAG5C,iBAAiB,UAAO;IAMpC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE;IAKnD,MAAM,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC,UAAU;CAOhD;AAED,eAAO,MAAM,aAAa,SAAU,MAAM,KAAG,MAAM,EAElD,CAAC;AAEF,OAAO,EAAE,UAAU,EAAE,CAAC;AAEtB,eAAO,MAAM,kBAAkB,SAAU,MAAM,KAAG,MAAM,EAEvD,CAAC"}
+{"version":3,"file":"basic.d.ts","sourceRoot":"","sources":["../../../src/tokenize/basic/basic.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,SAAS,MAAM,iBAAiB,CAAC;AAI7C,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAEvC,UAAU,gBAAgB;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,YAAY,EAAE,OAAO,CAAC;CACvB;AASD,qBAAa,iBAAkB,SAAQ,SAAS,CAAC,iBAAiB;;gBAGpD,OAAO,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC;IAS/C,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE;IAKnD,MAAM,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC,cAAc;CAQpD;AAED,qBAAa,aAAc,SAAQ,SAAS,CAAC,aAAa;;gBAG5C,iBAAiB,UAAO;IAMpC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE;IAKnD,MAAM,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC,UAAU;CAOhD;AAED,eAAO,MAAM,aAAa,SAAU,MAAM,KAAG,MAAM,EAElD,CAAC;AAEF,OAAO,EAAE,UAAU,EAAE,CAAC;AAEtB,eAAO,MAAM,kBAAkB,SAAU,MAAM,KAAG,MAAM,EAEvD,CAAC"}

package/dist/tokenize/basic/basic.js

@@ -4,14 +4,19 @@ import { hyphenator } from "./hyphenator.js";
 import { splitParagraphs } from "./paragraph.js";
 import { splitSentences } from "./sentence.js";
 import { splitWords } from "./word.js";
+const defaultTokenizerOptions = {
+  language: "en-US",
+  minSentenceLength: 20,
+  streamContextLength: 10,
+  retainFormat: false
+};
 class SentenceTokenizer extends tokenizer.SentenceTokenizer {
   #config;
-  constructor(language = "en-US", minSentenceLength = 20, streamContextLength = 10) {
+  constructor(options) {
     super();
     this.#config = {
-      language,
-      minSentenceLength,
-      streamContextLength
+      ...defaultTokenizerOptions,
+      ...options
     };
   }
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
@@ -21,7 +26,7 @@ class SentenceTokenizer extends tokenizer.SentenceTokenizer {
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
   stream(language) {
     return new BufferedSentenceStream(
-      (text) => splitSentences(text, this.#config.minSentenceLength),
+      (text) => splitSentences(text, this.#config.minSentenceLength, this.#config.retainFormat),
       this.#config.minSentenceLength,
       this.#config.streamContextLength
     );

package/dist/tokenize/basic/basic.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/tokenize/basic/basic.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { BufferedSentenceStream, BufferedWordStream } from '../token_stream.js';\nimport * as tokenizer from '../tokenizer.js';\nimport { hyphenator } from './hyphenator.js';\nimport { splitParagraphs } from './paragraph.js';\nimport { splitSentences } from './sentence.js';\nimport { splitWords } from './word.js';\n\ninterface TokenizerOptions {\n  language: string;\n  minSentenceLength: number;\n  streamContextLength: number;\n}\n\nexport class SentenceTokenizer extends tokenizer.SentenceTokenizer {\n  #config: TokenizerOptions;\n\n  constructor(language = 'en-US', minSentenceLength = 20, streamContextLength = 10) {\n    super();\n    this.#config = {\n      language,\n      minSentenceLength,\n      streamContextLength,\n    };\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  tokenize(text: string, language?: string): string[] {\n    return splitSentences(text, this.#config.minSentenceLength).map((tok) => tok[0]);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  stream(language?: string): tokenizer.SentenceStream {\n    return new BufferedSentenceStream(\n      (text: string) => splitSentences(text, this.#config.minSentenceLength),\n      this.#config.minSentenceLength,\n      this.#config.streamContextLength,\n    );\n  }\n}\n\nexport class WordTokenizer extends tokenizer.WordTokenizer {\n  #ignorePunctuation: boolean;\n\n  constructor(ignorePunctuation = true) {\n    super();\n    this.#ignorePunctuation = ignorePunctuation;\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  tokenize(text: string, language?: string): string[] {\n    return splitWords(text, this.#ignorePunctuation).map((tok) => tok[0]);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  stream(language?: string): tokenizer.WordStream {\n    return new BufferedWordStream(\n      (text: string) => splitWords(text, this.#ignorePunctuation),\n      1,\n      1,\n    );\n  }\n}\n\nexport const hyphenateWord = (word: string): string[] => {\n  return hyphenator.hyphenateWord(word);\n};\n\nexport { splitWords };\n\nexport const tokenizeParagraphs = (text: string): string[] => {\n  return splitParagraphs(text).map((tok) => tok[0]);\n};\n"],"mappings":"AAGA,SAAS,wBAAwB,0BAA0B;AAC3D,YAAY,eAAe;AAC3B,SAAS,kBAAkB;AAC3B,SAAS,uBAAuB;AAChC,SAAS,sBAAsB;AAC/B,SAAS,kBAAkB;AAQpB,MAAM,0BAA0B,UAAU,kBAAkB;AAAA,EACjE;AAAA,EAEA,YAAY,WAAW,SAAS,oBAAoB,IAAI,sBAAsB,IAAI;AAChF,UAAM;AACN,SAAK,UAAU;AAAA,MACb;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAGA,SAAS,MAAc,UAA6B;AAClD,WAAO,eAAe,MAAM,KAAK,QAAQ,iBAAiB,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;AAAA,EACjF;AAAA;AAAA,EAGA,OAAO,UAA6C;AAClD,WAAO,IAAI;AAAA,MACT,CAAC,SAAiB,eAAe,MAAM,KAAK,QAAQ,iBAAiB;AAAA,MACrE,KAAK,QAAQ;AAAA,MACb,KAAK,QAAQ;AAAA,IACf;AAAA,EACF;AACF;AAEO,MAAM,sBAAsB,UAAU,cAAc;AAAA,EACzD;AAAA,EAEA,YAAY,oBAAoB,MAAM;AACpC,UAAM;AACN,SAAK,qBAAqB;AAAA,EAC5B;AAAA;AAAA,EAGA,SAAS,MAAc,UAA6B;AAClD,WAAO,WAAW,MAAM,KAAK,kBAAkB,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;AAAA,EACtE;AAAA;AAAA,EAGA,OAAO,UAAyC;AAC9C,WAAO,IAAI;AAAA,MACT,CAAC,SAAiB,WAAW,MAAM,KAAK,kBAAkB;AAAA,MAC1D;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;AAEO,MAAM,gBAAgB,CAAC,SAA2B;AACvD,SAAO,WAAW,cAAc,IAAI;AACtC;AAIO,MAAM,qBAAqB,CAAC,SAA2B;AAC5D,SAAO,gBAAgB,IAAI,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;AAClD;","names":[]}
+{"version":3,"sources":["../../../src/tokenize/basic/basic.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { BufferedSentenceStream, BufferedWordStream } from '../token_stream.js';\nimport * as tokenizer from '../tokenizer.js';\nimport { hyphenator } from './hyphenator.js';\nimport { splitParagraphs } from './paragraph.js';\nimport { splitSentences } from './sentence.js';\nimport { splitWords } from './word.js';\n\ninterface TokenizerOptions {\n  language: string;\n  minSentenceLength: number;\n  streamContextLength: number;\n  retainFormat: boolean;\n}\n\nconst defaultTokenizerOptions: TokenizerOptions = {\n  language: 'en-US',\n  minSentenceLength: 20,\n  streamContextLength: 10,\n  retainFormat: false,\n};\n\nexport class SentenceTokenizer extends tokenizer.SentenceTokenizer {\n  #config: TokenizerOptions;\n\n  constructor(options?: Partial<TokenizerOptions>) {\n    super();\n    this.#config = {\n      ...defaultTokenizerOptions,\n      ...options,\n    };\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  tokenize(text: string, language?: string): string[] {\n    return splitSentences(text, this.#config.minSentenceLength).map((tok) => tok[0]);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  stream(language?: string): tokenizer.SentenceStream {\n    return new BufferedSentenceStream(\n      (text: string) =>\n        splitSentences(text, this.#config.minSentenceLength, this.#config.retainFormat),\n      this.#config.minSentenceLength,\n      this.#config.streamContextLength,\n    );\n  }\n}\n\nexport class WordTokenizer extends tokenizer.WordTokenizer {\n  #ignorePunctuation: boolean;\n\n  constructor(ignorePunctuation = true) {\n    super();\n    this.#ignorePunctuation = ignorePunctuation;\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  tokenize(text: string, language?: string): string[] {\n    return splitWords(text, this.#ignorePunctuation).map((tok) => tok[0]);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  stream(language?: string): tokenizer.WordStream {\n    return new BufferedWordStream(\n      (text: string) => splitWords(text, this.#ignorePunctuation),\n      1,\n      1,\n    );\n  }\n}\n\nexport const hyphenateWord = (word: string): string[] => {\n  return hyphenator.hyphenateWord(word);\n};\n\nexport { splitWords };\n\nexport const tokenizeParagraphs = (text: string): string[] => {\n  return splitParagraphs(text).map((tok) => tok[0]);\n};\n"],"mappings":"AAGA,SAAS,wBAAwB,0BAA0B;AAC3D,YAAY,eAAe;AAC3B,SAAS,kBAAkB;AAC3B,SAAS,uBAAuB;AAChC,SAAS,sBAAsB;AAC/B,SAAS,kBAAkB;AAS3B,MAAM,0BAA4C;AAAA,EAChD,UAAU;AAAA,EACV,mBAAmB;AAAA,EACnB,qBAAqB;AAAA,EACrB,cAAc;AAChB;AAEO,MAAM,0BAA0B,UAAU,kBAAkB;AAAA,EACjE;AAAA,EAEA,YAAY,SAAqC;AAC/C,UAAM;AACN,SAAK,UAAU;AAAA,MACb,GAAG;AAAA,MACH,GAAG;AAAA,IACL;AAAA,EACF;AAAA;AAAA,EAGA,SAAS,MAAc,UAA6B;AAClD,WAAO,eAAe,MAAM,KAAK,QAAQ,iBAAiB,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;AAAA,EACjF;AAAA;AAAA,EAGA,OAAO,UAA6C;AAClD,WAAO,IAAI;AAAA,MACT,CAAC,SACC,eAAe,MAAM,KAAK,QAAQ,mBAAmB,KAAK,QAAQ,YAAY;AAAA,MAChF,KAAK,QAAQ;AAAA,MACb,KAAK,QAAQ;AAAA,IACf;AAAA,EACF;AACF;AAEO,MAAM,sBAAsB,UAAU,cAAc;AAAA,EACzD;AAAA,EAEA,YAAY,oBAAoB,MAAM;AACpC,UAAM;AACN,SAAK,qBAAqB;AAAA,EAC5B;AAAA;AAAA,EAGA,SAAS,MAAc,UAA6B;AAClD,WAAO,WAAW,MAAM,KAAK,kBAAkB,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;AAAA,EACtE;AAAA;AAAA,EAGA,OAAO,UAAyC;AAC9C,WAAO,IAAI;AAAA,MACT,CAAC,SAAiB,WAAW,MAAM,KAAK,kBAAkB;AAAA,MAC1D;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;AAEO,MAAM,gBAAgB,CAAC,SAA2B;AACvD,SAAO,WAAW,cAAc,IAAI;AACtC;AAIO,MAAM,qBAAqB,CAAC,SAA2B;AAC5D,SAAO,gBAAgB,IAAI,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;AAClD;","names":[]}

package/dist/tokenize/basic/sentence.cjs

@@ -21,7 +21,7 @@ __export(sentence_exports, {
   splitSentences: () => splitSentences
 });
 module.exports = __toCommonJS(sentence_exports);
-const splitSentences = (text, minLength = 20) => {
+const splitSentences = (text, minLength = 20, retainFormat = false) => {
   const alphabets = /([A-Za-z])/g;
   const prefixes = /(Mr|St|Mrs|Ms|Dr)[.]/g;
   const suffixes = /(Inc|Ltd|Jr|Sr|Co)/g;
@@ -30,7 +30,11 @@ const splitSentences = (text, minLength = 20) => {
   const websites = /[.](com|net|org|io|gov|edu|me)/g;
   const digits = /([0-9])/g;
   const dots = /\.{2,}/g;
-  text = text.replaceAll("\n", " ");
+  if (retainFormat) {
+    text = text.replaceAll("\n", "<nel><stop>");
+  } else {
+    text = text.replaceAll("\n", " ");
+  }
   text = text.replaceAll(prefixes, "$1<prd>");
   text = text.replaceAll(websites, "<prd>$2");
   text = text.replaceAll(new RegExp(`${digits.source}[.]${digits.source}`, "g"), "$1<prd>$2");
@@ -60,25 +64,29 @@ const splitSentences = (text, minLength = 20) => {
   text = text.replaceAll("?", "?<stop>");
   text = text.replaceAll("!", "!<stop>");
   text = text.replaceAll("<prd>", ".");
+  if (retainFormat) {
+    text = text.replaceAll("<nel>", "\n");
+  }
   const split = text.split("<stop>");
   text = text.replaceAll("<stop>", "");
   const sentences = [];
   let buf = "";
   let start = 0;
   let end = 0;
+  const prePad = retainFormat ? "" : " ";
   for (const match of split) {
-    const sentence = match.trim();
+    const sentence = retainFormat ? match : match.trim();
     if (!sentence) continue;
-    buf += " " + sentence;
+    buf += prePad + sentence;
     end += match.length;
     if (buf.length > minLength) {
-      sentences.push([buf.slice(1), start, end]);
+      sentences.push([buf.slice(prePad.length), start, end]);
       start = end;
       buf = "";
     }
   }
   if (buf) {
-    sentences.push([buf.slice(1), start, text.length - 1]);
+    sentences.push([buf.slice(prePad.length), start, text.length - 1]);
   }
   return sentences;
 };

package/dist/tokenize/basic/sentence.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/tokenize/basic/sentence.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\n/**\n * Split the text into sentences.\n */\nexport const splitSentences = (text: string, minLength = 20): [string, number, number][] => {\n  const alphabets = /([A-Za-z])/g;\n  const prefixes = /(Mr|St|Mrs|Ms|Dr)[.]/g;\n  const suffixes = /(Inc|Ltd|Jr|Sr|Co)/g;\n  const starters =\n    /(Mr|Mrs|Ms|Dr|Prof|Capt|Cpt|Lt|He\\s|She\\s|It\\s|They\\s|Their\\s|Our\\s|We\\s|But\\s|However\\s|That\\s|This\\s|Wherever)/g;\n  const acronyms = /([A-Z][.][A-Z][.](?:[A-Z][.])?)/g;\n  const websites = /[.](com|net|org|io|gov|edu|me)/g;\n  const digits = /([0-9])/g;\n  const dots = /\\.{2,}/g;\n\n  text = text.replaceAll('\\n', ' ');\n  text = text.replaceAll(prefixes, '$1<prd>');\n  text = text.replaceAll(websites, '<prd>$2');\n  text = text.replaceAll(new RegExp(`${digits.source}[.]${digits.source}`, 'g'), '$1<prd>$2');\n  text = text.replaceAll(dots, (match) => '<prd>'.repeat(match.length));\n  text = text.replaceAll('Ph.D.', 'Ph<prd>D<prd>');\n  text = text.replaceAll(new RegExp(`\\\\s${alphabets.source}[.] `, 'g'), ' $1<prd> ');\n  text = text.replaceAll(new RegExp(`${acronyms.source} ${starters.source}`, 'g'), '$1<stop> $2');\n  text = text.replaceAll(\n    new RegExp(`${alphabets.source}[.]${alphabets.source}[.]${alphabets.source}[.]`, 'g'),\n    '$1<prd>$2<prd>$3<prd>',\n  );\n  text = text.replaceAll(\n    new RegExp(`${alphabets.source}[.]${alphabets.source}[.]`, 'g'),\n    '$1<prd>$2<prd>',\n  );\n  text = text.replaceAll(\n    new RegExp(` ${suffixes.source}[.] ${starters.source}`, 'g'),\n    '$1<stop> $2',\n  );\n  text = text.replaceAll(new RegExp(` ${suffixes.source}[.]`, 'g'), '$1<prd>');\n  text = text.replaceAll(new RegExp(` ${alphabets.source}[.]`, 'g'), '$1<prd>');\n  text = text.replaceAll('.”', '”.');\n  text = text.replaceAll('.\"', '\".');\n  text = text.replaceAll('!\"', '\"!');\n  text = text.replaceAll('?\"', '\"?');\n  text = text.replaceAll('.', '.<stop>');\n  text = text.replaceAll('?', '?<stop>');\n  text = text.replaceAll('!', '!<stop>');\n  text = text.replaceAll('<prd>', '.');\n\n  const split = text.split('<stop>');\n  text = text.replaceAll('<stop>', '');\n\n  const sentences: [string, number, number][] = [];\n  let buf = '';\n  let start = 0;\n  let end = 0;\n  for (const match of split) {\n    const sentence = match.trim();\n    if (!sentence) continue;\n\n    buf += ' ' + sentence;\n    end += match.length;\n    if (buf.length > minLength) {\n      sentences.push([buf.slice(1), start, end]);\n      start = end;\n      buf = '';\n    }\n  }\n\n  if (buf) {\n    sentences.push([buf.slice(1), start, text.length - 1]);\n  }\n\n  return sentences;\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAOO,MAAM,iBAAiB,CAAC,MAAc,YAAY,OAAmC;AAC1F,QAAM,YAAY;AAClB,QAAM,WAAW;AACjB,QAAM,WAAW;AACjB,QAAM,WACJ;AACF,QAAM,WAAW;AACjB,QAAM,WAAW;AACjB,QAAM,SAAS;AACf,QAAM,OAAO;AAEb,SAAO,KAAK,WAAW,MAAM,GAAG;AAChC,SAAO,KAAK,WAAW,UAAU,SAAS;AAC1C,SAAO,KAAK,WAAW,UAAU,SAAS;AAC1C,SAAO,KAAK,WAAW,IAAI,OAAO,GAAG,OAAO,MAAM,MAAM,OAAO,MAAM,IAAI,GAAG,GAAG,WAAW;AAC1F,SAAO,KAAK,WAAW,MAAM,CAAC,UAAU,QAAQ,OAAO,MAAM,MAAM,CAAC;AACpE,SAAO,KAAK,WAAW,SAAS,eAAe;AAC/C,SAAO,KAAK,WAAW,IAAI,OAAO,MAAM,UAAU,MAAM,QAAQ,GAAG,GAAG,WAAW;AACjF,SAAO,KAAK,WAAW,IAAI,OAAO,GAAG,SAAS,MAAM,IAAI,SAAS,MAAM,IAAI,GAAG,GAAG,aAAa;AAC9F,SAAO,KAAK;AAAA,IACV,IAAI,OAAO,GAAG,UAAU,MAAM,MAAM,UAAU,MAAM,MAAM,UAAU,MAAM,OAAO,GAAG;AAAA,IACpF;AAAA,EACF;AACA,SAAO,KAAK;AAAA,IACV,IAAI,OAAO,GAAG,UAAU,MAAM,MAAM,UAAU,MAAM,OAAO,GAAG;AAAA,IAC9D;AAAA,EACF;AACA,SAAO,KAAK;AAAA,IACV,IAAI,OAAO,IAAI,SAAS,MAAM,OAAO,SAAS,MAAM,IAAI,GAAG;AAAA,IAC3D;AAAA,EACF;AACA,SAAO,KAAK,WAAW,IAAI,OAAO,IAAI,SAAS,MAAM,OAAO,GAAG,GAAG,SAAS;AAC3E,SAAO,KAAK,WAAW,IAAI,OAAO,IAAI,UAAU,MAAM,OAAO,GAAG,GAAG,SAAS;AAC5E,SAAO,KAAK,WAAW,WAAM,SAAI;AACjC,SAAO,KAAK,WAAW,MAAM,IAAI;AACjC,SAAO,KAAK,WAAW,MAAM,IAAI;AACjC,SAAO,KAAK,WAAW,MAAM,IAAI;AACjC,SAAO,KAAK,WAAW,KAAK,SAAS;AACrC,SAAO,KAAK,WAAW,KAAK,SAAS;AACrC,SAAO,KAAK,WAAW,KAAK,SAAS;AACrC,SAAO,KAAK,WAAW,SAAS,GAAG;AAEnC,QAAM,QAAQ,KAAK,MAAM,QAAQ;AACjC,SAAO,KAAK,WAAW,UAAU,EAAE;AAEnC,QAAM,YAAwC,CAAC;AAC/C,MAAI,MAAM;AACV,MAAI,QAAQ;AACZ,MAAI,MAAM;AACV,aAAW,SAAS,OAAO;AACzB,UAAM,WAAW,MAAM,KAAK;AAC5B,QAAI,CAAC,SAAU;AAEf,WAAO,MAAM;AACb,WAAO,MAAM;AACb,QAAI,IAAI,SAAS,WAAW;AAC1B,gBAAU,KAAK,CAAC,IAAI,MAAM,CAAC,GAAG,OAAO,GAAG,CAAC;AACzC,cAAQ;AACR,YAAM;AAAA,IACR;AAAA,EACF;AAEA,MAAI,KAAK;AACP,cAAU,KAAK,CAAC,IAAI,MAAM,CAAC,GAAG,OAAO,KAAK,SAAS,CAAC,CAAC;AAAA,EACvD;AAEA,SAAO;AACT;","names":[]}
+{"version":3,"sources":["../../../src/tokenize/basic/sentence.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\n\n/**\n * Split the text into sentences.\n */\nexport const splitSentences = (\n  text: string,\n  minLength = 20,\n  retainFormat: boolean = false,\n): [string, number, number][] => {\n  const alphabets = /([A-Za-z])/g;\n  const prefixes = /(Mr|St|Mrs|Ms|Dr)[.]/g;\n  const suffixes = /(Inc|Ltd|Jr|Sr|Co)/g;\n  const starters =\n    /(Mr|Mrs|Ms|Dr|Prof|Capt|Cpt|Lt|He\\s|She\\s|It\\s|They\\s|Their\\s|Our\\s|We\\s|But\\s|However\\s|That\\s|This\\s|Wherever)/g;\n  const acronyms = /([A-Z][.][A-Z][.](?:[A-Z][.])?)/g;\n  const websites = /[.](com|net|org|io|gov|edu|me)/g;\n  const digits = /([0-9])/g;\n  const dots = /\\.{2,}/g;\n\n  if (retainFormat) {\n    text = text.replaceAll('\\n', '<nel><stop>');\n  } else {\n    text = text.replaceAll('\\n', ' ');\n  }\n\n  text = text.replaceAll(prefixes, '$1<prd>');\n  text = text.replaceAll(websites, '<prd>$2');\n  text = text.replaceAll(new RegExp(`${digits.source}[.]${digits.source}`, 'g'), '$1<prd>$2');\n  text = text.replaceAll(dots, (match) => '<prd>'.repeat(match.length));\n  text = text.replaceAll('Ph.D.', 'Ph<prd>D<prd>');\n  text = text.replaceAll(new RegExp(`\\\\s${alphabets.source}[.] `, 'g'), ' $1<prd> ');\n  text = text.replaceAll(new RegExp(`${acronyms.source} ${starters.source}`, 'g'), '$1<stop> $2');\n  text = text.replaceAll(\n    new RegExp(`${alphabets.source}[.]${alphabets.source}[.]${alphabets.source}[.]`, 'g'),\n    '$1<prd>$2<prd>$3<prd>',\n  );\n  text = text.replaceAll(\n    new RegExp(`${alphabets.source}[.]${alphabets.source}[.]`, 'g'),\n    '$1<prd>$2<prd>',\n  );\n  text = text.replaceAll(\n    new RegExp(` ${suffixes.source}[.] ${starters.source}`, 'g'),\n    '$1<stop> $2',\n  );\n  text = text.replaceAll(new RegExp(` ${suffixes.source}[.]`, 'g'), '$1<prd>');\n  text = text.replaceAll(new RegExp(` ${alphabets.source}[.]`, 'g'), '$1<prd>');\n  text = text.replaceAll('.”', '”.');\n  text = text.replaceAll('.\"', '\".');\n  text = text.replaceAll('!\"', '\"!');\n  text = text.replaceAll('?\"', '\"?');\n  text = text.replaceAll('.', '.<stop>');\n  text = text.replaceAll('?', '?<stop>');\n  text = text.replaceAll('!', '!<stop>');\n  text = text.replaceAll('<prd>', '.');\n\n  if (retainFormat) {\n    text = text.replaceAll('<nel>', '\\n');\n  }\n\n  const split = text.split('<stop>');\n  text = text.replaceAll('<stop>', '');\n\n  const sentences: [string, number, number][] = [];\n  let buf = '';\n  let start = 0;\n  let end = 0;\n  const prePad = retainFormat ? '' : ' ';\n  for (const match of split) {\n    const sentence = retainFormat ? match : match.trim();\n    if (!sentence) continue;\n\n    buf += prePad + sentence;\n    end += match.length;\n    if (buf.length > minLength) {\n      sentences.push([buf.slice(prePad.length), start, end]);\n      start = end;\n      buf = '';\n    }\n  }\n\n  if (buf) {\n    sentences.push([buf.slice(prePad.length), start, text.length - 1]);\n  }\n\n  return sentences;\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAOO,MAAM,iBAAiB,CAC5B,MACA,YAAY,IACZ,eAAwB,UACO;AAC/B,QAAM,YAAY;AAClB,QAAM,WAAW;AACjB,QAAM,WAAW;AACjB,QAAM,WACJ;AACF,QAAM,WAAW;AACjB,QAAM,WAAW;AACjB,QAAM,SAAS;AACf,QAAM,OAAO;AAEb,MAAI,cAAc;AAChB,WAAO,KAAK,WAAW,MAAM,aAAa;AAAA,EAC5C,OAAO;AACL,WAAO,KAAK,WAAW,MAAM,GAAG;AAAA,EAClC;AAEA,SAAO,KAAK,WAAW,UAAU,SAAS;AAC1C,SAAO,KAAK,WAAW,UAAU,SAAS;AAC1C,SAAO,KAAK,WAAW,IAAI,OAAO,GAAG,OAAO,MAAM,MAAM,OAAO,MAAM,IAAI,GAAG,GAAG,WAAW;AAC1F,SAAO,KAAK,WAAW,MAAM,CAAC,UAAU,QAAQ,OAAO,MAAM,MAAM,CAAC;AACpE,SAAO,KAAK,WAAW,SAAS,eAAe;AAC/C,SAAO,KAAK,WAAW,IAAI,OAAO,MAAM,UAAU,MAAM,QAAQ,GAAG,GAAG,WAAW;AACjF,SAAO,KAAK,WAAW,IAAI,OAAO,GAAG,SAAS,MAAM,IAAI,SAAS,MAAM,IAAI,GAAG,GAAG,aAAa;AAC9F,SAAO,KAAK;AAAA,IACV,IAAI,OAAO,GAAG,UAAU,MAAM,MAAM,UAAU,MAAM,MAAM,UAAU,MAAM,OAAO,GAAG;AAAA,IACpF;AAAA,EACF;AACA,SAAO,KAAK;AAAA,IACV,IAAI,OAAO,GAAG,UAAU,MAAM,MAAM,UAAU,MAAM,OAAO,GAAG;AAAA,IAC9D;AAAA,EACF;AACA,SAAO,KAAK;AAAA,IACV,IAAI,OAAO,IAAI,SAAS,MAAM,OAAO,SAAS,MAAM,IAAI,GAAG;AAAA,IAC3D;AAAA,EACF;AACA,SAAO,KAAK,WAAW,IAAI,OAAO,IAAI,SAAS,MAAM,OAAO,GAAG,GAAG,SAAS;AAC3E,SAAO,KAAK,WAAW,IAAI,OAAO,IAAI,UAAU,MAAM,OAAO,GAAG,GAAG,SAAS;AAC5E,SAAO,KAAK,WAAW,WAAM,SAAI;AACjC,SAAO,KAAK,WAAW,MAAM,IAAI;AACjC,SAAO,KAAK,WAAW,MAAM,IAAI;AACjC,SAAO,KAAK,WAAW,MAAM,IAAI;AACjC,SAAO,KAAK,WAAW,KAAK,SAAS;AACrC,SAAO,KAAK,WAAW,KAAK,SAAS;AACrC,SAAO,KAAK,WAAW,KAAK,SAAS;AACrC,SAAO,KAAK,WAAW,SAAS,GAAG;AAEnC,MAAI,cAAc;AAChB,WAAO,KAAK,WAAW,SAAS,IAAI;AAAA,EACtC;AAEA,QAAM,QAAQ,KAAK,MAAM,QAAQ;AACjC,SAAO,KAAK,WAAW,UAAU,EAAE;AAEnC,QAAM,YAAwC,CAAC;AAC/C,MAAI,MAAM;AACV,MAAI,QAAQ;AACZ,MAAI,MAAM;AACV,QAAM,SAAS,eAAe,KAAK;AACnC,aAAW,SAAS,OAAO;AACzB,UAAM,WAAW,eAAe,QAAQ,MAAM,KAAK;AACnD,QAAI,CAAC,SAAU;AAEf,WAAO,SAAS;AAChB,WAAO,MAAM;AACb,QAAI,IAAI,SAAS,WAAW;AAC1B,gBAAU,KAAK,CAAC,IAAI,MAAM,OAAO,MAAM,GAAG,OAAO,GAAG,CAAC;AACrD,cAAQ;AACR,YAAM;AAAA,IACR;AAAA,EACF;AAEA,MAAI,KAAK;AACP,cAAU,KAAK,CAAC,IAAI,MAAM,OAAO,MAAM,GAAG,OAAO,KAAK,SAAS,CAAC,CAAC;AAAA,EACnE;AAEA,SAAO;AACT;","names":[]}

package/dist/tokenize/basic/sentence.d.cts

@@ -1,5 +1,5 @@
 /**
  * Split the text into sentences.
  */
-export declare const splitSentences: (text: string, minLength?: number) => [string, number, number][];
+export declare const splitSentences: (text: string, minLength?: number, retainFormat?: boolean) => [string, number, number][];
 //# sourceMappingURL=sentence.d.ts.map

package/dist/tokenize/basic/sentence.d.ts

@@ -1,5 +1,5 @@
 /**
  * Split the text into sentences.
  */
-export declare const splitSentences: (text: string, minLength?: number) => [string, number, number][];
+export declare const splitSentences: (text: string, minLength?: number, retainFormat?: boolean) => [string, number, number][];
 //# sourceMappingURL=sentence.d.ts.map

package/dist/tokenize/basic/sentence.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"sentence.d.ts","sourceRoot":"","sources":["../../../src/tokenize/basic/sentence.ts"],"names":[],"mappings":"AAIA;;GAEG;AACH,eAAO,MAAM,cAAc,SAAU,MAAM,yBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAmErF,CAAC"}
+{"version":3,"file":"sentence.d.ts","sourceRoot":"","sources":["../../../src/tokenize/basic/sentence.ts"],"names":[],"mappings":"AAIA;;GAEG;AACH,eAAO,MAAM,cAAc,SACnB,MAAM,qCAEE,OAAO,KACpB,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EA6E1B,CAAC"}

package/dist/tokenize/basic/sentence.js

@@ -1,4 +1,4 @@
-const splitSentences = (text, minLength = 20) => {
+const splitSentences = (text, minLength = 20, retainFormat = false) => {
   const alphabets = /([A-Za-z])/g;
   const prefixes = /(Mr|St|Mrs|Ms|Dr)[.]/g;
   const suffixes = /(Inc|Ltd|Jr|Sr|Co)/g;
@@ -7,7 +7,11 @@ const splitSentences = (text, minLength = 20) => {
   const websites = /[.](com|net|org|io|gov|edu|me)/g;
   const digits = /([0-9])/g;
   const dots = /\.{2,}/g;
-  text = text.replaceAll("\n", " ");
+  if (retainFormat) {
+    text = text.replaceAll("\n", "<nel><stop>");
+  } else {
+    text = text.replaceAll("\n", " ");
+  }
   text = text.replaceAll(prefixes, "$1<prd>");
   text = text.replaceAll(websites, "<prd>$2");
   text = text.replaceAll(new RegExp(`${digits.source}[.]${digits.source}`, "g"), "$1<prd>$2");
@@ -37,25 +41,29 @@ const splitSentences = (text, minLength = 20) => {
   text = text.replaceAll("?", "?<stop>");
   text = text.replaceAll("!", "!<stop>");
   text = text.replaceAll("<prd>", ".");
+  if (retainFormat) {
+    text = text.replaceAll("<nel>", "\n");
+  }
   const split = text.split("<stop>");
   text = text.replaceAll("<stop>", "");
   const sentences = [];
   let buf = "";
   let start = 0;
   let end = 0;
+  const prePad = retainFormat ? "" : " ";
   for (const match of split) {
-    const sentence = match.trim();
+    const sentence = retainFormat ? match : match.trim();
     if (!sentence) continue;
-    buf += " " + sentence;
+    buf += prePad + sentence;
     end += match.length;
     if (buf.length > minLength) {
-      sentences.push([buf.slice(1), start, end]);
+      sentences.push([buf.slice(prePad.length), start, end]);
       start = end;
       buf = "";
     }
   }
   if (buf) {
-    sentences.push([buf.slice(1), start, text.length - 1]);
+    sentences.push([buf.slice(prePad.length), start, text.length - 1]);
   }
   return sentences;
 };