@livekit/agents 0.4.5 → 0.5.0

package/dist/tokenize/token_stream.js

@@ -1,144 +1,138 @@
-// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
-//
-// SPDX-License-Identifier: Apache-2.0
-import { randomUUID } from 'node:crypto';
-import { AsyncIterableQueue } from '../utils.js';
-import { SentenceStream, WordStream } from './tokenizer.js';
-export class BufferedTokenStream {
-    queue = new AsyncIterableQueue();
-    closed = false;
-    #func;
-    #minTokenLength;
-    #minContextLength;
-    #bufTokens = [];
-    #inBuf = '';
-    #outBuf = '';
-    #currentSegmentId;
-    constructor(func, minTokenLength, minContextLength) {
-        this.#func = func;
-        this.#minTokenLength = minTokenLength;
-        this.#minContextLength = minContextLength;
-        this.#currentSegmentId = randomUUID();
-    }
-    /** Push a string of text into the token stream */
-    pushText(text) {
-        if (this.closed) {
-            throw new Error('Stream is closed');
-        }
-        this.#inBuf += text;
-        if (this.#inBuf.length < this.#minContextLength)
-            return;
-        while (true) {
-            const tokens = this.#func(this.#inBuf);
-            if (tokens.length <= 1)
-                break;
-            if (this.#outBuf)
-                this.#outBuf += ' ';
-            const tok = tokens.shift();
-            let tokText = tok;
-            if (tok.length > 1 && typeof tok[1] === 'number') {
-                tokText = tok[0];
-            }
-            this.#outBuf += tokText;
-            if (this.#outBuf.length >= this.#minTokenLength) {
-                this.queue.put({ token: this.#outBuf, segmentId: this.#currentSegmentId });
-                this.#outBuf = '';
-            }
-            if (typeof tok !== 'string') {
-                this.#inBuf = this.#inBuf.slice(tok[2]);
-            }
-            else {
-                this.#inBuf = this.#inBuf
-                    .slice(Math.max(0, this.#inBuf.indexOf(tok)) + tok.length)
-                    .trimStart();
-            }
-        }
-    }
-    /** Flush the stream, causing it to process all pending text */
-    flush() {
-        if (this.closed) {
-            throw new Error('Stream is closed');
-        }
-        if (this.#inBuf || this.#outBuf) {
-            const tokens = this.#func(this.#inBuf);
-            if (tokens) {
-                if (this.#outBuf)
-                    this.#outBuf += ' ';
-                if (typeof tokens[0] !== 'string') {
-                    this.#outBuf += tokens.map((tok) => tok[0]).join(' ');
-                }
-                else {
-                    this.#outBuf += tokens.join(' ');
-                }
-            }
-            if (this.#outBuf) {
-                this.queue.put({ token: this.#outBuf, segmentId: this.#currentSegmentId });
-            }
-            this.#currentSegmentId = randomUUID();
+import { randomUUID } from "node:crypto";
+import { AsyncIterableQueue } from "../utils.js";
+import { SentenceStream, WordStream } from "./tokenizer.js";
+class BufferedTokenStream {
+  queue = new AsyncIterableQueue();
+  closed = false;
+  #func;
+  #minTokenLength;
+  #minContextLength;
+  #bufTokens = [];
+  #inBuf = "";
+  #outBuf = "";
+  #currentSegmentId;
+  constructor(func, minTokenLength, minContextLength) {
+    this.#func = func;
+    this.#minTokenLength = minTokenLength;
+    this.#minContextLength = minContextLength;
+    this.#currentSegmentId = randomUUID();
+  }
+  /** Push a string of text into the token stream */
+  pushText(text) {
+    if (this.closed) {
+      throw new Error("Stream is closed");
+    }
+    this.#inBuf += text;
+    if (this.#inBuf.length < this.#minContextLength) return;
+    while (true) {
+      const tokens = this.#func(this.#inBuf);
+      if (tokens.length <= 1) break;
+      if (this.#outBuf) this.#outBuf += " ";
+      const tok = tokens.shift();
+      let tokText = tok;
+      if (tok.length > 1 && typeof tok[1] === "number") {
+        tokText = tok[0];
+      }
+      this.#outBuf += tokText;
+      if (this.#outBuf.length >= this.#minTokenLength) {
+        this.queue.put({ token: this.#outBuf, segmentId: this.#currentSegmentId });
+        this.#outBuf = "";
+      }
+      if (typeof tok !== "string") {
+        this.#inBuf = this.#inBuf.slice(tok[2]);
+      } else {
+        this.#inBuf = this.#inBuf.slice(Math.max(0, this.#inBuf.indexOf(tok)) + tok.length).trimStart();
+      }
+    }
+  }
+  /** Flush the stream, causing it to process all pending text */
+  flush() {
+    if (this.closed) {
+      throw new Error("Stream is closed");
+    }
+    if (this.#inBuf || this.#outBuf) {
+      const tokens = this.#func(this.#inBuf);
+      if (tokens) {
+        if (this.#outBuf) this.#outBuf += " ";
+        if (typeof tokens[0] !== "string") {
+          this.#outBuf += tokens.map((tok) => tok[0]).join(" ");
+        } else {
+          this.#outBuf += tokens.join(" ");
         }
-        this.#inBuf = '';
-        this.#outBuf = '';
-    }
-    /** Mark the input as ended and forbid additional pushes */
-    endInput() {
-        if (this.closed) {
-            throw new Error('Stream is closed');
-        }
-        this.flush();
-        this.close();
-    }
-    next() {
-        return this.queue.next();
-    }
-    /** Close both the input and output of the token stream */
-    close() {
-        this.queue.close();
-        this.closed = true;
-    }
-    [Symbol.asyncIterator]() {
-        return this;
-    }
+      }
+      if (this.#outBuf) {
+        this.queue.put({ token: this.#outBuf, segmentId: this.#currentSegmentId });
+      }
+      this.#currentSegmentId = randomUUID();
+    }
+    this.#inBuf = "";
+    this.#outBuf = "";
+  }
+  /** Mark the input as ended and forbid additional pushes */
+  endInput() {
+    if (this.closed) {
+      throw new Error("Stream is closed");
+    }
+    this.flush();
+    this.close();
+  }
+  next() {
+    return this.queue.next();
+  }
+  /** Close both the input and output of the token stream */
+  close() {
+    this.queue.close();
+    this.closed = true;
+  }
+  [Symbol.asyncIterator]() {
+    return this;
+  }
 }
-export class BufferedSentenceStream extends SentenceStream {
-    #stream;
-    constructor(func, minTokenLength, minContextLength) {
-        super();
-        this.#stream = new BufferedTokenStream(func, minTokenLength, minContextLength);
-    }
-    pushText(text) {
-        this.#stream.pushText(text);
-    }
-    flush() {
-        this.#stream.flush();
-    }
-    close() {
-        super.close();
-        this.#stream.close();
-    }
-    next() {
-        return this.#stream.next();
-    }
+class BufferedSentenceStream extends SentenceStream {
+  #stream;
+  constructor(func, minTokenLength, minContextLength) {
+    super();
+    this.#stream = new BufferedTokenStream(func, minTokenLength, minContextLength);
+  }
+  pushText(text) {
+    this.#stream.pushText(text);
+  }
+  flush() {
+    this.#stream.flush();
+  }
+  close() {
+    super.close();
+    this.#stream.close();
+  }
+  next() {
+    return this.#stream.next();
+  }
 }
-export class BufferedWordStream extends WordStream {
-    #stream;
-    constructor(func, minTokenLength, minContextLength) {
-        super();
-        this.#stream = new BufferedTokenStream(func, minTokenLength, minContextLength);
-    }
-    pushText(text) {
-        this.#stream.pushText(text);
-    }
-    flush() {
-        this.#stream.flush();
-    }
-    endInput() {
-        this.#stream.endInput();
-    }
-    close() {
-        this.#stream.close();
-    }
-    next() {
-        return this.#stream.next();
-    }
+class BufferedWordStream extends WordStream {
+  #stream;
+  constructor(func, minTokenLength, minContextLength) {
+    super();
+    this.#stream = new BufferedTokenStream(func, minTokenLength, minContextLength);
+  }
+  pushText(text) {
+    this.#stream.pushText(text);
+  }
+  flush() {
+    this.#stream.flush();
+  }
+  endInput() {
+    this.#stream.endInput();
+  }
+  close() {
+    this.#stream.close();
+  }
+  next() {
+    return this.#stream.next();
+  }
 }
+export {
+  BufferedSentenceStream,
+  BufferedTokenStream,
+  BufferedWordStream
+};
 //# sourceMappingURL=token_stream.js.map

package/dist/tokenize/token_stream.js.map

@@ -1 +1 @@
-{"version":3,"file":"token_stream.js","sourceRoot":"","sources":["../../src/tokenize/token_stream.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C,EAAE;AACF,sCAAsC;AACtC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAEjD,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAI5D,MAAM,OAAO,mBAAmB;IACpB,KAAK,GAAG,IAAI,kBAAkB,EAAa,CAAC;IAC5C,MAAM,GAAG,KAAK,CAAC;IAEzB,KAAK,CAAe;IACpB,eAAe,CAAS;IACxB,iBAAiB,CAAS;IAC1B,UAAU,GAAa,EAAE,CAAC;IAC1B,MAAM,GAAG,EAAE,CAAC;IACZ,OAAO,GAAG,EAAE,CAAC;IACb,iBAAiB,CAAS;IAE1B,YAAY,IAAkB,EAAE,cAAsB,EAAE,gBAAwB;QAC9E,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAClB,IAAI,CAAC,eAAe,GAAG,cAAc,CAAC;QACtC,IAAI,CAAC,iBAAiB,GAAG,gBAAgB,CAAC;QAE1C,IAAI,CAAC,iBAAiB,GAAG,UAAU,EAAE,CAAC;IACxC,CAAC;IAED,kDAAkD;IAClD,QAAQ,CAAC,IAAY;QACnB,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACtC,CAAC;QAED,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC;QACpB,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,iBAAiB;YAAE,OAAO;QAExD,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvC,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC;gBAAE,MAAM;YAE9B,IAAI,IAAI,CAAC,OAAO;gBAAE,IAAI,CAAC,OAAO,IAAI,GAAG,CAAC;YAEtC,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,EAAG,CAAC;YAC5B,IAAI,OAAO,GAAG,GAAa,CAAC;YAC5B,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,GAAG,CAAC,CAAC,CAAC,KAAK,QAAQ,EAAE,CAAC;gBACjD,OAAO,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;YACnB,CAAC;YAED,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC;YACxB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;gBAChD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE,SAAS,EAAE,IAAI,CAAC,iBAAiB,EAAE,CAAC,CAAC;gBAC3E,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;YACpB,CAAC;YAED,IAAI,OAAO,GAAI,KAAK,QAAQ,EAAE,CAAC;gBAC7B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YAC3C,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM;qBACtB,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC;qBACzD,SAAS,EAAE,CAAC;YACjB,CAAC;QACH,CAAC;IACH,CAAC;IAED,+DAA+D;IAC/D,KAAK;QACH,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACtC,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAChC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvC,IAAI,MAAM,EAAE,CAAC;gBACX,IAAI,IAAI,CAAC,OAAO;oBAAE,IAAI,CAAC,OAAO,IAAI,GAAG,CAAC;gBAEtC,IAAI,OAAO,MAAM,CAAC,CAAC,CAAC,KAAK,QAAQ,EAAE,CAAC;oBAClC,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACxD,CAAC;qBAAM,CAAC;oBACN,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACnC,CAAC;YACH,CAAC;YAED,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBACjB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE,SAAS,EAAE,IAAI,CAAC,iBAAiB,EAAE,CAAC,CAAC;YAC7E,CAAC;YAED,IAAI,CAAC,iBAAiB,GAAG,UAAU,EAAE,CAAC;QACxC,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;QACjB,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;IACpB,CAAC;IAED,2DAA2D;IAC3D,QAAQ;QACN,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACtC,CAAC;QACD,IAAI,CAAC,KAAK,EAAE,CAAC;QACb,IAAI,CAAC,KAAK,EAAE,CAAC;IACf,CAAC;IAED,IAAI;QACF,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;IAC3B,CAAC;IAED,0DAA0D;IAC1D,KAAK;QACH,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;IACrB,CAAC;IAED,CAAC,MAAM,CAAC,aAAa,CAAC;QACpB,OAAO,IAAI,CAAC;IACd,CAAC;CACF;AAED,MAAM,OAAO,sBAAuB,SAAQ,cAAc;IACxD,OAAO,CAAsB;IAE7B,YAAY,IAAkB,EAAE,cAAsB,EAAE,gBAAwB;QAC9E,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,OAAO,GAAG,IAAI,mBAAmB,CAAC,IAAI,EAAE,cAAc,EAAE,gBAAgB,CAAC,CAAC;IACjF,CAAC;IAED,QAAQ,CAAC,IAAY;QACnB,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAED,KAAK;QACH,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;IAED,KAAK;QACH,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;IAED,IAAI;QACF,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IAC7B,CAAC;CACF;AAED,MAAM,OAAO,kBAAmB,SAAQ,UAAU;IAChD,OAAO,CAAsB;IAE7B,YAAY,IAAkB,EAAE,cAAsB,EAAE,gBAAwB;QAC9E,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,OAAO,GAAG,IAAI,mBAAmB,CAAC,IAAI,EAAE,cAAc,EAAE,gBAAgB,CAAC,CAAC;IACjF,CAAC;IAED,QAAQ,CAAC,IAAY;QACnB,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAED,KAAK;QACH,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;IAED,QAAQ;QACN,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;IAC1B,CAAC;IAED,KAAK;QACH,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;IAED,IAAI;QACF,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IAC7B,CAAC;CACF"}
+{"version":3,"sources":["../../src/tokenize/token_stream.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { randomUUID } from 'node:crypto';\nimport { AsyncIterableQueue } from '../utils.js';\nimport type { TokenData } from './tokenizer.js';\nimport { SentenceStream, WordStream } from './tokenizer.js';\n\ntype TokenizeFunc = (x: string) => string[] | [string, number, number][];\n\nexport class BufferedTokenStream implements AsyncIterableIterator<TokenData> {\n  protected queue = new AsyncIterableQueue<TokenData>();\n  protected closed = false;\n\n  #func: TokenizeFunc;\n  #minTokenLength: number;\n  #minContextLength: number;\n  #bufTokens: string[] = [];\n  #inBuf = '';\n  #outBuf = '';\n  #currentSegmentId: string;\n\n  constructor(func: TokenizeFunc, minTokenLength: number, minContextLength: number) {\n    this.#func = func;\n    this.#minTokenLength = minTokenLength;\n    this.#minContextLength = minContextLength;\n\n    this.#currentSegmentId = randomUUID();\n  }\n\n  /** Push a string of text into the token stream */\n  pushText(text: string) {\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n\n    this.#inBuf += text;\n    if (this.#inBuf.length < this.#minContextLength) return;\n\n    while (true) {\n      const tokens = this.#func(this.#inBuf);\n      if (tokens.length <= 1) break;\n\n      if (this.#outBuf) this.#outBuf += ' ';\n\n      const tok = tokens.shift()!;\n      let tokText = tok as string;\n      if (tok.length > 1 && typeof tok[1] === 'number') {\n        tokText = tok[0];\n      }\n\n      this.#outBuf += tokText;\n      if (this.#outBuf.length >= this.#minTokenLength) {\n        this.queue.put({ token: this.#outBuf, segmentId: this.#currentSegmentId });\n        this.#outBuf = '';\n      }\n\n      if (typeof tok! !== 'string') {\n        this.#inBuf = this.#inBuf.slice(tok![2]);\n      } else {\n        this.#inBuf = this.#inBuf\n          .slice(Math.max(0, this.#inBuf.indexOf(tok)) + tok.length)\n          .trimStart();\n      }\n    }\n  }\n\n  /** Flush the stream, causing it to process all pending text */\n  flush() {\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n\n    if (this.#inBuf || this.#outBuf) {\n      const tokens = this.#func(this.#inBuf);\n      if (tokens) {\n        if (this.#outBuf) this.#outBuf += ' ';\n\n        if (typeof tokens[0] !== 'string') {\n          this.#outBuf += tokens.map((tok) => tok[0]).join(' ');\n        } else {\n          this.#outBuf += tokens.join(' ');\n        }\n      }\n\n      if (this.#outBuf) {\n        this.queue.put({ token: this.#outBuf, segmentId: this.#currentSegmentId });\n      }\n\n      this.#currentSegmentId = randomUUID();\n    }\n\n    this.#inBuf = '';\n    this.#outBuf = '';\n  }\n\n  /** Mark the input as ended and forbid additional pushes */\n  endInput() {\n    if (this.closed) {\n      throw new Error('Stream is closed');\n    }\n    this.flush();\n    this.close();\n  }\n\n  next(): Promise<IteratorResult<TokenData>> {\n    return this.queue.next();\n  }\n\n  /** Close both the input and output of the token stream */\n  close() {\n    this.queue.close();\n    this.closed = true;\n  }\n\n  [Symbol.asyncIterator](): BufferedTokenStream {\n    return this;\n  }\n}\n\nexport class BufferedSentenceStream extends SentenceStream {\n  #stream: BufferedTokenStream;\n\n  constructor(func: TokenizeFunc, minTokenLength: number, minContextLength: number) {\n    super();\n    this.#stream = new BufferedTokenStream(func, minTokenLength, minContextLength);\n  }\n\n  pushText(text: string) {\n    this.#stream.pushText(text);\n  }\n\n  flush() {\n    this.#stream.flush();\n  }\n\n  close() {\n    super.close();\n    this.#stream.close();\n  }\n\n  next(): Promise<IteratorResult<TokenData>> {\n    return this.#stream.next();\n  }\n}\n\nexport class BufferedWordStream extends WordStream {\n  #stream: BufferedTokenStream;\n\n  constructor(func: TokenizeFunc, minTokenLength: number, minContextLength: number) {\n    super();\n    this.#stream = new BufferedTokenStream(func, minTokenLength, minContextLength);\n  }\n\n  pushText(text: string) {\n    this.#stream.pushText(text);\n  }\n\n  flush() {\n    this.#stream.flush();\n  }\n\n  endInput() {\n    this.#stream.endInput();\n  }\n\n  close() {\n    this.#stream.close();\n  }\n\n  next(): Promise<IteratorResult<TokenData>> {\n    return this.#stream.next();\n  }\n}\n"],"mappings":"AAGA,SAAS,kBAAkB;AAC3B,SAAS,0BAA0B;AAEnC,SAAS,gBAAgB,kBAAkB;AAIpC,MAAM,oBAAgE;AAAA,EACjE,QAAQ,IAAI,mBAA8B;AAAA,EAC1C,SAAS;AAAA,EAEnB;AAAA,EACA;AAAA,EACA;AAAA,EACA,aAAuB,CAAC;AAAA,EACxB,SAAS;AAAA,EACT,UAAU;AAAA,EACV;AAAA,EAEA,YAAY,MAAoB,gBAAwB,kBAA0B;AAChF,SAAK,QAAQ;AACb,SAAK,kBAAkB;AACvB,SAAK,oBAAoB;AAEzB,SAAK,oBAAoB,WAAW;AAAA,EACtC;AAAA;AAAA,EAGA,SAAS,MAAc;AACrB,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AAEA,SAAK,UAAU;AACf,QAAI,KAAK,OAAO,SAAS,KAAK,kBAAmB;AAEjD,WAAO,MAAM;AACX,YAAM,SAAS,KAAK,MAAM,KAAK,MAAM;AACrC,UAAI,OAAO,UAAU,EAAG;AAExB,UAAI,KAAK,QAAS,MAAK,WAAW;AAElC,YAAM,MAAM,OAAO,MAAM;AACzB,UAAI,UAAU;AACd,UAAI,IAAI,SAAS,KAAK,OAAO,IAAI,CAAC,MAAM,UAAU;AAChD,kBAAU,IAAI,CAAC;AAAA,MACjB;AAEA,WAAK,WAAW;AAChB,UAAI,KAAK,QAAQ,UAAU,KAAK,iBAAiB;AAC/C,aAAK,MAAM,IAAI,EAAE,OAAO,KAAK,SAAS,WAAW,KAAK,kBAAkB,CAAC;AACzE,aAAK,UAAU;AAAA,MACjB;AAEA,UAAI,OAAO,QAAS,UAAU;AAC5B,aAAK,SAAS,KAAK,OAAO,MAAM,IAAK,CAAC,CAAC;AAAA,MACzC,OAAO;AACL,aAAK,SAAS,KAAK,OAChB,MAAM,KAAK,IAAI,GAAG,KAAK,OAAO,QAAQ,GAAG,CAAC,IAAI,IAAI,MAAM,EACxD,UAAU;AAAA,MACf;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAGA,QAAQ;AACN,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AAEA,QAAI,KAAK,UAAU,KAAK,SAAS;AAC/B,YAAM,SAAS,KAAK,MAAM,KAAK,MAAM;AACrC,UAAI,QAAQ;AACV,YAAI,KAAK,QAAS,MAAK,WAAW;AAElC,YAAI,OAAO,OAAO,CAAC,MAAM,UAAU;AACjC,eAAK,WAAW,OAAO,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC,EAAE,KAAK,GAAG;AAAA,QACtD,OAAO;AACL,eAAK,WAAW,OAAO,KAAK,GAAG;AAAA,QACjC;AAAA,MACF;AAEA,UAAI,KAAK,SAAS;AAChB,aAAK,MAAM,IAAI,EAAE,OAAO,KAAK,SAAS,WAAW,KAAK,kBAAkB,CAAC;AAAA,MAC3E;AAEA,WAAK,oBAAoB,WAAW;AAAA,IACtC;AAEA,SAAK,SAAS;AACd,SAAK,UAAU;AAAA,EACjB;AAAA;AAAA,EAGA,WAAW;AACT,QAAI,KAAK,QAAQ;AACf,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM;AACX,SAAK,MAAM;AAAA,EACb;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,MAAM,KAAK;AAAA,EACzB;AAAA;AAAA,EAGA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAyB;AAC5C,WAAO;AAAA,EACT;AACF;AAEO,MAAM,+BAA+B,eAAe;AAAA,EACzD;AAAA,EAEA,YAAY,MAAoB,gBAAwB,kBAA0B;AAChF,UAAM;AACN,SAAK,UAAU,IAAI,oBAAoB,MAAM,gBAAgB,gBAAgB;AAAA,EAC/E;AAAA,EAEA,SAAS,MAAc;AACrB,SAAK,QAAQ,SAAS,IAAI;AAAA,EAC5B;AAAA,EAEA,QAAQ;AACN,SAAK,QAAQ,MAAM;AAAA,EACrB;AAAA,EAEA,QAAQ;AACN,UAAM,MAAM;AACZ,SAAK,QAAQ,MAAM;AAAA,EACrB;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,QAAQ,KAAK;AAAA,EAC3B;AACF;AAEO,MAAM,2BAA2B,WAAW;AAAA,EACjD;AAAA,EAEA,YAAY,MAAoB,gBAAwB,kBAA0B;AAChF,UAAM;AACN,SAAK,UAAU,IAAI,oBAAoB,MAAM,gBAAgB,gBAAgB;AAAA,EAC/E;AAAA,EAEA,SAAS,MAAc;AACrB,SAAK,QAAQ,SAAS,IAAI;AAAA,EAC5B;AAAA,EAEA,QAAQ;AACN,SAAK,QAAQ,MAAM;AAAA,EACrB;AAAA,EAEA,WAAW;AACT,SAAK,QAAQ,SAAS;AAAA,EACxB;AAAA,EAEA,QAAQ;AACN,SAAK,QAAQ,MAAM;AAAA,EACrB;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,QAAQ,KAAK;AAAA,EAC3B;AACF;","names":[]}

package/dist/tokenize/tokenizer.cjs

@@ -0,0 +1,184 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var tokenizer_exports = {};
+__export(tokenizer_exports, {
+  PUNCTUATIONS: () => PUNCTUATIONS,
+  SentenceStream: () => SentenceStream,
+  SentenceTokenizer: () => SentenceTokenizer,
+  WordStream: () => WordStream,
+  WordTokenizer: () => WordTokenizer
+});
+module.exports = __toCommonJS(tokenizer_exports);
+var import_utils = require("../utils.cjs");
+const PUNCTUATIONS = [
+  "!",
+  '"',
+  "#",
+  "$",
+  "%",
+  "&",
+  "'",
+  "(",
+  ")",
+  "*",
+  "+",
+  ",",
+  "-",
+  ".",
+  "/",
+  ":",
+  ";",
+  "<",
+  "=",
+  ">",
+  "?",
+  "@",
+  "[",
+  "\\",
+  "]",
+  "^",
+  "_",
+  "`",
+  "{",
+  "|",
+  "}",
+  "~",
+  "\xB1",
+  "\u2014",
+  "\u2018",
+  "\u2019",
+  "\u201C",
+  "\u201D",
+  "\u2026"
+];
+class SentenceTokenizer {
+}
+class SentenceStream {
+  static FLUSH_SENTINEL = Symbol("FLUSH_SENTINEL");
+  input = new import_utils.AsyncIterableQueue();
+  queue = new import_utils.AsyncIterableQueue();
+  #closed = false;
+  get closed() {
+    return this.#closed;
+  }
+  /** Push a string of text to the tokenizer */
+  pushText(text) {
+    if (this.input.closed) {
+      throw new Error("Input is closed");
+    }
+    if (this.#closed) {
+      throw new Error("Stream is closed");
+    }
+    this.input.put(text);
+  }
+  /** Flush the tokenizer, causing it to process all pending text */
+  flush() {
+    if (this.input.closed) {
+      throw new Error("Input is closed");
+    }
+    if (this.#closed) {
+      throw new Error("Stream is closed");
+    }
+    this.input.put(SentenceStream.FLUSH_SENTINEL);
+  }
+  /** Mark the input as ended and forbid additional pushes */
+  endInput() {
+    if (this.input.closed) {
+      throw new Error("Input is closed");
+    }
+    if (this.#closed) {
+      throw new Error("Stream is closed");
+    }
+    this.input.close();
+  }
+  next() {
+    return this.queue.next();
+  }
+  /** Close both the input and output of the tokenizer stream */
+  close() {
+    this.input.close();
+    this.queue.close();
+    this.#closed = true;
+  }
+  [Symbol.asyncIterator]() {
+    return this;
+  }
+}
+class WordTokenizer {
+}
+class WordStream {
+  static FLUSH_SENTINEL = Symbol("FLUSH_SENTINEL");
+  input = new import_utils.AsyncIterableQueue();
+  queue = new import_utils.AsyncIterableQueue();
+  #closed = false;
+  get closed() {
+    return this.#closed;
+  }
+  /** Push a string of text to the tokenizer */
+  pushText(text) {
+    if (this.input.closed) {
+      throw new Error("Input is closed");
+    }
+    if (this.#closed) {
+      throw new Error("Stream is closed");
+    }
+    this.input.put(text);
+  }
+  /** Flush the tokenizer, causing it to process all pending text */
+  flush() {
+    if (this.input.closed) {
+      throw new Error("Input is closed");
+    }
+    if (this.#closed) {
+      throw new Error("Stream is closed");
+    }
+    this.input.put(WordStream.FLUSH_SENTINEL);
+  }
+  /** Mark the input as ended and forbid additional pushes */
+  endInput() {
+    if (this.input.closed) {
+      throw new Error("Input is closed");
+    }
+    if (this.#closed) {
+      throw new Error("Stream is closed");
+    }
+    this.input.close();
+  }
+  next() {
+    return this.queue.next();
+  }
+  /** Close both the input and output of the tokenizer stream */
+  close() {
+    this.input.close();
+    this.queue.close();
+    this.#closed = true;
+  }
+  [Symbol.asyncIterator]() {
+    return this;
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  PUNCTUATIONS,
+  SentenceStream,
+  SentenceTokenizer,
+  WordStream,
+  WordTokenizer
+});
+//# sourceMappingURL=tokenizer.cjs.map

package/dist/tokenize/tokenizer.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/tokenize/tokenizer.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { AsyncIterableQueue } from '../utils.js';\n\n// prettier-ignore\nexport const PUNCTUATIONS = [\n  '!', '\"', '#', '$', '%', '&', \"'\", '(', ')', '*', '+', ',', '-', '.', '/', ':', ';', '<', '=',\n  '>', '?', '@', '[', '\\\\', ']', '^', '_', '`', '{', '|', '}', '~', '±', '—', '‘', '’', '“', '”',\n  '…',\n]\n\nexport interface TokenData {\n  segmentId: string;\n  token: string;\n}\n\nexport abstract class SentenceTokenizer {\n  abstract tokenize(text: string, language?: string): string[];\n\n  /**\n   * Returns a {@link SentenceStream} that can be used to push strings and receive smaller segments.\n   */\n  abstract stream(): SentenceStream;\n}\n\nexport abstract class SentenceStream {\n  protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');\n  protected input = new AsyncIterableQueue<string | typeof SentenceStream.FLUSH_SENTINEL>();\n  protected queue = new AsyncIterableQueue<TokenData>();\n  #closed = false;\n\n  get closed(): boolean {\n    return this.#closed;\n  }\n\n  /** Push a string of text to the tokenizer */\n  pushText(text: string) {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.#closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(text);\n  }\n\n  /** Flush the tokenizer, causing it to process all pending text */\n  flush() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.#closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(SentenceStream.FLUSH_SENTINEL);\n  }\n\n  /** Mark the input as ended and forbid additional pushes */\n  endInput() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.#closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.close();\n  }\n\n  next(): Promise<IteratorResult<TokenData>> {\n    return this.queue.next();\n  }\n\n  /** Close both the input and output of the tokenizer stream */\n  close() {\n    this.input.close();\n    this.queue.close();\n    this.#closed = true;\n  }\n\n  [Symbol.asyncIterator](): SentenceStream {\n    return this;\n  }\n}\n\nexport abstract class WordTokenizer {\n  abstract tokenize(text: string, language?: string): string[];\n\n  /**\n   * Returns a {@link WordStream} that can be used to push words and receive smaller segments.\n   */\n  abstract stream(): WordStream;\n}\n\nexport abstract class WordStream {\n  protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');\n  protected input = new AsyncIterableQueue<string | typeof WordStream.FLUSH_SENTINEL>();\n  protected queue = new AsyncIterableQueue<TokenData>();\n  #closed = false;\n\n  get closed(): boolean {\n    return this.#closed;\n  }\n\n  /** Push a string of text to the tokenizer */\n  pushText(text: string) {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.#closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(text);\n  }\n\n  /** Flush the tokenizer, causing it to process all pending text */\n  flush() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.#closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(WordStream.FLUSH_SENTINEL);\n  }\n\n  /** Mark the input as ended and forbid additional pushes */\n  endInput() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.#closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.close();\n  }\n\n  next(): Promise<IteratorResult<TokenData>> {\n    return this.queue.next();\n  }\n\n  /** Close both the input and output of the tokenizer stream */\n  close() {\n    this.input.close();\n    this.queue.close();\n    this.#closed = true;\n  }\n\n  [Symbol.asyncIterator](): WordStream {\n    return this;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,mBAAmC;AAG5B,MAAM,eAAe;AAAA,EAC1B;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAC1F;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAM;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAC3F;AACF;AAOO,MAAe,kBAAkB;AAOxC;AAEO,MAAe,eAAe;AAAA,EACnC,OAA0B,iBAAiB,OAAO,gBAAgB;AAAA,EACxD,QAAQ,IAAI,gCAAkE;AAAA,EAC9E,QAAQ,IAAI,gCAA8B;AAAA,EACpD,UAAU;AAAA,EAEV,IAAI,SAAkB;AACpB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,SAAS,MAAc;AACrB,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,SAAS;AAChB,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,IAAI;AAAA,EACrB;AAAA;AAAA,EAGA,QAAQ;AACN,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,SAAS;AAChB,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,eAAe,cAAc;AAAA,EAC9C;AAAA;AAAA,EAGA,WAAW;AACT,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,SAAS;AAChB,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,MAAM;AAAA,EACnB;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,MAAM,KAAK;AAAA,EACzB;AAAA;AAAA,EAGA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,MAAM,MAAM;AACjB,SAAK,UAAU;AAAA,EACjB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAoB;AACvC,WAAO;AAAA,EACT;AACF;AAEO,MAAe,cAAc;AAOpC;AAEO,MAAe,WAAW;AAAA,EAC/B,OAA0B,iBAAiB,OAAO,gBAAgB;AAAA,EACxD,QAAQ,IAAI,gCAA8D;AAAA,EAC1E,QAAQ,IAAI,gCAA8B;AAAA,EACpD,UAAU;AAAA,EAEV,IAAI,SAAkB;AACpB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,SAAS,MAAc;AACrB,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,SAAS;AAChB,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,IAAI;AAAA,EACrB;AAAA;AAAA,EAGA,QAAQ;AACN,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,SAAS;AAChB,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,WAAW,cAAc;AAAA,EAC1C;AAAA;AAAA,EAGA,WAAW;AACT,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,SAAS;AAChB,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,MAAM;AAAA,EACnB;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,MAAM,KAAK;AAAA,EACzB;AAAA;AAAA,EAGA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,MAAM,MAAM;AACjB,SAAK,UAAU;AAAA,EACjB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAgB;AACnC,WAAO;AAAA,EACT;AACF;","names":[]}