@livekit/agents 0.4.6 → 0.5.1

package/dist/tokenize/tokenizer.js

@@ -1,117 +1,156 @@
-// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
-//
-// SPDX-License-Identifier: Apache-2.0
-import { AsyncIterableQueue } from '../utils.js';
-// prettier-ignore
-export const PUNCTUATIONS = [
-    '!', '"', '#', '$', '%', '&', "'", '(', ')', '*', '+', ',', '-', '.', '/', ':', ';', '<', '=',
-    '>', '?', '@', '[', '\\', ']', '^', '_', '`', '{', '|', '}', '~', '±', '—', '‘', '’', '“', '”',
-    '…',
+import { AsyncIterableQueue } from "../utils.js";
+const PUNCTUATIONS = [
+  "!",
+  '"',
+  "#",
+  "$",
+  "%",
+  "&",
+  "'",
+  "(",
+  ")",
+  "*",
+  "+",
+  ",",
+  "-",
+  ".",
+  "/",
+  ":",
+  ";",
+  "<",
+  "=",
+  ">",
+  "?",
+  "@",
+  "[",
+  "\\",
+  "]",
+  "^",
+  "_",
+  "`",
+  "{",
+  "|",
+  "}",
+  "~",
+  "\xB1",
+  "\u2014",
+  "\u2018",
+  "\u2019",
+  "\u201C",
+  "\u201D",
+  "\u2026"
 ];
-export class SentenceTokenizer {
+class SentenceTokenizer {
 }
-export class SentenceStream {
-    static FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');
-    input = new AsyncIterableQueue();
-    queue = new AsyncIterableQueue();
-    #closed = false;
-    get closed() {
-        return this.#closed;
+class SentenceStream {
+  static FLUSH_SENTINEL = Symbol("FLUSH_SENTINEL");
+  input = new AsyncIterableQueue();
+  queue = new AsyncIterableQueue();
+  #closed = false;
+  get closed() {
+    return this.#closed;
+  }
+  /** Push a string of text to the tokenizer */
+  pushText(text) {
+    if (this.input.closed) {
+      throw new Error("Input is closed");
     }
-    /** Push a string of text to the tokenizer */
-    pushText(text) {
-        if (this.input.closed) {
-            throw new Error('Input is closed');
-        }
-        if (this.#closed) {
-            throw new Error('Stream is closed');
-        }
-        this.input.put(text);
+    if (this.#closed) {
+      throw new Error("Stream is closed");
     }
-    /** Flush the tokenizer, causing it to process all pending text */
-    flush() {
-        if (this.input.closed) {
-            throw new Error('Input is closed');
-        }
-        if (this.#closed) {
-            throw new Error('Stream is closed');
-        }
-        this.input.put(SentenceStream.FLUSH_SENTINEL);
+    this.input.put(text);
+  }
+  /** Flush the tokenizer, causing it to process all pending text */
+  flush() {
+    if (this.input.closed) {
+      throw new Error("Input is closed");
     }
-    /** Mark the input as ended and forbid additional pushes */
-    endInput() {
-        if (this.input.closed) {
-            throw new Error('Input is closed');
-        }
-        if (this.#closed) {
-            throw new Error('Stream is closed');
-        }
-        this.input.close();
+    if (this.#closed) {
+      throw new Error("Stream is closed");
     }
-    next() {
-        return this.queue.next();
+    this.input.put(SentenceStream.FLUSH_SENTINEL);
+  }
+  /** Mark the input as ended and forbid additional pushes */
+  endInput() {
+    if (this.input.closed) {
+      throw new Error("Input is closed");
     }
-    /** Close both the input and output of the tokenizer stream */
-    close() {
-        this.input.close();
-        this.queue.close();
-        this.#closed = true;
-    }
-    [Symbol.asyncIterator]() {
-        return this;
+    if (this.#closed) {
+      throw new Error("Stream is closed");
     }
+    this.input.close();
+  }
+  next() {
+    return this.queue.next();
+  }
+  /** Close both the input and output of the tokenizer stream */
+  close() {
+    this.input.close();
+    this.queue.close();
+    this.#closed = true;
+  }
+  [Symbol.asyncIterator]() {
+    return this;
+  }
 }
-export class WordTokenizer {
+class WordTokenizer {
 }
-export class WordStream {
-    static FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');
-    input = new AsyncIterableQueue();
-    queue = new AsyncIterableQueue();
-    #closed = false;
-    get closed() {
-        return this.#closed;
-    }
-    /** Push a string of text to the tokenizer */
-    pushText(text) {
-        if (this.input.closed) {
-            throw new Error('Input is closed');
-        }
-        if (this.#closed) {
-            throw new Error('Stream is closed');
-        }
-        this.input.put(text);
+class WordStream {
+  static FLUSH_SENTINEL = Symbol("FLUSH_SENTINEL");
+  input = new AsyncIterableQueue();
+  queue = new AsyncIterableQueue();
+  #closed = false;
+  get closed() {
+    return this.#closed;
+  }
+  /** Push a string of text to the tokenizer */
+  pushText(text) {
+    if (this.input.closed) {
+      throw new Error("Input is closed");
     }
-    /** Flush the tokenizer, causing it to process all pending text */
-    flush() {
-        if (this.input.closed) {
-            throw new Error('Input is closed');
-        }
-        if (this.#closed) {
-            throw new Error('Stream is closed');
-        }
-        this.input.put(WordStream.FLUSH_SENTINEL);
+    if (this.#closed) {
+      throw new Error("Stream is closed");
     }
-    /** Mark the input as ended and forbid additional pushes */
-    endInput() {
-        if (this.input.closed) {
-            throw new Error('Input is closed');
-        }
-        if (this.#closed) {
-            throw new Error('Stream is closed');
-        }
-        this.input.close();
+    this.input.put(text);
+  }
+  /** Flush the tokenizer, causing it to process all pending text */
+  flush() {
+    if (this.input.closed) {
+      throw new Error("Input is closed");
     }
-    next() {
-        return this.queue.next();
+    if (this.#closed) {
+      throw new Error("Stream is closed");
     }
-    /** Close both the input and output of the tokenizer stream */
-    close() {
-        this.input.close();
-        this.queue.close();
-        this.#closed = true;
+    this.input.put(WordStream.FLUSH_SENTINEL);
+  }
+  /** Mark the input as ended and forbid additional pushes */
+  endInput() {
+    if (this.input.closed) {
+      throw new Error("Input is closed");
     }
-    [Symbol.asyncIterator]() {
-        return this;
+    if (this.#closed) {
+      throw new Error("Stream is closed");
     }
+    this.input.close();
+  }
+  next() {
+    return this.queue.next();
+  }
+  /** Close both the input and output of the tokenizer stream */
+  close() {
+    this.input.close();
+    this.queue.close();
+    this.#closed = true;
+  }
+  [Symbol.asyncIterator]() {
+    return this;
+  }
 }
+export {
+  PUNCTUATIONS,
+  SentenceStream,
+  SentenceTokenizer,
+  WordStream,
+  WordTokenizer
+};
 //# sourceMappingURL=tokenizer.js.map

package/dist/tokenize/tokenizer.js.map

@@ -1 +1 @@
-{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../../src/tokenize/tokenizer.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C,EAAE;AACF,sCAAsC;AACtC,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAEjD,kBAAkB;AAClB,MAAM,CAAC,MAAM,YAAY,GAAG;IAC1B,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG;IAC7F,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG;IAC9F,GAAG;CACJ,CAAA;AAOD,MAAM,OAAgB,iBAAiB;CAOtC;AAED,MAAM,OAAgB,cAAc;IACxB,MAAM,CAAU,cAAc,GAAG,MAAM,CAAC,gBAAgB,CAAC,CAAC;IAC1D,KAAK,GAAG,IAAI,kBAAkB,EAAiD,CAAC;IAChF,KAAK,GAAG,IAAI,kBAAkB,EAAa,CAAC;IACtD,OAAO,GAAG,KAAK,CAAC;IAEhB,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,6CAA6C;IAC7C,QAAQ,CAAC,IAAY;QACnB,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;QACrC,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACtC,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACvB,CAAC;IAED,kEAAkE;IAClE,KAAK;QACH,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;QACrC,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACtC,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,cAAc,CAAC,cAAc,CAAC,CAAC;IAChD,CAAC;IAED,2DAA2D;IAC3D,QAAQ;QACN,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;QACrC,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACtC,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;IAED,IAAI;QACF,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;IAC3B,CAAC;IAED,8DAA8D;IAC9D,KAAK;QACH,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;IACtB,CAAC;IAED,CAAC,MAAM,CAAC,aAAa,CAAC;QACpB,OAAO,IAAI,CAAC;IACd,CAAC;;AAGH,MAAM,OAAgB,aAAa;CAOlC;AAED,MAAM,OAAgB,UAAU;IACpB,MAAM,CAAU,cAAc,GAAG,MAAM,CAAC,gBAAgB,CAAC,CAAC;IAC1D,KAAK,GAAG,IAAI,kBAAkB,EAA6C,CAAC;IAC5E,KAAK,GAAG,IAAI,kBAAkB,EAAa,CAAC;IACtD,OAAO,GAAG,KAAK,CAAC;IAEhB,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,6CAA6C;IAC7C,QAAQ,CAAC,IAAY;QACnB,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;QACrC,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACtC,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACvB,CAAC;IAED,kEAAkE;IAClE,KAAK;QACH,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;QACrC,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACtC,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;IAC5C,CAAC;IAED,2DAA2D;IAC3D,QAAQ;QACN,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC;QACrC,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACtC,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;IAED,IAAI;QACF,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;IAC3B,CAAC;IAED,8DAA8D;IAC9D,KAAK;QACH,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;IACtB,CAAC;IAED,CAAC,MAAM,CAAC,aAAa,CAAC;QACpB,OAAO,IAAI,CAAC;IACd,CAAC"}
+{"version":3,"sources":["../../src/tokenize/tokenizer.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { AsyncIterableQueue } from '../utils.js';\n\n// prettier-ignore\nexport const PUNCTUATIONS = [\n  '!', '\"', '#', '$', '%', '&', \"'\", '(', ')', '*', '+', ',', '-', '.', '/', ':', ';', '<', '=',\n  '>', '?', '@', '[', '\\\\', ']', '^', '_', '`', '{', '|', '}', '~', '±', '—', '‘', '’', '“', '”',\n  '…',\n]\n\nexport interface TokenData {\n  segmentId: string;\n  token: string;\n}\n\nexport abstract class SentenceTokenizer {\n  abstract tokenize(text: string, language?: string): string[];\n\n  /**\n   * Returns a {@link SentenceStream} that can be used to push strings and receive smaller segments.\n   */\n  abstract stream(): SentenceStream;\n}\n\nexport abstract class SentenceStream {\n  protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');\n  protected input = new AsyncIterableQueue<string | typeof SentenceStream.FLUSH_SENTINEL>();\n  protected queue = new AsyncIterableQueue<TokenData>();\n  #closed = false;\n\n  get closed(): boolean {\n    return this.#closed;\n  }\n\n  /** Push a string of text to the tokenizer */\n  pushText(text: string) {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.#closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(text);\n  }\n\n  /** Flush the tokenizer, causing it to process all pending text */\n  flush() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.#closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(SentenceStream.FLUSH_SENTINEL);\n  }\n\n  /** Mark the input as ended and forbid additional pushes */\n  endInput() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.#closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.close();\n  }\n\n  next(): Promise<IteratorResult<TokenData>> {\n    return this.queue.next();\n  }\n\n  /** Close both the input and output of the tokenizer stream */\n  close() {\n    this.input.close();\n    this.queue.close();\n    this.#closed = true;\n  }\n\n  [Symbol.asyncIterator](): SentenceStream {\n    return this;\n  }\n}\n\nexport abstract class WordTokenizer {\n  abstract tokenize(text: string, language?: string): string[];\n\n  /**\n   * Returns a {@link WordStream} that can be used to push words and receive smaller segments.\n   */\n  abstract stream(): WordStream;\n}\n\nexport abstract class WordStream {\n  protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');\n  protected input = new AsyncIterableQueue<string | typeof WordStream.FLUSH_SENTINEL>();\n  protected queue = new AsyncIterableQueue<TokenData>();\n  #closed = false;\n\n  get closed(): boolean {\n    return this.#closed;\n  }\n\n  /** Push a string of text to the tokenizer */\n  pushText(text: string) {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.#closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(text);\n  }\n\n  /** Flush the tokenizer, causing it to process all pending text */\n  flush() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.#closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.put(WordStream.FLUSH_SENTINEL);\n  }\n\n  /** Mark the input as ended and forbid additional pushes */\n  endInput() {\n    if (this.input.closed) {\n      throw new Error('Input is closed');\n    }\n    if (this.#closed) {\n      throw new Error('Stream is closed');\n    }\n    this.input.close();\n  }\n\n  next(): Promise<IteratorResult<TokenData>> {\n    return this.queue.next();\n  }\n\n  /** Close both the input and output of the tokenizer stream */\n  close() {\n    this.input.close();\n    this.queue.close();\n    this.#closed = true;\n  }\n\n  [Symbol.asyncIterator](): WordStream {\n    return this;\n  }\n}\n"],"mappings":"AAGA,SAAS,0BAA0B;AAG5B,MAAM,eAAe;AAAA,EAC1B;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAC1F;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAM;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAC3F;AACF;AAOO,MAAe,kBAAkB;AAOxC;AAEO,MAAe,eAAe;AAAA,EACnC,OAA0B,iBAAiB,OAAO,gBAAgB;AAAA,EACxD,QAAQ,IAAI,mBAAkE;AAAA,EAC9E,QAAQ,IAAI,mBAA8B;AAAA,EACpD,UAAU;AAAA,EAEV,IAAI,SAAkB;AACpB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,SAAS,MAAc;AACrB,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,SAAS;AAChB,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,IAAI;AAAA,EACrB;AAAA;AAAA,EAGA,QAAQ;AACN,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,SAAS;AAChB,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,eAAe,cAAc;AAAA,EAC9C;AAAA;AAAA,EAGA,WAAW;AACT,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,SAAS;AAChB,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,MAAM;AAAA,EACnB;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,MAAM,KAAK;AAAA,EACzB;AAAA;AAAA,EAGA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,MAAM,MAAM;AACjB,SAAK,UAAU;AAAA,EACjB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAoB;AACvC,WAAO;AAAA,EACT;AACF;AAEO,MAAe,cAAc;AAOpC;AAEO,MAAe,WAAW;AAAA,EAC/B,OAA0B,iBAAiB,OAAO,gBAAgB;AAAA,EACxD,QAAQ,IAAI,mBAA8D;AAAA,EAC1E,QAAQ,IAAI,mBAA8B;AAAA,EACpD,UAAU;AAAA,EAEV,IAAI,SAAkB;AACpB,WAAO,KAAK;AAAA,EACd;AAAA;AAAA,EAGA,SAAS,MAAc;AACrB,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,SAAS;AAChB,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,IAAI;AAAA,EACrB;AAAA;AAAA,EAGA,QAAQ;AACN,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,SAAS;AAChB,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,IAAI,WAAW,cAAc;AAAA,EAC1C;AAAA;AAAA,EAGA,WAAW;AACT,QAAI,KAAK,MAAM,QAAQ;AACrB,YAAM,IAAI,MAAM,iBAAiB;AAAA,IACnC;AACA,QAAI,KAAK,SAAS;AAChB,YAAM,IAAI,MAAM,kBAAkB;AAAA,IACpC;AACA,SAAK,MAAM,MAAM;AAAA,EACnB;AAAA,EAEA,OAA2C;AACzC,WAAO,KAAK,MAAM,KAAK;AAAA,EACzB;AAAA;AAAA,EAGA,QAAQ;AACN,SAAK,MAAM,MAAM;AACjB,SAAK,MAAM,MAAM;AACjB,SAAK,UAAU;AAAA,EACjB;AAAA,EAEA,CAAC,OAAO,aAAa,IAAgB;AACnC,WAAO;AAAA,EACT;AACF;","names":[]}

package/dist/tokenize/tokenizer.test.cjs

@@ -0,0 +1,220 @@
+"use strict";
+var import_vitest = require("vitest");
+var import_basic = require("./basic/index.cjs");
+var import_paragraph = require("./basic/paragraph.cjs");
+const TEXT = "Hi! LiveKit is a platform for live audio and video applications and services. R.T.C stands for Real-Time Communication... again R.T.C. Mr. Theo is testing the sentence tokenizer. This is a test. Another test. A short sentence. A longer sentence that is longer than the previous sentence. f(x) = x * 2.54 + 42. Hey! Hi! Hello! ";
+const EXPECTED_MIN_20 = [
+  "Hi! LiveKit is a platform for live audio and video applications and services.",
+  "R.T.C stands for Real-Time Communication... again R.T.C.",
+  "Mr. Theo is testing the sentence tokenizer.",
+  "This is a test. Another test.",
+  "A short sentence. A longer sentence that is longer than the previous sentence.",
+  "f(x) = x * 2.54 + 42.",
+  "Hey! Hi! Hello!"
+];
+const WORDS_TEXT = "This is a test. Blabla another test! multiple consecutive spaces:     done";
+const WORDS_EXPECTED = [
+  "This",
+  "is",
+  "a",
+  "test",
+  "Blabla",
+  "another",
+  "test",
+  "multiple",
+  "consecutive",
+  "spaces",
+  "done"
+];
+const WORDS_PUNCT_TEXT = 'This is <phoneme alphabet="cmu-arpabet" ph="AE K CH UW AH L IY">actually</phoneme> tricky to handle.';
+const WORDS_PUNCT_EXPECTED = [
+  "This",
+  "is",
+  "<phoneme",
+  'alphabet="cmu-arpabet"',
+  'ph="AE',
+  "K",
+  "CH",
+  "UW",
+  "AH",
+  "L",
+  'IY">actually</phoneme>',
+  "tricky",
+  "to",
+  "handle."
+];
+const HYPHENATOR_TEXT = ["Segment", "expected", "communication", "window", "welcome", "bedroom"];
+const HYPHENATOR_EXPECTED = [
+  ["Seg", "ment"],
+  ["ex", "pect", "ed"],
+  ["com", "mu", "ni", "ca", "tion"],
+  ["win", "dow"],
+  ["wel", "come"],
+  ["bed", "room"]
+];
+const PARAGRAPH_TEST_CASES = [
+  ["Single paragraph.", [["Single paragraph.", 0, 17]]],
+  [
+    "Paragraph 1.\n\nParagraph 2.",
+    [
+      ["Paragraph 1.", 0, 12],
+      ["Paragraph 2.", 14, 26]
+    ]
+  ],
+  [
+    "Para 1.\n\nPara 2.\n\nPara 3.",
+    [
+      ["Para 1.", 0, 7],
+      ["Para 2.", 9, 16],
+      ["Para 3.", 18, 25]
+    ]
+  ],
+  ["\n\nParagraph with leading newlines.", [["Paragraph with leading newlines.", 2, 34]]],
+  ["Paragraph with trailing newlines.\n\n", [["Paragraph with trailing newlines.", 0, 33]]],
+  [
+    "\n\n  Paragraph with leading and trailing spaces.  \n\n",
+    [["Paragraph with leading and trailing spaces.", 4, 47]]
+  ],
+  [
+    "Para 1.\n\n\n\nPara 2.",
+    // Multiple newlines between paragraphs
+    [
+      ["Para 1.", 0, 7],
+      ["Para 2.", 11, 18]
+    ]
+  ],
+  [
+    "Para 1.\n \n \nPara 2.",
+    // Newlines with spaces between paragraphs
+    [
+      ["Para 1.", 0, 7],
+      ["Para 2.", 12, 19]
+    ]
+  ],
+  [
+    "",
+    // Empty string
+    []
+  ],
+  [
+    "\n\n\n",
+    // Only newlines
+    []
+  ],
+  [
+    "Line 1\nLine 2\nLine 3",
+    // Single paragraph with newlines
+    [["Line 1\nLine 2\nLine 3", 0, 20]]
+  ]
+];
+(0, import_vitest.describe)("tokenizer", () => {
+  (0, import_vitest.describe)("SentenceTokenizer", () => {
+    const tokenizer = new import_basic.SentenceTokenizer();
+    (0, import_vitest.it)("should tokenize sentences correctly", () => {
+      (0, import_vitest.expect)(tokenizer.tokenize(TEXT).every((x, i) => EXPECTED_MIN_20[i] === x)).toBeTruthy();
+    });
+    (0, import_vitest.it)("should stream tokenize sentences correctly", async () => {
+      const pattern = [1, 2, 4];
+      let text = TEXT;
+      const chunks = [];
+      const patternIter = Array(Math.ceil(text.length / pattern.reduce((sum, num) => sum + num, 0))).fill(pattern).flat()[Symbol.iterator]();
+      for (const size of patternIter) {
+        if (!text) break;
+        chunks.push(text.slice(void 0, size));
+        text = text.slice(size);
+      }
+      const stream = tokenizer.stream();
+      for (const chunk of chunks) {
+        stream.pushText(chunk);
+      }
+      stream.endInput();
+      stream.close();
+      for (const x of EXPECTED_MIN_20) {
+        await stream.next().then((value) => {
+          if (value.value) {
+            (0, import_vitest.expect)(value.value.token).toStrictEqual(x);
+          }
+        });
+      }
+    });
+  });
+  (0, import_vitest.describe)("WordTokenizer", () => {
+    const tokenizer = new import_basic.WordTokenizer();
+    (0, import_vitest.it)("should tokenize words correctly", () => {
+      (0, import_vitest.expect)(tokenizer.tokenize(WORDS_TEXT).every((x, i) => WORDS_EXPECTED[i] === x)).toBeTruthy();
+    });
+    (0, import_vitest.it)("should stream tokenize words correctly", async () => {
+      const pattern = [1, 2, 4];
+      let text = WORDS_TEXT;
+      const chunks = [];
+      const patternIter = Array(Math.ceil(text.length / pattern.reduce((sum, num) => sum + num, 0))).fill(pattern).flat()[Symbol.iterator]();
+      for (const size of patternIter) {
+        if (!text) break;
+        chunks.push(text.slice(void 0, size));
+        text = text.slice(size);
+      }
+      const stream = tokenizer.stream();
+      for (const chunk of chunks) {
+        stream.pushText(chunk);
+      }
+      stream.endInput();
+      stream.close();
+      for (const x of WORDS_EXPECTED) {
+        await stream.next().then((value) => {
+          if (value.value) {
+            (0, import_vitest.expect)(value.value.token).toStrictEqual(x);
+          }
+        });
+      }
+    });
+    (0, import_vitest.describe)("punctuation handling", () => {
+      const tokenizerPunct = new import_basic.WordTokenizer(false);
+      (0, import_vitest.it)("should tokenize words correctly", () => {
+        (0, import_vitest.expect)(
+          tokenizerPunct.tokenize(WORDS_PUNCT_TEXT).every((x, i) => WORDS_PUNCT_EXPECTED[i] === x)
+        ).toBeTruthy();
+      });
+      (0, import_vitest.it)("should stream tokenize words correctly", async () => {
+        const pattern = [1, 2, 4];
+        let text = WORDS_PUNCT_TEXT;
+        const chunks = [];
+        const patternIter = Array(
+          Math.ceil(text.length / pattern.reduce((sum, num) => sum + num, 0))
+        ).fill(pattern).flat()[Symbol.iterator]();
+        for (const size of patternIter) {
+          if (!text) break;
+          chunks.push(text.slice(void 0, size));
+          text = text.slice(size);
+        }
+        const stream = tokenizerPunct.stream();
+        for (const chunk of chunks) {
+          stream.pushText(chunk);
+        }
+        stream.endInput();
+        stream.close();
+        for (const x of WORDS_PUNCT_EXPECTED) {
+          await stream.next().then((value) => {
+            if (value.value) {
+              (0, import_vitest.expect)(value.value.token).toStrictEqual(x);
+            }
+          });
+        }
+      });
+    });
+  });
+  (0, import_vitest.describe)("hyphenateWord", () => {
+    (0, import_vitest.it)("should hyphenate correctly", () => {
+      HYPHENATOR_TEXT.forEach((x, i) => {
+        (0, import_vitest.expect)((0, import_basic.hyphenateWord)(x)).toStrictEqual(HYPHENATOR_EXPECTED[i]);
+      });
+    });
+  });
+  (0, import_vitest.describe)("splitParagraphs", () => {
+    (0, import_vitest.it)("should tokenize paragraphs correctly", () => {
+      PARAGRAPH_TEST_CASES.forEach(([a, b]) => {
+        (0, import_vitest.expect)((0, import_paragraph.splitParagraphs)(a)).toStrictEqual(b);
+      });
+    });
+  });
+});
+//# sourceMappingURL=tokenizer.test.cjs.map

package/dist/tokenize/tokenizer.test.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/tokenize/tokenizer.test.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { describe, expect, it } from 'vitest';\nimport { SentenceTokenizer, WordTokenizer, hyphenateWord } from './basic/index.js';\nimport { splitParagraphs } from './basic/paragraph.js';\n\nconst TEXT =\n  'Hi! ' +\n  'LiveKit is a platform for live audio and video applications and services. ' +\n  'R.T.C stands for Real-Time Communication... again R.T.C. ' +\n  'Mr. Theo is testing the sentence tokenizer. ' +\n  'This is a test. Another test. ' +\n  'A short sentence. ' +\n  'A longer sentence that is longer than the previous sentence. ' +\n  'f(x) = x * 2.54 + 42. ' +\n  'Hey! Hi! Hello! ';\n\nconst EXPECTED_MIN_20 = [\n  'Hi! LiveKit is a platform for live audio and video applications and services.',\n  'R.T.C stands for Real-Time Communication... again R.T.C.',\n  'Mr. Theo is testing the sentence tokenizer.',\n  'This is a test. Another test.',\n  'A short sentence. A longer sentence that is longer than the previous sentence.',\n  'f(x) = x * 2.54 + 42.',\n  'Hey! Hi! Hello!',\n];\n\nconst WORDS_TEXT = 'This is a test. Blabla another test! multiple consecutive spaces:     done';\nconst WORDS_EXPECTED = [\n  'This',\n  'is',\n  'a',\n  'test',\n  'Blabla',\n  'another',\n  'test',\n  'multiple',\n  'consecutive',\n  'spaces',\n  'done',\n];\n\nconst WORDS_PUNCT_TEXT =\n  'This is <phoneme alphabet=\"cmu-arpabet\" ph=\"AE K CH UW AH L IY\">actually</phoneme> tricky to handle.';\nconst WORDS_PUNCT_EXPECTED = [\n  'This',\n  'is',\n  '<phoneme',\n  'alphabet=\"cmu-arpabet\"',\n  'ph=\"AE',\n  'K',\n  'CH',\n  'UW',\n  'AH',\n  'L',\n  'IY\">actually</phoneme>',\n  'tricky',\n  'to',\n  'handle.',\n];\n\nconst HYPHENATOR_TEXT = ['Segment', 'expected', 'communication', 'window', 'welcome', 'bedroom'];\nconst HYPHENATOR_EXPECTED = [\n  ['Seg', 'ment'],\n  ['ex', 'pect', 'ed'],\n  ['com', 'mu', 'ni', 'ca', 'tion'],\n  ['win', 'dow'],\n  ['wel', 'come'],\n  ['bed', 'room'],\n];\n\nconst PARAGRAPH_TEST_CASES: [string, [string, number, number][]][] = [\n  ['Single paragraph.', [['Single paragraph.', 0, 17]]],\n  [\n    'Paragraph 1.\\n\\nParagraph 2.',\n    [\n      ['Paragraph 1.', 0, 12],\n      ['Paragraph 2.', 14, 26],\n    ],\n  ],\n  [\n    'Para 1.\\n\\nPara 2.\\n\\nPara 3.',\n    [\n      ['Para 1.', 0, 7],\n      ['Para 2.', 9, 16],\n      ['Para 3.', 18, 25],\n    ],\n  ],\n  ['\\n\\nParagraph with leading newlines.', [['Paragraph with leading newlines.', 2, 34]]],\n  ['Paragraph with trailing newlines.\\n\\n', [['Paragraph with trailing newlines.', 0, 33]]],\n  [\n    '\\n\\n  Paragraph with leading and trailing spaces.  \\n\\n',\n    [['Paragraph with leading and trailing spaces.', 4, 47]],\n  ],\n  [\n    'Para 1.\\n\\n\\n\\nPara 2.', // Multiple newlines between paragraphs\n    [\n      ['Para 1.', 0, 7],\n      ['Para 2.', 11, 18],\n    ],\n  ],\n  [\n    'Para 1.\\n \\n \\nPara 2.', // Newlines with spaces between paragraphs\n    [\n      ['Para 1.', 0, 7],\n      ['Para 2.', 12, 19],\n    ],\n  ],\n  [\n    '', // Empty string\n    [],\n  ],\n  [\n    '\\n\\n\\n', // Only newlines\n    [],\n  ],\n  [\n    'Line 1\\nLine 2\\nLine 3', // Single paragraph with newlines\n    [['Line 1\\nLine 2\\nLine 3', 0, 20]],\n  ],\n];\n\ndescribe('tokenizer', () => {\n  describe('SentenceTokenizer', () => {\n    const tokenizer = new SentenceTokenizer();\n\n    it('should tokenize sentences correctly', () => {\n      expect(tokenizer.tokenize(TEXT).every((x, i) => EXPECTED_MIN_20[i] === x)).toBeTruthy();\n    });\n\n    it('should stream tokenize sentences correctly', async () => {\n      const pattern = [1, 2, 4];\n      let text = TEXT;\n      const chunks = [];\n      const patternIter = Array(Math.ceil(text.length / pattern.reduce((sum, num) => sum + num, 0)))\n        .fill(pattern)\n        .flat()\n        [Symbol.iterator]();\n\n      for (const size of patternIter) {\n        if (!text) break;\n        chunks.push(text.slice(undefined, size));\n        text = text.slice(size);\n      }\n      const stream = tokenizer.stream();\n      for (const chunk of chunks) {\n        stream.pushText(chunk);\n      }\n      stream.endInput();\n      stream.close();\n\n      for (const x of EXPECTED_MIN_20) {\n        await stream.next().then((value) => {\n          if (value.value) {\n            expect(value.value.token).toStrictEqual(x);\n          }\n        });\n      }\n    });\n  });\n  describe('WordTokenizer', () => {\n    const tokenizer = new WordTokenizer();\n\n    it('should tokenize words correctly', () => {\n      expect(tokenizer.tokenize(WORDS_TEXT).every((x, i) => WORDS_EXPECTED[i] === x)).toBeTruthy();\n    });\n\n    it('should stream tokenize words correctly', async () => {\n      const pattern = [1, 2, 4];\n      let text = WORDS_TEXT;\n      const chunks = [];\n      const patternIter = Array(Math.ceil(text.length / pattern.reduce((sum, num) => sum + num, 0)))\n        .fill(pattern)\n        .flat()\n        [Symbol.iterator]();\n\n      for (const size of patternIter) {\n        if (!text) break;\n        chunks.push(text.slice(undefined, size));\n        text = text.slice(size);\n      }\n      const stream = tokenizer.stream();\n      for (const chunk of chunks) {\n        stream.pushText(chunk);\n      }\n      stream.endInput();\n      stream.close();\n\n      for (const x of WORDS_EXPECTED) {\n        await stream.next().then((value) => {\n          if (value.value) {\n            expect(value.value.token).toStrictEqual(x);\n          }\n        });\n      }\n    });\n\n    describe('punctuation handling', () => {\n      const tokenizerPunct = new WordTokenizer(false);\n\n      it('should tokenize words correctly', () => {\n        expect(\n          tokenizerPunct.tokenize(WORDS_PUNCT_TEXT).every((x, i) => WORDS_PUNCT_EXPECTED[i] === x),\n        ).toBeTruthy();\n      });\n\n      it('should stream tokenize words correctly', async () => {\n        const pattern = [1, 2, 4];\n        let text = WORDS_PUNCT_TEXT;\n        const chunks = [];\n        const patternIter = Array(\n          Math.ceil(text.length / pattern.reduce((sum, num) => sum + num, 0)),\n        )\n          .fill(pattern)\n          .flat()\n          [Symbol.iterator]();\n\n        for (const size of patternIter) {\n          if (!text) break;\n          chunks.push(text.slice(undefined, size));\n          text = text.slice(size);\n        }\n        const stream = tokenizerPunct.stream();\n        for (const chunk of chunks) {\n          stream.pushText(chunk);\n        }\n        stream.endInput();\n        stream.close();\n\n        for (const x of WORDS_PUNCT_EXPECTED) {\n          await stream.next().then((value) => {\n            if (value.value) {\n              expect(value.value.token).toStrictEqual(x);\n            }\n          });\n        }\n      });\n    });\n  });\n  describe('hyphenateWord', () => {\n    it('should hyphenate correctly', () => {\n      HYPHENATOR_TEXT.forEach((x, i) => {\n        expect(hyphenateWord(x)).toStrictEqual(HYPHENATOR_EXPECTED[i]);\n      });\n    });\n  });\n  describe('splitParagraphs', () => {\n    it('should tokenize paragraphs correctly', () => {\n      PARAGRAPH_TEST_CASES.forEach(([a, b]) => {\n        expect(splitParagraphs(a)).toStrictEqual(b);\n      });\n    });\n  });\n});\n"],"mappings":";AAGA,oBAAqC;AACrC,mBAAgE;AAChE,uBAAgC;AAEhC,MAAM,OACJ;AAUF,MAAM,kBAAkB;AAAA,EACtB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEA,MAAM,aAAa;AACnB,MAAM,iBAAiB;AAAA,EACrB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEA,MAAM,mBACJ;AACF,MAAM,uBAAuB;AAAA,EAC3B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEA,MAAM,kBAAkB,CAAC,WAAW,YAAY,iBAAiB,UAAU,WAAW,SAAS;AAC/F,MAAM,sBAAsB;AAAA,EAC1B,CAAC,OAAO,MAAM;AAAA,EACd,CAAC,MAAM,QAAQ,IAAI;AAAA,EACnB,CAAC,OAAO,MAAM,MAAM,MAAM,MAAM;AAAA,EAChC,CAAC,OAAO,KAAK;AAAA,EACb,CAAC,OAAO,MAAM;AAAA,EACd,CAAC,OAAO,MAAM;AAChB;AAEA,MAAM,uBAA+D;AAAA,EACnE,CAAC,qBAAqB,CAAC,CAAC,qBAAqB,GAAG,EAAE,CAAC,CAAC;AAAA,EACpD;AAAA,IACE;AAAA,IACA;AAAA,MACE,CAAC,gBAAgB,GAAG,EAAE;AAAA,MACtB,CAAC,gBAAgB,IAAI,EAAE;AAAA,IACzB;AAAA,EACF;AAAA,EACA;AAAA,IACE;AAAA,IACA;AAAA,MACE,CAAC,WAAW,GAAG,CAAC;AAAA,MAChB,CAAC,WAAW,GAAG,EAAE;AAAA,MACjB,CAAC,WAAW,IAAI,EAAE;AAAA,IACpB;AAAA,EACF;AAAA,EACA,CAAC,wCAAwC,CAAC,CAAC,oCAAoC,GAAG,EAAE,CAAC,CAAC;AAAA,EACtF,CAAC,yCAAyC,CAAC,CAAC,qCAAqC,GAAG,EAAE,CAAC,CAAC;AAAA,EACxF;AAAA,IACE;AAAA,IACA,CAAC,CAAC,+CAA+C,GAAG,EAAE,CAAC;AAAA,EACzD;AAAA,EACA;AAAA,IACE;AAAA;AAAA,IACA;AAAA,MACE,CAAC,WAAW,GAAG,CAAC;AAAA,MAChB,CAAC,WAAW,IAAI,EAAE;AAAA,IACpB;AAAA,EACF;AAAA,EACA;AAAA,IACE;AAAA;AAAA,IACA;AAAA,MACE,CAAC,WAAW,GAAG,CAAC;AAAA,MAChB,CAAC,WAAW,IAAI,EAAE;AAAA,IACpB;AAAA,EACF;AAAA,EACA;AAAA,IACE;AAAA;AAAA,IACA,CAAC;AAAA,EACH;AAAA,EACA;AAAA,IACE;AAAA;AAAA,IACA,CAAC;AAAA,EACH;AAAA,EACA;AAAA,IACE;AAAA;AAAA,IACA,CAAC,CAAC,0BAA0B,GAAG,EAAE,CAAC;AAAA,EACpC;AACF;AAAA,IAEA,wBAAS,aAAa,MAAM;AAC1B,8BAAS,qBAAqB,MAAM;AAClC,UAAM,YAAY,IAAI,+BAAkB;AAExC,0BAAG,uCAAuC,MAAM;AAC9C,gCAAO,UAAU,SAAS,IAAI,EAAE,MAAM,CAAC,GAAG,MAAM,gBAAgB,CAAC,MAAM,CAAC,CAAC,EAAE,WAAW;AAAA,IACxF,CAAC;AAED,0BAAG,8CAA8C,YAAY;AAC3D,YAAM,UAAU,CAAC,GAAG,GAAG,CAAC;AACxB,UAAI,OAAO;AACX,YAAM,SAAS,CAAC;AAChB,YAAM,cAAc,MAAM,KAAK,KAAK,KAAK,SAAS,QAAQ,OAAO,CAAC,KAAK,QAAQ,MAAM,KAAK,CAAC,CAAC,CAAC,EAC1F,KAAK,OAAO,EACZ,KAAK,EACL,OAAO,QAAQ,EAAE;AAEpB,iBAAW,QAAQ,aAAa;AAC9B,YAAI,CAAC,KAAM;AACX,eAAO,KAAK,KAAK,MAAM,QAAW,IAAI,CAAC;AACvC,eAAO,KAAK,MAAM,IAAI;AAAA,MACxB;AACA,YAAM,SAAS,UAAU,OAAO;AAChC,iBAAW,SAAS,QAAQ;AAC1B,eAAO,SAAS,KAAK;AAAA,MACvB;AACA,aAAO,SAAS;AAChB,aAAO,MAAM;AAEb,iBAAW,KAAK,iBAAiB;AAC/B,cAAM,OAAO,KAAK,EAAE,KAAK,CAAC,UAAU;AAClC,cAAI,MAAM,OAAO;AACf,sCAAO,MAAM,MAAM,KAAK,EAAE,cAAc,CAAC;AAAA,UAC3C;AAAA,QACF,CAAC;AAAA,MACH;AAAA,IACF,CAAC;AAAA,EACH,CAAC;AACD,8BAAS,iBAAiB,MAAM;AAC9B,UAAM,YAAY,IAAI,2BAAc;AAEpC,0BAAG,mCAAmC,MAAM;AAC1C,gCAAO,UAAU,SAAS,UAAU,EAAE,MAAM,CAAC,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,CAAC,EAAE,WAAW;AAAA,IAC7F,CAAC;AAED,0BAAG,0CAA0C,YAAY;AACvD,YAAM,UAAU,CAAC,GAAG,GAAG,CAAC;AACxB,UAAI,OAAO;AACX,YAAM,SAAS,CAAC;AAChB,YAAM,cAAc,MAAM,KAAK,KAAK,KAAK,SAAS,QAAQ,OAAO,CAAC,KAAK,QAAQ,MAAM,KAAK,CAAC,CAAC,CAAC,EAC1F,KAAK,OAAO,EACZ,KAAK,EACL,OAAO,QAAQ,EAAE;AAEpB,iBAAW,QAAQ,aAAa;AAC9B,YAAI,CAAC,KAAM;AACX,eAAO,KAAK,KAAK,MAAM,QAAW,IAAI,CAAC;AACvC,eAAO,KAAK,MAAM,IAAI;AAAA,MACxB;AACA,YAAM,SAAS,UAAU,OAAO;AAChC,iBAAW,SAAS,QAAQ;AAC1B,eAAO,SAAS,KAAK;AAAA,MACvB;AACA,aAAO,SAAS;AAChB,aAAO,MAAM;AAEb,iBAAW,KAAK,gBAAgB;AAC9B,cAAM,OAAO,KAAK,EAAE,KAAK,CAAC,UAAU;AAClC,cAAI,MAAM,OAAO;AACf,sCAAO,MAAM,MAAM,KAAK,EAAE,cAAc,CAAC;AAAA,UAC3C;AAAA,QACF,CAAC;AAAA,MACH;AAAA,IACF,CAAC;AAED,gCAAS,wBAAwB,MAAM;AACrC,YAAM,iBAAiB,IAAI,2BAAc,KAAK;AAE9C,4BAAG,mCAAmC,MAAM;AAC1C;AAAA,UACE,eAAe,SAAS,gBAAgB,EAAE,MAAM,CAAC,GAAG,MAAM,qBAAqB,CAAC,MAAM,CAAC;AAAA,QACzF,EAAE,WAAW;AAAA,MACf,CAAC;AAED,4BAAG,0CAA0C,YAAY;AACvD,cAAM,UAAU,CAAC,GAAG,GAAG,CAAC;AACxB,YAAI,OAAO;AACX,cAAM,SAAS,CAAC;AAChB,cAAM,cAAc;AAAA,UAClB,KAAK,KAAK,KAAK,SAAS,QAAQ,OAAO,CAAC,KAAK,QAAQ,MAAM,KAAK,CAAC,CAAC;AAAA,QACpE,EACG,KAAK,OAAO,EACZ,KAAK,EACL,OAAO,QAAQ,EAAE;AAEpB,mBAAW,QAAQ,aAAa;AAC9B,cAAI,CAAC,KAAM;AACX,iBAAO,KAAK,KAAK,MAAM,QAAW,IAAI,CAAC;AACvC,iBAAO,KAAK,MAAM,IAAI;AAAA,QACxB;AACA,cAAM,SAAS,eAAe,OAAO;AACrC,mBAAW,SAAS,QAAQ;AAC1B,iBAAO,SAAS,KAAK;AAAA,QACvB;AACA,eAAO,SAAS;AAChB,eAAO,MAAM;AAEb,mBAAW,KAAK,sBAAsB;AACpC,gBAAM,OAAO,KAAK,EAAE,KAAK,CAAC,UAAU;AAClC,gBAAI,MAAM,OAAO;AACf,wCAAO,MAAM,MAAM,KAAK,EAAE,cAAc,CAAC;AAAA,YAC3C;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF,CAAC;AAAA,IACH,CAAC;AAAA,EACH,CAAC;AACD,8BAAS,iBAAiB,MAAM;AAC9B,0BAAG,8BAA8B,MAAM;AACrC,sBAAgB,QAAQ,CAAC,GAAG,MAAM;AAChC,sCAAO,4BAAc,CAAC,CAAC,EAAE,cAAc,oBAAoB,CAAC,CAAC;AAAA,MAC/D,CAAC;AAAA,IACH,CAAC;AAAA,EACH,CAAC;AACD,8BAAS,mBAAmB,MAAM;AAChC,0BAAG,wCAAwC,MAAM;AAC/C,2BAAqB,QAAQ,CAAC,CAAC,GAAG,CAAC,MAAM;AACvC,sCAAO,kCAAgB,CAAC,CAAC,EAAE,cAAc,CAAC;AAAA,MAC5C,CAAC;AAAA,IACH,CAAC;AAAA,EACH,CAAC;AACH,CAAC;","names":[]}

package/dist/tokenize/tokenizer.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=tokenizer.test.d.ts.map

package/dist/tokenize/tokenizer.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer.test.d.ts","sourceRoot":"","sources":["../../src/tokenize/tokenizer.test.ts"],"names":[],"mappings":""}