@livekit/agents 0.0.0-20260120144724

package/dist/tokenize/basic/basic.cjs

@@ -0,0 +1,105 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var basic_exports = {};
+__export(basic_exports, {
+  SentenceTokenizer: () => SentenceTokenizer,
+  WordTokenizer: () => WordTokenizer,
+  hyphenateWord: () => hyphenateWord,
+  splitWords: () => import_word.splitWords,
+  tokenizeParagraphs: () => tokenizeParagraphs
+});
+module.exports = __toCommonJS(basic_exports);
+var import_token_stream = require("../token_stream.cjs");
+var tokenizer = __toESM(require("../tokenizer.cjs"), 1);
+var import_hyphenator = require("./hyphenator.cjs");
+var import_paragraph = require("./paragraph.cjs");
+var import_sentence = require("./sentence.cjs");
+var import_word = require("./word.cjs");
+const defaultTokenizerOptions = {
+  language: "en-US",
+  minSentenceLength: 20,
+  streamContextLength: 10,
+  retainFormat: false
+};
+class SentenceTokenizer extends tokenizer.SentenceTokenizer {
+  #config;
+  constructor(options) {
+    super();
+    this.#config = {
+      ...defaultTokenizerOptions,
+      ...options
+    };
+  }
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  tokenize(text, language) {
+    return (0, import_sentence.splitSentences)(text, this.#config.minSentenceLength).map((tok) => tok[0]);
+  }
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  stream(language) {
+    return new import_token_stream.BufferedSentenceStream(
+      (text) => (0, import_sentence.splitSentences)(text, this.#config.minSentenceLength, this.#config.retainFormat),
+      this.#config.minSentenceLength,
+      this.#config.streamContextLength
+    );
+  }
+}
+class WordTokenizer extends tokenizer.WordTokenizer {
+  #ignorePunctuation;
+  constructor(ignorePunctuation = true) {
+    super();
+    this.#ignorePunctuation = ignorePunctuation;
+  }
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  tokenize(text, language) {
+    return (0, import_word.splitWords)(text, this.#ignorePunctuation).map((tok) => tok[0]);
+  }
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  stream(language) {
+    return new import_token_stream.BufferedWordStream(
+      (text) => (0, import_word.splitWords)(text, this.#ignorePunctuation),
+      1,
+      1
+    );
+  }
+}
+const hyphenateWord = (word) => {
+  return import_hyphenator.hyphenator.hyphenateWord(word);
+};
+const tokenizeParagraphs = (text) => {
+  return (0, import_paragraph.splitParagraphs)(text).map((tok) => tok[0]);
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  SentenceTokenizer,
+  WordTokenizer,
+  hyphenateWord,
+  splitWords,
+  tokenizeParagraphs
+});
+//# sourceMappingURL=basic.cjs.map

package/dist/tokenize/basic/basic.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/tokenize/basic/basic.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { BufferedSentenceStream, BufferedWordStream } from '../token_stream.js';\nimport * as tokenizer from '../tokenizer.js';\nimport { hyphenator } from './hyphenator.js';\nimport { splitParagraphs } from './paragraph.js';\nimport { splitSentences } from './sentence.js';\nimport { splitWords } from './word.js';\n\ninterface TokenizerOptions {\n  language: string;\n  minSentenceLength: number;\n  streamContextLength: number;\n  retainFormat: boolean;\n}\n\nconst defaultTokenizerOptions: TokenizerOptions = {\n  language: 'en-US',\n  minSentenceLength: 20,\n  streamContextLength: 10,\n  retainFormat: false,\n};\n\nexport class SentenceTokenizer extends tokenizer.SentenceTokenizer {\n  #config: TokenizerOptions;\n\n  constructor(options?: Partial<TokenizerOptions>) {\n    super();\n    this.#config = {\n      ...defaultTokenizerOptions,\n      ...options,\n    };\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  tokenize(text: string, language?: string): string[] {\n    return splitSentences(text, this.#config.minSentenceLength).map((tok) => tok[0]);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  stream(language?: string): tokenizer.SentenceStream {\n    return new BufferedSentenceStream(\n      (text: string) =>\n        splitSentences(text, this.#config.minSentenceLength, this.#config.retainFormat),\n      this.#config.minSentenceLength,\n      this.#config.streamContextLength,\n    );\n  }\n}\n\nexport class WordTokenizer extends tokenizer.WordTokenizer {\n  #ignorePunctuation: boolean;\n\n  constructor(ignorePunctuation = true) {\n    super();\n    this.#ignorePunctuation = ignorePunctuation;\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  tokenize(text: string, language?: string): string[] {\n    return splitWords(text, this.#ignorePunctuation).map((tok) => tok[0]);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  stream(language?: string): tokenizer.WordStream {\n    return new BufferedWordStream(\n      (text: string) => splitWords(text, this.#ignorePunctuation),\n      1,\n      1,\n    );\n  }\n}\n\nexport const hyphenateWord = (word: string): string[] => {\n  return hyphenator.hyphenateWord(word);\n};\n\nexport { splitWords };\n\nexport const tokenizeParagraphs = (text: string): string[] => {\n  return splitParagraphs(text).map((tok) => tok[0]);\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,0BAA2D;AAC3D,gBAA2B;AAC3B,wBAA2B;AAC3B,uBAAgC;AAChC,sBAA+B;AAC/B,kBAA2B;AAS3B,MAAM,0BAA4C;AAAA,EAChD,UAAU;AAAA,EACV,mBAAmB;AAAA,EACnB,qBAAqB;AAAA,EACrB,cAAc;AAChB;AAEO,MAAM,0BAA0B,UAAU,kBAAkB;AAAA,EACjE;AAAA,EAEA,YAAY,SAAqC;AAC/C,UAAM;AACN,SAAK,UAAU;AAAA,MACb,GAAG;AAAA,MACH,GAAG;AAAA,IACL;AAAA,EACF;AAAA;AAAA,EAGA,SAAS,MAAc,UAA6B;AAClD,eAAO,gCAAe,MAAM,KAAK,QAAQ,iBAAiB,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;AAAA,EACjF;AAAA;AAAA,EAGA,OAAO,UAA6C;AAClD,WAAO,IAAI;AAAA,MACT,CAAC,aACC,gCAAe,MAAM,KAAK,QAAQ,mBAAmB,KAAK,QAAQ,YAAY;AAAA,MAChF,KAAK,QAAQ;AAAA,MACb,KAAK,QAAQ;AAAA,IACf;AAAA,EACF;AACF;AAEO,MAAM,sBAAsB,UAAU,cAAc;AAAA,EACzD;AAAA,EAEA,YAAY,oBAAoB,MAAM;AACpC,UAAM;AACN,SAAK,qBAAqB;AAAA,EAC5B;AAAA;AAAA,EAGA,SAAS,MAAc,UAA6B;AAClD,eAAO,wBAAW,MAAM,KAAK,kBAAkB,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;AAAA,EACtE;AAAA;AAAA,EAGA,OAAO,UAAyC;AAC9C,WAAO,IAAI;AAAA,MACT,CAAC,aAAiB,wBAAW,MAAM,KAAK,kBAAkB;AAAA,MAC1D;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;AAEO,MAAM,gBAAgB,CAAC,SAA2B;AACvD,SAAO,6BAAW,cAAc,IAAI;AACtC;AAIO,MAAM,qBAAqB,CAAC,SAA2B;AAC5D,aAAO,kCAAgB,IAAI,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;AAClD;","names":[]}

package/dist/tokenize/basic/basic.d.cts

@@ -0,0 +1,24 @@
+import * as tokenizer from '../tokenizer.js';
+import { splitWords } from './word.js';
+interface TokenizerOptions {
+    language: string;
+    minSentenceLength: number;
+    streamContextLength: number;
+    retainFormat: boolean;
+}
+export declare class SentenceTokenizer extends tokenizer.SentenceTokenizer {
+    #private;
+    constructor(options?: Partial<TokenizerOptions>);
+    tokenize(text: string, language?: string): string[];
+    stream(language?: string): tokenizer.SentenceStream;
+}
+export declare class WordTokenizer extends tokenizer.WordTokenizer {
+    #private;
+    constructor(ignorePunctuation?: boolean);
+    tokenize(text: string, language?: string): string[];
+    stream(language?: string): tokenizer.WordStream;
+}
+export declare const hyphenateWord: (word: string) => string[];
+export { splitWords };
+export declare const tokenizeParagraphs: (text: string) => string[];
+//# sourceMappingURL=basic.d.ts.map

package/dist/tokenize/basic/basic.d.ts

@@ -0,0 +1,24 @@
+import * as tokenizer from '../tokenizer.js';
+import { splitWords } from './word.js';
+interface TokenizerOptions {
+    language: string;
+    minSentenceLength: number;
+    streamContextLength: number;
+    retainFormat: boolean;
+}
+export declare class SentenceTokenizer extends tokenizer.SentenceTokenizer {
+    #private;
+    constructor(options?: Partial<TokenizerOptions>);
+    tokenize(text: string, language?: string): string[];
+    stream(language?: string): tokenizer.SentenceStream;
+}
+export declare class WordTokenizer extends tokenizer.WordTokenizer {
+    #private;
+    constructor(ignorePunctuation?: boolean);
+    tokenize(text: string, language?: string): string[];
+    stream(language?: string): tokenizer.WordStream;
+}
+export declare const hyphenateWord: (word: string) => string[];
+export { splitWords };
+export declare const tokenizeParagraphs: (text: string) => string[];
+//# sourceMappingURL=basic.d.ts.map

package/dist/tokenize/basic/basic.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"basic.d.ts","sourceRoot":"","sources":["../../../src/tokenize/basic/basic.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,SAAS,MAAM,iBAAiB,CAAC;AAI7C,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAEvC,UAAU,gBAAgB;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,YAAY,EAAE,OAAO,CAAC;CACvB;AASD,qBAAa,iBAAkB,SAAQ,SAAS,CAAC,iBAAiB;;gBAGpD,OAAO,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC;IAS/C,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE;IAKnD,MAAM,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC,cAAc;CAQpD;AAED,qBAAa,aAAc,SAAQ,SAAS,CAAC,aAAa;;gBAG5C,iBAAiB,UAAO;IAMpC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE;IAKnD,MAAM,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC,UAAU;CAOhD;AAED,eAAO,MAAM,aAAa,SAAU,MAAM,KAAG,MAAM,EAElD,CAAC;AAEF,OAAO,EAAE,UAAU,EAAE,CAAC;AAEtB,eAAO,MAAM,kBAAkB,SAAU,MAAM,KAAG,MAAM,EAEvD,CAAC"}

package/dist/tokenize/basic/basic.js

@@ -0,0 +1,67 @@
+import { BufferedSentenceStream, BufferedWordStream } from "../token_stream.js";
+import * as tokenizer from "../tokenizer.js";
+import { hyphenator } from "./hyphenator.js";
+import { splitParagraphs } from "./paragraph.js";
+import { splitSentences } from "./sentence.js";
+import { splitWords } from "./word.js";
+const defaultTokenizerOptions = {
+  language: "en-US",
+  minSentenceLength: 20,
+  streamContextLength: 10,
+  retainFormat: false
+};
+class SentenceTokenizer extends tokenizer.SentenceTokenizer {
+  #config;
+  constructor(options) {
+    super();
+    this.#config = {
+      ...defaultTokenizerOptions,
+      ...options
+    };
+  }
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  tokenize(text, language) {
+    return splitSentences(text, this.#config.minSentenceLength).map((tok) => tok[0]);
+  }
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  stream(language) {
+    return new BufferedSentenceStream(
+      (text) => splitSentences(text, this.#config.minSentenceLength, this.#config.retainFormat),
+      this.#config.minSentenceLength,
+      this.#config.streamContextLength
+    );
+  }
+}
+class WordTokenizer extends tokenizer.WordTokenizer {
+  #ignorePunctuation;
+  constructor(ignorePunctuation = true) {
+    super();
+    this.#ignorePunctuation = ignorePunctuation;
+  }
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  tokenize(text, language) {
+    return splitWords(text, this.#ignorePunctuation).map((tok) => tok[0]);
+  }
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  stream(language) {
+    return new BufferedWordStream(
+      (text) => splitWords(text, this.#ignorePunctuation),
+      1,
+      1
+    );
+  }
+}
+const hyphenateWord = (word) => {
+  return hyphenator.hyphenateWord(word);
+};
+const tokenizeParagraphs = (text) => {
+  return splitParagraphs(text).map((tok) => tok[0]);
+};
+export {
+  SentenceTokenizer,
+  WordTokenizer,
+  hyphenateWord,
+  splitWords,
+  tokenizeParagraphs
+};
+//# sourceMappingURL=basic.js.map

package/dist/tokenize/basic/basic.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/tokenize/basic/basic.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { BufferedSentenceStream, BufferedWordStream } from '../token_stream.js';\nimport * as tokenizer from '../tokenizer.js';\nimport { hyphenator } from './hyphenator.js';\nimport { splitParagraphs } from './paragraph.js';\nimport { splitSentences } from './sentence.js';\nimport { splitWords } from './word.js';\n\ninterface TokenizerOptions {\n  language: string;\n  minSentenceLength: number;\n  streamContextLength: number;\n  retainFormat: boolean;\n}\n\nconst defaultTokenizerOptions: TokenizerOptions = {\n  language: 'en-US',\n  minSentenceLength: 20,\n  streamContextLength: 10,\n  retainFormat: false,\n};\n\nexport class SentenceTokenizer extends tokenizer.SentenceTokenizer {\n  #config: TokenizerOptions;\n\n  constructor(options?: Partial<TokenizerOptions>) {\n    super();\n    this.#config = {\n      ...defaultTokenizerOptions,\n      ...options,\n    };\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  tokenize(text: string, language?: string): string[] {\n    return splitSentences(text, this.#config.minSentenceLength).map((tok) => tok[0]);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  stream(language?: string): tokenizer.SentenceStream {\n    return new BufferedSentenceStream(\n      (text: string) =>\n        splitSentences(text, this.#config.minSentenceLength, this.#config.retainFormat),\n      this.#config.minSentenceLength,\n      this.#config.streamContextLength,\n    );\n  }\n}\n\nexport class WordTokenizer extends tokenizer.WordTokenizer {\n  #ignorePunctuation: boolean;\n\n  constructor(ignorePunctuation = true) {\n    super();\n    this.#ignorePunctuation = ignorePunctuation;\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  tokenize(text: string, language?: string): string[] {\n    return splitWords(text, this.#ignorePunctuation).map((tok) => tok[0]);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  stream(language?: string): tokenizer.WordStream {\n    return new BufferedWordStream(\n      (text: string) => splitWords(text, this.#ignorePunctuation),\n      1,\n      1,\n    );\n  }\n}\n\nexport const hyphenateWord = (word: string): string[] => {\n  return hyphenator.hyphenateWord(word);\n};\n\nexport { splitWords };\n\nexport const tokenizeParagraphs = (text: string): string[] => {\n  return splitParagraphs(text).map((tok) => tok[0]);\n};\n"],"mappings":"AAGA,SAAS,wBAAwB,0BAA0B;AAC3D,YAAY,eAAe;AAC3B,SAAS,kBAAkB;AAC3B,SAAS,uBAAuB;AAChC,SAAS,sBAAsB;AAC/B,SAAS,kBAAkB;AAS3B,MAAM,0BAA4C;AAAA,EAChD,UAAU;AAAA,EACV,mBAAmB;AAAA,EACnB,qBAAqB;AAAA,EACrB,cAAc;AAChB;AAEO,MAAM,0BAA0B,UAAU,kBAAkB;AAAA,EACjE;AAAA,EAEA,YAAY,SAAqC;AAC/C,UAAM;AACN,SAAK,UAAU;AAAA,MACb,GAAG;AAAA,MACH,GAAG;AAAA,IACL;AAAA,EACF;AAAA;AAAA,EAGA,SAAS,MAAc,UAA6B;AAClD,WAAO,eAAe,MAAM,KAAK,QAAQ,iBAAiB,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;AAAA,EACjF;AAAA;AAAA,EAGA,OAAO,UAA6C;AAClD,WAAO,IAAI;AAAA,MACT,CAAC,SACC,eAAe,MAAM,KAAK,QAAQ,mBAAmB,KAAK,QAAQ,YAAY;AAAA,MAChF,KAAK,QAAQ;AAAA,MACb,KAAK,QAAQ;AAAA,IACf;AAAA,EACF;AACF;AAEO,MAAM,sBAAsB,UAAU,cAAc;AAAA,EACzD;AAAA,EAEA,YAAY,oBAAoB,MAAM;AACpC,UAAM;AACN,SAAK,qBAAqB;AAAA,EAC5B;AAAA;AAAA,EAGA,SAAS,MAAc,UAA6B;AAClD,WAAO,WAAW,MAAM,KAAK,kBAAkB,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;AAAA,EACtE;AAAA;AAAA,EAGA,OAAO,UAAyC;AAC9C,WAAO,IAAI;AAAA,MACT,CAAC,SAAiB,WAAW,MAAM,KAAK,kBAAkB;AAAA,MAC1D;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;AAEO,MAAM,gBAAgB,CAAC,SAA2B;AACvD,SAAO,WAAW,cAAc,IAAI;AACtC;AAIO,MAAM,qBAAqB,CAAC,SAA2B;AAC5D,SAAO,gBAAgB,IAAI,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;AAClD;","names":[]}