@livekit/agents 0.4.6 → 0.5.0

package/dist/tokenize/basic/basic.cjs

@@ -0,0 +1,98 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var basic_exports = {};
+__export(basic_exports, {
+  SentenceTokenizer: () => SentenceTokenizer,
+  WordTokenizer: () => WordTokenizer,
+  hyphenateWord: () => hyphenateWord,
+  tokenizeParagraphs: () => tokenizeParagraphs
+});
+module.exports = __toCommonJS(basic_exports);
+var tokenizer = __toESM(require("../index.cjs"), 1);
+var import_token_stream = require("../token_stream.cjs");
+var import_hyphenator = require("./hyphenator.cjs");
+var import_paragraph = require("./paragraph.cjs");
+var import_sentence = require("./sentence.cjs");
+var import_word = require("./word.cjs");
+class SentenceTokenizer extends tokenizer.SentenceTokenizer {
+  #config;
+  constructor(language = "en-US", minSentenceLength = 20, streamContextLength = 10) {
+    super();
+    this.#config = {
+      language,
+      minSentenceLength,
+      streamContextLength
+    };
+  }
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  tokenize(text, language) {
+    return (0, import_sentence.splitSentences)(text, this.#config.minSentenceLength).map((tok) => tok[0]);
+  }
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  stream(language) {
+    return new import_token_stream.BufferedSentenceStream(
+      (text) => (0, import_sentence.splitSentences)(text, this.#config.minSentenceLength),
+      this.#config.minSentenceLength,
+      this.#config.streamContextLength
+    );
+  }
+}
+class WordTokenizer extends tokenizer.WordTokenizer {
+  #ignorePunctuation;
+  constructor(ignorePunctuation = true) {
+    super();
+    this.#ignorePunctuation = ignorePunctuation;
+  }
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  tokenize(text, language) {
+    return (0, import_word.splitWords)(text, this.#ignorePunctuation).map((tok) => tok[0]);
+  }
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  stream(language) {
+    return new import_token_stream.BufferedWordStream(
+      (text) => (0, import_word.splitWords)(text, this.#ignorePunctuation),
+      1,
+      1
+    );
+  }
+}
+const hyphenateWord = (word) => {
+  return import_hyphenator.hyphenator.hyphenateWord(word);
+};
+const tokenizeParagraphs = (text) => {
+  return (0, import_paragraph.splitParagraphs)(text).map((tok) => tok[0]);
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  SentenceTokenizer,
+  WordTokenizer,
+  hyphenateWord,
+  tokenizeParagraphs
+});
+//# sourceMappingURL=basic.cjs.map

package/dist/tokenize/basic/basic.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/tokenize/basic/basic.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport * as tokenizer from '../index.js';\nimport { BufferedSentenceStream, BufferedWordStream } from '../token_stream.js';\nimport { hyphenator } from './hyphenator.js';\nimport { splitParagraphs } from './paragraph.js';\nimport { splitSentences } from './sentence.js';\nimport { splitWords } from './word.js';\n\ninterface TokenizerOptions {\n  language: string;\n  minSentenceLength: number;\n  streamContextLength: number;\n}\n\nexport class SentenceTokenizer extends tokenizer.SentenceTokenizer {\n  #config: TokenizerOptions;\n\n  constructor(language = 'en-US', minSentenceLength = 20, streamContextLength = 10) {\n    super();\n    this.#config = {\n      language,\n      minSentenceLength,\n      streamContextLength,\n    };\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  tokenize(text: string, language?: string): string[] {\n    return splitSentences(text, this.#config.minSentenceLength).map((tok) => tok[0]);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  stream(language?: string): tokenizer.SentenceStream {\n    return new BufferedSentenceStream(\n      (text: string) => splitSentences(text, this.#config.minSentenceLength),\n      this.#config.minSentenceLength,\n      this.#config.streamContextLength,\n    );\n  }\n}\n\nexport class WordTokenizer extends tokenizer.WordTokenizer {\n  #ignorePunctuation: boolean;\n\n  constructor(ignorePunctuation = true) {\n    super();\n    this.#ignorePunctuation = ignorePunctuation;\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  tokenize(text: string, language?: string): string[] {\n    return splitWords(text, this.#ignorePunctuation).map((tok) => tok[0]);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  stream(language?: string): tokenizer.WordStream {\n    return new BufferedWordStream(\n      (text: string) => splitWords(text, this.#ignorePunctuation),\n      1,\n      1,\n    );\n  }\n}\n\nexport const hyphenateWord = (word: string): string[] => {\n  return hyphenator.hyphenateWord(word);\n};\n\nexport const tokenizeParagraphs = (text: string): string[] => {\n  return splitParagraphs(text).map((tok) => tok[0]);\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,gBAA2B;AAC3B,0BAA2D;AAC3D,wBAA2B;AAC3B,uBAAgC;AAChC,sBAA+B;AAC/B,kBAA2B;AAQpB,MAAM,0BAA0B,UAAU,kBAAkB;AAAA,EACjE;AAAA,EAEA,YAAY,WAAW,SAAS,oBAAoB,IAAI,sBAAsB,IAAI;AAChF,UAAM;AACN,SAAK,UAAU;AAAA,MACb;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAGA,SAAS,MAAc,UAA6B;AAClD,eAAO,gCAAe,MAAM,KAAK,QAAQ,iBAAiB,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;AAAA,EACjF;AAAA;AAAA,EAGA,OAAO,UAA6C;AAClD,WAAO,IAAI;AAAA,MACT,CAAC,aAAiB,gCAAe,MAAM,KAAK,QAAQ,iBAAiB;AAAA,MACrE,KAAK,QAAQ;AAAA,MACb,KAAK,QAAQ;AAAA,IACf;AAAA,EACF;AACF;AAEO,MAAM,sBAAsB,UAAU,cAAc;AAAA,EACzD;AAAA,EAEA,YAAY,oBAAoB,MAAM;AACpC,UAAM;AACN,SAAK,qBAAqB;AAAA,EAC5B;AAAA;AAAA,EAGA,SAAS,MAAc,UAA6B;AAClD,eAAO,wBAAW,MAAM,KAAK,kBAAkB,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;AAAA,EACtE;AAAA;AAAA,EAGA,OAAO,UAAyC;AAC9C,WAAO,IAAI;AAAA,MACT,CAAC,aAAiB,wBAAW,MAAM,KAAK,kBAAkB;AAAA,MAC1D;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;AAEO,MAAM,gBAAgB,CAAC,SAA2B;AACvD,SAAO,6BAAW,cAAc,IAAI;AACtC;AAEO,MAAM,qBAAqB,CAAC,SAA2B;AAC5D,aAAO,kCAAgB,IAAI,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;AAClD;","names":[]}

package/dist/tokenize/basic/basic.js

@@ -1,50 +1,61 @@
-// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
-//
-// SPDX-License-Identifier: Apache-2.0
-import * as tokenizer from '../index.js';
-import { BufferedSentenceStream, BufferedWordStream } from '../token_stream.js';
-import { hyphenator } from './hyphenator.js';
-import { splitParagraphs } from './paragraph.js';
-import { splitSentences } from './sentence.js';
-import { splitWords } from './word.js';
-export class SentenceTokenizer extends tokenizer.SentenceTokenizer {
-    #config;
-    constructor(language = 'en-US', minSentenceLength = 20, streamContextLength = 10) {
-        super();
-        this.#config = {
-            language,
-            minSentenceLength,
-            streamContextLength,
-        };
-    }
-    // eslint-disable-next-line @typescript-eslint/no-unused-vars
-    tokenize(text, language) {
-        return splitSentences(text, this.#config.minSentenceLength).map((tok) => tok[0]);
-    }
-    // eslint-disable-next-line @typescript-eslint/no-unused-vars
-    stream(language) {
-        return new BufferedSentenceStream((text) => splitSentences(text, this.#config.minSentenceLength), this.#config.minSentenceLength, this.#config.streamContextLength);
-    }
+import * as tokenizer from "../index.js";
+import { BufferedSentenceStream, BufferedWordStream } from "../token_stream.js";
+import { hyphenator } from "./hyphenator.js";
+import { splitParagraphs } from "./paragraph.js";
+import { splitSentences } from "./sentence.js";
+import { splitWords } from "./word.js";
+class SentenceTokenizer extends tokenizer.SentenceTokenizer {
+  #config;
+  constructor(language = "en-US", minSentenceLength = 20, streamContextLength = 10) {
+    super();
+    this.#config = {
+      language,
+      minSentenceLength,
+      streamContextLength
+    };
+  }
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  tokenize(text, language) {
+    return splitSentences(text, this.#config.minSentenceLength).map((tok) => tok[0]);
+  }
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  stream(language) {
+    return new BufferedSentenceStream(
+      (text) => splitSentences(text, this.#config.minSentenceLength),
+      this.#config.minSentenceLength,
+      this.#config.streamContextLength
+    );
+  }
 }
-export class WordTokenizer extends tokenizer.WordTokenizer {
-    #ignorePunctuation;
-    constructor(ignorePunctuation = true) {
-        super();
-        this.#ignorePunctuation = ignorePunctuation;
-    }
-    // eslint-disable-next-line @typescript-eslint/no-unused-vars
-    tokenize(text, language) {
-        return splitWords(text, this.#ignorePunctuation).map((tok) => tok[0]);
-    }
-    // eslint-disable-next-line @typescript-eslint/no-unused-vars
-    stream(language) {
-        return new BufferedWordStream((text) => splitWords(text, this.#ignorePunctuation), 1, 1);
-    }
+class WordTokenizer extends tokenizer.WordTokenizer {
+  #ignorePunctuation;
+  constructor(ignorePunctuation = true) {
+    super();
+    this.#ignorePunctuation = ignorePunctuation;
+  }
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  tokenize(text, language) {
+    return splitWords(text, this.#ignorePunctuation).map((tok) => tok[0]);
+  }
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  stream(language) {
+    return new BufferedWordStream(
+      (text) => splitWords(text, this.#ignorePunctuation),
+      1,
+      1
+    );
+  }
 }
-export const hyphenateWord = (word) => {
-    return hyphenator.hyphenateWord(word);
+const hyphenateWord = (word) => {
+  return hyphenator.hyphenateWord(word);
 };
-export const tokenizeParagraphs = (text) => {
-    return splitParagraphs(text).map((tok) => tok[0]);
+const tokenizeParagraphs = (text) => {
+  return splitParagraphs(text).map((tok) => tok[0]);
+};
+export {
+  SentenceTokenizer,
+  WordTokenizer,
+  hyphenateWord,
+  tokenizeParagraphs
 };
 //# sourceMappingURL=basic.js.map

package/dist/tokenize/basic/basic.js.map

@@ -1 +1 @@
-{"version":3,"file":"basic.js","sourceRoot":"","sources":["../../../src/tokenize/basic/basic.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C,EAAE;AACF,sCAAsC;AACtC,OAAO,KAAK,SAAS,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,sBAAsB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAC/C,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAQvC,MAAM,OAAO,iBAAkB,SAAQ,SAAS,CAAC,iBAAiB;IAChE,OAAO,CAAmB;IAE1B,YAAY,QAAQ,GAAG,OAAO,EAAE,iBAAiB,GAAG,EAAE,EAAE,mBAAmB,GAAG,EAAE;QAC9E,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,OAAO,GAAG;YACb,QAAQ;YACR,iBAAiB;YACjB,mBAAmB;SACpB,CAAC;IACJ,CAAC;IAED,6DAA6D;IAC7D,QAAQ,CAAC,IAAY,EAAE,QAAiB;QACtC,OAAO,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACnF,CAAC;IAED,6DAA6D;IAC7D,MAAM,CAAC,QAAiB;QACtB,OAAO,IAAI,sBAAsB,CAC/B,CAAC,IAAY,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,EACtE,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAC9B,IAAI,CAAC,OAAO,CAAC,mBAAmB,CACjC,CAAC;IACJ,CAAC;CACF;AAED,MAAM,OAAO,aAAc,SAAQ,SAAS,CAAC,aAAa;IACxD,kBAAkB,CAAU;IAE5B,YAAY,iBAAiB,GAAG,IAAI;QAClC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,kBAAkB,GAAG,iBAAiB,CAAC;IAC9C,CAAC;IAED,6DAA6D;IAC7D,QAAQ,CAAC,IAAY,EAAE,QAAiB;QACtC,OAAO,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,kBAAkB,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACxE,CAAC;IAED,6DAA6D;IAC7D,MAAM,CAAC,QAAiB;QACtB,OAAO,IAAI,kBAAkB,CAC3B,CAAC,IAAY,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,kBAAkB,CAAC,EAC3D,CAAC,EACD,CAAC,CACF,CAAC;IACJ,CAAC;CACF;AAED,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,IAAY,EAAY,EAAE;IACtD,OAAO,UAAU,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;AACxC,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,IAAY,EAAY,EAAE;IAC3D,OAAO,eAAe,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;AACpD,CAAC,CAAC"}
+{"version":3,"sources":["../../../src/tokenize/basic/basic.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport * as tokenizer from '../index.js';\nimport { BufferedSentenceStream, BufferedWordStream } from '../token_stream.js';\nimport { hyphenator } from './hyphenator.js';\nimport { splitParagraphs } from './paragraph.js';\nimport { splitSentences } from './sentence.js';\nimport { splitWords } from './word.js';\n\ninterface TokenizerOptions {\n  language: string;\n  minSentenceLength: number;\n  streamContextLength: number;\n}\n\nexport class SentenceTokenizer extends tokenizer.SentenceTokenizer {\n  #config: TokenizerOptions;\n\n  constructor(language = 'en-US', minSentenceLength = 20, streamContextLength = 10) {\n    super();\n    this.#config = {\n      language,\n      minSentenceLength,\n      streamContextLength,\n    };\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  tokenize(text: string, language?: string): string[] {\n    return splitSentences(text, this.#config.minSentenceLength).map((tok) => tok[0]);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  stream(language?: string): tokenizer.SentenceStream {\n    return new BufferedSentenceStream(\n      (text: string) => splitSentences(text, this.#config.minSentenceLength),\n      this.#config.minSentenceLength,\n      this.#config.streamContextLength,\n    );\n  }\n}\n\nexport class WordTokenizer extends tokenizer.WordTokenizer {\n  #ignorePunctuation: boolean;\n\n  constructor(ignorePunctuation = true) {\n    super();\n    this.#ignorePunctuation = ignorePunctuation;\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  tokenize(text: string, language?: string): string[] {\n    return splitWords(text, this.#ignorePunctuation).map((tok) => tok[0]);\n  }\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  stream(language?: string): tokenizer.WordStream {\n    return new BufferedWordStream(\n      (text: string) => splitWords(text, this.#ignorePunctuation),\n      1,\n      1,\n    );\n  }\n}\n\nexport const hyphenateWord = (word: string): string[] => {\n  return hyphenator.hyphenateWord(word);\n};\n\nexport const tokenizeParagraphs = (text: string): string[] => {\n  return splitParagraphs(text).map((tok) => tok[0]);\n};\n"],"mappings":"AAGA,YAAY,eAAe;AAC3B,SAAS,wBAAwB,0BAA0B;AAC3D,SAAS,kBAAkB;AAC3B,SAAS,uBAAuB;AAChC,SAAS,sBAAsB;AAC/B,SAAS,kBAAkB;AAQpB,MAAM,0BAA0B,UAAU,kBAAkB;AAAA,EACjE;AAAA,EAEA,YAAY,WAAW,SAAS,oBAAoB,IAAI,sBAAsB,IAAI;AAChF,UAAM;AACN,SAAK,UAAU;AAAA,MACb;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAGA,SAAS,MAAc,UAA6B;AAClD,WAAO,eAAe,MAAM,KAAK,QAAQ,iBAAiB,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;AAAA,EACjF;AAAA;AAAA,EAGA,OAAO,UAA6C;AAClD,WAAO,IAAI;AAAA,MACT,CAAC,SAAiB,eAAe,MAAM,KAAK,QAAQ,iBAAiB;AAAA,MACrE,KAAK,QAAQ;AAAA,MACb,KAAK,QAAQ;AAAA,IACf;AAAA,EACF;AACF;AAEO,MAAM,sBAAsB,UAAU,cAAc;AAAA,EACzD;AAAA,EAEA,YAAY,oBAAoB,MAAM;AACpC,UAAM;AACN,SAAK,qBAAqB;AAAA,EAC5B;AAAA;AAAA,EAGA,SAAS,MAAc,UAA6B;AAClD,WAAO,WAAW,MAAM,KAAK,kBAAkB,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;AAAA,EACtE;AAAA;AAAA,EAGA,OAAO,UAAyC;AAC9C,WAAO,IAAI;AAAA,MACT,CAAC,SAAiB,WAAW,MAAM,KAAK,kBAAkB;AAAA,MAC1D;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;AAEO,MAAM,gBAAgB,CAAC,SAA2B;AACvD,SAAO,WAAW,cAAc,IAAI;AACtC;AAEO,MAAM,qBAAqB,CAAC,SAA2B;AAC5D,SAAO,gBAAgB,IAAI,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;AAClD;","names":[]}