@livekit/agents-plugin-livekit 0.1.0

package/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

package/README.md

@@ -0,0 +1,17 @@
+<!--
+SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+
+SPDX-License-Identifier: Apache-2.0
+-->
+# Additional utilities for LiveKit Agents
+
+The Agents Framework is designed for building realtime, programmable
+participants that run on servers. Use it to create conversational, multi-modal
+voice agents that can see, hear, and understand.
+
+This package contains additional utilities provided by LiveKit.
+Refer to the [documentation](https://docs.livekit.io/agents/overview/) for
+information on how to use it, or browse the [API
+reference](https://docs.livekit.io/agents-js/modules/plugins_agents_plugin_livekit.html).
+See the [repository](https://github.com/livekit/agents-js) for more information
+about the framework as a whole.

package/dist/index.cjs

@@ -0,0 +1,45 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var src_exports = {};
+__export(src_exports, {
+  turnDetector: () => turnDetector
+});
+module.exports = __toCommonJS(src_exports);
+var import_agents = require("@livekit/agents");
+var turnDetector = __toESM(require("./turn_detector.cjs"), 1);
+const import_meta = {};
+import_agents.InferenceRunner.registerRunner(
+  turnDetector.EOURunner.INFERENCE_METHOD,
+  import_meta.resolve("./turn_detector.js")
+);
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  turnDetector
+});
+//# sourceMappingURL=index.cjs.map

package/dist/index.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { InferenceRunner } from '@livekit/agents';\nimport * as turnDetector from './turn_detector.js';\n\nInferenceRunner.registerRunner(\n  turnDetector.EOURunner.INFERENCE_METHOD,\n  import.meta.resolve('./turn_detector.js'),\n);\nexport { turnDetector };\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,oBAAgC;AAChC,mBAA8B;AAJ9B;AAMA,8BAAgB;AAAA,EACd,aAAa,UAAU;AAAA,EACvB,YAAY,QAAQ,oBAAoB;AAC1C;","names":[]}

package/dist/index.d.ts

@@ -0,0 +1,3 @@
+import * as turnDetector from './turn_detector.js';
+export { turnDetector };
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,YAAY,MAAM,oBAAoB,CAAC;AAMnD,OAAO,EAAE,YAAY,EAAE,CAAC"}

package/dist/index.js

@@ -0,0 +1,10 @@
+import { InferenceRunner } from "@livekit/agents";
+import * as turnDetector from "./turn_detector.js";
+InferenceRunner.registerRunner(
+  turnDetector.EOURunner.INFERENCE_METHOD,
+  import.meta.resolve("./turn_detector.js")
+);
+export {
+  turnDetector
+};
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/index.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport { InferenceRunner } from '@livekit/agents';\nimport * as turnDetector from './turn_detector.js';\n\nInferenceRunner.registerRunner(\n  turnDetector.EOURunner.INFERENCE_METHOD,\n  import.meta.resolve('./turn_detector.js'),\n);\nexport { turnDetector };\n"],"mappings":"AAGA,SAAS,uBAAuB;AAChC,YAAY,kBAAkB;AAE9B,gBAAgB;AAAA,EACd,aAAa,UAAU;AAAA,EACvB,YAAY,QAAQ,oBAAoB;AAC1C;","names":[]}

package/dist/onnxruntime.d.cjs

@@ -0,0 +1,2 @@
+"use strict";
+//# sourceMappingURL=onnxruntime.d.cjs.map

package/dist/onnxruntime.d.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/onnxruntime.d.js

@@ -0,0 +1 @@
+//# sourceMappingURL=onnxruntime.d.js.map

package/dist/onnxruntime.d.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/turn_detector.cjs

@@ -0,0 +1,126 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var turn_detector_exports = {};
+__export(turn_detector_exports, {
+  EOUModel: () => EOUModel,
+  EOURunner: () => EOURunner,
+  default: () => turn_detector_default
+});
+module.exports = __toCommonJS(turn_detector_exports);
+var getImportMetaUrl = () => typeof document === "undefined" ? new URL(`file:${__filename}`).href : document.currentScript && document.currentScript.src || new URL("main.js", document.baseURI).href;
+var importMetaUrl = /* @__PURE__ */ getImportMetaUrl();
+var import_transformers = require("@huggingface/transformers");
+var import_agents = require("@livekit/agents");
+var import_node_url = require("node:url");
+var import_onnxruntime_node = require("onnxruntime-node");
+const MAX_HISTORY = 4;
+class EOURunner extends import_agents.InferenceRunner {
+  static INFERENCE_METHOD = "lk_end_of_utterance";
+  #tokenizerPromise;
+  #session;
+  #tokenizer;
+  #logger = (0, import_agents.log)();
+  constructor() {
+    super();
+    this.#tokenizerPromise = import_transformers.AutoTokenizer.from_pretrained("livekit/turn-detector", {
+      revision: "v1.2.0"
+      // local_files_only: true, // TODO(nbsp): can't find it
+    });
+    this.#session = import_onnxruntime_node.InferenceSession.create(
+      (0, import_node_url.fileURLToPath)(new URL("turn_detector.onnx", importMetaUrl).href),
+      {
+        executionProviders: [{ name: "cpu" }]
+      }
+    );
+  }
+  async initialize() {
+    this.#tokenizer = await this.#tokenizerPromise;
+  }
+  async run(data) {
+    const text = this.#formatChatContext(data);
+    const startTime = Date.now();
+    const inputs = this.#tokenizer.encode(text, { add_special_tokens: false });
+    const outputs = await this.#session.then(
+      (session) => session.run({ input_ids: new import_onnxruntime_node.Tensor("int64", inputs, [1, inputs.length]) }, ["prob"])
+    );
+    const endTime = Date.now();
+    const logits = outputs.prob;
+    const eouProbability = logits.data[0];
+    this.#logger.child({ eouProbability, input: text, duration: endTime - startTime }).debug("eou prediction");
+    return eouProbability;
+  }
+  #formatChatContext(ctx) {
+    const newCtx = [];
+    for (const msg of ctx) {
+      if (!msg.content) continue;
+      newCtx.push(msg);
+    }
+    const convoText = this.#tokenizer.apply_chat_template(newCtx, {
+      add_generation_prompt: false,
+      tokenize: false
+    });
+    return convoText.slice(0, convoText.lastIndexOf("<|im_end|>"));
+  }
+}
+class EOUModel {
+  unlikelyThreshold;
+  #executor;
+  constructor(unlikelyThreshold = 0.15) {
+    this.unlikelyThreshold = unlikelyThreshold;
+    this.#executor = import_agents.CurrentJobContext.getCurrent().inferenceExecutor;
+  }
+  supportsLanguage(language) {
+    if (!language) return false;
+    const parts = language.toLowerCase().split("-");
+    return parts[0] === "en" || parts[0] === "english";
+  }
+  async predictEndOfTurn(chatCtx) {
+    let messages = [];
+    for (const msg of chatCtx.messages) {
+      if (msg.role !== import_agents.llm.ChatRole.ASSISTANT && msg.role !== import_agents.llm.ChatRole.USER) {
+        continue;
+      }
+      if (typeof msg.content === "string") {
+        messages.push({
+          role: msg.role === import_agents.llm.ChatRole.ASSISTANT ? "assistant" : "user",
+          content: msg.content
+        });
+      } else if (Array.isArray(msg.content)) {
+        for (const content of msg.content) {
+          if (typeof content === "string") {
+            messages.push({
+              role: msg.role === import_agents.llm.ChatRole.ASSISTANT ? "assistant" : "user",
+              content
+            });
+          }
+        }
+      }
+    }
+    messages = messages.slice(-MAX_HISTORY);
+    const result = await this.#executor.doInference(EOURunner.INFERENCE_METHOD, messages);
+    return result;
+  }
+}
+var turn_detector_default = EOURunner;
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  EOUModel,
+  EOURunner
+});
+//# sourceMappingURL=turn_detector.cjs.map

package/dist/turn_detector.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/turn_detector.ts","../../../node_modules/.pnpm/tsup@8.3.5_@microsoft+api-extractor@7.43.7_@types+node@22.5.5__postcss@8.4.38_tsx@4.19.2_typescript@5.4.5/node_modules/tsup/assets/cjs_shims.js"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { PreTrainedTokenizer } from '@huggingface/transformers';\nimport { AutoTokenizer } from '@huggingface/transformers';\nimport type { ipc } from '@livekit/agents';\nimport { CurrentJobContext, InferenceRunner, llm, log } from '@livekit/agents';\nimport { fileURLToPath } from 'node:url';\nimport { InferenceSession, Tensor } from 'onnxruntime-node';\n\nconst MAX_HISTORY = 4;\n\ntype RawChatContext = { role: string; content: string }[];\n\nexport class EOURunner extends InferenceRunner {\n  static INFERENCE_METHOD = 'lk_end_of_utterance';\n  #tokenizerPromise: Promise<PreTrainedTokenizer>;\n  #session: Promise<InferenceSession>;\n  #tokenizer?: PreTrainedTokenizer;\n  #logger = log();\n\n  constructor() {\n    super();\n    this.#tokenizerPromise = AutoTokenizer.from_pretrained('livekit/turn-detector', {\n      revision: 'v1.2.0',\n      // local_files_only: true, // TODO(nbsp): can't find it\n    });\n    this.#session = InferenceSession.create(\n      fileURLToPath(new URL('turn_detector.onnx', import.meta.url).href),\n      {\n        executionProviders: [{ name: 'cpu' }],\n      },\n    );\n  }\n\n  async initialize() {\n    this.#tokenizer = await this.#tokenizerPromise;\n  }\n\n  async run(data: RawChatContext): Promise<number | undefined> {\n    const text = this.#formatChatContext(data);\n    const startTime = Date.now();\n    const inputs = this.#tokenizer!.encode(text, { add_special_tokens: false });\n    const outputs = await this.#session.then((session) =>\n      session.run({ input_ids: new Tensor('int64', inputs, [1, inputs.length]) }, ['prob']),\n    );\n    const endTime = Date.now();\n    const logits = outputs.prob!;\n    const eouProbability = logits.data[0] as number;\n    this.#logger\n      .child({ eouProbability, input: text, duration: endTime - startTime })\n      .debug('eou prediction');\n    return eouProbability;\n  }\n\n  #formatChatContext(ctx: RawChatContext): string {\n    const newCtx: RawChatContext = [];\n    for (const msg of ctx) {\n      if (!msg.content) continue;\n      newCtx.push(msg);\n    }\n\n    const convoText = this.#tokenizer!.apply_chat_template(newCtx, {\n      add_generation_prompt: false,\n      tokenize: false,\n    }) as string;\n    // remove EOU token from current utterance\n    return convoText.slice(0, convoText.lastIndexOf('<|im_end|>'));\n  }\n}\n\nexport class EOUModel {\n  readonly unlikelyThreshold: number;\n  #executor: ipc.InferenceExecutor;\n\n  constructor(unlikelyThreshold = 0.15) {\n    this.unlikelyThreshold = unlikelyThreshold;\n    this.#executor = CurrentJobContext.getCurrent().inferenceExecutor;\n  }\n\n  supportsLanguage(language?: string) {\n    if (!language) return false;\n    const parts = language.toLowerCase().split('-');\n    return parts[0] === 'en' || parts[0] === 'english';\n  }\n\n  async predictEndOfTurn(chatCtx: llm.ChatContext): Promise<number> {\n    let messages: RawChatContext = [];\n\n    for (const msg of chatCtx.messages) {\n      if (msg.role !== llm.ChatRole.ASSISTANT && msg.role !== llm.ChatRole.USER) {\n        continue;\n      }\n\n      if (typeof msg.content === 'string') {\n        messages.push({\n          role: msg.role === llm.ChatRole.ASSISTANT ? 'assistant' : 'user',\n          content: msg.content,\n        });\n      } else if (Array.isArray(msg.content)) {\n        for (const content of msg.content) {\n          if (typeof content === 'string') {\n            messages.push({\n              role: msg.role === llm.ChatRole.ASSISTANT ? 'assistant' : 'user',\n              content: content,\n            });\n          }\n        }\n      }\n    }\n    messages = messages.slice(-MAX_HISTORY);\n    const result = await this.#executor.doInference(EOURunner.INFERENCE_METHOD, messages);\n    return result as any;\n  }\n}\n\nexport default EOURunner;\n","// Shim globals in cjs bundle\n// There's a weird bug that esbuild will always inject importMetaUrl\n// if we export it as `const importMetaUrl = ... __filename ...`\n// But using a function will not cause this issue\n\nconst getImportMetaUrl = () =>\n  typeof document === 'undefined'\n    ? new URL(`file:${__filename}`).href\n    : (document.currentScript && document.currentScript.src) ||\n      new URL('main.js', document.baseURI).href\n\nexport const importMetaUrl = /* @__PURE__ */ getImportMetaUrl()\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;ACKA,IAAM,mBAAmB,MACvB,OAAO,aAAa,cAChB,IAAI,IAAI,QAAQ,UAAU,EAAE,EAAE,OAC7B,SAAS,iBAAiB,SAAS,cAAc,OAClD,IAAI,IAAI,WAAW,SAAS,OAAO,EAAE;AAEpC,IAAM,gBAAgC,iCAAiB;ADP9D,0BAA8B;AAE9B,oBAA6D;AAC7D,sBAA8B;AAC9B,8BAAyC;AAEzC,MAAM,cAAc;AAIb,MAAM,kBAAkB,8BAAgB;AAAA,EAC7C,OAAO,mBAAmB;AAAA,EAC1B;AAAA,EACA;AAAA,EACA;AAAA,EACA,cAAU,mBAAI;AAAA,EAEd,cAAc;AACZ,UAAM;AACN,SAAK,oBAAoB,kCAAc,gBAAgB,yBAAyB;AAAA,MAC9E,UAAU;AAAA;AAAA,IAEZ,CAAC;AACD,SAAK,WAAW,yCAAiB;AAAA,UAC/B,+BAAc,IAAI,IAAI,sBAAsB,aAAe,EAAE,IAAI;AAAA,MACjE;AAAA,QACE,oBAAoB,CAAC,EAAE,MAAM,MAAM,CAAC;AAAA,MACtC;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,aAAa;AACjB,SAAK,aAAa,MAAM,KAAK;AAAA,EAC/B;AAAA,EAEA,MAAM,IAAI,MAAmD;AAC3D,UAAM,OAAO,KAAK,mBAAmB,IAAI;AACzC,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,SAAS,KAAK,WAAY,OAAO,MAAM,EAAE,oBAAoB,MAAM,CAAC;AAC1E,UAAM,UAAU,MAAM,KAAK,SAAS;AAAA,MAAK,CAAC,YACxC,QAAQ,IAAI,EAAE,WAAW,IAAI,+BAAO,SAAS,QAAQ,CAAC,GAAG,OAAO,MAAM,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC;AAAA,IACtF;AACA,UAAM,UAAU,KAAK,IAAI;AACzB,UAAM,SAAS,QAAQ;AACvB,UAAM,iBAAiB,OAAO,KAAK,CAAC;AACpC,SAAK,QACF,MAAM,EAAE,gBAAgB,OAAO,MAAM,UAAU,UAAU,UAAU,CAAC,EACpE,MAAM,gBAAgB;AACzB,WAAO;AAAA,EACT;AAAA,EAEA,mBAAmB,KAA6B;AAC9C,UAAM,SAAyB,CAAC;AAChC,eAAW,OAAO,KAAK;AACrB,UAAI,CAAC,IAAI,QAAS;AAClB,aAAO,KAAK,GAAG;AAAA,IACjB;AAEA,UAAM,YAAY,KAAK,WAAY,oBAAoB,QAAQ;AAAA,MAC7D,uBAAuB;AAAA,MACvB,UAAU;AAAA,IACZ,CAAC;AAED,WAAO,UAAU,MAAM,GAAG,UAAU,YAAY,YAAY,CAAC;AAAA,EAC/D;AACF;AAEO,MAAM,SAAS;AAAA,EACX;AAAA,EACT;AAAA,EAEA,YAAY,oBAAoB,MAAM;AACpC,SAAK,oBAAoB;AACzB,SAAK,YAAY,gCAAkB,WAAW,EAAE;AAAA,EAClD;AAAA,EAEA,iBAAiB,UAAmB;AAClC,QAAI,CAAC,SAAU,QAAO;AACtB,UAAM,QAAQ,SAAS,YAAY,EAAE,MAAM,GAAG;AAC9C,WAAO,MAAM,CAAC,MAAM,QAAQ,MAAM,CAAC,MAAM;AAAA,EAC3C;AAAA,EAEA,MAAM,iBAAiB,SAA2C;AAChE,QAAI,WAA2B,CAAC;AAEhC,eAAW,OAAO,QAAQ,UAAU;AAClC,UAAI,IAAI,SAAS,kBAAI,SAAS,aAAa,IAAI,SAAS,kBAAI,SAAS,MAAM;AACzE;AAAA,MACF;AAEA,UAAI,OAAO,IAAI,YAAY,UAAU;AACnC,iBAAS,KAAK;AAAA,UACZ,MAAM,IAAI,SAAS,kBAAI,SAAS,YAAY,cAAc;AAAA,UAC1D,SAAS,IAAI;AAAA,QACf,CAAC;AAAA,MACH,WAAW,MAAM,QAAQ,IAAI,OAAO,GAAG;AACrC,mBAAW,WAAW,IAAI,SAAS;AACjC,cAAI,OAAO,YAAY,UAAU;AAC/B,qBAAS,KAAK;AAAA,cACZ,MAAM,IAAI,SAAS,kBAAI,SAAS,YAAY,cAAc;AAAA,cAC1D;AAAA,YACF,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAAA,IACF;AACA,eAAW,SAAS,MAAM,CAAC,WAAW;AACtC,UAAM,SAAS,MAAM,KAAK,UAAU,YAAY,UAAU,kBAAkB,QAAQ;AACpF,WAAO;AAAA,EACT;AACF;AAEA,IAAO,wBAAQ;","names":[]}

package/dist/turn_detector.d.ts

@@ -0,0 +1,21 @@
+import { InferenceRunner, llm } from '@livekit/agents';
+type RawChatContext = {
+    role: string;
+    content: string;
+}[];
+export declare class EOURunner extends InferenceRunner {
+    #private;
+    static INFERENCE_METHOD: string;
+    constructor();
+    initialize(): Promise<void>;
+    run(data: RawChatContext): Promise<number | undefined>;
+}
+export declare class EOUModel {
+    #private;
+    readonly unlikelyThreshold: number;
+    constructor(unlikelyThreshold?: number);
+    supportsLanguage(language?: string): boolean;
+    predictEndOfTurn(chatCtx: llm.ChatContext): Promise<number>;
+}
+export default EOURunner;
+//# sourceMappingURL=turn_detector.d.ts.map

package/dist/turn_detector.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"turn_detector.d.ts","sourceRoot":"","sources":["../src/turn_detector.ts"],"names":[],"mappings":"AAMA,OAAO,EAAqB,eAAe,EAAE,GAAG,EAAO,MAAM,iBAAiB,CAAC;AAM/E,KAAK,cAAc,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,EAAE,CAAC;AAE1D,qBAAa,SAAU,SAAQ,eAAe;;IAC5C,MAAM,CAAC,gBAAgB,SAAyB;;IAoB1C,UAAU;IAIV,GAAG,CAAC,IAAI,EAAE,cAAc,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;CA8B7D;AAED,qBAAa,QAAQ;;IACnB,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;gBAGvB,iBAAiB,SAAO;IAKpC,gBAAgB,CAAC,QAAQ,CAAC,EAAE,MAAM;IAM5B,gBAAgB,CAAC,OAAO,EAAE,GAAG,CAAC,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;CA4BlE;AAED,eAAe,SAAS,CAAC"}

package/dist/turn_detector.js

@@ -0,0 +1,99 @@
+import { AutoTokenizer } from "@huggingface/transformers";
+import { CurrentJobContext, InferenceRunner, llm, log } from "@livekit/agents";
+import { fileURLToPath } from "node:url";
+import { InferenceSession, Tensor } from "onnxruntime-node";
+const MAX_HISTORY = 4;
+class EOURunner extends InferenceRunner {
+  static INFERENCE_METHOD = "lk_end_of_utterance";
+  #tokenizerPromise;
+  #session;
+  #tokenizer;
+  #logger = log();
+  constructor() {
+    super();
+    this.#tokenizerPromise = AutoTokenizer.from_pretrained("livekit/turn-detector", {
+      revision: "v1.2.0"
+      // local_files_only: true, // TODO(nbsp): can't find it
+    });
+    this.#session = InferenceSession.create(
+      fileURLToPath(new URL("turn_detector.onnx", import.meta.url).href),
+      {
+        executionProviders: [{ name: "cpu" }]
+      }
+    );
+  }
+  async initialize() {
+    this.#tokenizer = await this.#tokenizerPromise;
+  }
+  async run(data) {
+    const text = this.#formatChatContext(data);
+    const startTime = Date.now();
+    const inputs = this.#tokenizer.encode(text, { add_special_tokens: false });
+    const outputs = await this.#session.then(
+      (session) => session.run({ input_ids: new Tensor("int64", inputs, [1, inputs.length]) }, ["prob"])
+    );
+    const endTime = Date.now();
+    const logits = outputs.prob;
+    const eouProbability = logits.data[0];
+    this.#logger.child({ eouProbability, input: text, duration: endTime - startTime }).debug("eou prediction");
+    return eouProbability;
+  }
+  #formatChatContext(ctx) {
+    const newCtx = [];
+    for (const msg of ctx) {
+      if (!msg.content) continue;
+      newCtx.push(msg);
+    }
+    const convoText = this.#tokenizer.apply_chat_template(newCtx, {
+      add_generation_prompt: false,
+      tokenize: false
+    });
+    return convoText.slice(0, convoText.lastIndexOf("<|im_end|>"));
+  }
+}
+class EOUModel {
+  unlikelyThreshold;
+  #executor;
+  constructor(unlikelyThreshold = 0.15) {
+    this.unlikelyThreshold = unlikelyThreshold;
+    this.#executor = CurrentJobContext.getCurrent().inferenceExecutor;
+  }
+  supportsLanguage(language) {
+    if (!language) return false;
+    const parts = language.toLowerCase().split("-");
+    return parts[0] === "en" || parts[0] === "english";
+  }
+  async predictEndOfTurn(chatCtx) {
+    let messages = [];
+    for (const msg of chatCtx.messages) {
+      if (msg.role !== llm.ChatRole.ASSISTANT && msg.role !== llm.ChatRole.USER) {
+        continue;
+      }
+      if (typeof msg.content === "string") {
+        messages.push({
+          role: msg.role === llm.ChatRole.ASSISTANT ? "assistant" : "user",
+          content: msg.content
+        });
+      } else if (Array.isArray(msg.content)) {
+        for (const content of msg.content) {
+          if (typeof content === "string") {
+            messages.push({
+              role: msg.role === llm.ChatRole.ASSISTANT ? "assistant" : "user",
+              content
+            });
+          }
+        }
+      }
+    }
+    messages = messages.slice(-MAX_HISTORY);
+    const result = await this.#executor.doInference(EOURunner.INFERENCE_METHOD, messages);
+    return result;
+  }
+}
+var turn_detector_default = EOURunner;
+export {
+  EOUModel,
+  EOURunner,
+  turn_detector_default as default
+};
+//# sourceMappingURL=turn_detector.js.map

package/dist/turn_detector.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/turn_detector.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { PreTrainedTokenizer } from '@huggingface/transformers';\nimport { AutoTokenizer } from '@huggingface/transformers';\nimport type { ipc } from '@livekit/agents';\nimport { CurrentJobContext, InferenceRunner, llm, log } from '@livekit/agents';\nimport { fileURLToPath } from 'node:url';\nimport { InferenceSession, Tensor } from 'onnxruntime-node';\n\nconst MAX_HISTORY = 4;\n\ntype RawChatContext = { role: string; content: string }[];\n\nexport class EOURunner extends InferenceRunner {\n  static INFERENCE_METHOD = 'lk_end_of_utterance';\n  #tokenizerPromise: Promise<PreTrainedTokenizer>;\n  #session: Promise<InferenceSession>;\n  #tokenizer?: PreTrainedTokenizer;\n  #logger = log();\n\n  constructor() {\n    super();\n    this.#tokenizerPromise = AutoTokenizer.from_pretrained('livekit/turn-detector', {\n      revision: 'v1.2.0',\n      // local_files_only: true, // TODO(nbsp): can't find it\n    });\n    this.#session = InferenceSession.create(\n      fileURLToPath(new URL('turn_detector.onnx', import.meta.url).href),\n      {\n        executionProviders: [{ name: 'cpu' }],\n      },\n    );\n  }\n\n  async initialize() {\n    this.#tokenizer = await this.#tokenizerPromise;\n  }\n\n  async run(data: RawChatContext): Promise<number | undefined> {\n    const text = this.#formatChatContext(data);\n    const startTime = Date.now();\n    const inputs = this.#tokenizer!.encode(text, { add_special_tokens: false });\n    const outputs = await this.#session.then((session) =>\n      session.run({ input_ids: new Tensor('int64', inputs, [1, inputs.length]) }, ['prob']),\n    );\n    const endTime = Date.now();\n    const logits = outputs.prob!;\n    const eouProbability = logits.data[0] as number;\n    this.#logger\n      .child({ eouProbability, input: text, duration: endTime - startTime })\n      .debug('eou prediction');\n    return eouProbability;\n  }\n\n  #formatChatContext(ctx: RawChatContext): string {\n    const newCtx: RawChatContext = [];\n    for (const msg of ctx) {\n      if (!msg.content) continue;\n      newCtx.push(msg);\n    }\n\n    const convoText = this.#tokenizer!.apply_chat_template(newCtx, {\n      add_generation_prompt: false,\n      tokenize: false,\n    }) as string;\n    // remove EOU token from current utterance\n    return convoText.slice(0, convoText.lastIndexOf('<|im_end|>'));\n  }\n}\n\nexport class EOUModel {\n  readonly unlikelyThreshold: number;\n  #executor: ipc.InferenceExecutor;\n\n  constructor(unlikelyThreshold = 0.15) {\n    this.unlikelyThreshold = unlikelyThreshold;\n    this.#executor = CurrentJobContext.getCurrent().inferenceExecutor;\n  }\n\n  supportsLanguage(language?: string) {\n    if (!language) return false;\n    const parts = language.toLowerCase().split('-');\n    return parts[0] === 'en' || parts[0] === 'english';\n  }\n\n  async predictEndOfTurn(chatCtx: llm.ChatContext): Promise<number> {\n    let messages: RawChatContext = [];\n\n    for (const msg of chatCtx.messages) {\n      if (msg.role !== llm.ChatRole.ASSISTANT && msg.role !== llm.ChatRole.USER) {\n        continue;\n      }\n\n      if (typeof msg.content === 'string') {\n        messages.push({\n          role: msg.role === llm.ChatRole.ASSISTANT ? 'assistant' : 'user',\n          content: msg.content,\n        });\n      } else if (Array.isArray(msg.content)) {\n        for (const content of msg.content) {\n          if (typeof content === 'string') {\n            messages.push({\n              role: msg.role === llm.ChatRole.ASSISTANT ? 'assistant' : 'user',\n              content: content,\n            });\n          }\n        }\n      }\n    }\n    messages = messages.slice(-MAX_HISTORY);\n    const result = await this.#executor.doInference(EOURunner.INFERENCE_METHOD, messages);\n    return result as any;\n  }\n}\n\nexport default EOURunner;\n"],"mappings":"AAIA,SAAS,qBAAqB;AAE9B,SAAS,mBAAmB,iBAAiB,KAAK,WAAW;AAC7D,SAAS,qBAAqB;AAC9B,SAAS,kBAAkB,cAAc;AAEzC,MAAM,cAAc;AAIb,MAAM,kBAAkB,gBAAgB;AAAA,EAC7C,OAAO,mBAAmB;AAAA,EAC1B;AAAA,EACA;AAAA,EACA;AAAA,EACA,UAAU,IAAI;AAAA,EAEd,cAAc;AACZ,UAAM;AACN,SAAK,oBAAoB,cAAc,gBAAgB,yBAAyB;AAAA,MAC9E,UAAU;AAAA;AAAA,IAEZ,CAAC;AACD,SAAK,WAAW,iBAAiB;AAAA,MAC/B,cAAc,IAAI,IAAI,sBAAsB,YAAY,GAAG,EAAE,IAAI;AAAA,MACjE;AAAA,QACE,oBAAoB,CAAC,EAAE,MAAM,MAAM,CAAC;AAAA,MACtC;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,aAAa;AACjB,SAAK,aAAa,MAAM,KAAK;AAAA,EAC/B;AAAA,EAEA,MAAM,IAAI,MAAmD;AAC3D,UAAM,OAAO,KAAK,mBAAmB,IAAI;AACzC,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,SAAS,KAAK,WAAY,OAAO,MAAM,EAAE,oBAAoB,MAAM,CAAC;AAC1E,UAAM,UAAU,MAAM,KAAK,SAAS;AAAA,MAAK,CAAC,YACxC,QAAQ,IAAI,EAAE,WAAW,IAAI,OAAO,SAAS,QAAQ,CAAC,GAAG,OAAO,MAAM,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC;AAAA,IACtF;AACA,UAAM,UAAU,KAAK,IAAI;AACzB,UAAM,SAAS,QAAQ;AACvB,UAAM,iBAAiB,OAAO,KAAK,CAAC;AACpC,SAAK,QACF,MAAM,EAAE,gBAAgB,OAAO,MAAM,UAAU,UAAU,UAAU,CAAC,EACpE,MAAM,gBAAgB;AACzB,WAAO;AAAA,EACT;AAAA,EAEA,mBAAmB,KAA6B;AAC9C,UAAM,SAAyB,CAAC;AAChC,eAAW,OAAO,KAAK;AACrB,UAAI,CAAC,IAAI,QAAS;AAClB,aAAO,KAAK,GAAG;AAAA,IACjB;AAEA,UAAM,YAAY,KAAK,WAAY,oBAAoB,QAAQ;AAAA,MAC7D,uBAAuB;AAAA,MACvB,UAAU;AAAA,IACZ,CAAC;AAED,WAAO,UAAU,MAAM,GAAG,UAAU,YAAY,YAAY,CAAC;AAAA,EAC/D;AACF;AAEO,MAAM,SAAS;AAAA,EACX;AAAA,EACT;AAAA,EAEA,YAAY,oBAAoB,MAAM;AACpC,SAAK,oBAAoB;AACzB,SAAK,YAAY,kBAAkB,WAAW,EAAE;AAAA,EAClD;AAAA,EAEA,iBAAiB,UAAmB;AAClC,QAAI,CAAC,SAAU,QAAO;AACtB,UAAM,QAAQ,SAAS,YAAY,EAAE,MAAM,GAAG;AAC9C,WAAO,MAAM,CAAC,MAAM,QAAQ,MAAM,CAAC,MAAM;AAAA,EAC3C;AAAA,EAEA,MAAM,iBAAiB,SAA2C;AAChE,QAAI,WAA2B,CAAC;AAEhC,eAAW,OAAO,QAAQ,UAAU;AAClC,UAAI,IAAI,SAAS,IAAI,SAAS,aAAa,IAAI,SAAS,IAAI,SAAS,MAAM;AACzE;AAAA,MACF;AAEA,UAAI,OAAO,IAAI,YAAY,UAAU;AACnC,iBAAS,KAAK;AAAA,UACZ,MAAM,IAAI,SAAS,IAAI,SAAS,YAAY,cAAc;AAAA,UAC1D,SAAS,IAAI;AAAA,QACf,CAAC;AAAA,MACH,WAAW,MAAM,QAAQ,IAAI,OAAO,GAAG;AACrC,mBAAW,WAAW,IAAI,SAAS;AACjC,cAAI,OAAO,YAAY,UAAU;AAC/B,qBAAS,KAAK;AAAA,cACZ,MAAM,IAAI,SAAS,IAAI,SAAS,YAAY,cAAc;AAAA,cAC1D;AAAA,YACF,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAAA,IACF;AACA,eAAW,SAAS,MAAM,CAAC,WAAW;AACtC,UAAM,SAAS,MAAM,KAAK,UAAU,YAAY,UAAU,kBAAkB,QAAQ;AACpF,WAAO;AAAA,EACT;AACF;AAEA,IAAO,wBAAQ;","names":[]}

package/package.json

@@ -0,0 +1,46 @@
+{
+  "name": "@livekit/agents-plugin-livekit",
+  "version": "0.1.0",
+  "description": "Additional utilities for LiveKit Node Agents",
+  "main": "dist/index.js",
+  "require": "dist/index.cjs",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "require": "./dist/index.cjs"
+    }
+  },
+  "author": "LiveKit",
+  "type": "module",
+  "repository": "git@github.com:livekit/agents-js.git",
+  "license": "Apache-2.0",
+  "files": [
+    "dist",
+    "src",
+    "README.md"
+  ],
+  "devDependencies": {
+    "@livekit/agents": "^x",
+    "@microsoft/api-extractor": "^7.35.0",
+    "onnxruntime-common": "^1.19.2",
+    "tsup": "^8.3.5",
+    "typescript": "^5.0.0"
+  },
+  "peerDependencies": {
+    "@livekit/agents": "^0.7.0x"
+  },
+  "dependencies": {
+    "@huggingface/transformers": "^3.2.1",
+    "onnxruntime-node": "^1.19.2"
+  },
+  "scripts": {
+    "build": "tsup --onSuccess \"tsc --declaration --emitDeclarationOnly\" && cp src/turn_detector.onnx dist/",
+    "clean": "rm -rf dist",
+    "clean:build": "pnpm clean && pnpm build",
+    "lint": "eslint -f unix \"src/**/*.{ts,js}\"",
+    "api:check": "api-extractor run --typescript-compiler-folder ../../node_modules/typescript",
+    "api:update": "api-extractor run --local --typescript-compiler-folder ../../node_modules/typescript --verbose"
+  }
+}

package/src/index.ts

@@ -0,0 +1,11 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { InferenceRunner } from '@livekit/agents';
+import * as turnDetector from './turn_detector.js';
+
+InferenceRunner.registerRunner(
+  turnDetector.EOURunner.INFERENCE_METHOD,
+  import.meta.resolve('./turn_detector.js'),
+);
+export { turnDetector };

package/src/onnxruntime.d.ts

@@ -0,0 +1,8 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+// https://github.com/microsoft/onnxruntime/issues/17979
+declare module 'onnxruntime-node' {
+  export * from 'onnxruntime-common';
+}

package/src/turn_detector.ts

@@ -0,0 +1,117 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { PreTrainedTokenizer } from '@huggingface/transformers';
+import { AutoTokenizer } from '@huggingface/transformers';
+import type { ipc } from '@livekit/agents';
+import { CurrentJobContext, InferenceRunner, llm, log } from '@livekit/agents';
+import { fileURLToPath } from 'node:url';
+import { InferenceSession, Tensor } from 'onnxruntime-node';
+
+const MAX_HISTORY = 4;
+
+type RawChatContext = { role: string; content: string }[];
+
+export class EOURunner extends InferenceRunner {
+  static INFERENCE_METHOD = 'lk_end_of_utterance';
+  #tokenizerPromise: Promise<PreTrainedTokenizer>;
+  #session: Promise<InferenceSession>;
+  #tokenizer?: PreTrainedTokenizer;
+  #logger = log();
+
+  constructor() {
+    super();
+    this.#tokenizerPromise = AutoTokenizer.from_pretrained('livekit/turn-detector', {
+      revision: 'v1.2.0',
+      // local_files_only: true, // TODO(nbsp): can't find it
+    });
+    this.#session = InferenceSession.create(
+      fileURLToPath(new URL('turn_detector.onnx', import.meta.url).href),
+      {
+        executionProviders: [{ name: 'cpu' }],
+      },
+    );
+  }
+
+  async initialize() {
+    this.#tokenizer = await this.#tokenizerPromise;
+  }
+
+  async run(data: RawChatContext): Promise<number | undefined> {
+    const text = this.#formatChatContext(data);
+    const startTime = Date.now();
+    const inputs = this.#tokenizer!.encode(text, { add_special_tokens: false });
+    const outputs = await this.#session.then((session) =>
+      session.run({ input_ids: new Tensor('int64', inputs, [1, inputs.length]) }, ['prob']),
+    );
+    const endTime = Date.now();
+    const logits = outputs.prob!;
+    const eouProbability = logits.data[0] as number;
+    this.#logger
+      .child({ eouProbability, input: text, duration: endTime - startTime })
+      .debug('eou prediction');
+    return eouProbability;
+  }
+
+  #formatChatContext(ctx: RawChatContext): string {
+    const newCtx: RawChatContext = [];
+    for (const msg of ctx) {
+      if (!msg.content) continue;
+      newCtx.push(msg);
+    }
+
+    const convoText = this.#tokenizer!.apply_chat_template(newCtx, {
+      add_generation_prompt: false,
+      tokenize: false,
+    }) as string;
+    // remove EOU token from current utterance
+    return convoText.slice(0, convoText.lastIndexOf('<|im_end|>'));
+  }
+}
+
+export class EOUModel {
+  readonly unlikelyThreshold: number;
+  #executor: ipc.InferenceExecutor;
+
+  constructor(unlikelyThreshold = 0.15) {
+    this.unlikelyThreshold = unlikelyThreshold;
+    this.#executor = CurrentJobContext.getCurrent().inferenceExecutor;
+  }
+
+  supportsLanguage(language?: string) {
+    if (!language) return false;
+    const parts = language.toLowerCase().split('-');
+    return parts[0] === 'en' || parts[0] === 'english';
+  }
+
+  async predictEndOfTurn(chatCtx: llm.ChatContext): Promise<number> {
+    let messages: RawChatContext = [];
+
+    for (const msg of chatCtx.messages) {
+      if (msg.role !== llm.ChatRole.ASSISTANT && msg.role !== llm.ChatRole.USER) {
+        continue;
+      }
+
+      if (typeof msg.content === 'string') {
+        messages.push({
+          role: msg.role === llm.ChatRole.ASSISTANT ? 'assistant' : 'user',
+          content: msg.content,
+        });
+      } else if (Array.isArray(msg.content)) {
+        for (const content of msg.content) {
+          if (typeof content === 'string') {
+            messages.push({
+              role: msg.role === llm.ChatRole.ASSISTANT ? 'assistant' : 'user',
+              content: content,
+            });
+          }
+        }
+      }
+    }
+    messages = messages.slice(-MAX_HISTORY);
+    const result = await this.#executor.doInference(EOURunner.INFERENCE_METHOD, messages);
+    return result as any;
+  }
+}
+
+export default EOURunner;