@livekit/agents-plugin-livekit 0.1.3 → 1.0.0-next.1

package/dist/turn_detector.js

@@ -1,102 +0,0 @@
-import { AutoTokenizer } from "@huggingface/transformers";
-import { CurrentJobContext, InferenceRunner, llm, log } from "@livekit/agents";
-import { fileURLToPath } from "node:url";
-import { InferenceSession, Tensor } from "onnxruntime-node";
-const MAX_HISTORY = 4;
-class EOURunner extends InferenceRunner {
-  static INFERENCE_METHOD = "lk_end_of_utterance";
-  #tokenizerPromise;
-  #session;
-  #tokenizer;
-  #logger = log();
-  constructor() {
-    super();
-    this.#tokenizerPromise = AutoTokenizer.from_pretrained("livekit/turn-detector", {
-      revision: "v1.2.0"
-      // local_files_only: true, // TODO(nbsp): can't find it
-    });
-    this.#session = InferenceSession.create(
-      fileURLToPath(new URL("turn_detector.onnx", import.meta.url).href),
-      {
-        executionProviders: [{ name: "cpu" }]
-      }
-    );
-  }
-  async initialize() {
-    this.#tokenizer = await this.#tokenizerPromise;
-  }
-  async run(data) {
-    const text = this.#formatChatContext(data);
-    const startTime = Date.now();
-    const inputs = this.#tokenizer.encode(text, { add_special_tokens: false });
-    const outputs = await this.#session.then(
-      (session) => session.run({ input_ids: new Tensor("int64", inputs, [1, inputs.length]) }, ["prob"])
-    );
-    const endTime = Date.now();
-    const logits = outputs.prob;
-    const eouProbability = logits.data[0];
-    this.#logger.child({ eouProbability, input: text, duration: endTime - startTime }).debug("eou prediction");
-    return eouProbability;
-  }
-  #formatChatContext(ctx) {
-    const newCtx = [];
-    for (const msg of ctx) {
-      if (!msg.content) continue;
-      newCtx.push(msg);
-    }
-    const convoText = this.#tokenizer.apply_chat_template(newCtx, {
-      add_generation_prompt: false,
-      tokenize: false
-    });
-    return convoText.slice(0, convoText.lastIndexOf("<|im_end|>"));
-  }
-  async close() {
-    await this.#session.then((session) => session.release());
-  }
-}
-class EOUModel {
-  unlikelyThreshold;
-  #executor;
-  constructor(unlikelyThreshold = 0.15) {
-    this.unlikelyThreshold = unlikelyThreshold;
-    this.#executor = CurrentJobContext.getCurrent().inferenceExecutor;
-  }
-  supportsLanguage(language) {
-    if (!language) return false;
-    const parts = language.toLowerCase().split("-");
-    return parts[0] === "en" || parts[0] === "english";
-  }
-  async predictEndOfTurn(chatCtx) {
-    let messages = [];
-    for (const msg of chatCtx.messages) {
-      if (msg.role !== llm.ChatRole.ASSISTANT && msg.role !== llm.ChatRole.USER) {
-        continue;
-      }
-      if (typeof msg.content === "string") {
-        messages.push({
-          role: msg.role === llm.ChatRole.ASSISTANT ? "assistant" : "user",
-          content: msg.content
-        });
-      } else if (Array.isArray(msg.content)) {
-        for (const content of msg.content) {
-          if (typeof content === "string") {
-            messages.push({
-              role: msg.role === llm.ChatRole.ASSISTANT ? "assistant" : "user",
-              content
-            });
-          }
-        }
-      }
-    }
-    messages = messages.slice(-MAX_HISTORY);
-    const result = await this.#executor.doInference(EOURunner.INFERENCE_METHOD, messages);
-    return result;
-  }
-}
-var turn_detector_default = EOURunner;
-export {
-  EOUModel,
-  EOURunner,
-  turn_detector_default as default
-};
-//# sourceMappingURL=turn_detector.js.map

package/dist/turn_detector.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/turn_detector.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { PreTrainedTokenizer } from '@huggingface/transformers';\nimport { AutoTokenizer } from '@huggingface/transformers';\nimport type { ipc } from '@livekit/agents';\nimport { CurrentJobContext, InferenceRunner, llm, log } from '@livekit/agents';\nimport { fileURLToPath } from 'node:url';\nimport { InferenceSession, Tensor } from 'onnxruntime-node';\n\nconst MAX_HISTORY = 4;\n\ntype RawChatContext = { role: string; content: string }[];\n\nexport class EOURunner extends InferenceRunner {\n  static INFERENCE_METHOD = 'lk_end_of_utterance';\n  #tokenizerPromise: Promise<PreTrainedTokenizer>;\n  #session: Promise<InferenceSession>;\n  #tokenizer?: PreTrainedTokenizer;\n  #logger = log();\n\n  constructor() {\n    super();\n    this.#tokenizerPromise = AutoTokenizer.from_pretrained('livekit/turn-detector', {\n      revision: 'v1.2.0',\n      // local_files_only: true, // TODO(nbsp): can't find it\n    });\n    this.#session = InferenceSession.create(\n      fileURLToPath(new URL('turn_detector.onnx', import.meta.url).href),\n      {\n        executionProviders: [{ name: 'cpu' }],\n      },\n    );\n  }\n\n  async initialize() {\n    this.#tokenizer = await this.#tokenizerPromise;\n  }\n\n  async run(data: RawChatContext): Promise<number | undefined> {\n    const text = this.#formatChatContext(data);\n    const startTime = Date.now();\n    const inputs = this.#tokenizer!.encode(text, { add_special_tokens: false });\n    const outputs = await this.#session.then((session) =>\n      session.run({ input_ids: new Tensor('int64', inputs, [1, inputs.length]) }, ['prob']),\n    );\n    const endTime = Date.now();\n    const logits = outputs.prob!;\n    const eouProbability = logits.data[0] as number;\n    this.#logger\n      .child({ eouProbability, input: text, duration: endTime - startTime })\n      .debug('eou prediction');\n    return eouProbability;\n  }\n\n  #formatChatContext(ctx: RawChatContext): string {\n    const newCtx: RawChatContext = [];\n    for (const msg of ctx) {\n      if (!msg.content) continue;\n      newCtx.push(msg);\n    }\n\n    const convoText = this.#tokenizer!.apply_chat_template(newCtx, {\n      add_generation_prompt: false,\n      tokenize: false,\n    }) as string;\n    // remove EOU token from current utterance\n    return convoText.slice(0, convoText.lastIndexOf('<|im_end|>'));\n  }\n\n  async close() {\n    await this.#session.then((session) => session.release());\n  }\n}\n\nexport class EOUModel {\n  readonly unlikelyThreshold: number;\n  #executor: ipc.InferenceExecutor;\n\n  constructor(unlikelyThreshold = 0.15) {\n    this.unlikelyThreshold = unlikelyThreshold;\n    this.#executor = CurrentJobContext.getCurrent().inferenceExecutor;\n  }\n\n  supportsLanguage(language?: string) {\n    if (!language) return false;\n    const parts = language.toLowerCase().split('-');\n    return parts[0] === 'en' || parts[0] === 'english';\n  }\n\n  async predictEndOfTurn(chatCtx: llm.ChatContext): Promise<number> {\n    let messages: RawChatContext = [];\n\n    for (const msg of chatCtx.messages) {\n      if (msg.role !== llm.ChatRole.ASSISTANT && msg.role !== llm.ChatRole.USER) {\n        continue;\n      }\n\n      if (typeof msg.content === 'string') {\n        messages.push({\n          role: msg.role === llm.ChatRole.ASSISTANT ? 'assistant' : 'user',\n          content: msg.content,\n        });\n      } else if (Array.isArray(msg.content)) {\n        for (const content of msg.content) {\n          if (typeof content === 'string') {\n            messages.push({\n              role: msg.role === llm.ChatRole.ASSISTANT ? 'assistant' : 'user',\n              content: content,\n            });\n          }\n        }\n      }\n    }\n    messages = messages.slice(-MAX_HISTORY);\n    const result = await this.#executor.doInference(EOURunner.INFERENCE_METHOD, messages);\n    return result as any;\n  }\n}\n\nexport default EOURunner;\n"],"mappings":"AAIA,SAAS,qBAAqB;AAE9B,SAAS,mBAAmB,iBAAiB,KAAK,WAAW;AAC7D,SAAS,qBAAqB;AAC9B,SAAS,kBAAkB,cAAc;AAEzC,MAAM,cAAc;AAIb,MAAM,kBAAkB,gBAAgB;AAAA,EAC7C,OAAO,mBAAmB;AAAA,EAC1B;AAAA,EACA;AAAA,EACA;AAAA,EACA,UAAU,IAAI;AAAA,EAEd,cAAc;AACZ,UAAM;AACN,SAAK,oBAAoB,cAAc,gBAAgB,yBAAyB;AAAA,MAC9E,UAAU;AAAA;AAAA,IAEZ,CAAC;AACD,SAAK,WAAW,iBAAiB;AAAA,MAC/B,cAAc,IAAI,IAAI,sBAAsB,YAAY,GAAG,EAAE,IAAI;AAAA,MACjE;AAAA,QACE,oBAAoB,CAAC,EAAE,MAAM,MAAM,CAAC;AAAA,MACtC;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,aAAa;AACjB,SAAK,aAAa,MAAM,KAAK;AAAA,EAC/B;AAAA,EAEA,MAAM,IAAI,MAAmD;AAC3D,UAAM,OAAO,KAAK,mBAAmB,IAAI;AACzC,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,SAAS,KAAK,WAAY,OAAO,MAAM,EAAE,oBAAoB,MAAM,CAAC;AAC1E,UAAM,UAAU,MAAM,KAAK,SAAS;AAAA,MAAK,CAAC,YACxC,QAAQ,IAAI,EAAE,WAAW,IAAI,OAAO,SAAS,QAAQ,CAAC,GAAG,OAAO,MAAM,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC;AAAA,IACtF;AACA,UAAM,UAAU,KAAK,IAAI;AACzB,UAAM,SAAS,QAAQ;AACvB,UAAM,iBAAiB,OAAO,KAAK,CAAC;AACpC,SAAK,QACF,MAAM,EAAE,gBAAgB,OAAO,MAAM,UAAU,UAAU,UAAU,CAAC,EACpE,MAAM,gBAAgB;AACzB,WAAO;AAAA,EACT;AAAA,EAEA,mBAAmB,KAA6B;AAC9C,UAAM,SAAyB,CAAC;AAChC,eAAW,OAAO,KAAK;AACrB,UAAI,CAAC,IAAI,QAAS;AAClB,aAAO,KAAK,GAAG;AAAA,IACjB;AAEA,UAAM,YAAY,KAAK,WAAY,oBAAoB,QAAQ;AAAA,MAC7D,uBAAuB;AAAA,MACvB,UAAU;AAAA,IACZ,CAAC;AAED,WAAO,UAAU,MAAM,GAAG,UAAU,YAAY,YAAY,CAAC;AAAA,EAC/D;AAAA,EAEA,MAAM,QAAQ;AACZ,UAAM,KAAK,SAAS,KAAK,CAAC,YAAY,QAAQ,QAAQ,CAAC;AAAA,EACzD;AACF;AAEO,MAAM,SAAS;AAAA,EACX;AAAA,EACT;AAAA,EAEA,YAAY,oBAAoB,MAAM;AACpC,SAAK,oBAAoB;AACzB,SAAK,YAAY,kBAAkB,WAAW,EAAE;AAAA,EAClD;AAAA,EAEA,iBAAiB,UAAmB;AAClC,QAAI,CAAC,SAAU,QAAO;AACtB,UAAM,QAAQ,SAAS,YAAY,EAAE,MAAM,GAAG;AAC9C,WAAO,MAAM,CAAC,MAAM,QAAQ,MAAM,CAAC,MAAM;AAAA,EAC3C;AAAA,EAEA,MAAM,iBAAiB,SAA2C;AAChE,QAAI,WAA2B,CAAC;AAEhC,eAAW,OAAO,QAAQ,UAAU;AAClC,UAAI,IAAI,SAAS,IAAI,SAAS,aAAa,IAAI,SAAS,IAAI,SAAS,MAAM;AACzE;AAAA,MACF;AAEA,UAAI,OAAO,IAAI,YAAY,UAAU;AACnC,iBAAS,KAAK;AAAA,UACZ,MAAM,IAAI,SAAS,IAAI,SAAS,YAAY,cAAc;AAAA,UAC1D,SAAS,IAAI;AAAA,QACf,CAAC;AAAA,MACH,WAAW,MAAM,QAAQ,IAAI,OAAO,GAAG;AACrC,mBAAW,WAAW,IAAI,SAAS;AACjC,cAAI,OAAO,YAAY,UAAU;AAC/B,qBAAS,KAAK;AAAA,cACZ,MAAM,IAAI,SAAS,IAAI,SAAS,YAAY,cAAc;AAAA,cAC1D;AAAA,YACF,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAAA,IACF;AACA,eAAW,SAAS,MAAM,CAAC,WAAW;AACtC,UAAM,SAAS,MAAM,KAAK,UAAU,YAAY,UAAU,kBAAkB,QAAQ;AACpF,WAAO;AAAA,EACT;AACF;AAEA,IAAO,wBAAQ;","names":[]}

package/src/turn_detector.ts

@@ -1,121 +0,0 @@
-// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
-//
-// SPDX-License-Identifier: Apache-2.0
-import type { PreTrainedTokenizer } from '@huggingface/transformers';
-import { AutoTokenizer } from '@huggingface/transformers';
-import type { ipc } from '@livekit/agents';
-import { CurrentJobContext, InferenceRunner, llm, log } from '@livekit/agents';
-import { fileURLToPath } from 'node:url';
-import { InferenceSession, Tensor } from 'onnxruntime-node';
-
-const MAX_HISTORY = 4;
-
-type RawChatContext = { role: string; content: string }[];
-
-export class EOURunner extends InferenceRunner {
-  static INFERENCE_METHOD = 'lk_end_of_utterance';
-  #tokenizerPromise: Promise<PreTrainedTokenizer>;
-  #session: Promise<InferenceSession>;
-  #tokenizer?: PreTrainedTokenizer;
-  #logger = log();
-
-  constructor() {
-    super();
-    this.#tokenizerPromise = AutoTokenizer.from_pretrained('livekit/turn-detector', {
-      revision: 'v1.2.0',
-      // local_files_only: true, // TODO(nbsp): can't find it
-    });
-    this.#session = InferenceSession.create(
-      fileURLToPath(new URL('turn_detector.onnx', import.meta.url).href),
-      {
-        executionProviders: [{ name: 'cpu' }],
-      },
-    );
-  }
-
-  async initialize() {
-    this.#tokenizer = await this.#tokenizerPromise;
-  }
-
-  async run(data: RawChatContext): Promise<number | undefined> {
-    const text = this.#formatChatContext(data);
-    const startTime = Date.now();
-    const inputs = this.#tokenizer!.encode(text, { add_special_tokens: false });
-    const outputs = await this.#session.then((session) =>
-      session.run({ input_ids: new Tensor('int64', inputs, [1, inputs.length]) }, ['prob']),
-    );
-    const endTime = Date.now();
-    const logits = outputs.prob!;
-    const eouProbability = logits.data[0] as number;
-    this.#logger
-      .child({ eouProbability, input: text, duration: endTime - startTime })
-      .debug('eou prediction');
-    return eouProbability;
-  }
-
-  #formatChatContext(ctx: RawChatContext): string {
-    const newCtx: RawChatContext = [];
-    for (const msg of ctx) {
-      if (!msg.content) continue;
-      newCtx.push(msg);
-    }
-
-    const convoText = this.#tokenizer!.apply_chat_template(newCtx, {
-      add_generation_prompt: false,
-      tokenize: false,
-    }) as string;
-    // remove EOU token from current utterance
-    return convoText.slice(0, convoText.lastIndexOf('<|im_end|>'));
-  }
-
-  async close() {
-    await this.#session.then((session) => session.release());
-  }
-}
-
-export class EOUModel {
-  readonly unlikelyThreshold: number;
-  #executor: ipc.InferenceExecutor;
-
-  constructor(unlikelyThreshold = 0.15) {
-    this.unlikelyThreshold = unlikelyThreshold;
-    this.#executor = CurrentJobContext.getCurrent().inferenceExecutor;
-  }
-
-  supportsLanguage(language?: string) {
-    if (!language) return false;
-    const parts = language.toLowerCase().split('-');
-    return parts[0] === 'en' || parts[0] === 'english';
-  }
-
-  async predictEndOfTurn(chatCtx: llm.ChatContext): Promise<number> {
-    let messages: RawChatContext = [];
-
-    for (const msg of chatCtx.messages) {
-      if (msg.role !== llm.ChatRole.ASSISTANT && msg.role !== llm.ChatRole.USER) {
-        continue;
-      }
-
-      if (typeof msg.content === 'string') {
-        messages.push({
-          role: msg.role === llm.ChatRole.ASSISTANT ? 'assistant' : 'user',
-          content: msg.content,
-        });
-      } else if (Array.isArray(msg.content)) {
-        for (const content of msg.content) {
-          if (typeof content === 'string') {
-            messages.push({
-              role: msg.role === llm.ChatRole.ASSISTANT ? 'assistant' : 'user',
-              content: content,
-            });
-          }
-        }
-      }
-    }
-    messages = messages.slice(-MAX_HISTORY);
-    const result = await this.#executor.doInference(EOURunner.INFERENCE_METHOD, messages);
-    return result as any;
-  }
-}
-
-export default EOURunner;