@livekit/agents-plugin-livekit 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/turn_detector.cjs
CHANGED
|
@@ -77,6 +77,9 @@ class EOURunner extends import_agents.InferenceRunner {
|
|
|
77
77
|
});
|
|
78
78
|
return convoText.slice(0, convoText.lastIndexOf("<|im_end|>"));
|
|
79
79
|
}
|
|
80
|
+
async close() {
|
|
81
|
+
await this.#session.then((session) => session.release());
|
|
82
|
+
}
|
|
80
83
|
}
|
|
81
84
|
class EOUModel {
|
|
82
85
|
unlikelyThreshold;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/turn_detector.ts","../../../node_modules/.pnpm/tsup@8.3.5_@microsoft+api-extractor@7.43.7_@types+node@22.5.5__postcss@8.4.38_tsx@4.19.2_typescript@5.4.5/node_modules/tsup/assets/cjs_shims.js"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { PreTrainedTokenizer } from '@huggingface/transformers';\nimport { AutoTokenizer } from '@huggingface/transformers';\nimport type { ipc } from '@livekit/agents';\nimport { CurrentJobContext, InferenceRunner, llm, log } from '@livekit/agents';\nimport { fileURLToPath } from 'node:url';\nimport { InferenceSession, Tensor } from 'onnxruntime-node';\n\nconst MAX_HISTORY = 4;\n\ntype RawChatContext = { role: string; content: string }[];\n\nexport class EOURunner extends InferenceRunner {\n static INFERENCE_METHOD = 'lk_end_of_utterance';\n #tokenizerPromise: Promise<PreTrainedTokenizer>;\n #session: Promise<InferenceSession>;\n #tokenizer?: PreTrainedTokenizer;\n #logger = log();\n\n constructor() {\n super();\n this.#tokenizerPromise = AutoTokenizer.from_pretrained('livekit/turn-detector', {\n revision: 'v1.2.0',\n // local_files_only: true, // TODO(nbsp): can't find it\n });\n this.#session = InferenceSession.create(\n fileURLToPath(new URL('turn_detector.onnx', import.meta.url).href),\n {\n executionProviders: [{ name: 'cpu' }],\n },\n );\n }\n\n async initialize() {\n this.#tokenizer = await this.#tokenizerPromise;\n }\n\n async run(data: RawChatContext): Promise<number | undefined> {\n const text = this.#formatChatContext(data);\n const startTime = Date.now();\n const inputs = this.#tokenizer!.encode(text, { add_special_tokens: false });\n const outputs = await this.#session.then((session) =>\n session.run({ input_ids: new Tensor('int64', inputs, [1, inputs.length]) }, ['prob']),\n );\n const endTime = Date.now();\n const logits = outputs.prob!;\n const eouProbability = logits.data[0] as number;\n this.#logger\n .child({ eouProbability, input: text, duration: endTime - startTime })\n .debug('eou prediction');\n return eouProbability;\n }\n\n #formatChatContext(ctx: RawChatContext): string {\n const newCtx: RawChatContext = [];\n for (const msg of ctx) {\n if (!msg.content) continue;\n newCtx.push(msg);\n }\n\n const convoText = this.#tokenizer!.apply_chat_template(newCtx, {\n add_generation_prompt: false,\n tokenize: false,\n }) as string;\n // remove EOU token from current utterance\n return convoText.slice(0, convoText.lastIndexOf('<|im_end|>'));\n }\n}\n\nexport class EOUModel {\n readonly unlikelyThreshold: number;\n #executor: ipc.InferenceExecutor;\n\n constructor(unlikelyThreshold = 0.15) {\n this.unlikelyThreshold = unlikelyThreshold;\n this.#executor = CurrentJobContext.getCurrent().inferenceExecutor;\n }\n\n supportsLanguage(language?: string) {\n if (!language) return false;\n const parts = language.toLowerCase().split('-');\n return parts[0] === 'en' || parts[0] === 'english';\n }\n\n async predictEndOfTurn(chatCtx: llm.ChatContext): Promise<number> {\n let messages: RawChatContext = [];\n\n for (const msg of chatCtx.messages) {\n if (msg.role !== llm.ChatRole.ASSISTANT && msg.role !== llm.ChatRole.USER) {\n continue;\n }\n\n if (typeof msg.content === 'string') {\n messages.push({\n role: msg.role === llm.ChatRole.ASSISTANT ? 'assistant' : 'user',\n content: msg.content,\n });\n } else if (Array.isArray(msg.content)) {\n for (const content of msg.content) {\n if (typeof content === 'string') {\n messages.push({\n role: msg.role === llm.ChatRole.ASSISTANT ? 'assistant' : 'user',\n content: content,\n });\n }\n }\n }\n }\n messages = messages.slice(-MAX_HISTORY);\n const result = await this.#executor.doInference(EOURunner.INFERENCE_METHOD, messages);\n return result as any;\n }\n}\n\nexport default EOURunner;\n","// Shim globals in cjs bundle\n// There's a weird bug that esbuild will always inject importMetaUrl\n// if we export it as `const importMetaUrl = ... __filename ...`\n// But using a function will not cause this issue\n\nconst getImportMetaUrl = () =>\n typeof document === 'undefined'\n ? new URL(`file:${__filename}`).href\n : (document.currentScript && document.currentScript.src) ||\n new URL('main.js', document.baseURI).href\n\nexport const importMetaUrl = /* @__PURE__ */ getImportMetaUrl()\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;ACKA,IAAM,mBAAmB,MACvB,OAAO,aAAa,cAChB,IAAI,IAAI,QAAQ,UAAU,EAAE,EAAE,OAC7B,SAAS,iBAAiB,SAAS,cAAc,OAClD,IAAI,IAAI,WAAW,SAAS,OAAO,EAAE;AAEpC,IAAM,gBAAgC,iCAAiB;ADP9D,0BAA8B;AAE9B,oBAA6D;AAC7D,sBAA8B;AAC9B,8BAAyC;AAEzC,MAAM,cAAc;AAIb,MAAM,kBAAkB,8BAAgB;AAAA,EAC7C,OAAO,mBAAmB;AAAA,EAC1B;AAAA,EACA;AAAA,EACA;AAAA,EACA,cAAU,mBAAI;AAAA,EAEd,cAAc;AACZ,UAAM;AACN,SAAK,oBAAoB,kCAAc,gBAAgB,yBAAyB;AAAA,MAC9E,UAAU;AAAA;AAAA,IAEZ,CAAC;AACD,SAAK,WAAW,yCAAiB;AAAA,UAC/B,+BAAc,IAAI,IAAI,sBAAsB,aAAe,EAAE,IAAI;AAAA,MACjE;AAAA,QACE,oBAAoB,CAAC,EAAE,MAAM,MAAM,CAAC;AAAA,MACtC;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,aAAa;AACjB,SAAK,aAAa,MAAM,KAAK;AAAA,EAC/B;AAAA,EAEA,MAAM,IAAI,MAAmD;AAC3D,UAAM,OAAO,KAAK,mBAAmB,IAAI;AACzC,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,SAAS,KAAK,WAAY,OAAO,MAAM,EAAE,oBAAoB,MAAM,CAAC;AAC1E,UAAM,UAAU,MAAM,KAAK,SAAS;AAAA,MAAK,CAAC,YACxC,QAAQ,IAAI,EAAE,WAAW,IAAI,+BAAO,SAAS,QAAQ,CAAC,GAAG,OAAO,MAAM,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC;AAAA,IACtF;AACA,UAAM,UAAU,KAAK,IAAI;AACzB,UAAM,SAAS,QAAQ;AACvB,UAAM,iBAAiB,OAAO,KAAK,CAAC;AACpC,SAAK,QACF,MAAM,EAAE,gBAAgB,OAAO,MAAM,UAAU,UAAU,UAAU,CAAC,EACpE,MAAM,gBAAgB;AACzB,WAAO;AAAA,EACT;AAAA,EAEA,mBAAmB,KAA6B;AAC9C,UAAM,SAAyB,CAAC;AAChC,eAAW,OAAO,KAAK;AACrB,UAAI,CAAC,IAAI,QAAS;AAClB,aAAO,KAAK,GAAG;AAAA,IACjB;AAEA,UAAM,YAAY,KAAK,WAAY,oBAAoB,QAAQ;AAAA,MAC7D,uBAAuB;AAAA,MACvB,UAAU;AAAA,IACZ,CAAC;AAED,WAAO,UAAU,MAAM,GAAG,UAAU,YAAY,YAAY,CAAC;AAAA,EAC/D;AACF;AAEO,MAAM,SAAS;AAAA,EACX;AAAA,EACT;AAAA,EAEA,YAAY,oBAAoB,MAAM;AACpC,SAAK,oBAAoB;AACzB,SAAK,YAAY,gCAAkB,WAAW,EAAE;AAAA,EAClD;AAAA,EAEA,iBAAiB,UAAmB;AAClC,QAAI,CAAC,SAAU,QAAO;AACtB,UAAM,QAAQ,SAAS,YAAY,EAAE,MAAM,GAAG;AAC9C,WAAO,MAAM,CAAC,MAAM,QAAQ,MAAM,CAAC,MAAM;AAAA,EAC3C;AAAA,EAEA,MAAM,iBAAiB,SAA2C;AAChE,QAAI,WAA2B,CAAC;AAEhC,eAAW,OAAO,QAAQ,UAAU;AAClC,UAAI,IAAI,SAAS,kBAAI,SAAS,aAAa,IAAI,SAAS,kBAAI,SAAS,MAAM;AACzE;AAAA,MACF;AAEA,UAAI,OAAO,IAAI,YAAY,UAAU;AACnC,iBAAS,KAAK;AAAA,UACZ,MAAM,IAAI,SAAS,kBAAI,SAAS,YAAY,cAAc;AAAA,UAC1D,SAAS,IAAI;AAAA,QACf,CAAC;AAAA,MACH,WAAW,MAAM,QAAQ,IAAI,OAAO,GAAG;AACrC,mBAAW,WAAW,IAAI,SAAS;AACjC,cAAI,OAAO,YAAY,UAAU;AAC/B,qBAAS,KAAK;AAAA,cACZ,MAAM,IAAI,SAAS,kBAAI,SAAS,YAAY,cAAc;AAAA,cAC1D;AAAA,YACF,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAAA,IACF;AACA,eAAW,SAAS,MAAM,CAAC,WAAW;AACtC,UAAM,SAAS,MAAM,KAAK,UAAU,YAAY,UAAU,kBAAkB,QAAQ;AACpF,WAAO;AAAA,EACT;AACF;AAEA,IAAO,wBAAQ;","names":[]}
|
|
1
|
+
{"version":3,"sources":["../src/turn_detector.ts","../../../node_modules/.pnpm/tsup@8.3.5_@microsoft+api-extractor@7.43.7_@types+node@22.5.5__postcss@8.4.38_tsx@4.19.2_typescript@5.4.5/node_modules/tsup/assets/cjs_shims.js"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { PreTrainedTokenizer } from '@huggingface/transformers';\nimport { AutoTokenizer } from '@huggingface/transformers';\nimport type { ipc } from '@livekit/agents';\nimport { CurrentJobContext, InferenceRunner, llm, log } from '@livekit/agents';\nimport { fileURLToPath } from 'node:url';\nimport { InferenceSession, Tensor } from 'onnxruntime-node';\n\nconst MAX_HISTORY = 4;\n\ntype RawChatContext = { role: string; content: string }[];\n\nexport class EOURunner extends InferenceRunner {\n static INFERENCE_METHOD = 'lk_end_of_utterance';\n #tokenizerPromise: Promise<PreTrainedTokenizer>;\n #session: Promise<InferenceSession>;\n #tokenizer?: PreTrainedTokenizer;\n #logger = log();\n\n constructor() {\n super();\n this.#tokenizerPromise = AutoTokenizer.from_pretrained('livekit/turn-detector', {\n revision: 'v1.2.0',\n // local_files_only: true, // TODO(nbsp): can't find it\n });\n this.#session = InferenceSession.create(\n fileURLToPath(new URL('turn_detector.onnx', import.meta.url).href),\n {\n executionProviders: [{ name: 'cpu' }],\n },\n );\n }\n\n async initialize() {\n this.#tokenizer = await this.#tokenizerPromise;\n }\n\n async run(data: RawChatContext): Promise<number | undefined> {\n const text = this.#formatChatContext(data);\n const startTime = Date.now();\n const inputs = this.#tokenizer!.encode(text, { add_special_tokens: false });\n const outputs = await this.#session.then((session) =>\n session.run({ input_ids: new Tensor('int64', inputs, [1, inputs.length]) }, ['prob']),\n );\n const endTime = Date.now();\n const logits = outputs.prob!;\n const eouProbability = logits.data[0] as number;\n this.#logger\n .child({ eouProbability, input: text, duration: endTime - startTime })\n .debug('eou prediction');\n return eouProbability;\n }\n\n #formatChatContext(ctx: RawChatContext): string {\n const newCtx: RawChatContext = [];\n for (const msg of ctx) {\n if (!msg.content) continue;\n newCtx.push(msg);\n }\n\n const convoText = this.#tokenizer!.apply_chat_template(newCtx, {\n add_generation_prompt: false,\n tokenize: false,\n }) as string;\n // remove EOU token from current utterance\n return convoText.slice(0, convoText.lastIndexOf('<|im_end|>'));\n }\n\n async close() {\n await this.#session.then((session) => session.release());\n }\n}\n\nexport class EOUModel {\n readonly unlikelyThreshold: number;\n #executor: ipc.InferenceExecutor;\n\n constructor(unlikelyThreshold = 0.15) {\n this.unlikelyThreshold = unlikelyThreshold;\n this.#executor = CurrentJobContext.getCurrent().inferenceExecutor;\n }\n\n supportsLanguage(language?: string) {\n if (!language) return false;\n const parts = language.toLowerCase().split('-');\n return parts[0] === 'en' || parts[0] === 'english';\n }\n\n async predictEndOfTurn(chatCtx: llm.ChatContext): Promise<number> {\n let messages: RawChatContext = [];\n\n for (const msg of chatCtx.messages) {\n if (msg.role !== llm.ChatRole.ASSISTANT && msg.role !== llm.ChatRole.USER) {\n continue;\n }\n\n if (typeof msg.content === 'string') {\n messages.push({\n role: msg.role === llm.ChatRole.ASSISTANT ? 'assistant' : 'user',\n content: msg.content,\n });\n } else if (Array.isArray(msg.content)) {\n for (const content of msg.content) {\n if (typeof content === 'string') {\n messages.push({\n role: msg.role === llm.ChatRole.ASSISTANT ? 'assistant' : 'user',\n content: content,\n });\n }\n }\n }\n }\n messages = messages.slice(-MAX_HISTORY);\n const result = await this.#executor.doInference(EOURunner.INFERENCE_METHOD, messages);\n return result as any;\n }\n}\n\nexport default EOURunner;\n","// Shim globals in cjs bundle\n// There's a weird bug that esbuild will always inject importMetaUrl\n// if we export it as `const importMetaUrl = ... __filename ...`\n// But using a function will not cause this issue\n\nconst getImportMetaUrl = () =>\n typeof document === 'undefined'\n ? new URL(`file:${__filename}`).href\n : (document.currentScript && document.currentScript.src) ||\n new URL('main.js', document.baseURI).href\n\nexport const importMetaUrl = /* @__PURE__ */ getImportMetaUrl()\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;ACKA,IAAM,mBAAmB,MACvB,OAAO,aAAa,cAChB,IAAI,IAAI,QAAQ,UAAU,EAAE,EAAE,OAC7B,SAAS,iBAAiB,SAAS,cAAc,OAClD,IAAI,IAAI,WAAW,SAAS,OAAO,EAAE;AAEpC,IAAM,gBAAgC,iCAAiB;ADP9D,0BAA8B;AAE9B,oBAA6D;AAC7D,sBAA8B;AAC9B,8BAAyC;AAEzC,MAAM,cAAc;AAIb,MAAM,kBAAkB,8BAAgB;AAAA,EAC7C,OAAO,mBAAmB;AAAA,EAC1B;AAAA,EACA;AAAA,EACA;AAAA,EACA,cAAU,mBAAI;AAAA,EAEd,cAAc;AACZ,UAAM;AACN,SAAK,oBAAoB,kCAAc,gBAAgB,yBAAyB;AAAA,MAC9E,UAAU;AAAA;AAAA,IAEZ,CAAC;AACD,SAAK,WAAW,yCAAiB;AAAA,UAC/B,+BAAc,IAAI,IAAI,sBAAsB,aAAe,EAAE,IAAI;AAAA,MACjE;AAAA,QACE,oBAAoB,CAAC,EAAE,MAAM,MAAM,CAAC;AAAA,MACtC;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,aAAa;AACjB,SAAK,aAAa,MAAM,KAAK;AAAA,EAC/B;AAAA,EAEA,MAAM,IAAI,MAAmD;AAC3D,UAAM,OAAO,KAAK,mBAAmB,IAAI;AACzC,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,SAAS,KAAK,WAAY,OAAO,MAAM,EAAE,oBAAoB,MAAM,CAAC;AAC1E,UAAM,UAAU,MAAM,KAAK,SAAS;AAAA,MAAK,CAAC,YACxC,QAAQ,IAAI,EAAE,WAAW,IAAI,+BAAO,SAAS,QAAQ,CAAC,GAAG,OAAO,MAAM,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC;AAAA,IACtF;AACA,UAAM,UAAU,KAAK,IAAI;AACzB,UAAM,SAAS,QAAQ;AACvB,UAAM,iBAAiB,OAAO,KAAK,CAAC;AACpC,SAAK,QACF,MAAM,EAAE,gBAAgB,OAAO,MAAM,UAAU,UAAU,UAAU,CAAC,EACpE,MAAM,gBAAgB;AACzB,WAAO;AAAA,EACT;AAAA,EAEA,mBAAmB,KAA6B;AAC9C,UAAM,SAAyB,CAAC;AAChC,eAAW,OAAO,KAAK;AACrB,UAAI,CAAC,IAAI,QAAS;AAClB,aAAO,KAAK,GAAG;AAAA,IACjB;AAEA,UAAM,YAAY,KAAK,WAAY,oBAAoB,QAAQ;AAAA,MAC7D,uBAAuB;AAAA,MACvB,UAAU;AAAA,IACZ,CAAC;AAED,WAAO,UAAU,MAAM,GAAG,UAAU,YAAY,YAAY,CAAC;AAAA,EAC/D;AAAA,EAEA,MAAM,QAAQ;AACZ,UAAM,KAAK,SAAS,KAAK,CAAC,YAAY,QAAQ,QAAQ,CAAC;AAAA,EACzD;AACF;AAEO,MAAM,SAAS;AAAA,EACX;AAAA,EACT;AAAA,EAEA,YAAY,oBAAoB,MAAM;AACpC,SAAK,oBAAoB;AACzB,SAAK,YAAY,gCAAkB,WAAW,EAAE;AAAA,EAClD;AAAA,EAEA,iBAAiB,UAAmB;AAClC,QAAI,CAAC,SAAU,QAAO;AACtB,UAAM,QAAQ,SAAS,YAAY,EAAE,MAAM,GAAG;AAC9C,WAAO,MAAM,CAAC,MAAM,QAAQ,MAAM,CAAC,MAAM;AAAA,EAC3C;AAAA,EAEA,MAAM,iBAAiB,SAA2C;AAChE,QAAI,WAA2B,CAAC;AAEhC,eAAW,OAAO,QAAQ,UAAU;AAClC,UAAI,IAAI,SAAS,kBAAI,SAAS,aAAa,IAAI,SAAS,kBAAI,SAAS,MAAM;AACzE;AAAA,MACF;AAEA,UAAI,OAAO,IAAI,YAAY,UAAU;AACnC,iBAAS,KAAK;AAAA,UACZ,MAAM,IAAI,SAAS,kBAAI,SAAS,YAAY,cAAc;AAAA,UAC1D,SAAS,IAAI;AAAA,QACf,CAAC;AAAA,MACH,WAAW,MAAM,QAAQ,IAAI,OAAO,GAAG;AACrC,mBAAW,WAAW,IAAI,SAAS;AACjC,cAAI,OAAO,YAAY,UAAU;AAC/B,qBAAS,KAAK;AAAA,cACZ,MAAM,IAAI,SAAS,kBAAI,SAAS,YAAY,cAAc;AAAA,cAC1D;AAAA,YACF,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAAA,IACF;AACA,eAAW,SAAS,MAAM,CAAC,WAAW;AACtC,UAAM,SAAS,MAAM,KAAK,UAAU,YAAY,UAAU,kBAAkB,QAAQ;AACpF,WAAO;AAAA,EACT;AACF;AAEA,IAAO,wBAAQ;","names":[]}
|
package/dist/turn_detector.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"turn_detector.d.ts","sourceRoot":"","sources":["../src/turn_detector.ts"],"names":[],"mappings":"AAMA,OAAO,EAAqB,eAAe,EAAE,GAAG,EAAO,MAAM,iBAAiB,CAAC;AAM/E,KAAK,cAAc,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,EAAE,CAAC;AAE1D,qBAAa,SAAU,SAAQ,eAAe;;IAC5C,MAAM,CAAC,gBAAgB,SAAyB;;IAoB1C,UAAU;IAIV,GAAG,CAAC,IAAI,EAAE,cAAc,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"turn_detector.d.ts","sourceRoot":"","sources":["../src/turn_detector.ts"],"names":[],"mappings":"AAMA,OAAO,EAAqB,eAAe,EAAE,GAAG,EAAO,MAAM,iBAAiB,CAAC;AAM/E,KAAK,cAAc,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,EAAE,CAAC;AAE1D,qBAAa,SAAU,SAAQ,eAAe;;IAC5C,MAAM,CAAC,gBAAgB,SAAyB;;IAoB1C,UAAU;IAIV,GAAG,CAAC,IAAI,EAAE,cAAc,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IA+BtD,KAAK;CAGZ;AAED,qBAAa,QAAQ;;IACnB,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;gBAGvB,iBAAiB,SAAO;IAKpC,gBAAgB,CAAC,QAAQ,CAAC,EAAE,MAAM;IAM5B,gBAAgB,CAAC,OAAO,EAAE,GAAG,CAAC,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;CA4BlE;AAED,eAAe,SAAS,CAAC"}
|
package/dist/turn_detector.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/turn_detector.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { PreTrainedTokenizer } from '@huggingface/transformers';\nimport { AutoTokenizer } from '@huggingface/transformers';\nimport type { ipc } from '@livekit/agents';\nimport { CurrentJobContext, InferenceRunner, llm, log } from '@livekit/agents';\nimport { fileURLToPath } from 'node:url';\nimport { InferenceSession, Tensor } from 'onnxruntime-node';\n\nconst MAX_HISTORY = 4;\n\ntype RawChatContext = { role: string; content: string }[];\n\nexport class EOURunner extends InferenceRunner {\n static INFERENCE_METHOD = 'lk_end_of_utterance';\n #tokenizerPromise: Promise<PreTrainedTokenizer>;\n #session: Promise<InferenceSession>;\n #tokenizer?: PreTrainedTokenizer;\n #logger = log();\n\n constructor() {\n super();\n this.#tokenizerPromise = AutoTokenizer.from_pretrained('livekit/turn-detector', {\n revision: 'v1.2.0',\n // local_files_only: true, // TODO(nbsp): can't find it\n });\n this.#session = InferenceSession.create(\n fileURLToPath(new URL('turn_detector.onnx', import.meta.url).href),\n {\n executionProviders: [{ name: 'cpu' }],\n },\n );\n }\n\n async initialize() {\n this.#tokenizer = await this.#tokenizerPromise;\n }\n\n async run(data: RawChatContext): Promise<number | undefined> {\n const text = this.#formatChatContext(data);\n const startTime = Date.now();\n const inputs = this.#tokenizer!.encode(text, { add_special_tokens: false });\n const outputs = await this.#session.then((session) =>\n session.run({ input_ids: new Tensor('int64', inputs, [1, inputs.length]) }, ['prob']),\n );\n const endTime = Date.now();\n const logits = outputs.prob!;\n const eouProbability = logits.data[0] as number;\n this.#logger\n .child({ eouProbability, input: text, duration: endTime - startTime })\n .debug('eou prediction');\n return eouProbability;\n }\n\n #formatChatContext(ctx: RawChatContext): string {\n const newCtx: RawChatContext = [];\n for (const msg of ctx) {\n if (!msg.content) continue;\n newCtx.push(msg);\n }\n\n const convoText = this.#tokenizer!.apply_chat_template(newCtx, {\n add_generation_prompt: false,\n tokenize: false,\n }) as string;\n // remove EOU token from current utterance\n return convoText.slice(0, convoText.lastIndexOf('<|im_end|>'));\n }\n}\n\nexport class EOUModel {\n readonly unlikelyThreshold: number;\n #executor: ipc.InferenceExecutor;\n\n constructor(unlikelyThreshold = 0.15) {\n this.unlikelyThreshold = unlikelyThreshold;\n this.#executor = CurrentJobContext.getCurrent().inferenceExecutor;\n }\n\n supportsLanguage(language?: string) {\n if (!language) return false;\n const parts = language.toLowerCase().split('-');\n return parts[0] === 'en' || parts[0] === 'english';\n }\n\n async predictEndOfTurn(chatCtx: llm.ChatContext): Promise<number> {\n let messages: RawChatContext = [];\n\n for (const msg of chatCtx.messages) {\n if (msg.role !== llm.ChatRole.ASSISTANT && msg.role !== llm.ChatRole.USER) {\n continue;\n }\n\n if (typeof msg.content === 'string') {\n messages.push({\n role: msg.role === llm.ChatRole.ASSISTANT ? 'assistant' : 'user',\n content: msg.content,\n });\n } else if (Array.isArray(msg.content)) {\n for (const content of msg.content) {\n if (typeof content === 'string') {\n messages.push({\n role: msg.role === llm.ChatRole.ASSISTANT ? 'assistant' : 'user',\n content: content,\n });\n }\n }\n }\n }\n messages = messages.slice(-MAX_HISTORY);\n const result = await this.#executor.doInference(EOURunner.INFERENCE_METHOD, messages);\n return result as any;\n }\n}\n\nexport default EOURunner;\n"],"mappings":"AAIA,SAAS,qBAAqB;AAE9B,SAAS,mBAAmB,iBAAiB,KAAK,WAAW;AAC7D,SAAS,qBAAqB;AAC9B,SAAS,kBAAkB,cAAc;AAEzC,MAAM,cAAc;AAIb,MAAM,kBAAkB,gBAAgB;AAAA,EAC7C,OAAO,mBAAmB;AAAA,EAC1B;AAAA,EACA;AAAA,EACA;AAAA,EACA,UAAU,IAAI;AAAA,EAEd,cAAc;AACZ,UAAM;AACN,SAAK,oBAAoB,cAAc,gBAAgB,yBAAyB;AAAA,MAC9E,UAAU;AAAA;AAAA,IAEZ,CAAC;AACD,SAAK,WAAW,iBAAiB;AAAA,MAC/B,cAAc,IAAI,IAAI,sBAAsB,YAAY,GAAG,EAAE,IAAI;AAAA,MACjE;AAAA,QACE,oBAAoB,CAAC,EAAE,MAAM,MAAM,CAAC;AAAA,MACtC;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,aAAa;AACjB,SAAK,aAAa,MAAM,KAAK;AAAA,EAC/B;AAAA,EAEA,MAAM,IAAI,MAAmD;AAC3D,UAAM,OAAO,KAAK,mBAAmB,IAAI;AACzC,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,SAAS,KAAK,WAAY,OAAO,MAAM,EAAE,oBAAoB,MAAM,CAAC;AAC1E,UAAM,UAAU,MAAM,KAAK,SAAS;AAAA,MAAK,CAAC,YACxC,QAAQ,IAAI,EAAE,WAAW,IAAI,OAAO,SAAS,QAAQ,CAAC,GAAG,OAAO,MAAM,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC;AAAA,IACtF;AACA,UAAM,UAAU,KAAK,IAAI;AACzB,UAAM,SAAS,QAAQ;AACvB,UAAM,iBAAiB,OAAO,KAAK,CAAC;AACpC,SAAK,QACF,MAAM,EAAE,gBAAgB,OAAO,MAAM,UAAU,UAAU,UAAU,CAAC,EACpE,MAAM,gBAAgB;AACzB,WAAO;AAAA,EACT;AAAA,EAEA,mBAAmB,KAA6B;AAC9C,UAAM,SAAyB,CAAC;AAChC,eAAW,OAAO,KAAK;AACrB,UAAI,CAAC,IAAI,QAAS;AAClB,aAAO,KAAK,GAAG;AAAA,IACjB;AAEA,UAAM,YAAY,KAAK,WAAY,oBAAoB,QAAQ;AAAA,MAC7D,uBAAuB;AAAA,MACvB,UAAU;AAAA,IACZ,CAAC;AAED,WAAO,UAAU,MAAM,GAAG,UAAU,YAAY,YAAY,CAAC;AAAA,EAC/D;AACF;AAEO,MAAM,SAAS;AAAA,EACX;AAAA,EACT;AAAA,EAEA,YAAY,oBAAoB,MAAM;AACpC,SAAK,oBAAoB;AACzB,SAAK,YAAY,kBAAkB,WAAW,EAAE;AAAA,EAClD;AAAA,EAEA,iBAAiB,UAAmB;AAClC,QAAI,CAAC,SAAU,QAAO;AACtB,UAAM,QAAQ,SAAS,YAAY,EAAE,MAAM,GAAG;AAC9C,WAAO,MAAM,CAAC,MAAM,QAAQ,MAAM,CAAC,MAAM;AAAA,EAC3C;AAAA,EAEA,MAAM,iBAAiB,SAA2C;AAChE,QAAI,WAA2B,CAAC;AAEhC,eAAW,OAAO,QAAQ,UAAU;AAClC,UAAI,IAAI,SAAS,IAAI,SAAS,aAAa,IAAI,SAAS,IAAI,SAAS,MAAM;AACzE;AAAA,MACF;AAEA,UAAI,OAAO,IAAI,YAAY,UAAU;AACnC,iBAAS,KAAK;AAAA,UACZ,MAAM,IAAI,SAAS,IAAI,SAAS,YAAY,cAAc;AAAA,UAC1D,SAAS,IAAI;AAAA,QACf,CAAC;AAAA,MACH,WAAW,MAAM,QAAQ,IAAI,OAAO,GAAG;AACrC,mBAAW,WAAW,IAAI,SAAS;AACjC,cAAI,OAAO,YAAY,UAAU;AAC/B,qBAAS,KAAK;AAAA,cACZ,MAAM,IAAI,SAAS,IAAI,SAAS,YAAY,cAAc;AAAA,cAC1D;AAAA,YACF,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAAA,IACF;AACA,eAAW,SAAS,MAAM,CAAC,WAAW;AACtC,UAAM,SAAS,MAAM,KAAK,UAAU,YAAY,UAAU,kBAAkB,QAAQ;AACpF,WAAO;AAAA,EACT;AACF;AAEA,IAAO,wBAAQ;","names":[]}
|
|
1
|
+
{"version":3,"sources":["../src/turn_detector.ts"],"sourcesContent":["// SPDX-FileCopyrightText: 2024 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport type { PreTrainedTokenizer } from '@huggingface/transformers';\nimport { AutoTokenizer } from '@huggingface/transformers';\nimport type { ipc } from '@livekit/agents';\nimport { CurrentJobContext, InferenceRunner, llm, log } from '@livekit/agents';\nimport { fileURLToPath } from 'node:url';\nimport { InferenceSession, Tensor } from 'onnxruntime-node';\n\nconst MAX_HISTORY = 4;\n\ntype RawChatContext = { role: string; content: string }[];\n\nexport class EOURunner extends InferenceRunner {\n static INFERENCE_METHOD = 'lk_end_of_utterance';\n #tokenizerPromise: Promise<PreTrainedTokenizer>;\n #session: Promise<InferenceSession>;\n #tokenizer?: PreTrainedTokenizer;\n #logger = log();\n\n constructor() {\n super();\n this.#tokenizerPromise = AutoTokenizer.from_pretrained('livekit/turn-detector', {\n revision: 'v1.2.0',\n // local_files_only: true, // TODO(nbsp): can't find it\n });\n this.#session = InferenceSession.create(\n fileURLToPath(new URL('turn_detector.onnx', import.meta.url).href),\n {\n executionProviders: [{ name: 'cpu' }],\n },\n );\n }\n\n async initialize() {\n this.#tokenizer = await this.#tokenizerPromise;\n }\n\n async run(data: RawChatContext): Promise<number | undefined> {\n const text = this.#formatChatContext(data);\n const startTime = Date.now();\n const inputs = this.#tokenizer!.encode(text, { add_special_tokens: false });\n const outputs = await this.#session.then((session) =>\n session.run({ input_ids: new Tensor('int64', inputs, [1, inputs.length]) }, ['prob']),\n );\n const endTime = Date.now();\n const logits = outputs.prob!;\n const eouProbability = logits.data[0] as number;\n this.#logger\n .child({ eouProbability, input: text, duration: endTime - startTime })\n .debug('eou prediction');\n return eouProbability;\n }\n\n #formatChatContext(ctx: RawChatContext): string {\n const newCtx: RawChatContext = [];\n for (const msg of ctx) {\n if (!msg.content) continue;\n newCtx.push(msg);\n }\n\n const convoText = this.#tokenizer!.apply_chat_template(newCtx, {\n add_generation_prompt: false,\n tokenize: false,\n }) as string;\n // remove EOU token from current utterance\n return convoText.slice(0, convoText.lastIndexOf('<|im_end|>'));\n }\n\n async close() {\n await this.#session.then((session) => session.release());\n }\n}\n\nexport class EOUModel {\n readonly unlikelyThreshold: number;\n #executor: ipc.InferenceExecutor;\n\n constructor(unlikelyThreshold = 0.15) {\n this.unlikelyThreshold = unlikelyThreshold;\n this.#executor = CurrentJobContext.getCurrent().inferenceExecutor;\n }\n\n supportsLanguage(language?: string) {\n if (!language) return false;\n const parts = language.toLowerCase().split('-');\n return parts[0] === 'en' || parts[0] === 'english';\n }\n\n async predictEndOfTurn(chatCtx: llm.ChatContext): Promise<number> {\n let messages: RawChatContext = [];\n\n for (const msg of chatCtx.messages) {\n if (msg.role !== llm.ChatRole.ASSISTANT && msg.role !== llm.ChatRole.USER) {\n continue;\n }\n\n if (typeof msg.content === 'string') {\n messages.push({\n role: msg.role === llm.ChatRole.ASSISTANT ? 'assistant' : 'user',\n content: msg.content,\n });\n } else if (Array.isArray(msg.content)) {\n for (const content of msg.content) {\n if (typeof content === 'string') {\n messages.push({\n role: msg.role === llm.ChatRole.ASSISTANT ? 'assistant' : 'user',\n content: content,\n });\n }\n }\n }\n }\n messages = messages.slice(-MAX_HISTORY);\n const result = await this.#executor.doInference(EOURunner.INFERENCE_METHOD, messages);\n return result as any;\n }\n}\n\nexport default EOURunner;\n"],"mappings":"AAIA,SAAS,qBAAqB;AAE9B,SAAS,mBAAmB,iBAAiB,KAAK,WAAW;AAC7D,SAAS,qBAAqB;AAC9B,SAAS,kBAAkB,cAAc;AAEzC,MAAM,cAAc;AAIb,MAAM,kBAAkB,gBAAgB;AAAA,EAC7C,OAAO,mBAAmB;AAAA,EAC1B;AAAA,EACA;AAAA,EACA;AAAA,EACA,UAAU,IAAI;AAAA,EAEd,cAAc;AACZ,UAAM;AACN,SAAK,oBAAoB,cAAc,gBAAgB,yBAAyB;AAAA,MAC9E,UAAU;AAAA;AAAA,IAEZ,CAAC;AACD,SAAK,WAAW,iBAAiB;AAAA,MAC/B,cAAc,IAAI,IAAI,sBAAsB,YAAY,GAAG,EAAE,IAAI;AAAA,MACjE;AAAA,QACE,oBAAoB,CAAC,EAAE,MAAM,MAAM,CAAC;AAAA,MACtC;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,aAAa;AACjB,SAAK,aAAa,MAAM,KAAK;AAAA,EAC/B;AAAA,EAEA,MAAM,IAAI,MAAmD;AAC3D,UAAM,OAAO,KAAK,mBAAmB,IAAI;AACzC,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,SAAS,KAAK,WAAY,OAAO,MAAM,EAAE,oBAAoB,MAAM,CAAC;AAC1E,UAAM,UAAU,MAAM,KAAK,SAAS;AAAA,MAAK,CAAC,YACxC,QAAQ,IAAI,EAAE,WAAW,IAAI,OAAO,SAAS,QAAQ,CAAC,GAAG,OAAO,MAAM,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC;AAAA,IACtF;AACA,UAAM,UAAU,KAAK,IAAI;AACzB,UAAM,SAAS,QAAQ;AACvB,UAAM,iBAAiB,OAAO,KAAK,CAAC;AACpC,SAAK,QACF,MAAM,EAAE,gBAAgB,OAAO,MAAM,UAAU,UAAU,UAAU,CAAC,EACpE,MAAM,gBAAgB;AACzB,WAAO;AAAA,EACT;AAAA,EAEA,mBAAmB,KAA6B;AAC9C,UAAM,SAAyB,CAAC;AAChC,eAAW,OAAO,KAAK;AACrB,UAAI,CAAC,IAAI,QAAS;AAClB,aAAO,KAAK,GAAG;AAAA,IACjB;AAEA,UAAM,YAAY,KAAK,WAAY,oBAAoB,QAAQ;AAAA,MAC7D,uBAAuB;AAAA,MACvB,UAAU;AAAA,IACZ,CAAC;AAED,WAAO,UAAU,MAAM,GAAG,UAAU,YAAY,YAAY,CAAC;AAAA,EAC/D;AAAA,EAEA,MAAM,QAAQ;AACZ,UAAM,KAAK,SAAS,KAAK,CAAC,YAAY,QAAQ,QAAQ,CAAC;AAAA,EACzD;AACF;AAEO,MAAM,SAAS;AAAA,EACX;AAAA,EACT;AAAA,EAEA,YAAY,oBAAoB,MAAM;AACpC,SAAK,oBAAoB;AACzB,SAAK,YAAY,kBAAkB,WAAW,EAAE;AAAA,EAClD;AAAA,EAEA,iBAAiB,UAAmB;AAClC,QAAI,CAAC,SAAU,QAAO;AACtB,UAAM,QAAQ,SAAS,YAAY,EAAE,MAAM,GAAG;AAC9C,WAAO,MAAM,CAAC,MAAM,QAAQ,MAAM,CAAC,MAAM;AAAA,EAC3C;AAAA,EAEA,MAAM,iBAAiB,SAA2C;AAChE,QAAI,WAA2B,CAAC;AAEhC,eAAW,OAAO,QAAQ,UAAU;AAClC,UAAI,IAAI,SAAS,IAAI,SAAS,aAAa,IAAI,SAAS,IAAI,SAAS,MAAM;AACzE;AAAA,MACF;AAEA,UAAI,OAAO,IAAI,YAAY,UAAU;AACnC,iBAAS,KAAK;AAAA,UACZ,MAAM,IAAI,SAAS,IAAI,SAAS,YAAY,cAAc;AAAA,UAC1D,SAAS,IAAI;AAAA,QACf,CAAC;AAAA,MACH,WAAW,MAAM,QAAQ,IAAI,OAAO,GAAG;AACrC,mBAAW,WAAW,IAAI,SAAS;AACjC,cAAI,OAAO,YAAY,UAAU;AAC/B,qBAAS,KAAK;AAAA,cACZ,MAAM,IAAI,SAAS,IAAI,SAAS,YAAY,cAAc;AAAA,cAC1D;AAAA,YACF,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAAA,IACF;AACA,eAAW,SAAS,MAAM,CAAC,WAAW;AACtC,UAAM,SAAS,MAAM,KAAK,UAAU,YAAY,UAAU,kBAAkB,QAAQ;AACpF,WAAO;AAAA,EACT;AACF;AAEA,IAAO,wBAAQ;","names":[]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@livekit/agents-plugin-livekit",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.1",
|
|
4
4
|
"description": "Additional utilities for LiveKit Node Agents",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"require": "dist/index.cjs",
|
|
@@ -29,7 +29,7 @@
|
|
|
29
29
|
"typescript": "^5.0.0"
|
|
30
30
|
},
|
|
31
31
|
"peerDependencies": {
|
|
32
|
-
"@livekit/agents": "^0.7.
|
|
32
|
+
"@livekit/agents": "^0.7.5x"
|
|
33
33
|
},
|
|
34
34
|
"dependencies": {
|
|
35
35
|
"@huggingface/transformers": "^3.2.1",
|
package/src/turn_detector.ts
CHANGED
|
@@ -67,6 +67,10 @@ export class EOURunner extends InferenceRunner {
|
|
|
67
67
|
// remove EOU token from current utterance
|
|
68
68
|
return convoText.slice(0, convoText.lastIndexOf('<|im_end|>'));
|
|
69
69
|
}
|
|
70
|
+
|
|
71
|
+
async close() {
|
|
72
|
+
await this.#session.then((session) => session.release());
|
|
73
|
+
}
|
|
70
74
|
}
|
|
71
75
|
|
|
72
76
|
export class EOUModel {
|