@livekit/agents-plugin-livekit 0.1.2 → 1.0.0-next.0

package/src/hf_utils.ts

@@ -0,0 +1,365 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * Fixed version of HuggingFace's downloadFileToCacheDir that matches Python's behavior
+ *
+ * Key fix: Uses branch/tag HEAD commit for snapshot paths, not file's last commit
+ * This ensures all files from the same revision end up in the same snapshot folder
+ */
+import type { CommitInfo, PathInfo, RepoDesignation } from '@huggingface/hub';
+import { downloadFile, listCommits, pathsInfo } from '@huggingface/hub';
+import { log } from '@livekit/agents';
+import { createWriteStream, writeFileSync } from 'node:fs';
+import { lstat, mkdir, rename, stat } from 'node:fs/promises';
+import { homedir } from 'node:os';
+import { dirname, join, relative, resolve } from 'node:path';
+import { Readable } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
+import type { ReadableStream } from 'node:stream/web';
+
+// Define CredentialsParams if not exported
+interface CredentialsParams {
+  accessToken?: string;
+}
+
+export const REGEX_COMMIT_HASH: RegExp = new RegExp('^[0-9a-f]{40}$');
+
+// Helper functions from HuggingFace's cache-management
+function getHFHubCachePath(customCacheDir?: string): string {
+  return customCacheDir || join(homedir(), '.cache', 'huggingface', 'hub');
+}
+
+function getRepoFolderName(repoId: string): string {
+  return `models--${repoId.replace(/\//g, '--')}`;
+}
+
+function toRepoId(repo: RepoDesignation | string): string {
+  if (typeof repo === 'string') {
+    return repo;
+  }
+  return `${repo.name}`;
+}
+
+/**
+ * Get the HEAD commit hash for a branch/tag (matching Python's behavior)
+ */
+async function getBranchHeadCommit(
+  repo: RepoDesignation,
+  revision: string,
+  params: { accessToken?: string; hubUrl?: string; fetch?: typeof fetch },
+): Promise<string | null> {
+  const logger = log();
+
+  try {
+    // If already a commit hash, return it
+    if (REGEX_COMMIT_HASH.test(revision)) {
+      return revision;
+    }
+
+    // Get the first commit from listCommits - this is the HEAD
+    for await (const commit of listCommits({
+      repo,
+      revision,
+      ...params,
+    })) {
+      // The commit object structure varies, so we check multiple possible properties
+      const commitHash = (commit as any).oid || (commit as any).id || (commit as any).commitId;
+      if (commitHash) {
+        return commitHash;
+      }
+      break; // Only need the first one
+    }
+
+    logger.error({ repo: toRepoId(repo), revision }, 'No commits found for revision');
+    return null;
+  } catch (error) {
+    logger.error(
+      { error: (error as Error).message, repo: toRepoId(repo), revision },
+      'Error getting HEAD commit',
+    );
+    throw error;
+  }
+}
+
+/**
+ * Create a symbolic link following HuggingFace's implementation
+ */
+async function createSymlink(sourcePath: string, targetPath: string): Promise<void> {
+  const logger = log();
+  const { symlink, rm, copyFile } = await import('node:fs/promises');
+
+  // Expand ~ to home directory
+  function expandUser(path: string): string {
+    if (path.startsWith('~')) {
+      return path.replace('~', homedir());
+    }
+    return path;
+  }
+
+  const absSrc = resolve(expandUser(sourcePath));
+  const absDst = resolve(expandUser(targetPath));
+
+  // Remove existing file/symlink if it exists
+  try {
+    await rm(absDst);
+  } catch {
+    // Ignore - file might not exist
+  }
+
+  try {
+    // Create relative symlink (better for portability)
+    const relativePath = relative(dirname(absDst), absSrc);
+    await symlink(relativePath, absDst);
+    logger.debug({ source: absSrc, target: absDst, relative: relativePath }, 'Created symlink');
+  } catch (symlinkError) {
+    // Symlink failed (common on Windows without admin rights)
+    // Fall back to copying the file
+    logger.warn({ source: absSrc, target: absDst }, 'Symlink not supported, falling back to copy');
+    try {
+      await copyFile(absSrc, absDst);
+      logger.debug({ source: absSrc, target: absDst }, 'File copied successfully');
+    } catch (copyError) {
+      logger.error(
+        { error: (copyError as Error).message, source: absSrc, target: absDst },
+        'Failed to copy file',
+      );
+      // If copy also fails, throw the original symlink error
+      throw symlinkError;
+    }
+  }
+}
+
+function getFilePointer(storageFolder: string, revision: string, relativeFilename: string): string {
+  const snapshotPath = join(storageFolder, 'snapshots');
+  return join(snapshotPath, revision, relativeFilename);
+}
+
+/**
+ * handy method to check if a file exists, or the pointer of a symlinks exists
+ */
+async function exists(path: string, followSymlinks?: boolean): Promise<boolean> {
+  try {
+    if (followSymlinks) {
+      await stat(path);
+    } else {
+      await lstat(path);
+    }
+    return true;
+  } catch (err: unknown) {
+    return false;
+  }
+}
+
+async function saveRevisionMapping({
+  storageFolder,
+  revision,
+  commitHash,
+}: {
+  storageFolder: string;
+  revision: string;
+  commitHash: string;
+}): Promise<void> {
+  if (!REGEX_COMMIT_HASH.test(revision) && revision !== commitHash) {
+    const refsPath = join(storageFolder, 'refs');
+    await mkdir(refsPath, { recursive: true });
+    writeFileSync(join(refsPath, revision), commitHash);
+  }
+}
+
+/**
+ * Download a given file if it's not already present in the local cache.
+ * Matches Python's hf_hub_download behavior by using branch HEAD commits.
+ */
+export async function downloadFileToCacheDir(
+  params: {
+    repo: RepoDesignation;
+    path: string;
+    /**
+     * If true, will download the raw git file.
+     */
+    raw?: boolean;
+    /**
+     * An optional Git revision id which can be a branch name, a tag, or a commit hash.
+     * @default "main"
+     */
+    revision?: string;
+    hubUrl?: string;
+    cacheDir?: string;
+    /**
+     * Custom fetch function to use instead of the default one
+     */
+    fetch?: typeof fetch;
+    /**
+     * If true, only return cached files, don't download
+     */
+    localFileOnly?: boolean;
+  } & Partial<CredentialsParams>,
+): Promise<string> {
+  const logger = log();
+
+  // get revision provided or default to main
+  const revision = params.revision ?? 'main';
+  const cacheDir = params.cacheDir ?? getHFHubCachePath();
+  // get repo id
+  const repoId = toRepoId(params.repo);
+  // get storage folder
+  const storageFolder = join(cacheDir, getRepoFolderName(repoId));
+
+  let branchHeadCommit: string | undefined;
+
+  // if user provides a commitHash as revision, use it directly
+  if (REGEX_COMMIT_HASH.test(revision)) {
+    branchHeadCommit = revision;
+    const pointerPath = getFilePointer(storageFolder, revision, params.path);
+    if (await exists(pointerPath, true)) {
+      logger.debug(
+        { pointerPath, commitHash: branchHeadCommit },
+        'File found in cache (commit hash)',
+      );
+      return pointerPath;
+    }
+  }
+
+  // If localFileOnly, check cache without making API calls
+  if (params.localFileOnly) {
+    logger.debug({ repoId, path: params.path, revision }, 'Local file only mode - checking cache');
+
+    // Check with revision as-is (in case it's a commit hash)
+    const directPath = getFilePointer(storageFolder, revision, params.path);
+    if (await exists(directPath, true)) {
+      logger.debug({ directPath }, 'File found in cache (direct path)');
+      return directPath;
+    }
+
+    // If revision is not a commit hash, try to resolve from refs
+    if (!REGEX_COMMIT_HASH.test(revision)) {
+      const refsPath = join(storageFolder, 'refs', revision);
+      try {
+        const { readFileSync } = await import('fs');
+        const resolvedHash = readFileSync(refsPath, 'utf-8').trim();
+        const resolvedPath = getFilePointer(storageFolder, resolvedHash, params.path);
+        if (await exists(resolvedPath, true)) {
+          logger.debug({ resolvedPath, resolvedHash }, 'File found in cache (via refs)');
+          return resolvedPath;
+        }
+      } catch {
+        logger.debug({ revision }, 'No ref mapping found for revision');
+      }
+    }
+
+    const error = `File not found in cache: ${repoId}/${params.path} (revision: ${revision}). Make sure to run the download-files command before running the agent worker.`;
+    logger.error({ repoId, path: params.path, revision }, error);
+    throw new Error(error);
+  }
+
+  // Get the branch HEAD commit if not already a commit hash
+  if (!branchHeadCommit) {
+    const headCommit = await getBranchHeadCommit(params.repo, revision, params);
+    if (!headCommit) {
+      throw new Error(`Failed to resolve revision ${revision} to commit hash`);
+    }
+    branchHeadCommit = headCommit;
+  }
+
+  // Check if file exists with the branch HEAD commit
+  const pointerPath = getFilePointer(storageFolder, branchHeadCommit, params.path);
+  if (await exists(pointerPath, true)) {
+    logger.debug({ pointerPath, branchHeadCommit }, 'File found in cache (branch HEAD)');
+
+    await saveRevisionMapping({
+      storageFolder,
+      revision,
+      commitHash: branchHeadCommit,
+    });
+
+    return pointerPath;
+  }
+
+  // Now get file metadata to download it
+  logger.debug(
+    { repoId, path: params.path, revision: branchHeadCommit },
+    'Fetching path info from HF API',
+  );
+  const pathsInformation: (PathInfo & { lastCommit: CommitInfo })[] = await pathsInfo({
+    ...params,
+    paths: [params.path],
+    revision: branchHeadCommit, // Use HEAD commit for consistency
+    expand: true,
+  });
+
+  if (!pathsInformation || pathsInformation.length !== 1) {
+    const error = `cannot get path info for ${params.path}`;
+    logger.error({ repoId, path: params.path, pathsInfoLength: pathsInformation?.length }, error);
+    throw new Error(error);
+  }
+
+  const pathInfo = pathsInformation[0];
+  if (!pathInfo) {
+    const error = `No path info returned for ${params.path}`;
+    logger.error({ repoId, path: params.path }, error);
+    throw new Error(error);
+  }
+
+  let etag: string;
+  if (pathInfo.lfs) {
+    etag = pathInfo.lfs.oid; // get the LFS pointed file oid
+    logger.debug({ etag, path: params.path }, 'File is LFS pointer');
+  } else {
+    etag = pathInfo.oid; // get the repo file if not a LFS pointer
+    logger.debug({ etag, path: params.path }, 'File is regular git object');
+  }
+
+  const blobPath = join(storageFolder, 'blobs', etag);
+
+  logger.debug({ branchHeadCommit, pointerPath, blobPath }, 'Computed cache paths');
+
+  // mkdir blob and pointer path parent directory
+  await mkdir(dirname(blobPath), { recursive: true });
+  await mkdir(dirname(pointerPath), { recursive: true });
+
+  // We might already have the blob but not the pointer
+  // shortcut the download if needed
+  if (await exists(blobPath)) {
+    logger.debug({ blobPath, etag }, 'Blob already exists in cache, creating symlink only');
+    // create symlinks in snapshot folder to blob object
+    await createSymlink(blobPath, pointerPath);
+    return pointerPath;
+  }
+
+  const incomplete = `${blobPath}.incomplete`;
+  logger.debug({ path: params.path, incomplete }, 'Starting file download');
+
+  // Use enhanced download with retry - use branch HEAD commit for download
+  const blob: Blob | null = await downloadFile({
+    ...params,
+    revision: branchHeadCommit,
+  });
+
+  if (!blob) {
+    const error = `invalid response for file ${params.path}`;
+    logger.error({ path: params.path }, error);
+    throw new Error(error);
+  }
+
+  logger.debug({ size: blob.size }, 'Writing blob to disk');
+  await pipeline(Readable.fromWeb(blob.stream() as ReadableStream), createWriteStream(incomplete));
+
+  // rename .incomplete file to expected blob
+  await rename(incomplete, blobPath);
+  logger.debug({ blobPath }, 'Renamed incomplete file to final blob');
+
+  // create symlinks in snapshot folder to blob object
+  await createSymlink(blobPath, pointerPath);
+  logger.debug({ blobPath, pointerPath }, 'Created symlink from snapshot to blob');
+
+  await saveRevisionMapping({
+    storageFolder,
+    revision,
+    commitHash: branchHeadCommit,
+  });
+
+  logger.debug({ pointerPath, size: blob.size }, 'File download completed successfully');
+  return pointerPath;
+}

package/src/index.ts

@@ -1,11 +1,34 @@
-// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
-import { InferenceRunner } from '@livekit/agents';
-import * as turnDetector from './turn_detector.js';
-
-InferenceRunner.registerRunner(
-  turnDetector.EOURunner.INFERENCE_METHOD,
-  new URL('./turn_detector.js', import.meta.url).toString(),
-);
-export { turnDetector };
+import { Plugin } from '@livekit/agents';
+import { downloadFileToCacheDir as hfDownload } from './hf_utils.js';
+import { HG_MODEL_REPO, MODEL_REVISIONS, ONNX_FILEPATH } from './turn_detector/constants.js';
+
+export { downloadFileToCacheDir } from './hf_utils.js';
+export * as turnDetector from './turn_detector/index.js';
+
+class EOUPlugin extends Plugin {
+  constructor() {
+    super({
+      title: 'turn-detector',
+      version: '0.1.1',
+      package: '@livekit/agents-plugin-livekit',
+    });
+  }
+
+  async downloadFiles(): Promise<void> {
+    const { AutoTokenizer } = await import('@huggingface/transformers');
+
+    for (const revision of Object.values(MODEL_REVISIONS)) {
+      // Ensure tokenizer is cached
+      await AutoTokenizer.from_pretrained(HG_MODEL_REPO, { revision });
+
+      // Ensure ONNX model and language data are cached
+      await hfDownload({ repo: HG_MODEL_REPO, path: ONNX_FILEPATH, revision });
+      await hfDownload({ repo: HG_MODEL_REPO, path: 'languages.json', revision });
+    }
+  }
+}
+
+Plugin.registerPlugin(new EOUPlugin());

package/src/turn_detector/base.ts

@@ -0,0 +1,238 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { type PreTrainedTokenizer } from '@huggingface/transformers';
+import type { ipc, llm } from '@livekit/agents';
+import { CurrentJobContext, Future, InferenceRunner, log } from '@livekit/agents';
+import { readFileSync } from 'node:fs';
+import os from 'node:os';
+import { InferenceSession, Tensor } from 'onnxruntime-node';
+import { downloadFileToCacheDir } from '../hf_utils.js';
+import {
+  type EOUModelType,
+  HG_MODEL_REPO,
+  MAX_HISTORY_TURNS,
+  MODEL_REVISIONS,
+  ONNX_FILEPATH,
+} from './constants.js';
+import { normalizeText } from './utils.js';
+
+type RawChatItem = { role: string; content: string };
+
+type EOUOutput = { eouProbability: number; input: string; duration: number };
+
+export abstract class EOURunnerBase extends InferenceRunner<RawChatItem[], EOUOutput> {
+  private modelType: EOUModelType;
+  private modelRevision: string;
+
+  private session?: InferenceSession;
+  private tokenizer?: PreTrainedTokenizer;
+
+  #logger = log();
+
+  constructor(modelType: EOUModelType) {
+    super();
+    this.modelType = modelType;
+    this.modelRevision = MODEL_REVISIONS[modelType];
+  }
+
+  async initialize() {
+    const { AutoTokenizer } = await import('@huggingface/transformers');
+
+    const onnxModelPath = await downloadFileToCacheDir({
+      repo: HG_MODEL_REPO,
+      path: ONNX_FILEPATH,
+      revision: this.modelRevision,
+      localFileOnly: true,
+    });
+
+    try {
+      // TODO(brian): support session config once onnxruntime-node supports it
+      const sessOptions: InferenceSession.SessionOptions = {
+        intraOpNumThreads: Math.max(1, Math.floor(os.cpus().length / 2)),
+        interOpNumThreads: 1,
+        executionProviders: [{ name: 'cpu' }],
+      };
+
+      this.session = await InferenceSession.create(onnxModelPath, sessOptions);
+
+      this.tokenizer = await AutoTokenizer.from_pretrained('livekit/turn-detector', {
+        revision: this.modelRevision,
+        local_files_only: true,
+      });
+    } catch (e) {
+      throw new Error(
+        `agents-plugins-livekit failed to initialize ${this.modelType} EOU turn detector: ${e}`,
+      );
+    }
+  }
+
+  async run(data: RawChatItem[]) {
+    const startTime = Date.now();
+
+    const text = this.formatChatCtx(data);
+
+    const inputs = this.tokenizer!.encode(text, { add_special_tokens: false });
+    this.#logger.debug({ inputs: JSON.stringify(inputs), text }, 'EOU inputs');
+
+    const outputs = await this.session!.run(
+      { input_ids: new Tensor('int64', inputs, [1, inputs.length]) },
+      ['prob'],
+    );
+
+    const probData = outputs.prob!.data;
+    // should be the logits of the last token
+    const eouProbability = probData[probData.length - 1] as number;
+    const endTime = Date.now();
+
+    const result = {
+      eouProbability,
+      input: text,
+      duration: (endTime - startTime) / 1000,
+    };
+
+    this.#logger.child({ result }).debug('eou prediction');
+    return result;
+  }
+
+  async close() {
+    await this.session?.release();
+  }
+
+  private formatChatCtx(chatCtx: RawChatItem[]): string {
+    const newChatCtx: RawChatItem[] = [];
+    let lastMsg: RawChatItem | undefined = undefined;
+
+    for (const msg of chatCtx) {
+      const content = msg.content;
+      if (!content) continue;
+
+      const norm = normalizeText(content);
+
+      // need to combine adjacent turns together to match training data
+      if (lastMsg !== undefined && lastMsg.role === msg.role) {
+        lastMsg.content += ` ${norm}`;
+      } else {
+        newChatCtx.push({ role: msg.role, content: norm });
+        lastMsg = newChatCtx[newChatCtx.length - 1]!;
+      }
+    }
+
+    // TODO(brian): investigate add_special_tokens options
+    const convoText = this.tokenizer!.apply_chat_template(newChatCtx, {
+      add_generation_prompt: false,
+      tokenize: false,
+    }) as string;
+
+    // remove the EOU token from current utterance
+    return convoText.slice(0, convoText.lastIndexOf('<|im_end|>'));
+  }
+}
+
+export interface EOUModelOptions {
+  modelType: EOUModelType;
+  executor?: ipc.InferenceExecutor;
+  unlikelyThreshold?: number;
+  loadLanguages?: boolean;
+}
+
+type LanguageData = {
+  threshold: number;
+};
+
+export abstract class EOUModel {
+  private modelType: EOUModelType;
+  private executor: ipc.InferenceExecutor;
+  private threshold: number | undefined;
+  private loadLanguages: boolean;
+
+  protected languagesFuture: Future<Record<string, LanguageData>> = new Future();
+
+  #logger = log();
+
+  constructor(opts: EOUModelOptions) {
+    const {
+      modelType = 'en',
+      executor = CurrentJobContext.getCurrent().inferenceExecutor,
+      unlikelyThreshold,
+      loadLanguages = true,
+    } = opts;
+
+    this.modelType = modelType;
+    this.executor = executor;
+    this.threshold = unlikelyThreshold;
+    this.loadLanguages = loadLanguages;
+
+    if (loadLanguages) {
+      downloadFileToCacheDir({
+        repo: HG_MODEL_REPO,
+        path: 'languages.json',
+        revision: MODEL_REVISIONS[modelType],
+        localFileOnly: true,
+      }).then((path) => {
+        this.languagesFuture.resolve(JSON.parse(readFileSync(path, 'utf8')));
+      });
+    }
+  }
+
+  async unlikelyThreshold(language?: string): Promise<number | undefined> {
+    if (language === undefined) {
+      return this.threshold;
+    }
+
+    const lang = language.toLowerCase();
+    const languages = await this.languagesFuture.await;
+
+    // try the full language code first
+    let langData = languages[lang];
+
+    if (langData === undefined && lang.includes('-')) {
+      const baseLang = lang.split('-')[0]!;
+      langData = languages[baseLang];
+    }
+
+    if (langData === undefined) {
+      this.#logger.warn(`Language ${language} not supported by EOU model`);
+      return undefined;
+    }
+
+    // if a custom threshold is provided, use it
+    return this.threshold !== undefined ? this.threshold : langData.threshold;
+  }
+
+  async supportsLanguage(language?: string): Promise<boolean> {
+    return (await this.unlikelyThreshold(language)) !== undefined;
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  async predictEndOfTurn(chatCtx: llm.ChatContext, timeout: number = 3): Promise<number> {
+    let messages: RawChatItem[] = [];
+
+    for (const message of chatCtx.items) {
+      // skip system and developer messages or tool call messages
+      if (message.type !== 'message' || message.role in ['system', 'developer']) {
+        continue;
+      }
+
+      for (const content of message.content) {
+        if (typeof content === 'string') {
+          messages.push({
+            role: message.role === 'assistant' ? 'assistant' : 'user',
+            content: content,
+          });
+        }
+      }
+    }
+
+    messages = messages.slice(-MAX_HISTORY_TURNS);
+
+    const result = await this.executor.doInference(this.inferenceMethod(), messages);
+    if (result === undefined) {
+      throw new Error('EOU inference should always returns a result');
+    }
+
+    return (result as EOUOutput).eouProbability;
+  }
+
+  abstract inferenceMethod(): string;
+}

package/src/turn_detector/constants.ts

@@ -0,0 +1,16 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+export type EOUModelType = 'en' | 'multilingual';
+
+export const MAX_HISTORY_TOKENS = 128;
+export const MAX_HISTORY_TURNS = 6;
+
+export const MODEL_REVISIONS: Record<EOUModelType, string> = {
+  en: 'v1.2.2-en',
+  multilingual: 'v0.3.0-intl',
+};
+
+export const HG_MODEL_REPO = 'livekit/turn-detector';
+
+export const ONNX_FILEPATH = 'onnx/model_q8.onnx';

package/src/turn_detector/english.ts

@@ -0,0 +1,27 @@
+// SPDX-FileCopyrightText: 2025 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { EOUModel, EOURunnerBase } from './base.js';
+
+export const INFERENCE_METHOD_EN = 'lk_end_of_utterance_en';
+
+export class EOURunnerEn extends EOURunnerBase {
+  constructor() {
+    super('en');
+  }
+}
+
+export class EnglishModel extends EOUModel {
+  constructor(unlikelyThreshold?: number) {
+    super({
+      modelType: 'en',
+      unlikelyThreshold,
+    });
+  }
+
+  inferenceMethod(): string {
+    return INFERENCE_METHOD_EN;
+  }
+}
+
+export default EOURunnerEn;