optave-codegraph 3.13.0

package/src/domain/queries.ts

@@ -0,0 +1,50 @@
+/**
+ * queries.ts — Barrel re-export file.
+ *
+ * All query logic lives in the sub-modules under src/analysis/ and src/shared/.
+ * This file exists purely for backward compatibility so that all existing
+ * importers continue to work without changes.
+ */
+
+// ── Re-export from dedicated module for backward compat ───────────────────
+export { isTestFile, TEST_PATTERN } from '../infrastructure/test-filter.js';
+export { diffImpactMermaid } from '../presentation/diff-impact-mermaid.js';
+export { iterListFunctions, iterRoles, iterWhere } from '../shared/generators.js';
+// ── Kind/edge constants (canonical source: kinds.js) ─────────────────────
+export {
+  ALL_SYMBOL_KINDS,
+  CORE_EDGE_KINDS,
+  CORE_SYMBOL_KINDS,
+  EVERY_EDGE_KIND,
+  EVERY_SYMBOL_KIND,
+  EXTENDED_SYMBOL_KINDS,
+  STRUCTURAL_EDGE_KINDS,
+  VALID_ROLES,
+} from '../shared/kinds.js';
+// ── Shared utilities ─────────────────────────────────────────────────────
+export { kindIcon, normalizeSymbol, toSymbolRef } from '../shared/normalize.js';
+export { briefData } from './analysis/brief.js';
+export { contextData, explainData } from './analysis/context.js';
+export { fileDepsData, filePathData, fnDepsData, pathData } from './analysis/dependencies.js';
+export { exportsData } from './analysis/exports.js';
+export {
+  diffImpactData,
+  fnImpactData,
+  impactAnalysisData,
+} from './analysis/impact.js';
+export { implementationsData, interfacesData } from './analysis/implementations.js';
+export {
+  FALSE_POSITIVE_CALLER_THRESHOLD,
+  FALSE_POSITIVE_NAMES,
+  moduleMapData,
+  statsData,
+} from './analysis/module-map.js';
+export { dynamicCallsData, rolesData } from './analysis/roles.js';
+// ── Analysis modules ─────────────────────────────────────────────────────
+export {
+  childrenData,
+  findMatchingNodes,
+  listFunctionsData,
+  queryNameData,
+  whereData,
+} from './analysis/symbol-lookup.js';

package/src/domain/search/generator.ts

@@ -0,0 +1,260 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { closeDb, findDbPath, getBuildMeta, openDb } from '../../db/index.js';
+import { warn } from '../../infrastructure/logger.js';
+import { DbError } from '../../shared/errors.js';
+import type { BetterSqlite3Database, NodeRow } from '../../types.js';
+import { type EmbedOptions, embed, getModelConfig } from './models.js';
+import { buildSourceText } from './strategies/source.js';
+import { buildStructuredText } from './strategies/structured.js';
+
+type EmbeddingNode = NodeRow & { id: number };
+type EmbeddingStrategy = 'structured' | 'source';
+
+interface PreparedEmbeddings {
+  texts: string[];
+  nodeIds: number[];
+  nodeNames: string[];
+  previews: string[];
+  overflowCount: number;
+  filesRead: number;
+  filesSkipped: number;
+}
+
+/**
+ * Rough token estimate (~4 chars per token for code/English).
+ * Conservative — avoids adding a tokenizer dependency.
+ */
+export function estimateTokens(text: string): number {
+  return Math.ceil(text.length / 4);
+}
+
+function initEmbeddingsSchema(db: BetterSqlite3Database): void {
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS embeddings (
+      node_id INTEGER PRIMARY KEY,
+      vector BLOB NOT NULL,
+      text_preview TEXT,
+      FOREIGN KEY(node_id) REFERENCES nodes(id)
+    );
+    CREATE TABLE IF NOT EXISTS embedding_meta (
+      key TEXT PRIMARY KEY,
+      value TEXT
+    );
+  `);
+
+  // Add full_text column (idempotent — ignore if already exists)
+  try {
+    db.exec('ALTER TABLE embeddings ADD COLUMN full_text TEXT');
+  } catch {
+    /* column already exists */
+  }
+
+  // FTS5 virtual table for BM25 keyword search
+  db.exec(`
+    CREATE VIRTUAL TABLE IF NOT EXISTS fts_index USING fts5(
+      name,
+      content,
+      tokenize='unicode61'
+    );
+  `);
+}
+
+/**
+ * Resolve the repo root for embedding. Prefer the root recorded at build time;
+ * fall back to `<dbParent>` only when the DB lives at the conventional
+ * `<root>/.codegraph/graph.db` layout — otherwise trust the caller's rootDir.
+ */
+function resolveRoot(db: BetterSqlite3Database, dbPath: string, rootDir: string): string {
+  const metaRoot = getBuildMeta(db, 'root_dir');
+  const resolvedDbPath = path.resolve(dbPath);
+  const dbDirName = path.basename(path.dirname(resolvedDbPath));
+  const dbParent =
+    dbDirName === '.codegraph' ? path.dirname(path.dirname(resolvedDbPath)) : undefined;
+  return metaRoot || dbParent || rootDir;
+}
+
+/** Reset embedding tables and load eligible symbols grouped by file. */
+function loadNodesByFile(db: BetterSqlite3Database): Map<string, EmbeddingNode[]> {
+  db.exec('DELETE FROM embeddings');
+  db.exec('DELETE FROM embedding_meta');
+  db.exec('DELETE FROM fts_index');
+
+  const nodes = db
+    .prepare(
+      `SELECT * FROM nodes WHERE kind IN ('function', 'method', 'class') ORDER BY file, line`,
+    )
+    .all() as EmbeddingNode[];
+
+  const byFile = new Map<string, EmbeddingNode[]>();
+  for (const node of nodes) {
+    if (!byFile.has(node.file)) byFile.set(node.file, []);
+    byFile.get(node.file)?.push(node);
+  }
+  return byFile;
+}
+
+/** Build embedding text for a single node, truncating if it would overflow. */
+function buildNodeText(
+  node: EmbeddingNode,
+  file: string,
+  lines: string[],
+  db: BetterSqlite3Database,
+  strategy: EmbeddingStrategy,
+  contextWindow: number,
+): { text: string; overflowed: boolean } {
+  let text =
+    strategy === 'structured'
+      ? buildStructuredText(node, file, lines, db)
+      : buildSourceText(node, file, lines);
+  const tokens = estimateTokens(text);
+  if (tokens > contextWindow) {
+    text = text.slice(0, contextWindow * 4);
+    return { text, overflowed: true };
+  }
+  return { text, overflowed: false };
+}
+
+/**
+ * Walk files in the graph, read source, and produce parallel arrays of
+ * texts / nodeIds / nodeNames / previews ready for embedding.
+ */
+function prepareEmbeddingTexts(
+  byFile: Map<string, EmbeddingNode[]>,
+  db: BetterSqlite3Database,
+  resolvedRoot: string,
+  strategy: EmbeddingStrategy,
+  contextWindow: number,
+): PreparedEmbeddings {
+  const texts: string[] = [];
+  const nodeIds: number[] = [];
+  const nodeNames: string[] = [];
+  const previews: string[] = [];
+  let overflowCount = 0;
+  let filesRead = 0;
+  let filesSkipped = 0;
+
+  for (const [file, fileNodes] of byFile) {
+    const fullPath = path.isAbsolute(file) ? file : path.join(resolvedRoot, file);
+    let lines: string[];
+    try {
+      lines = fs.readFileSync(fullPath, 'utf-8').split('\n');
+      filesRead++;
+    } catch (err: unknown) {
+      filesSkipped++;
+      warn(`Cannot read ${file} for embeddings: ${(err as Error).message}`);
+      continue;
+    }
+
+    for (const node of fileNodes) {
+      const { text, overflowed } = buildNodeText(node, file, lines, db, strategy, contextWindow);
+      if (overflowed) overflowCount++;
+      texts.push(text);
+      nodeIds.push(node.id);
+      nodeNames.push(node.name);
+      previews.push(`${node.name} (${node.kind}) -- ${file}:${node.line}`);
+    }
+  }
+
+  return { texts, nodeIds, nodeNames, previews, overflowCount, filesRead, filesSkipped };
+}
+
+/** Persist vectors, FTS rows, and embedding metadata in a single transaction. */
+function persistEmbeddings(
+  db: BetterSqlite3Database,
+  prepared: PreparedEmbeddings,
+  vectors: Float32Array[],
+  dim: number,
+  modelName: string,
+  strategy: EmbeddingStrategy,
+): void {
+  const { nodeIds, nodeNames, previews, texts, overflowCount } = prepared;
+  const insert = db.prepare(
+    'INSERT OR REPLACE INTO embeddings (node_id, vector, text_preview, full_text) VALUES (?, ?, ?, ?)',
+  );
+  const insertFts = db.prepare('INSERT INTO fts_index(rowid, name, content) VALUES (?, ?, ?)');
+  const insertMeta = db.prepare('INSERT OR REPLACE INTO embedding_meta (key, value) VALUES (?, ?)');
+  const insertAll = db.transaction(() => {
+    for (let i = 0; i < vectors.length; i++) {
+      const vec = vectors[i] as Float32Array;
+      insert.run(nodeIds[i], Buffer.from(vec.buffer), previews[i], texts[i]);
+      insertFts.run(nodeIds[i], nodeNames[i], texts[i]);
+    }
+    insertMeta.run('model', modelName);
+    insertMeta.run('dim', String(dim));
+    insertMeta.run('count', String(vectors.length));
+    insertMeta.run('fts_count', String(vectors.length));
+    insertMeta.run('strategy', strategy);
+    insertMeta.run('built_at', new Date().toISOString());
+    if (overflowCount > 0) {
+      insertMeta.run('truncated_count', String(overflowCount));
+    }
+  });
+  insertAll();
+}
+
+export interface BuildEmbeddingsOptions extends EmbedOptions {
+  strategy?: EmbeddingStrategy;
+}
+
+/**
+ * Build embeddings for all functions/methods/classes in the graph.
+ */
+export async function buildEmbeddings(
+  rootDir: string,
+  modelKey: string,
+  customDbPath?: string,
+  options: BuildEmbeddingsOptions = {},
+): Promise<void> {
+  const { strategy = 'structured', ...embOpts } = options;
+  //const strategy = options.strategy || 'structured';
+  const dbPath = customDbPath || findDbPath(undefined);
+
+  if (!fs.existsSync(dbPath)) {
+    throw new DbError(
+      `No codegraph database found at ${dbPath}.\nRun "codegraph build" first to analyze your codebase.`,
+      { file: dbPath },
+    );
+  }
+
+  const db = openDb(dbPath) as BetterSqlite3Database;
+  initEmbeddingsSchema(db);
+
+  const resolvedRoot = resolveRoot(db, dbPath, rootDir);
+  const byFile = loadNodesByFile(db);
+
+  const nodeCount = [...byFile.values()].reduce((acc, list) => acc + list.length, 0);
+  console.log(`Building embeddings for ${nodeCount} symbols (strategy: ${strategy})...`);
+
+  const config = getModelConfig(modelKey);
+  const prepared = prepareEmbeddingTexts(byFile, db, resolvedRoot, strategy, config.contextWindow);
+
+  if (prepared.overflowCount > 0) {
+    warn(
+      `${prepared.overflowCount} symbol(s) exceeded model context window (${config.contextWindow} tokens) and were truncated`,
+    );
+  }
+
+  // If there were symbols to embed but every file failed to read, the DB was
+  // almost certainly built from a different location than the current cwd.
+  // Surface this clearly instead of emitting a silent "Stored 0 embeddings".
+  if (byFile.size > 0 && prepared.filesRead === 0) {
+    closeDb(db);
+    throw new DbError(
+      `embed: could not read any of the ${prepared.filesSkipped} source files recorded in the graph — the DB may have been built from a different location than the current working directory.\n` +
+        `Tried resolving against: ${resolvedRoot}\n` +
+        'Pass a positional <dir> argument pointing at the original repo root, or re-run "codegraph build" from that directory.',
+      { file: dbPath },
+    );
+  }
+
+  console.log(`Embedding ${prepared.texts.length} symbols...`);
+  const { vectors, dim } = await embed(prepared.texts, modelKey, embOpts);
+
+  persistEmbeddings(db, prepared, vectors as Float32Array[], dim, config.name, strategy);
+
+  console.log(
+    `\nStored ${vectors.length} embeddings (${dim}d, ${config.name}, strategy: ${strategy}) in graph.db`,
+  );
+  closeDb(db);
+}

package/src/domain/search/index.ts

@@ -0,0 +1,15 @@
+/**
+ * Embeddings subsystem — public API barrel.
+ *
+ * Re-exports everything consumers previously imported from `../embedder.js`.
+ */
+
+export type { BuildEmbeddingsOptions } from './generator.js';
+export { buildEmbeddings, estimateTokens } from './generator.js';
+export type { EmbedOptions, ModelConfig } from './models.js';
+export { DEFAULT_MODEL, disposeModel, EMBEDDING_STRATEGIES, embed, MODELS } from './models.js';
+export { search } from './search/cli-formatter.js';
+export { hybridSearchData } from './search/hybrid.js';
+export { ftsSearchData } from './search/keyword.js';
+export { multiSearchData, searchData } from './search/semantic.js';
+export { cosineSim } from './stores/sqlite-blob.js';

package/src/domain/search/models.ts

@@ -0,0 +1,356 @@
+import { execFileSync } from 'node:child_process';
+import { createRequire } from 'node:module';
+import path from 'node:path';
+import { createInterface } from 'node:readline';
+import { setTimeout } from 'node:timers/promises';
+import { info } from '../../infrastructure/logger.js';
+import { ConfigError, EngineError } from '../../shared/errors.js';
+
+const _require = createRequire(import.meta.url);
+
+/**
+ * Resolve the directory where `npm install` should run so the installed
+ * package ends up reachable by `await import(pkg)` from inside this module.
+ *
+ * Without a `cwd`, `execFileSync('npm', ['install', ...])` operates on
+ * `process.cwd()` — when the user runs codegraph against a repo that is *not*
+ * the directory where codegraph itself is installed, npm installs into the
+ * wrong `node_modules`, the dynamic import still fails, and the user gets
+ * `ENGINE_UNAVAILABLE: ... installed but failed to load`.
+ *
+ * Pin cwd to the directory that contains @optave/codegraph's `node_modules`
+ * so the install lands where Node's resolution algorithm will find it.
+ *
+ * @internal Exported for unit tests; not part of the public barrel.
+ */
+export function resolveNpmInstallCwd(): string | undefined {
+  try {
+    const pkgJsonPath = _require.resolve('optave-codegraph/package.json');
+    // pkgJsonPath = <host>/node_modules/optave-codegraph/package.json
+    // dirname x4: package.json → codegraph → @optave → node_modules → <host>
+    return path.dirname(path.dirname(path.dirname(path.dirname(pkgJsonPath))));
+  } catch {
+    // Source-of-truth checkout (no @optave/codegraph in node_modules) — fall back
+    // to process.cwd() so legacy behavior survives in tests.
+    return undefined;
+  }
+}
+
+export interface ModelConfig {
+  name: string;
+  dim: number;
+  contextWindow: number;
+  desc: string;
+  quantized: boolean;
+  /** Pooling strategy passed to the transformers pipeline. Defaults to 'mean'. */
+  pooling?: 'mean' | 'cls';
+}
+
+// Lazy-load transformers (heavy, optional module)
+let pipeline: unknown = null;
+let extractor: null | {
+  dispose(): Promise<void>;
+  (batch: string[], opts: Record<string, unknown>): Promise<{ data: number[] }>;
+} = null;
+let activeModel: string | null = null;
+
+export const MODELS: Record<string, ModelConfig> = {
+  minilm: {
+    name: 'Xenova/all-MiniLM-L6-v2',
+    dim: 384,
+    contextWindow: 256,
+    desc: 'Smallest, fastest (~23MB). General text.',
+    quantized: true,
+  },
+  'jina-small': {
+    name: 'Xenova/jina-embeddings-v2-small-en',
+    dim: 512,
+    contextWindow: 8192,
+    desc: 'Small, good quality (~33MB). General text.',
+    quantized: false,
+  },
+  'jina-base': {
+    name: 'Xenova/jina-embeddings-v2-base-en',
+    dim: 768,
+    contextWindow: 8192,
+    desc: 'Good quality (~137MB). General text, 8192 token context.',
+    quantized: false,
+  },
+  'jina-code': {
+    name: 'jinaai/jina-embeddings-v2-base-code',
+    dim: 768,
+    contextWindow: 8192,
+    desc: 'Code-aware (~137MB). Trained on code+text, best for code search.',
+    quantized: false,
+  },
+  nomic: {
+    name: 'Xenova/nomic-embed-text-v1',
+    dim: 768,
+    contextWindow: 8192,
+    desc: 'Good local quality (~137MB). 8192 context.',
+    quantized: false,
+  },
+  'nomic-v1.5': {
+    name: 'nomic-ai/nomic-embed-text-v1.5',
+    dim: 768,
+    contextWindow: 8192,
+    desc: 'Matryoshka MRL trained (~137MB). 8192 context. Codegraph stores full 768d (no truncation); v1 scores higher on our benchmark.',
+    quantized: false,
+  },
+  'bge-large': {
+    name: 'Xenova/bge-large-en-v1.5',
+    dim: 1024,
+    contextWindow: 512,
+    desc: 'Best general retrieval (~335MB). Top MTEB scores.',
+    quantized: false,
+  },
+  'mxbai-xsmall': {
+    name: 'mixedbread-ai/mxbai-embed-xsmall-v1',
+    dim: 384,
+    contextWindow: 4096,
+    desc: 'Tiny model with long context (~50MB). 4096 ctx.',
+    quantized: false,
+    pooling: 'cls',
+  },
+  'mxbai-large': {
+    name: 'mixedbread-ai/mxbai-embed-large-v1',
+    dim: 1024,
+    contextWindow: 512,
+    desc: 'Top MTEB BERT-large, Matryoshka dimensions (~400MB). 512 ctx.',
+    quantized: false,
+    pooling: 'cls',
+  },
+  'bge-m3': {
+    name: 'Xenova/bge-m3',
+    dim: 1024,
+    contextWindow: 8192,
+    desc: 'Multilingual, multi-task (~600MB). 100+ languages, 8192 context.',
+    quantized: false,
+  },
+  modernbert: {
+    name: 'nomic-ai/modernbert-embed-base',
+    dim: 768,
+    contextWindow: 8192,
+    desc: 'ModernBERT base (~150MB). Newer architecture, 8192 ctx, English.',
+    quantized: false,
+  },
+};
+
+export const EMBEDDING_STRATEGIES: readonly string[] = ['structured', 'source'];
+
+export const DEFAULT_MODEL: string = 'nomic';
+const NPM_BIN = process.platform === 'win32' ? 'npm.cmd' : 'npm';
+const BATCH_SIZE_MAP: Record<string, number> = {
+  minilm: 32,
+  'jina-small': 16,
+  'jina-base': 8,
+  'jina-code': 8,
+  nomic: 8,
+  'nomic-v1.5': 8,
+  'bge-large': 4,
+  'mxbai-xsmall': 32,
+  'mxbai-large': 4,
+  'bge-m3': 4,
+  modernbert: 8,
+};
+const DEFAULT_BATCH_SIZE = 32;
+
+/** @internal Used by generator.js — not part of the public barrel. */
+export function getModelConfig(modelKey?: string): ModelConfig {
+  const key = modelKey || DEFAULT_MODEL;
+  const config = MODELS[key];
+  if (!config) {
+    throw new ConfigError(`Unknown model: ${key}. Available: ${Object.keys(MODELS).join(', ')}`);
+  }
+  return config;
+}
+
+/**
+ * Attempt to install a missing package.
+ * In TTY environments, prompts the user for confirmation first.
+ * In non-TTY environments (CI, piped stdin), installs automatically with a log message.
+ * Returns true if the package was installed, false otherwise.
+ * @internal Not part of the public barrel.
+ */
+export function promptInstall(packageName: string): Promise<boolean> {
+  const installCwd = resolveNpmInstallCwd();
+  if (!process.stdin.isTTY) {
+    info(`Installing ${packageName} (optional dependency for semantic search)…`);
+    try {
+      execFileSync(NPM_BIN, ['install', '--no-save', packageName], {
+        stdio: 'inherit',
+        timeout: 300_000,
+        cwd: installCwd,
+      });
+      return Promise.resolve(true);
+    } catch (err) {
+      info(
+        `Auto-install of ${packageName} failed (${err instanceof Error ? err.message : String(err)}). Install it manually with:\n  npm install ${packageName}`,
+      );
+      return Promise.resolve(false);
+    }
+  }
+
+  return new Promise((resolve) => {
+    const rl = createInterface({ input: process.stdin, output: process.stderr });
+    rl.question(
+      `Semantic search requires ${packageName}. Install it now? [y/N] `,
+      (answer: string) => {
+        rl.close();
+        if (answer.trim().toLowerCase() !== 'y') return resolve(false);
+        try {
+          execFileSync(NPM_BIN, ['install', '--no-save', packageName], {
+            stdio: 'inherit',
+            timeout: 300_000,
+            cwd: installCwd,
+          });
+          resolve(true);
+        } catch (err) {
+          info(
+            `Install of ${packageName} failed (${err instanceof Error ? err.message : String(err)}). Install it manually with:\n  npm install ${packageName}`,
+          );
+          resolve(false);
+        }
+      },
+    );
+  });
+}
+
+/**
+ * Lazy-load @huggingface/transformers.
+ * If the package is missing, prompts the user to install it interactively.
+ * In non-TTY environments, attempts automatic installation.
+ * @internal Not part of the public barrel.
+ */
+export async function loadTransformers(): Promise<unknown> {
+  try {
+    return await import('@huggingface/transformers');
+  } catch {
+    const pkg = '@huggingface/transformers';
+    const installed = await promptInstall(pkg);
+    if (installed) {
+      try {
+        return await import(pkg);
+      } catch (loadErr) {
+        throw new EngineError(
+          `${pkg} was installed but failed to load. Please check your environment.`,
+          { cause: loadErr instanceof Error ? loadErr : undefined },
+        );
+      }
+    }
+    throw new EngineError(`Semantic search requires ${pkg}.\nInstall it with: npm install ${pkg}`);
+  }
+}
+
+/**
+ * Dispose the current ONNX session and free memory.
+ * Safe to call when no model is loaded (no-op).
+ */
+export async function disposeModel(): Promise<void> {
+  if (extractor) {
+    await extractor.dispose();
+    extractor = null;
+  }
+  activeModel = null;
+}
+
+async function loadModel(modelKey?: string): Promise<{ extractor: unknown; config: ModelConfig }> {
+  const config = getModelConfig(modelKey);
+
+  if (extractor && activeModel === config.name) return { extractor, config };
+
+  // Dispose previous model before loading a different one
+  await disposeModel();
+
+  const transformers = (await loadTransformers()) as { pipeline: unknown };
+  pipeline = transformers.pipeline;
+
+  info(`Loading embedding model: ${config.name} (${config.dim}d)...`);
+  const pipelineOpts = config.quantized ? { dtype: 'q8' } : {};
+  try {
+    extractor =
+      await // biome-ignore lint/complexity/noBannedTypes: dynamically loaded transformers pipeline is untyped
+      (pipeline as Function)('feature-extraction', config.name, pipelineOpts);
+  } catch (err: unknown) {
+    const cause = err instanceof Error ? err : undefined;
+    const msg = cause?.message || String(err);
+    if (msg.includes('Unauthorized') || msg.includes('401') || msg.includes('gated')) {
+      throw new EngineError(
+        `Model "${config.name}" requires authentication.\n` +
+          `This model is gated on HuggingFace and needs an access token.\n\n` +
+          `Options:\n` +
+          `  1. Set HF_TOKEN env var: export HF_TOKEN=hf_...\n` +
+          `  2. Use a public model instead: codegraph embed --model minilm`,
+        { cause },
+      );
+    }
+    throw new EngineError(
+      `Failed to load model "${config.name}": ${msg}\n` +
+        `Try a different model: codegraph embed --model minilm`,
+      { cause },
+    );
+  }
+  activeModel = config.name;
+  info('Model loaded.');
+  return { extractor, config };
+}
+
+export interface EmbedOptions {
+  batchSize?: number;
+  throttlePerBatchInMs?: number;
+  onBatchComplete?: (batchSize: number, embedded: number, total: number) => void;
+}
+/**
+ * Generate embeddings for an array of texts.
+ */
+export async function embed(
+  texts: string[],
+  modelKey?: string,
+  options: EmbedOptions = {},
+): Promise<{ vectors: Float32Array[]; dim: number }> {
+  const { extractor: ext, config } = await loadModel(modelKey);
+  const dim = config.dim;
+  const results: Float32Array[] = [];
+  const batchSize = (() => {
+    if (options.batchSize) {
+      return options.batchSize;
+    } else {
+      return BATCH_SIZE_MAP[modelKey || DEFAULT_MODEL] ?? DEFAULT_BATCH_SIZE;
+    }
+  })();
+
+  for (let i = 0; i < texts.length; i += batchSize) {
+    const batch = texts.slice(i, i + batchSize);
+    const output =
+      (await // biome-ignore lint/complexity/noBannedTypes: dynamically loaded extractor is untyped
+      (ext as Function)(batch, { pooling: config.pooling ?? 'mean', normalize: true })) as {
+        data: number[];
+      };
+
+    for (let j = 0; j < batch.length; j++) {
+      const start = j * dim;
+      const vec = new Float32Array(dim);
+      for (let k = 0; k < dim; k++) {
+        vec[k] = output.data[start + k] ?? 0;
+      }
+      results.push(vec);
+    }
+
+    if (texts.length > batchSize) {
+      if (global.gc) {
+        global.gc();
+      }
+
+      const embedded = Math.min(i + batchSize, texts.length);
+      process.stderr.write(`  Embedded ${embedded}/${texts.length}\r`);
+      if (options.onBatchComplete) {
+        options.onBatchComplete(batchSize, embedded, texts.length);
+      }
+      if (options.throttlePerBatchInMs) {
+        await setTimeout(options.throttlePerBatchInMs);
+      }
+    }
+  }
+
+  return { vectors: results, dim };
+}