@mseep/core 3.0.0

package/lib/engine/corrections.js

@@ -0,0 +1,194 @@
+// lib/engine/corrections.js
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { randomUUID } from 'node:crypto';
+import { createEmbeddingPipeline } from './embedding-provider.js';
+
+// ── Constants ────────────────────────────────────────────────────────────────
+
+export const LEVEL_THRESHOLDS = [
+  { level: 4, min: 10 },
+  { level: 3, min: 5 },
+  { level: 2, min: 3 },
+  { level: 1, min: 1 },
+];
+
+const DEDUP_THRESHOLD = 0.85;
+const MAX_INJECTED    = 20;
+
+export const MARKER_START = '<!-- booklib-learned-start -->';
+export const MARKER_END   = '<!-- booklib-learned-end -->';
+
+// ── Embedding model (lazy-loaded, module-level singleton) ────────────────────
+
+async function _getEmbedding(text) {
+  const { extractor } = await createEmbeddingPipeline({ quiet: true });
+  const output = await extractor(text, { pooling: 'mean', normalize: true });
+  return Array.from(output.data);
+}
+
+// ── Pure functions ────────────────────────────────────────────────────────────
+
+export function levelFromMentions(n) {
+  for (const { level, min } of LEVEL_THRESHOLDS) {
+    if (n >= min) return level;
+  }
+  return 1;
+}
+
+export function cosine(a, b) {
+  let dot = 0, na = 0, nb = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    na  += a[i] * a[i];
+    nb  += b[i] * b[i];
+  }
+  const denom = Math.sqrt(na) * Math.sqrt(nb);
+  return denom === 0 ? 0 : dot / denom;
+}
+
+function _generateId() {
+  return randomUUID().replace(/-/g, '').slice(0, 8);
+}
+
+function _escapeRegex(s) {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+// ── File paths ────────────────────────────────────────────────────────────────
+
+function _correctionsPath(home) {
+  return path.join(home, '.booklib', 'corrections.jsonl');
+}
+
+function _claudeMdPath(home) {
+  return path.join(home, '.claude', 'CLAUDE.md');
+}
+
+// ── Storage ───────────────────────────────────────────────────────────────────
+
+export function loadCorrections(home = os.homedir()) {
+  const p = _correctionsPath(home);
+  if (!fs.existsSync(p)) return [];
+  return fs.readFileSync(p, 'utf8')
+    .split('\n')
+    .filter(Boolean)
+    .reduce((acc, line) => {
+      try {
+        acc.push(JSON.parse(line));
+      } catch {
+        process.stderr.write(`Warning: skipping corrupt line in corrections.jsonl: ${line.slice(0, 40)}\n`);
+      }
+      return acc;
+    }, []);
+}
+
+function _saveCorrections(corrections, home) {
+  const p = _correctionsPath(home);
+  fs.mkdirSync(path.dirname(p), { recursive: true });
+  const lines = corrections.map(c => JSON.stringify(c)).join('\n');
+  fs.writeFileSync(p, corrections.length ? lines + '\n' : '');
+}
+
+// ── CLAUDE.md injection ────────────────────────────────────────────────────────
+
+export function rebuildLearnedSection(home = os.homedir()) {
+  const corrections = loadCorrections(home);
+  const active = corrections
+    .filter(c => c.level >= 3)
+    .sort((a, b) => b.mentions - a.mentions)
+    .slice(0, MAX_INJECTED);
+
+  const claudeFile = _claudeMdPath(home);
+  fs.mkdirSync(path.dirname(claudeFile), { recursive: true });
+
+  let existing = '';
+  try { existing = fs.readFileSync(claudeFile, 'utf8'); } catch (e) {
+    if (e.code !== 'ENOENT') throw e;
+  }
+
+  if (active.length === 0) {
+    const re = new RegExp(
+      `\\n?${_escapeRegex(MARKER_START)}[\\s\\S]*?${_escapeRegex(MARKER_END)}\\n?`
+    );
+    const updated = existing.replace(re, '').trimEnd();
+    fs.writeFileSync(claudeFile, updated ? updated + '\n' : '');
+    return;
+  }
+
+  const bullets = active.map(c => `- ${c.text.slice(0, 120)}`).join('\n');
+  const section = [
+    MARKER_START,
+    '## Learned Corrections (BookLib)',
+    '',
+    '> When the user corrects your approach, run: booklib correction add "brief rule"',
+    '',
+    bullets,
+    '',
+    MARKER_END,
+  ].join('\n');
+
+  const re = new RegExp(
+    `${_escapeRegex(MARKER_START)}[\\s\\S]*?${_escapeRegex(MARKER_END)}`
+  );
+  const updated = existing.includes(MARKER_START)
+    ? existing.replace(re, section)
+    : (existing.trimEnd() ? `${existing.trimEnd()}\n\n${section}\n` : `${section}\n`);
+
+  fs.writeFileSync(claudeFile, updated);
+}
+
+// ── Public API ────────────────────────────────────────────────────────────────
+
+export function listCorrections(home = os.homedir()) {
+  return loadCorrections(home).sort((a, b) => b.mentions - a.mentions);
+}
+
+export function removeCorrection(id, home = os.homedir()) {
+  const corrections = loadCorrections(home);
+  const idx = corrections.findIndex(c => c.id === id);
+  if (idx === -1) return null;
+  const [removed] = corrections.splice(idx, 1);
+  _saveCorrections(corrections, home);
+  if (removed.level >= 3) rebuildLearnedSection(home);
+  return removed;
+}
+
+export async function addCorrection(text, home = os.homedir(), embedFn = _getEmbedding) {
+  const corrections = loadCorrections(home);
+  const now = new Date().toISOString();
+  const newVec = await embedFn(text);
+
+  for (const c of corrections) {
+    const existVec = c.embedding ?? await embedFn(c.text);
+    if (!c.embedding) c.embedding = existVec; // backfill on first encounter
+    const sim = cosine(newVec, existVec);
+    if (sim >= DEDUP_THRESHOLD) {
+      const oldLevel = c.level;
+      c.mentions += 1;
+      c.level = levelFromMentions(c.mentions);
+      c.lastSeen = now;
+      c.sessions.push(now);
+      _saveCorrections(corrections, home);
+      if (c.level >= 3 && c.level !== oldLevel) rebuildLearnedSection(home);
+      const { embedding: _emb, ...rest } = c;
+      return { ...rest, wasExisting: true };
+    }
+  }
+
+  const entry = {
+    id: _generateId(),
+    text,
+    mentions: 1,
+    level: levelFromMentions(1),
+    sessions: [now],
+    firstSeen: now,
+    lastSeen: now,
+    embedding: newVec,
+  };
+  corrections.push(entry);
+  _saveCorrections(corrections, home);
+  const { embedding: _emb, ...rest } = entry;
+  return { ...rest, wasExisting: false };
+}

package/lib/engine/decision-checker.js

@@ -0,0 +1,203 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { parseImports, detectLanguage } from './import-parser.js';
+
+const MAX_FILE_SIZE = 1_000_000;
+
+/**
+ * Prohibition patterns -- regex that captures what's being prohibited.
+ * Group 1 should capture the prohibited thing.
+ */
+const PROHIBITION_PATTERNS = [
+  // "do not use X", "don't use X", "never use X", "avoid X"
+  /(?:do\s+not|don't|never|avoid)\s+(?:use|using)\s+['"`]?([^'"`.,;\n]+)/gi,
+  // "deprecated: X", "X is deprecated"
+  /deprecated:?\s+['"`]?([^'"`.,;\n]+)/gi,
+  /(['"`]?[A-Z][\w.]*['"`]?)\s+is\s+deprecated/gi,
+  // "prefer X over Y" -- Y is prohibited
+  /prefer\s+\S+\s+over\s+['"`]?([^'"`.,;\n]+)/gi,
+  // "replaced X with Y" -- X is prohibited
+  /replaced?\s+['"`]?([^'"`.,;\n]+?)['"`]?\s+with\s/gi,
+  // "decided against X"
+  /decided\s+against\s+['"`]?([^'"`.,;\n]+)/gi,
+  // "instead of X" — only after a decision verb to reduce false positives
+  /(?:decided|agreed|chose|chosen|use)\s+\S+\s+instead\s+of\s+['"`]?([^'"`.,;\n]+)/gi,
+  // "must not X", "should not X"
+  /(?:must|should)\s+not\s+(?:use|import|call|reference)\s+['"`]?([^'"`.,;\n]+)/gi,
+];
+
+export class DecisionChecker {
+  /**
+   * @param {object} [opts]
+   * @param {object} [opts.searcher] - BookLibSearcher instance
+   * @param {number} [opts.minScore] - minimum search relevance score
+   */
+  constructor(opts = {}) {
+    this.searcher = opts.searcher ?? null;
+    this.minScore = opts.minScore ?? 0.4;
+  }
+
+  /**
+   * Check a file for contradictions against indexed team decisions.
+   * @param {string} filePath
+   * @returns {Promise<{contradictions: Array<{identifier: string, decision: string, source: string, pattern: string}>, checked: number}>}
+   */
+  async checkFile(filePath) {
+    if (!this.searcher) return { contradictions: [], checked: 0 };
+
+    const language = detectLanguage(filePath);
+    if (!language) return { contradictions: [], checked: 0 };
+
+    const resolved = path.resolve(filePath);
+    const stat = fs.statSync(resolved, { throwIfNoEntry: false });
+    if (!stat || stat.size > MAX_FILE_SIZE) return { contradictions: [], checked: 0 };
+
+    const code = fs.readFileSync(resolved, 'utf8');
+    const identifiers = this._extractIdentifiers(code, language);
+    if (identifiers.length === 0) return { contradictions: [], checked: 0 };
+
+    return this._findContradictions(identifiers);
+  }
+
+  /**
+   * Extract meaningful identifiers from code -- import names, API calls, etc.
+   * @param {string} code
+   * @param {string} language
+   * @returns {string[]} unique identifiers
+   */
+  _extractIdentifiers(code, language) {
+    const imports = parseImports(code, language).map(i => i.module);
+    const apiCalls = [];
+    // Match dot-notation API calls: stripe.charges.create(), db.collection.find()
+    const dotPattern = /\b([a-zA-Z_$][\w]*(?:\.[a-zA-Z_$]\w*)+)\s*\(/g;
+    let match;
+    while ((match = dotPattern.exec(code)) !== null) {
+      apiCalls.push(match[1]);
+    }
+    return [...new Set([...imports, ...apiCalls])];
+  }
+
+  /**
+   * Search the index for each identifier and check for contradictions.
+   * @param {string[]} identifiers
+   * @returns {Promise<{contradictions: Array, checked: number}>}
+   */
+  async _findContradictions(identifiers) {
+    const contradictions = [];
+    const seen = new Set();
+
+    for (const id of identifiers) {
+      try {
+        const results = await this.searcher.search(id, 3, this.minScore);
+        for (const result of results) {
+          this._checkResult(result, identifiers, seen, contradictions);
+        }
+      } catch {
+        // Search failure -- skip this identifier
+      }
+    }
+
+    return { contradictions, checked: identifiers.length };
+  }
+
+  /**
+   * Check a single search result for prohibition matches.
+   * @param {object} result - search result with text and metadata
+   * @param {string[]} identifiers - code identifiers to match against
+   * @param {Set} seen - dedup tracker
+   * @param {Array} contradictions - output array
+   */
+  _checkResult(result, identifiers, seen, contradictions) {
+    const text = result.item?.text ?? result.text ?? '';
+    const prohibited = this._extractProhibitions(text);
+
+    for (const { target, pattern } of prohibited) {
+      const matchedId = this._matchesIdentifier(target, identifiers);
+      const key = `${matchedId}:${text.slice(0, 50)}`;
+      if (matchedId && !seen.has(key)) {
+        seen.add(key);
+        contradictions.push({
+          identifier: matchedId,
+          decision: text.slice(0, 300),
+          source: result.metadata?.sourceName ?? result.metadata?.title ?? 'unknown',
+          pattern,
+        });
+      }
+    }
+  }
+
+  /**
+   * Extract prohibition targets from a text snippet.
+   * @param {string} text
+   * @returns {Array<{target: string, pattern: string}>}
+   */
+  _extractProhibitions(text) {
+    const results = [];
+    for (const pattern of PROHIBITION_PATTERNS) {
+      const regex = new RegExp(pattern.source, pattern.flags);
+      let match;
+      while ((match = regex.exec(text)) !== null) {
+        const target = match[1].trim();
+        if (target.length >= 2 && target.length <= 100) {
+          results.push({ target, pattern: pattern.source.slice(0, 40) });
+        }
+      }
+    }
+    return results;
+  }
+
+  /**
+   * Check if a prohibition target matches any code identifier.
+   * Splits both on word boundaries (dots, spaces) for segment-level matching.
+   * "Charges API" matches "stripe.charges" because "charges" overlaps.
+   * @param {string} target - the prohibited thing from decision text
+   * @param {string[]} identifiers - identifiers from the code
+   * @returns {string|null} the matched identifier, or null
+   */
+  _matchesIdentifier(target, identifiers) {
+    const targetSegments = this._toSegments(target);
+    for (const id of identifiers) {
+      if (this._segmentsOverlap(targetSegments, this._toSegments(id))) {
+        return id;
+      }
+    }
+    return null;
+  }
+
+  /**
+   * Split a string into lowercase segments by dots, spaces, and camelCase.
+   * @param {string} str
+   * @returns {string[]}
+   */
+  _toSegments(str) {
+    return str
+      .replace(/([a-z])([A-Z])/g, '$1 $2')
+      .toLowerCase()
+      .split(/[\s./:@-]+/)
+      .filter(s => s.length >= 2);
+  }
+
+  /**
+   * Check if any significant segment from A exactly matches one from B.
+   * Exact equality only — substring matching produces too many false positives
+   * on short common segments like "api", "sql", "log".
+   * @param {string[]} segsA
+   * @param {string[]} segsB
+   * @returns {boolean}
+   */
+  _segmentsOverlap(segsA, segsB) {
+    const setB = new Set(segsB);
+    // Require exact match on segments of 4+ chars, or at least 2 short-segment matches
+    let shortMatches = 0;
+    for (const a of segsA) {
+      if (setB.has(a)) {
+        if (a.length >= 4) return true;
+        shortMatches++;
+        if (shortMatches >= 2) return true;
+      }
+    }
+    return false;
+  }
+}
+
+export { PROHIBITION_PATTERNS };

package/lib/engine/doctor.js

@@ -0,0 +1,207 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+import { resolveBookLibPaths } from '../paths.js';
+import { SKILL_LIMIT } from '../wizard/skill-recommender.js';
+import { countAllSlots, countInstalledSlots, listInstalledSkillNames } from '../skill-fetcher.js';
+
+const MARKER_START = '<!-- booklib-standards-start -->';
+
+/**
+ * Runs all diagnostic checks and returns an array of findings.
+ * @param {string} cwd - project root
+ * @returns {Array<{ check: string, severity: 'error'|'warning'|'info', message: string, fixable: boolean, data?: object }>}
+ */
+export function runDiagnostics(cwd = process.cwd()) {
+  const findings = [];
+
+  checkSlotOverload(findings);
+  checkOversizedConfigs(findings, cwd);
+  checkMissingIndex(findings, cwd);
+  checkMissingConfigFiles(findings, cwd);
+  checkStaleSkills(findings);
+  checkOrphanedSkills(findings);
+
+  return findings;
+}
+
+/** Check 1: Slot overload */
+function checkSlotOverload(findings) {
+  const slotsUsed = countAllSlots();
+  if (slotsUsed > SKILL_LIMIT) {
+    findings.push({
+      check: 'slot-overload',
+      severity: 'warning',
+      message: `${slotsUsed} skills installed (limit: ${SKILL_LIMIT}). Agent context is overloaded. Run: booklib doctor --usage`,
+      fixable: false,
+      data: { slotsUsed },
+    });
+  }
+}
+
+/** Check 2: Oversized config files */
+function checkOversizedConfigs(findings, cwd) {
+  const configFiles = [
+    { file: 'CLAUDE.md', path: path.join(cwd, 'CLAUDE.md') },
+    { file: '.github/copilot-instructions.md', path: path.join(cwd, '.github', 'copilot-instructions.md') },
+    { file: '.gemini/context.md', path: path.join(cwd, '.gemini', 'context.md') },
+  ];
+
+  for (const cf of configFiles) {
+    if (fs.existsSync(cf.path)) {
+      const content = fs.readFileSync(cf.path, 'utf8');
+      const lines = content.split('\n').length;
+      if (lines > 500 && content.includes(MARKER_START)) {
+        findings.push({
+          check: 'oversized-config',
+          severity: 'warning',
+          message: `${cf.file} is ${lines} lines. Recommended: under 200.`,
+          fixable: true,
+          data: { file: cf.file, absPath: cf.path, lines },
+        });
+      }
+    }
+  }
+}
+
+/** Check 3: Missing index */
+function checkMissingIndex(findings, cwd) {
+  try {
+    const { indexPath } = resolveBookLibPaths(cwd);
+    if (!fs.existsSync(indexPath)) {
+      findings.push({
+        check: 'missing-index',
+        severity: 'error',
+        message: 'No search index found. Search and recommendations won\'t work.',
+        fixable: true,
+      });
+    }
+  } catch {
+    findings.push({
+      check: 'missing-index',
+      severity: 'error',
+      message: 'No search index found. Search and recommendations won\'t work.',
+      fixable: true,
+    });
+  }
+}
+
+/** Check 4: Missing config files for configured tools */
+function checkMissingConfigFiles(findings, cwd) {
+  const configPath = path.join(cwd, 'booklib.config.json');
+  if (!fs.existsSync(configPath)) return;
+
+  try {
+    const config = JSON.parse(fs.readFileSync(configPath, 'utf8'));
+    const toolFileMap = {
+      claude: 'CLAUDE.md',
+      copilot: '.github/copilot-instructions.md',
+      gemini: '.gemini/context.md',
+      codex: 'AGENTS.md',
+      cursor: '.cursor/rules/booklib-standards.mdc',
+    };
+
+    for (const tool of (config.tools ?? [])) {
+      const expectedFile = toolFileMap[tool];
+      if (expectedFile && !fs.existsSync(path.join(cwd, expectedFile))) {
+        findings.push({
+          check: 'missing-config',
+          severity: 'warning',
+          message: `${tool} configured but ${expectedFile} not found.`,
+          fixable: true,
+          data: { tool, file: expectedFile },
+        });
+      }
+    }
+  } catch { /* malformed config, skip */ }
+}
+
+/** Check 5: Stale skills (installed but never used) */
+function checkStaleSkills(findings) {
+  const usagePath = path.join(os.homedir(), '.booklib', 'usage.json');
+  if (!fs.existsSync(usagePath)) return;
+
+  try {
+    const usage = JSON.parse(fs.readFileSync(usagePath, 'utf8'));
+    const usedNames = new Set(
+      usage.map(u => u.skill?.toLowerCase()).filter(Boolean),
+    );
+    const installed = listInstalledSkillNames();
+    const stale = installed.filter(n => !usedNames.has(n.toLowerCase()));
+
+    if (stale.length > 5) {
+      findings.push({
+        check: 'stale-skills',
+        severity: 'info',
+        message: `${stale.length} skills have no recorded usage.`,
+        fixable: true,
+        data: { staleNames: stale },
+      });
+    }
+  } catch { /* no usage data, skip */ }
+}
+
+/** Check 6: Orphaned skills (have .booklib marker but not in any catalog/index) */
+function checkOrphanedSkills(findings) {
+  const claudeSkillsDir = path.join(os.homedir(), '.claude', 'skills');
+  if (!fs.existsSync(claudeSkillsDir)) return;
+
+  try {
+    const installed = listInstalledSkillNames();
+    // Read the bundled skills directory to find known skill names
+    const packageRoot = path.resolve(
+      new URL('.', import.meta.url).pathname, '..', '..',
+    );
+    const bundledSkillsDir = path.join(packageRoot, 'skills');
+    const knownNames = new Set();
+    if (fs.existsSync(bundledSkillsDir)) {
+      for (const entry of fs.readdirSync(bundledSkillsDir, { withFileTypes: true })) {
+        if (entry.isDirectory()) knownNames.add(entry.name);
+      }
+    }
+
+    // Also load community registry skill names
+    const registryPath = path.join(packageRoot, 'community', 'registry.json');
+    if (fs.existsSync(registryPath)) {
+      try {
+        const registry = JSON.parse(fs.readFileSync(registryPath, 'utf8'));
+        for (const skill of (registry.skills ?? [])) {
+          knownNames.add(skill.name);
+        }
+      } catch { /* malformed registry, skip */ }
+    }
+
+    const orphaned = installed.filter(n => !knownNames.has(n));
+    for (const name of orphaned) {
+      findings.push({
+        check: 'orphaned-skill',
+        severity: 'info',
+        message: `${name} -- not found in any catalog. May be outdated.`,
+        fixable: false,
+      });
+    }
+  } catch { /* skip orphan check on error */ }
+}
+
+/**
+ * Prints diagnostic results to stdout.
+ * @param {Array<{ check: string, severity: string, message: string, fixable: boolean }>} findings
+ */
+export function printDiagnostics(findings) {
+  if (findings.length === 0) {
+    console.log('  No issues found. BookLib is healthy.\n');
+    return;
+  }
+
+  for (const f of findings) {
+    const icon = f.severity === 'error' ? '\u2717' : f.severity === 'warning' ? '\u26A0' : '\u2139';
+    console.log(`  ${icon} ${f.message}`);
+  }
+
+  const fixable = findings.filter(f => f.fixable).length;
+  if (fixable > 0) {
+    console.log(`\n  ${fixable} issue(s) fixable. Run: booklib doctor --cure\n`);
+  } else {
+    console.log('');
+  }
+}

package/lib/engine/embedding-provider.js

@@ -0,0 +1,72 @@
+// lib/engine/embedding-provider.js
+// Suppress ONNX runtime JS-level warnings
+process.env.ORT_LOG_LEVEL = 'ERROR';
+
+import { pipeline } from '@huggingface/transformers';
+
+const MODEL_NAME = 'Xenova/all-MiniLM-L6-v2';
+const BATCH_SIZE = 32;
+
+let _cachedPipeline = null;
+
+/**
+ * Returns a shared embedding pipeline (singleton per process).
+ * First call downloads/loads the model; subsequent calls return the cached instance.
+ *
+ * @param {Object} [opts]
+ * @param {boolean} [opts.quiet=false] - Suppress loading message.
+ * @returns {Promise<{ extractor: Function }>}
+ */
+export async function createEmbeddingPipeline(opts = {}) {
+  if (_cachedPipeline) return _cachedPipeline;
+
+  const { quiet = false } = opts;
+  if (!quiet) {
+    console.log('Loading local embedding model...');
+  }
+
+  // Try CoreML (Apple Silicon) with stderr suppressed during load
+  // CoreML spams native warnings ("Context leak", "IsInputSupported") — harmless but noisy
+  let extractor;
+  const isMac = process.platform === 'darwin';
+
+  // CPU int8 quantization is faster than CoreML for this model (73ms vs 1550ms).
+  // CoreML partitions 231/323 nodes to GPU, but the CPU↔GPU transfer overhead
+  // makes it slower than pure CPU for MiniLM-L6 (384-dim, small model).
+  // CPU q8 also produces zero native warnings.
+  extractor = await pipeline('feature-extraction', MODEL_NAME, { dtype: 'q8' });
+
+  // Warm up the model with a dummy embedding — triggers native CoreML warnings NOW
+  // so they don't appear during user-facing operations (search, lookup, index).
+  try {
+    await extractor('warmup', { pooling: 'mean', normalize: true });
+  } catch { /* warmup is best-effort */ }
+
+  _cachedPipeline = { extractor };
+  return _cachedPipeline;
+}
+
+/**
+ * Embed multiple texts in batches.
+ * @param {Function} extractor - the pipeline function
+ * @param {string[]} texts - array of texts to embed
+ * @param {number} [batchSize=32]
+ * @param {Function} [onBatch] - called after each batch with { done, total }
+ * @returns {Promise<number[][]>} array of embedding vectors
+ */
+export async function batchEmbed(extractor, texts, batchSize = BATCH_SIZE, onBatch) {
+  const allVectors = [];
+  for (let i = 0; i < texts.length; i += batchSize) {
+    const batch = texts.slice(i, i + batchSize);
+    const outputs = await extractor(batch, { pooling: 'mean', normalize: true });
+    const dims = outputs.dims[outputs.dims.length - 1];
+    for (let j = 0; j < batch.length; j++) {
+      const start = j * dims;
+      allVectors.push(Array.from(outputs.data.slice(start, start + dims)));
+    }
+    onBatch?.({ done: Math.min(i + batchSize, texts.length), total: texts.length });
+  }
+  return allVectors;
+}
+
+export { BATCH_SIZE, MODEL_NAME };