@monoes/monograph 1.2.8 → 1.2.9

package/src/search/embed-batch.ts

@@ -1,151 +0,0 @@
-/**
- * Batch-embed all symbol nodes that don't yet have an embedding stored.
- */
-
-import type Database from 'better-sqlite3';
-import type { EmbedderFn } from './embedder.js';
-import type { EmbedDeviceConfig } from './device-config.js';
-import type { HttpEmbedderConfig } from './http-embedder.js';
-import { upsertEmbedding, ensureEmbeddingSchema, countEmbeddings } from '../storage/embedding-store.js';
-import { embedText } from './embedder.js';
-
-const BATCH_SIZE = 32;
-
-export interface EmbedBatchConfig {
-  device?: EmbedDeviceConfig;
-  remote?: HttpEmbedderConfig; // if set, use HttpEmbedder instead of local
-  batchSize?: number;          // override default batch size
-}
-
-/**
- * Embed a list of text strings and return an array of embedding vectors.
- *
- * If `config.remote` is provided, delegates to `HttpEmbedder`.
- * Otherwise falls back to the local HuggingFace embedder.
- *
- * @param texts  - Strings to embed
- * @param config - Optional device/remote/batchSize config
- */
-export async function embedBatch(
-  texts: string[],
-  config?: EmbedBatchConfig,
-): Promise<number[][]> {
-  // Store device config in a local variable (passed through for future use)
-  const _device = config?.device;
-  void _device;
-
-  if (config?.remote) {
-    const { HttpEmbedder } = await import('./http-embedder.js');
-    const embedder = new HttpEmbedder(config.remote);
-    return embedder.embedBatch(texts);
-  }
-
-  // Local embedder path
-  const { getEmbedder } = await import('./embedder.js');
-  const embedderFn = await getEmbedder();
-  const batchSize = config?.batchSize ?? BATCH_SIZE;
-  const results: number[][] = [];
-
-  for (let i = 0; i < texts.length; i += batchSize) {
-    const batch = texts.slice(i, i + batchSize);
-    const batchResults = await Promise.all(
-      batch.map(async (text) => {
-        const vector = await embedText(text, embedderFn);
-        return Array.from(vector);
-      }),
-    );
-    results.push(...batchResults);
-  }
-
-  return results;
-}
-
-export interface EmbedAllResult {
-  embedded: number;
-  skipped: number;
-}
-
-/**
- * Embed all nodes that are missing embeddings.
- *
- * Nodes are fetched in batches of BATCH_SIZE to avoid loading the entire DB
- * into memory at once.
- *
- * @param db       - Open monograph database
- * @param embedder - Feature extraction pipeline function
- * @param force    - When true, re-embed nodes even if they already have an embedding
- */
-const CODE_LABELS = new Set(['Function', 'Method', 'Class', 'Module', 'Interface', 'Enum', 'Struct', 'Constructor', 'Variable', 'Const', 'TypeAlias']);
-
-export async function embedAll(
-  db: Database.Database,
-  embedder: EmbedderFn,
-  force = false,
-  codeOnly = false,
-): Promise<EmbedAllResult> {
-  let embedded = 0;
-  let skipped = 0;
-
-  // Fetch all node IDs and their name+normLabel (used as the text to embed)
-  const rows = db
-    .prepare(
-      `SELECT id, name, norm_label, label, file_path
-       FROM nodes
-       ORDER BY rowid`,
-    )
-    .all() as { id: string; name: string; norm_label: string; label: string; file_path: string | null }[];
-
-  const filteredRows = codeOnly ? rows.filter((r) => CODE_LABELS.has(r.label)) : rows;
-
-  // Build set of node IDs that already have embeddings
-  const existingIds = force
-    ? new Set<string>()
-    : new Set(
-        (
-          db.prepare('SELECT node_id FROM embeddings').all() as { node_id: string }[]
-        ).map((r) => r.node_id),
-      );
-
-  // Filter to only rows that need embedding
-  const toEmbed = filteredRows.filter((r) => !existingIds.has(r.id));
-  skipped = filteredRows.length - toEmbed.length;
-
-  // Ensure schema migration runs once before any upsert — not per-row.
-  if (toEmbed.length > 0) ensureEmbeddingSchema(db);
-
-  // Process in batches
-  for (let i = 0; i < toEmbed.length; i += BATCH_SIZE) {
-    const batch = toEmbed.slice(i, i + BATCH_SIZE);
-    await Promise.all(
-      batch.map(async (row) => {
-        const text = buildNodeText(row);
-        try {
-          const vector = await embedText(text, embedder);
-          upsertEmbedding(db, row.id, vector);
-          embedded++;
-        } catch {
-          // Skip nodes that fail to embed (e.g. empty name)
-          skipped++;
-        }
-      }),
-    );
-  }
-
-  return { embedded, skipped };
-}
-
-// ── Helpers ───────────────────────────────────────────────────────────────────
-
-function buildNodeText(row: {
-  name: string;
-  norm_label: string;
-  label: string;
-  file_path: string | null;
-}): string {
-  const parts: string[] = [row.name, row.label];
-  if (row.norm_label && row.norm_label !== row.name) parts.push(row.norm_label);
-  if (row.file_path) parts.push(row.file_path);
-  return parts.join(' ');
-}
-
-export { countEmbeddings };

package/src/search/embedder.ts

@@ -1,60 +0,0 @@
-/**
- * Lazy singleton for the HuggingFace FeatureExtractionPipeline.
- *
- * Uses dynamic import so that:
- *  1. Tests can mock `getEmbedder` without installing the package.
- *  2. @huggingface/transformers is optional at import time; callers receive a
- *     clear error if the package is unavailable.
- *
- * Model: Snowflake/snowflake-arctic-embed-xs (384 dimensions)
- */
-
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-export type EmbedderFn = (text: string | string[], options?: Record<string, unknown>) => Promise<any>;
-
-let cachedEmbedder: EmbedderFn | null = null;
-
-/**
- * Returns (and caches) a FeatureExtractionPipeline for
- * 'Snowflake/snowflake-arctic-embed-xs'.
- *
- * Throws a descriptive error if @huggingface/transformers is not installed.
- */
-export async function getEmbedder(): Promise<EmbedderFn> {
-  if (cachedEmbedder) return cachedEmbedder;
-
-  let pipeline: (task: string, model: string, opts?: Record<string, unknown>) => Promise<EmbedderFn>;
-  try {
-    // Dynamic import so the dep is optional at bundle time
-    const hf = await import('@huggingface/transformers');
-    pipeline = hf.pipeline as typeof pipeline;
-  } catch {
-    throw new Error(
-      '@huggingface/transformers is not installed. ' +
-        'Run `npm install @huggingface/transformers` to enable embedding support.',
-    );
-  }
-
-  cachedEmbedder = await pipeline('feature-extraction', 'Snowflake/snowflake-arctic-embed-xs', {
-    dtype: 'fp32',
-  });
-  return cachedEmbedder;
-}
-
-/** Reset the cached singleton — used only in tests. */
-export function resetEmbedderCache(): void {
-  cachedEmbedder = null;
-}
-
-/**
- * Embed a single text string and return a 384-dim Float32Array.
- * Pooling: mean. Normalisation: L2 (so cosine ≡ dot product).
- */
-export async function embedText(
-  text: string,
-  embedder: EmbedderFn,
-): Promise<Float32Array> {
-  const output = await embedder(text, { pooling: 'mean', normalize: true });
-  // HuggingFace Tensor: output.data is Float32Array
-  return output.data as Float32Array;
-}

package/src/search/exact-search.ts

@@ -1,45 +0,0 @@
-import type Database from 'better-sqlite3';
-
-export interface ExactSearchResult {
-  id: string;
-  score: number;
-}
-
-export interface ExactSearchOptions {
-  limit?: number;
-}
-
-function cosineSimilarity(a: Float32Array, b: Float32Array): number {
-  let dot = 0, normA = 0, normB = 0;
-  const len = Math.min(a.length, b.length);
-  for (let i = 0; i < len; i++) {
-    dot += a[i]! * b[i]!;
-    normA += a[i]! * a[i]!;
-    normB += b[i]! * b[i]!;
-  }
-  const denom = Math.sqrt(normA) * Math.sqrt(normB);
-  return denom === 0 ? 0 : dot / denom;
-}
-
-export function exactVectorSearch(
-  db: Database.Database,
-  queryVector: Float32Array,
-  options: ExactSearchOptions = {},
-): ExactSearchResult[] {
-  const { limit = 10 } = options;
-
-  const rows = db.prepare(`
-    SELECT e.node_id as id, e.vector FROM embeddings e
-    INNER JOIN nodes n ON n.id = e.node_id
-  `).all() as { id: string; vector: Buffer }[];
-
-  if (rows.length === 0) return [];
-
-  return rows
-    .map(row => {
-      const vec = new Float32Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength / 4);
-      return { id: row.id, score: cosineSimilarity(queryVector, vec) };
-    })
-    .sort((a, b) => b.score - a.score)
-    .slice(0, limit);
-}

package/src/search/http-embedder.ts

@@ -1,78 +0,0 @@
-export interface HttpEmbedderConfig {
-  endpoint: string;
-  model?: string;
-  apiKey?: string;
-  timeoutMs?: number;
-  batchSize?: number;
-}
-
-export interface EmbedResponse {
-  embedding: number[];
-}
-
-export class HttpEmbedder {
-  private config: Required<HttpEmbedderConfig>;
-
-  constructor(config: HttpEmbedderConfig) {
-    this.config = {
-      model: '',
-      apiKey: '',
-      timeoutMs: 30_000,
-      batchSize: 32,
-      ...config,
-    };
-  }
-
-  async embedOne(text: string): Promise<number[]> {
-    const controller = new AbortController();
-    const timer = setTimeout(() => controller.abort(), this.config.timeoutMs);
-
-    try {
-      const headers: Record<string, string> = {
-        'Content-Type': 'application/json',
-      };
-      if (this.config.apiKey) {
-        headers['Authorization'] = `Bearer ${this.config.apiKey}`;
-      }
-
-      const body: Record<string, unknown> = { prompt: text };
-      if (this.config.model) {
-        body['model'] = this.config.model;
-      }
-
-      const response = await fetch(this.config.endpoint, {
-        method: 'POST',
-        headers,
-        body: JSON.stringify(body),
-        signal: controller.signal,
-      });
-
-      if (!response.ok) {
-        throw new Error(`HTTP embedder: ${response.status} ${response.statusText}`);
-      }
-
-      const data = (await response.json()) as Partial<EmbedResponse>;
-      if (!data.embedding) {
-        throw new Error('HTTP embedder: no embedding in response');
-      }
-
-      return data.embedding;
-    } finally {
-      clearTimeout(timer);
-    }
-  }
-
-  async embedBatch(texts: string[]): Promise<number[][]> {
-    const results: number[][] = [];
-    const batchSize = this.config.batchSize;
-
-    for (let i = 0; i < texts.length; i += batchSize) {
-      const batch = texts.slice(i, i + batchSize);
-      for (const text of batch) {
-        results.push(await this.embedOne(text));
-      }
-    }
-
-    return results;
-  }
-}

package/src/storage/embedding-store.ts

@@ -1,108 +0,0 @@
-import type Database from 'better-sqlite3';
-
-/**
- * CRUD operations for the embeddings table.
- *
- * Vectors are stored as BLOBs (raw Float32Array bytes) and reconstructed on read.
- */
-
-/**
- * Ensure the embeddings table has the content_hash column introduced in a later
- * schema version. Call this ONCE before a batch of upserts rather than inside
- * upsertEmbedding itself to avoid running ALTER TABLE on every row write.
- */
-export function ensureEmbeddingSchema(db: Database.Database): void {
-  try {
-    db.exec('ALTER TABLE embeddings ADD COLUMN content_hash TEXT');
-  } catch {
-    // Column already exists — ignore.
-  }
-}
-
-export function upsertEmbedding(
-  db: Database.Database,
-  nodeId: string,
-  vector: Float32Array,
-  contentHash?: string,
-): void {
-  const buf = Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength);
-  db
-    .prepare(
-      'INSERT OR REPLACE INTO embeddings (node_id, vector, content_hash) VALUES (?, ?, ?)',
-    )
-    .run(nodeId, buf, contentHash ?? null);
-}
-
-/**
- * Bulk-upsert multiple embeddings in a single transaction.
- * Calls ensureEmbeddingSchema once before writing, then wraps all inserts in
- * a transaction for 10-100x faster throughput vs per-row upsertEmbedding calls.
- */
-export function batchUpsertEmbeddings(
-  db: Database.Database,
-  entries: Array<{ nodeId: string; vector: Float32Array; contentHash?: string }>,
-): void {
-  if (entries.length === 0) return;
-  ensureEmbeddingSchema(db);
-  const stmt = db.prepare(
-    'INSERT OR REPLACE INTO embeddings (node_id, vector, content_hash) VALUES (?, ?, ?)',
-  );
-  const insertMany = db.transaction(
-    (rows: Array<{ nodeId: string; vector: Float32Array; contentHash?: string }>) => {
-      for (const e of rows) {
-        const buf = Buffer.from(e.vector.buffer, e.vector.byteOffset, e.vector.byteLength);
-        stmt.run(e.nodeId, buf, e.contentHash ?? null);
-      }
-    },
-  );
-  insertMany(entries);
-}
-
-export function getEmbeddingContentHash(db: Database.Database, nodeId: string): string | null {
-  const row = db
-    .prepare('SELECT content_hash FROM embeddings WHERE node_id = ?')
-    .get(nodeId) as { content_hash: string | null } | undefined;
-  return row?.content_hash ?? null;
-}
-
-export function isEmbeddingStale(
-  db: Database.Database,
-  nodeId: string,
-  currentHash: string,
-): boolean {
-  const stored = getEmbeddingContentHash(db, nodeId);
-  return stored !== currentHash;
-}
-
-export function getEmbedding(db: Database.Database, nodeId: string): Float32Array | null {
-  const row = db.prepare('SELECT vector FROM embeddings WHERE node_id = ?').get(nodeId) as
-    | { vector: Buffer }
-    | undefined;
-  if (!row) return null;
-  return bufToFloat32(row.vector);
-}
-
-export function getAllEmbeddings(db: Database.Database): Map<string, Float32Array> {
-  const rows = db.prepare('SELECT node_id, vector FROM embeddings').all() as {
-    node_id: string;
-    vector: Buffer;
-  }[];
-  const result = new Map<string, Float32Array>();
-  for (const row of rows) {
-    result.set(row.node_id, bufToFloat32(row.vector));
-  }
-  return result;
-}
-
-export function countEmbeddings(db: Database.Database): number {
-  const row = db.prepare('SELECT COUNT(*) AS n FROM embeddings').get() as { n: number };
-  return row.n;
-}
-
-// ── Internal helpers ──────────────────────────────────────────────────────────
-
-function bufToFloat32(buf: Buffer): Float32Array {
-  // Copy to a fresh ArrayBuffer so slice offset doesn't cause issues
-  const ab = buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength);
-  return new Float32Array(ab);
-}