codebase-contextualizer-cli 1.0.0

package/scripts/benchmark.js

@@ -0,0 +1,189 @@
+#!/usr/bin/env node
+
+const path = require("path");
+const { performance } = require("perf_hooks");
+const Database = require("better-sqlite3");
+const { pipeline } = require("@xenova/transformers");
+
+const ITERATIONS = 1000;
+const DEFAULT_QUERY = "authentication logic";
+const MODEL_ID = "Xenova/all-MiniLM-L6-v2";
+
+let embedderPromise;
+
+function getEmbedder() {
+  if (!embedderPromise) {
+    embedderPromise = pipeline("feature-extraction", MODEL_ID);
+  }
+
+  return embedderPromise;
+}
+
+function toFloat32Array(value) {
+  if (value instanceof Float32Array) {
+    return new Float32Array(value);
+  }
+
+  if (ArrayBuffer.isView(value)) {
+    return Float32Array.from(value);
+  }
+
+  if (Array.isArray(value)) {
+    return Float32Array.from(value);
+  }
+
+  throw new Error("Embedding output could not be converted to Float32Array");
+}
+
+async function embedQuery(query) {
+  const embedder = await getEmbedder();
+  const output = await embedder(query, {
+    pooling: "mean",
+    normalize: true,
+  });
+
+  return toFloat32Array(output.data);
+}
+
+function blobToFloat32Array(blob) {
+  return new Float32Array(
+    blob.buffer,
+    blob.byteOffset,
+    blob.byteLength / Float32Array.BYTES_PER_ELEMENT,
+  );
+}
+
+function cosineSimilarity(left, right) {
+  if (left.length !== right.length) {
+    return Number.NEGATIVE_INFINITY;
+  }
+
+  let dot = 0;
+  let magnitudeLeft = 0;
+  let magnitudeRight = 0;
+
+  for (let index = 0; index < left.length; index += 1) {
+    dot += left[index] * right[index];
+    magnitudeLeft += left[index] * left[index];
+    magnitudeRight += right[index] * right[index];
+  }
+
+  if (magnitudeLeft === 0 || magnitudeRight === 0) {
+    return 0;
+  }
+
+  return dot / (Math.sqrt(magnitudeLeft) * Math.sqrt(magnitudeRight));
+}
+
+function loadChunks(databasePath) {
+  const db = new Database(databasePath, { readonly: true, fileMustExist: true });
+
+  try {
+    return db.prepare(`
+      SELECT
+        chunks.id,
+        files.path AS file,
+        chunks.name,
+        chunks.startLine,
+        chunks.endLine,
+        chunks.embedding
+      FROM chunks
+      INNER JOIN files ON files.id = chunks.file_id
+    `).all().map((row) => ({
+      ...row,
+      embedding: blobToFloat32Array(row.embedding),
+    }));
+  } finally {
+    db.close();
+  }
+}
+
+function searchChunks(queryEmbedding, chunks) {
+  let best = null;
+
+  for (const chunk of chunks) {
+    const score = cosineSimilarity(queryEmbedding, chunk.embedding);
+
+    if (!best || score > best.score) {
+      best = {
+        chunk,
+        score,
+      };
+    }
+  }
+
+  return best;
+}
+
+function percentile(sortedValues, percentileValue) {
+  if (sortedValues.length === 0) {
+    return 0;
+  }
+
+  const index = Math.ceil((percentileValue / 100) * sortedValues.length) - 1;
+  return sortedValues[Math.max(0, Math.min(index, sortedValues.length - 1))];
+}
+
+function summarizeLatencies(latencies) {
+  const sorted = [...latencies].sort((left, right) => left - right);
+  const total = latencies.reduce((sum, value) => sum + value, 0);
+
+  return {
+    average: total / latencies.length,
+    p50: percentile(sorted, 50),
+    p90: percentile(sorted, 90),
+    p95: percentile(sorted, 95),
+  };
+}
+
+async function main() {
+  const target = path.resolve(process.argv[2] || ".");
+  const query = process.argv[3] || DEFAULT_QUERY;
+  const databasePath = path.join(target, ".contextualizer", "vector.db");
+  const chunks = loadChunks(databasePath);
+
+  if (chunks.length === 0) {
+    throw new Error(`No chunks found in ${databasePath}. Run "node index.js index ${target}" first.`);
+  }
+
+  console.log(`Database: ${databasePath}`);
+  console.log(`Chunks: ${chunks.length}`);
+  console.log(`Query: ${query}`);
+  console.log(`Loading model: ${MODEL_ID}`);
+
+  const queryEmbedding = await embedQuery(query);
+  const latencies = [];
+
+  // Warm the tight loop once so the reported run is less sensitive to first-call overhead.
+  searchChunks(queryEmbedding, chunks);
+
+  for (let index = 0; index < ITERATIONS; index += 1) {
+    const start = performance.now();
+    searchChunks(queryEmbedding, chunks);
+    latencies.push(performance.now() - start);
+  }
+
+  const summary = summarizeLatencies(latencies);
+  const best = searchChunks(queryEmbedding, chunks);
+
+  console.log("");
+  console.log(`Iterations: ${ITERATIONS}`);
+  console.log(`Average: ${summary.average.toFixed(4)} ms`);
+  console.log(`p50: ${summary.p50.toFixed(4)} ms`);
+  console.log(`p90: ${summary.p90.toFixed(4)} ms`);
+  console.log(`p95: ${summary.p95.toFixed(4)} ms`);
+
+  if (best) {
+    console.log("");
+    console.log("Best match:");
+    console.log(`  File: ${best.chunk.file}`);
+    console.log(`  Line: ${best.chunk.startLine}-${best.chunk.endLine}`);
+    console.log(`  Symbol: ${best.chunk.name}`);
+    console.log(`  Score: ${best.score.toFixed(4)}`);
+  }
+}
+
+main().catch((error) => {
+  console.error(`Benchmark failed: ${error.message}`);
+  process.exitCode = 1;
+});

package/src/abort.js

@@ -0,0 +1,27 @@
+function createAbortError(signal) {
+  const reason = signal && signal.reason;
+
+  if (reason instanceof Error) {
+    return reason;
+  }
+
+  const error = new Error(reason ? String(reason) : "Operation aborted");
+  error.name = "AbortError";
+  return error;
+}
+
+function throwIfAborted(signal) {
+  if (signal && signal.aborted) {
+    throw createAbortError(signal);
+  }
+}
+
+function isAbortError(error) {
+  return Boolean(error && (error.name === "AbortError" || /aborted|interrupted/i.test(error.message || "")));
+}
+
+module.exports = {
+  createAbortError,
+  isAbortError,
+  throwIfAborted,
+};

package/src/cache.js

@@ -0,0 +1,100 @@
+const fs = require("fs/promises");
+const path = require("path");
+
+const CACHE_DIR = ".contextualizer";
+const CACHE_FILE = "cache.json";
+const CACHE_VERSION = 1;
+
+function getCachePath(root) {
+  return path.join(root, CACHE_DIR, CACHE_FILE);
+}
+
+function createEmptyCache(root) {
+  return {
+    version: CACHE_VERSION,
+    root,
+    updatedAt: null,
+    files: {},
+  };
+}
+
+function normalizeCache(cache, root) {
+  if (!cache || typeof cache !== "object") {
+    return createEmptyCache(root);
+  }
+
+  return {
+    version: cache.version || CACHE_VERSION,
+    root: cache.root || root,
+    updatedAt: cache.updatedAt || null,
+    files: cache.files && typeof cache.files === "object" ? cache.files : {},
+  };
+}
+
+function sortObjectByKey(value) {
+  return Object.keys(value)
+    .sort()
+    .reduce((sorted, key) => {
+      sorted[key] = value[key];
+      return sorted;
+    }, {});
+}
+
+async function loadCache(root) {
+  const cachePath = getCachePath(root);
+
+  try {
+    const raw = await fs.readFile(cachePath, "utf8");
+    const parsed = JSON.parse(raw);
+
+    return {
+      cache: normalizeCache(parsed, root),
+      exists: true,
+      cachePath,
+    };
+  } catch (error) {
+    if (error.code === "ENOENT") {
+      return {
+        cache: createEmptyCache(root),
+        exists: false,
+        cachePath,
+      };
+    }
+
+    if (error instanceof SyntaxError) {
+      throw new Error(`Cache file is not valid JSON: ${cachePath}`);
+    }
+
+    throw error;
+  }
+}
+
+async function saveCache(root, files) {
+  const cachePath = getCachePath(root);
+  const cacheDir = path.dirname(cachePath);
+  const cache = {
+    version: CACHE_VERSION,
+    root,
+    updatedAt: new Date().toISOString(),
+    files: sortObjectByKey(files),
+  };
+  const tmpPath = `${cachePath}.${process.pid}.${Date.now()}.tmp`;
+
+  await fs.mkdir(cacheDir, { recursive: true });
+  await fs.writeFile(tmpPath, `${JSON.stringify(cache, null, 2)}\n`, "utf8");
+  await fs.rename(tmpPath, cachePath);
+
+  return {
+    cache,
+    cachePath,
+  };
+}
+
+module.exports = {
+  CACHE_DIR,
+  CACHE_FILE,
+  CACHE_VERSION,
+  getCachePath,
+  loadCache,
+  saveCache,
+};

package/src/concurrency.js

@@ -0,0 +1,20 @@
+async function mapLimit(items, limit, mapper) {
+  const results = new Array(items.length);
+  const workerCount = Math.max(1, Math.min(limit, items.length || 1));
+  let nextIndex = 0;
+
+  async function runWorker() {
+    while (nextIndex < items.length) {
+      const currentIndex = nextIndex;
+      nextIndex += 1;
+      results[currentIndex] = await mapper(items[currentIndex], currentIndex);
+    }
+  }
+
+  await Promise.all(Array.from({ length: workerCount }, runWorker));
+  return results;
+}
+
+module.exports = {
+  mapLimit,
+};

package/src/database.js

@@ -0,0 +1,172 @@
+const fs = require("fs");
+const path = require("path");
+const Database = require("better-sqlite3");
+const { CACHE_DIR } = require("./cache");
+const { registerCleanup } = require("./shutdown");
+
+const DATABASE_FILE = "vector.db";
+const openDatabases = new Set();
+
+function closeOpenDatabases() {
+  for (const db of Array.from(openDatabases)) {
+    try {
+      db.close();
+    } catch {
+      openDatabases.delete(db);
+    }
+  }
+}
+
+registerCleanup(closeOpenDatabases);
+
+function getDatabasePath(root) {
+  return path.join(root, CACHE_DIR, DATABASE_FILE);
+}
+
+function databaseExists(root) {
+  return fs.existsSync(getDatabasePath(root));
+}
+
+function openDatabase(root) {
+  const databasePath = getDatabasePath(root);
+
+  fs.mkdirSync(path.dirname(databasePath), { recursive: true });
+
+  const db = new Database(databasePath);
+  const originalClose = db.close.bind(db);
+  let closed = false;
+
+  db.close = () => {
+    if (closed) {
+      return undefined;
+    }
+
+    closed = true;
+    openDatabases.delete(db);
+    return originalClose();
+  };
+
+  openDatabases.add(db);
+
+  db.pragma("journal_mode = WAL");
+  db.pragma("foreign_keys = ON");
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS files (
+      id INTEGER PRIMARY KEY,
+      path TEXT NOT NULL UNIQUE,
+      hash TEXT NOT NULL
+    );
+
+    CREATE TABLE IF NOT EXISTS chunks (
+      id INTEGER PRIMARY KEY,
+      file_id INTEGER NOT NULL,
+      name TEXT NOT NULL,
+      type TEXT NOT NULL,
+      startLine INTEGER NOT NULL,
+      endLine INTEGER NOT NULL,
+      code TEXT NOT NULL,
+      embedding BLOB NOT NULL,
+      FOREIGN KEY (file_id) REFERENCES files(id) ON DELETE CASCADE
+    );
+
+    CREATE INDEX IF NOT EXISTS idx_chunks_file_id ON chunks(file_id);
+  `);
+
+  return db;
+}
+
+function upsertFile(statements, file) {
+  const existing = statements.selectFile.get(file.path);
+
+  if (existing) {
+    statements.updateFile.run(file.hash, existing.id);
+    return existing.id;
+  }
+
+  const result = statements.insertFile.run(file.path, file.hash);
+  return Number(result.lastInsertRowid);
+}
+
+function writeEmbeddingResults(root, filesByRelativePath, embeddingSummary, removedPaths = []) {
+  const db = openDatabase(root);
+
+  try {
+    const statements = {
+      selectFile: db.prepare("SELECT id FROM files WHERE path = ?"),
+      insertFile: db.prepare("INSERT INTO files (path, hash) VALUES (?, ?)"),
+      updateFile: db.prepare("UPDATE files SET hash = ? WHERE id = ?"),
+      deleteFile: db.prepare("DELETE FROM files WHERE path = ?"),
+      deleteChunks: db.prepare("DELETE FROM chunks WHERE file_id = ?"),
+      insertChunk: db.prepare(`
+        INSERT INTO chunks (
+          file_id,
+          name,
+          type,
+          startLine,
+          endLine,
+          code,
+          embedding
+        ) VALUES (?, ?, ?, ?, ?, ?, ?)
+      `),
+    };
+    const persist = db.transaction((fileResults, deletedPaths) => {
+      for (const removedPath of deletedPaths) {
+        statements.deleteFile.run(removedPath);
+      }
+
+      for (const fileResult of fileResults) {
+        if (!fileResult.ok) {
+          continue;
+        }
+
+        const file = filesByRelativePath[fileResult.relativePath];
+
+        if (!file) {
+          continue;
+        }
+
+        const fileId = upsertFile(statements, {
+          path: fileResult.relativePath,
+          hash: file.hash,
+        });
+
+        statements.deleteChunks.run(fileId);
+
+        for (const chunk of fileResult.chunks) {
+          statements.insertChunk.run(
+            fileId,
+            chunk.name,
+            chunk.type,
+            chunk.startLine,
+            chunk.endLine,
+            chunk.code,
+            Buffer.from(chunk.embedding.buffer),
+          );
+        }
+      }
+    });
+
+    persist(embeddingSummary.files, removedPaths);
+
+    return {
+      databasePath: getDatabasePath(root),
+      filesRemoved: removedPaths.length,
+      filesWritten: embeddingSummary.files.filter((fileResult) => fileResult.ok).length,
+      chunksWritten: embeddingSummary.files.reduce(
+        (total, fileResult) => total + (fileResult.ok ? fileResult.chunks.length : 0),
+        0,
+      ),
+    };
+  } finally {
+    db.close();
+  }
+}
+
+module.exports = {
+  DATABASE_FILE,
+  closeOpenDatabases,
+  databaseExists,
+  getDatabasePath,
+  openDatabase,
+  writeEmbeddingResults,
+};

package/src/file-filter.js

@@ -0,0 +1,113 @@
+const path = require("path");
+
+const MAX_FILE_SIZE = 1024 * 1024;
+const MAX_LINE_LENGTH = 20_000;
+const MINIFIED_SOURCE_PATTERN = /(?:^|[.-])min\.(?:cjs|js|jsx|mjs|ts|tsx)$/i;
+const BINARY_EXTENSIONS = new Set([
+  ".7z",
+  ".a",
+  ".app",
+  ".avi",
+  ".bmp",
+  ".class",
+  ".dll",
+  ".dmg",
+  ".doc",
+  ".docx",
+  ".dylib",
+  ".eot",
+  ".exe",
+  ".gif",
+  ".gz",
+  ".ico",
+  ".jar",
+  ".jpeg",
+  ".jpg",
+  ".mov",
+  ".mp3",
+  ".mp4",
+  ".o",
+  ".obj",
+  ".otf",
+  ".pdf",
+  ".png",
+  ".ppt",
+  ".pptx",
+  ".rar",
+  ".so",
+  ".sqlite",
+  ".tar",
+  ".ttf",
+  ".wasm",
+  ".webm",
+  ".webp",
+  ".woff",
+  ".woff2",
+  ".xls",
+  ".xlsx",
+  ".zip",
+]);
+
+function isBinaryExtension(fileName) {
+  return BINARY_EXTENSIONS.has(path.extname(fileName).toLowerCase());
+}
+
+function isMinifiedSourceName(fileName) {
+  return MINIFIED_SOURCE_PATTERN.test(fileName);
+}
+
+function getFileSizeSkipReason(size) {
+  if (size > MAX_FILE_SIZE) {
+    return `file is larger than ${MAX_FILE_SIZE} bytes`;
+  }
+
+  return null;
+}
+
+function getSourcePathSkipReason(fileName) {
+  if (isMinifiedSourceName(fileName)) {
+    return "minified source file";
+  }
+
+  return null;
+}
+
+function getContentSkipReason(code) {
+  if (code.includes("\0")) {
+    return "binary content";
+  }
+
+  let currentLineLength = 0;
+  let maxLineLength = 0;
+
+  for (let index = 0; index < code.length; index += 1) {
+    const character = code[index];
+
+    if (character === "\n" || character === "\r") {
+      maxLineLength = Math.max(maxLineLength, currentLineLength);
+      currentLineLength = 0;
+      continue;
+    }
+
+    currentLineLength += 1;
+  }
+
+  maxLineLength = Math.max(maxLineLength, currentLineLength);
+
+  if (maxLineLength > MAX_LINE_LENGTH) {
+    return `line is longer than ${MAX_LINE_LENGTH} characters`;
+  }
+
+  return null;
+}
+
+module.exports = {
+  BINARY_EXTENSIONS,
+  MAX_FILE_SIZE,
+  MAX_LINE_LENGTH,
+  getContentSkipReason,
+  getFileSizeSkipReason,
+  getSourcePathSkipReason,
+  isBinaryExtension,
+  isMinifiedSourceName,
+};

package/src/file-hash.js

@@ -0,0 +1,17 @@
+const { createHash } = require("crypto");
+const { createReadStream } = require("fs");
+
+function hashFile(filePath) {
+  return new Promise((resolve, reject) => {
+    const hash = createHash("sha256");
+    const stream = createReadStream(filePath);
+
+    stream.on("data", (chunk) => hash.update(chunk));
+    stream.on("error", reject);
+    stream.on("end", () => resolve(hash.digest("hex")));
+  });
+}
+
+module.exports = {
+  hashFile,
+};