ctxloom-pro 1.5.5 → 1.7.0

package/apps/dashboard/package.json

@@ -15,7 +15,7 @@
   "dependencies": {
     "@ctxloom/core": "*",
     "cors": "^2.8.5",
-    "express": "^4.18.0",
+    "express": "^4.22.2",
     "open": "^10.1.0",
     "react-syntax-highlighter": "^16.1.1"
   },
@@ -36,7 +36,7 @@
     "clsx": "^2.1.1",
     "d3": "^7.9.0",
     "jsdom": "^24.0.0",
-    "postcss": "^8.4.0",
+    "postcss": "^8.5.15",
     "react": "^18.3.0",
     "react-dom": "^18.3.0",
     "react-router-dom": "^6.23.0",

package/dist/VectorStore-WDL3H7QT.js

@@ -0,0 +1,9 @@
+import {
+  VectorStore
+} from "./chunk-7S2ELKNU.js";
+import "./chunk-6FGTNOCP.js";
+import "./chunk-TYDMSHV7.js";
+export {
+  VectorStore
+};
+//# sourceMappingURL=VectorStore-WDL3H7QT.js.map

package/dist/chunk-6FGTNOCP.js

@@ -0,0 +1,397 @@
+import {
+  logger
+} from "./chunk-TYDMSHV7.js";
+
+// packages/core/src/indexer/embedder.ts
+import fs from "fs";
+import path from "path";
+var CHUNK_SIZE = 4096;
+var MODEL_REGISTRY = {
+  // The historical default. General English, 384-dim, ~90 MB.
+  // Kept as the free-tier default so existing users see zero change.
+  minilm: {
+    hfId: "sentence-transformers/all-MiniLM-L6-v2",
+    dim: 384,
+    minBytes: 80 * 1024 * 1024,
+    description: "General-purpose English sentence embedder (2020). 384-dim. The legacy default."
+  },
+  // Code-specific embedding model (Jina AI). 768-dim, ~140 MB.
+  // Empirically 20-40% better recall on code-similarity queries than
+  // MiniLM — the upgrade path recommended in the v1.7.0 analysis.
+  // Runs through the same @huggingface/transformers pipeline so the
+  // privacy story (fully local, no network at inference time) is
+  // preserved.
+  "jina-code": {
+    hfId: "jinaai/jina-embeddings-v2-base-code",
+    dim: 768,
+    minBytes: 130 * 1024 * 1024,
+    description: "Code-specific embedder (Jina, 2024). 768-dim. Better recall on code-similarity tasks."
+  }
+};
+function resolveEmbeddingModel(env = process.env) {
+  const envModel = env.CTXLOOM_EMBEDDING_MODEL?.trim();
+  if (!envModel) return MODEL_REGISTRY.minilm;
+  const registered = MODEL_REGISTRY[envModel];
+  if (registered) return registered;
+  const envDim = env.CTXLOOM_EMBEDDING_DIM ? Number.parseInt(env.CTXLOOM_EMBEDDING_DIM, 10) : NaN;
+  if (!Number.isFinite(envDim) || envDim <= 0) {
+    throw new Error(
+      `CTXLOOM_EMBEDDING_MODEL=${envModel} is not a known alias. Either use one of [${Object.keys(MODEL_REGISTRY).join(", ")}] or set CTXLOOM_EMBEDDING_DIM=<vector-length> alongside a raw HF id.`
+    );
+  }
+  return {
+    hfId: envModel,
+    dim: envDim,
+    // Without a known artifact size we can't enforce the truncated-download
+    // guard. Use 1 MB as the minimum; the worst case is a redundant retry
+    // rather than a hung process.
+    minBytes: 1024 * 1024,
+    description: `User-supplied model: ${envModel} (${envDim}-dim)`
+  };
+}
+var ACTIVE_MODEL = resolveEmbeddingModel();
+var EMBEDDING_DIMENSION = ACTIVE_MODEL.dim;
+var MODEL_ID = ACTIVE_MODEL.hfId;
+var MIN_MODEL_BYTES = ACTIVE_MODEL.minBytes;
+var embedder = null;
+var embedderInitInFlight = null;
+async function loadEmbedder() {
+  const { pipeline } = await import("@huggingface/transformers");
+  return await pipeline("feature-extraction", MODEL_ID, {
+    dtype: "fp32"
+  });
+}
+function extractModelPathFromProtobufError(message) {
+  const match = /Load model from (.+) failed:Protobuf parsing failed/i.exec(message);
+  return match ? match[1] : null;
+}
+function tryRemoveTruncatedModel(modelPath) {
+  try {
+    const stat = fs.statSync(modelPath);
+    if (stat.size >= MIN_MODEL_BYTES) return false;
+    fs.unlinkSync(modelPath);
+    logger.warn("Removed truncated embedding model; next attempt will re-download", {
+      path: modelPath,
+      sizeBytes: stat.size,
+      minBytes: MIN_MODEL_BYTES
+    });
+    return true;
+  } catch (err) {
+    logger.warn("Could not inspect/remove suspected truncated model", {
+      path: modelPath,
+      detail: err instanceof Error ? err.message : String(err)
+    });
+    return false;
+  }
+}
+async function getEmbedder() {
+  if (embedder) return embedder;
+  if (embedderInitInFlight) return embedderInitInFlight;
+  embedderInitInFlight = (async () => {
+    const MAX_ATTEMPTS = 3;
+    let lastErr;
+    for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
+      try {
+        const pipe = await loadEmbedder();
+        embedder = pipe;
+        return pipe;
+      } catch (err) {
+        lastErr = err;
+        const msg = err instanceof Error ? err.message : String(err);
+        const isProtobufRace = /protobuf parsing failed/i.test(msg);
+        if (!isProtobufRace || attempt === MAX_ATTEMPTS) break;
+        const modelPath = extractModelPathFromProtobufError(msg);
+        if (modelPath && tryRemoveTruncatedModel(modelPath)) {
+          logger.warn("Retrying embedding model load after truncated-cache removal", {
+            attempt
+          });
+          continue;
+        }
+        const delay = attempt * 1e3;
+        logger.warn("Embedding model load failed; retrying after FS settle", {
+          attempt,
+          delayMs: delay
+        });
+        await new Promise((resolve) => setTimeout(resolve, delay));
+      }
+    }
+    embedderInitInFlight = null;
+    throw lastErr;
+  })();
+  try {
+    return await embedderInitInFlight;
+  } finally {
+    if (embedder) embedderInitInFlight = null;
+  }
+}
+async function generateEmbedding(text) {
+  const pipe = await getEmbedder();
+  const output = await pipe(text.slice(0, CHUNK_SIZE), {
+    pooling: "mean",
+    normalize: true
+  });
+  const data = output.tolist();
+  if (Array.isArray(data[0])) {
+    return data[0];
+  }
+  return data;
+}
+async function generateEmbeddingBatch(texts) {
+  if (texts.length === 0) return [];
+  const pipe = await getEmbedder();
+  const truncated = texts.map((t) => t.slice(0, CHUNK_SIZE));
+  const output = await pipe(truncated, {
+    pooling: "mean",
+    normalize: true
+  });
+  const data = output.tolist();
+  if (!Array.isArray(data[0])) {
+    throw new Error(
+      `generateEmbeddingBatch: pipeline returned 1D tensor for ${texts.length} inputs (expected 2D [batch_size, embedding_dim]). Likely a transformers.js version regression.`
+    );
+  }
+  return data;
+}
+function collectFiles(dir, results = []) {
+  const IGNORED_DIRS = /* @__PURE__ */ new Set([
+    // Build artifacts + dependency caches
+    "node_modules",
+    "dist",
+    "build",
+    "out",
+    "target",
+    "coverage",
+    ".cache",
+    ".turbo",
+    ".next",
+    ".nuxt",
+    // Version control + ctxloom state
+    ".git",
+    ".ctxloom",
+    // Other tools' working state (often contains duplicated source)
+    ".claude",
+    ".code-review-graph",
+    ".vscode-test"
+  ]);
+  const SUPPORTED_EXTENSIONS = /* @__PURE__ */ new Set([
+    ".ts",
+    ".tsx",
+    ".js",
+    ".jsx",
+    ".mjs",
+    ".vue",
+    ".py",
+    ".rs",
+    ".go",
+    ".java",
+    ".cs",
+    ".rb",
+    ".kt",
+    ".kts",
+    ".swift",
+    ".php",
+    ".dart",
+    ".c",
+    ".cpp",
+    ".h",
+    ".md",
+    ".json",
+    ".yaml",
+    ".yml",
+    ".toml",
+    ".ipynb"
+  ]);
+  const entries = fs.readdirSync(dir, { withFileTypes: true });
+  for (const entry of entries) {
+    const fullPath = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      if (!IGNORED_DIRS.has(entry.name)) {
+        collectFiles(fullPath, results);
+      }
+    } else if (entry.isFile()) {
+      const ext = path.extname(entry.name);
+      if (SUPPORTED_EXTENSIONS.has(ext)) {
+        results.push(fullPath);
+      }
+    }
+  }
+  return results;
+}
+async function* collectFilesStream(dir) {
+  const entries = fs.readdirSync(dir, { withFileTypes: true });
+  for (const entry of entries) {
+    const fullPath = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      if (!INDEX_IGNORED_DIRS.has(entry.name)) {
+        yield* collectFilesStream(fullPath);
+      }
+    } else if (entry.isFile()) {
+      if (INDEX_SUPPORTED_EXTENSIONS.has(path.extname(entry.name))) {
+        yield fullPath;
+      }
+    }
+  }
+}
+var INDEXER_IGNORED_DIRS = /* @__PURE__ */ new Set([
+  // Build artifacts + dependency caches
+  "node_modules",
+  "dist",
+  "build",
+  "out",
+  "target",
+  "coverage",
+  ".cache",
+  ".turbo",
+  ".next",
+  ".nuxt",
+  // Version control + ctxloom state
+  ".git",
+  ".ctxloom",
+  // Other tools' working state (often contains duplicated source)
+  ".claude",
+  ".code-review-graph",
+  ".vscode-test"
+]);
+var INDEX_IGNORED_DIRS = INDEXER_IGNORED_DIRS;
+var INDEX_SUPPORTED_EXTENSIONS = /* @__PURE__ */ new Set([
+  ".ts",
+  ".tsx",
+  ".js",
+  ".jsx",
+  ".mjs",
+  ".vue",
+  ".py",
+  ".rs",
+  ".go",
+  ".java",
+  ".cs",
+  ".rb",
+  ".kt",
+  ".kts",
+  ".swift",
+  ".php",
+  ".dart",
+  ".c",
+  ".cpp",
+  ".h",
+  ".md",
+  ".json",
+  ".yaml",
+  ".yml",
+  ".toml",
+  ".ipynb"
+]);
+async function indexDirectory(rootDir, onProgress) {
+  const { VectorStore } = await import("./VectorStore-WDL3H7QT.js");
+  const store = new VectorStore(path.join(rootDir, ".ctxloom", "vectors.lancedb"));
+  await store.init();
+  let indexed = 0;
+  let errors = 0;
+  let processed = 0;
+  const total = 0;
+  const CONCURRENCY = 4;
+  const BATCH_SIZE = 50;
+  try {
+    let batch = [];
+    let chunk = [];
+    for await (const filePath of collectFilesStream(rootDir)) {
+      chunk.push(filePath);
+      if (chunk.length < CONCURRENCY) continue;
+      await processChunk(chunk);
+      chunk = [];
+    }
+    if (chunk.length > 0) {
+      await processChunk(chunk);
+    }
+    if (batch.length > 0) {
+      await store.upsertBatch(batch);
+      batch = [];
+    }
+    async function processChunk(paths) {
+      const readResults = await Promise.allSettled(
+        paths.map((filePath) => {
+          const MAX_INDEX_SIZE = 5 * 1024 * 1024;
+          const stat = fs.statSync(filePath);
+          if (stat.size > MAX_INDEX_SIZE) {
+            logger.warn("Skipping oversized file", { file: filePath, size: stat.size });
+            return null;
+          }
+          const content = fs.readFileSync(filePath, "utf-8");
+          if (!content.trim()) return null;
+          const relPath = path.relative(rootDir, filePath);
+          return { filePath: relPath, content };
+        })
+      );
+      const ready = [];
+      for (const r of readResults) {
+        processed++;
+        if (r.status === "fulfilled") {
+          if (r.value !== null) ready.push(r.value);
+        } else {
+          errors++;
+          logger.error("Failed to read file for indexing", {
+            detail: r.reason instanceof Error ? r.reason.message : String(r.reason)
+          });
+        }
+      }
+      if (ready.length === 0) return;
+      let embeddings;
+      try {
+        embeddings = await generateEmbeddingBatch(ready.map((r) => r.content));
+      } catch (err) {
+        errors += ready.length;
+        logger.error("Batch embedding failed; chunk lost", {
+          chunkSize: ready.length,
+          detail: err instanceof Error ? err.message : String(err)
+        });
+        return;
+      }
+      if (embeddings.length !== ready.length) {
+        errors += ready.length;
+        logger.error("Embedding batch length mismatch \u2014 chunk lost", {
+          expected: ready.length,
+          got: embeddings.length
+        });
+        return;
+      }
+      for (let i = 0; i < ready.length; i++) {
+        batch.push({
+          filePath: ready[i].filePath,
+          embedding: embeddings[i],
+          content: ready[i].content
+        });
+        indexed++;
+        onProgress?.(ready[i].filePath, processed, total);
+        if (batch.length >= BATCH_SIZE) {
+          await store.upsertBatch(batch);
+          batch = [];
+        }
+      }
+    }
+  } finally {
+    await store.close();
+  }
+  return { indexed, errors };
+}
+var EMBEDDING_MODEL_ID = MODEL_ID;
+function getActiveEmbeddingModel() {
+  return {
+    hfId: ACTIVE_MODEL.hfId,
+    dim: ACTIVE_MODEL.dim,
+    description: ACTIVE_MODEL.description
+  };
+}
+
+export {
+  resolveEmbeddingModel,
+  EMBEDDING_DIMENSION,
+  generateEmbedding,
+  generateEmbeddingBatch,
+  collectFiles,
+  collectFilesStream,
+  INDEXER_IGNORED_DIRS,
+  indexDirectory,
+  EMBEDDING_MODEL_ID,
+  getActiveEmbeddingModel
+};
+//# sourceMappingURL=chunk-6FGTNOCP.js.map

package/dist/{chunk-R56D54Y7.js → chunk-7S2ELKNU.js}

@@ -1,3 +1,7 @@
+import {
+  EMBEDDING_DIMENSION,
+  EMBEDDING_MODEL_ID
+} from "./chunk-6FGTNOCP.js";
 import {
   logger
 } from "./chunk-TYDMSHV7.js";
@@ -34,8 +38,12 @@ var VectorStore = class {
       fs.mkdirSync(dir, { recursive: true });
     }
     const lancedb = await import("@lancedb/lancedb");
+    const lanceModule = lancedb.default ?? lancedb;
+    const lance = lanceModule;
     const { makeArrowTable } = lancedb;
-    this.db = await lancedb.default.connect(this.dbPath);
+    this.db = await lance.connect(this.dbPath);
+    const markerPath = path.join(path.dirname(this.dbPath), "embedding-model.json");
+    this.assertModelCompatibility(markerPath);
     const existingTables = await this.db.tableNames();
     if (existingTables.includes("code_embeddings")) {
       this.table = await this.db.openTable("code_embeddings");
@@ -44,7 +52,7 @@ var VectorStore = class {
         {
           id: "__seed__",
           filePath: "__seed__",
-          vector: new Array(384).fill(0),
+          vector: new Array(EMBEDDING_DIMENSION).fill(0),
           content: ""
         }
       ]);
@@ -53,6 +61,47 @@ var VectorStore = class {
     }
     this.initialized = true;
   }
+  /**
+   * Compare the active embedding model against the marker file written
+   * when the index was first built. Three cases:
+   *
+   *   1. Marker missing  → legacy index (pre-v1.7.0). Assume MiniLM
+   *      and write the marker so future runs are guarded.
+   *   2. Marker matches  → proceed silently.
+   *   3. Marker differs  → throw with a clear migration instruction.
+   *      We don't auto-wipe — silently dropping a user's index because
+   *      they set an env var is exactly the footgun the marker exists
+   *      to prevent.
+   */
+  assertModelCompatibility(markerPath) {
+    const active = { model: EMBEDDING_MODEL_ID, dim: EMBEDDING_DIMENSION };
+    let existing = null;
+    if (fs.existsSync(markerPath)) {
+      try {
+        existing = JSON.parse(fs.readFileSync(markerPath, "utf-8"));
+      } catch (err) {
+        logger.warn("Embedding-model marker is corrupt; treating as missing", {
+          path: markerPath,
+          detail: err instanceof Error ? err.message : String(err)
+        });
+      }
+    }
+    if (!existing) {
+      fs.writeFileSync(markerPath, JSON.stringify(active, null, 2));
+      return;
+    }
+    if (existing.model === active.model && existing.dim === active.dim) {
+      return;
+    }
+    throw new Error(
+      `Embedding-model mismatch: vector index at ${this.dbPath} was built with "${existing.model}" (${existing.dim}-dim) but the active model is "${active.model}" (${active.dim}-dim). Re-index required:
+
+    ctxloom vectors-cleanup --reset
+    ctxloom index
+
+Or revert CTXLOOM_EMBEDDING_MODEL to "${existing.model}" to keep the existing index.`
+    );
+  }
   /**
    * Release LanceDB resources (file descriptors held by the underlying
    * connection / table handles). Must be called at the end of long-lived
@@ -101,6 +150,48 @@ var VectorStore = class {
       await this.compact();
     }
   }
+  /**
+   * Batch upsert — one LanceDB transaction for N file records.
+   *
+   * Why this exists (v1.7.0 monorepo support): the per-file `upsert()`
+   * does 2 LanceDB transactions (delete + add). On a 50k-file repo
+   * that's 100k transactions, each writing a manifest + transaction
+   * file. Compaction reclaims them every 200 upserts, but during the
+   * indexing burst the FD churn is enormous (observed: ~10k FDs/sec
+   * peak on Next.js, hitting the 256-default macOS process limit).
+   *
+   * With BATCH_SIZE=50 callers (the streaming indexer), this drops to
+   * ~2k transactions across the same corpus — 50× fewer manifests
+   * written, ~5× faster wall time, and the FD ceiling stops mattering.
+   *
+   * Compaction semantics unchanged: `upsertsSinceCompact` increments by
+   * one per RECORD (not per batch), so the compact trigger fires at
+   * the same total-record cadence as the per-file path.
+   */
+  async upsertBatch(records) {
+    if (!this.table) throw new Error("VectorStore not initialized. Call init() first.");
+    if (records.length === 0) return;
+    const filter = records.map((r) => `filePath = '${sanitizeFilterPath(r.filePath)}'`).join(" OR ");
+    try {
+      await this.table.delete(filter);
+    } catch (err) {
+      logger.warn("Batch delete before upsert failed, continuing", {
+        detail: err instanceof Error ? err.message : String(err)
+      });
+    }
+    const rows = records.map((r) => ({
+      id: r.filePath,
+      filePath: r.filePath,
+      vector: r.embedding,
+      content: r.content.slice(0, 512)
+    }));
+    await this.table.add(rows);
+    this.upsertsSinceCompact += records.length;
+    if (this.upsertsSinceCompact >= this.compactEvery) {
+      this.upsertsSinceCompact = 0;
+      await this.compact();
+    }
+  }
   /**
    * Merge fragments and prune old LanceDB versions. Idempotent and safe to
    * call mid-flight; the Table API serializes writes internally. Called
@@ -114,15 +205,52 @@ var VectorStore = class {
     if (!this.table) return;
     try {
       const optimizable = this.table;
-      await optimizable.optimize?.({
+      const result = await optimizable.optimize?.({
         cleanupOlderThan: new Date(Date.now() - this.cleanupOlderThanMs)
       });
+      if (result) {
+        logger.info("VectorStore.compact", {
+          fragmentsRemoved: result.compaction?.fragmentsRemoved ?? 0,
+          fragmentsAdded: result.compaction?.fragmentsAdded ?? 0,
+          oldVersionsRemoved: result.prune?.oldVersionsRemoved ?? 0,
+          bytesRemoved: result.prune?.bytesRemoved ?? 0
+        });
+      }
     } catch (err) {
       logger.warn("VectorStore.compact failed (non-fatal)", {
         detail: err instanceof Error ? err.message : String(err)
       });
     }
   }
+  /**
+   * Retrieve the stored embedding for a known file. Returns null if the
+   * file isn't indexed. Used by blast-radius semantic-similarity search
+   * (v1.6.x) to find files semantically related to a seed without
+   * re-embedding the seed at query time.
+   */
+  async findEmbeddingByPath(filePath) {
+    if (!this.table) throw new Error("VectorStore not initialized. Call init() first.");
+    try {
+      const safe = sanitizeFilterPath(filePath);
+      const rows = await this.table.query().where(`filePath = '${safe}'`).limit(1).toArray();
+      const first = rows[0];
+      if (!first) return null;
+      const vec = first.vector;
+      if (vec === null || vec === void 0) return null;
+      if (Array.isArray(vec)) return vec;
+      if (vec instanceof Float32Array) return Array.from(vec);
+      if (typeof vec.length === "number") {
+        return Array.from(vec);
+      }
+      return null;
+    } catch (err) {
+      logger.warn("VectorStore.findEmbeddingByPath failed", {
+        detail: err instanceof Error ? err.message : String(err),
+        filePath
+      });
+      return null;
+    }
+  }
   /**
    * Search for the top-K most similar code records using vector search.
    */
@@ -179,4 +307,4 @@ var VectorStore = class {
 export {
   VectorStore
 };
-//# sourceMappingURL=chunk-R56D54Y7.js.map
+//# sourceMappingURL=chunk-7S2ELKNU.js.map