ctxloom-pro 1.5.6 → 1.7.1

package/apps/dashboard/package.json

@@ -15,7 +15,7 @@
   "dependencies": {
     "@ctxloom/core": "*",
     "cors": "^2.8.5",
-    "express": "^4.18.0",
+    "express": "^4.22.2",
     "open": "^10.1.0",
     "react-syntax-highlighter": "^16.1.1"
   },
@@ -36,7 +36,7 @@
     "clsx": "^2.1.1",
     "d3": "^7.9.0",
     "jsdom": "^24.0.0",
-    "postcss": "^8.4.0",
+    "postcss": "^8.5.15",
     "react": "^18.3.0",
     "react-dom": "^18.3.0",
     "react-router-dom": "^6.23.0",

package/dist/VectorStore-WDL3H7QT.js

@@ -0,0 +1,9 @@
+import {
+  VectorStore
+} from "./chunk-7S2ELKNU.js";
+import "./chunk-6FGTNOCP.js";
+import "./chunk-TYDMSHV7.js";
+export {
+  VectorStore
+};
+//# sourceMappingURL=VectorStore-WDL3H7QT.js.map

package/dist/chunk-6FGTNOCP.js

@@ -0,0 +1,397 @@
+import {
+  logger
+} from "./chunk-TYDMSHV7.js";
+
+// packages/core/src/indexer/embedder.ts
+import fs from "fs";
+import path from "path";
+var CHUNK_SIZE = 4096;
+var MODEL_REGISTRY = {
+  // The historical default. General English, 384-dim, ~90 MB.
+  // Kept as the free-tier default so existing users see zero change.
+  minilm: {
+    hfId: "sentence-transformers/all-MiniLM-L6-v2",
+    dim: 384,
+    minBytes: 80 * 1024 * 1024,
+    description: "General-purpose English sentence embedder (2020). 384-dim. The legacy default."
+  },
+  // Code-specific embedding model (Jina AI). 768-dim, ~140 MB.
+  // Empirically 20-40% better recall on code-similarity queries than
+  // MiniLM — the upgrade path recommended in the v1.7.0 analysis.
+  // Runs through the same @huggingface/transformers pipeline so the
+  // privacy story (fully local, no network at inference time) is
+  // preserved.
+  "jina-code": {
+    hfId: "jinaai/jina-embeddings-v2-base-code",
+    dim: 768,
+    minBytes: 130 * 1024 * 1024,
+    description: "Code-specific embedder (Jina, 2024). 768-dim. Better recall on code-similarity tasks."
+  }
+};
+function resolveEmbeddingModel(env = process.env) {
+  const envModel = env.CTXLOOM_EMBEDDING_MODEL?.trim();
+  if (!envModel) return MODEL_REGISTRY.minilm;
+  const registered = MODEL_REGISTRY[envModel];
+  if (registered) return registered;
+  const envDim = env.CTXLOOM_EMBEDDING_DIM ? Number.parseInt(env.CTXLOOM_EMBEDDING_DIM, 10) : NaN;
+  if (!Number.isFinite(envDim) || envDim <= 0) {
+    throw new Error(
+      `CTXLOOM_EMBEDDING_MODEL=${envModel} is not a known alias. Either use one of [${Object.keys(MODEL_REGISTRY).join(", ")}] or set CTXLOOM_EMBEDDING_DIM=<vector-length> alongside a raw HF id.`
+    );
+  }
+  return {
+    hfId: envModel,
+    dim: envDim,
+    // Without a known artifact size we can't enforce the truncated-download
+    // guard. Use 1 MB as the minimum; the worst case is a redundant retry
+    // rather than a hung process.
+    minBytes: 1024 * 1024,
+    description: `User-supplied model: ${envModel} (${envDim}-dim)`
+  };
+}
+var ACTIVE_MODEL = resolveEmbeddingModel();
+var EMBEDDING_DIMENSION = ACTIVE_MODEL.dim;
+var MODEL_ID = ACTIVE_MODEL.hfId;
+var MIN_MODEL_BYTES = ACTIVE_MODEL.minBytes;
+var embedder = null;
+var embedderInitInFlight = null;
+async function loadEmbedder() {
+  const { pipeline } = await import("@huggingface/transformers");
+  return await pipeline("feature-extraction", MODEL_ID, {
+    dtype: "fp32"
+  });
+}
+function extractModelPathFromProtobufError(message) {
+  const match = /Load model from (.+) failed:Protobuf parsing failed/i.exec(message);
+  return match ? match[1] : null;
+}
+function tryRemoveTruncatedModel(modelPath) {
+  try {
+    const stat = fs.statSync(modelPath);
+    if (stat.size >= MIN_MODEL_BYTES) return false;
+    fs.unlinkSync(modelPath);
+    logger.warn("Removed truncated embedding model; next attempt will re-download", {
+      path: modelPath,
+      sizeBytes: stat.size,
+      minBytes: MIN_MODEL_BYTES
+    });
+    return true;
+  } catch (err) {
+    logger.warn("Could not inspect/remove suspected truncated model", {
+      path: modelPath,
+      detail: err instanceof Error ? err.message : String(err)
+    });
+    return false;
+  }
+}
+async function getEmbedder() {
+  if (embedder) return embedder;
+  if (embedderInitInFlight) return embedderInitInFlight;
+  embedderInitInFlight = (async () => {
+    const MAX_ATTEMPTS = 3;
+    let lastErr;
+    for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
+      try {
+        const pipe = await loadEmbedder();
+        embedder = pipe;
+        return pipe;
+      } catch (err) {
+        lastErr = err;
+        const msg = err instanceof Error ? err.message : String(err);
+        const isProtobufRace = /protobuf parsing failed/i.test(msg);
+        if (!isProtobufRace || attempt === MAX_ATTEMPTS) break;
+        const modelPath = extractModelPathFromProtobufError(msg);
+        if (modelPath && tryRemoveTruncatedModel(modelPath)) {
+          logger.warn("Retrying embedding model load after truncated-cache removal", {
+            attempt
+          });
+          continue;
+        }
+        const delay = attempt * 1e3;
+        logger.warn("Embedding model load failed; retrying after FS settle", {
+          attempt,
+          delayMs: delay
+        });
+        await new Promise((resolve) => setTimeout(resolve, delay));
+      }
+    }
+    embedderInitInFlight = null;
+    throw lastErr;
+  })();
+  try {
+    return await embedderInitInFlight;
+  } finally {
+    if (embedder) embedderInitInFlight = null;
+  }
+}
+async function generateEmbedding(text) {
+  const pipe = await getEmbedder();
+  const output = await pipe(text.slice(0, CHUNK_SIZE), {
+    pooling: "mean",
+    normalize: true
+  });
+  const data = output.tolist();
+  if (Array.isArray(data[0])) {
+    return data[0];
+  }
+  return data;
+}
+async function generateEmbeddingBatch(texts) {
+  if (texts.length === 0) return [];
+  const pipe = await getEmbedder();
+  const truncated = texts.map((t) => t.slice(0, CHUNK_SIZE));
+  const output = await pipe(truncated, {
+    pooling: "mean",
+    normalize: true
+  });
+  const data = output.tolist();
+  if (!Array.isArray(data[0])) {
+    throw new Error(
+      `generateEmbeddingBatch: pipeline returned 1D tensor for ${texts.length} inputs (expected 2D [batch_size, embedding_dim]). Likely a transformers.js version regression.`
+    );
+  }
+  return data;
+}
+function collectFiles(dir, results = []) {
+  const IGNORED_DIRS = /* @__PURE__ */ new Set([
+    // Build artifacts + dependency caches
+    "node_modules",
+    "dist",
+    "build",
+    "out",
+    "target",
+    "coverage",
+    ".cache",
+    ".turbo",
+    ".next",
+    ".nuxt",
+    // Version control + ctxloom state
+    ".git",
+    ".ctxloom",
+    // Other tools' working state (often contains duplicated source)
+    ".claude",
+    ".code-review-graph",
+    ".vscode-test"
+  ]);
+  const SUPPORTED_EXTENSIONS = /* @__PURE__ */ new Set([
+    ".ts",
+    ".tsx",
+    ".js",
+    ".jsx",
+    ".mjs",
+    ".vue",
+    ".py",
+    ".rs",
+    ".go",
+    ".java",
+    ".cs",
+    ".rb",
+    ".kt",
+    ".kts",
+    ".swift",
+    ".php",
+    ".dart",
+    ".c",
+    ".cpp",
+    ".h",
+    ".md",
+    ".json",
+    ".yaml",
+    ".yml",
+    ".toml",
+    ".ipynb"
+  ]);
+  const entries = fs.readdirSync(dir, { withFileTypes: true });
+  for (const entry of entries) {
+    const fullPath = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      if (!IGNORED_DIRS.has(entry.name)) {
+        collectFiles(fullPath, results);
+      }
+    } else if (entry.isFile()) {
+      const ext = path.extname(entry.name);
+      if (SUPPORTED_EXTENSIONS.has(ext)) {
+        results.push(fullPath);
+      }
+    }
+  }
+  return results;
+}
+async function* collectFilesStream(dir) {
+  const entries = fs.readdirSync(dir, { withFileTypes: true });
+  for (const entry of entries) {
+    const fullPath = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      if (!INDEX_IGNORED_DIRS.has(entry.name)) {
+        yield* collectFilesStream(fullPath);
+      }
+    } else if (entry.isFile()) {
+      if (INDEX_SUPPORTED_EXTENSIONS.has(path.extname(entry.name))) {
+        yield fullPath;
+      }
+    }
+  }
+}
+var INDEXER_IGNORED_DIRS = /* @__PURE__ */ new Set([
+  // Build artifacts + dependency caches
+  "node_modules",
+  "dist",
+  "build",
+  "out",
+  "target",
+  "coverage",
+  ".cache",
+  ".turbo",
+  ".next",
+  ".nuxt",
+  // Version control + ctxloom state
+  ".git",
+  ".ctxloom",
+  // Other tools' working state (often contains duplicated source)
+  ".claude",
+  ".code-review-graph",
+  ".vscode-test"
+]);
+var INDEX_IGNORED_DIRS = INDEXER_IGNORED_DIRS;
+var INDEX_SUPPORTED_EXTENSIONS = /* @__PURE__ */ new Set([
+  ".ts",
+  ".tsx",
+  ".js",
+  ".jsx",
+  ".mjs",
+  ".vue",
+  ".py",
+  ".rs",
+  ".go",
+  ".java",
+  ".cs",
+  ".rb",
+  ".kt",
+  ".kts",
+  ".swift",
+  ".php",
+  ".dart",
+  ".c",
+  ".cpp",
+  ".h",
+  ".md",
+  ".json",
+  ".yaml",
+  ".yml",
+  ".toml",
+  ".ipynb"
+]);
+async function indexDirectory(rootDir, onProgress) {
+  const { VectorStore } = await import("./VectorStore-WDL3H7QT.js");
+  const store = new VectorStore(path.join(rootDir, ".ctxloom", "vectors.lancedb"));
+  await store.init();
+  let indexed = 0;
+  let errors = 0;
+  let processed = 0;
+  const total = 0;
+  const CONCURRENCY = 4;
+  const BATCH_SIZE = 50;
+  try {
+    let batch = [];
+    let chunk = [];
+    for await (const filePath of collectFilesStream(rootDir)) {
+      chunk.push(filePath);
+      if (chunk.length < CONCURRENCY) continue;
+      await processChunk(chunk);
+      chunk = [];
+    }
+    if (chunk.length > 0) {
+      await processChunk(chunk);
+    }
+    if (batch.length > 0) {
+      await store.upsertBatch(batch);
+      batch = [];
+    }
+    async function processChunk(paths) {
+      const readResults = await Promise.allSettled(
+        paths.map((filePath) => {
+          const MAX_INDEX_SIZE = 5 * 1024 * 1024;
+          const stat = fs.statSync(filePath);
+          if (stat.size > MAX_INDEX_SIZE) {
+            logger.warn("Skipping oversized file", { file: filePath, size: stat.size });
+            return null;
+          }
+          const content = fs.readFileSync(filePath, "utf-8");
+          if (!content.trim()) return null;
+          const relPath = path.relative(rootDir, filePath);
+          return { filePath: relPath, content };
+        })
+      );
+      const ready = [];
+      for (const r of readResults) {
+        processed++;
+        if (r.status === "fulfilled") {
+          if (r.value !== null) ready.push(r.value);
+        } else {
+          errors++;
+          logger.error("Failed to read file for indexing", {
+            detail: r.reason instanceof Error ? r.reason.message : String(r.reason)
+          });
+        }
+      }
+      if (ready.length === 0) return;
+      let embeddings;
+      try {
+        embeddings = await generateEmbeddingBatch(ready.map((r) => r.content));
+      } catch (err) {
+        errors += ready.length;
+        logger.error("Batch embedding failed; chunk lost", {
+          chunkSize: ready.length,
+          detail: err instanceof Error ? err.message : String(err)
+        });
+        return;
+      }
+      if (embeddings.length !== ready.length) {
+        errors += ready.length;
+        logger.error("Embedding batch length mismatch \u2014 chunk lost", {
+          expected: ready.length,
+          got: embeddings.length
+        });
+        return;
+      }
+      for (let i = 0; i < ready.length; i++) {
+        batch.push({
+          filePath: ready[i].filePath,
+          embedding: embeddings[i],
+          content: ready[i].content
+        });
+        indexed++;
+        onProgress?.(ready[i].filePath, processed, total);
+        if (batch.length >= BATCH_SIZE) {
+          await store.upsertBatch(batch);
+          batch = [];
+        }
+      }
+    }
+  } finally {
+    await store.close();
+  }
+  return { indexed, errors };
+}
+var EMBEDDING_MODEL_ID = MODEL_ID;
+function getActiveEmbeddingModel() {
+  return {
+    hfId: ACTIVE_MODEL.hfId,
+    dim: ACTIVE_MODEL.dim,
+    description: ACTIVE_MODEL.description
+  };
+}
+
+export {
+  resolveEmbeddingModel,
+  EMBEDDING_DIMENSION,
+  generateEmbedding,
+  generateEmbeddingBatch,
+  collectFiles,
+  collectFilesStream,
+  INDEXER_IGNORED_DIRS,
+  indexDirectory,
+  EMBEDDING_MODEL_ID,
+  getActiveEmbeddingModel
+};
+//# sourceMappingURL=chunk-6FGTNOCP.js.map