ctxloom-pro 1.5.6 → 1.7.0

package/dist/chunk-WDX4PJGL.js

@@ -1,214 +0,0 @@
-import {
-  logger
-} from "./chunk-TYDMSHV7.js";
-
-// packages/core/src/indexer/embedder.ts
-import fs from "fs";
-import path from "path";
-var EMBEDDING_DIMENSION = 384;
-var MODEL_ID = "sentence-transformers/all-MiniLM-L6-v2";
-var CHUNK_SIZE = 4096;
-var MIN_MODEL_BYTES = 80 * 1024 * 1024;
-var embedder = null;
-var embedderInitInFlight = null;
-async function loadEmbedder() {
-  const { pipeline } = await import("@huggingface/transformers");
-  return await pipeline("feature-extraction", MODEL_ID, {
-    dtype: "fp32"
-  });
-}
-function extractModelPathFromProtobufError(message) {
-  const match = /Load model from (.+) failed:Protobuf parsing failed/i.exec(message);
-  return match ? match[1] : null;
-}
-function tryRemoveTruncatedModel(modelPath) {
-  try {
-    const stat = fs.statSync(modelPath);
-    if (stat.size >= MIN_MODEL_BYTES) return false;
-    fs.unlinkSync(modelPath);
-    logger.warn("Removed truncated embedding model; next attempt will re-download", {
-      path: modelPath,
-      sizeBytes: stat.size,
-      minBytes: MIN_MODEL_BYTES
-    });
-    return true;
-  } catch (err) {
-    logger.warn("Could not inspect/remove suspected truncated model", {
-      path: modelPath,
-      detail: err instanceof Error ? err.message : String(err)
-    });
-    return false;
-  }
-}
-async function getEmbedder() {
-  if (embedder) return embedder;
-  if (embedderInitInFlight) return embedderInitInFlight;
-  embedderInitInFlight = (async () => {
-    const MAX_ATTEMPTS = 3;
-    let lastErr;
-    for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
-      try {
-        const pipe = await loadEmbedder();
-        embedder = pipe;
-        return pipe;
-      } catch (err) {
-        lastErr = err;
-        const msg = err instanceof Error ? err.message : String(err);
-        const isProtobufRace = /protobuf parsing failed/i.test(msg);
-        if (!isProtobufRace || attempt === MAX_ATTEMPTS) break;
-        const modelPath = extractModelPathFromProtobufError(msg);
-        if (modelPath && tryRemoveTruncatedModel(modelPath)) {
-          logger.warn("Retrying embedding model load after truncated-cache removal", {
-            attempt
-          });
-          continue;
-        }
-        const delay = attempt * 1e3;
-        logger.warn("Embedding model load failed; retrying after FS settle", {
-          attempt,
-          delayMs: delay
-        });
-        await new Promise((resolve) => setTimeout(resolve, delay));
-      }
-    }
-    embedderInitInFlight = null;
-    throw lastErr;
-  })();
-  try {
-    return await embedderInitInFlight;
-  } finally {
-    if (embedder) embedderInitInFlight = null;
-  }
-}
-async function generateEmbedding(text) {
-  const pipe = await getEmbedder();
-  const output = await pipe(text.slice(0, CHUNK_SIZE), {
-    pooling: "mean",
-    normalize: true
-  });
-  const data = output.tolist();
-  if (Array.isArray(data[0])) {
-    return data[0];
-  }
-  return data;
-}
-function collectFiles(dir, results = []) {
-  const IGNORED_DIRS = /* @__PURE__ */ new Set([
-    // Build artifacts + dependency caches
-    "node_modules",
-    "dist",
-    "build",
-    "out",
-    "target",
-    "coverage",
-    ".cache",
-    ".turbo",
-    ".next",
-    ".nuxt",
-    // Version control + ctxloom state
-    ".git",
-    ".ctxloom",
-    // Other tools' working state (often contains duplicated source)
-    ".claude",
-    ".code-review-graph",
-    ".vscode-test"
-  ]);
-  const SUPPORTED_EXTENSIONS = /* @__PURE__ */ new Set([
-    ".ts",
-    ".tsx",
-    ".js",
-    ".jsx",
-    ".mjs",
-    ".vue",
-    ".py",
-    ".rs",
-    ".go",
-    ".java",
-    ".cs",
-    ".rb",
-    ".kt",
-    ".kts",
-    ".swift",
-    ".php",
-    ".dart",
-    ".c",
-    ".cpp",
-    ".h",
-    ".md",
-    ".json",
-    ".yaml",
-    ".yml",
-    ".toml",
-    ".ipynb"
-  ]);
-  const entries = fs.readdirSync(dir, { withFileTypes: true });
-  for (const entry of entries) {
-    const fullPath = path.join(dir, entry.name);
-    if (entry.isDirectory()) {
-      if (!IGNORED_DIRS.has(entry.name)) {
-        collectFiles(fullPath, results);
-      }
-    } else if (entry.isFile()) {
-      const ext = path.extname(entry.name);
-      if (SUPPORTED_EXTENSIONS.has(ext)) {
-        results.push(fullPath);
-      }
-    }
-  }
-  return results;
-}
-async function indexDirectory(rootDir, onProgress) {
-  const { VectorStore } = await import("./VectorStore-2LVECRTY.js");
-  const store = new VectorStore(path.join(rootDir, ".ctxloom", "vectors.lancedb"));
-  await store.init();
-  const files = collectFiles(rootDir);
-  const total = files.length;
-  let indexed = 0;
-  let errors = 0;
-  let processed = 0;
-  const CONCURRENCY = 4;
-  try {
-    for (let i = 0; i < files.length; i += CONCURRENCY) {
-      const batch = files.slice(i, i + CONCURRENCY);
-      const results = await Promise.allSettled(
-        batch.map(async (filePath) => {
-          const MAX_INDEX_SIZE = 5 * 1024 * 1024;
-          const stat = fs.statSync(filePath);
-          if (stat.size > MAX_INDEX_SIZE) {
-            logger.warn("Skipping oversized file", { file: filePath, size: stat.size });
-            return null;
-          }
-          const content = fs.readFileSync(filePath, "utf-8");
-          if (!content.trim()) return null;
-          const relPath = path.relative(rootDir, filePath);
-          const embedding = await generateEmbedding(content);
-          await store.upsert(relPath, embedding, content);
-          return relPath;
-        })
-      );
-      for (const result of results) {
-        processed++;
-        if (result.status === "fulfilled") {
-          if (result.value !== null) {
-            indexed++;
-            onProgress?.(result.value, processed, total);
-          }
-        } else {
-          errors++;
-          logger.error("Failed to index file", { detail: result.reason instanceof Error ? result.reason.message : String(result.reason) });
-        }
-      }
-    }
-  } finally {
-    await store.close();
-  }
-  return { indexed, errors };
-}
-
-export {
-  EMBEDDING_DIMENSION,
-  generateEmbedding,
-  collectFiles,
-  indexDirectory
-};
-//# sourceMappingURL=chunk-WDX4PJGL.js.map

package/dist/embedder-3AE4CSR7.js

@@ -1,14 +0,0 @@
-import {
-  EMBEDDING_DIMENSION,
-  collectFiles,
-  generateEmbedding,
-  indexDirectory
-} from "./chunk-WDX4PJGL.js";
-import "./chunk-TYDMSHV7.js";
-export {
-  EMBEDDING_DIMENSION,
-  collectFiles,
-  generateEmbedding,
-  indexDirectory
-};
-//# sourceMappingURL=embedder-3AE4CSR7.js.map