ctxloom-pro 1.0.4

package/dist/chunk-XNKTZGDX.js

@@ -0,0 +1,125 @@
+import {
+  logger
+} from "./chunk-IHXVD5SO.js";
+
+// src/db/VectorStore.js
+import lancedb from "@lancedb/lancedb";
+import { makeArrowTable } from "@lancedb/lancedb";
+import path from "path";
+import fs from "fs";
+function sanitizeFilterPath(filePath) {
+  return filePath.replace(/[^a-zA-Z0-9/._\- ]/g, "_");
+}
+var VectorStore = class {
+  dbPath;
+  db = null;
+  table = null;
+  initialized = false;
+  constructor(dbPath) {
+    this.dbPath = dbPath ?? path.join(process.cwd(), ".ctxloom", "vectors.lancedb");
+  }
+  async init() {
+    if (this.initialized)
+      return;
+    const dir = path.dirname(this.dbPath);
+    if (!fs.existsSync(dir)) {
+      fs.mkdirSync(dir, { recursive: true });
+    }
+    this.db = await lancedb.connect(this.dbPath);
+    const existingTables = await this.db.tableNames();
+    if (existingTables.includes("code_embeddings")) {
+      this.table = await this.db.openTable("code_embeddings");
+    } else {
+      const seedTable = makeArrowTable([
+        {
+          id: "__seed__",
+          filePath: "__seed__",
+          vector: new Array(384).fill(0),
+          content: ""
+        }
+      ]);
+      this.table = await this.db.createTable("code_embeddings", seedTable);
+      await this.table.delete("id = '__seed__'");
+    }
+    this.initialized = true;
+  }
+  /**
+   * Insert or update a code record.
+   */
+  async upsert(filePath, embedding, content) {
+    if (!this.table)
+      throw new Error("VectorStore not initialized. Call init() first.");
+    const safe = sanitizeFilterPath(filePath);
+    try {
+      await this.table.delete(`filePath = '${safe}'`);
+    } catch (err) {
+      logger.warn("Delete before upsert failed, continuing", { detail: err instanceof Error ? err.message : String(err) });
+    }
+    const record = {
+      id: filePath,
+      filePath,
+      vector: embedding,
+      content: content.slice(0, 512)
+    };
+    await this.table.add([record]);
+  }
+  /**
+   * Search for the top-K most similar code records using vector search.
+   */
+  async search(queryEmbedding, limit = 10) {
+    if (!this.table)
+      throw new Error("VectorStore not initialized. Call init() first.");
+    try {
+      const results = await this.table.vectorSearch(queryEmbedding).limit(limit).toArray();
+      return results.filter((r) => r.id !== "__seed__").map((r) => ({
+        filePath: String(r.filePath ?? r.id),
+        content: String(r.content ?? ""),
+        score: Number(r._distance ?? 0)
+      }));
+    } catch (err) {
+      logger.warn("Search failed, attempting to create index", { detail: String(err) });
+      try {
+        await this.table.createIndex("vector");
+        const results = await this.table.vectorSearch(queryEmbedding).limit(limit).toArray();
+        return results.filter((r) => r.id !== "__seed__").map((r) => ({
+          filePath: String(r.filePath ?? r.id),
+          content: String(r.content ?? ""),
+          score: Number(r._distance ?? 0)
+        }));
+      } catch {
+        return [];
+      }
+    }
+  }
+  /**
+   * Remove a file's embedding from the store.
+   */
+  async remove(filePath) {
+    if (!this.table)
+      throw new Error("VectorStore not initialized. Call init() first.");
+    const safe = sanitizeFilterPath(filePath);
+    try {
+      await this.table.delete(`filePath = '${safe}'`);
+    } catch (err) {
+      logger.error("Remove failed", { detail: err instanceof Error ? err.message : String(err) });
+    }
+  }
+  /**
+   * Get the total number of records.
+   */
+  async count() {
+    if (!this.table)
+      return 0;
+    try {
+      return await this.table.countRows();
+    } catch (err) {
+      logger.error("countRows failed", { detail: err instanceof Error ? err.message : String(err) });
+      return 0;
+    }
+  }
+};
+
+export {
+  VectorStore
+};
+//# sourceMappingURL=chunk-XNKTZGDX.js.map

package/dist/chunk-ZYDVY7VZ.js

@@ -0,0 +1,140 @@
+import {
+  logger
+} from "./chunk-IHXVD5SO.js";
+
+// src/indexer/embedder.js
+import { pipeline } from "@huggingface/transformers";
+import fs from "fs";
+import path from "path";
+var EMBEDDING_DIMENSION = 384;
+var MODEL_ID = "sentence-transformers/all-MiniLM-L6-v2";
+var CHUNK_SIZE = 4096;
+var embedder = null;
+async function getEmbedder() {
+  if (!embedder) {
+    embedder = await pipeline("feature-extraction", MODEL_ID, {
+      dtype: "fp32"
+    });
+  }
+  return embedder;
+}
+async function generateEmbedding(text) {
+  const pipe = await getEmbedder();
+  const output = await pipe(text.slice(0, CHUNK_SIZE), {
+    pooling: "mean",
+    normalize: true
+  });
+  const data = output.tolist();
+  if (Array.isArray(data[0])) {
+    return data[0];
+  }
+  return data;
+}
+function collectFiles(dir, results = []) {
+  const IGNORED_DIRS = /* @__PURE__ */ new Set([
+    "node_modules",
+    ".git",
+    "dist",
+    "build",
+    ".ctxloom",
+    "coverage",
+    ".next",
+    ".nuxt",
+    "out",
+    ".cache",
+    ".turbo"
+  ]);
+  const SUPPORTED_EXTENSIONS = /* @__PURE__ */ new Set([
+    ".ts",
+    ".tsx",
+    ".js",
+    ".jsx",
+    ".mjs",
+    ".vue",
+    ".py",
+    ".rs",
+    ".go",
+    ".java",
+    ".cs",
+    ".rb",
+    ".kt",
+    ".kts",
+    ".swift",
+    ".php",
+    ".dart",
+    ".c",
+    ".cpp",
+    ".h",
+    ".md",
+    ".json",
+    ".yaml",
+    ".yml",
+    ".toml",
+    ".ipynb"
+  ]);
+  const entries = fs.readdirSync(dir, { withFileTypes: true });
+  for (const entry of entries) {
+    const fullPath = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      if (!IGNORED_DIRS.has(entry.name)) {
+        collectFiles(fullPath, results);
+      }
+    } else if (entry.isFile()) {
+      const ext = path.extname(entry.name);
+      if (SUPPORTED_EXTENSIONS.has(ext)) {
+        results.push(fullPath);
+      }
+    }
+  }
+  return results;
+}
+async function indexDirectory(rootDir, onProgress) {
+  const { VectorStore } = await import("./VectorStore-UQNBYPBV.js");
+  const store = new VectorStore(path.join(rootDir, ".ctxloom", "vectors.lancedb"));
+  await store.init();
+  const files = collectFiles(rootDir);
+  const total = files.length;
+  let indexed = 0;
+  let errors = 0;
+  let processed = 0;
+  const CONCURRENCY = 4;
+  for (let i = 0; i < files.length; i += CONCURRENCY) {
+    const batch = files.slice(i, i + CONCURRENCY);
+    const results = await Promise.allSettled(batch.map(async (filePath) => {
+      const MAX_INDEX_SIZE = 5 * 1024 * 1024;
+      const stat = fs.statSync(filePath);
+      if (stat.size > MAX_INDEX_SIZE) {
+        logger.warn("Skipping oversized file", { file: filePath, size: stat.size });
+        return null;
+      }
+      const content = fs.readFileSync(filePath, "utf-8");
+      if (!content.trim())
+        return null;
+      const relPath = path.relative(rootDir, filePath);
+      const embedding = await generateEmbedding(content);
+      await store.upsert(relPath, embedding, content);
+      return relPath;
+    }));
+    for (const result of results) {
+      processed++;
+      if (result.status === "fulfilled") {
+        if (result.value !== null) {
+          indexed++;
+          onProgress?.(result.value, processed, total);
+        }
+      } else {
+        errors++;
+        logger.error("Failed to index file", { detail: result.reason instanceof Error ? result.reason.message : String(result.reason) });
+      }
+    }
+  }
+  return { indexed, errors };
+}
+
+export {
+  EMBEDDING_DIMENSION,
+  generateEmbedding,
+  collectFiles,
+  indexDirectory
+};
+//# sourceMappingURL=chunk-ZYDVY7VZ.js.map

package/dist/dashboard-LVSRXGZN.js

@@ -0,0 +1,13 @@
+// src/dashboard.ts
+import path from "path";
+import { fileURLToPath } from "url";
+var __dirname = path.dirname(fileURLToPath(import.meta.url));
+async function startDashboard(options) {
+  const serverPath = path.resolve(__dirname, "../apps/dashboard/server/index.js");
+  const mod = await import(serverPath);
+  await mod.startDashboard(options);
+}
+export {
+  startDashboard
+};
+//# sourceMappingURL=dashboard-LVSRXGZN.js.map

package/dist/embedder-RECRKXTB.js

@@ -0,0 +1,14 @@
+import {
+  EMBEDDING_DIMENSION,
+  collectFiles,
+  generateEmbedding,
+  indexDirectory
+} from "./chunk-ZYDVY7VZ.js";
+import "./chunk-IHXVD5SO.js";
+export {
+  EMBEDDING_DIMENSION,
+  collectFiles,
+  generateEmbedding,
+  indexDirectory
+};
+//# sourceMappingURL=embedder-RECRKXTB.js.map

package/dist/index.d.ts

@@ -0,0 +1 @@
+#!/usr/bin/env node