@vpxa/kb 0.1.1 → 0.1.3

package/packages/indexer/dist/incremental-indexer.js

@@ -1,278 +1 @@
-import { availableParallelism } from "node:os";
-import { createChunker } from "@kb/chunker";
-import { detectContentType } from "@kb/core";
-import { generateRecordId, hashContent } from "./file-hasher.js";
-import { FilesystemCrawler } from "./filesystem-crawler.js";
-import { extractGraph } from "./graph-extractor.js";
-async function runConcurrent(items, fn, concurrency, onError) {
-  let nextIndex = 0;
-  async function worker() {
-    while (nextIndex < items.length) {
-      const i = nextIndex++;
-      try {
-        await fn(items[i]);
-      } catch (err) {
-        onError?.(items[i], err);
-      }
-    }
-  }
-  await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, () => worker()));
-}
-const DEFAULT_CONCURRENCY = Math.max(1, Math.floor(availableParallelism() / 2));
-class IncrementalIndexer {
-  constructor(embedder, store) {
-    this.embedder = embedder;
-    this.store = store;
-    this.crawler = new FilesystemCrawler();
-  }
-  crawler;
-  indexing = false;
-  graphStore;
-  /** Set the graph store for auto-population during indexing and cleanup on re-index. */
-  setGraphStore(graphStore) {
-    this.graphStore = graphStore;
-  }
-  /**
-   * Index all configured sources. Only re-indexes files that have changed.
-   * Sources are crawled in parallel, and file processing runs concurrently
-   * up to `config.indexing.concurrency` (default: half of CPU cores).
-   */
-  async index(config, onProgress) {
-    if (this.indexing) {
-      throw new Error("Indexing is already in progress");
-    }
-    this.indexing = true;
-    try {
-      return await this.doIndex(config, onProgress, {});
-    } finally {
-      this.indexing = false;
-    }
-  }
-  async doIndex(config, onProgress, opts = {}) {
-    const startTime = Date.now();
-    let filesProcessed = 0;
-    let filesSkipped = 0;
-    let chunksCreated = 0;
-    let filesRemoved = 0;
-    const concurrency = config.indexing.concurrency ?? DEFAULT_CONCURRENCY;
-    onProgress?.({
-      phase: "crawling",
-      filesTotal: 0,
-      filesProcessed: 0,
-      chunksTotal: 0,
-      chunksProcessed: 0
-    });
-    const crawlResults = await Promise.all(
-      config.sources.map(
-        (source) => this.crawler.crawl({
-          rootDir: source.path,
-          excludePatterns: source.excludePatterns
-        })
-      )
-    );
-    const allFiles = crawlResults.flat();
-    let filesToProcess;
-    let pathsToRemove;
-    if (opts.skipHashCheck) {
-      filesToProcess = allFiles;
-      pathsToRemove = [];
-    } else {
-      const existingPaths = await this.store.listSourcePaths();
-      const currentPathSet = new Set(allFiles.map((f) => f.relativePath));
-      pathsToRemove = existingPaths.filter(
-        (p) => !currentPathSet.has(p) && !p.startsWith("curated/")
-      );
-      filesToProcess = [];
-      await runConcurrent(
-        allFiles,
-        async (file) => {
-          const contentHash = hashContent(file.content);
-          const existingRecords = await this.store.getBySourcePath(file.relativePath);
-          if (existingRecords.length > 0 && existingRecords[0].fileHash === contentHash) {
-            filesSkipped++;
-            return;
-          }
-          filesToProcess.push(file);
-        },
-        concurrency,
-        (file, err) => console.error(`[indexer] hash check failed for ${file.relativePath}:`, err)
-      );
-    }
-    const totalFiles = filesToProcess.length;
-    const GRAPH_FLUSH_INTERVAL = 50;
-    let pendingNodes = [];
-    let pendingEdges = [];
-    let graphFilesAccumulated = 0;
-    const flushGraph = async () => {
-      if (!this.graphStore) return;
-      try {
-        if (pendingNodes.length > 0) {
-          await this.graphStore.upsertNodes(pendingNodes);
-        }
-        if (pendingEdges.length > 0) {
-          await this.graphStore.upsertEdges(pendingEdges);
-        }
-      } catch (err) {
-        console.error("[indexer] graph batch flush failed:", err);
-      }
-      pendingNodes = [];
-      pendingEdges = [];
-      graphFilesAccumulated = 0;
-    };
-    await runConcurrent(
-      filesToProcess,
-      async (file) => {
-        onProgress?.({
-          phase: "chunking",
-          filesTotal: totalFiles,
-          filesProcessed,
-          chunksTotal: chunksCreated,
-          chunksProcessed: chunksCreated,
-          currentFile: file.relativePath
-        });
-        const contentType = detectContentType(file.relativePath);
-        const chunker = createChunker(file.extension);
-        const chunks = chunker.chunk(file.content, {
-          sourcePath: file.relativePath,
-          contentType
-        });
-        if (chunks.length === 0) return;
-        onProgress?.({
-          phase: "embedding",
-          filesTotal: totalFiles,
-          filesProcessed,
-          chunksTotal: chunksCreated + chunks.length,
-          chunksProcessed: chunksCreated,
-          currentFile: file.relativePath
-        });
-        const vectors = await this.embedder.embedBatch(chunks.map((c) => c.text));
-        const fileHash = hashContent(file.content);
-        const records = chunks.map((chunk, i) => ({
-          id: generateRecordId(file.relativePath, i),
-          content: chunk.text,
-          sourcePath: chunk.sourcePath,
-          contentType: chunk.contentType,
-          headingPath: chunk.headingPath,
-          chunkIndex: chunk.chunkIndex,
-          totalChunks: chunk.totalChunks,
-          startLine: chunk.startLine,
-          endLine: chunk.endLine,
-          fileHash,
-          indexedAt: (/* @__PURE__ */ new Date()).toISOString(),
-          origin: "indexed",
-          tags: [],
-          version: 1
-        }));
-        onProgress?.({
-          phase: "storing",
-          filesTotal: totalFiles,
-          filesProcessed,
-          chunksTotal: chunksCreated + chunks.length,
-          chunksProcessed: chunksCreated,
-          currentFile: file.relativePath
-        });
-        await this.store.upsert(records, vectors);
-        if (this.graphStore) {
-          try {
-            if (!opts.graphCleared) {
-              await this.graphStore.deleteBySourcePath(file.relativePath);
-            }
-            const graph = extractGraph(file.content, file.relativePath);
-            if (graph.nodes.length > 0) pendingNodes.push(...graph.nodes);
-            if (graph.edges.length > 0) pendingEdges.push(...graph.edges);
-            graphFilesAccumulated++;
-            if (graphFilesAccumulated >= GRAPH_FLUSH_INTERVAL) {
-              await flushGraph();
-            }
-          } catch (err) {
-            console.error(`[indexer] graph extraction failed for ${file.relativePath}:`, err);
-          }
-        }
-        filesProcessed++;
-        chunksCreated += chunks.length;
-      },
-      concurrency,
-      (file, err) => console.error(`[indexer] processing failed for ${file.relativePath}:`, err)
-    );
-    await flushGraph();
-    if (pathsToRemove.length > 0) {
-      onProgress?.({
-        phase: "cleanup",
-        filesTotal: totalFiles,
-        filesProcessed,
-        chunksTotal: chunksCreated,
-        chunksProcessed: chunksCreated
-      });
-      await runConcurrent(
-        pathsToRemove,
-        async (path) => {
-          await this.store.deleteBySourcePath(path);
-          if (this.graphStore) {
-            await this.graphStore.deleteBySourcePath(path).catch((err) => console.error(`[indexer] graph cleanup failed for ${path}:`, err));
-          }
-          filesRemoved++;
-        },
-        concurrency,
-        (path, err) => console.error(`[indexer] cleanup failed for ${path}:`, err)
-      );
-    }
-    onProgress?.({
-      phase: "done",
-      filesTotal: totalFiles,
-      filesProcessed,
-      chunksTotal: chunksCreated,
-      chunksProcessed: chunksCreated
-    });
-    return {
-      filesProcessed,
-      filesSkipped,
-      chunksCreated,
-      filesRemoved,
-      durationMs: Date.now() - startTime
-    };
-  }
-  /**
-   * Force re-index all files (ignoring hashes).
-   */
-  async reindexAll(config, onProgress) {
-    await this.store.dropTable();
-    if (this.graphStore) {
-      try {
-        const stats = await this.graphStore.getStats();
-        if (stats.nodeCount > 0) {
-          await this.graphStore.clear();
-          console.error(
-            `[indexer] Graph store cleared (was ${stats.nodeCount} nodes, ${stats.edgeCount} edges)`
-          );
-        }
-      } catch (err) {
-        console.error("[indexer] Graph store clear failed:", err);
-      }
-    }
-    return this.doReindex(config, onProgress);
-  }
-  async doReindex(config, onProgress) {
-    if (this.indexing) {
-      throw new Error("Indexing is already in progress");
-    }
-    this.indexing = true;
-    try {
-      return await this.doIndex(config, onProgress, {
-        skipHashCheck: true,
-        graphCleared: true
-      });
-    } finally {
-      this.indexing = false;
-    }
-  }
-  /**
-   * Get current index statistics.
-   */
-  async getStats() {
-    return this.store.getStats();
-  }
-}
-export {
-  IncrementalIndexer
-};
-//# sourceMappingURL=incremental-indexer.js.map
+import{availableParallelism as G}from"node:os";import{createChunker as F}from"../../chunker/dist/index.js";import{detectContentType as B}from"../../core/dist/index.js";import{generateRecordId as K,hashContent as I}from"./file-hasher.js";import{FilesystemCrawler as H}from"./filesystem-crawler.js";import{extractGraph as A}from"./graph-extractor.js";async function S(c,s,r,o){let g=0;async function l(){for(;g<c.length;){const d=g++;try{await s(c[d])}catch(a){o?.(c[d],a)}}}await Promise.all(Array.from({length:Math.min(r,c.length)},()=>l()))}const L=Math.max(1,Math.floor(G()/2));class O{constructor(s,r){this.embedder=s;this.store=r;this.crawler=new H}crawler;indexing=!1;graphStore;setGraphStore(s){this.graphStore=s}async index(s,r){if(this.indexing)throw new Error("Indexing is already in progress");this.indexing=!0;try{return await this.doIndex(s,r,{})}finally{this.indexing=!1}}async doIndex(s,r,o={}){const g=Date.now();let l=0,d=0,a=0,C=0;const w=s.indexing.concurrency??L;r?.({phase:"crawling",filesTotal:0,filesProcessed:0,chunksTotal:0,chunksProcessed:0});const y=(await Promise.all(s.sources.map(e=>this.crawler.crawl({rootDir:e.path,excludePatterns:e.excludePatterns})))).flat();let p,f;if(o.skipHashCheck)p=y,f=[];else{const e=await this.store.listSourcePaths(),h=new Set(y.map(n=>n.relativePath));f=e.filter(n=>!h.has(n)&&!n.startsWith("curated/")),p=[],await S(y,async n=>{const i=I(n.content),P=await this.store.getBySourcePath(n.relativePath);if(P.length>0&&P[0].fileHash===i){d++;return}p.push(n)},w,(n,i)=>console.error(`[indexer] hash check failed for ${n.relativePath}:`,i))}const u=p.length,b=50;let m=[],x=[],k=0;const v=async()=>{if(this.graphStore){try{m.length>0&&await this.graphStore.upsertNodes(m),x.length>0&&await this.graphStore.upsertEdges(x)}catch(e){console.error("[indexer] graph batch flush failed:",e)}m=[],x=[],k=0}};return await S(p,async e=>{r?.({phase:"chunking",filesTotal:u,filesProcessed:l,chunksTotal:a,chunksProcessed:a,currentFile:e.relativePath});const h=B(e.relativePath),i=F(e.extension).chunk(e.content,{sourcePath:e.relativePath,contentType:h});if(i.length===0)return;r?.({phase:"embedding",filesTotal:u,filesProcessed:l,chunksTotal:a+i.length,chunksProcessed:a,currentFile:e.relativePath});const P=await this.embedder.embedBatch(i.map(t=>t.text)),T=I(e.content),R=i.map((t,E)=>({id:K(e.relativePath,E),content:t.text,sourcePath:t.sourcePath,contentType:t.contentType,headingPath:t.headingPath,chunkIndex:t.chunkIndex,totalChunks:t.totalChunks,startLine:t.startLine,endLine:t.endLine,fileHash:T,indexedAt:new Date().toISOString(),origin:"indexed",tags:[],version:1}));if(r?.({phase:"storing",filesTotal:u,filesProcessed:l,chunksTotal:a+i.length,chunksProcessed:a,currentFile:e.relativePath}),await this.store.upsert(R,P),this.graphStore)try{o.graphCleared||await this.graphStore.deleteBySourcePath(e.relativePath);const t=A(e.content,e.relativePath);t.nodes.length>0&&m.push(...t.nodes),t.edges.length>0&&x.push(...t.edges),k++,k>=b&&await v()}catch(t){console.error(`[indexer] graph extraction failed for ${e.relativePath}:`,t)}l++,a+=i.length},w,(e,h)=>console.error(`[indexer] processing failed for ${e.relativePath}:`,h)),await v(),f.length>0&&(r?.({phase:"cleanup",filesTotal:u,filesProcessed:l,chunksTotal:a,chunksProcessed:a}),await S(f,async e=>{await this.store.deleteBySourcePath(e),this.graphStore&&await this.graphStore.deleteBySourcePath(e).catch(h=>console.error(`[indexer] graph cleanup failed for ${e}:`,h)),C++},w,(e,h)=>console.error(`[indexer] cleanup failed for ${e}:`,h))),r?.({phase:"done",filesTotal:u,filesProcessed:l,chunksTotal:a,chunksProcessed:a}),{filesProcessed:l,filesSkipped:d,chunksCreated:a,filesRemoved:C,durationMs:Date.now()-g}}async reindexAll(s,r){if(await this.store.dropTable(),this.graphStore)try{const o=await this.graphStore.getStats();o.nodeCount>0&&(await this.graphStore.clear(),console.error(`[indexer] Graph store cleared (was ${o.nodeCount} nodes, ${o.edgeCount} edges)`))}catch(o){console.error("[indexer] Graph store clear failed:",o)}return this.doReindex(s,r)}async doReindex(s,r){if(this.indexing)throw new Error("Indexing is already in progress");this.indexing=!0;try{return await this.doIndex(s,r,{skipHashCheck:!0,graphCleared:!0})}finally{this.indexing=!1}}async getStats(){return this.store.getStats()}}export{O as IncrementalIndexer};

package/packages/indexer/dist/index.js

@@ -1,14 +1 @@
-import { generateRecordId, hashContent } from "./file-hasher.js";
-import { FilesystemCrawler } from "./filesystem-crawler.js";
-import { extractGraph } from "./graph-extractor.js";
-import {
-  IncrementalIndexer
-} from "./incremental-indexer.js";
-export {
-  FilesystemCrawler,
-  IncrementalIndexer,
-  extractGraph,
-  generateRecordId,
-  hashContent
-};
-//# sourceMappingURL=index.js.map
+import{generateRecordId as t,hashContent as o}from"./file-hasher.js";import{FilesystemCrawler as a}from"./filesystem-crawler.js";import{extractGraph as l}from"./graph-extractor.js";import{IncrementalIndexer as x}from"./incremental-indexer.js";export{a as FilesystemCrawler,x as IncrementalIndexer,l as extractGraph,t as generateRecordId,o as hashContent};

package/packages/server/dist/api.js

@@ -1,9 +1 @@
-import { loadConfig } from "./config.js";
-import { createMcpServer, createServer, initializeKnowledgeBase } from "./server.js";
-export {
-  createMcpServer,
-  createServer,
-  initializeKnowledgeBase,
-  loadConfig
-};
-//# sourceMappingURL=api.js.map
+import{loadConfig as o}from"./config.js";import{createMcpServer as a,createServer as n,initializeKnowledgeBase as p}from"./server.js";export{a as createMcpServer,n as createServer,p as initializeKnowledgeBase,o as loadConfig};

package/packages/server/dist/config.js

@@ -1,75 +1 @@
-import { existsSync, readFileSync } from "node:fs";
-import { dirname, resolve } from "node:path";
-import { fileURLToPath } from "node:url";
-const __dirname = dirname(fileURLToPath(import.meta.url));
-function guardContainedPath(resolved, root, label) {
-  const normalizedResolved = resolve(resolved);
-  const normalizedRoot = resolve(root);
-  if (!normalizedResolved.startsWith(normalizedRoot)) {
-    throw new Error(
-      `Config ${label} path escapes workspace root: ${resolved} is not under ${root}`
-    );
-  }
-  return normalizedResolved;
-}
-function loadConfig() {
-  const configPath = process.env.KB_CONFIG_PATH ?? (existsSync(resolve(process.cwd(), "kb.config.json")) ? resolve(process.cwd(), "kb.config.json") : resolve(__dirname, "..", "..", "..", "kb.config.json"));
-  try {
-    const raw = readFileSync(configPath, "utf-8");
-    const config = JSON.parse(raw);
-    if (!config.sources || !Array.isArray(config.sources) || config.sources.length === 0) {
-      throw new Error("Config must have at least one source");
-    }
-    if (!config.store?.path) {
-      throw new Error("Config must specify store.path");
-    }
-    const workspaceRoot = dirname(configPath);
-    config.sources = config.sources.map((source) => ({
-      ...source,
-      path: guardContainedPath(resolve(workspaceRoot, source.path), workspaceRoot, "source")
-    }));
-    config.store.path = guardContainedPath(
-      resolve(workspaceRoot, config.store.path),
-      workspaceRoot,
-      "store"
-    );
-    config.curated = config.curated ?? { path: "curated" };
-    config.curated.path = guardContainedPath(
-      resolve(workspaceRoot, config.curated.path),
-      workspaceRoot,
-      "curated"
-    );
-    return config;
-  } catch (err) {
-    const message = err instanceof Error ? err.message : String(err);
-    console.error(`[KB] Failed to load config from ${configPath}: ${message}`);
-    console.error("[KB] Falling back to default configuration");
-    return getDefaultConfig();
-  }
-}
-function getDefaultConfig() {
-  const workspaceRoot = process.env.KB_WORKSPACE_ROOT ?? process.cwd();
-  return {
-    sources: [
-      {
-        path: workspaceRoot,
-        excludePatterns: [
-          "node_modules/**",
-          "dist/**",
-          ".git/**",
-          "coverage/**",
-          "*.lock",
-          "pnpm-lock.yaml"
-        ]
-      }
-    ],
-    indexing: { chunkSize: 1500, chunkOverlap: 200, minChunkSize: 100 },
-    embedding: { model: "mixedbread-ai/mxbai-embed-large-v1", dimensions: 1024 },
-    store: { backend: "lancedb", path: resolve(workspaceRoot, ".kb-data") },
-    curated: { path: resolve(workspaceRoot, "curated") }
-  };
-}
-export {
-  loadConfig
-};
-//# sourceMappingURL=config.js.map
+import{existsSync as i,readFileSync as d}from"node:fs";import{dirname as c,resolve as r}from"node:path";import{fileURLToPath as u}from"node:url";const p=c(u(import.meta.url));function a(e,n,o){const t=r(e),s=r(n);if(!t.startsWith(s))throw new Error(`Config ${o} path escapes workspace root: ${e} is not under ${n}`);return t}function l(){const e=process.env.KB_CONFIG_PATH??(i(r(process.cwd(),"kb.config.json"))?r(process.cwd(),"kb.config.json"):r(p,"..","..","..","kb.config.json"));try{const n=d(e,"utf-8"),o=JSON.parse(n);if(!o.sources||!Array.isArray(o.sources)||o.sources.length===0)throw new Error("Config must have at least one source");if(!o.store?.path)throw new Error("Config must specify store.path");const t=c(e);return o.sources=o.sources.map(s=>({...s,path:a(r(t,s.path),t,"source")})),o.store.path=a(r(t,o.store.path),t,"store"),o.curated=o.curated??{path:"curated"},o.curated.path=a(r(t,o.curated.path),t,"curated"),o}catch(n){const o=n instanceof Error?n.message:String(n);return console.error(`[KB] Failed to load config from ${e}: ${o}`),console.error("[KB] Falling back to default configuration"),f()}}function f(){const e=process.env.KB_WORKSPACE_ROOT??process.cwd();return{sources:[{path:e,excludePatterns:["node_modules/**","dist/**",".git/**","coverage/**","*.lock","pnpm-lock.yaml"]}],indexing:{chunkSize:1500,chunkOverlap:200,minChunkSize:100},embedding:{model:"mixedbread-ai/mxbai-embed-large-v1",dimensions:1024},store:{backend:"lancedb",path:r(e,".kb-data")},curated:{path:r(e,"curated")}}}export{l as loadConfig};