@vpxa/kb 0.1.1

package/packages/indexer/dist/filesystem-crawler.js

@@ -0,0 +1,125 @@
+import { lstat, readdir, readFile, stat } from "node:fs/promises";
+import { extname, join, relative } from "node:path";
+import { FILE_LIMITS } from "@kb/core";
+import { minimatch } from "minimatch";
+class FilesystemCrawler {
+  /** Binary file extensions that should always be skipped */
+  static BINARY_EXTENSIONS = /* @__PURE__ */ new Set([
+    ".node",
+    ".so",
+    ".dylib",
+    ".dll",
+    ".wasm",
+    ".bin",
+    ".exe",
+    ".png",
+    ".jpg",
+    ".jpeg",
+    ".gif",
+    ".bmp",
+    ".ico",
+    ".webp",
+    ".svg",
+    ".mp3",
+    ".mp4",
+    ".wav",
+    ".avi",
+    ".mov",
+    ".flac",
+    ".zip",
+    ".gz",
+    ".tar",
+    ".bz2",
+    ".7z",
+    ".rar",
+    ".pdf",
+    ".doc",
+    ".docx",
+    ".xls",
+    ".xlsx",
+    ".ppt",
+    ".pptx",
+    ".ttf",
+    ".otf",
+    ".woff",
+    ".woff2",
+    ".eot",
+    ".pyc",
+    ".class",
+    ".o",
+    ".obj",
+    ".a",
+    ".lib"
+  ]);
+  async crawl(options) {
+    const results = [];
+    const visited = /* @__PURE__ */ new Set();
+    await this.walkDir(options.rootDir, options.rootDir, options.excludePatterns, results, visited);
+    return results;
+  }
+  async walkDir(dir, rootDir, excludePatterns, results, visited) {
+    let entries;
+    try {
+      entries = await readdir(dir, { withFileTypes: true });
+    } catch (err) {
+      const code = err.code;
+      if (code === "EACCES" || code === "EPERM") {
+        console.error(`[KB] Permission denied, skipping directory: ${dir}`);
+      }
+      return;
+    }
+    for (const entry of entries) {
+      const fullPath = join(dir, entry.name);
+      const relPath = relative(rootDir, fullPath).replace(/\\/g, "/");
+      if (this.isExcluded(relPath, excludePatterns)) {
+        continue;
+      }
+      if (entry.isDirectory()) {
+        if (entry.name.startsWith(".")) {
+          continue;
+        }
+        try {
+          const linkStat = await lstat(fullPath);
+          if (linkStat.isSymbolicLink()) {
+            continue;
+          }
+        } catch {
+          continue;
+        }
+        const realPath = fullPath;
+        if (visited.has(realPath)) continue;
+        visited.add(realPath);
+        await this.walkDir(fullPath, rootDir, excludePatterns, results, visited);
+      } else if (entry.isFile()) {
+        const ext = extname(entry.name).toLowerCase();
+        if (FilesystemCrawler.BINARY_EXTENSIONS.has(ext)) {
+          continue;
+        }
+        try {
+          const fileStat = await stat(fullPath);
+          if (fileStat.size > FILE_LIMITS.maxFileSizeBytes) {
+            continue;
+          }
+          const content = await readFile(fullPath, "utf-8");
+          if (content.includes("\0")) {
+            continue;
+          }
+          results.push({
+            relativePath: relPath,
+            absolutePath: fullPath,
+            content,
+            extension: ext
+          });
+        } catch {
+        }
+      }
+    }
+  }
+  isExcluded(relativePath, patterns) {
+    return patterns.some((pattern) => minimatch(relativePath, pattern, { dot: true }));
+  }
+}
+export {
+  FilesystemCrawler
+};
+//# sourceMappingURL=filesystem-crawler.js.map

package/packages/indexer/dist/graph-extractor.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Lightweight graph extraction for automatic population during indexing.
+ * Extracts symbols (functions, classes, interfaces, etc.) and import relationships
+ * from source code, producing GraphNode[] and GraphEdge[] for the graph store.
+ *
+ * Focuses on TS/JS — the primary use case. Non-code files return empty graphs.
+ */
+import type { GraphEdge, GraphNode } from '@kb/store';
+export interface ExtractedGraph {
+    nodes: GraphNode[];
+    edges: GraphEdge[];
+}
+/**
+ * Extract graph nodes and edges from a single file's content.
+ * Produces:
+ * - One "module" node per file
+ * - Symbol nodes (function, class, interface, type, const, enum)
+ * - "defines" edges from module → symbol
+ * - "imports" edges from module → imported module
+ */
+export declare function extractGraph(content: string, sourcePath: string): ExtractedGraph;
+//# sourceMappingURL=graph-extractor.d.ts.map

package/packages/indexer/dist/graph-extractor.js

@@ -0,0 +1,111 @@
+import { createHash } from "node:crypto";
+import { dirname, extname, join } from "node:path";
+const CODE_EXTENSIONS = /* @__PURE__ */ new Set([".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs"]);
+const SYMBOL_PATTERNS = [
+  // Exported
+  { pattern: /^export\s+(?:async\s+)?function\s+(\w+)/gm, kind: "function", exported: true },
+  { pattern: /^export\s+(?:default\s+)?class\s+(\w+)/gm, kind: "class", exported: true },
+  { pattern: /^export\s+interface\s+(\w+)/gm, kind: "interface", exported: true },
+  { pattern: /^export\s+type\s+(\w+)/gm, kind: "type", exported: true },
+  { pattern: /^export\s+(?:const|let)\s+(\w+)/gm, kind: "const", exported: true },
+  { pattern: /^export\s+enum\s+(\w+)/gm, kind: "enum", exported: true },
+  // Non-exported (top-level only)
+  { pattern: /^(?:async\s+)?function\s+(\w+)/gm, kind: "function", exported: false },
+  { pattern: /^class\s+(\w+)/gm, kind: "class", exported: false },
+  { pattern: /^interface\s+(\w+)/gm, kind: "interface", exported: false },
+  { pattern: /^type\s+(\w+)/gm, kind: "type", exported: false },
+  { pattern: /^enum\s+(\w+)/gm, kind: "enum", exported: false }
+];
+const IMPORT_PATTERNS = [
+  // ES import: import { x } from 'module' / import x from 'module' / import type { x } from '...'
+  /import\s+(?:(?:type\s+)?(?:(?:\{[^}]*\}|[\w*]+)\s+from\s+)?)['"]([^'"]+)['"]/g,
+  // Dynamic import: import('module')
+  /import\(\s*['"]([^'"]+)['"]\s*\)/g,
+  // CommonJS require: require('module')
+  /require\(\s*['"]([^'"]+)['"]\s*\)/g
+];
+function nodeId(type, name, sourcePath) {
+  return createHash("sha256").update(`${type}:${name}:${sourcePath}`).digest("hex").slice(0, 16);
+}
+function edgeId(fromId, toId, type) {
+  return createHash("sha256").update(`${fromId}-${type}-${toId}`).digest("hex").slice(0, 16);
+}
+function resolveImportPath(importSource, fromPath) {
+  const dir = dirname(fromPath);
+  const resolved = join(dir, importSource).replace(/\\/g, "/");
+  return resolved.replace(/\.(js|jsx|ts|tsx|mjs|cjs)$/, "");
+}
+function moduleKey(sourcePath) {
+  return sourcePath.replace(/\.(js|jsx|ts|tsx|mjs|cjs)$/, "");
+}
+function extractGraph(content, sourcePath) {
+  const ext = extname(sourcePath).toLowerCase();
+  if (!CODE_EXTENSIONS.has(ext)) {
+    return { nodes: [], edges: [] };
+  }
+  const nodes = [];
+  const edges = [];
+  const now = (/* @__PURE__ */ new Date()).toISOString();
+  const seenSymbols = /* @__PURE__ */ new Set();
+  const mk = moduleKey(sourcePath);
+  const modId = nodeId("module", mk, mk);
+  nodes.push({
+    id: modId,
+    type: "module",
+    name: sourcePath,
+    properties: { ext },
+    sourcePath,
+    createdAt: now
+  });
+  for (const { pattern, kind, exported } of SYMBOL_PATTERNS) {
+    const regex = new RegExp(pattern.source, pattern.flags);
+    let match;
+    while ((match = regex.exec(content)) !== null) {
+      const name = match[1];
+      const key = `${kind}:${name}`;
+      if (seenSymbols.has(key)) continue;
+      seenSymbols.add(key);
+      const symId = nodeId(kind, name, sourcePath);
+      nodes.push({
+        id: symId,
+        type: kind,
+        name,
+        properties: { exported },
+        sourcePath,
+        createdAt: now
+      });
+      edges.push({
+        id: edgeId(modId, symId, "defines"),
+        fromId: modId,
+        toId: symId,
+        type: "defines",
+        weight: exported ? 1 : 0.5
+      });
+    }
+  }
+  const seenImports = /* @__PURE__ */ new Set();
+  for (const pattern of IMPORT_PATTERNS) {
+    const regex = new RegExp(pattern.source, pattern.flags);
+    let match;
+    while ((match = regex.exec(content)) !== null) {
+      const source = match[1];
+      if (!source.startsWith(".")) continue;
+      if (seenImports.has(source)) continue;
+      seenImports.add(source);
+      const targetKey = resolveImportPath(source, sourcePath);
+      const targetModId = nodeId("module", targetKey, targetKey);
+      edges.push({
+        id: edgeId(modId, targetModId, "imports"),
+        fromId: modId,
+        toId: targetModId,
+        type: "imports",
+        properties: { source }
+      });
+    }
+  }
+  return { nodes, edges };
+}
+export {
+  extractGraph
+};
+//# sourceMappingURL=graph-extractor.js.map

package/packages/indexer/dist/incremental-indexer.d.ts

@@ -0,0 +1,47 @@
+import type { IndexStats, KBConfig } from '@kb/core';
+import type { IEmbedder } from '@kb/embeddings';
+import type { IGraphStore, IKnowledgeStore } from '@kb/store';
+export interface IndexProgress {
+    phase: 'crawling' | 'chunking' | 'embedding' | 'storing' | 'cleanup' | 'done';
+    filesTotal: number;
+    filesProcessed: number;
+    chunksTotal: number;
+    chunksProcessed: number;
+    /** The file currently being processed */
+    currentFile?: string;
+}
+export type ProgressCallback = (progress: IndexProgress) => void;
+export interface IndexResult {
+    filesProcessed: number;
+    filesSkipped: number;
+    chunksCreated: number;
+    filesRemoved: number;
+    durationMs: number;
+}
+export declare class IncrementalIndexer {
+    private readonly embedder;
+    private readonly store;
+    private readonly crawler;
+    private indexing;
+    private graphStore?;
+    constructor(embedder: IEmbedder, store: IKnowledgeStore);
+    /** Set the graph store for auto-population during indexing and cleanup on re-index. */
+    setGraphStore(graphStore: IGraphStore): void;
+    /**
+     * Index all configured sources. Only re-indexes files that have changed.
+     * Sources are crawled in parallel, and file processing runs concurrently
+     * up to `config.indexing.concurrency` (default: half of CPU cores).
+     */
+    index(config: KBConfig, onProgress?: ProgressCallback): Promise<IndexResult>;
+    private doIndex;
+    /**
+     * Force re-index all files (ignoring hashes).
+     */
+    reindexAll(config: KBConfig, onProgress?: ProgressCallback): Promise<IndexResult>;
+    private doReindex;
+    /**
+     * Get current index statistics.
+     */
+    getStats(): Promise<IndexStats>;
+}
+//# sourceMappingURL=incremental-indexer.d.ts.map

package/packages/indexer/dist/incremental-indexer.js

@@ -0,0 +1,278 @@
+import { availableParallelism } from "node:os";
+import { createChunker } from "@kb/chunker";
+import { detectContentType } from "@kb/core";
+import { generateRecordId, hashContent } from "./file-hasher.js";
+import { FilesystemCrawler } from "./filesystem-crawler.js";
+import { extractGraph } from "./graph-extractor.js";
+async function runConcurrent(items, fn, concurrency, onError) {
+  let nextIndex = 0;
+  async function worker() {
+    while (nextIndex < items.length) {
+      const i = nextIndex++;
+      try {
+        await fn(items[i]);
+      } catch (err) {
+        onError?.(items[i], err);
+      }
+    }
+  }
+  await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, () => worker()));
+}
+const DEFAULT_CONCURRENCY = Math.max(1, Math.floor(availableParallelism() / 2));
+class IncrementalIndexer {
+  constructor(embedder, store) {
+    this.embedder = embedder;
+    this.store = store;
+    this.crawler = new FilesystemCrawler();
+  }
+  crawler;
+  indexing = false;
+  graphStore;
+  /** Set the graph store for auto-population during indexing and cleanup on re-index. */
+  setGraphStore(graphStore) {
+    this.graphStore = graphStore;
+  }
+  /**
+   * Index all configured sources. Only re-indexes files that have changed.
+   * Sources are crawled in parallel, and file processing runs concurrently
+   * up to `config.indexing.concurrency` (default: half of CPU cores).
+   */
+  async index(config, onProgress) {
+    if (this.indexing) {
+      throw new Error("Indexing is already in progress");
+    }
+    this.indexing = true;
+    try {
+      return await this.doIndex(config, onProgress, {});
+    } finally {
+      this.indexing = false;
+    }
+  }
+  async doIndex(config, onProgress, opts = {}) {
+    const startTime = Date.now();
+    let filesProcessed = 0;
+    let filesSkipped = 0;
+    let chunksCreated = 0;
+    let filesRemoved = 0;
+    const concurrency = config.indexing.concurrency ?? DEFAULT_CONCURRENCY;
+    onProgress?.({
+      phase: "crawling",
+      filesTotal: 0,
+      filesProcessed: 0,
+      chunksTotal: 0,
+      chunksProcessed: 0
+    });
+    const crawlResults = await Promise.all(
+      config.sources.map(
+        (source) => this.crawler.crawl({
+          rootDir: source.path,
+          excludePatterns: source.excludePatterns
+        })
+      )
+    );
+    const allFiles = crawlResults.flat();
+    let filesToProcess;
+    let pathsToRemove;
+    if (opts.skipHashCheck) {
+      filesToProcess = allFiles;
+      pathsToRemove = [];
+    } else {
+      const existingPaths = await this.store.listSourcePaths();
+      const currentPathSet = new Set(allFiles.map((f) => f.relativePath));
+      pathsToRemove = existingPaths.filter(
+        (p) => !currentPathSet.has(p) && !p.startsWith("curated/")
+      );
+      filesToProcess = [];
+      await runConcurrent(
+        allFiles,
+        async (file) => {
+          const contentHash = hashContent(file.content);
+          const existingRecords = await this.store.getBySourcePath(file.relativePath);
+          if (existingRecords.length > 0 && existingRecords[0].fileHash === contentHash) {
+            filesSkipped++;
+            return;
+          }
+          filesToProcess.push(file);
+        },
+        concurrency,
+        (file, err) => console.error(`[indexer] hash check failed for ${file.relativePath}:`, err)
+      );
+    }
+    const totalFiles = filesToProcess.length;
+    const GRAPH_FLUSH_INTERVAL = 50;
+    let pendingNodes = [];
+    let pendingEdges = [];
+    let graphFilesAccumulated = 0;
+    const flushGraph = async () => {
+      if (!this.graphStore) return;
+      try {
+        if (pendingNodes.length > 0) {
+          await this.graphStore.upsertNodes(pendingNodes);
+        }
+        if (pendingEdges.length > 0) {
+          await this.graphStore.upsertEdges(pendingEdges);
+        }
+      } catch (err) {
+        console.error("[indexer] graph batch flush failed:", err);
+      }
+      pendingNodes = [];
+      pendingEdges = [];
+      graphFilesAccumulated = 0;
+    };
+    await runConcurrent(
+      filesToProcess,
+      async (file) => {
+        onProgress?.({
+          phase: "chunking",
+          filesTotal: totalFiles,
+          filesProcessed,
+          chunksTotal: chunksCreated,
+          chunksProcessed: chunksCreated,
+          currentFile: file.relativePath
+        });
+        const contentType = detectContentType(file.relativePath);
+        const chunker = createChunker(file.extension);
+        const chunks = chunker.chunk(file.content, {
+          sourcePath: file.relativePath,
+          contentType
+        });
+        if (chunks.length === 0) return;
+        onProgress?.({
+          phase: "embedding",
+          filesTotal: totalFiles,
+          filesProcessed,
+          chunksTotal: chunksCreated + chunks.length,
+          chunksProcessed: chunksCreated,
+          currentFile: file.relativePath
+        });
+        const vectors = await this.embedder.embedBatch(chunks.map((c) => c.text));
+        const fileHash = hashContent(file.content);
+        const records = chunks.map((chunk, i) => ({
+          id: generateRecordId(file.relativePath, i),
+          content: chunk.text,
+          sourcePath: chunk.sourcePath,
+          contentType: chunk.contentType,
+          headingPath: chunk.headingPath,
+          chunkIndex: chunk.chunkIndex,
+          totalChunks: chunk.totalChunks,
+          startLine: chunk.startLine,
+          endLine: chunk.endLine,
+          fileHash,
+          indexedAt: (/* @__PURE__ */ new Date()).toISOString(),
+          origin: "indexed",
+          tags: [],
+          version: 1
+        }));
+        onProgress?.({
+          phase: "storing",
+          filesTotal: totalFiles,
+          filesProcessed,
+          chunksTotal: chunksCreated + chunks.length,
+          chunksProcessed: chunksCreated,
+          currentFile: file.relativePath
+        });
+        await this.store.upsert(records, vectors);
+        if (this.graphStore) {
+          try {
+            if (!opts.graphCleared) {
+              await this.graphStore.deleteBySourcePath(file.relativePath);
+            }
+            const graph = extractGraph(file.content, file.relativePath);
+            if (graph.nodes.length > 0) pendingNodes.push(...graph.nodes);
+            if (graph.edges.length > 0) pendingEdges.push(...graph.edges);
+            graphFilesAccumulated++;
+            if (graphFilesAccumulated >= GRAPH_FLUSH_INTERVAL) {
+              await flushGraph();
+            }
+          } catch (err) {
+            console.error(`[indexer] graph extraction failed for ${file.relativePath}:`, err);
+          }
+        }
+        filesProcessed++;
+        chunksCreated += chunks.length;
+      },
+      concurrency,
+      (file, err) => console.error(`[indexer] processing failed for ${file.relativePath}:`, err)
+    );
+    await flushGraph();
+    if (pathsToRemove.length > 0) {
+      onProgress?.({
+        phase: "cleanup",
+        filesTotal: totalFiles,
+        filesProcessed,
+        chunksTotal: chunksCreated,
+        chunksProcessed: chunksCreated
+      });
+      await runConcurrent(
+        pathsToRemove,
+        async (path) => {
+          await this.store.deleteBySourcePath(path);
+          if (this.graphStore) {
+            await this.graphStore.deleteBySourcePath(path).catch((err) => console.error(`[indexer] graph cleanup failed for ${path}:`, err));
+          }
+          filesRemoved++;
+        },
+        concurrency,
+        (path, err) => console.error(`[indexer] cleanup failed for ${path}:`, err)
+      );
+    }
+    onProgress?.({
+      phase: "done",
+      filesTotal: totalFiles,
+      filesProcessed,
+      chunksTotal: chunksCreated,
+      chunksProcessed: chunksCreated
+    });
+    return {
+      filesProcessed,
+      filesSkipped,
+      chunksCreated,
+      filesRemoved,
+      durationMs: Date.now() - startTime
+    };
+  }
+  /**
+   * Force re-index all files (ignoring hashes).
+   */
+  async reindexAll(config, onProgress) {
+    await this.store.dropTable();
+    if (this.graphStore) {
+      try {
+        const stats = await this.graphStore.getStats();
+        if (stats.nodeCount > 0) {
+          await this.graphStore.clear();
+          console.error(
+            `[indexer] Graph store cleared (was ${stats.nodeCount} nodes, ${stats.edgeCount} edges)`
+          );
+        }
+      } catch (err) {
+        console.error("[indexer] Graph store clear failed:", err);
+      }
+    }
+    return this.doReindex(config, onProgress);
+  }
+  async doReindex(config, onProgress) {
+    if (this.indexing) {
+      throw new Error("Indexing is already in progress");
+    }
+    this.indexing = true;
+    try {
+      return await this.doIndex(config, onProgress, {
+        skipHashCheck: true,
+        graphCleared: true
+      });
+    } finally {
+      this.indexing = false;
+    }
+  }
+  /**
+   * Get current index statistics.
+   */
+  async getStats() {
+    return this.store.getStats();
+  }
+}
+export {
+  IncrementalIndexer
+};
+//# sourceMappingURL=incremental-indexer.js.map

package/packages/indexer/dist/index.d.ts

@@ -0,0 +1,5 @@
+export { generateRecordId, hashContent } from './file-hasher.js';
+export { type CrawlOptions, type CrawlResult, FilesystemCrawler } from './filesystem-crawler.js';
+export { type ExtractedGraph, extractGraph } from './graph-extractor.js';
+export { IncrementalIndexer, type IndexProgress, type IndexResult, type ProgressCallback, } from './incremental-indexer.js';
+//# sourceMappingURL=index.d.ts.map

package/packages/indexer/dist/index.js

@@ -0,0 +1,14 @@
+import { generateRecordId, hashContent } from "./file-hasher.js";
+import { FilesystemCrawler } from "./filesystem-crawler.js";
+import { extractGraph } from "./graph-extractor.js";
+import {
+  IncrementalIndexer
+} from "./incremental-indexer.js";
+export {
+  FilesystemCrawler,
+  IncrementalIndexer,
+  extractGraph,
+  generateRecordId,
+  hashContent
+};
+//# sourceMappingURL=index.js.map

package/packages/server/dist/api.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Public API for @kb/server — imported by CLI and other consumers.
+ * Does NOT start the server (unlike index.ts which auto-calls main()).
+ */
+export { loadConfig } from './config.js';
+export type { KnowledgeBaseComponents } from './server.js';
+export { createMcpServer, createServer, initializeKnowledgeBase } from './server.js';
+//# sourceMappingURL=api.d.ts.map

package/packages/server/dist/api.js

@@ -0,0 +1,9 @@
+import { loadConfig } from "./config.js";
+import { createMcpServer, createServer, initializeKnowledgeBase } from "./server.js";
+export {
+  createMcpServer,
+  createServer,
+  initializeKnowledgeBase,
+  loadConfig
+};
+//# sourceMappingURL=api.js.map

package/packages/server/dist/config.d.ts

@@ -0,0 +1,3 @@
+import type { KBConfig } from '@kb/core';
+export declare function loadConfig(): KBConfig;
+//# sourceMappingURL=config.d.ts.map