@vpxa/aikit 0.1.1

package/packages/indexer/dist/incremental-indexer.d.ts

@@ -0,0 +1,56 @@
+import { FileHashCache } from "./hash-cache.js";
+import { IndexStats, KBConfig } from "../../core/dist/index.js";
+import { IGraphStore, IKnowledgeStore } from "../../store/dist/index.js";
+import { IEmbedder } from "../../embeddings/dist/index.js";
+
+//#region packages/indexer/src/incremental-indexer.d.ts
+interface IndexProgress {
+  phase: 'crawling' | 'chunking' | 'embedding' | 'storing' | 'cleanup' | 'done';
+  filesTotal: number;
+  filesProcessed: number;
+  chunksTotal: number;
+  chunksProcessed: number;
+  /** The file currently being processed */
+  currentFile?: string;
+}
+type ProgressCallback = (progress: IndexProgress) => void;
+interface IndexResult {
+  filesProcessed: number;
+  filesSkipped: number;
+  chunksCreated: number;
+  filesRemoved: number;
+  durationMs: number;
+}
+declare class IncrementalIndexer {
+  private readonly embedder;
+  private readonly store;
+  private readonly crawler;
+  private indexing;
+  private graphStore?;
+  private hashCache?;
+  /** Whether an index operation is currently in progress. */
+  get isIndexing(): boolean;
+  constructor(embedder: IEmbedder, store: IKnowledgeStore);
+  /** Set the graph store for auto-population during indexing and cleanup on re-index. */
+  setGraphStore(graphStore: IGraphStore): void;
+  /** Set the hash cache for faster incremental checks. */
+  setHashCache(cache: FileHashCache): void;
+  /**
+   * Index all configured sources. Only re-indexes files that have changed.
+   * Sources are crawled in parallel, and file processing runs concurrently
+   * up to `config.indexing.concurrency` (default: 50% of CPU cores, minimum 2, maximum 4).
+   */
+  index(config: KBConfig, onProgress?: ProgressCallback): Promise<IndexResult>;
+  private doIndex;
+  /**
+   * Force re-index all files (ignoring hashes).
+   */
+  reindexAll(config: KBConfig, onProgress?: ProgressCallback): Promise<IndexResult>;
+  private doReindex;
+  /**
+   * Get current index statistics.
+   */
+  getStats(): Promise<IndexStats>;
+}
+//#endregion
+export { IncrementalIndexer, IndexProgress, IndexResult, ProgressCallback };

package/packages/indexer/dist/incremental-indexer.js

@@ -0,0 +1 @@
+import{generateRecordId as e,hashContent as t}from"./file-hasher.js";import{FilesystemCrawler as n}from"./filesystem-crawler.js";import{extractGraph as r}from"./graph-extractor.js";import{AIKIT_PATHS as i,createLogger as a,detectContentType as o,serializeError as s}from"../../core/dist/index.js";import{availableParallelism as c,loadavg as l}from"node:os";import{createChunkerSync as u}from"../../chunker/dist/index.js";const d=a(`indexer`);async function f(e,t,n,r){let i=0;async function a(){for(;i<e.length;){let n=i++;try{await t(e[n])}catch(t){r?.(e[n],t)}}}await Promise.all(Array.from({length:Math.min(n,e.length)},()=>a()))}function p(e){let t=c(),n=l()[0]/t;return n>1.5?2:n>1?Math.max(2,Math.floor(e/2)):e}const m=Math.max(2,Math.min(4,Math.floor(c()*.5)));var h=class{crawler;indexing=!1;graphStore;hashCache;get isIndexing(){return this.indexing}constructor(e,t){this.embedder=e,this.store=t,this.crawler=new n}setGraphStore(e){this.graphStore=e}setHashCache(e){this.hashCache=e}async index(e,t){if(this.indexing)throw Error(`Indexing is already in progress`);this.indexing=!0;try{return await this.doIndex(e,t,{})}finally{this.indexing=!1}}async doIndex(n,a,c={}){let l=Date.now(),h=0,g=0,_=0,v=0,y=n.indexing.concurrency??m;a?.({phase:`crawling`,filesTotal:0,filesProcessed:0,chunksTotal:0,chunksProcessed:0});let b=(await Promise.all(n.sources.map(e=>this.crawler.crawl({rootDir:e.path,excludePatterns:e.excludePatterns})))).flat(),x,S;if(c.skipHashCheck)x=b,S=[];else{let e=await this.store.listSourcePaths(),n=new Set(b.map(e=>e.relativePath));S=e.filter(e=>!n.has(e)&&!e.startsWith(`${i.aiCurated}/`)),x=[],await f(b,async e=>{let n=t(e.content);if(this.hashCache){if(this.hashCache.get(e.relativePath)===n){g++;return}}else{let t=await this.store.getBySourcePath(e.relativePath);if(t.length>0&&t[0].fileHash===n){g++;return}}x.push(e)},p(y),(e,t)=>d.error(`Hash check failed`,{sourcePath:e.relativePath,...s(t)}))}let C=x.length,w=[],T=[],E=0,D=[],O=[],k=new Map,A=0,j=async()=>{if(D.length===0)return;let e=D,t=O,n=k;D=[],O=[],k=new Map,A=0,await this.store.upsert(e,t);for(let[e,t]of n)this.hashCache?.set(e,t)},M=async()=>{if(this.graphStore){try{w.length>0&&await this.graphStore.upsertNodes(w),T.length>0&&await this.graphStore.upsertEdges(T)}catch(e){d.warn(`Graph batch flush failed`,s(e))}w=[],T=[],E=0}};return await f(x,async n=>{a?.({phase:`chunking`,filesTotal:C,filesProcessed:h,chunksTotal:_,chunksProcessed:_,currentFile:n.relativePath});let i=o(n.relativePath),l=u(n.extension).chunk(n.content,{sourcePath:n.relativePath,contentType:i});if(l.length===0)return;a?.({phase:`embedding`,filesTotal:C,filesProcessed:h,chunksTotal:_+l.length,chunksProcessed:_,currentFile:n.relativePath});let f=await this.embedder.embedBatch(l.map(e=>e.text)),p=t(n.content),m=l.map((t,r)=>({id:e(n.relativePath,r),content:t.text,sourcePath:t.sourcePath,contentType:t.contentType,headingPath:t.headingPath,chunkIndex:t.chunkIndex,totalChunks:t.totalChunks,startLine:t.startLine,endLine:t.endLine,fileHash:p,indexedAt:new Date().toISOString(),origin:`indexed`,tags:[],version:1}));if(a?.({phase:`storing`,filesTotal:C,filesProcessed:h,chunksTotal:_+l.length,chunksProcessed:_,currentFile:n.relativePath}),D.push(...m),O.push(...f),k.set(n.relativePath,p),A++,A>=20&&await j(),this.graphStore)try{c.graphCleared||await this.graphStore.deleteBySourcePath(n.relativePath);let e=r(n.content,n.relativePath);e.nodes.length>0&&w.push(...e.nodes),e.edges.length>0&&T.push(...e.edges),E++,E>=50&&await M()}catch(e){d.warn(`Graph extraction failed`,{sourcePath:n.relativePath,...s(e)})}h++,_+=l.length},p(y),(e,t)=>d.error(`Processing failed`,{sourcePath:e.relativePath,...s(t)})),await j(),await M(),S.length>0&&(a?.({phase:`cleanup`,filesTotal:C,filesProcessed:h,chunksTotal:_,chunksProcessed:_}),await f(S,async e=>{await this.store.deleteBySourcePath(e),this.hashCache?.delete(e),this.graphStore&&await this.graphStore.deleteBySourcePath(e).catch(t=>d.warn(`Graph cleanup failed`,{sourcePath:e,...s(t)})),v++},p(y),(e,t)=>d.error(`Cleanup failed`,{sourcePath:e,...s(t)}))),this.hashCache?.flush(),a?.({phase:`done`,filesTotal:C,filesProcessed:h,chunksTotal:_,chunksProcessed:_}),{filesProcessed:h,filesSkipped:g,chunksCreated:_,filesRemoved:v,durationMs:Date.now()-l}}async reindexAll(e,t){if(this.indexing)throw Error(`Indexing is already in progress`);this.indexing=!0;try{if(await this.store.dropTable(),this.graphStore)try{let e=await this.graphStore.getStats();e.nodeCount>0&&(await this.graphStore.clear(),d.info(`Graph store cleared`,{nodeCount:e.nodeCount,edgeCount:e.edgeCount}))}catch(e){d.warn(`Graph store clear failed`,s(e))}return await this.doReindex(e,t)}catch(e){throw this.indexing=!1,e}}async doReindex(e,t){try{return await this.doIndex(e,t,{skipHashCheck:!0,graphCleared:!0})}finally{this.indexing=!1}}async getStats(){return this.store.getStats()}};export{h as IncrementalIndexer};

package/packages/indexer/dist/index.d.ts

@@ -0,0 +1,6 @@
+import { generateRecordId, hashContent } from "./file-hasher.js";
+import { CrawlOptions, CrawlResult, FilesystemCrawler } from "./filesystem-crawler.js";
+import { ExtractedGraph, extractGraph } from "./graph-extractor.js";
+import { FileHashCache } from "./hash-cache.js";
+import { IncrementalIndexer, IndexProgress, IndexResult, ProgressCallback } from "./incremental-indexer.js";
+export { type CrawlOptions, type CrawlResult, type ExtractedGraph, FileHashCache, FilesystemCrawler, IncrementalIndexer, type IndexProgress, type IndexResult, type ProgressCallback, extractGraph, generateRecordId, hashContent };

package/packages/indexer/dist/index.js

@@ -0,0 +1 @@
+import{generateRecordId as e,hashContent as t}from"./file-hasher.js";import{FilesystemCrawler as n}from"./filesystem-crawler.js";import{extractGraph as r}from"./graph-extractor.js";import{FileHashCache as i}from"./hash-cache.js";import{IncrementalIndexer as a}from"./incremental-indexer.js";export{i as FileHashCache,n as FilesystemCrawler,a as IncrementalIndexer,r as extractGraph,e as generateRecordId,t as hashContent};