@vpxa/aikit 0.1.58 → 0.1.60

package/packages/indexer/dist/index.js

@@ -1 +1 @@
-import{generateRecordId as e,hashContent as t}from"./file-hasher.js";import{FilesystemCrawler as n}from"./filesystem-crawler.js";import{extractGraph as r}from"./graph-extractor.js";import{FileHashCache as i}from"./hash-cache.js";import{IncrementalIndexer as a}from"./incremental-indexer.js";import{SmartIndexScheduler as o}from"./smart-index-scheduler.js";export{i as FileHashCache,n as FilesystemCrawler,a as IncrementalIndexer,o as SmartIndexScheduler,r as extractGraph,e as generateRecordId,t as hashContent};
+import{createHash as e}from"node:crypto";import{existsSync as t,readFileSync as n,realpathSync as r,statSync as i,writeFileSync as a}from"node:fs";import{lstat as o,readFile as s,readdir as c,stat as l}from"node:fs/promises";import{dirname as u,extname as d,join as f,relative as p,resolve as m}from"node:path";import{AIKIT_PATHS as h,FILE_LIMITS as g,createLogger as _,detectContentType as v,serializeError as y}from"../../core/dist/index.js";import{minimatch as b}from"minimatch";import{availableParallelism as x,loadavg as S}from"node:os";import{createChunkerSync as C}from"../../chunker/dist/index.js";function w(t){return e(`sha256`).update(t).digest(`hex`).slice(0,16)}function T(t,n){let r=`${t}:${n}`;return e(`sha256`).update(r).digest(`hex`).slice(0,16)}const E=_(`indexer`);var D=class e{static BINARY_EXTENSIONS=new Set(`.node,.so,.dylib,.dll,.wasm,.bin,.exe,.png,.jpg,.jpeg,.gif,.bmp,.ico,.webp,.svg,.mp3,.mp4,.wav,.avi,.mov,.flac,.zip,.gz,.tar,.bz2,.7z,.rar,.pdf,.doc,.docx,.xls,.xlsx,.ppt,.pptx,.ttf,.otf,.woff,.woff2,.eot,.pyc,.class,.o,.obj,.a,.lib`.split(`,`));async crawl(e){let t=[],n=new Set;return await this.walkDir(e.rootDir,e.rootDir,e.excludePatterns,t,n),t}async walkDir(t,n,i,a,u){let m;try{m=await c(t,{withFileTypes:!0})}catch(e){let n=e.code;(n===`EACCES`||n===`EPERM`)&&E.warn(`Permission denied, skipping directory`,{dir:t});return}for(let c of m){let m=f(t,c.name),_=p(n,m).replace(/\\/g,`/`);if(!this.isExcluded(_,i)){if(c.isDirectory()){if(c.name.startsWith(`.`)&&!(c.name===h.ai.slice(1)&&_.startsWith(h.ai)))continue;try{if((await o(m)).isSymbolicLink())continue}catch{continue}let e;try{e=r(m)}catch{continue}if(u.has(e))continue;u.add(e),await this.walkDir(m,n,i,a,u)}else if(c.isFile()){let t=d(c.name).toLowerCase();if(e.BINARY_EXTENSIONS.has(t))continue;try{if((await l(m)).size>g.maxFileSizeBytes)continue;let e=await s(m,`utf-8`);if(e.includes(`\0`))continue;a.push({relativePath:_,absolutePath:m,content:e,extension:t})}catch{}}}}}isExcluded(e,t){return t.some(t=>b(e,t,{dot:!0}))}};const O=new Set([`.ts`,`.tsx`,`.js`,`.jsx`,`.mjs`,`.cjs`]),k=[{pattern:/^export\s+(?:async\s+)?function\s+(\w+)/gm,kind:`function`,exported:!0},{pattern:/^export\s+(?:default\s+)?class\s+(\w+)/gm,kind:`class`,exported:!0},{pattern:/^export\s+interface\s+(\w+)/gm,kind:`interface`,exported:!0},{pattern:/^export\s+type\s+(\w+)/gm,kind:`type`,exported:!0},{pattern:/^export\s+(?:const|let)\s+(\w+)/gm,kind:`const`,exported:!0},{pattern:/^export\s+enum\s+(\w+)/gm,kind:`enum`,exported:!0},{pattern:/^(?:async\s+)?function\s+(\w+)/gm,kind:`function`,exported:!1},{pattern:/^class\s+(\w+)/gm,kind:`class`,exported:!1},{pattern:/^interface\s+(\w+)/gm,kind:`interface`,exported:!1},{pattern:/^type\s+(\w+)/gm,kind:`type`,exported:!1},{pattern:/^enum\s+(\w+)/gm,kind:`enum`,exported:!1}],A=[/import\s+(?:(?:type\s+)?(?:(?:\{[^}]*\}|[\w*]+)\s+from\s+)?)['"]([^'"]+)['"]/g,/import\(\s*['"]([^'"]+)['"]\s*\)/g,/require\(\s*['"]([^'"]+)['"]\s*\)/g];function j(t,n,r){return e(`sha256`).update(`${t}:${n}:${r}`).digest(`hex`).slice(0,16)}function M(t,n,r){return e(`sha256`).update(`${t}-${r}-${n}`).digest(`hex`).slice(0,16)}function N(e,t){return f(u(t),e).replace(/\\/g,`/`).replace(/\.(js|jsx|ts|tsx|mjs|cjs)$/,``)}function P(e){return e.replace(/\.(js|jsx|ts|tsx|mjs|cjs)$/,``)}function F(e,t,n){let r=d(t).toLowerCase();if(!O.has(r))return{nodes:[],edges:[]};let i=[],a=[],o=new Date().toISOString(),s=new Set,c=P(t),l=j(`module`,c,c);i.push({id:l,type:`module`,name:t,properties:{ext:r},sourcePath:t,createdAt:o});for(let{pattern:n,kind:r,exported:c}of k){let u=new RegExp(n.source,n.flags),d;for(;(d=u.exec(e))!==null;){let e=d[1],n=`${r}:${e}`;if(s.has(n))continue;s.add(n);let u=j(r,e,t);i.push({id:u,type:r,name:e,properties:{exported:c},sourcePath:t,createdAt:o}),a.push({id:M(l,u,`defines`),fromId:l,toId:u,type:`defines`,weight:c?1:.5})}}let u=n?.workspacePackages,f=new Set;for(let n of A){let r=new RegExp(n.source,n.flags),i;for(;(i=r.exec(e))!==null;){let e=i[1];if(f.has(e))continue;let n;if(e.startsWith(`.`))n=N(e,t);else if(u){let t=e.startsWith(`@`)?e.split(`/`).slice(0,2).join(`/`):e.split(`/`)[0],r=u.get(t);if(!r)continue;n=r}else continue;f.add(e);let r=P(n),o=j(`module`,r,r);a.push({id:M(l,o,`imports`),fromId:l,toId:o,type:`imports`,properties:{source:e}})}}return{nodes:i,edges:a}}const I=_(`hash-cache`);var L=class{cache;filePath;dirty=!1;constructor(e){this.filePath=m(e,`file-hashes.json`),this.cache=new Map}load(){if(t(this.filePath))try{let e=n(this.filePath,`utf-8`),t=JSON.parse(e);this.cache=new Map(Object.entries(t)),I.info(`Hash cache loaded`,{entries:this.cache.size})}catch(e){I.warn(`Hash cache load failed, starting fresh`,{err:e}),this.cache=new Map}}get(e){return this.cache.get(e)}set(e,t){this.cache.set(e,t),this.dirty=!0}delete(e){this.cache.delete(e)&&(this.dirty=!0)}flush(){if(this.dirty)try{let e={};for(let[t,n]of this.cache)e[t]=n;a(this.filePath,JSON.stringify(e),`utf-8`),this.dirty=!1}catch(e){I.warn(`Hash cache flush failed`,{err:e})}}clear(){this.cache.clear(),this.dirty=!0,this.flush()}get size(){return this.cache.size}};const R=_(`indexer`),z=()=>new Promise(e=>setImmediate(e));async function B(e,t,n,r){let i=0;async function a(){for(;i<e.length;){let n=i++;try{await t(e[n])}catch(t){r?.(e[n],t)}n%10==0&&await z()}}await Promise.all(Array.from({length:Math.min(n,e.length)},()=>a()))}function V(e){let t=x(),n=S()[0]/t;return n>1.5?2:n>1?Math.max(2,Math.floor(e/2)):e}const H=Math.max(2,Math.min(4,Math.floor(x()*.5)));var U=class{crawler;indexing=!1;graphStore;hashCache;get isIndexing(){return this.indexing}constructor(e,t){this.embedder=e,this.store=t,this.crawler=new D}setGraphStore(e){this.graphStore=e}setHashCache(e){this.hashCache=e}async index(e,t){if(this.indexing)throw Error(`Indexing is already in progress`);this.indexing=!0;try{return await this.doIndex(e,t,{})}finally{this.indexing=!1}}async getChangedFiles(e){let t=e.indexing.concurrency??H,n=await this.crawlSources(e),{filesToProcess:r}=await this.planIndexWork(n,t,{});return r.map(e=>e.relativePath)}async indexFiles(e,t,n){if(this.indexing)throw Error(`Indexing is already in progress`);this.indexing=!0;try{return await this.doIndexFiles(e,t,n)}finally{this.indexing=!1}}async doIndex(e,t,n={}){let r=Date.now(),i=e.indexing.concurrency??H;t?.({phase:`crawling`,filesTotal:0,filesProcessed:0,chunksTotal:0,chunksProcessed:0});let a=await this.crawlSources(e),{filesToProcess:o,filesSkipped:s,pathsToRemove:c}=await this.planIndexWork(a,i,{skipHashCheck:n.skipHashCheck}),{filesProcessed:l,chunksCreated:u}=await this.processFiles(a,o,i,t,{graphCleared:n.graphCleared}),d=await this.cleanupRemovedFiles(c,i,o.length,l,u,t);return this.hashCache?.flush(),t?.({phase:`done`,filesTotal:o.length,filesProcessed:l,chunksTotal:u,chunksProcessed:u}),{filesProcessed:l,filesSkipped:s,chunksCreated:u,filesRemoved:d,durationMs:Date.now()-r}}async doIndexFiles(e,t,n){let r=Date.now(),i=e.indexing.concurrency??H,a=new Set(t);if(a.size===0)return{filesProcessed:0,filesSkipped:0,chunksCreated:0,filesRemoved:0,durationMs:Date.now()-r};n?.({phase:`crawling`,filesTotal:0,filesProcessed:0,chunksTotal:0,chunksProcessed:0});let o=await this.crawlSources(e),{filesToProcess:s,filesSkipped:c}=await this.planIndexWork(o,i,{requestedPaths:a}),{filesProcessed:l,chunksCreated:u}=await this.processFiles(o,s,i,n,{});return this.hashCache?.flush(),n?.({phase:`done`,filesTotal:s.length,filesProcessed:l,chunksTotal:u,chunksProcessed:u}),{filesProcessed:l,filesSkipped:c,chunksCreated:u,filesRemoved:0,durationMs:Date.now()-r}}async crawlSources(e){return(await Promise.all(e.sources.map(e=>this.crawler.crawl({rootDir:e.path,excludePatterns:e.excludePatterns})))).flat()}async planIndexWork(e,t,n){let r=n.requestedPaths,i=r?e.filter(e=>r.has(e.relativePath)):e;if(n.skipHashCheck)return{filesToProcess:i,filesSkipped:0,pathsToRemove:[]};let a=r?[]:await this.getPathsToRemove(e),o=0,s=[];return await B(i,async e=>{let t=w(e.content);if(this.hashCache){if(this.hashCache.get(e.relativePath)===t){o++;return}}else{let n=await this.store.getBySourcePath(e.relativePath);if(n.length>0&&n[0].fileHash===t){o++;return}}s.push(e)},V(t),(e,t)=>R.error(`Hash check failed`,{sourcePath:e.relativePath,...y(t)})),{filesToProcess:s,filesSkipped:o,pathsToRemove:a}}async getPathsToRemove(e){let t=await this.store.listSourcePaths(),n=new Set(e.map(e=>e.relativePath));return t.filter(e=>!n.has(e)&&!e.startsWith(`${h.aiCurated}/`))}async buildWorkspacePackageMap(e){let t=new Map;for(let n of e)if(n.relativePath.endsWith(`package.json`))try{let e=JSON.parse(n.content);if(typeof e.name!=`string`||e.name.length===0)continue;let r=n.relativePath===`package.json`?``:n.relativePath.replace(/\/package\.json$/,``),i=``;if(typeof e.exports==`string`)i=e.exports;else if(e.exports&&typeof e.exports==`object`){let t=e.exports,n=t[`.`]??t;if(typeof n==`string`)i=n;else if(n&&typeof n==`object`){let e=n,t=e.import??e.module??e.default??e.require;typeof t==`string`&&(i=t)}}!i&&typeof e.module==`string`&&(i=e.module),!i&&typeof e.main==`string`&&(i=e.main);let a=i.replace(/^\.\//,``).replace(/^dist\//,`src/`).replace(/\.(ts|js|mjs|cjs|tsx|jsx)$/,``),o=a?r?`${r}/${a}`:a:r?`${r}/src/index`:`src/index`;t.set(e.name,o)}catch{}return t}async processFiles(e,t,n,r,i={}){let a=0,o=0,s=t.length,c=this.graphStore?await this.buildWorkspacePackageMap(e):void 0,l=[],u=[],d=0,f=[],p=[],m=new Map,h=0,g=async()=>{if(f.length===0)return;let e=f,t=p,n=m;f=[],p=[],m=new Map,h=0,await this.store.upsert(e,t);for(let[e,t]of n)this.hashCache?.set(e,t)},_=async()=>{if(this.graphStore){try{l.length>0&&await this.graphStore.upsertNodes(l),u.length>0&&await this.graphStore.upsertEdges(u)}catch(e){R.warn(`Graph batch flush failed`,y(e))}l=[],u=[],d=0}};return await B(t,async e=>{r?.({phase:`chunking`,filesTotal:s,filesProcessed:a,chunksTotal:o,chunksProcessed:o,currentFile:e.relativePath});let t=v(e.relativePath),n=C(e.extension).chunk(e.content,{sourcePath:e.relativePath,contentType:t});if(n.length===0)return;r?.({phase:`embedding`,filesTotal:s,filesProcessed:a,chunksTotal:o+n.length,chunksProcessed:o,currentFile:e.relativePath});let b=await this.embedder.embedBatch(n.map(e=>e.text)),x=w(e.content),S=n.map((t,n)=>({id:T(e.relativePath,n),content:t.text,sourcePath:t.sourcePath,contentType:t.contentType,headingPath:t.headingPath,chunkIndex:t.chunkIndex,totalChunks:t.totalChunks,startLine:t.startLine,endLine:t.endLine,fileHash:x,indexedAt:new Date().toISOString(),origin:`indexed`,tags:[],version:1}));if(r?.({phase:`storing`,filesTotal:s,filesProcessed:a,chunksTotal:o+n.length,chunksProcessed:o,currentFile:e.relativePath}),f.push(...S),p.push(...b),m.set(e.relativePath,x),h++,h>=20&&await g(),this.graphStore)try{i.graphCleared||await this.graphStore.deleteBySourcePath(e.relativePath);let t=F(e.content,e.relativePath,{workspacePackages:c});t.nodes.length>0&&l.push(...t.nodes),t.edges.length>0&&u.push(...t.edges),d++,d>=50&&await _()}catch(t){R.warn(`Graph extraction failed`,{sourcePath:e.relativePath,...y(t)})}a++,o+=n.length},V(n),(e,t)=>R.error(`Processing failed`,{sourcePath:e.relativePath,...y(t)})),await g(),await _(),{filesProcessed:a,chunksCreated:o}}async cleanupRemovedFiles(e,t,n,r,i,a){if(e.length===0)return 0;let o=0;return a?.({phase:`cleanup`,filesTotal:n,filesProcessed:r,chunksTotal:i,chunksProcessed:i}),await B(e,async e=>{await this.store.deleteBySourcePath(e),this.hashCache?.delete(e),this.graphStore&&await this.graphStore.deleteBySourcePath(e).catch(t=>R.warn(`Graph cleanup failed`,{sourcePath:e,...y(t)})),o++},V(t),(e,t)=>R.error(`Cleanup failed`,{sourcePath:e,...y(t)})),o}async reindexAll(e,t){if(this.indexing)throw Error(`Indexing is already in progress`);this.indexing=!0;try{if(await this.store.dropTable(),this.graphStore)try{let e=await this.graphStore.getStats();e.nodeCount>0&&(await this.graphStore.clear(),R.info(`Graph store cleared`,{nodeCount:e.nodeCount,edgeCount:e.edgeCount}))}catch(e){R.warn(`Graph store clear failed`,y(e))}return await this.doReindex(e,t)}catch(e){throw this.indexing=!1,e}}async doReindex(e,t){try{return await this.doIndex(e,t,{skipHashCheck:!0,graphCleared:!0})}finally{this.indexing=!1}}async getStats(){return this.store.getStats()}};const W=_(`smart-index`),G=1.5;var K=class{trickleTimer=null;trickleIntervalMs;batchSize;priorityQueue=[];changedFiles=[];lastRefreshTime=0;refreshing=!1;constructor(e,t,n){this.indexer=e,this.config=t,this.store=n,this.trickleIntervalMs=t.indexing.trickleIntervalMs??this.readPositiveIntEnv(`AIKIT_SMART_TRICKLE_MS`,3e4),this.batchSize=t.indexing.trickleBatchSize??this.readPositiveIntEnv(`AIKIT_SMART_BATCH_SIZE`,1)}start(){this.stop(),W.info(`Smart index scheduler started (trickle mode)`,{intervalMs:this.trickleIntervalMs,batchSize:this.batchSize}),this.scheduleTick()}stop(){this.trickleTimer&&=(clearTimeout(this.trickleTimer),null)}prioritize(...e){let t=[...new Set(e.filter(Boolean))].filter(e=>{try{return!i(e).isDirectory()}catch{return console.debug(`smart-index: skipping non-existent path: ${e}`),!1}});for(let e of t){let t=this.priorityQueue.indexOf(e);t>=0&&this.priorityQueue.splice(t,1)}for(let e of t.reverse())this.priorityQueue.unshift(e);this.priorityQueue.length>500&&(this.priorityQueue.length=500),t.length>0&&W.info(`Files prioritized for trickle indexing`,{added:t.length,queued:this.priorityQueue.length})}getState(){return{mode:`smart`,queueSize:this.priorityQueue.length,changedFilesSize:this.changedFiles.length,intervalMs:this.trickleIntervalMs,batchSize:this.batchSize,running:this.trickleTimer!==null}}readPositiveIntEnv(e,t){let n=Number(process.env[e]);return Number.isFinite(n)&&n>0?n:t}scheduleTick(){this.trickleTimer=setTimeout(()=>void this.tick(),this.trickleIntervalMs),this.trickleTimer.unref&&this.trickleTimer.unref()}async tick(){try{if(this.indexer.isIndexing){W.info(`Skipping trickle tick — indexing already in progress`);return}let e=this.getCpuCount(),t=S()[0];if(e>0&&t/e>G){W.info(`Skipping trickle tick — system load too high`,{load:t.toFixed(2),cpuCount:e,threshold:G});return}let n=await this.pickFiles();if(n.length===0){await this.maybeRefreshChangedFiles();return}W.info(`Trickle indexing tick started`,{count:n.length,files:n});let r=await this.indexer.indexFiles(this.config,n);if(this.store)try{await this.store.createFtsIndex()}catch(e){W.warn(`FTS index rebuild failed after trickle tick`,{error:String(e)})}this.changedFiles=this.changedFiles.filter(e=>!n.includes(e)),W.info(`Trickle indexing tick complete`,{filesProcessed:r.filesProcessed,filesSkipped:r.filesSkipped,chunksCreated:r.chunksCreated})}catch(e){W.error(`Trickle indexing tick failed`,{error:String(e)})}finally{this.scheduleTick()}}getCpuCount(){try{return typeof x==`function`?x():4}catch{return 4}}async pickFiles(){let e=[];for(;e.length<this.batchSize&&this.priorityQueue.length>0;){let t=this.priorityQueue.shift();t&&!e.includes(t)&&e.push(t)}if(e.length<this.batchSize)for(await this.maybeRefreshChangedFiles();e.length<this.batchSize&&this.changedFiles.length>0;){let t=this.changedFiles.shift();t&&!e.includes(t)&&e.push(t)}return e}async maybeRefreshChangedFiles(){let e=Date.now();if(!(this.refreshing||this.changedFiles.length>0&&e-this.lastRefreshTime<6e5)){this.refreshing=!0;try{this.changedFiles=await this.indexer.getChangedFiles(this.config),this.lastRefreshTime=e,this.changedFiles.length>0&&W.info(`Refreshed changed files for trickle indexing`,{count:this.changedFiles.length})}catch(e){W.error(`Failed to refresh changed files for trickle indexing`,{error:String(e)})}finally{this.refreshing=!1}}}};export{L as FileHashCache,D as FilesystemCrawler,U as IncrementalIndexer,K as SmartIndexScheduler,F as extractGraph,T as generateRecordId,w as hashContent};