@vpxa/aikit 0.1.249 → 0.1.250
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/packages/blocks-core/package.json +1 -0
- package/packages/indexer/dist/index.d.ts +26 -7
- package/packages/indexer/dist/index.js +3 -3
- package/packages/server/dist/bin.js +1 -1
- package/packages/server/dist/index.js +1 -1
- package/packages/server/dist/{server-CoohgKA7.js → server-0JRtGGti.js} +155 -155
- package/packages/server/dist/{server-CWkmhYOe.js → server-CBs_q9S9.js} +155 -155
- package/packages/store/dist/index.d.ts +3 -1
- package/packages/store/dist/index.js +36 -30
- package/packages/store/dist/lance-store-B1Gim3h9.js +1 -0
- package/packages/tools/dist/index.d.ts +1 -0
- package/packages/tools/dist/index.js +2 -2
- package/packages/viewers/README.md +1 -0
- package/scaffold/dist/definitions/bodies.mjs +2 -0
- package/packages/store/dist/lance-store-BRKcJXVO.js +0 -1
package/package.json
CHANGED
|
@@ -20,10 +20,16 @@ interface CrawlResult {
|
|
|
20
20
|
relativePath: string;
|
|
21
21
|
/** Absolute file path */
|
|
22
22
|
absolutePath: string;
|
|
23
|
-
/** File content */
|
|
24
|
-
content: string;
|
|
25
23
|
/** File extension (lowercase, with dot) */
|
|
26
24
|
extension: string;
|
|
25
|
+
/** File size in bytes */
|
|
26
|
+
size: number;
|
|
27
|
+
/** Last modified time in milliseconds since epoch */
|
|
28
|
+
mtimeMs: number;
|
|
29
|
+
/** Lazily read file content when downstream processing actually needs it. */
|
|
30
|
+
readContent: () => Promise<string>;
|
|
31
|
+
/** Cached file content after the first read. */
|
|
32
|
+
content?: string;
|
|
27
33
|
}
|
|
28
34
|
interface CrawlOptions {
|
|
29
35
|
/** Glob patterns to exclude */
|
|
@@ -38,11 +44,13 @@ declare class FilesystemCrawler {
|
|
|
38
44
|
/** Binary file extensions that should always be skipped */
|
|
39
45
|
private static readonly BINARY_EXTENSIONS;
|
|
40
46
|
private static readonly GENERATED_PATH_PATTERN;
|
|
41
|
-
private
|
|
47
|
+
private static readonly GENERATED_SAMPLE_LIMIT;
|
|
42
48
|
crawl(options: CrawlOptions): Promise<CrawlResult[]>;
|
|
43
49
|
private walkDir;
|
|
44
50
|
private isExcluded;
|
|
45
|
-
|
|
51
|
+
static isLikelyGeneratedPath(relativePath: string): boolean;
|
|
52
|
+
static isLikelyGeneratedContent(content: string, relativePath: string, _size: number): boolean;
|
|
53
|
+
private static readTextPreview;
|
|
46
54
|
}
|
|
47
55
|
//#endregion
|
|
48
56
|
//#region packages/indexer/src/graph-extractor.d.ts
|
|
@@ -67,9 +75,14 @@ declare function extractGraph(content: string, sourcePath: string, options?: Ext
|
|
|
67
75
|
//#region packages/indexer/src/hash-cache.d.ts
|
|
68
76
|
/**
|
|
69
77
|
* Persistent file hash cache.
|
|
70
|
-
* Stores path->hash mappings in a JSON file to avoid
|
|
71
|
-
* when checking which files
|
|
78
|
+
* Stores path->hash + metadata mappings in a JSON file to avoid repeated
|
|
79
|
+
* content reads and LanceDB round-trips when checking which files changed.
|
|
72
80
|
*/
|
|
81
|
+
interface FileHashCacheEntry {
|
|
82
|
+
hash: string;
|
|
83
|
+
size?: number;
|
|
84
|
+
mtimeMs?: number;
|
|
85
|
+
}
|
|
73
86
|
declare class FileHashCache {
|
|
74
87
|
private cache;
|
|
75
88
|
private readonly filePath;
|
|
@@ -78,7 +91,11 @@ declare class FileHashCache {
|
|
|
78
91
|
/** Load cache from disk. Non-fatal if missing or corrupt. */
|
|
79
92
|
load(): void;
|
|
80
93
|
get(path: string): string | undefined;
|
|
81
|
-
|
|
94
|
+
getEntry(path: string): FileHashCacheEntry | undefined;
|
|
95
|
+
set(path: string, hash: string, metadata?: {
|
|
96
|
+
size: number;
|
|
97
|
+
mtimeMs: number;
|
|
98
|
+
}): void;
|
|
82
99
|
delete(path: string): void;
|
|
83
100
|
/** Persist cache to disk if changed. */
|
|
84
101
|
flush(): void;
|
|
@@ -122,6 +139,8 @@ declare class IncrementalIndexer {
|
|
|
122
139
|
* Modular — swap this to change dedup normalization strategy.
|
|
123
140
|
*/
|
|
124
141
|
private normalizeChunkText;
|
|
142
|
+
private loadFileContent;
|
|
143
|
+
private isMetadataUnchanged;
|
|
125
144
|
/** Set the graph store for auto-population during indexing and cleanup on re-index. */
|
|
126
145
|
setGraphStore(graphStore: IGraphStore): void;
|
|
127
146
|
/** Set the hash cache for faster incremental checks. */
|
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import{createHash as e}from"node:crypto";import{existsSync as t,readFileSync as n,realpathSync as r,statSync as i,writeFileSync as a}from"node:fs";import{lstat as o,
|
|
2
|
-
`),a=i.slice(0,100);if(a.length>0&&a.reduce((e,t)=>e+t.length,0)/a.length>500)return!0;let o=i.slice(1,201);if(o.length>=20){let e=new Map,t=0;for(let n of o){let r=n.replace(/\d/g,`#`).slice(0,40),i=(e.get(r)??0)+1;e.set(r,i),i>t&&(t=i)}if(t/o.length>.8)return!0}if(
|
|
3
|
-
`&&(e+=1,e>=10))break;if(e<10)return!0}return!1}};const O=new Set([`.ts`,`.tsx`,`.js`,`.jsx`,`.mjs`,`.cjs`]),k=[{pattern:/^export\s+(?:async\s+)?function\s+(\w+)/gm,kind:`function`,exported:!0},{pattern:/^export\s+(?:default\s+)?class\s+(\w+)/gm,kind:`class`,exported:!0},{pattern:/^export\s+interface\s+(\w+)/gm,kind:`interface`,exported:!0},{pattern:/^export\s+type\s+(\w+)/gm,kind:`type`,exported:!0},{pattern:/^export\s+(?:const|let)\s+(\w+)/gm,kind:`const`,exported:!0},{pattern:/^export\s+enum\s+(\w+)/gm,kind:`enum`,exported:!0},{pattern:/^(?:async\s+)?function\s+(\w+)/gm,kind:`function`,exported:!1},{pattern:/^class\s+(\w+)/gm,kind:`class`,exported:!1},{pattern:/^interface\s+(\w+)/gm,kind:`interface`,exported:!1},{pattern:/^type\s+(\w+)/gm,kind:`type`,exported:!1},{pattern:/^enum\s+(\w+)/gm,kind:`enum`,exported:!1}],A=[/import\s+(?:(?:type\s+)?(?:(?:\{[^}]*\}|[\w*]+)\s+from\s+)?)['"]([^'"]+)['"]/g,/import\(\s*['"]([^'"]+)['"]\s*\)/g,/require\(\s*['"]([^'"]+)['"]\s*\)/g];function j(t,n,r){return e(`sha256`).update(`${t}:${n}:${r}`).digest(`hex`).slice(0,16)}function M(t,n,r){return e(`sha256`).update(`${t}-${r}-${n}`).digest(`hex`).slice(0,16)}function N(e,t){return f(u(t),e).replace(/\\/g,`/`).replace(/\.(js|jsx|ts|tsx|mjs|cjs)$/,``)}function P(e){return e.replace(/\.(js|jsx|ts|tsx|mjs|cjs)$/,``)}function F(e,t,n){let r=d(t).toLowerCase();if(!O.has(r))return{nodes:[],edges:[]};let i=[],a=[],o=new Date().toISOString(),s=new Set,c=P(t),l=j(`module`,c,c);i.push({id:l,type:`module`,name:t,properties:{ext:r},sourcePath:t,createdAt:o});for(let{pattern:n,kind:r,exported:c}of k){let u=new RegExp(n.source,n.flags),d;for(;(d=u.exec(e))!==null;){let e=d[1],n=`${r}:${e}`;if(s.has(n))continue;s.add(n);let u=j(r,e,t);i.push({id:u,type:r,name:e,properties:{exported:c},sourcePath:t,createdAt:o}),a.push({id:M(l,u,`defines`),fromId:l,toId:u,type:`defines`,weight:c?1:.5})}}let u=n?.workspacePackages,f=new Set;for(let n of A){let r=new RegExp(n.source,n.flags),i;for(;(i=r.exec(e))!==null;){let e=i[1];if(f.has(e))continue;let n;if(e.startsWith(`.`))n=N(e,t);else if(u){let t=e.startsWith(`@`)?e.split(`/`).slice(0,2).join(`/`):e.split(`/`)[0],r=u.get(t);if(!r)continue;n=r}else continue;f.add(e);let r=P(n),o=j(`module`,r,r);a.push({id:M(l,o,`imports`),fromId:l,toId:o,type:`imports`,properties:{source:e}})}}return{nodes:i,edges:a}}const I=_(`hash-cache`);var L=class{cache;filePath;dirty=!1;constructor(e){this.filePath=m(e,`file-hashes.json`),this.cache=new Map}load(){if(t(this.filePath))try{let e=n(this.filePath,`utf-8`),t=JSON.parse(e);this.cache=new Map(Object.entries(t)),I.debug(`Hash cache loaded`,{entries:this.cache.size})}catch(e){I.warn(`Hash cache load failed, starting fresh`,{err:e}),this.cache=new Map}}get(e){return this.cache.get(e)}set(e,t){this.cache.set(e,t),this.dirty=!0}delete(e){this.cache.delete(e)&&(this.dirty=!0)}flush(){if(this.dirty)try{let e={};for(let[t,n]of this.cache)e[t]=n;a(this.filePath,JSON.stringify(e),`utf-8`),this.dirty=!1}catch(e){I.warn(`Hash cache flush failed`,{err:e})}}clear(){this.cache.clear(),this.dirty=!0,this.flush()}get size(){return this.cache.size}};const R=_(`indexer`),z=()=>new Promise(e=>setImmediate(e));async function B(e,t,n,r){let i=0;async function a(){for(;i<e.length;){let n=i++;try{await t(e[n])}catch(t){r?.(e[n],t)}n%10==0&&await z()}}await Promise.all(Array.from({length:Math.min(n,e.length)},()=>a()))}function V(e){let t=x(),n=S()[0]/t;return n>1.5?2:n>1?Math.max(2,Math.floor(e/2)):e}const H=Math.max(2,Math.min(4,Math.floor(x()*.5)));var U=class{embedder;store;crawler;indexing=!1;graphStore;hashCache;get isIndexing(){return this.indexing}constructor(e,t){this.embedder=e,this.store=t,this.crawler=new D}normalizeChunkText(e){return e.replace(/^\/\/ File: [^\n]*\n?/,``).toLowerCase().replace(/\s+/g,` `).trim()}setGraphStore(e){this.graphStore=e}setHashCache(e){this.hashCache=e}async index(e,t){if(this.indexing)throw Error(`Indexing is already in progress`);this.indexing=!0;try{return await this.doIndex(e,t,{})}finally{this.indexing=!1}}async getChangedFiles(e){let t=e.indexing.concurrency??H,n=await this.crawlSources(e),{filesToProcess:r}=await this.planIndexWork(n,t,{});return r.map(e=>e.relativePath)}async indexFiles(e,t,n){if(this.indexing)throw Error(`Indexing is already in progress`);this.indexing=!0;try{return await this.doIndexFiles(e,t,n)}finally{this.indexing=!1}}async doIndex(e,t,n={}){let r=Date.now(),i=e.indexing.concurrency??H;t?.({phase:`crawling`,filesTotal:0,filesProcessed:0,chunksTotal:0,chunksProcessed:0});let a=await this.crawlSources(e),{filesToProcess:o,filesSkipped:s,pathsToRemove:c}=await this.planIndexWork(a,i,{skipHashCheck:n.skipHashCheck}),{filesProcessed:l,chunksCreated:u,chunksDeduped:d}=await this.processFiles(a,o,i,t,{graphCleared:n.graphCleared}),f=await this.cleanupRemovedFiles(c,i,o.length,l,u,t);return this.hashCache?.flush(),t?.({phase:`done`,filesTotal:o.length,filesProcessed:l,chunksTotal:u,chunksProcessed:u}),{filesProcessed:l,filesSkipped:s,chunksCreated:u,chunksDeduped:d,filesRemoved:f,durationMs:Date.now()-r}}async doIndexFiles(e,t,n){let r=Date.now(),i=e.indexing.concurrency??H,a=new Set(t);if(a.size===0)return{filesProcessed:0,filesSkipped:0,chunksCreated:0,chunksDeduped:0,filesRemoved:0,durationMs:Date.now()-r};n?.({phase:`crawling`,filesTotal:0,filesProcessed:0,chunksTotal:0,chunksProcessed:0});let o=await this.crawlSources(e),{filesToProcess:s,filesSkipped:c}=await this.planIndexWork(o,i,{requestedPaths:a}),{filesProcessed:l,chunksCreated:u,chunksDeduped:d}=await this.processFiles(o,s,i,n,{});return this.hashCache?.flush(),n?.({phase:`done`,filesTotal:s.length,filesProcessed:l,chunksTotal:u,chunksProcessed:u}),{filesProcessed:l,filesSkipped:c,chunksCreated:u,chunksDeduped:d,filesRemoved:0,durationMs:Date.now()-r}}async crawlSources(e){return(await Promise.all(e.sources.map(e=>this.crawler.crawl({rootDir:e.path,excludePatterns:e.excludePatterns})))).flat()}async planIndexWork(e,t,n){let r=n.requestedPaths,i;if(r){let t=e=>e.replace(/\\/g,`/`).toLowerCase(),n=new Set([...r].map(t));i=e.filter(e=>n.has(t(e.relativePath))||n.has(t(e.absolutePath)))}else i=e;if(n.skipHashCheck)return{filesToProcess:i,filesSkipped:0,pathsToRemove:[]};let a=r?[]:await this.getPathsToRemove(e),o=0,s=[];return await B(i,async e=>{let t=w(e.content);if(this.hashCache){if(this.hashCache.get(e.relativePath)===t){o++;return}}else{let n=await this.store.getBySourcePath(e.relativePath);if(n.length>0&&n[0].fileHash===t){o++;return}}s.push(e)},V(t),(e,t)=>R.error(`Hash check failed`,{sourcePath:e.relativePath,...y(t)})),{filesToProcess:s,filesSkipped:o,pathsToRemove:a}}async getPathsToRemove(e){let t=await this.store.listSourcePaths(),n=new Set(e.map(e=>e.relativePath));return t.filter(e=>!n.has(e)&&!e.startsWith(`${h.aiCurated}/`))}async buildWorkspacePackageMap(e){let t=new Map;for(let n of e)if(n.relativePath.endsWith(`package.json`))try{let e=JSON.parse(n.content);if(typeof e.name!=`string`||e.name.length===0)continue;let r=n.relativePath===`package.json`?``:n.relativePath.replace(/\/package\.json$/,``),i=``;if(typeof e.exports==`string`)i=e.exports;else if(e.exports&&typeof e.exports==`object`){let t=e.exports,n=t[`.`]??t;if(typeof n==`string`)i=n;else if(n&&typeof n==`object`){let e=n,t=e.import??e.module??e.default??e.require;typeof t==`string`&&(i=t)}}!i&&typeof e.module==`string`&&(i=e.module),!i&&typeof e.main==`string`&&(i=e.main);let a=i.replace(/^\.\//,``).replace(/^dist\//,`src/`).replace(/\.(ts|js|mjs|cjs|tsx|jsx)$/,``),o=a?r?`${r}/${a}`:a:r?`${r}/src/index`:`src/index`;t.set(e.name,o)}catch{}return t}async processFiles(t,n,r,i,a={}){let o=0,s=0,c=0,l=n.length,u=this.graphStore?await this.buildWorkspacePackageMap(t):void 0,d=new Map,f=[],p=[],m=0,h=[],g=[],_=new Map,b=0,x=async()=>{if(h.length===0)return;let e=h,t=g,n=_;h=[],g=[],_=new Map,b=0;try{await this.store.upsert(e,t);for(let[e,t]of n)this.hashCache?.set(e,t)}catch(r){h.push(...e),g.push(...t);for(let[e,t]of n)_.set(e,t);throw r}},S=async()=>{if(this.graphStore){try{f.length>0&&await this.graphStore.upsertNodes(f),p.length>0&&await this.graphStore.upsertEdges(p)}catch(e){R.warn(`Graph batch flush failed`,y(e))}f=[],p=[],m=0}};return await B(n,async t=>{i?.({phase:`chunking`,filesTotal:l,filesProcessed:o,chunksTotal:s,chunksProcessed:s,currentFile:t.relativePath});let n=v(t.relativePath),r=C(t.extension).chunk(t.content,{sourcePath:t.relativePath,contentType:n});if(r.length===0)return;let E=w(t.content),D=new Date().toISOString(),O=(e,n,r)=>({id:T(t.relativePath,n),content:e.text,sourcePath:e.sourcePath,contentType:e.contentType,headingPath:e.headingPath,chunkIndex:e.chunkIndex,totalChunks:e.totalChunks,startLine:e.startLine,endLine:e.endLine,fileHash:E,contentHash:r,indexedAt:D,origin:`indexed`,tags:[],version:1}),k=[],A=[];for(let t=0;t<r.length;t++){let n=r[t],i=e(`sha256`).update(this.normalizeChunkText(n.text)).digest(`hex`),a=O(n,t,i),o=d.get(i);if(o&&this.store.upsertWithoutVector){A.push({record:a,sourceRecordId:o}),c++;continue}d.set(i,a.id),k.push({chunk:n,record:a})}i?.({phase:`embedding`,filesTotal:l,filesProcessed:o,chunksTotal:s+r.length,chunksProcessed:s,currentFile:t.relativePath});let j=k.length>0?await this.embedder.embedBatch(k.map(({chunk:e})=>e.text)):[],M=k.map(({record:e})=>e);if(i?.({phase:`storing`,filesTotal:l,filesProcessed:o,chunksTotal:s+r.length,chunksProcessed:s,currentFile:t.relativePath}),M.length>0&&(h.push(...M),g.push(...j),_.set(t.relativePath,E),b++,b>=20&&await x()),A.length>0&&h.length>0&&await x(),this.store.upsertWithoutVector)for(let{record:e,sourceRecordId:t}of A)await this.store.upsertWithoutVector(e,t);if(M.length===0&&A.length>0&&this.hashCache?.set(t.relativePath,E),this.graphStore)try{a.graphCleared||await this.graphStore.deleteBySourcePath(t.relativePath);let e=F(t.content,t.relativePath,{workspacePackages:u});e.nodes.length>0&&f.push(...e.nodes),e.edges.length>0&&p.push(...e.edges),m++,m>=50&&await S()}catch(e){R.warn(`Graph extraction failed`,{sourcePath:t.relativePath,...y(e)})}o++,s+=r.length},V(r),(e,t)=>R.error(`Processing failed`,{sourcePath:e.relativePath,...y(t)})),await x(),await S(),{filesProcessed:o,chunksCreated:s,chunksDeduped:c}}async cleanupRemovedFiles(e,t,n,r,i,a){if(e.length===0)return 0;let o=0;return a?.({phase:`cleanup`,filesTotal:n,filesProcessed:r,chunksTotal:i,chunksProcessed:i}),await B(e,async e=>{await this.store.deleteBySourcePath(e),this.hashCache?.delete(e),this.graphStore&&await this.graphStore.deleteBySourcePath(e).catch(t=>R.warn(`Graph cleanup failed`,{sourcePath:e,...y(t)})),o++},V(t),(e,t)=>R.error(`Cleanup failed`,{sourcePath:e,...y(t)})),o}async reindexAll(e,t){if(this.indexing)throw Error(`Indexing is already in progress`);this.indexing=!0;try{if(await this.store.dropTable(),this.graphStore)try{let e=await this.graphStore.getStats();e.nodeCount>0&&(await this.graphStore.clear(),R.info(`Graph store cleared`,{nodeCount:e.nodeCount,edgeCount:e.edgeCount}))}catch(e){R.warn(`Graph store clear failed`,y(e))}return await this.doReindex(e,t)}catch(e){throw this.indexing=!1,e}}async doReindex(e,t){try{return await this.doIndex(e,t,{skipHashCheck:!0,graphCleared:!0})}finally{this.indexing=!1}}async getStats(){return this.store.getStats()}};const W=_(`smart-index`),G=1.5;var K=class{indexer;config;store;trickleTimer=null;stopped=!1;trickleIntervalMs;batchSize;priorityQueue=[];changedFiles=[];lastRefreshTime=0;refreshing=!1;constructor(e,t,n){this.indexer=e,this.config=t,this.store=n,this.trickleIntervalMs=t.indexing.trickleIntervalMs??this.readPositiveIntEnv(`AIKIT_SMART_TRICKLE_MS`,3e4),this.batchSize=t.indexing.trickleBatchSize??this.readPositiveIntEnv(`AIKIT_SMART_BATCH_SIZE`,1)}start(){this.trickleTimer&&=(clearTimeout(this.trickleTimer),null),this.stopped=!1,W.debug(`Smart index scheduler started (trickle mode)`,{intervalMs:this.trickleIntervalMs,batchSize:this.batchSize}),this.scheduleTick()}stop(){this.stopped=!0,this.trickleTimer&&=(clearTimeout(this.trickleTimer),null)}prioritize(...e){let t=[...new Set(e.filter(Boolean))].filter(e=>{try{return!i(e).isDirectory()}catch{return W.debug(`Skipping non-existent path`,{path:e}),!1}});for(let e of t){let t=this.priorityQueue.indexOf(e);t>=0&&this.priorityQueue.splice(t,1)}for(let e of t.reverse())this.priorityQueue.unshift(e);this.priorityQueue.length>500&&(this.priorityQueue.length=500),t.length>0&&W.debug(`Files prioritized for trickle indexing`,{added:t.length,queued:this.priorityQueue.length})}getState(){return{mode:`smart`,queueSize:this.priorityQueue.length,changedFilesSize:this.changedFiles.length,intervalMs:this.trickleIntervalMs,batchSize:this.batchSize,running:this.trickleTimer!==null}}readPositiveIntEnv(e,t){let n=Number(process.env[e]);return Number.isFinite(n)&&n>0?n:t}scheduleTick(){this.trickleTimer=setTimeout(()=>void this.tick(),this.trickleIntervalMs),this.trickleTimer.unref&&this.trickleTimer.unref()}async tick(){if(!this.stopped)try{if(this.indexer.isIndexing){W.info(`Skipping trickle tick — indexing already in progress`);return}let e=this.getCpuCount(),t=S()[0];if(e>0&&t/e>G){W.info(`Skipping trickle tick — system load too high`,{load:t.toFixed(2),cpuCount:e,threshold:G});return}let n=await this.pickFiles();if(n.length===0){await this.maybeRefreshChangedFiles();return}W.debug(`Trickle indexing tick started`,{count:n.length,files:n});let r=await this.indexer.indexFiles(this.config,n);if(this.store)try{await this.store.createFtsIndex()}catch(e){W.warn(`FTS index rebuild failed after trickle tick`,{error:String(e)})}this.changedFiles=this.changedFiles.filter(e=>!n.includes(e)),W.debug(`Trickle indexing tick complete`,{filesProcessed:r.filesProcessed,filesSkipped:r.filesSkipped,chunksCreated:r.chunksCreated})}catch(e){let t=String(e);if(t.includes(`not initialized`)||t.includes(`has been closed`)){W.warn(`Store closed — stopping smart index scheduler`,{error:t}),this.stop();return}W.error(`Trickle indexing tick failed`,{error:t})}finally{this.stopped||this.scheduleTick()}}getCpuCount(){try{return typeof x==`function`?x():4}catch{return 4}}async pickFiles(){let e=[];for(;e.length<this.batchSize&&this.priorityQueue.length>0;){let t=this.priorityQueue.shift();t&&!e.includes(t)&&e.push(t)}if(e.length<this.batchSize)for(await this.maybeRefreshChangedFiles();e.length<this.batchSize&&this.changedFiles.length>0;){let t=this.changedFiles.shift();t&&!e.includes(t)&&e.push(t)}return e}async maybeRefreshChangedFiles(){let e=Date.now();if(!(this.refreshing||this.changedFiles.length>0&&e-this.lastRefreshTime<6e5)){this.refreshing=!0;try{this.changedFiles=await this.indexer.getChangedFiles(this.config),this.lastRefreshTime=e,this.changedFiles.length>0&&W.debug(`Refreshed changed files for trickle indexing`,{count:this.changedFiles.length})}catch(e){let t=String(e);if(t.includes(`not initialized`)||t.includes(`has been closed`)){W.warn(`Store closed — stopping smart index scheduler`,{error:t}),this.stop();return}W.error(`Failed to refresh changed files for trickle indexing`,{error:t})}finally{this.refreshing=!1}}}};export{L as FileHashCache,D as FilesystemCrawler,U as IncrementalIndexer,K as SmartIndexScheduler,F as extractGraph,T as generateRecordId,w as hashContent};
|
|
1
|
+
import{createHash as e}from"node:crypto";import{existsSync as t,readFileSync as n,realpathSync as r,statSync as i,writeFileSync as a}from"node:fs";import{lstat as o,open as s,readFile as c,readdir as l,stat as u}from"node:fs/promises";import{dirname as d,extname as f,join as p,relative as m,resolve as h}from"node:path";import{AIKIT_PATHS as g,FILE_LIMITS as _,createLogger as v,detectContentType as y,serializeError as b}from"../../core/dist/index.js";import{minimatch as x}from"minimatch";import{availableParallelism as S,loadavg as C}from"node:os";import{createChunkerSync as w}from"../../chunker/dist/index.js";function T(t){return e(`sha256`).update(t).digest(`hex`).slice(0,16)}function E(t,n){let r=`${t}:${n}`;return e(`sha256`).update(r).digest(`hex`).slice(0,16)}const D=v(`indexer`);var O=class e{static BINARY_EXTENSIONS=new Set(`.node,.so,.dylib,.dll,.wasm,.bin,.exe,.png,.jpg,.jpeg,.gif,.bmp,.ico,.webp,.svg,.mp3,.mp4,.wav,.avi,.mov,.flac,.zip,.gz,.tar,.bz2,.7z,.rar,.pdf,.doc,.docx,.xls,.xlsx,.ppt,.pptx,.ttf,.otf,.woff,.woff2,.eot,.pyc,.class,.o,.obj,.a,.lib`.split(`,`));static GENERATED_PATH_PATTERN=/(?:lcov\.info|\.min\.(?:js|css)|\.css\.map|\.js\.map|\.map)$/i;static GENERATED_SAMPLE_LIMIT=5e4;async crawl(e){let t=[],n=new Set;return await this.walkDir(e.rootDir,e.rootDir,e.excludePatterns,t,n),t}async walkDir(t,n,i,a,s){let d;try{d=await l(t,{withFileTypes:!0})}catch(e){let n=e.code;(n===`EACCES`||n===`EPERM`)&&D.warn(`Permission denied, skipping directory`,{dir:t});return}for(let l of d){let d=p(t,l.name),h=m(n,d).replace(/\\/g,`/`);if(!this.isExcluded(h,i)){if(l.isDirectory()){if(l.name.startsWith(`.`)&&!(l.name===g.ai.slice(1)&&h.startsWith(g.ai)))continue;try{if((await o(d)).isSymbolicLink())continue}catch{continue}let e;try{e=r(d)}catch{continue}if(s.has(e))continue;s.add(e),await this.walkDir(d,n,i,a,s)}else if(l.isFile()){let t=f(l.name).toLowerCase();if(e.BINARY_EXTENSIONS.has(t))continue;try{let n=await u(d);if(n.size>_.maxFileSizeBytes)continue;if(e.isLikelyGeneratedPath(h)){D.debug(`Skipping likely generated file`,{path:h});continue}if(t===`.snap`&&n.size>e.GENERATED_SAMPLE_LIMIT){D.debug(`Skipping likely generated snapshot`,{path:h});continue}let r;try{r=await e.readTextPreview(d)}catch{continue}if(r.includes(`\0`))continue;if(e.isLikelyGeneratedContent(r,h,n.size)){D.debug(`Skipping likely generated file`,{path:h});continue}let i,o,s={relativePath:h,absolutePath:d,extension:t,size:n.size,mtimeMs:n.mtimeMs,content:i,readContent:async()=>i===void 0?(o??=c(d,`utf-8`).then(e=>(i=e,s.content=e,e)),o):i};a.push(s)}catch{}}}}}isExcluded(e,t){return t.some(t=>x(e,t,{dot:!0}))}static isLikelyGeneratedPath(t){return e.GENERATED_PATH_PATTERN.test(t)}static isLikelyGeneratedContent(e,t,n){let r=f(t).toLowerCase();if(t.split(`/`).includes(`coverage`)&&r===``)return!0;let i=e.slice(0,5e4).split(`
|
|
2
|
+
`),a=i.slice(0,100);if(a.length>0&&a.reduce((e,t)=>e+t.length,0)/a.length>500)return!0;let o=i.slice(1,201);if(o.length>=20){let e=new Map,t=0;for(let n of o){let r=n.replace(/\d/g,`#`).slice(0,40),i=(e.get(r)??0)+1;e.set(r,i),i>t&&(t=i)}if(t/o.length>.8)return!0}if(e.length>1e5){let t=0;for(let n of e)if(n===`
|
|
3
|
+
`&&(t+=1,t>=10))break;if(t<10)return!0}return!1}static async readTextPreview(t){let n=await s(t,`r`);try{let t=Buffer.alloc(e.GENERATED_SAMPLE_LIMIT),{bytesRead:r}=await n.read(t,0,t.length,0);return t.toString(`utf8`,0,r)}finally{await n.close()}}};const k=new Set([`.ts`,`.tsx`,`.js`,`.jsx`,`.mjs`,`.cjs`]),A=[{pattern:/^export\s+(?:async\s+)?function\s+(\w+)/gm,kind:`function`,exported:!0},{pattern:/^export\s+(?:default\s+)?class\s+(\w+)/gm,kind:`class`,exported:!0},{pattern:/^export\s+interface\s+(\w+)/gm,kind:`interface`,exported:!0},{pattern:/^export\s+type\s+(\w+)/gm,kind:`type`,exported:!0},{pattern:/^export\s+(?:const|let)\s+(\w+)/gm,kind:`const`,exported:!0},{pattern:/^export\s+enum\s+(\w+)/gm,kind:`enum`,exported:!0},{pattern:/^(?:async\s+)?function\s+(\w+)/gm,kind:`function`,exported:!1},{pattern:/^class\s+(\w+)/gm,kind:`class`,exported:!1},{pattern:/^interface\s+(\w+)/gm,kind:`interface`,exported:!1},{pattern:/^type\s+(\w+)/gm,kind:`type`,exported:!1},{pattern:/^enum\s+(\w+)/gm,kind:`enum`,exported:!1}],j=[/import\s+(?:(?:type\s+)?(?:(?:\{[^}]*\}|[\w*]+)\s+from\s+)?)['"]([^'"]+)['"]/g,/import\(\s*['"]([^'"]+)['"]\s*\)/g,/require\(\s*['"]([^'"]+)['"]\s*\)/g];function M(t,n,r){return e(`sha256`).update(`${t}:${n}:${r}`).digest(`hex`).slice(0,16)}function N(t,n,r){return e(`sha256`).update(`${t}-${r}-${n}`).digest(`hex`).slice(0,16)}function P(e,t){return p(d(t),e).replace(/\\/g,`/`).replace(/\.(js|jsx|ts|tsx|mjs|cjs)$/,``)}function F(e){return e.replace(/\.(js|jsx|ts|tsx|mjs|cjs)$/,``)}function I(e,t,n){let r=f(t).toLowerCase();if(!k.has(r))return{nodes:[],edges:[]};let i=[],a=[],o=new Date().toISOString(),s=new Set,c=F(t),l=M(`module`,c,c);i.push({id:l,type:`module`,name:t,properties:{ext:r},sourcePath:t,createdAt:o});for(let{pattern:n,kind:r,exported:c}of A){let u=new RegExp(n.source,n.flags),d;for(;(d=u.exec(e))!==null;){let e=d[1],n=`${r}:${e}`;if(s.has(n))continue;s.add(n);let u=M(r,e,t);i.push({id:u,type:r,name:e,properties:{exported:c},sourcePath:t,createdAt:o}),a.push({id:N(l,u,`defines`),fromId:l,toId:u,type:`defines`,weight:c?1:.5})}}let u=n?.workspacePackages,d=new Set;for(let n of j){let r=new RegExp(n.source,n.flags),i;for(;(i=r.exec(e))!==null;){let e=i[1];if(d.has(e))continue;let n;if(e.startsWith(`.`))n=P(e,t);else if(u){let t=e.startsWith(`@`)?e.split(`/`).slice(0,2).join(`/`):e.split(`/`)[0],r=u.get(t);if(!r)continue;n=r}else continue;d.add(e);let r=F(n),o=M(`module`,r,r);a.push({id:N(l,o,`imports`),fromId:l,toId:o,type:`imports`,properties:{source:e}})}}return{nodes:i,edges:a}}const L=v(`hash-cache`);var R=class{cache;filePath;dirty=!1;constructor(e){this.filePath=h(e,`file-hashes.json`),this.cache=new Map}load(){if(t(this.filePath))try{let e=n(this.filePath,`utf-8`),t=JSON.parse(e);this.cache=new Map(Object.entries(t).map(([e,t])=>[e,typeof t==`string`?{hash:t}:t])),L.debug(`Hash cache loaded`,{entries:this.cache.size})}catch(e){L.warn(`Hash cache load failed, starting fresh`,{err:e}),this.cache=new Map}}get(e){return this.cache.get(e)?.hash}getEntry(e){return this.cache.get(e)}set(e,t,n){this.cache.set(e,{hash:t,...n?{size:n.size,mtimeMs:n.mtimeMs}:{}}),this.dirty=!0}delete(e){this.cache.delete(e)&&(this.dirty=!0)}flush(){if(this.dirty)try{let e={};for(let[t,n]of this.cache)e[t]=n;a(this.filePath,JSON.stringify(e),`utf-8`),this.dirty=!1}catch(e){L.warn(`Hash cache flush failed`,{err:e})}}clear(){this.cache.clear(),this.dirty=!0,this.flush()}get size(){return this.cache.size}};const z=v(`indexer`),B=()=>new Promise(e=>setImmediate(e));async function V(e,t,n,r){let i=0;async function a(){for(;i<e.length;){let n=i++;try{await t(e[n])}catch(t){r?.(e[n],t)}n%10==0&&await B()}}await Promise.all(Array.from({length:Math.min(n,e.length)},()=>a()))}function H(e){let t=S(),n=C()[0]/t;return n>1.5?2:n>1?Math.max(2,Math.floor(e/2)):e}const U=Math.max(2,Math.min(4,Math.floor(S()*.5)));var W=class{embedder;store;crawler;indexing=!1;graphStore;hashCache;get isIndexing(){return this.indexing}constructor(e,t){this.embedder=e,this.store=t,this.crawler=new O}normalizeChunkText(e){return e.replace(/^\/\/ File: [^\n]*\n?/,``).toLowerCase().replace(/\s+/g,` `).trim()}async loadFileContent(e){if(e.content!==void 0)return e.content;let t=await e.readContent();return e.content=t,t}isMetadataUnchanged(e,t){return t?.size!==void 0&&t?.mtimeMs!==void 0&&t.size===e.size&&t.mtimeMs===e.mtimeMs}setGraphStore(e){this.graphStore=e}setHashCache(e){this.hashCache=e}async index(e,t){if(this.indexing)throw Error(`Indexing is already in progress`);this.indexing=!0;try{return await this.doIndex(e,t,{})}finally{this.indexing=!1}}async getChangedFiles(e){let t=e.indexing.concurrency??U,n=await this.crawlSources(e),{filesToProcess:r}=await this.planIndexWork(n,t,{});return r.map(e=>e.relativePath)}async indexFiles(e,t,n){if(this.indexing)throw Error(`Indexing is already in progress`);this.indexing=!0;try{return await this.doIndexFiles(e,t,n)}finally{this.indexing=!1}}async doIndex(e,t,n={}){let r=Date.now(),i=e.indexing.concurrency??U;t?.({phase:`crawling`,filesTotal:0,filesProcessed:0,chunksTotal:0,chunksProcessed:0});let a=await this.crawlSources(e),{filesToProcess:o,filesSkipped:s,pathsToRemove:c}=await this.planIndexWork(a,i,{skipHashCheck:n.skipHashCheck}),{filesProcessed:l,chunksCreated:u,chunksDeduped:d}=await this.processFiles(a,o,i,t,{graphCleared:n.graphCleared}),f=await this.cleanupRemovedFiles(c,i,o.length,l,u,t);return this.hashCache?.flush(),t?.({phase:`done`,filesTotal:o.length,filesProcessed:l,chunksTotal:u,chunksProcessed:u}),{filesProcessed:l,filesSkipped:s,chunksCreated:u,chunksDeduped:d,filesRemoved:f,durationMs:Date.now()-r}}async doIndexFiles(e,t,n){let r=Date.now(),i=e.indexing.concurrency??U,a=new Set(t);if(a.size===0)return{filesProcessed:0,filesSkipped:0,chunksCreated:0,chunksDeduped:0,filesRemoved:0,durationMs:Date.now()-r};n?.({phase:`crawling`,filesTotal:0,filesProcessed:0,chunksTotal:0,chunksProcessed:0});let o=await this.crawlSources(e),{filesToProcess:s,filesSkipped:c}=await this.planIndexWork(o,i,{requestedPaths:a}),{filesProcessed:l,chunksCreated:u,chunksDeduped:d}=await this.processFiles(o,s,i,n,{});return this.hashCache?.flush(),n?.({phase:`done`,filesTotal:s.length,filesProcessed:l,chunksTotal:u,chunksProcessed:u}),{filesProcessed:l,filesSkipped:c,chunksCreated:u,chunksDeduped:d,filesRemoved:0,durationMs:Date.now()-r}}async crawlSources(e){return(await Promise.all(e.sources.map(e=>this.crawler.crawl({rootDir:e.path,excludePatterns:e.excludePatterns})))).flat()}async planIndexWork(e,t,n){let r=n.requestedPaths,i;if(r){let t=e=>e.replace(/\\/g,`/`).toLowerCase(),n=new Set([...r].map(t));i=e.filter(e=>n.has(t(e.relativePath))||n.has(t(e.absolutePath)))}else i=e;if(n.skipHashCheck)return{filesToProcess:i,filesSkipped:0,pathsToRemove:[]};let a=r?[]:await this.getPathsToRemove(e),o=0,s=[];return await V(i,async e=>{let t=this.hashCache?.getEntry(e.relativePath);if(this.isMetadataUnchanged(e,t)){o++;return}let n=await this.loadFileContent(e);if(n.includes(`\0`)){o++;return}if(O.isLikelyGeneratedContent(n,e.relativePath,e.size)){o++;return}let r=T(n);if(this.hashCache){if((t?.hash??this.hashCache.get(e.relativePath))===r){o++;return}}else{let t=await this.store.getBySourcePath(e.relativePath);if(t.length>0&&t[0].fileHash===r){o++;return}}s.push(e)},H(t),(e,t)=>z.error(`Hash check failed`,{sourcePath:e.relativePath,...b(t)})),{filesToProcess:s,filesSkipped:o,pathsToRemove:a}}async getPathsToRemove(e){let t=await this.store.listSourcePaths(),n=new Set(e.map(e=>e.relativePath));return t.filter(e=>!n.has(e)&&!e.startsWith(`${g.aiCurated}/`))}async buildWorkspacePackageMap(e){let t=new Map;for(let n of e)if(n.relativePath.endsWith(`package.json`))try{let e=JSON.parse(await this.loadFileContent(n));if(typeof e.name!=`string`||e.name.length===0)continue;let r=n.relativePath===`package.json`?``:n.relativePath.replace(/\/package\.json$/,``),i=``;if(typeof e.exports==`string`)i=e.exports;else if(e.exports&&typeof e.exports==`object`){let t=e.exports,n=t[`.`]??t;if(typeof n==`string`)i=n;else if(n&&typeof n==`object`){let e=n,t=e.import??e.module??e.default??e.require;typeof t==`string`&&(i=t)}}!i&&typeof e.module==`string`&&(i=e.module),!i&&typeof e.main==`string`&&(i=e.main);let a=i.replace(/^\.\//,``).replace(/^dist\//,`src/`).replace(/\.(ts|js|mjs|cjs|tsx|jsx)$/,``),o=a?r?`${r}/${a}`:a:r?`${r}/src/index`:`src/index`;t.set(e.name,o)}catch{}return t}async processFiles(t,n,r,i,a={}){let o=0,s=0,c=0,l=n.length,u=this.graphStore?await this.buildWorkspacePackageMap(t):void 0,d=new Map,f=[],p=[],m=0,h=[],g=[],_=new Map,v=0,x=async()=>{if(h.length===0)return;let e=h,t=g,n=_;h=[],g=[],_=new Map,v=0;try{await this.store.upsert(e,t);for(let[e,t]of n)this.hashCache?.set(e,t.hash,{size:t.size,mtimeMs:t.mtimeMs})}catch(r){h.push(...e),g.push(...t);for(let[e,t]of n)_.set(e,t);throw r}},S=async()=>{if(this.graphStore){try{f.length>0&&await this.graphStore.upsertNodes(f),p.length>0&&await this.graphStore.upsertEdges(p)}catch(e){z.warn(`Graph batch flush failed`,b(e))}f=[],p=[],m=0}};return await V(n,async t=>{i?.({phase:`chunking`,filesTotal:l,filesProcessed:o,chunksTotal:s,chunksProcessed:s,currentFile:t.relativePath});let n=await this.loadFileContent(t),r=y(t.relativePath),C=w(t.extension).chunk(n,{sourcePath:t.relativePath,contentType:r});if(C.length===0)return;let D=T(n),O=new Date().toISOString(),k=(e,n,r)=>({id:E(t.relativePath,n),content:e.text,sourcePath:e.sourcePath,contentType:e.contentType,headingPath:e.headingPath,chunkIndex:e.chunkIndex,totalChunks:e.totalChunks,startLine:e.startLine,endLine:e.endLine,fileHash:D,contentHash:r,indexedAt:O,origin:`indexed`,tags:[],version:1}),A=[],j=[];for(let t=0;t<C.length;t++){let n=C[t],r=e(`sha256`).update(this.normalizeChunkText(n.text)).digest(`hex`),i=k(n,t,r),a=d.get(r);if(a&&this.store.upsertWithoutVector){j.push({record:i,sourceRecordId:a}),c++;continue}d.set(r,i.id),A.push({chunk:n,record:i})}i?.({phase:`embedding`,filesTotal:l,filesProcessed:o,chunksTotal:s+C.length,chunksProcessed:s,currentFile:t.relativePath});let M=A.length>0?await this.embedder.embedBatch(A.map(({chunk:e})=>e.text)):[],N=A.map(({record:e})=>e);if(i?.({phase:`storing`,filesTotal:l,filesProcessed:o,chunksTotal:s+C.length,chunksProcessed:s,currentFile:t.relativePath}),N.length>0&&(h.push(...N),g.push(...M),_.set(t.relativePath,{hash:D,size:t.size,mtimeMs:t.mtimeMs}),v++,v>=20&&await x()),j.length>0&&h.length>0&&await x(),this.store.upsertWithoutVector)for(let{record:e,sourceRecordId:t}of j)await this.store.upsertWithoutVector(e,t);if(N.length===0&&j.length>0&&this.hashCache?.set(t.relativePath,D,{size:t.size,mtimeMs:t.mtimeMs}),this.graphStore)try{a.graphCleared||await this.graphStore.deleteBySourcePath(t.relativePath);let e=I(n,t.relativePath,{workspacePackages:u});e.nodes.length>0&&f.push(...e.nodes),e.edges.length>0&&p.push(...e.edges),m++,m>=50&&await S()}catch(e){z.warn(`Graph extraction failed`,{sourcePath:t.relativePath,...b(e)})}o++,s+=C.length},H(r),(e,t)=>z.error(`Processing failed`,{sourcePath:e.relativePath,...b(t)})),await x(),await S(),{filesProcessed:o,chunksCreated:s,chunksDeduped:c}}async cleanupRemovedFiles(e,t,n,r,i,a){if(e.length===0)return 0;let o=0;return a?.({phase:`cleanup`,filesTotal:n,filesProcessed:r,chunksTotal:i,chunksProcessed:i}),await V(e,async e=>{await this.store.deleteBySourcePath(e),this.hashCache?.delete(e),this.graphStore&&await this.graphStore.deleteBySourcePath(e).catch(t=>z.warn(`Graph cleanup failed`,{sourcePath:e,...b(t)})),o++},H(t),(e,t)=>z.error(`Cleanup failed`,{sourcePath:e,...b(t)})),o}async reindexAll(e,t){if(this.indexing)throw Error(`Indexing is already in progress`);this.indexing=!0;try{if(await this.store.dropTable(),this.graphStore)try{let e=await this.graphStore.getStats();e.nodeCount>0&&(await this.graphStore.clear(),z.info(`Graph store cleared`,{nodeCount:e.nodeCount,edgeCount:e.edgeCount}))}catch(e){z.warn(`Graph store clear failed`,b(e))}return await this.doReindex(e,t)}catch(e){throw this.indexing=!1,e}}async doReindex(e,t){try{return await this.doIndex(e,t,{skipHashCheck:!0,graphCleared:!0})}finally{this.indexing=!1}}async getStats(){return this.store.getStats()}};const G=v(`smart-index`),K=1.5;var q=class{indexer;config;store;trickleTimer=null;stopped=!1;trickleIntervalMs;batchSize;priorityQueue=[];changedFiles=[];lastRefreshTime=0;refreshing=!1;constructor(e,t,n){this.indexer=e,this.config=t,this.store=n,this.trickleIntervalMs=t.indexing.trickleIntervalMs??this.readPositiveIntEnv(`AIKIT_SMART_TRICKLE_MS`,3e4),this.batchSize=t.indexing.trickleBatchSize??this.readPositiveIntEnv(`AIKIT_SMART_BATCH_SIZE`,1)}start(){this.trickleTimer&&=(clearTimeout(this.trickleTimer),null),this.stopped=!1,G.debug(`Smart index scheduler started (trickle mode)`,{intervalMs:this.trickleIntervalMs,batchSize:this.batchSize}),this.scheduleTick()}stop(){this.stopped=!0,this.trickleTimer&&=(clearTimeout(this.trickleTimer),null)}prioritize(...e){let t=[...new Set(e.filter(Boolean))].filter(e=>{try{return!i(e).isDirectory()}catch{return G.debug(`Skipping non-existent path`,{path:e}),!1}});for(let e of t){let t=this.priorityQueue.indexOf(e);t>=0&&this.priorityQueue.splice(t,1)}for(let e of t.reverse())this.priorityQueue.unshift(e);this.priorityQueue.length>500&&(this.priorityQueue.length=500),t.length>0&&G.debug(`Files prioritized for trickle indexing`,{added:t.length,queued:this.priorityQueue.length})}getState(){return{mode:`smart`,queueSize:this.priorityQueue.length,changedFilesSize:this.changedFiles.length,intervalMs:this.trickleIntervalMs,batchSize:this.batchSize,running:this.trickleTimer!==null}}readPositiveIntEnv(e,t){let n=Number(process.env[e]);return Number.isFinite(n)&&n>0?n:t}scheduleTick(){this.trickleTimer=setTimeout(()=>void this.tick(),this.trickleIntervalMs),this.trickleTimer.unref&&this.trickleTimer.unref()}async tick(){if(!this.stopped)try{if(this.indexer.isIndexing){G.info(`Skipping trickle tick — indexing already in progress`);return}let e=this.getCpuCount(),t=C()[0];if(e>0&&t/e>K){G.info(`Skipping trickle tick — system load too high`,{load:t.toFixed(2),cpuCount:e,threshold:K});return}let n=await this.pickFiles();if(n.length===0){await this.maybeRefreshChangedFiles();return}G.debug(`Trickle indexing tick started`,{count:n.length,files:n});let r=await this.indexer.indexFiles(this.config,n);if(this.store)try{await this.store.createFtsIndex()}catch(e){G.warn(`FTS index rebuild failed after trickle tick`,{error:String(e)})}this.changedFiles=this.changedFiles.filter(e=>!n.includes(e)),G.debug(`Trickle indexing tick complete`,{filesProcessed:r.filesProcessed,filesSkipped:r.filesSkipped,chunksCreated:r.chunksCreated})}catch(e){let t=String(e);if(t.includes(`not initialized`)||t.includes(`has been closed`)){G.warn(`Store closed — stopping smart index scheduler`,{error:t}),this.stop();return}G.error(`Trickle indexing tick failed`,{error:t})}finally{this.stopped||this.scheduleTick()}}getCpuCount(){try{return typeof S==`function`?S():4}catch{return 4}}async pickFiles(){let e=[];for(;e.length<this.batchSize&&this.priorityQueue.length>0;){let t=this.priorityQueue.shift();t&&!e.includes(t)&&e.push(t)}if(e.length<this.batchSize)for(await this.maybeRefreshChangedFiles();e.length<this.batchSize&&this.changedFiles.length>0;){let t=this.changedFiles.shift();t&&!e.includes(t)&&e.push(t)}return e}async maybeRefreshChangedFiles(){let e=Date.now();if(!(this.refreshing||this.changedFiles.length>0&&e-this.lastRefreshTime<6e5)){this.refreshing=!0;try{this.changedFiles=await this.indexer.getChangedFiles(this.config),this.lastRefreshTime=e,this.changedFiles.length>0&&G.debug(`Refreshed changed files for trickle indexing`,{count:this.changedFiles.length})}catch(e){let t=String(e);if(t.includes(`not initialized`)||t.includes(`has been closed`)){G.warn(`Store closed — stopping smart index scheduler`,{error:t}),this.stop();return}G.error(`Failed to refresh changed files for trickle indexing`,{error:t})}finally{this.refreshing=!1}}}};export{R as FileHashCache,O as FilesystemCrawler,W as IncrementalIndexer,q as SmartIndexScheduler,I as extractGraph,E as generateRecordId,T as hashContent};
|
|
@@ -5,4 +5,4 @@ import{t as e}from"./rolldown-runtime-yuFVEuWy.js";import{createHash as t,random
|
|
|
5
5
|
`).length,fileHash:this.hash(e.content),indexedAt:t,origin:`curated`,tags:e.frontmatter.tags,category:e.frontmatter.category,version:e.frontmatter.version}});try{return await this.store.upsert(i,r),e.length}catch(t){I.error(`Failed to upsert curated batch`,{batchSize:e.length,...p(t)});for(let t of e)n.push(`${t.relativePath}: upsert failed`);return 0}}catch(r){if(e.length===1)return I.error(`Failed to embed curated item`,{relativePath:e[0].relativePath,...p(r)}),n.push(`${e[0].relativePath}: reindex failed`),0;I.warn(`Curated embed batch failed, retrying with smaller chunks`,{batchSize:e.length,...p(r)});let i=Math.ceil(e.length/2),a=e.slice(0,i),o=e.slice(i);return await this.embedAndUpsertBatch(a,t,n)+await this.embedAndUpsertBatch(o,t,n)}}gitCommitKnowledge(e,t,n){try{if(!h(this.curatedDir))return;let r=this.knowledgeRefForPath(e);if(!r)return;g(r,`entry.md`,t,n,this.curatedDir)}catch{}}gitDeleteKnowledgeRef(e){try{if(!h(this.curatedDir))return;let t=this.knowledgeRefForPath(e);if(!t)return;_([`update-ref`,`-d`,t],this.curatedDir)}catch{}}knowledgeRefForPath(e){let t=e.replace(/\.md$/,``).split(`/`).map(e=>v(e)).join(`/`);return t.split(`/`).every(e=>m.test(e))?`${F}/${t}`:null}async indexCuratedFile(e,t,n){let r=await this.embedder.embed(t),i=`.ai/curated/${e}`,a=new Date().toISOString(),o={id:this.hashId(i,0),content:t,sourcePath:i,contentType:`curated-knowledge`,headingPath:n.title,chunkIndex:0,totalChunks:1,startLine:1,endLine:t.split(`
|
|
6
6
|
`).length,fileHash:this.hash(t),indexedAt:a,origin:`curated`,tags:n.tags,category:n.category,version:n.version};await this.store.upsert([o],[r])}async indexCuratedFileBestEffort(e,t,n,r){if(d.instance().isDegraded(`embedder`)){I.debug(`Skipping vector indexing — embedder degraded`,{relativePath:e,operation:r,subsystem:`embedder`});return}try{await this.indexCuratedFile(e,t,n)}catch(t){I.warn(`Curated file persisted but vector indexing deferred`,{relativePath:e,operation:r,...p(t)})}}async discoverCategories(){return this.adapter.listDirectories()}guardPath(e){let t=e.replace(/^\.ai\/curated\//,``);if(t.endsWith(`.md`)||(t+=`.md`),t.includes(`..`)||a(t))throw Error(`Invalid path: ${t}. Must be relative within .ai/curated/ directory.`);let n=t.split(`/`)[0];return this.validateCategoryName(n),t}validateCategoryName(e){if(!/^[a-z][a-z0-9-]*$/.test(e))throw Error(`Invalid category name: "${e}". Must be lowercase kebab-case (e.g., "decisions", "api-contracts").`)}validateContentSize(e){if(Buffer.byteLength(e,`utf-8`)>P)throw Error(`Content exceeds maximum size of ${P/1024}KB`)}slugify(e){return e.toLowerCase().replace(/[^a-z0-9]+/g,`-`).replace(/^-|-$/g,``).slice(0,80)}async uniqueRelativePath(e,t){let n=`${e}/${t}.md`;if(!await this.adapter.exists(n))return n;for(let n=2;n<=100;n++){let r=`${e}/${t}-${n}.md`;if(!await this.adapter.exists(r))return r}throw Error(`Too many entries with slug "${t}" in category "${e}"`)}hash(e){return t(`sha256`).update(e).digest(`hex`).slice(0,16)}hashId(e,t){return this.hash(`${e}::${t}`)}serializeFile(e,t){return`${[`---`,`title: "${t.title.replace(/"/g,`\\"`)}"`,`category: ${t.category}`,`tags: [${t.tags.map(e=>`"${e}"`).join(`, `)}]`,`created: ${t.created}`,`updated: ${t.updated}`,`version: ${t.version}`,`origin: ${t.origin}`,`changelog:`,...t.changelog.map(e=>` - version: ${e.version}\n date: ${e.date}\n reason: "${e.reason.replace(/"/g,`\\"`)}"`),`---`].join(`
|
|
7
7
|
`)}\n\n${e}\n`}parseFile(e){let t=e.match(/^---\n([\s\S]*?)\n---\n\n?([\s\S]*)$/);if(!t)return{frontmatter:{title:`Untitled`,category:`notes`,tags:[],created:``,updated:``,version:1,origin:`curated`,changelog:[]},content:e};let n=t[1],r=t[2].trim(),i={},a=[],o=n.split(`
|
|
8
|
-
`),s=!1,c={};for(let e of o){if(/^changelog:\s*$/.test(e)){s=!0;continue}if(s){let t=e.match(/^\s+-\s+version:\s*(\d+)$/);if(t){c.version!=null&&a.push(c),c={version:parseInt(t[1],10)};continue}let n=e.match(/^\s+date:\s*(.+)$/);if(n){c.date=n[1].trim();continue}let r=e.match(/^\s+reason:\s*"?(.*?)"?\s*$/);if(r){c.reason=r[1];continue}/^\w/.test(e)&&(s=!1,c.version!=null&&a.push(c),c={});continue}let t=e.match(/^(\w+):\s*(.*)$/);if(t){let e=t[1],n=t[2];typeof n==`string`&&n.startsWith(`[`)&&n.endsWith(`]`)?n=n.slice(1,-1).split(`,`).map(e=>e.trim().replace(/^"|"$/g,``)).filter(e=>e.length>0):typeof n==`string`&&/^\d+$/.test(n)?n=parseInt(n,10):typeof n==`string`&&n.startsWith(`"`)&&n.endsWith(`"`)&&(n=n.slice(1,-1)),i[e]=n}}return c.version!=null&&a.push(c),{frontmatter:{title:i.title??`Untitled`,category:i.category??`notes`,tags:i.tags??[],created:i.created??``,updated:i.updated??``,version:i.version??1,origin:`curated`,changelog:a},content:r}}};function R(){try{let e=s(i(c(import.meta.url)),`..`,`..`,`..`,`package.json`);return JSON.parse(r(e,`utf-8`)).version??`0.0.0`}catch{return`0.0.0`}}const z=f(`server`),B=/^https?:\/\/(localhost|127\.0\.0\.1)(:\d+)?$/i;function V({requestOrigin:e,configuredOrigin:t,allowAnyOrigin:n,fallbackOrigin:r}){let i=t??r;return i===`*`?n?{allowOrigin:`*`,warn:!1}:e?B.test(e)?{allowOrigin:e,warn:!1}:{allowOrigin:null,warn:!0}:{allowOrigin:r,warn:!1}:{allowOrigin:i,warn:!1}}function H({limit:e,windowMs:t}){let n=new Map,r=(e,r)=>{let i=r-t,a=n.get(e)?.filter(e=>e>i)??[];return a.length===0?(n.delete(e),[]):(n.set(e,a),a)};return{allow(t,i=Date.now()){let a=r(t,i);return a.length>=e?!1:(a.push(i),n.set(t,a),!0)},getRetryAfterMs(n,i=Date.now()){let a=r(n,i);return a.length<e?0:Math.max(0,a[0]+t-i)}}}function U(e){return e.startsWith(`file://`)?c(e):e}function W(e){let t=s(e);return process.platform===`win32`?t.toLowerCase():t}function G(e){let t=new Map;for(let n of e){let e=U(n.uri);t.set(W(e),e)}return[...t.values()].sort((e,t)=>W(e).localeCompare(W(t)))}function K({config:e,roots:t}){let n=G(t);if(n.length===0)return null;let r=e.sources?.[0]?.path;if(r){let e=W(r),t=n.find(t=>W(t)===e);if(t)return t}return n[0]}function q({config:e,log:t,reconfigureForWorkspace:n,roots:r}){let i=G(r);if(i.length===0)return!1;let a=K({config:e,roots:i.map(e=>({uri:e}))});if(!a)return!1;let o=e.sources?.[0]?.path,s=o&&W(o)===W(a)?`configured-source`:`stable-sorted-root`;return t.debug(`MCP roots resolved`,{rootPath:a,rootCount:i.length,selectedBy:s}),n(e,a),e.allRoots=i,!0}async function J({config:e,indexMode:t,log:n,rootsChangedNotificationSchema:r,reconfigureForWorkspace:i,runInitialIndex:a,server:o,startInit:s,timeoutMs:c=5e3,getCwd:l=()=>process.cwd()}){let u=!1,d=!1,f,m=o.server,h=e=>{let t=p(e),n=`${String(e)} ${String(t.message??``)}`.toLowerCase();return n.includes(`method not found`)||n.includes(`not supported`)||n.includes(`unsupported`)||n.includes(`unknown method`)||n.includes(`roots/list`)},g=()=>{u||(u=!0,s())},_=()=>{d||t!==`auto`||(d=!0,(async()=>{try{await a()}catch(e){n.error(`Initial index failed`,p(e))}})())},v=t=>t.length===0||(f&&=(clearTimeout(f),void 0),!q({config:e,log:n,reconfigureForWorkspace:i,roots:t}))?!1:(g(),_(),!0);try{if(v((await m.listRoots()).roots))return;n.debug(`No MCP roots yet; waiting for roots/list_changed notification`)}catch(e){if(h(e)){n.warn(`MCP roots/list not supported by client; using cwd fallback`,{cwd:l(),...p(e)}),g(),_();return}n.warn(`MCP roots/list failed during bootstrap; waiting for roots/list_changed notification`,{cwd:l(),...p(e)})}m.setNotificationHandler(r,async()=>{try{v((await m.listRoots()).roots)}catch(e){n.warn(`roots/list retry failed after notification`,p(e))}}),f=setTimeout(()=>{let t=l();n.debug(`Timed out waiting for MCP roots/list_changed; falling back to cwd workspace`,{cwd:t}),q({config:e,log:n,reconfigureForWorkspace:i,roots:[{uri:t}]})&&(g(),_())},c)}function Y(){return process.env.AIKIT_TRANSPORT?process.env.AIKIT_TRANSPORT:process.stdin.isTTY?`http`:`stdio`}function X(){let e=process.argv[1];if(!e)return!1;try{return import.meta.url===l(e).href}catch{return!1}}function Z(e){let t=e.headers[`mcp-session-id`];return Array.isArray(t)?t[0]:t}function Q(e,t){let n=process.env[e];if(!n)return t;let r=Number.parseInt(n,10);return Number.isFinite(r)&&r>0?r:t}function $(){return X()?u({allowPositionals:!0,options:{transport:{type:`string`,default:Y()},port:{type:`string`,default:process.env.AIKIT_PORT??`3210`}}}).values:{transport:Y(),port:process.env.AIKIT_PORT??`3210`}}async function ee(){let e=R(),t=$();process.on(`unhandledRejection`,e=>{z.error(`Unhandled rejection`,p(e))}),process.on(`uncaughtException`,e=>{z.error(`Uncaught exception — exiting`,p(e)),process.exit(1)});let r=(e,t)=>{e.then(async()=>{try{let{markPromoteRun:e,markPruneRun:n,prune:r,shouldRunStartupPrune:i,shouldRunWeeklyPromote:a}=await import(`../../tools/dist/index.js`),o=t();if(!o)return;if(i()){let e=await r({});n(),e.totalBytesFreed>0&&z.info(`Storage maintenance complete`,{forgeOrphans:e.forgeGroundOrphans.count,legacyLance:e.legacyLance.count,bytesFreed:e.totalBytesFreed});let{groupLessons:t,pruneLessons:i}=await import(`./evolution-DWaEE6XW.js`).then(e=>e.t),a=await i(o.curated,o.stateStore,{dryRun:!1});a.pruned.length>0&&z.info(`Startup lesson prune complete`,{pruned:a.pruned.length});let s=await t(o.curated,o.stateStore,{dryRun:!1});(s.groupsCreated>0||s.lessonsGrouped>0)&&z.info(`Startup lesson grouping complete`,{groupsCreated:s.groupsCreated,lessonsGrouped:s.lessonsGrouped})}if(a()){let{DEFAULT_PROMOTE_CONFIG:t,collectWorkspaceLessons:n,getGlobalCuratedDir:r,promoteLessons:i,scanForDuplicates:a}=await import(`./promotion-CJFYv4Ye.js`).then(e=>e.o),{CuratedKnowledgeManager:s}=await Promise.resolve().then(()=>N),c=o.curated,l=new s(r(),c.store,c.embedder),u=await n(),d={...t,dryRun:!1},f=await i(a(u,d),l,d);e(),f.promoted.length>0&&z.info(`Weekly lesson promotion complete`,{promoted:f.promoted.length,candidates:f.candidates.length})}}catch(e){z.warn(`Startup maintenance failed (non-critical)`,p(e))}}).catch(()=>{})};if(z.info(`Starting MCP AI Kit server`,{version:e}),t.transport===`http`){let[{default:e},{loadConfig:i,resolveIndexMode:a},{registerDashboardRoutes:o,resolveDashboardDir:s},{registerSettingsRoutes:c,resolveSettingsDir:l},{createSettingsRouter:u},{authMiddleware:d,getOrCreateToken:f}]=await Promise.all([import(`express`),import(`./config-CCrLxwqC.js`),import(`./dashboard-static-CRfR1yKU.js`),import(`./settings-static-B3lnYvcb.js`),import(`./routes-Afg7J7xK.js`),import(`./auth-lzZKfxlV.js`)]),m=i();z.info(`Config loaded`,{sourceCount:m.sources.length,storePath:m.store.path});let h=e();h.use(e.json({limit:`1mb`}));let g=Number(t.port),_=`http://localhost:${g}`,v=process.env.AIKIT_CORS_ORIGIN??_,y=process.env.AIKIT_ALLOW_ANY_ORIGIN===`true`,b=Q(`AIKIT_HTTP_MAX_SESSIONS`,8),x=Q(`AIKIT_HTTP_SESSION_TIMEOUT_MINUTES`,30),S=Q(`AIKIT_HTTP_SESSION_GC_INTERVAL_MINUTES`,5),C=H({limit:100,windowMs:6e4}),w=!1;h.use((e,t,n)=>{let r=Array.isArray(e.headers.origin)?e.headers.origin[0]:e.headers.origin,i=V({requestOrigin:r,configuredOrigin:v,allowAnyOrigin:y,fallbackOrigin:_});if(i.warn&&!w&&(w=!0,z.warn(`Rejected non-local CORS origin while AIKIT_CORS_ORIGIN=*`,{origin:r,allowAnyOriginEnv:`AIKIT_ALLOW_ANY_ORIGIN=true`})),r&&!i.allowOrigin){t.status(403).json({error:`Origin not allowed`});return}if(i.allowOrigin&&t.setHeader(`Access-Control-Allow-Origin`,i.allowOrigin),t.setHeader(`Access-Control-Allow-Methods`,`GET, POST, PUT, PATCH, DELETE, OPTIONS`),t.setHeader(`Access-Control-Allow-Headers`,`Content-Type, Authorization, Mcp-Session-Id, Mcp-Protocol-Version, Last-Event-ID`),t.setHeader(`Access-Control-Expose-Headers`,`Mcp-Session-Id`),e.method===`OPTIONS`){t.status(204).end();return}n()});let T=f();console.error(`[aikit] Auth token: ~/.aikit/token`),h.use(d(T)),h.use(`/mcp`,(e,t,n)=>{let r=Z(e)??e.ip??e.socket.remoteAddress??`anonymous`;if(C.allow(r)){n();return}let i=Math.max(1,Math.ceil(C.getRetryAfterMs(r)/1e3));t.setHeader(`Retry-After`,String(i)),t.status(429).json({jsonrpc:`2.0`,error:{code:-32003,message:`Rate limit exceeded`},id:null})}),o(h,s(),z);let E=new Date().toISOString();h.use(`/settings/api`,u({log:z,mcpInfo:()=>({transport:`http`,port:g,pid:process.pid,startedAt:E})})),c(h,l(),z),h.get(`/health`,(e,t)=>{t.json({status:`ok`})});let D=!1,O=null,A=null,M=null,N=null,P=null,F=null,I=null,L=Promise.resolve(),R=async(e,t)=>{if(!D||!M||!N){t.status(503).json({jsonrpc:`2.0`,error:{code:-32603,message:`Server initializing — please retry in a few seconds`},id:null});return}let r=L,i;L=new Promise(e=>{i=e}),await r;try{let r=Z(e);if(!F){if(r){t.status(404).json({jsonrpc:`2.0`,error:{code:-32001,message:`Session not found`},id:null});return}let e=new N({sessionIdGenerator:()=>n(),onsessioninitialized:async e=>{I=e,A?.onSessionStart(e,{transport:`http`})},onsessionclosed:async e=>{e&&A?.onSessionEnd(e),I=null}});e.onclose=()=>{F===e&&(F=null),I===e.sessionId&&(I=null)},F=e,await M.connect(e)}let i=F;await i.handleRequest(e,t,e.body),e.method!==`DELETE`&&(!r&&i.sessionId?(I=i.sessionId,A?.onSessionStart(i.sessionId,{transport:`http`}),A?.onSessionActivity(i.sessionId)):r&&A?.onSessionActivity(r))}catch(e){if(z.error(`MCP handler error`,p(e)),!t.headersSent){let n=e instanceof Error?e.message:String(e),r=n.includes(`Not Acceptable`);t.status(r?406:500).json({jsonrpc:`2.0`,error:{code:r?-32e3:-32603,message:r?n:`Internal server error`},id:null})}}finally{i()}},B=async(e,t)=>{let n=Z(e);if(P&&(!F||n!==I)){await P.handleRequest(e,t,e.body);return}await R(e,t)};h.post(`/mcp`,B),h.get(`/mcp`,B),h.delete(`/mcp`,B);let U=h.listen(g,`127.0.0.1`,()=>{z.info(`MCP server listening`,{url:`http://127.0.0.1:${g}/mcp`,port:g}),setTimeout(async()=>{try{let[{createLazyServer:e,createMcpServer:t,ALL_TOOL_NAMES:n},{StreamableHTTPServerTransport:i},{checkForUpdates:o,autoUpgradeScaffold:s}]=await Promise.all([import(`./server-CWkmhYOe.js`),import(`@modelcontextprotocol/sdk/server/streamableHttp.js`),import(`./version-check-gazMo-D4.js`)]);o(),s();let c=a(m),l=e(m,c);M=l.server,N=i,D=!0,z.debug(`MCP server configured (lazy — AI Kit initializing in background)`,{toolCount:n.length,resourceCount:2}),l.startInit(),l.ready.then(()=>{if(!l.aikit)throw Error(`AI Kit components are not available after initialization`);A=new j(l.aikit.stateStore,{staleTimeoutMinutes:x,gcIntervalMinutes:S,onBeforeSessionDelete:e=>{if(I===e&&F){let e=F;F=null,I=null,e.close().catch(()=>void 0)}P?.closeSession(e,{notifySessionEnd:!1})},onSessionEndMaintenance:async()=>{if(!l.aikit?.curated||!l.aikit?.stateStore)return;let{pruneLessons:e}=await import(`./evolution-DWaEE6XW.js`).then(e=>e.t),t=await e(l.aikit.curated,l.aikit.stateStore,{dryRun:!1});t.pruned.length>0&&z.info(`Session-end lesson prune`,{pruned:t.pruned.length})}}),P=new k({createServer:()=>{if(!l.aikit)throw Error(`AI Kit components are not available after initialization`);return t(l.aikit,m)},createTransport:e=>new i(e),maxSessions:b,sessionTimeoutMinutes:x,onSessionStart:e=>A?.onSessionStart(e,{transport:`http`}),onSessionActivity:e=>A?.onSessionActivity(e),onSessionEnd:e=>A?.onSessionEnd(e)}),A.startGC(),I&&(A.onSessionStart(I,{transport:`http`}),A.onSessionActivity(I)),z.info(`HTTP session runtime ready`,{maxSessions:b,sessionTimeoutMinutes:x,gcIntervalMinutes:S})}).catch(e=>z.error(`Failed to start session manager`,p(e))),c===`auto`?l.ready.then(async()=>{try{let e=m.sources.map(e=>e.path).join(`, `);z.info(`Running initial index`,{sourcePaths:e}),await l.runInitialIndex(),z.info(`Initial index complete`)}catch(e){z.error(`Initial index failed; will retry on aikit_reindex`,p(e))}}).catch(e=>z.error(`AI Kit init or indexing failed`,p(e))):c===`smart`?l.ready.then(async()=>{try{if(!l.aikit)throw Error(`AI Kit components are not available after initialization`);let{SmartIndexScheduler:e}=await import(`../../indexer/dist/index.js`),t=new e(l.aikit.indexer,m,l.aikit.store),n=l.aikit.store;O=t,t.start(),n.onBeforeClose&&n.onBeforeClose(()=>t.stop()),l.setSmartScheduler(t),z.debug(`Smart index scheduler started (HTTP mode)`)}catch(e){z.error(`Failed to start smart index scheduler`,p(e))}}).catch(e=>z.error(`AI Kit initialization failed`,p(e))):(l.ready.catch(e=>z.error(`AI Kit initialization failed`,p(e))),z.info(`Initial full indexing skipped in HTTP mode`,{indexMode:c})),r(l.ready,()=>l.aikit?{curated:l.aikit.curated,stateStore:l.aikit.stateStore}:null)}catch(e){z.error(`Failed to load server modules`,p(e))}},100)}),W=async e=>{z.info(`Shutdown signal received`,{signal:e}),O?.stop(),A?.stop(),await P?.closeAll().catch(()=>void 0),I&&A?.onSessionEnd(I),F&&(await F.close().catch(()=>void 0),F=null,I=null),U.close(),M&&await M.close(),process.exit(0)};process.on(`SIGINT`,()=>W(`SIGINT`)),process.on(`SIGTERM`,()=>W(`SIGTERM`))}else{let[{loadConfig:e,reconfigureForWorkspace:t,resolveIndexMode:n},{createLazyServer:i},{checkForUpdates:a,autoUpgradeScaffold:o},{RootsListChangedNotificationSchema:s}]=await Promise.all([import(`./config-CCrLxwqC.js`),import(`./server-CWkmhYOe.js`),import(`./version-check-gazMo-D4.js`),import(`@modelcontextprotocol/sdk/types.js`)]),c=e();z.info(`Config loaded`,{sourceCount:c.sources.length,storePath:c.store.path}),a(),o();let l=n(c),u=i(c,l),{server:d,startInit:f,ready:m,runInitialIndex:h}=u,{StdioServerTransport:g}=await import(`@modelcontextprotocol/sdk/server/stdio.js`),_=new g;await d.connect(_),z.debug(`MCP server started`,{transport:`stdio`}),await J({config:c,indexMode:l,log:z,rootsChangedNotificationSchema:s,reconfigureForWorkspace:t,runInitialIndex:h,server:d,startInit:f});let v=null,y=()=>{v&&clearTimeout(v),v=setTimeout(async()=>{z.info(`Auto-shutdown: no activity for 30 minutes — shutting down gracefully`);try{let e=u.aikit;e&&await Promise.all([e.embedder.shutdown?.().catch(()=>{})??Promise.resolve(),e.graphStore.close().catch(()=>{}),e.store.close().catch(()=>{})])}catch{}process.exit(0)},18e5),v.unref&&v.unref()};y(),process.stdin.on(`data`,()=>y()),m.catch(e=>{z.error(`Initialization failed — server will continue with limited tools`,p(e))}),l===`smart`?m.then(async()=>{try{if(!u.aikit)throw Error(`AI Kit components are not available after initialization`);let{SmartIndexScheduler:e}=await import(`../../indexer/dist/index.js`),t=new e(u.aikit.indexer,c,u.aikit.store),n=u.aikit.store;t.start(),n.onBeforeClose&&n.onBeforeClose(()=>t.stop()),u.setSmartScheduler(t),z.debug(`Smart index scheduler started (stdio mode)`)}catch(e){z.error(`Failed to start smart index scheduler`,p(e))}}).catch(e=>z.error(`AI Kit init failed for smart scheduler`,p(e))):z.warn(`Initial full indexing skipped; use aikit_reindex to index manually`,{indexMode:l}),r(m,()=>u.aikit?{curated:u.aikit.curated,stateStore:u.aikit.stateStore}:null)}}ee();export{M as n,L as t};
|
|
8
|
+
`),s=!1,c={};for(let e of o){if(/^changelog:\s*$/.test(e)){s=!0;continue}if(s){let t=e.match(/^\s+-\s+version:\s*(\d+)$/);if(t){c.version!=null&&a.push(c),c={version:parseInt(t[1],10)};continue}let n=e.match(/^\s+date:\s*(.+)$/);if(n){c.date=n[1].trim();continue}let r=e.match(/^\s+reason:\s*"?(.*?)"?\s*$/);if(r){c.reason=r[1];continue}/^\w/.test(e)&&(s=!1,c.version!=null&&a.push(c),c={});continue}let t=e.match(/^(\w+):\s*(.*)$/);if(t){let e=t[1],n=t[2];typeof n==`string`&&n.startsWith(`[`)&&n.endsWith(`]`)?n=n.slice(1,-1).split(`,`).map(e=>e.trim().replace(/^"|"$/g,``)).filter(e=>e.length>0):typeof n==`string`&&/^\d+$/.test(n)?n=parseInt(n,10):typeof n==`string`&&n.startsWith(`"`)&&n.endsWith(`"`)&&(n=n.slice(1,-1)),i[e]=n}}return c.version!=null&&a.push(c),{frontmatter:{title:i.title??`Untitled`,category:i.category??`notes`,tags:i.tags??[],created:i.created??``,updated:i.updated??``,version:i.version??1,origin:`curated`,changelog:a},content:r}}};function R(){try{let e=s(i(c(import.meta.url)),`..`,`..`,`..`,`package.json`);return JSON.parse(r(e,`utf-8`)).version??`0.0.0`}catch{return`0.0.0`}}const z=f(`server`),B=/^https?:\/\/(localhost|127\.0\.0\.1)(:\d+)?$/i;function V({requestOrigin:e,configuredOrigin:t,allowAnyOrigin:n,fallbackOrigin:r}){let i=t??r;return i===`*`?n?{allowOrigin:`*`,warn:!1}:e?B.test(e)?{allowOrigin:e,warn:!1}:{allowOrigin:null,warn:!0}:{allowOrigin:r,warn:!1}:{allowOrigin:i,warn:!1}}function H({limit:e,windowMs:t}){let n=new Map,r=(e,r)=>{let i=r-t,a=n.get(e)?.filter(e=>e>i)??[];return a.length===0?(n.delete(e),[]):(n.set(e,a),a)};return{allow(t,i=Date.now()){let a=r(t,i);return a.length>=e?!1:(a.push(i),n.set(t,a),!0)},getRetryAfterMs(n,i=Date.now()){let a=r(n,i);return a.length<e?0:Math.max(0,a[0]+t-i)}}}function U(e){return e.startsWith(`file://`)?c(e):e}function W(e){let t=s(e);return process.platform===`win32`?t.toLowerCase():t}function G(e){let t=new Map;for(let n of e){let e=U(n.uri);t.set(W(e),e)}return[...t.values()].sort((e,t)=>W(e).localeCompare(W(t)))}function K({config:e,roots:t}){let n=G(t);if(n.length===0)return null;let r=e.sources?.[0]?.path;if(r){let e=W(r),t=n.find(t=>W(t)===e);if(t)return t}return n[0]}function q({config:e,log:t,reconfigureForWorkspace:n,roots:r}){let i=G(r);if(i.length===0)return!1;let a=K({config:e,roots:i.map(e=>({uri:e}))});if(!a)return!1;let o=e.sources?.[0]?.path,s=o&&W(o)===W(a)?`configured-source`:`stable-sorted-root`;return t.debug(`MCP roots resolved`,{rootPath:a,rootCount:i.length,selectedBy:s}),n(e,a),e.allRoots=i,!0}async function J({config:e,indexMode:t,log:n,rootsChangedNotificationSchema:r,reconfigureForWorkspace:i,runInitialIndex:a,server:o,startInit:s,timeoutMs:c=5e3,getCwd:l=()=>process.cwd()}){let u=!1,d=!1,f,m=o.server,h=e=>{let t=p(e),n=`${String(e)} ${String(t.message??``)}`.toLowerCase();return n.includes(`method not found`)||n.includes(`not supported`)||n.includes(`unsupported`)||n.includes(`unknown method`)||n.includes(`roots/list`)},g=()=>{u||(u=!0,s())},_=()=>{d||t!==`auto`||(d=!0,(async()=>{try{await a()}catch(e){n.error(`Initial index failed`,p(e))}})())},v=t=>t.length===0||(f&&=(clearTimeout(f),void 0),!q({config:e,log:n,reconfigureForWorkspace:i,roots:t}))?!1:(g(),_(),!0);try{if(v((await m.listRoots()).roots))return;n.debug(`No MCP roots yet; waiting for roots/list_changed notification`)}catch(e){if(h(e)){n.warn(`MCP roots/list not supported by client; using cwd fallback`,{cwd:l(),...p(e)}),g(),_();return}n.warn(`MCP roots/list failed during bootstrap; waiting for roots/list_changed notification`,{cwd:l(),...p(e)})}m.setNotificationHandler(r,async()=>{try{v((await m.listRoots()).roots)}catch(e){n.warn(`roots/list retry failed after notification`,p(e))}}),f=setTimeout(()=>{let t=l();n.debug(`Timed out waiting for MCP roots/list_changed; falling back to cwd workspace`,{cwd:t}),q({config:e,log:n,reconfigureForWorkspace:i,roots:[{uri:t}]})&&(g(),_())},c)}function Y(){return process.env.AIKIT_TRANSPORT?process.env.AIKIT_TRANSPORT:process.stdin.isTTY?`http`:`stdio`}function X(){let e=process.argv[1];if(!e)return!1;try{return import.meta.url===l(e).href}catch{return!1}}function Z(e){let t=e.headers[`mcp-session-id`];return Array.isArray(t)?t[0]:t}function Q(e,t){let n=process.env[e];if(!n)return t;let r=Number.parseInt(n,10);return Number.isFinite(r)&&r>0?r:t}function $(){return X()?u({allowPositionals:!0,options:{transport:{type:`string`,default:Y()},port:{type:`string`,default:process.env.AIKIT_PORT??`3210`}}}).values:{transport:Y(),port:process.env.AIKIT_PORT??`3210`}}async function ee(){let e=R(),t=$();process.on(`unhandledRejection`,e=>{z.error(`Unhandled rejection`,p(e))}),process.on(`uncaughtException`,e=>{z.error(`Uncaught exception — exiting`,p(e)),process.exit(1)});let r=(e,t)=>{e.then(async()=>{try{let{markPromoteRun:e,markPruneRun:n,prune:r,shouldRunStartupPrune:i,shouldRunWeeklyPromote:a}=await import(`../../tools/dist/index.js`),o=t();if(!o)return;if(i()){let e=await r({});n(),e.totalBytesFreed>0&&z.info(`Storage maintenance complete`,{forgeOrphans:e.forgeGroundOrphans.count,legacyLance:e.legacyLance.count,bytesFreed:e.totalBytesFreed});let{groupLessons:t,pruneLessons:i}=await import(`./evolution-DWaEE6XW.js`).then(e=>e.t),a=await i(o.curated,o.stateStore,{dryRun:!1});a.pruned.length>0&&z.info(`Startup lesson prune complete`,{pruned:a.pruned.length});let s=await t(o.curated,o.stateStore,{dryRun:!1});(s.groupsCreated>0||s.lessonsGrouped>0)&&z.info(`Startup lesson grouping complete`,{groupsCreated:s.groupsCreated,lessonsGrouped:s.lessonsGrouped})}if(a()){let{DEFAULT_PROMOTE_CONFIG:t,collectWorkspaceLessons:n,getGlobalCuratedDir:r,promoteLessons:i,scanForDuplicates:a}=await import(`./promotion-CJFYv4Ye.js`).then(e=>e.o),{CuratedKnowledgeManager:s}=await Promise.resolve().then(()=>N),c=o.curated,l=new s(r(),c.store,c.embedder),u=await n(),d={...t,dryRun:!1},f=await i(a(u,d),l,d);e(),f.promoted.length>0&&z.info(`Weekly lesson promotion complete`,{promoted:f.promoted.length,candidates:f.candidates.length})}}catch(e){z.warn(`Startup maintenance failed (non-critical)`,p(e))}}).catch(()=>{})};if(z.info(`Starting MCP AI Kit server`,{version:e}),t.transport===`http`){let[{default:e},{loadConfig:i,resolveIndexMode:a},{registerDashboardRoutes:o,resolveDashboardDir:s},{registerSettingsRoutes:c,resolveSettingsDir:l},{createSettingsRouter:u},{authMiddleware:d,getOrCreateToken:f}]=await Promise.all([import(`express`),import(`./config-CCrLxwqC.js`),import(`./dashboard-static-CRfR1yKU.js`),import(`./settings-static-B3lnYvcb.js`),import(`./routes-Afg7J7xK.js`),import(`./auth-lzZKfxlV.js`)]),m=i();z.info(`Config loaded`,{sourceCount:m.sources.length,storePath:m.store.path});let h=e();h.use(e.json({limit:`1mb`}));let g=Number(t.port),_=`http://localhost:${g}`,v=process.env.AIKIT_CORS_ORIGIN??_,y=process.env.AIKIT_ALLOW_ANY_ORIGIN===`true`,b=Q(`AIKIT_HTTP_MAX_SESSIONS`,8),x=Q(`AIKIT_HTTP_SESSION_TIMEOUT_MINUTES`,30),S=Q(`AIKIT_HTTP_SESSION_GC_INTERVAL_MINUTES`,5),C=H({limit:100,windowMs:6e4}),w=!1;h.use((e,t,n)=>{let r=Array.isArray(e.headers.origin)?e.headers.origin[0]:e.headers.origin,i=V({requestOrigin:r,configuredOrigin:v,allowAnyOrigin:y,fallbackOrigin:_});if(i.warn&&!w&&(w=!0,z.warn(`Rejected non-local CORS origin while AIKIT_CORS_ORIGIN=*`,{origin:r,allowAnyOriginEnv:`AIKIT_ALLOW_ANY_ORIGIN=true`})),r&&!i.allowOrigin){t.status(403).json({error:`Origin not allowed`});return}if(i.allowOrigin&&t.setHeader(`Access-Control-Allow-Origin`,i.allowOrigin),t.setHeader(`Access-Control-Allow-Methods`,`GET, POST, PUT, PATCH, DELETE, OPTIONS`),t.setHeader(`Access-Control-Allow-Headers`,`Content-Type, Authorization, Mcp-Session-Id, Mcp-Protocol-Version, Last-Event-ID`),t.setHeader(`Access-Control-Expose-Headers`,`Mcp-Session-Id`),e.method===`OPTIONS`){t.status(204).end();return}n()});let T=f();console.error(`[aikit] Auth token: ~/.aikit/token`),h.use(d(T)),h.use(`/mcp`,(e,t,n)=>{let r=Z(e)??e.ip??e.socket.remoteAddress??`anonymous`;if(C.allow(r)){n();return}let i=Math.max(1,Math.ceil(C.getRetryAfterMs(r)/1e3));t.setHeader(`Retry-After`,String(i)),t.status(429).json({jsonrpc:`2.0`,error:{code:-32003,message:`Rate limit exceeded`},id:null})}),o(h,s(),z);let E=new Date().toISOString();h.use(`/settings/api`,u({log:z,mcpInfo:()=>({transport:`http`,port:g,pid:process.pid,startedAt:E})})),c(h,l(),z),h.get(`/health`,(e,t)=>{t.json({status:`ok`})});let D=!1,O=null,A=null,M=null,N=null,P=null,F=null,I=null,L=Promise.resolve(),R=async(e,t)=>{if(!D||!M||!N){t.status(503).json({jsonrpc:`2.0`,error:{code:-32603,message:`Server initializing — please retry in a few seconds`},id:null});return}let r=L,i;L=new Promise(e=>{i=e}),await r;try{let r=Z(e);if(!F){if(r){t.status(404).json({jsonrpc:`2.0`,error:{code:-32001,message:`Session not found`},id:null});return}let e=new N({sessionIdGenerator:()=>n(),onsessioninitialized:async e=>{I=e,A?.onSessionStart(e,{transport:`http`})},onsessionclosed:async e=>{e&&A?.onSessionEnd(e),I=null}});e.onclose=()=>{F===e&&(F=null),I===e.sessionId&&(I=null)},F=e,await M.connect(e)}let i=F;await i.handleRequest(e,t,e.body),e.method!==`DELETE`&&(!r&&i.sessionId?(I=i.sessionId,A?.onSessionStart(i.sessionId,{transport:`http`}),A?.onSessionActivity(i.sessionId)):r&&A?.onSessionActivity(r))}catch(e){if(z.error(`MCP handler error`,p(e)),!t.headersSent){let n=e instanceof Error?e.message:String(e),r=n.includes(`Not Acceptable`);t.status(r?406:500).json({jsonrpc:`2.0`,error:{code:r?-32e3:-32603,message:r?n:`Internal server error`},id:null})}}finally{i()}},B=async(e,t)=>{let n=Z(e);if(P&&(!F||n!==I)){await P.handleRequest(e,t,e.body);return}await R(e,t)};h.post(`/mcp`,B),h.get(`/mcp`,B),h.delete(`/mcp`,B);let U=h.listen(g,`127.0.0.1`,()=>{z.info(`MCP server listening`,{url:`http://127.0.0.1:${g}/mcp`,port:g}),setTimeout(async()=>{try{let[{createLazyServer:e,createMcpServer:t,ALL_TOOL_NAMES:n},{StreamableHTTPServerTransport:i},{checkForUpdates:o,autoUpgradeScaffold:s}]=await Promise.all([import(`./server-CBs_q9S9.js`),import(`@modelcontextprotocol/sdk/server/streamableHttp.js`),import(`./version-check-gazMo-D4.js`)]);o(),s();let c=a(m),l=e(m,c);M=l.server,N=i,D=!0,z.debug(`MCP server configured (lazy — AI Kit initializing in background)`,{toolCount:n.length,resourceCount:2}),l.startInit(),l.ready.then(()=>{if(!l.aikit)throw Error(`AI Kit components are not available after initialization`);A=new j(l.aikit.stateStore,{staleTimeoutMinutes:x,gcIntervalMinutes:S,onBeforeSessionDelete:e=>{if(I===e&&F){let e=F;F=null,I=null,e.close().catch(()=>void 0)}P?.closeSession(e,{notifySessionEnd:!1})},onSessionEndMaintenance:async()=>{if(!l.aikit?.curated||!l.aikit?.stateStore)return;let{pruneLessons:e}=await import(`./evolution-DWaEE6XW.js`).then(e=>e.t),t=await e(l.aikit.curated,l.aikit.stateStore,{dryRun:!1});t.pruned.length>0&&z.info(`Session-end lesson prune`,{pruned:t.pruned.length})}}),P=new k({createServer:()=>{if(!l.aikit)throw Error(`AI Kit components are not available after initialization`);return t(l.aikit,m)},createTransport:e=>new i(e),maxSessions:b,sessionTimeoutMinutes:x,onSessionStart:e=>A?.onSessionStart(e,{transport:`http`}),onSessionActivity:e=>A?.onSessionActivity(e),onSessionEnd:e=>A?.onSessionEnd(e)}),A.startGC(),I&&(A.onSessionStart(I,{transport:`http`}),A.onSessionActivity(I)),z.info(`HTTP session runtime ready`,{maxSessions:b,sessionTimeoutMinutes:x,gcIntervalMinutes:S})}).catch(e=>z.error(`Failed to start session manager`,p(e))),c===`auto`?l.ready.then(async()=>{try{let e=m.sources.map(e=>e.path).join(`, `);z.info(`Running initial index`,{sourcePaths:e}),await l.runInitialIndex(),z.info(`Initial index complete`)}catch(e){z.error(`Initial index failed; will retry on aikit_reindex`,p(e))}}).catch(e=>z.error(`AI Kit init or indexing failed`,p(e))):c===`smart`?l.ready.then(async()=>{try{if(!l.aikit)throw Error(`AI Kit components are not available after initialization`);let{SmartIndexScheduler:e}=await import(`../../indexer/dist/index.js`),t=new e(l.aikit.indexer,m,l.aikit.store),n=l.aikit.store;O=t,t.start(),n.onBeforeClose&&n.onBeforeClose(()=>t.stop()),l.setSmartScheduler(t),z.debug(`Smart index scheduler started (HTTP mode)`)}catch(e){z.error(`Failed to start smart index scheduler`,p(e))}}).catch(e=>z.error(`AI Kit initialization failed`,p(e))):(l.ready.catch(e=>z.error(`AI Kit initialization failed`,p(e))),z.info(`Initial full indexing skipped in HTTP mode`,{indexMode:c})),r(l.ready,()=>l.aikit?{curated:l.aikit.curated,stateStore:l.aikit.stateStore}:null)}catch(e){z.error(`Failed to load server modules`,p(e))}},100)}),W=async e=>{z.info(`Shutdown signal received`,{signal:e}),O?.stop(),A?.stop(),await P?.closeAll().catch(()=>void 0),I&&A?.onSessionEnd(I),F&&(await F.close().catch(()=>void 0),F=null,I=null),U.close(),M&&await M.close(),process.exit(0)};process.on(`SIGINT`,()=>W(`SIGINT`)),process.on(`SIGTERM`,()=>W(`SIGTERM`))}else{let[{loadConfig:e,reconfigureForWorkspace:t,resolveIndexMode:n},{createLazyServer:i},{checkForUpdates:a,autoUpgradeScaffold:o},{RootsListChangedNotificationSchema:s}]=await Promise.all([import(`./config-CCrLxwqC.js`),import(`./server-CBs_q9S9.js`),import(`./version-check-gazMo-D4.js`),import(`@modelcontextprotocol/sdk/types.js`)]),c=e();z.info(`Config loaded`,{sourceCount:c.sources.length,storePath:c.store.path}),a(),o();let l=n(c),u=i(c,l),{server:d,startInit:f,ready:m,runInitialIndex:h}=u,{StdioServerTransport:g}=await import(`@modelcontextprotocol/sdk/server/stdio.js`),_=new g;await d.connect(_),z.debug(`MCP server started`,{transport:`stdio`}),await J({config:c,indexMode:l,log:z,rootsChangedNotificationSchema:s,reconfigureForWorkspace:t,runInitialIndex:h,server:d,startInit:f});let v=null,y=()=>{v&&clearTimeout(v),v=setTimeout(async()=>{z.info(`Auto-shutdown: no activity for 30 minutes — shutting down gracefully`);try{let e=u.aikit;e&&await Promise.all([e.embedder.shutdown?.().catch(()=>{})??Promise.resolve(),e.graphStore.close().catch(()=>{}),e.store.close().catch(()=>{})])}catch{}process.exit(0)},18e5),v.unref&&v.unref()};y(),process.stdin.on(`data`,()=>y()),m.catch(e=>{z.error(`Initialization failed — server will continue with limited tools`,p(e))}),l===`smart`?m.then(async()=>{try{if(!u.aikit)throw Error(`AI Kit components are not available after initialization`);let{SmartIndexScheduler:e}=await import(`../../indexer/dist/index.js`),t=new e(u.aikit.indexer,c,u.aikit.store),n=u.aikit.store;t.start(),n.onBeforeClose&&n.onBeforeClose(()=>t.stop()),u.setSmartScheduler(t),z.debug(`Smart index scheduler started (stdio mode)`)}catch(e){z.error(`Failed to start smart index scheduler`,p(e))}}).catch(e=>z.error(`AI Kit init failed for smart scheduler`,p(e))):z.warn(`Initial full indexing skipped; use aikit_reindex to index manually`,{indexMode:l}),r(m,()=>u.aikit?{curated:u.aikit.curated,stateStore:u.aikit.stateStore}:null)}}ee();export{M as n,L as t};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{t as e}from"./curated-manager-xTfPLFMR.js";import{randomUUID as t}from"node:crypto";import{readFileSync as n}from"node:fs";import{dirname as r,resolve as i}from"node:path";import{fileURLToPath as a,pathToFileURL as o}from"node:url";import{parseArgs as s}from"node:util";import{createLogger as c,serializeError as l}from"../../core/dist/index.js";const u=`__pending__:`;function d(e){return e.startsWith(u)}function f(e){let t=e.headers[`mcp-session-id`];return Array.isArray(t)?t[0]:t}function p(e,t,n,r){e.status(t).json({jsonrpc:`2.0`,error:{code:n,message:r},id:null})}var m=class{options;runtimes=new Map;maxSessions;sessionTimeoutMs;now;constructor(e){this.options=e,this.maxSessions=e.maxSessions??8,this.sessionTimeoutMs=(e.sessionTimeoutMinutes??30)*60*1e3,this.now=e.now??(()=>Date.now())}hasSession(e){return this.runtimes.has(e)}getSessionCount(){return this.runtimes.size}async handleRequest(e,t,n=e.body){let r=f(e),i=r?this.runtimes.get(r):void 0;if(r&&!i){p(t,404,-32001,`Session not found`);return}if(!i){if(e.method!==`POST`){p(t,400,-32e3,`Session required`);return}if(i=await this.createRuntime(t),!i)return}await this.withRuntimeLock(i,async()=>{await i.transport.handleRequest(e,t,n),i.lastAccessAt=this.now();let r=i.transport.sessionId??i.id;e.method!==`DELETE`&&!d(r)&&this.options.onSessionActivity?.(r),e.method===`DELETE`&&!d(r)&&await this.closeSession(r,{closeTransport:!1})})}async closeExpiredSessions(){let e=[...this.runtimes.values()].filter(e=>this.now()-e.lastAccessAt>=this.sessionTimeoutMs).map(e=>e.id);for(let t of e)await this.closeSession(t);return e.length}async closeSession(e,t={}){let n=this.runtimes.get(e);return n?(this.runtimes.delete(e),t.notifySessionEnd!==!1&&!d(e)&&this.options.onSessionEnd?.(e),t.closeTransport!==!1&&await n.transport.close().catch(()=>void 0),await n.server.close().catch(()=>void 0),!0):!1}async closeAll(){let e=[...this.runtimes.keys()];for(let t of e)await this.closeSession(t)}async createRuntime(e){if(await this.closeExpiredSessions(),this.runtimes.size>=this.maxSessions){p(e,503,-32003,`Session capacity reached`);return}let n=this.now(),r=await this.options.createServer(),i={id:`${u}${t()}`,transport:void 0,createdAt:n,lastAccessAt:n,server:r,requestChain:Promise.resolve()},a=this.options.createTransport({sessionIdGenerator:()=>t(),onsessioninitialized:async e=>{this.runtimes.delete(i.id),i.id=e,this.runtimes.set(e,i),this.options.onSessionStart?.(e)},onsessionclosed:async e=>{e&&await this.closeSession(e,{closeTransport:!1})}});return i.transport=a,a.onclose=()=>{let e=i.transport.sessionId??i.id;this.closeSession(e,{closeTransport:!1})},this.runtimes.set(i.id,i),await r.connect(a),i}async withRuntimeLock(e,t){let n=e.requestChain,r;e.requestChain=new Promise(e=>{r=e}),await n;try{await t()}finally{r()}}};function h(e){let t=e.includes(`T`)?e:`${e.replace(` `,`T`)}Z`;return Date.parse(t)}var g=class{stateStore;options;gcTimer=null;constructor(e,t={}){this.stateStore=e,this.options=t}onSessionStart(e,t){this.stateStore.sessionCreate(e,t)}onSessionActivity(e){this.stateStore.sessionTouch(e)}onSessionEnd(e){this.stateStore.sessionDelete(e),Promise.resolve(this.options.onSessionEndMaintenance?.(e)).catch(()=>{})}startGC(){if(this.gcTimer)return;let e=(this.options.gcIntervalMinutes??5)*60*1e3;this.gcTimer=setInterval(()=>{this.runGC()},e),this.gcTimer.unref()}runGC(){let e=this.getStaleSessionIds();for(let t of e)this.options.onBeforeSessionDelete?.(t),Promise.resolve(this.options.onSessionEndMaintenance?.(t)).catch(()=>{}),this.stateStore.sessionDelete(t);return e.length}stop(){this.gcTimer&&=(clearInterval(this.gcTimer),null)}getActiveSessions(){return this.stateStore.sessionList().length}listSessions(){return this.stateStore.sessionList()}getStaleSessionIds(e=Date.now()){let t=(this.options.staleTimeoutMinutes??30)*60*1e3;return this.stateStore.sessionList().filter(n=>{let r=h(n.lastActivity);return Number.isFinite(r)&&e-r>=t}).map(e=>e.sessionId)}};function _(){try{let e=i(r(a(import.meta.url)),`..`,`..`,`..`,`package.json`);return JSON.parse(n(e,`utf-8`)).version??`0.0.0`}catch{return`0.0.0`}}const v=c(`server`),y=/^https?:\/\/(localhost|127\.0\.0\.1)(:\d+)?$/i;function b({requestOrigin:e,configuredOrigin:t,allowAnyOrigin:n,fallbackOrigin:r}){let i=t??r;return i===`*`?n?{allowOrigin:`*`,warn:!1}:e?y.test(e)?{allowOrigin:e,warn:!1}:{allowOrigin:null,warn:!0}:{allowOrigin:r,warn:!1}:{allowOrigin:i,warn:!1}}function x({limit:e,windowMs:t}){let n=new Map,r=(e,r)=>{let i=r-t,a=n.get(e)?.filter(e=>e>i)??[];return a.length===0?(n.delete(e),[]):(n.set(e,a),a)};return{allow(t,i=Date.now()){let a=r(t,i);return a.length>=e?!1:(a.push(i),n.set(t,a),!0)},getRetryAfterMs(n,i=Date.now()){let a=r(n,i);return a.length<e?0:Math.max(0,a[0]+t-i)}}}function S(e){return e.startsWith(`file://`)?a(e):e}function C(e){let t=i(e);return process.platform===`win32`?t.toLowerCase():t}function w(e){let t=new Map;for(let n of e){let e=S(n.uri);t.set(C(e),e)}return[...t.values()].sort((e,t)=>C(e).localeCompare(C(t)))}function T({config:e,roots:t}){let n=w(t);if(n.length===0)return null;let r=e.sources?.[0]?.path;if(r){let e=C(r),t=n.find(t=>C(t)===e);if(t)return t}return n[0]}function E({config:e,log:t,reconfigureForWorkspace:n,roots:r}){let i=w(r);if(i.length===0)return!1;let a=T({config:e,roots:i.map(e=>({uri:e}))});if(!a)return!1;let o=e.sources?.[0]?.path,s=o&&C(o)===C(a)?`configured-source`:`stable-sorted-root`;return t.debug(`MCP roots resolved`,{rootPath:a,rootCount:i.length,selectedBy:s}),n(e,a),e.allRoots=i,!0}async function D({config:e,indexMode:t,log:n,rootsChangedNotificationSchema:r,reconfigureForWorkspace:i,runInitialIndex:a,server:o,startInit:s,timeoutMs:c=5e3,getCwd:u=()=>process.cwd()}){let d=!1,f=!1,p,m=o.server,h=e=>{let t=l(e),n=`${String(e)} ${String(t.message??``)}`.toLowerCase();return n.includes(`method not found`)||n.includes(`not supported`)||n.includes(`unsupported`)||n.includes(`unknown method`)||n.includes(`roots/list`)},g=()=>{d||(d=!0,s())},_=()=>{f||t!==`auto`||(f=!0,(async()=>{try{await a()}catch(e){n.error(`Initial index failed`,l(e))}})())},v=t=>t.length===0||(p&&=(clearTimeout(p),void 0),!E({config:e,log:n,reconfigureForWorkspace:i,roots:t}))?!1:(g(),_(),!0);try{if(v((await m.listRoots()).roots))return;n.debug(`No MCP roots yet; waiting for roots/list_changed notification`)}catch(e){if(h(e)){n.warn(`MCP roots/list not supported by client; using cwd fallback`,{cwd:u(),...l(e)}),g(),_();return}n.warn(`MCP roots/list failed during bootstrap; waiting for roots/list_changed notification`,{cwd:u(),...l(e)})}m.setNotificationHandler(r,async()=>{try{v((await m.listRoots()).roots)}catch(e){n.warn(`roots/list retry failed after notification`,l(e))}}),p=setTimeout(()=>{let t=u();n.debug(`Timed out waiting for MCP roots/list_changed; falling back to cwd workspace`,{cwd:t}),E({config:e,log:n,reconfigureForWorkspace:i,roots:[{uri:t}]})&&(g(),_())},c)}function O(){return process.env.AIKIT_TRANSPORT?process.env.AIKIT_TRANSPORT:process.stdin.isTTY?`http`:`stdio`}function k(){let e=process.argv[1];if(!e)return!1;try{return import.meta.url===o(e).href}catch{return!1}}function A(e){let t=e.headers[`mcp-session-id`];return Array.isArray(t)?t[0]:t}function j(e,t){let n=process.env[e];if(!n)return t;let r=Number.parseInt(n,10);return Number.isFinite(r)&&r>0?r:t}function M(){return k()?s({allowPositionals:!0,options:{transport:{type:`string`,default:O()},port:{type:`string`,default:process.env.AIKIT_PORT??`3210`}}}).values:{transport:O(),port:process.env.AIKIT_PORT??`3210`}}async function N(){let e=_(),n=M();process.on(`unhandledRejection`,e=>{v.error(`Unhandled rejection`,l(e))}),process.on(`uncaughtException`,e=>{v.error(`Uncaught exception — exiting`,l(e)),process.exit(1)});let r=(e,t)=>{e.then(async()=>{try{let{markPromoteRun:e,markPruneRun:n,prune:r,shouldRunStartupPrune:i,shouldRunWeeklyPromote:a}=await import(`../../tools/dist/index.js`),o=t();if(!o)return;if(i()){let e=await r({});n(),e.totalBytesFreed>0&&v.info(`Storage maintenance complete`,{forgeOrphans:e.forgeGroundOrphans.count,legacyLance:e.legacyLance.count,bytesFreed:e.totalBytesFreed});let{groupLessons:t,pruneLessons:i}=await import(`./evolution-BX_zTSdj.js`).then(e=>e.t),a=await i(o.curated,o.stateStore,{dryRun:!1});a.pruned.length>0&&v.info(`Startup lesson prune complete`,{pruned:a.pruned.length});let s=await t(o.curated,o.stateStore,{dryRun:!1});(s.groupsCreated>0||s.lessonsGrouped>0)&&v.info(`Startup lesson grouping complete`,{groupsCreated:s.groupsCreated,lessonsGrouped:s.lessonsGrouped})}if(a()){let{DEFAULT_PROMOTE_CONFIG:t,collectWorkspaceLessons:n,getGlobalCuratedDir:r,promoteLessons:i,scanForDuplicates:a}=await import(`./promotion-PdKQQktE.js`).then(e=>e.o),{CuratedKnowledgeManager:s}=await import(`./curated-manager-xTfPLFMR.js`).then(e=>e.n),c=o.curated,l=new s(r(),c.store,c.embedder),u=await n(),d={...t,dryRun:!1},f=await i(a(u,d),l,d);e(),f.promoted.length>0&&v.info(`Weekly lesson promotion complete`,{promoted:f.promoted.length,candidates:f.candidates.length})}}catch(e){v.warn(`Startup maintenance failed (non-critical)`,l(e))}}).catch(()=>{})};if(v.info(`Starting MCP AI Kit server`,{version:e}),n.transport===`http`){let[{default:e},{loadConfig:i,resolveIndexMode:a},{registerDashboardRoutes:o,resolveDashboardDir:s},{registerSettingsRoutes:c,resolveSettingsDir:u},{createSettingsRouter:d},{authMiddleware:f,getOrCreateToken:p}]=await Promise.all([import(`express`),import(`./config-D_MQ_9Q7.js`),import(`./dashboard-static-FmfoS46e.js`),import(`./settings-static-BtvyIrza.js`),import(`./routes-CR3fI-HJ.js`),import(`./auth-Bz5dmZgR.js`).then(e=>e.t)]),h=i();v.info(`Config loaded`,{sourceCount:h.sources.length,storePath:h.store.path});let _=e();_.use(e.json({limit:`1mb`}));let y=Number(n.port),S=`http://localhost:${y}`,C=process.env.AIKIT_CORS_ORIGIN??S,w=process.env.AIKIT_ALLOW_ANY_ORIGIN===`true`,T=j(`AIKIT_HTTP_MAX_SESSIONS`,8),E=j(`AIKIT_HTTP_SESSION_TIMEOUT_MINUTES`,30),D=j(`AIKIT_HTTP_SESSION_GC_INTERVAL_MINUTES`,5),O=x({limit:100,windowMs:6e4}),k=!1;_.use((e,t,n)=>{let r=Array.isArray(e.headers.origin)?e.headers.origin[0]:e.headers.origin,i=b({requestOrigin:r,configuredOrigin:C,allowAnyOrigin:w,fallbackOrigin:S});if(i.warn&&!k&&(k=!0,v.warn(`Rejected non-local CORS origin while AIKIT_CORS_ORIGIN=*`,{origin:r,allowAnyOriginEnv:`AIKIT_ALLOW_ANY_ORIGIN=true`})),r&&!i.allowOrigin){t.status(403).json({error:`Origin not allowed`});return}if(i.allowOrigin&&t.setHeader(`Access-Control-Allow-Origin`,i.allowOrigin),t.setHeader(`Access-Control-Allow-Methods`,`GET, POST, PUT, PATCH, DELETE, OPTIONS`),t.setHeader(`Access-Control-Allow-Headers`,`Content-Type, Authorization, Mcp-Session-Id, Mcp-Protocol-Version, Last-Event-ID`),t.setHeader(`Access-Control-Expose-Headers`,`Mcp-Session-Id`),e.method===`OPTIONS`){t.status(204).end();return}n()});let M=p();console.error(`[aikit] Auth token: ~/.aikit/token`),_.use(f(M)),_.use(`/mcp`,(e,t,n)=>{let r=A(e)??e.ip??e.socket.remoteAddress??`anonymous`;if(O.allow(r)){n();return}let i=Math.max(1,Math.ceil(O.getRetryAfterMs(r)/1e3));t.setHeader(`Retry-After`,String(i)),t.status(429).json({jsonrpc:`2.0`,error:{code:-32003,message:`Rate limit exceeded`},id:null})}),o(_,s(),v);let N=new Date().toISOString();_.use(`/settings/api`,d({log:v,mcpInfo:()=>({transport:`http`,port:y,pid:process.pid,startedAt:N})})),c(_,u(),v),_.get(`/health`,(e,t)=>{t.json({status:`ok`})});let P=!1,F=null,I=null,L=null,R=null,z=null,B=null,V=null,H=Promise.resolve(),U=async(e,n)=>{if(!P||!L||!R){n.status(503).json({jsonrpc:`2.0`,error:{code:-32603,message:`Server initializing — please retry in a few seconds`},id:null});return}let r=H,i;H=new Promise(e=>{i=e}),await r;try{let r=A(e);if(!B){if(r){n.status(404).json({jsonrpc:`2.0`,error:{code:-32001,message:`Session not found`},id:null});return}let e=new R({sessionIdGenerator:()=>t(),onsessioninitialized:async e=>{V=e,I?.onSessionStart(e,{transport:`http`})},onsessionclosed:async e=>{e&&I?.onSessionEnd(e),V=null}});e.onclose=()=>{B===e&&(B=null),V===e.sessionId&&(V=null)},B=e,await L.connect(e)}let i=B;await i.handleRequest(e,n,e.body),e.method!==`DELETE`&&(!r&&i.sessionId?(V=i.sessionId,I?.onSessionStart(i.sessionId,{transport:`http`}),I?.onSessionActivity(i.sessionId)):r&&I?.onSessionActivity(r))}catch(e){if(v.error(`MCP handler error`,l(e)),!n.headersSent){let t=e instanceof Error?e.message:String(e),r=t.includes(`Not Acceptable`);n.status(r?406:500).json({jsonrpc:`2.0`,error:{code:r?-32e3:-32603,message:r?t:`Internal server error`},id:null})}}finally{i()}},W=async(e,t)=>{let n=A(e);if(z&&(!B||n!==V)){await z.handleRequest(e,t,e.body);return}await U(e,t)};_.post(`/mcp`,W),_.get(`/mcp`,W),_.delete(`/mcp`,W);let G=_.listen(y,`127.0.0.1`,()=>{v.info(`MCP server listening`,{url:`http://127.0.0.1:${y}/mcp`,port:y}),setTimeout(async()=>{try{let[{createLazyServer:e,createMcpServer:t,ALL_TOOL_NAMES:n},{StreamableHTTPServerTransport:i},{checkForUpdates:o,autoUpgradeScaffold:s}]=await Promise.all([import(`./server-CoohgKA7.js`),import(`@modelcontextprotocol/sdk/server/streamableHttp.js`),import(`./version-check-BgHzxxCW.js`)]);o(),s();let c=a(h),u=e(h,c);L=u.server,R=i,P=!0,v.debug(`MCP server configured (lazy — AI Kit initializing in background)`,{toolCount:n.length,resourceCount:2}),u.startInit(),u.ready.then(()=>{if(!u.aikit)throw Error(`AI Kit components are not available after initialization`);I=new g(u.aikit.stateStore,{staleTimeoutMinutes:E,gcIntervalMinutes:D,onBeforeSessionDelete:e=>{if(V===e&&B){let e=B;B=null,V=null,e.close().catch(()=>void 0)}z?.closeSession(e,{notifySessionEnd:!1})},onSessionEndMaintenance:async()=>{if(!u.aikit?.curated||!u.aikit?.stateStore)return;let{pruneLessons:e}=await import(`./evolution-BX_zTSdj.js`).then(e=>e.t),t=await e(u.aikit.curated,u.aikit.stateStore,{dryRun:!1});t.pruned.length>0&&v.info(`Session-end lesson prune`,{pruned:t.pruned.length})}}),z=new m({createServer:()=>{if(!u.aikit)throw Error(`AI Kit components are not available after initialization`);return t(u.aikit,h)},createTransport:e=>new i(e),maxSessions:T,sessionTimeoutMinutes:E,onSessionStart:e=>I?.onSessionStart(e,{transport:`http`}),onSessionActivity:e=>I?.onSessionActivity(e),onSessionEnd:e=>I?.onSessionEnd(e)}),I.startGC(),V&&(I.onSessionStart(V,{transport:`http`}),I.onSessionActivity(V)),v.info(`HTTP session runtime ready`,{maxSessions:T,sessionTimeoutMinutes:E,gcIntervalMinutes:D})}).catch(e=>v.error(`Failed to start session manager`,l(e))),c===`auto`?u.ready.then(async()=>{try{let e=h.sources.map(e=>e.path).join(`, `);v.info(`Running initial index`,{sourcePaths:e}),await u.runInitialIndex(),v.info(`Initial index complete`)}catch(e){v.error(`Initial index failed; will retry on aikit_reindex`,l(e))}}).catch(e=>v.error(`AI Kit init or indexing failed`,l(e))):c===`smart`?u.ready.then(async()=>{try{if(!u.aikit)throw Error(`AI Kit components are not available after initialization`);let{SmartIndexScheduler:e}=await import(`../../indexer/dist/index.js`),t=new e(u.aikit.indexer,h,u.aikit.store),n=u.aikit.store;F=t,t.start(),n.onBeforeClose&&n.onBeforeClose(()=>t.stop()),u.setSmartScheduler(t),v.debug(`Smart index scheduler started (HTTP mode)`)}catch(e){v.error(`Failed to start smart index scheduler`,l(e))}}).catch(e=>v.error(`AI Kit initialization failed`,l(e))):(u.ready.catch(e=>v.error(`AI Kit initialization failed`,l(e))),v.info(`Initial full indexing skipped in HTTP mode`,{indexMode:c})),r(u.ready,()=>u.aikit?{curated:u.aikit.curated,stateStore:u.aikit.stateStore}:null)}catch(e){v.error(`Failed to load server modules`,l(e))}},100)}),K=async e=>{v.info(`Shutdown signal received`,{signal:e}),F?.stop(),I?.stop(),await z?.closeAll().catch(()=>void 0),V&&I?.onSessionEnd(V),B&&(await B.close().catch(()=>void 0),B=null,V=null),G.close(),L&&await L.close(),process.exit(0)};process.on(`SIGINT`,()=>K(`SIGINT`)),process.on(`SIGTERM`,()=>K(`SIGTERM`))}else{let[{loadConfig:e,reconfigureForWorkspace:t,resolveIndexMode:n},{createLazyServer:i},{checkForUpdates:a,autoUpgradeScaffold:o},{RootsListChangedNotificationSchema:s}]=await Promise.all([import(`./config-D_MQ_9Q7.js`),import(`./server-CoohgKA7.js`),import(`./version-check-BgHzxxCW.js`),import(`@modelcontextprotocol/sdk/types.js`)]),c=e();v.info(`Config loaded`,{sourceCount:c.sources.length,storePath:c.store.path}),a(),o();let u=n(c),d=i(c,u),{server:f,startInit:p,ready:m,runInitialIndex:h}=d,{StdioServerTransport:g}=await import(`@modelcontextprotocol/sdk/server/stdio.js`),_=new g;await f.connect(_),v.debug(`MCP server started`,{transport:`stdio`}),await D({config:c,indexMode:u,log:v,rootsChangedNotificationSchema:s,reconfigureForWorkspace:t,runInitialIndex:h,server:f,startInit:p});let y=null,b=()=>{y&&clearTimeout(y),y=setTimeout(async()=>{v.info(`Auto-shutdown: no activity for 30 minutes — shutting down gracefully`);try{let e=d.aikit;e&&await Promise.all([e.embedder.shutdown?.().catch(()=>{})??Promise.resolve(),e.graphStore.close().catch(()=>{}),e.store.close().catch(()=>{})])}catch{}process.exit(0)},18e5),y.unref&&y.unref()};b(),process.stdin.on(`data`,()=>b()),m.catch(e=>{v.error(`Initialization failed — server will continue with limited tools`,l(e))}),u===`smart`?m.then(async()=>{try{if(!d.aikit)throw Error(`AI Kit components are not available after initialization`);let{SmartIndexScheduler:e}=await import(`../../indexer/dist/index.js`),t=new e(d.aikit.indexer,c,d.aikit.store),n=d.aikit.store;t.start(),n.onBeforeClose&&n.onBeforeClose(()=>t.stop()),d.setSmartScheduler(t),v.debug(`Smart index scheduler started (stdio mode)`)}catch(e){v.error(`Failed to start smart index scheduler`,l(e))}}).catch(e=>v.error(`AI Kit init failed for smart scheduler`,l(e))):v.warn(`Initial full indexing skipped; use aikit_reindex to index manually`,{indexMode:u}),r(m,()=>d.aikit?{curated:d.aikit.curated,stateStore:d.aikit.stateStore}:null)}}export{e as CuratedKnowledgeManager,E as applyWorkspaceRoots,D as bootstrapWorkspaceRoots,x as createSlidingWindowRateLimiter,N as main,b as resolveCorsOrigin,T as selectWorkspaceRoot};
|
|
1
|
+
import{t as e}from"./curated-manager-xTfPLFMR.js";import{randomUUID as t}from"node:crypto";import{readFileSync as n}from"node:fs";import{dirname as r,resolve as i}from"node:path";import{fileURLToPath as a,pathToFileURL as o}from"node:url";import{parseArgs as s}from"node:util";import{createLogger as c,serializeError as l}from"../../core/dist/index.js";const u=`__pending__:`;function d(e){return e.startsWith(u)}function f(e){let t=e.headers[`mcp-session-id`];return Array.isArray(t)?t[0]:t}function p(e,t,n,r){e.status(t).json({jsonrpc:`2.0`,error:{code:n,message:r},id:null})}var m=class{options;runtimes=new Map;maxSessions;sessionTimeoutMs;now;constructor(e){this.options=e,this.maxSessions=e.maxSessions??8,this.sessionTimeoutMs=(e.sessionTimeoutMinutes??30)*60*1e3,this.now=e.now??(()=>Date.now())}hasSession(e){return this.runtimes.has(e)}getSessionCount(){return this.runtimes.size}async handleRequest(e,t,n=e.body){let r=f(e),i=r?this.runtimes.get(r):void 0;if(r&&!i){p(t,404,-32001,`Session not found`);return}if(!i){if(e.method!==`POST`){p(t,400,-32e3,`Session required`);return}if(i=await this.createRuntime(t),!i)return}await this.withRuntimeLock(i,async()=>{await i.transport.handleRequest(e,t,n),i.lastAccessAt=this.now();let r=i.transport.sessionId??i.id;e.method!==`DELETE`&&!d(r)&&this.options.onSessionActivity?.(r),e.method===`DELETE`&&!d(r)&&await this.closeSession(r,{closeTransport:!1})})}async closeExpiredSessions(){let e=[...this.runtimes.values()].filter(e=>this.now()-e.lastAccessAt>=this.sessionTimeoutMs).map(e=>e.id);for(let t of e)await this.closeSession(t);return e.length}async closeSession(e,t={}){let n=this.runtimes.get(e);return n?(this.runtimes.delete(e),t.notifySessionEnd!==!1&&!d(e)&&this.options.onSessionEnd?.(e),t.closeTransport!==!1&&await n.transport.close().catch(()=>void 0),await n.server.close().catch(()=>void 0),!0):!1}async closeAll(){let e=[...this.runtimes.keys()];for(let t of e)await this.closeSession(t)}async createRuntime(e){if(await this.closeExpiredSessions(),this.runtimes.size>=this.maxSessions){p(e,503,-32003,`Session capacity reached`);return}let n=this.now(),r=await this.options.createServer(),i={id:`${u}${t()}`,transport:void 0,createdAt:n,lastAccessAt:n,server:r,requestChain:Promise.resolve()},a=this.options.createTransport({sessionIdGenerator:()=>t(),onsessioninitialized:async e=>{this.runtimes.delete(i.id),i.id=e,this.runtimes.set(e,i),this.options.onSessionStart?.(e)},onsessionclosed:async e=>{e&&await this.closeSession(e,{closeTransport:!1})}});return i.transport=a,a.onclose=()=>{let e=i.transport.sessionId??i.id;this.closeSession(e,{closeTransport:!1})},this.runtimes.set(i.id,i),await r.connect(a),i}async withRuntimeLock(e,t){let n=e.requestChain,r;e.requestChain=new Promise(e=>{r=e}),await n;try{await t()}finally{r()}}};function h(e){let t=e.includes(`T`)?e:`${e.replace(` `,`T`)}Z`;return Date.parse(t)}var g=class{stateStore;options;gcTimer=null;constructor(e,t={}){this.stateStore=e,this.options=t}onSessionStart(e,t){this.stateStore.sessionCreate(e,t)}onSessionActivity(e){this.stateStore.sessionTouch(e)}onSessionEnd(e){this.stateStore.sessionDelete(e),Promise.resolve(this.options.onSessionEndMaintenance?.(e)).catch(()=>{})}startGC(){if(this.gcTimer)return;let e=(this.options.gcIntervalMinutes??5)*60*1e3;this.gcTimer=setInterval(()=>{this.runGC()},e),this.gcTimer.unref()}runGC(){let e=this.getStaleSessionIds();for(let t of e)this.options.onBeforeSessionDelete?.(t),Promise.resolve(this.options.onSessionEndMaintenance?.(t)).catch(()=>{}),this.stateStore.sessionDelete(t);return e.length}stop(){this.gcTimer&&=(clearInterval(this.gcTimer),null)}getActiveSessions(){return this.stateStore.sessionList().length}listSessions(){return this.stateStore.sessionList()}getStaleSessionIds(e=Date.now()){let t=(this.options.staleTimeoutMinutes??30)*60*1e3;return this.stateStore.sessionList().filter(n=>{let r=h(n.lastActivity);return Number.isFinite(r)&&e-r>=t}).map(e=>e.sessionId)}};function _(){try{let e=i(r(a(import.meta.url)),`..`,`..`,`..`,`package.json`);return JSON.parse(n(e,`utf-8`)).version??`0.0.0`}catch{return`0.0.0`}}const v=c(`server`),y=/^https?:\/\/(localhost|127\.0\.0\.1)(:\d+)?$/i;function b({requestOrigin:e,configuredOrigin:t,allowAnyOrigin:n,fallbackOrigin:r}){let i=t??r;return i===`*`?n?{allowOrigin:`*`,warn:!1}:e?y.test(e)?{allowOrigin:e,warn:!1}:{allowOrigin:null,warn:!0}:{allowOrigin:r,warn:!1}:{allowOrigin:i,warn:!1}}function x({limit:e,windowMs:t}){let n=new Map,r=(e,r)=>{let i=r-t,a=n.get(e)?.filter(e=>e>i)??[];return a.length===0?(n.delete(e),[]):(n.set(e,a),a)};return{allow(t,i=Date.now()){let a=r(t,i);return a.length>=e?!1:(a.push(i),n.set(t,a),!0)},getRetryAfterMs(n,i=Date.now()){let a=r(n,i);return a.length<e?0:Math.max(0,a[0]+t-i)}}}function S(e){return e.startsWith(`file://`)?a(e):e}function C(e){let t=i(e);return process.platform===`win32`?t.toLowerCase():t}function w(e){let t=new Map;for(let n of e){let e=S(n.uri);t.set(C(e),e)}return[...t.values()].sort((e,t)=>C(e).localeCompare(C(t)))}function T({config:e,roots:t}){let n=w(t);if(n.length===0)return null;let r=e.sources?.[0]?.path;if(r){let e=C(r),t=n.find(t=>C(t)===e);if(t)return t}return n[0]}function E({config:e,log:t,reconfigureForWorkspace:n,roots:r}){let i=w(r);if(i.length===0)return!1;let a=T({config:e,roots:i.map(e=>({uri:e}))});if(!a)return!1;let o=e.sources?.[0]?.path,s=o&&C(o)===C(a)?`configured-source`:`stable-sorted-root`;return t.debug(`MCP roots resolved`,{rootPath:a,rootCount:i.length,selectedBy:s}),n(e,a),e.allRoots=i,!0}async function D({config:e,indexMode:t,log:n,rootsChangedNotificationSchema:r,reconfigureForWorkspace:i,runInitialIndex:a,server:o,startInit:s,timeoutMs:c=5e3,getCwd:u=()=>process.cwd()}){let d=!1,f=!1,p,m=o.server,h=e=>{let t=l(e),n=`${String(e)} ${String(t.message??``)}`.toLowerCase();return n.includes(`method not found`)||n.includes(`not supported`)||n.includes(`unsupported`)||n.includes(`unknown method`)||n.includes(`roots/list`)},g=()=>{d||(d=!0,s())},_=()=>{f||t!==`auto`||(f=!0,(async()=>{try{await a()}catch(e){n.error(`Initial index failed`,l(e))}})())},v=t=>t.length===0||(p&&=(clearTimeout(p),void 0),!E({config:e,log:n,reconfigureForWorkspace:i,roots:t}))?!1:(g(),_(),!0);try{if(v((await m.listRoots()).roots))return;n.debug(`No MCP roots yet; waiting for roots/list_changed notification`)}catch(e){if(h(e)){n.warn(`MCP roots/list not supported by client; using cwd fallback`,{cwd:u(),...l(e)}),g(),_();return}n.warn(`MCP roots/list failed during bootstrap; waiting for roots/list_changed notification`,{cwd:u(),...l(e)})}m.setNotificationHandler(r,async()=>{try{v((await m.listRoots()).roots)}catch(e){n.warn(`roots/list retry failed after notification`,l(e))}}),p=setTimeout(()=>{let t=u();n.debug(`Timed out waiting for MCP roots/list_changed; falling back to cwd workspace`,{cwd:t}),E({config:e,log:n,reconfigureForWorkspace:i,roots:[{uri:t}]})&&(g(),_())},c)}function O(){return process.env.AIKIT_TRANSPORT?process.env.AIKIT_TRANSPORT:process.stdin.isTTY?`http`:`stdio`}function k(){let e=process.argv[1];if(!e)return!1;try{return import.meta.url===o(e).href}catch{return!1}}function A(e){let t=e.headers[`mcp-session-id`];return Array.isArray(t)?t[0]:t}function j(e,t){let n=process.env[e];if(!n)return t;let r=Number.parseInt(n,10);return Number.isFinite(r)&&r>0?r:t}function M(){return k()?s({allowPositionals:!0,options:{transport:{type:`string`,default:O()},port:{type:`string`,default:process.env.AIKIT_PORT??`3210`}}}).values:{transport:O(),port:process.env.AIKIT_PORT??`3210`}}async function N(){let e=_(),n=M();process.on(`unhandledRejection`,e=>{v.error(`Unhandled rejection`,l(e))}),process.on(`uncaughtException`,e=>{v.error(`Uncaught exception — exiting`,l(e)),process.exit(1)});let r=(e,t)=>{e.then(async()=>{try{let{markPromoteRun:e,markPruneRun:n,prune:r,shouldRunStartupPrune:i,shouldRunWeeklyPromote:a}=await import(`../../tools/dist/index.js`),o=t();if(!o)return;if(i()){let e=await r({});n(),e.totalBytesFreed>0&&v.info(`Storage maintenance complete`,{forgeOrphans:e.forgeGroundOrphans.count,legacyLance:e.legacyLance.count,bytesFreed:e.totalBytesFreed});let{groupLessons:t,pruneLessons:i}=await import(`./evolution-BX_zTSdj.js`).then(e=>e.t),a=await i(o.curated,o.stateStore,{dryRun:!1});a.pruned.length>0&&v.info(`Startup lesson prune complete`,{pruned:a.pruned.length});let s=await t(o.curated,o.stateStore,{dryRun:!1});(s.groupsCreated>0||s.lessonsGrouped>0)&&v.info(`Startup lesson grouping complete`,{groupsCreated:s.groupsCreated,lessonsGrouped:s.lessonsGrouped})}if(a()){let{DEFAULT_PROMOTE_CONFIG:t,collectWorkspaceLessons:n,getGlobalCuratedDir:r,promoteLessons:i,scanForDuplicates:a}=await import(`./promotion-PdKQQktE.js`).then(e=>e.o),{CuratedKnowledgeManager:s}=await import(`./curated-manager-xTfPLFMR.js`).then(e=>e.n),c=o.curated,l=new s(r(),c.store,c.embedder),u=await n(),d={...t,dryRun:!1},f=await i(a(u,d),l,d);e(),f.promoted.length>0&&v.info(`Weekly lesson promotion complete`,{promoted:f.promoted.length,candidates:f.candidates.length})}}catch(e){v.warn(`Startup maintenance failed (non-critical)`,l(e))}}).catch(()=>{})};if(v.info(`Starting MCP AI Kit server`,{version:e}),n.transport===`http`){let[{default:e},{loadConfig:i,resolveIndexMode:a},{registerDashboardRoutes:o,resolveDashboardDir:s},{registerSettingsRoutes:c,resolveSettingsDir:u},{createSettingsRouter:d},{authMiddleware:f,getOrCreateToken:p}]=await Promise.all([import(`express`),import(`./config-D_MQ_9Q7.js`),import(`./dashboard-static-FmfoS46e.js`),import(`./settings-static-BtvyIrza.js`),import(`./routes-CR3fI-HJ.js`),import(`./auth-Bz5dmZgR.js`).then(e=>e.t)]),h=i();v.info(`Config loaded`,{sourceCount:h.sources.length,storePath:h.store.path});let _=e();_.use(e.json({limit:`1mb`}));let y=Number(n.port),S=`http://localhost:${y}`,C=process.env.AIKIT_CORS_ORIGIN??S,w=process.env.AIKIT_ALLOW_ANY_ORIGIN===`true`,T=j(`AIKIT_HTTP_MAX_SESSIONS`,8),E=j(`AIKIT_HTTP_SESSION_TIMEOUT_MINUTES`,30),D=j(`AIKIT_HTTP_SESSION_GC_INTERVAL_MINUTES`,5),O=x({limit:100,windowMs:6e4}),k=!1;_.use((e,t,n)=>{let r=Array.isArray(e.headers.origin)?e.headers.origin[0]:e.headers.origin,i=b({requestOrigin:r,configuredOrigin:C,allowAnyOrigin:w,fallbackOrigin:S});if(i.warn&&!k&&(k=!0,v.warn(`Rejected non-local CORS origin while AIKIT_CORS_ORIGIN=*`,{origin:r,allowAnyOriginEnv:`AIKIT_ALLOW_ANY_ORIGIN=true`})),r&&!i.allowOrigin){t.status(403).json({error:`Origin not allowed`});return}if(i.allowOrigin&&t.setHeader(`Access-Control-Allow-Origin`,i.allowOrigin),t.setHeader(`Access-Control-Allow-Methods`,`GET, POST, PUT, PATCH, DELETE, OPTIONS`),t.setHeader(`Access-Control-Allow-Headers`,`Content-Type, Authorization, Mcp-Session-Id, Mcp-Protocol-Version, Last-Event-ID`),t.setHeader(`Access-Control-Expose-Headers`,`Mcp-Session-Id`),e.method===`OPTIONS`){t.status(204).end();return}n()});let M=p();console.error(`[aikit] Auth token: ~/.aikit/token`),_.use(f(M)),_.use(`/mcp`,(e,t,n)=>{let r=A(e)??e.ip??e.socket.remoteAddress??`anonymous`;if(O.allow(r)){n();return}let i=Math.max(1,Math.ceil(O.getRetryAfterMs(r)/1e3));t.setHeader(`Retry-After`,String(i)),t.status(429).json({jsonrpc:`2.0`,error:{code:-32003,message:`Rate limit exceeded`},id:null})}),o(_,s(),v);let N=new Date().toISOString();_.use(`/settings/api`,d({log:v,mcpInfo:()=>({transport:`http`,port:y,pid:process.pid,startedAt:N})})),c(_,u(),v),_.get(`/health`,(e,t)=>{t.json({status:`ok`})});let P=!1,F=null,I=null,L=null,R=null,z=null,B=null,V=null,H=Promise.resolve(),U=async(e,n)=>{if(!P||!L||!R){n.status(503).json({jsonrpc:`2.0`,error:{code:-32603,message:`Server initializing — please retry in a few seconds`},id:null});return}let r=H,i;H=new Promise(e=>{i=e}),await r;try{let r=A(e);if(!B){if(r){n.status(404).json({jsonrpc:`2.0`,error:{code:-32001,message:`Session not found`},id:null});return}let e=new R({sessionIdGenerator:()=>t(),onsessioninitialized:async e=>{V=e,I?.onSessionStart(e,{transport:`http`})},onsessionclosed:async e=>{e&&I?.onSessionEnd(e),V=null}});e.onclose=()=>{B===e&&(B=null),V===e.sessionId&&(V=null)},B=e,await L.connect(e)}let i=B;await i.handleRequest(e,n,e.body),e.method!==`DELETE`&&(!r&&i.sessionId?(V=i.sessionId,I?.onSessionStart(i.sessionId,{transport:`http`}),I?.onSessionActivity(i.sessionId)):r&&I?.onSessionActivity(r))}catch(e){if(v.error(`MCP handler error`,l(e)),!n.headersSent){let t=e instanceof Error?e.message:String(e),r=t.includes(`Not Acceptable`);n.status(r?406:500).json({jsonrpc:`2.0`,error:{code:r?-32e3:-32603,message:r?t:`Internal server error`},id:null})}}finally{i()}},W=async(e,t)=>{let n=A(e);if(z&&(!B||n!==V)){await z.handleRequest(e,t,e.body);return}await U(e,t)};_.post(`/mcp`,W),_.get(`/mcp`,W),_.delete(`/mcp`,W);let G=_.listen(y,`127.0.0.1`,()=>{v.info(`MCP server listening`,{url:`http://127.0.0.1:${y}/mcp`,port:y}),setTimeout(async()=>{try{let[{createLazyServer:e,createMcpServer:t,ALL_TOOL_NAMES:n},{StreamableHTTPServerTransport:i},{checkForUpdates:o,autoUpgradeScaffold:s}]=await Promise.all([import(`./server-0JRtGGti.js`),import(`@modelcontextprotocol/sdk/server/streamableHttp.js`),import(`./version-check-BgHzxxCW.js`)]);o(),s();let c=a(h),u=e(h,c);L=u.server,R=i,P=!0,v.debug(`MCP server configured (lazy — AI Kit initializing in background)`,{toolCount:n.length,resourceCount:2}),u.startInit(),u.ready.then(()=>{if(!u.aikit)throw Error(`AI Kit components are not available after initialization`);I=new g(u.aikit.stateStore,{staleTimeoutMinutes:E,gcIntervalMinutes:D,onBeforeSessionDelete:e=>{if(V===e&&B){let e=B;B=null,V=null,e.close().catch(()=>void 0)}z?.closeSession(e,{notifySessionEnd:!1})},onSessionEndMaintenance:async()=>{if(!u.aikit?.curated||!u.aikit?.stateStore)return;let{pruneLessons:e}=await import(`./evolution-BX_zTSdj.js`).then(e=>e.t),t=await e(u.aikit.curated,u.aikit.stateStore,{dryRun:!1});t.pruned.length>0&&v.info(`Session-end lesson prune`,{pruned:t.pruned.length})}}),z=new m({createServer:()=>{if(!u.aikit)throw Error(`AI Kit components are not available after initialization`);return t(u.aikit,h)},createTransport:e=>new i(e),maxSessions:T,sessionTimeoutMinutes:E,onSessionStart:e=>I?.onSessionStart(e,{transport:`http`}),onSessionActivity:e=>I?.onSessionActivity(e),onSessionEnd:e=>I?.onSessionEnd(e)}),I.startGC(),V&&(I.onSessionStart(V,{transport:`http`}),I.onSessionActivity(V)),v.info(`HTTP session runtime ready`,{maxSessions:T,sessionTimeoutMinutes:E,gcIntervalMinutes:D})}).catch(e=>v.error(`Failed to start session manager`,l(e))),c===`auto`?u.ready.then(async()=>{try{let e=h.sources.map(e=>e.path).join(`, `);v.info(`Running initial index`,{sourcePaths:e}),await u.runInitialIndex(),v.info(`Initial index complete`)}catch(e){v.error(`Initial index failed; will retry on aikit_reindex`,l(e))}}).catch(e=>v.error(`AI Kit init or indexing failed`,l(e))):c===`smart`?u.ready.then(async()=>{try{if(!u.aikit)throw Error(`AI Kit components are not available after initialization`);let{SmartIndexScheduler:e}=await import(`../../indexer/dist/index.js`),t=new e(u.aikit.indexer,h,u.aikit.store),n=u.aikit.store;F=t,t.start(),n.onBeforeClose&&n.onBeforeClose(()=>t.stop()),u.setSmartScheduler(t),v.debug(`Smart index scheduler started (HTTP mode)`)}catch(e){v.error(`Failed to start smart index scheduler`,l(e))}}).catch(e=>v.error(`AI Kit initialization failed`,l(e))):(u.ready.catch(e=>v.error(`AI Kit initialization failed`,l(e))),v.info(`Initial full indexing skipped in HTTP mode`,{indexMode:c})),r(u.ready,()=>u.aikit?{curated:u.aikit.curated,stateStore:u.aikit.stateStore}:null)}catch(e){v.error(`Failed to load server modules`,l(e))}},100)}),K=async e=>{v.info(`Shutdown signal received`,{signal:e}),F?.stop(),I?.stop(),await z?.closeAll().catch(()=>void 0),V&&I?.onSessionEnd(V),B&&(await B.close().catch(()=>void 0),B=null,V=null),G.close(),L&&await L.close(),process.exit(0)};process.on(`SIGINT`,()=>K(`SIGINT`)),process.on(`SIGTERM`,()=>K(`SIGTERM`))}else{let[{loadConfig:e,reconfigureForWorkspace:t,resolveIndexMode:n},{createLazyServer:i},{checkForUpdates:a,autoUpgradeScaffold:o},{RootsListChangedNotificationSchema:s}]=await Promise.all([import(`./config-D_MQ_9Q7.js`),import(`./server-0JRtGGti.js`),import(`./version-check-BgHzxxCW.js`),import(`@modelcontextprotocol/sdk/types.js`)]),c=e();v.info(`Config loaded`,{sourceCount:c.sources.length,storePath:c.store.path}),a(),o();let u=n(c),d=i(c,u),{server:f,startInit:p,ready:m,runInitialIndex:h}=d,{StdioServerTransport:g}=await import(`@modelcontextprotocol/sdk/server/stdio.js`),_=new g;await f.connect(_),v.debug(`MCP server started`,{transport:`stdio`}),await D({config:c,indexMode:u,log:v,rootsChangedNotificationSchema:s,reconfigureForWorkspace:t,runInitialIndex:h,server:f,startInit:p});let y=null,b=()=>{y&&clearTimeout(y),y=setTimeout(async()=>{v.info(`Auto-shutdown: no activity for 30 minutes — shutting down gracefully`);try{let e=d.aikit;e&&await Promise.all([e.embedder.shutdown?.().catch(()=>{})??Promise.resolve(),e.graphStore.close().catch(()=>{}),e.store.close().catch(()=>{})])}catch{}process.exit(0)},18e5),y.unref&&y.unref()};b(),process.stdin.on(`data`,()=>b()),m.catch(e=>{v.error(`Initialization failed — server will continue with limited tools`,l(e))}),u===`smart`?m.then(async()=>{try{if(!d.aikit)throw Error(`AI Kit components are not available after initialization`);let{SmartIndexScheduler:e}=await import(`../../indexer/dist/index.js`),t=new e(d.aikit.indexer,c,d.aikit.store),n=d.aikit.store;t.start(),n.onBeforeClose&&n.onBeforeClose(()=>t.stop()),d.setSmartScheduler(t),v.debug(`Smart index scheduler started (stdio mode)`)}catch(e){v.error(`Failed to start smart index scheduler`,l(e))}}).catch(e=>v.error(`AI Kit init failed for smart scheduler`,l(e))):v.warn(`Initial full indexing skipped; use aikit_reindex to index manually`,{indexMode:u}),r(m,()=>d.aikit?{curated:d.aikit.curated,stateStore:d.aikit.stateStore}:null)}}export{e as CuratedKnowledgeManager,E as applyWorkspaceRoots,D as bootstrapWorkspaceRoots,x as createSlidingWindowRateLimiter,N as main,b as resolveCorsOrigin,T as selectWorkspaceRoot};
|