@karmaniverous/jeeves-watcher 0.1.0

package/dist/index.iife.min.js

@@ -0,0 +1 @@
+!function(e,t,i,r,o,n,s,a,c,l,h,u,d,p,f,g,m,y){"use strict";function w(e){var t=Object.create(null);return e&&Object.keys(e).forEach((function(i){if("default"!==i){var r=Object.getOwnPropertyDescriptor(e,i);Object.defineProperty(t,i,r.get?r:{enumerable:!0,get:function(){return e[i]}})}})),t.default=e,Object.freeze(t)}var b=w(d);function v(e,t){const r=function(e){return e.replace(/\\/g,"/").replace(/^([A-Za-z]):/,((e,t)=>t.toLowerCase())).toLowerCase()}(e),n=i.createHash("sha256").update(r,"utf8").digest("hex");return o.join(t,`${n}.meta.json`)}async function P(e,t){try{const i=await r.readFile(v(e,t),"utf8");return JSON.parse(i)}catch{return null}}async function k(e,t,i){const n=v(e,t);await r.mkdir(o.dirname(n),{recursive:!0}),await r.writeFile(n,JSON.stringify(i,null,2),"utf8")}async function x(e,t){try{await r.rm(v(e,t))}catch{}}function S(e){const t=e.replace(/\\/g,"/"),i=t.search(/[*?\[]/);if(-1===i)return o.resolve(e);const r=t.slice(0,i),n=r.endsWith("/")?r.slice(0,-1):o.dirname(r);return o.resolve(n)}async function*M(e){let t;try{t=(await r.readdir(e,{withFileTypes:!0})).map((e=>({name:e.name,isDirectory:e.isDirectory()})))}catch{return}for(const i of t){const t=o.resolve(e,i.name);if(i.isDirectory)yield*M(t);else try{(await r.stat(t)).isFile()&&(yield t)}catch{}}}async function j(e,t=[]){const i=e.map((e=>e.replace(/\\/g,"/"))),r=t.map((e=>e.replace(/\\/g,"/"))),o=n(i,{dot:!0}),s=r.length?n(r,{dot:!0}):()=>!1,a=Array.from(new Set(e.map(S))),c=new Set;for(const e of a)for await(const t of M(e)){const e=t.replace(/\\/g,"/");s(e)||o(e)&&c.add(t)}return Array.from(c)}function F(e){const{processor:i,vectorStore:r,embeddingProvider:o,logger:n}=e,s=t({logger:!1});return s.get("/status",(()=>({status:"ok",uptime:process.uptime()}))),s.post("/metadata",(async(e,t)=>{try{const{path:t,metadata:r}=e.body;return await i.processMetadataUpdate(t,r),{ok:!0}}catch(e){return n.error({error:e},"Metadata update failed"),t.status(500).send({error:"Internal server error"})}})),s.post("/search",(async(e,t)=>{try{const{query:t,limit:i=10}=e.body,n=await o.embed([t]);return await r.search(n[0],i)}catch(e){return n.error({error:e},"Search failed"),t.status(500).send({error:"Internal server error"})}})),s.post("/reindex",(async(t,r)=>{try{const t=await j(e.config.watch.paths,e.config.watch.ignored);for(const e of t)await i.processFile(e);return await r.status(200).send({ok:!0,filesIndexed:t.length})}catch(e){return n.error({error:e},"Reindex failed"),await r.status(500).send({error:"Internal server error"})}})),s.post("/rebuild-metadata",(async(t,i)=>{try{const t=e.config.metadataDir??".jeeves-metadata";for await(const e of r.scroll()){const i=e.payload,r=i.file_path;if("string"!=typeof r||0===r.length)continue;const o={...i};delete o.file_path,delete o.chunk_index,delete o.total_chunks,delete o.content_hash,delete o.chunk_text,await k(r,t,o)}return await i.status(200).send({ok:!0})}catch(e){return n.error({error:e},"Rebuild metadata failed"),await i.status(500).send({error:"Internal server error"})}})),s.post("/config-reindex",(async(t,r)=>{try{const o=t.body.scope??"rules";return(async()=>{try{if("rules"===o){const t=await j(e.config.watch.paths,e.config.watch.ignored);for(const e of t)await i.processRulesUpdate(e);n.info({scope:o,filesProcessed:t.length},"Config reindex (rules) completed")}else{const t=await j(e.config.watch.paths,e.config.watch.ignored);for(const e of t)await i.processFile(e);n.info({scope:o,filesProcessed:t.length},"Config reindex (full) completed")}}catch(e){n.error({error:e,scope:o},"Config reindex failed")}})(),await r.status(200).send({status:"started",scope:o})}catch(e){return n.error({error:e},"Config reindex request failed"),await r.status(500).send({error:"Internal server error"})}})),s}const T="jeeves-watcher",C=new a({allErrors:!0}).compile({type:"object",required:["watch","embedding","vectorStore"],properties:{watch:{type:"object",required:["paths"],properties:{paths:{type:"array",items:{type:"string"},minItems:1},ignored:{type:"array",items:{type:"string"}},pollIntervalMs:{type:"number"},usePolling:{type:"boolean"},debounceMs:{type:"number"},stabilityThresholdMs:{type:"number"}},additionalProperties:!1},configWatch:{type:"object",properties:{enabled:{type:"boolean"},debounceMs:{type:"number"}},additionalProperties:!1},embedding:{type:"object",required:["provider","model"],properties:{provider:{type:"string"},model:{type:"string"},chunkSize:{type:"number"},chunkOverlap:{type:"number"},dimensions:{type:"number"},apiKey:{type:"string"},rateLimitPerMinute:{type:"number"},concurrency:{type:"number"}},additionalProperties:!1},vectorStore:{type:"object",required:["url","collectionName"],properties:{url:{type:"string"},collectionName:{type:"string"},apiKey:{type:"string"}},additionalProperties:!1},metadataDir:{type:"string"},api:{type:"object",properties:{host:{type:"string"},port:{type:"number"}},additionalProperties:!1},extractors:{type:"object"},inferenceRules:{type:"array",items:{type:"object",required:["match","set"],properties:{match:{type:"object"},set:{type:"object"}},additionalProperties:!1}},logging:{type:"object",properties:{level:{type:"string"},file:{type:"string"}},additionalProperties:!1},shutdownTimeoutMs:{type:"number"}},additionalProperties:!1}),I={configWatch:{enabled:!0,debounceMs:1e3},metadataDir:".jeeves-watcher",api:{host:"127.0.0.1",port:3100},logging:{level:"info"},shutdownTimeoutMs:1e4},_={debounceMs:300,stabilityThresholdMs:500,usePolling:!1,pollIntervalMs:1e3},D={chunkSize:1e3,chunkOverlap:200,dimensions:768,rateLimitPerMinute:300,concurrency:5};async function R(e){const t=c.cosmiconfig(T),i=e?await t.load(e):await t.search();if(!i||i.isEmpty)throw new Error("No jeeves-watcher configuration found. Create a .jeeves-watcherrc or jeeves-watcher.config.{js,ts,json,yaml} file.");const r=i.config;if(!C(r)){const e=C.errors?.map((e=>`${("instancePath"in e?e.instancePath:void 0)??"/"}: ${e.message??"unknown error"}`)).join("; ");throw new Error(`Invalid jeeves-watcher configuration: ${e??"unknown error"}`)}return function(e){return{...I,...e,watch:{..._,...e.watch},configWatch:{...I.configWatch,...e.configWatch},embedding:{...D,...e.embedding},api:{...I.api,...e.api},logging:{...I.logging,...e.logging}}}(r)}function W(e){const t=e.dimensions??768;switch(e.provider){case"mock":return function(e){return{dimensions:e,embed:t=>Promise.resolve(t.map((t=>{const r=i.createHash("sha256").update(t,"utf8").digest(),o=[];for(let t=0;t<e;t++){const e=r[t%r.length];o.push(e/127.5-1)}return o})))}}(t);case"gemini":return function(e){if(!e.apiKey)throw new Error("Gemini embedding provider requires config.embedding.apiKey");const t=e.dimensions??3072,i=new l.GoogleGenerativeAIEmbeddings({apiKey:e.apiKey,model:e.model});return{dimensions:t,async embed(e){const r=await i.embedDocuments(e);for(const e of r)if(e.length!==t)throw new Error(`Gemini embedding returned invalid dimensions: expected ${String(t)}, got ${String(e.length)}`);return r}}}(e);default:throw new Error(`Unsupported embedding provider: ${e.provider}`)}}function N(e){const t=e?.level??"info";if(e?.file){const i=h.transport({target:"pino/file",options:{destination:e.file,mkdir:!0}});return h({level:t},i)}return h({level:t})}const q=["content","body","text","snippet","subject","description","summary","transcript"];function O(e){if(!e||"object"!=typeof e)return JSON.stringify(e);const t=e;for(const e of q){const i=t[e];if("string"==typeof i&&i.trim())return i}return JSON.stringify(e)}async function L(e,t){const i=t.toLowerCase();if(".md"===i||".markdown"===i){const t=await r.readFile(e,"utf8"),{frontmatter:i,body:o}=function(e){const t=e.replace(/^\uFEFF/,""),i=/^---\s*\n([\s\S]*?)\n---\s*\n?([\s\S]*)$/m.exec(t);if(!i)return{body:e};const[,r,o]=i,n=p.load(r);return{frontmatter:n&&"object"==typeof n&&!Array.isArray(n)?n:void 0,body:o}}(t);return{text:o,frontmatter:i}}if(".txt"===i||".text"===i){return{text:await r.readFile(e,"utf8")}}if(".json"===i){const t=await r.readFile(e,"utf8"),i=JSON.parse(t),o=i&&"object"==typeof i&&!Array.isArray(i)?i:void 0;return{text:O(i),json:o}}if(".pdf"===i){const t=await r.readFile(e),i=new Uint8Array(t),{extractText:o}=await import("unpdf"),{text:n}=await o(i);return{text:Array.isArray(n)?n.join("\n\n"):n}}if(".docx"===i){const t=await r.readFile(e);return{text:(await f.extractRawText({buffer:t})).value}}if(".html"===i||".htm"===i){const t=await r.readFile(e,"utf8"),i=b.load(t);i("script, style").remove();return{text:i("body").text().trim()||i.text().trim()}}return{text:await r.readFile(e,"utf8")}}function $(e){return i.createHash("sha256").update(e,"utf8").digest("hex")}function E(e){return e.replace(/\\/g,"/").toLowerCase()}function A(e,t){const i=void 0!==t?`${E(e)}#${String(t)}`:E(e);return g.v5(i,"6a6f686e-6761-4c74-ad6a-656576657321")}function K(e,t,i,r){const n=e.replace(/\\/g,"/"),s={file:{path:n,directory:o.dirname(n).replace(/\\/g,"/"),filename:o.basename(n),extension:o.extname(n),sizeBytes:t.size,modified:t.mtime.toISOString()}};return i&&(s.frontmatter=i),r&&(s.json=r),s}function z(e){const t=function(){const e=new a({allErrors:!0});return m(e),e.addKeyword({keyword:"glob",type:"string",schemaType:"string",validate:(e,t)=>n.isMatch(t,e)}),e}();return e.map(((e,i)=>({rule:e,validate:t.compile({$id:`rule-${String(i)}`,...e.match})})))}function Q(e,t){return"string"!=typeof e?e:e.replace(/\$\{([^}]+)\}/g,((e,i)=>{const r=i.split(".");let o=t;for(const e of r){if(null==o)return"";o=o[e]}return null==o?"":"string"==typeof o?o:JSON.stringify(o)}))}function G(e,t){const i={};for(const[r,o]of Object.entries(e))i[r]=Q(o,t);return i}function J(e,t){let i={};for(const{rule:r,validate:o}of e)o(t)&&(i={...i,...G(r.set,t)});return i}class B{config;embeddingProvider;vectorStore;compiledRules;logger;metadataDir;constructor(e,t,i,r,o){this.config=e,this.embeddingProvider=t,this.vectorStore=i,this.compiledRules=r,this.logger=o,this.metadataDir=e.metadataDir??".jeeves-metadata"}async processFile(e){try{const t=o.extname(e),i=await r.stat(e),n=await L(e,t);if(!n.text.trim())return void this.logger.debug({filePath:e},"Skipping empty file");const s=$(n.text),a=A(e,0),c=await this.vectorStore.getPayload(a);if(c&&c.content_hash===s)return void this.logger.debug({filePath:e},"Content unchanged, skipping");const l="number"==typeof c?.total_chunks?c.total_chunks:0,h=K(e,i,n.frontmatter,n.json),u=J(this.compiledRules,h),d=await P(e,this.metadataDir),p={...u,...d??{}},f=this.config.embedding.chunkSize??1e3,g=this.config.embedding.chunkOverlap??200,m=this.createSplitter(t,f,g),y=await m.splitText(n.text),w=await this.embeddingProvider.embed(y),b=y.map(((t,i)=>({id:A(e,i),vector:w[i],payload:{...p,file_path:e.replace(/\\/g,"/"),chunk_index:i,total_chunks:y.length,content_hash:s,chunk_text:t}})));if(await this.vectorStore.upsert(b),l>y.length){const t=[];for(let i=y.length;i<l;i++)t.push(A(e,i));await this.vectorStore.delete(t)}this.logger.info({filePath:e,chunks:y.length},"File processed successfully")}catch(t){this.logger.error({filePath:e,error:t},"Failed to process file")}}async deleteFile(e){try{const t=A(e,0),i=await this.vectorStore.getPayload(t),r="number"==typeof i?.total_chunks?i.total_chunks:1,o=[];for(let t=0;t<r;t++)o.push(A(e,t));await this.vectorStore.delete(o),await x(e,this.metadataDir),this.logger.info({filePath:e},"File deleted from index")}catch(t){this.logger.error({filePath:e,error:t},"Failed to delete file")}}async processMetadataUpdate(e,t){try{const i={...await P(e,this.metadataDir)??{},...t};await k(e,this.metadataDir,i);const r=A(e,0),o=await this.vectorStore.getPayload(r);if(!o)return null;const n="number"==typeof o.total_chunks?o.total_chunks:1,s=[];for(let t=0;t<n;t++)s.push(A(e,t));return await this.vectorStore.setPayload(s,i),this.logger.info({filePath:e,chunks:n},"Metadata updated"),i}catch(t){return this.logger.error({filePath:e,error:t},"Failed to update metadata"),null}}async processRulesUpdate(e){try{const t=A(e,0),i=await this.vectorStore.getPayload(t);if(!i)return this.logger.debug({filePath:e},"File not indexed, skipping"),null;const n=o.extname(e),s=await r.stat(e),a=await L(e,n),c=K(e,s,a.frontmatter,a.json),l=J(this.compiledRules,c),h=await P(e,this.metadataDir),u={...l,...h??{}},d="number"==typeof i.total_chunks?i.total_chunks:1,p=[];for(let t=0;t<d;t++)p.push(A(e,t));return await this.vectorStore.setPayload(p,u),this.logger.info({filePath:e,chunks:d},"Rules re-applied"),u}catch(t){return this.logger.error({filePath:e,error:t},"Failed to re-apply rules"),null}}updateRules(e){this.compiledRules=e,this.logger.info({rules:e.length},"Inference rules updated")}createSplitter(e,t,i){const r=e.toLowerCase();return".md"===r||".markdown"===r?new u.MarkdownTextSplitter({chunkSize:t,chunkOverlap:i}):new u.RecursiveCharacterTextSplitter({chunkSize:t,chunkOverlap:i})}}class U{debounceMs;concurrency;rateLimitPerMinute;started=!1;active=0;debounceTimers=new Map;latestByKey=new Map;normalQueue=[];lowQueue=[];tokens;lastRefillMs=Date.now();drainWaiters=[];constructor(e){this.debounceMs=e.debounceMs,this.concurrency=e.concurrency,this.rateLimitPerMinute=e.rateLimitPerMinute,this.tokens=this.rateLimitPerMinute??Number.POSITIVE_INFINITY}enqueue(e,t){const i=`${e.priority}:${e.path}`;this.latestByKey.set(i,{event:e,fn:t});const r=this.debounceTimers.get(i);r&&clearTimeout(r);const o=setTimeout((()=>{this.debounceTimers.delete(i);const e=this.latestByKey.get(i);e&&(this.latestByKey.delete(i),this.push(e),this.pump())}),this.debounceMs);this.debounceTimers.set(i,o)}process(){this.started=!0,this.pump()}async drain(){this.isIdle()||await new Promise((e=>{this.drainWaiters.push(e)}))}push(e){"low"===e.event.priority?this.lowQueue.push(e):this.normalQueue.push(e)}refillTokens(e){if(void 0===this.rateLimitPerMinute)return;const t=Math.max(0,e-this.lastRefillMs)*(this.rateLimitPerMinute/6e4);this.tokens=Math.min(this.rateLimitPerMinute,this.tokens+t),this.lastRefillMs=e}takeToken(){const e=Date.now();return this.refillTokens(e),!(this.tokens<1)&&(this.tokens-=1,!0)}nextItem(){return this.normalQueue.shift()??this.lowQueue.shift()}pump(){if(this.started){for(;this.active<this.concurrency;){const e=this.nextItem();if(!e)break;if(!this.takeToken()){"low"===e.event.priority?this.lowQueue.unshift(e):this.normalQueue.unshift(e),setTimeout((()=>{this.pump()}),250);break}this.active+=1,Promise.resolve().then((()=>e.fn(e.event))).finally((()=>{this.active-=1,this.pump(),this.maybeResolveDrain()}))}this.maybeResolveDrain()}}isIdle(){return 0===this.active&&0===this.normalQueue.length&&0===this.lowQueue.length&&0===this.debounceTimers.size&&0===this.latestByKey.size}maybeResolveDrain(){if(!this.isIdle())return;const e=this.drainWaiters;this.drainWaiters=[];for(const t of e)t()}}class H{client;collectionName;dims;constructor(e,t){this.client=new y.QdrantClient({url:e.url,apiKey:e.apiKey}),this.collectionName=e.collectionName,this.dims=t}async ensureCollection(){try{const e=await this.client.getCollections();e.collections.some((e=>e.name===this.collectionName))||await this.client.createCollection(this.collectionName,{vectors:{size:this.dims,distance:"Cosine"}})}catch(e){throw new Error(`Failed to ensure collection "${this.collectionName}": ${String(e)}`)}}async upsert(e){0!==e.length&&await this.client.upsert(this.collectionName,{wait:!0,points:e.map((e=>({id:e.id,vector:e.vector,payload:e.payload})))})}async delete(e){0!==e.length&&await this.client.delete(this.collectionName,{wait:!0,points:e})}async setPayload(e,t){0!==e.length&&await this.client.setPayload(this.collectionName,{wait:!0,points:e,payload:t})}async getPayload(e){try{const t=await this.client.retrieve(this.collectionName,{ids:[e],with_payload:!0,with_vector:!1});return 0===t.length?null:t[0].payload}catch{return null}}async search(e,t,i){return(await this.client.search(this.collectionName,{vector:e,limit:t,with_payload:!0,...i?{filter:i}:{}})).map((e=>({id:String(e.id),score:e.score,payload:e.payload})))}async*scroll(e,t=100){let i;for(;;){const r=await this.client.scroll(this.collectionName,{limit:t,with_payload:!0,with_vector:!1,...e?{filter:e}:{},...void 0!==i?{offset:i}:{}});for(const e of r.points)yield{id:String(e.id),payload:e.payload};const o=r.next_page_offset;if(null==o)break;if("string"!=typeof o&&"number"!=typeof o)break;i=o}}}class V{config;queue;processor;logger;watcher;constructor(e,t,i,r){this.config=e,this.queue=t,this.processor=i,this.logger=r}start(){this.watcher=s.watch(this.config.paths,{ignored:this.config.ignored,usePolling:this.config.usePolling,interval:this.config.pollIntervalMs,awaitWriteFinish:!!this.config.stabilityThresholdMs&&{stabilityThreshold:this.config.stabilityThresholdMs},ignoreInitial:!1}),this.watcher.on("add",(e=>{this.logger.debug({path:e},"File added"),this.queue.enqueue({type:"create",path:e,priority:"normal"},(()=>this.processor.processFile(e)))})),this.watcher.on("change",(e=>{this.logger.debug({path:e},"File changed"),this.queue.enqueue({type:"modify",path:e,priority:"normal"},(()=>this.processor.processFile(e)))})),this.watcher.on("unlink",(e=>{this.logger.debug({path:e},"File removed"),this.queue.enqueue({type:"delete",path:e,priority:"normal"},(()=>this.processor.deleteFile(e)))})),this.watcher.on("error",(e=>{this.logger.error({error:e},"Watcher error")})),this.queue.process(),this.logger.info({paths:this.config.paths},"Filesystem watcher started")}async stop(){this.watcher&&(await this.watcher.close(),this.watcher=void 0,this.logger.info("Filesystem watcher stopped"))}}class Y{config;configPath;logger;watcher;queue;server;processor;configWatcher;configDebounce;constructor(e,t){this.config=e,this.configPath=t}async start(){const e=N(this.config.logging);let t;this.logger=e;try{t=W(this.config.embedding)}catch(t){throw e.fatal({error:t},"Failed to create embedding provider"),t}const i=new H(this.config.vectorStore,t.dimensions);await i.ensureCollection();const r=z(this.config.inferenceRules??[]),o=new B(this.config,t,i,r,e);this.processor=o;const n=new U({debounceMs:this.config.watch.debounceMs??2e3,concurrency:this.config.embedding.concurrency??5,rateLimitPerMinute:this.config.embedding.rateLimitPerMinute});this.queue=n;const s=new V(this.config.watch,n,o,e);this.watcher=s;const a=F({processor:o,vectorStore:i,embeddingProvider:t,queue:n,config:this.config,logger:e});this.server=a,await a.listen({host:this.config.api?.host??"127.0.0.1",port:this.config.api?.port??3458}),s.start(),this.startConfigWatch(),e.info("jeeves-watcher started")}async stop(){if(await this.stopConfigWatch(),this.watcher&&await this.watcher.stop(),this.queue){const e=this.config.shutdownTimeoutMs??1e4;await Promise.race([this.queue.drain(),new Promise((t=>{setTimeout(t,e)}))])}this.server&&await this.server.close(),this.logger?.info("jeeves-watcher stopped")}startConfigWatch(){const e=this.logger;if(!e)return;if(!(this.config.configWatch?.enabled??!0))return;if(!this.configPath)return void e.debug("Config watch enabled, but no config path was provided");const t=this.config.configWatch?.debounceMs??1e4;this.configWatcher=s.watch(this.configPath,{ignoreInitial:!0}),this.configWatcher.on("change",(()=>{this.configDebounce&&clearTimeout(this.configDebounce),this.configDebounce=setTimeout((()=>{this.reloadConfig()}),t)})),this.configWatcher.on("error",(t=>{e.error({error:t},"Config watcher error")})),e.info({configPath:this.configPath,debounceMs:t},"Config watcher started")}async stopConfigWatch(){this.configDebounce&&(clearTimeout(this.configDebounce),this.configDebounce=void 0),this.configWatcher&&(await this.configWatcher.close(),this.configWatcher=void 0)}async reloadConfig(){const e=this.logger,t=this.processor;if(e&&t&&this.configPath)try{const i=await R(this.configPath);this.config=i;const r=z(i.inferenceRules??[]);t.updateRules(r),e.info({configPath:this.configPath,rules:r.length},"Config reloaded")}catch(t){e.error({error:t},"Failed to reload config")}}}e.DocumentProcessor=B,e.EventQueue=U,e.FileSystemWatcher=V,e.JeevesWatcher=Y,e.VectorStoreClient=H,e.applyRules=J,e.buildAttributes=K,e.compileRules=z,e.contentHash=$,e.createApiServer=F,e.createEmbeddingProvider=W,e.createLogger=N,e.deleteMetadata=x,e.extractText=L,e.loadConfig=R,e.metadataPath=v,e.pointId=A,e.readMetadata=P,e.startFromConfig=async function(e){const t=await R(e),i=new Y(t,e),r=async()=>{await i.stop(),process.exit(0)};return process.on("SIGTERM",(()=>{r()})),process.on("SIGINT",(()=>{r()})),await i.start(),i},e.writeMetadata=k}(this["jeeves-watcher"]=this["jeeves-watcher"]||{},Fastify,node_crypto,promises,node_path,picomatch,chokidar,Ajv,cosmiconfig,googleGenai,pino,textsplitters,cheerio,yaml,mammoth,uuid,addFormats,jsClientRest);