@augmentcode/onison 1.2.0-prerelease.202608201510 → 1.2.0-prerelease.202608210912
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/auggie-v2.mjs +1356 -1357
- package/indexing-worker.mjs +1 -1
- package/package.json +1 -1
package/indexing-worker.mjs
CHANGED
|
@@ -238,7 +238,7 @@ ${i.join(`
|
|
|
238
238
|
`).map(e=>e.endsWith("\r")?e.slice(0,-1):e).filter(e=>e.trim()!==""&&!e.trim().startsWith("#"))}function dQ(t){let e=[],n=_p(t);for(;n!==".";)e.push(n),n=_p(n);return e.push(""),e}function Pf(t,e,n,r){for(let i of t){let a=e.get(i);if(!a)continue;let o=a[r];if(!o)continue;let s=o.test(n);if(s.unignored||s.ignored)return s}}var Qc=class{workspaceRoot;maxFileSize;excludedExtensions;builtinIgnore;rulesMap=new Map;constructor(e){if(this.workspaceRoot=e.workspaceRoot,this.maxFileSize=e.maxFileSize??pse,this.excludedExtensions=new Set(Vse),e.additionalExcludedExtensions)for(let n of e.additionalExcludedExtensions)this.excludedExtensions.add(n.startsWith(".")?n:`.${n}`);this.builtinIgnore=(0,qf.default)(),this.builtinIgnore.add(Rse)}loadDirectory(e){let n=e?kp(this.workspaceRoot,e):this.workspaceRoot,r={},i=!1,a=kp(n,".augmentignore");if(lQ(a))try{let s=cQ(a,"utf-8"),A=AQ(s);if(A.length>0){let l=(0,qf.default)();l.add(A),r.augmentignore=l,i=!0}}catch(s){xp.warn(`Failed to read ${a}`,{error:s})}let o=kp(n,".gitignore");if(lQ(o))try{let s=cQ(o,"utf-8"),A=AQ(s);if(A.length>0){let l=(0,qf.default)();l.add(A),r.gitignore=l,i=!0}}catch(s){xp.warn(`Failed to read ${o}`,{error:s})}i?this.rulesMap.set(e,r):this.rulesMap.delete(e)}updateIgnoreFile(e){let n=_p(e),r=n==="."?"":n;this.loadDirectory(r),xp.debug(`Updated ignore rules for directory: ${r||"root"}`)}removeDirectory(e){this.rulesMap.delete(e)}isDirectoryIgnored(e){let n=e.endsWith("/")?e:`${e}/`,r=dQ(e),i=Pf(r,this.rulesMap,n,"augmentignore");if(i)return i.ignored;let a=Pf(r,this.rulesMap,n,"gitignore");return a?a.ignored:!1}shouldAcceptBlob(e){let{path:n,size:r}=e,i=dQ(n),a=Pf(i,this.rulesMap,n,"augmentignore");if(a)return a.unignored?Promise.resolve({accepted:!0}):Promise.resolve({accepted:!1,reason:"Matched .augmentignore rule"});if(this.builtinIgnore.ignores(n))return Promise.resolve({accepted:!1,reason:"Matched builtin security pattern"});let o=Ese(n).toLowerCase();if(this.excludedExtensions.has(o))return Promise.resolve({accepted:!1,reason:`Excluded extension: ${o}`});if(r>this.maxFileSize)return Promise.resolve({accepted:!1,reason:`File too large: ${r} > ${this.maxFileSize}`});let s=Pf(i,this.rulesMap,n,"gitignore");return s?s.unignored?Promise.resolve({accepted:!0}):Promise.resolve({accepted:!1,reason:"Matched .gitignore rule"}):Promise.resolve({accepted:!0})}};var is=class{blobs={};disposables=[];log=Dt.child({prefix:"IndexOrchestrator"});traversalDone=!1;hadFailures=!1;finalized=!1;doneCallback;_filesAccepted=0;_filesSettled=0;_filesIndexed=0;_filesFailed=0;_filesRejected=0;uploadBatchSize;settledPending=!1;blobFilter;blobUploader;checkpointBuilder;onBlobSettled=new fn;constructor(e,n,r,{uploadBatchSize:i=1e3}={}){this.blobFilter=e,this.blobUploader=n,this.checkpointBuilder=r,this.uploadBatchSize=i,this.wireEvents()}get filesAccepted(){return this._filesAccepted}get filesSettled(){return this._filesSettled}get filesIndexed(){return this._filesIndexed}get filesFailed(){return this._filesFailed}get filesRejected(){return this._filesRejected}get pathToBlobName(){return this.checkpointBuilder.getPathToBlobName()}async addFile(e,n,r){if(!(await this.blobFilter.shouldAcceptBlob({path:e,size:n,readContent:r})).accepted)return;let a=await r(),o=this.blobUploader.enqueueUpload({path:e,readContent:r},a);o&&(this.blobs[o]=e,this._filesAccepted++,this._filesAccepted%this.uploadBatchSize===0&&this.blobUploader.startUpload())}removeFile(e){this.log.debug(`removeFile: ${e}`),this.checkpointBuilder.removeByPath(e)}traversalCompleted(){this.log.debug(`traversalCompleted: ${this._filesAccepted} accepted, ${this._filesSettled} settled, ${Object.keys(this.blobs).length} in-flight`),this.traversalDone=!0,this.blobUploader.startUpload(),this.finalizeIfNeeded()}onDone(e){this.doneCallback=e}scheduleSettledFire(){this.settledPending||(this.settledPending=!0,queueMicrotask(()=>{this.settledPending=!1,this.onBlobSettled.fire()}))}wireEvents(){this.disposables.push(this.blobUploader.onBlobsIndexed(e=>{for(let n of e.blobNames){let r=this.blobs[n];r&&this.checkpointBuilder.add(r,n),delete this.blobs[n],this._filesSettled++,this._filesIndexed++}this.scheduleSettledFire(),this.finalizeIfNeeded()})),this.disposables.push(this.blobUploader.onBlobsUploaded(e=>{for(let[n,r]of e.expectedToActualBlobNameMap)n!==r&&this.blobs[n]&&(this.blobs[r]=this.blobs[n],delete this.blobs[n])})),this.disposables.push(this.blobUploader.onBlobUploadFailed(e=>{delete this.blobs[e.blobName],this._filesSettled++,this._filesFailed++,this.hadFailures=!0,this.scheduleSettledFire(),this.finalizeIfNeeded()})),this.disposables.push(this.blobUploader.onBlobRejected(e=>{delete this.blobs[e.blobName],this._filesSettled++,this._filesRejected++,this.scheduleSettledFire(),this.finalizeIfNeeded()}))}finalizeIfNeeded(){if(!this.finalized&&!(!this.traversalDone||Object.keys(this.blobs).length>0)){this.finalized=!0,this.log.info("All blobs settled. Finalizing.");for(let e of this.disposables)e.dispose();this.disposables.length=0,this.checkpointBuilder.flush().then(()=>{this.doneCallback?.({hadFailures:this.hadFailures})}).catch(e=>{this.log.error("Failed to flush checkpoint",{error:e}),this.doneCallback?.({hadFailures:!0})})}}};import{execFile as age}from"node:child_process";import{readFile as EJ}from"node:fs/promises";import{join as VJ}from"node:path";import{promisify as oge}from"node:util";import{execFile as oJ,spawn as Mhe}from"node:child_process";import Lhe from"node:fs";import{promisify as Hhe}from"node:util";var ka=GA(S0(),1);var sJ=Hhe(oJ),lJ=20,cJ=200,Ou=1024*1024,Vs=class extends Error{blobSize;constructor(e,n,r){super(`Blob too large: ${e} (${(n/1024/1024).toFixed(1)} MB, max ${(r/1024/1024).toFixed(1)} MB)`),this.name="BlobTooLargeError",this.blobSize=n}};function AJ(t){return t.baseSha&&t.baseFileInfoMap?new y0(t):new F0(t)}function dJ(t){let e=ns(t);if(!e.isGitRepo||!e.dir||!e.gitdir)throw new Error(`Not a git repository: ${t}`);return{fs:Lhe,dir:e.dir,gitdir:e.gitdir,cache:{}}}function uJ(t){let e=ns(t);if(!e.isGitRepo||!e.dir||!e.gitdir)throw new Error(`Not a git repository: ${t}`);return{dir:e.dir,gitdir:e.gitdir}}function hJ(t,e,n){return new Promise((r,i)=>{oJ("git",["--git-dir",t,"cat-file","blob",n],{cwd:e,encoding:"buffer",maxBuffer:50*1024*1024},(a,o)=>{a?i(a instanceof Error?a:new Error(JSON.stringify(a))):r(new Uint8Array(o))})})}var Hb=class{process;maxBlobSize;alive=!0;buffer=Buffer.alloc(0);resolveWait=null;rejectWait=null;constructor(e,n,r){this.maxBlobSize=r??Ou,this.process=Mhe("git",["--git-dir",e,"cat-file","--batch"],{cwd:n,stdio:["pipe","pipe","pipe"]}),this.process.stdout?.on("data",i=>{if(this.buffer=Buffer.concat([this.buffer,i]),this.resolveWait){let a=this.resolveWait;this.resolveWait=null,a()}}),this.process.on("error",i=>{if(this.alive=!1,this.rejectWait){let a=this.rejectWait;this.rejectWait=null,a(i)}}),this.process.on("close",i=>{if(this.alive=!1,this.rejectWait){let a=this.rejectWait;this.rejectWait=null,a(new Error(`git cat-file process exited unexpectedly with code ${i}`))}})}get isAlive(){return this.alive}waitForBytes(e){return this.buffer.length>=e?Promise.resolve():new Promise((n,r)=>{let i=()=>{this.buffer.length>=e?(this.resolveWait=null,this.rejectWait=null,n()):this.resolveWait=i};this.resolveWait=i,this.rejectWait=r,i()})}async readBlob(e){for(this.process.stdin?.write(`${e}
|
|
239
239
|
`);;){await this.waitForBytes(1);let n=this.buffer.indexOf(10);if(n!==-1){let r=this.buffer.subarray(0,n).toString("utf8");this.buffer=this.buffer.subarray(n+1);let i=r.split(" ");if(i.length<3)throw new Error(`Unexpected cat-file header: ${r}`);let a=Number.parseInt(i[2],10);if(Number.isNaN(a))throw new Error(`Invalid size in cat-file header: ${r}`);if(a>this.maxBlobSize)throw await this.drainBytes(a+1),new Vs(e,a,this.maxBlobSize);await this.waitForBytes(a+1);let o=this.buffer.subarray(0,a),s=new Uint8Array(o);return this.buffer=this.buffer.subarray(a+1),s}await this.waitForBytes(this.buffer.length+1)}}async drainBytes(e){let n=e,r=64*1024;for(;n>0;){let i=Math.min(n,r);await this.waitForBytes(i),this.buffer=this.buffer.subarray(i),n-=i}}dispose(){if(this.alive=!1,this.rejectWait){let e=this.rejectWait;this.resolveWait=null,this.rejectWait=null,e(new Error("BlobBatchReader disposed"))}this.process.stdout?.removeAllListeners(),this.process.stderr?.removeAllListeners(),this.process.removeAllListeners(),this.process.stdin?.end(),this.process.kill(),this.process.unref()}},W0=class{config;constructor(e){this.config=e}async walk(e){let n=dJ(this.config.repoRoot),r=this.config.maxBlobSize??Ou,i=this.config.yieldMs??lJ,a=Date.now();await(0,ka.walk)({...n,trees:[(0,ka.TREE)({ref:this.config.targetSha})],iterate:async(o,s)=>{let A=[];for(let l of s)A.push(await o(l)),Date.now()-a>=i&&(await new Promise(h=>setTimeout(h,cJ)),a=Date.now());return A},map:async(o,[s])=>{if(!s||await s.type()!=="blob")return;let A=await s.oid(),l=async()=>{let{blob:u}=await(0,ka.readBlob)({...n,oid:A});if(u.byteLength>r)throw new Vs(o,u.byteLength,r);return u};await e.onFileAdded(o,l)}})}},X0=class{config;constructor(e){this.config=e}async walk(e){let n=dJ(this.config.repoRoot),r=this.config.maxBlobSize??Ou,i=this.config.yieldMs??lJ,a=Date.now(),o=this.config.baseFileInfoMap??new Map,s=(l,u)=>async()=>{let{blob:h}=await(0,ka.readBlob)({...n,oid:l});if(h.byteLength>r)throw new Vs(u,h.byteLength,r);return h},A=this.config.baseSha??this.config.targetSha;await(0,ka.walk)({...n,trees:[(0,ka.TREE)({ref:this.config.targetSha}),(0,ka.TREE)({ref:A})],iterate:async(l,u)=>{let h=[];for(let b of u)h.push(await l(b)),Date.now()-a>=i&&(await new Promise(f=>setTimeout(f,cJ)),a=Date.now());return h},map:async(l,[u,h])=>{let b=u&&await u.type()==="blob"?await u.oid():null,m=h&&await h.type()==="blob"?await h.oid():null;if(b!==m){if(b&&!m){await e.onFileAdded(l,s(b,l));return}if(!b&&m){let f=o.get(l);f&&await e.onFileDeleted(l,f);return}if(b&&m){let f=o.get(l);f?await e.onFileChanged(l,f,s(b,l)):await e.onFileAdded(l,s(b,l))}}}})}},G0=class{config;constructor(e){this.config=e}async walk(e){let{dir:n,gitdir:r}=uJ(this.config.repoRoot),i=this.config.maxBlobSize??Ou,{stdout:a}=await sJ("git",["--git-dir",r,"ls-tree","-r","-l","-z",this.config.targetSha],{cwd:n,maxBuffer:100*1024*1024}),o=a.split("\0").filter(Boolean),s=new Hb(r,n,i);try{for(let A of o){let l=A.indexOf(" ");if(l===-1)continue;let u=A.slice(0,l),h=A.slice(l+1),b=u.split(/\s+/);if(b[1]!=="blob")continue;let m=b[2],f=Number.parseInt(b[3],10);if(!Number.isNaN(f)&&f>i)continue;let E=m,R=()=>s.isAlive?s.readBlob(E):hJ(r,n,E);await e.onFileAdded(h,R)}}finally{s.dispose()}}},v0=class{config;constructor(e){this.config=e}async walk(e){let{dir:n,gitdir:r}=uJ(this.config.repoRoot),i=this.config.baseFileInfoMap??new Map,a=this.config.baseSha??this.config.targetSha,{stdout:o}=await sJ("git",["--git-dir",r,"diff-tree","-r","-z",a,this.config.targetSha],{cwd:n,maxBuffer:100*1024*1024}),s=o.split("\0"),A=this.config.maxBlobSize??Ou,l=new Hb(r,n,A),u=h=>()=>l.isAlive?l.readBlob(h):hJ(r,n,h);try{let h=0;for(;h<s.length;){let b=s[h];if(!b?.startsWith(":")){h++;continue}let m=b.slice(1).split(" "),f=m[3],E=m[4]?.[0],R=s[h+1];if(!R||!E){h+=2;continue}switch(E){case"A":{await e.onFileAdded(R,u(f)),h+=2;break}case"D":{let V=i.get(R);V&&await e.onFileDeleted(R,V),h+=2;break}case"M":case"T":{let V=i.get(R);V?await e.onFileChanged(R,V,u(f)):await e.onFileAdded(R,u(f)),h+=2;break}case"R":{let V=s[h+2],I=i.get(R);I&&await e.onFileDeleted(R,I),V&&await e.onFileAdded(V,u(f)),h+=3;break}case"C":{let V=s[h+2];V&&await e.onFileAdded(V,u(f)),h+=3;break}default:h+=2}}}finally{l.dispose()}}},gJ=Dt.child({prefix:"git-tree-walker"}),F0=class{config;constructor(e){this.config=e}async walk(e){try{await new G0(this.config).walk(e)}catch(n){gJ.info(`native git full walk failed, falling back to isomorphic-git: ${n}`),await new W0(this.config).walk(e)}}},y0=class{config;constructor(e){this.config=e}async walk(e){try{await new v0(this.config).walk(e)}catch(n){gJ.info(`native git incremental walk failed, falling back to isomorphic-git: ${n}`),await new X0(this.config).walk(e)}}};var fJ=GA(S0(),1);import{execFile as jhe}from"node:child_process";import Ohe from"node:fs";import{promisify as Phe}from"node:util";var qhe=Phe(jhe),$he=1;async function bJ(t,e,n,r,i){i.info(`getAncestorCommitShas: targetOid=${n}, depth=${r}`);try{let{stdout:A}=await qhe("git",["--git-dir",e,"log","--format=%H","--first-parent","-n",`${r}`,n],{cwd:t,timeout:1e4}),l=A.trim().split(`
|
|
240
240
|
`).filter(Boolean);if(l.length>0)return i.info(`getAncestorCommitShas via native git: ${l.length} commits`),l}catch(A){i.info(`Native git unavailable, falling back to isomorphic-git: ${A}`)}let a={fs:Ohe,dir:t,gitdir:e,cache:{}},o=[],s=n;for(let A=0;A<r;A++){let{commit:l}=await(0,fJ.readCommit)({...a,oid:s});if(o.push(s),await new Promise(u=>setTimeout(u,$he)),!l.parent||l.parent.length===0)break;s=l.parent[0]}return i.info(`getAncestorCommitShas via isomorphic-git: ${o.length} commits`),o}import{extname as ege}from"node:path";var mJ=GA(Jp(),1),tge=new Set([".png",".jpg",".jpeg",".gif",".ico",".webp",".svg",".bmp",".tiff",".psd",".ai",".mp3",".mp4",".wav",".mov",".avi",".flv",".mkv",".m4a",".aac",".ogg",".webm",".zip",".tar",".gz",".rar",".7z",".bz2",".xz",".iso",".exe",".dll",".so",".dylib",".o",".a",".lib",".bin",".app",".db",".sqlite",".sqlite3",".mdb",".dat",".class",".pyc",".pyo",".obj",".out",".woff",".woff2",".ttf",".otf",".eot",".fon",".jar",".war",".ear",".apk",".dmg",".deb",".rpm",".crt",".cer",".pem",".key",".lock"]),nge=[".git","*.pem","*.key","*.pfx","*.p12","*.jks","*.keystore","*.pkcs12","*.crt","*.cer","id_rsa","id_ed25519","id_ecdsa","id_dsa",".augment-guidelines"],rge=1024*1024,jb=class{log=Dt.child({prefix:"IgnoreRulesBlobFilter"});maxFileSize;excludedExtensions;ignoreObj;constructor(e={}){if(this.maxFileSize=e.maxFileSize??rge,this.excludedExtensions=new Set(tge),e.additionalExcludedExtensions)for(let n of e.additionalExcludedExtensions)this.excludedExtensions.add(n.startsWith(".")?n:`.${n}`);if(this.ignoreObj=(0,mJ.default)(),this.ignoreObj.add(nge),e.gitignoreContent){let n=Q0(e.gitignoreContent);this.ignoreObj.add(n)}}setAugmentIgnoreRules(e){let n=Q0(e);n.length>0&&(this.ignoreObj.add(n),this.log.info(`Added ${n.length} .augmentignore rules`))}addGitignoreRules(e,n=""){let r=Q0(e,n);r.length>0&&this.ignoreObj.add(r)}shouldAcceptBlob(e){let n=ege(e.path).toLowerCase();return this.excludedExtensions.has(n)?Promise.resolve({accepted:!1,reason:`Excluded extension: ${n}`}):e.size>this.maxFileSize?Promise.resolve({accepted:!1,reason:`File too large: ${e.size} > ${this.maxFileSize}`}):this.ignoreObj.ignores(e.path)?Promise.resolve({accepted:!1,reason:"Matched ignore rule"}):Promise.resolve({accepted:!0})}};function Q0(t,e=""){let n=t.split(`
|
|
241
|
-
`).map(i=>i.endsWith("\r")?i.slice(0,-1):i).filter(i=>i.trim()!==""&&!i.trim().startsWith("#"));if(e==="")return n;let r=e.endsWith("/")?e.slice(0,-1):e;return n.map(i=>ige(i,r))}function ige(t,e){if(t.startsWith("!")){let n=t.slice(1);return n.startsWith("/")?`!${e}${n}`:`!${e}/**/${n}`}return t.startsWith("/")?e+t:`${e}/**/${t}`}var RJ=oge(age),Ob=class{blobNameCalc;blobUploader;log=Dt.child({prefix:"IsoGitIndexingService"});onProgress=new fn;orchestrator;filesDiscovered=0;currentWorkspaceRoot;indexingStartTime;apiServer;constructor(e,n=1024*1024,r=$d){this.apiServer=e,this.blobNameCalc=new es(n),this.blobUploader=new ts(this.blobNameCalc,e,r)}dispose(){this.blobUploader.dispose(),this.onProgress.dispose()}async createBlobFilter(e){let n;try{n=await EJ(VJ(e,".gitignore"),"utf-8")}catch{}let r=new jb({gitignoreContent:n});try{let i=await EJ(VJ(e,".augmentignore"),"utf-8");r.setAugmentIgnoreRules(i)}catch{}return r}getIndexingStatus(){let e=this.orchestrator?.filesAccepted??0,n=this.orchestrator?.filesSettled??0;return{itemsInFlight:Math.max(0,e-n),calculateQueueSize:0,probeQueueSize:0,uploadQueueSize:0,probeRetryQueueSize:0,probeRetryBackoffQueueSize:0,calculateErrors:0,probeErrors:0,mtimeCacheHits:0,mtimeCacheMisses:0,pathsAccepted:e,currentFile:void 0,consecutiveNoneIndexedProbes:0,calculateAvgMs:0,calculateMaxMs:0}}emitProgress(e){let n=this.orchestrator?.filesAccepted??0,r=this.orchestrator?.filesSettled??0,i=n>0?Math.round(r/n*100):0;this.onProgress.fire({phase:e,percentage:i,filesDiscovered:this.filesDiscovered,filesTracked:n,filesProcessed:r,filesIndexed:this.orchestrator?.filesIndexed??0,filesFailed:this.orchestrator?.filesFailed??0,filesRejected:this.orchestrator?.filesRejected??0,filesPreIndexed:0,workspaceRoot:this.currentWorkspaceRoot,durationMs:e==="complete"&&this.indexingStartTime?Date.now()-this.indexingStartTime:void 0})}async findIndexedBaseCommit(e,n,r=20){let i=ns(e);if(!i.isGitRepo||!i.dir||!i.gitdir)return this.log.warn(`Not a git repository: ${e}`),null;try{let a=await bJ(i.dir,i.gitdir,n,r,this.log);if(a.length===0)return null;this.log.info(`Looking for indexed checkpoint among ${a.length} ancestor commits`);let o=this.apiServer.getLatestIndexedCommitBlobset({commit_shas:a}),s=null,A,l=new Map;for await(let u of o){u.commit_sha&&!s&&(s=u.commit_sha,A=u.checkpoint_id,this.log.info(`Found indexed base commit: ${s}`+(A?` (checkpoint: ${A})`:"")));for(let h of u.file_infos)l.set(h.file_path,h.blob_name)}return s?(this.log.info(`Collected ${l.size} file mappings from base commit`),{commitSha:s,checkpointId:A,fileInfoMap:l}):(this.log.info("No indexed checkpoint found \u2014 will do full tree walk"),null)}catch(a){return this.log.info("No indexed base commit found",{error:a}),null}}async createGitShaCommitIndex(e,n){this.log.info(`Starting indexing for commit ${n} in repo ${e}`),this.currentWorkspaceRoot=e,this.indexingStartTime=Date.now(),this.emitProgress("qualifying");let r=ns(e);if(!r.isGitRepo||!r.dir||!r.gitdir)throw new Error(`Not a git repository: ${e}`);let{dir:i,gitdir:a,headRef:o}=r,s=n==="HEAD"&&o?o:n,A;try{let{stdout:U}=await RJ("git",["--git-dir",a,"rev-parse",s],{cwd:i});A=U.trim(),this.log.info(`Resolved commit ${n} to OID ${A}`)}catch(U){throw new Error(`Failed to resolve commit ${n}: ${U}`)}let l=await this.createBlobFilter(e),u;try{let{stdout:U}=await RJ("git",["--git-dir",a,"log","-1","--format=%ct",A],{cwd:i});u=Number.parseInt(U.trim(),10)*1e3}catch(U){throw new Error(`Failed to read commit ${A}: ${U}`)}let h=await this.findIndexedBaseCommit(e,A);if(h&&h.commitSha===A)return this.log.info("Target commit already indexed, nothing to do"),this.emitProgress("complete"),{commitSha:A,filesIndexed:0,alreadyIndexed:!0,registered:!1,durationMs:Date.now()-(this.indexingStartTime??Date.now())};this.emitProgress("indexing");let b=new rs(this.apiServer,h?.checkpointId);if(h)if(h.checkpointId)this.log.info(`Resuming from checkpoint ${h.checkpointId} (${h.fileInfoMap.size} base files)`);else{let U=Array.from(h.fileInfoMap.values());this.log.info(`Seeding checkpoint with ${U.length} blobs from base commit`),b.update(U,[]),await b.flushToLimit()}let m=new is(l,this.blobUploader,b);this.orchestrator=m;let f=!1,E=new Promise(U=>{m.onDone(async S=>{if(S.hadFailures)this.log.warn("Not registering commit blobset due to failures");else try{await this.apiServer.registerIndexedCommitBlobset({commit:{commit_sha:A,commit_time:new Date(u).toISOString()},blobs:b.blobs}),f=!0,this.log.info(`Checkpoint registered: commit ${A}, ${m.filesSettled} files indexed`)}catch(W){this.log.error("Failed to register commit blobset",{error:W})}this.emitProgress("complete"),U()})}),R=m.onBlobSettled.event(()=>{this.emitProgress("indexing")}),V=()=>{this.filesDiscovered++,this.filesDiscovered%1e3===0&&this.emitProgress("indexing")},I={async onFileAdded(U,S){let W;try{W=await S()}catch(C){if(C instanceof Vs)return;throw C}await m.addFile(U,W.length,async()=>W),V()},async onFileChanged(U,S,W){let C;try{C=await W()}catch(Y){if(Y instanceof Vs)return;throw Y}await m.addFile(U,C.length,async()=>C),V()},onFileDeleted(U){m.removeFile(U)}},G=AJ({repoRoot:e,targetSha:A,baseSha:h?.commitSha,baseFileInfoMap:h?.fileInfoMap});this.log.info(h?`Incremental walk from base ${h.commitSha} to ${A}`:`Full tree walk for ${A}`),await G.walk(I),this.emitProgress("indexing"),m.traversalCompleted(),await E,R.dispose();let F=m.filesSettled,J=Date.now()-(this.indexingStartTime??Date.now());return this.log.info(`Git SHA indexing complete: ${F} files, ${(J/1e3).toFixed(1)}s, registered=${f}`),{commitSha:A,filesIndexed:F,alreadyIndexed:!1,registered:f,durationMs:J}}};import{readFile as uge}from"node:fs/promises";import{basename as hge}from"node:path";import{readdir as sge,stat as lge}from"node:fs/promises";import{join as cge,relative as pJ}from"node:path";var Age=new Set([".git","node_modules",".hg",".svn","__pycache__",".tox",".mypy_cache",".pytest_cache",".next",".nuxt",".output",".turbo",".vercel",".cache",".parcel-cache","dist","build","out","coverage",".idea",".vscode"]),dge=Dt.child({prefix:"FileWalker"});async function*BJ(t,e,n={}){let r=new Set([...Age,...n.skipDirectories??[]]),i=n.signal,a=n.shouldEnterDirectory;async function*o(s){if(i?.aborted)return;let A;try{A=await sge(s,{withFileTypes:!0})}catch(l){dge.warn(`Cannot read directory ${s}`,{error:l});return}for(let l of A){if(i?.aborted)return;let u=cge(s,l.name);if(l.isDirectory()){if(r.has(l.name))continue;if(a){let h=pJ(t,u);if(!a(h))continue}yield*o(u)}else if(l.isFile())try{let h=await lge(u);yield{relativePath:pJ(t,u),absolutePath:u,size:h.size}}catch{}}}yield*o(e)}var gge=2e5,Pb=class{blobNameCalc;blobUploader;log=Dt.child({prefix:"WorkspaceIndexingService"});onProgress=new fn;onCheckpoint=new fn;orchestrator;walkComplete=!1;filesDiscovered=0;filesPreIndexed=0;currentWorkspaceRoot;indexingStartTime;apiServer;constructor(e,n=1024*1024,r=$d){this.apiServer=e,this.blobNameCalc=new es(n),this.blobUploader=new ts(this.blobNameCalc,e,r)}dispose(){this.blobUploader.dispose(),this.onProgress.dispose()}createBlobFilter(e){let n=new Qc({workspaceRoot:e});return n.loadDirectory(""),n}getIndexingStatus(){let e=this.orchestrator?.filesAccepted??0,n=this.orchestrator?.filesSettled??0;return{itemsInFlight:Math.max(0,e-n),calculateQueueSize:0,probeQueueSize:0,uploadQueueSize:0,probeRetryQueueSize:0,probeRetryBackoffQueueSize:0,calculateErrors:0,probeErrors:0,mtimeCacheHits:0,mtimeCacheMisses:0,pathsAccepted:e,currentFile:void 0,consecutiveNoneIndexedProbes:0,calculateAvgMs:0,calculateMaxMs:0}}emitProgress(e){let n=this.orchestrator?.filesAccepted??0,r=this.orchestrator?.filesSettled??0,i=n+this.filesPreIndexed,a=r+this.filesPreIndexed,o=i>0?Math.round(a/i*100):0;this.onProgress.fire({phase:e,percentage:o,filesDiscovered:this.filesDiscovered,filesTracked:n,filesProcessed:r,filesIndexed:this.orchestrator?.filesIndexed??0,filesFailed:this.orchestrator?.filesFailed??0,filesRejected:this.orchestrator?.filesRejected??0,filesPreIndexed:this.filesPreIndexed,walkComplete:this.walkComplete,workspaceRoot:this.currentWorkspaceRoot,durationMs:e==="complete"&&this.indexingStartTime?Date.now()-this.indexingStartTime:void 0})}async indexWorkspace(e,n,r={}){this.log.info(`Starting workspace indexing for ${n}`+(n===e?"":` (workspace root: ${e})`)),this.walkComplete=!1,this.filesDiscovered=0,this.filesPreIndexed=0,this.currentWorkspaceRoot=e,this.indexingStartTime=Date.now(),this.emitProgress("qualifying");let i=this.createBlobFilter(e),a=r.baselineCheckpointId,o=new rs(this.apiServer,a),s=!1,A=o.onCheckpoint.event(F=>{s=!0,this.onCheckpoint.fire(F)}),l=new is(i,this.blobUploader,o);this.orchestrator=l;let u=r.preIndexedFiles;if(u&&u.size>0)if(o.seed(u),this.filesPreIndexed=u.size,a)this.log.info(`Resuming from checkpoint ${a} (${u.size} pre-indexed files)`);else{let F=Array.from(u.values());this.log.info(`Seeding checkpoint with ${F.length} pre-indexed blobs`),o.update(F,[]),await o.flushToLimit()}let h=new Promise((F,J)=>{l.onDone(U=>{U.hadFailures&&this.log.warn("Workspace indexing completed with failures"),this.emitProgress("complete"),F()})}),b=l.onBlobSettled.event(()=>{this.emitProgress("indexing")});this.emitProgress("indexing");let m={skipDirectories:r.skipDirectories,signal:r.signal,shouldEnterDirectory:F=>(i.loadDirectory(F),!i.isDirectoryIgnored(F))},f=r.maxTrackableFiles??gge,E=0;for await(let F of BJ(e,n,m)){if(r.signal?.aborted)break;if(l.filesAccepted>=f){this.log.warn(`Workspace exceeds ${f} trackable files, stopping walk`),this.emitProgress("workspace-too-large");break}this.filesDiscovered++;let J=hge(F.relativePath);if(J===".gitignore"||J===".augmentignore"){i.updateIgnoreFile(F.relativePath);continue}if(u?.has(F.relativePath)){E++;continue}await l.addFile(F.relativePath,F.size,()=>uge(F.absolutePath)),this.filesDiscovered%1e3===0&&this.emitProgress("indexing")}E>0&&this.log.info(`Skipped ${E} pre-indexed files (incremental)`),this.walkComplete=!0,this.emitProgress("indexing"),l.traversalCompleted(),await h,b.dispose(),o.flushCheckpointEvent();let R=o.blobs.checkpointId;!s&&R&&this.onCheckpoint.fire({checkpointId:R,addedBlobs:[],deletedBlobs:[]}),A.dispose();let V=Date.now()-(this.indexingStartTime??Date.now()),I=l.filesAccepted,G=l.filesSettled;return this.log.info(`Workspace indexing complete: ${I} tracked, ${G} processed, ${E} skipped (pre-indexed), ${(V/1e3).toFixed(1)}s, checkpoint: ${R??"none"}`),{checkpointId:R,filesTracked:I,filesProcessed:G,filesSkipped:E,durationMs:V,pathToBlobName:l.pathToBlobName}}};import{watch as IJ}from"node:fs";import{readdir as fge,stat as bge}from"node:fs/promises";import{join as ZJ,relative as N0}from"node:path";var mge=500,Ege=new Set([".git","node_modules",".hg",".svn","__pycache__",".tox",".mypy_cache",".pytest_cache",".next",".nuxt",".output",".turbo",".vercel",".cache",".parcel-cache"]),qb=Dt.child({prefix:"FileWatcher"}),$b=class{workspaceRoot;debounceMs;skipDirs;shouldWatch;onChange=new fn;watchers=[];pendingEvents=new Map;debounceTimer=null;running=!1;constructor(e){this.workspaceRoot=e.workspaceRoot,this.debounceMs=e.debounceMs??mge,this.skipDirs=new Set([...Ege,...e.skipDirectories??[]]),this.shouldWatch=e.shouldWatchDirectory}async start(){if(this.running)return;this.running=!0;let e=process.platform==="darwin"||process.platform==="win32";e?this.startRecursive():await this.startPerDirectory(this.workspaceRoot),qb.info(`Started watching ${this.workspaceRoot} (${e?"recursive":"per-directory"}, ${this.watchers.length} watcher(s))`)}stop(){if(this.running){this.running=!1;for(let e of this.watchers)e.close();this.watchers=[],this.flush(),qb.info("Stopped watching")}}startRecursive(){let e=IJ(this.workspaceRoot,{recursive:!0},(n,r)=>{if(!r||!this.running)return;let i=r.toString();this.shouldSkip(i)||this.enqueue(i,n==="rename"?"rename":"change")});e.on("error",n=>{qb.warn("Watcher error",{error:n})}),this.watchers.push(e)}async startPerDirectory(e){if(!this.running)return;let n=N0(this.workspaceRoot,e);if(n&&this.shouldSkipDir(n))return;this.watchSingleDirectory(e);let r;try{r=await fge(e,{withFileTypes:!0})}catch{return}let i=[];for(let a of r)if(a.isDirectory()&&!this.skipDirs.has(a.name)){let o=n?`${n}/${a.name}`:a.name;if(this.shouldWatch&&!this.shouldWatch(o))continue;i.push(this.startPerDirectory(ZJ(e,a.name)))}await Promise.all(i)}watchSingleDirectory(e){let n=IJ(e,(r,i)=>{if(!i||!this.running)return;let a=ZJ(e,i.toString()),o=N0(this.workspaceRoot,a);this.shouldSkip(o)||(this.enqueue(o,r==="rename"?"rename":"change"),r==="rename"&&this.maybeWatchNewDirectory(a))});n.on("error",r=>{qb.warn(`Watcher error on ${e}`,{error:r})}),this.watchers.push(n)}maybeWatchNewDirectory(e){bge(e).then(n=>{if(n.isDirectory()&&this.running){let r=N0(this.workspaceRoot,e);if(this.shouldSkipDir(r)||this.shouldWatch&&!this.shouldWatch(r))return;this.watchSingleDirectory(e)}}).catch(()=>{})}shouldSkip(e){let n=e.split("/");if(n.some(r=>this.skipDirs.has(r)))return!0;if(this.shouldWatch&&n.length>1){let r=n.slice(0,-1).join("/");if(!this.shouldWatch(r))return!0}return!1}shouldSkipDir(e){return e.split("/").some(r=>this.skipDirs.has(r))}enqueue(e,n){this.pendingEvents.set(e,n),this.debounceTimer&&clearTimeout(this.debounceTimer),this.debounceTimer=setTimeout(()=>this.flush(),this.debounceMs)}flush(){if(this.debounceTimer&&(clearTimeout(this.debounceTimer),this.debounceTimer=null),this.pendingEvents.size===0)return;let e=[];for(let[n,r]of this.pendingEvents)e.push({type:r,relativePath:n});this.pendingEvents.clear(),this.onChange.fire(e)}};import{readFile as Vge,stat as Rge}from"node:fs/promises";import{basename as pge,join as Bge}from"node:path";var Ige=new Set([".gitignore",".augmentignore"]),CJ=Dt.child({prefix:"WatcherProcessor"}),em=class{workspaceRoot;blobFilter;orchestrator;subscription=null;onUpdate=new fn;constructor(e){this.workspaceRoot=e.workspaceRoot,this.blobFilter=e.blobFilter,this.orchestrator=e.orchestrator}attach(e){this.detach(),this.subscription=e.onChange.event(n=>{this.processBatch(n).catch(r=>{CJ.error("Failed to process watcher batch",{error:r})})})}detach(){this.subscription?.dispose(),this.subscription=null}dispose(){this.detach(),this.onUpdate.dispose()}async processBatch(e){let n=[],r=[];for(let o of e){let s=pge(o.relativePath);Ige.has(s)?n.push(o):r.push(o)}for(let o of n)this.blobFilter.updateIgnoreFile(o.relativePath);if(r.length===0)return;let i=0,a=0;for(let o of r){let s=Bge(this.workspaceRoot,o.relativePath),A=null;try{A=await Rge(s)}catch{}if(!A?.isFile()){this.orchestrator.removeFile(o.relativePath),a++;continue}await this.orchestrator.addFile(o.relativePath,A.size,()=>Vge(s)),i++}(i>0||a>0)&&(CJ.info(`Processed batch: ${i} added, ${a} deleted`),this.onUpdate.fire({checkpointId:this.orchestrator.checkpointBuilder.blobs.checkpointId}))}};var tm=class{gitService;workspaceService;deltaResolver;apiServer;maxBlobSize;log=Dt.child({prefix:"ContextIndexer"});fileWatcher=null;watcherProcessor=null;onIndexing=new fn;onCheckpoint=new fn;onWatcherUpdate=new fn;constructor(e,n={}){this.apiServer=e,this.maxBlobSize=n.maxBlobSize??1024*1024,this.gitService=new Ob(e,n.maxBlobSize),this.workspaceService=new Pb(e,n.maxBlobSize),this.deltaResolver=new Mf(e,n.deltaResolverOptions),this.workspaceService.onProgress.event(r=>{this.onIndexing.fire(r)}),this.workspaceService.onCheckpoint.event(r=>{this.onCheckpoint.fire(r)})}getGitRepoDetails(e){return ns(e)}async indexWorkspace(e,n,r){let i=WJ(e),a=WJ(n),o=$y(i),s,A,l=null;this.log.info("Looking for previously indexed git SHA...");try{let h=o?await this.deltaResolver.resolveIndexedCommitDelta(o):null;if(h&&o){this.log.info(`Found indexed commit ${h.indexedCommitSha}: ${h.fileInfoMap.size} unchanged, ${h.modifiedFiles.size} modified, ${h.addedFiles.size} added, ${h.deletedFiles.size} deleted, ${h.untrackedFiles.size} untracked`);let b=o===i?"":`${SJ(o,i)}/`,m=a===i?b:`${SJ(o,a)}/`,f=new Map;for(let[E,R]of h.fileInfoMap)E.startsWith(m)&&f.set(E,R);l={commitSha:h.indexedCommitSha,fileCount:f.size},f.size!==h.fileInfoMap.size&&this.log.info(`Filtered delta to ${f.size} files under ${m} (was ${h.fileInfoMap.size})`),s=f,o===i&&a===i&&(A=h.checkpointId)}else this.log.info(o?"No previously indexed commit found \u2014 doing full walk":"Not inside a git repository \u2014 doing full walk")}catch(h){this.log.warn("Delta resolution failed, falling back to full walk",h)}return{...await this.workspaceService.indexWorkspace(e,n,{...r,preIndexedFiles:s,baselineCheckpointId:A}),baseline:l}}createGitShaCommitIndex(e,n){return this.gitService.createGitShaCommitIndex(e,n)}getIndexingStatus(){return this.gitService.getIndexingStatus()}async startWatcher(e,n){this.stopWatcher();let r=new es(this.maxBlobSize),i=new ts(r,this.apiServer),a=new rs(this.apiServer);n&&a.seed(n);let o=new Qc({workspaceRoot:e});o.loadDirectory("");let s=new is(o,i,a);this.fileWatcher=new $b({workspaceRoot:e,shouldWatchDirectory:A=>(o.loadDirectory(A),!o.isDirectoryIgnored(A))}),this.watcherProcessor=new em({workspaceRoot:e,blobFilter:o,orchestrator:s}),this.watcherProcessor.onUpdate.event(A=>{this.onWatcherUpdate.fire(A)}),a.onCheckpoint.event(A=>{this.onCheckpoint.fire(A)}),this.watcherProcessor.attach(this.fileWatcher),await this.fileWatcher.start(),this.log.info(`File watcher started for ${e}`)}stopWatcher(){this.fileWatcher?.stop(),this.watcherProcessor?.dispose(),this.fileWatcher=null,this.watcherProcessor=null}dispose(){this.stopWatcher(),this.gitService.dispose(),this.workspaceService.dispose(),this.onIndexing.dispose(),this.onCheckpoint.dispose(),this.onWatcherUpdate.dispose()}};if(!GJ)throw new Error("indexing-worker.ts must be run as a worker thread");var im=GJ,Y0=new Zge;process.on("uncaughtException",t=>{im.postMessage({type:"fatalError",message:t.message,stack:t.stack,errorType:"uncaughtException"})});process.on("unhandledRejection",t=>{let e=t instanceof Error?t:new Error(String(t));im.postMessage({type:"fatalError",message:e.message,stack:e.stack,errorType:"unhandledRejection"})});var rm=null,Il=new Map,nm=new Map,Pu=new Map,AA=null,Cge=2e3;function Ii(t){im.postMessage(t)}function Zl(t,e,n,r){Ii({type:"log",workspaceId:r??Y0.getStore(),level:t,prefix:e,message:n,timestamp:Date.now()})}function XJ(t){let e=Il.get(t);if(e)return e;if(!rm)throw new Error("Worker not initialized \u2014 send 'configure' first");let n=rm();return n.onIndexing.event(r=>{nm.set(t,r),Ii({type:"progress",workspaceId:t,progress:r})}),n.onCheckpoint.event(r=>{Ii({type:"checkpointUpdate",workspaceId:t,checkpoint:r})}),n.onWatcherUpdate.event(r=>{Ii({type:"watcherUpdate",workspaceId:t,checkpointId:r.checkpointId})}),Il.set(t,n),n}Hy();Dt.on("data",t=>{let e=t.level??"info";Zl(e,t.prefix??"worker",String(t.message??""),Y0.getStore())});function Sge(){AA||(AA=setInterval(()=>{let t=process.memoryUsage();Ii({type:"metrics",heapUsedMb:Math.round(t.heapUsed/1024/1024),rssMb:Math.round(t.rss/1024/1024),activeHandles:process._getActiveHandles?.().length??0,timestamp:Date.now()})},Cge),AA.unref())}function Wge(){AA&&(clearInterval(AA),AA=null)}async function Xge(t){switch(t.type){case"configure":{let{apiToken:e,apiUrl:n,indexerOptions:r}=t.config;Zl("info","IndexingWorker",`Creating API client for ${n}`);let i=mX({apiToken:e,apiUrl:n}),a=new zf(i,n,e);rm=()=>new tm(a,r),Sge(),Zl("info","IndexingWorker","Configured successfully"),Ii({type:"result",requestId:t.requestId});break}case"indexWorkspace":{let e=XJ(t.workspaceId),n=t.skipDirectories?new Set(t.skipDirectories):void 0,r=t.folderToIndex??t.workspaceRoot,i=await e.indexWorkspace(t.workspaceRoot,r,{skipDirectories:n}),{pathToBlobName:a,...o}=i;try{let s=await e.createGitShaCommitIndex(t.workspaceRoot,"HEAD");Zl("info","IndexingWorker",`Git SHA commit index created: ${s.commitSha}`)}catch(s){Zl("warn","IndexingWorker",`Git SHA commit index failed (non-fatal): ${s}`)}Zl("info","IndexingWorker","Starting file watcher"),await e.startWatcher(t.workspaceRoot,a),Ii({type:"result",requestId:t.requestId,data:o});break}case"createGitShaCommitIndex":{let n=await XJ(t.workspaceId).createGitShaCommitIndex(t.repoRoot,t.revision);Ii({type:"result",requestId:t.requestId,data:n});break}case"statusRequest":{let e=Il.get(t.workspaceId)?.getIndexingStatus()??{itemsInFlight:0,calculateQueueSize:0,probeQueueSize:0,uploadQueueSize:0,probeRetryQueueSize:0,probeRetryBackoffQueueSize:0,calculateErrors:0,probeErrors:0,mtimeCacheHits:0,mtimeCacheMisses:0,pathsAccepted:0,currentFile:void 0,consecutiveNoneIndexedProbes:0,calculateAvgMs:0,calculateMaxMs:0};Ii({type:"statusResponse",requestId:t.requestId,workspaceId:t.workspaceId,status:e,progress:nm.get(t.workspaceId)??null});break}case"dispose":{Il.get(t.workspaceId)?.dispose(),Il.delete(t.workspaceId),nm.delete(t.workspaceId),Ii({type:"result",requestId:t.requestId});break}case"shutdown":{for(let e of Il.values())e.dispose();Il.clear(),nm.clear(),Pu.clear(),rm=null,Wge(),Ii({type:"result",requestId:t.requestId}),setTimeout(()=>process.exit(0),100);break}}}im.on("message",t=>{let e="workspaceId"in t?t.workspaceId:void 0,n=async()=>{try{await Xge(t)}catch(a){let o=a instanceof Error?a:new Error(String(a));Ii({type:"error",requestId:t.requestId,message:o.message,stack:o.stack})}};if(!e){n();return}let i=(Pu.get(e)??Promise.resolve()).then(()=>Y0.run(e,n));Pu.set(e,i),i.finally(()=>{Pu.get(e)===i&&Pu.delete(e)})});Ii({type:"ready"});Zl("info","IndexingWorker","single worker indexing enabled");
|
|
241
|
+
`).map(i=>i.endsWith("\r")?i.slice(0,-1):i).filter(i=>i.trim()!==""&&!i.trim().startsWith("#"));if(e==="")return n;let r=e.endsWith("/")?e.slice(0,-1):e;return n.map(i=>ige(i,r))}function ige(t,e){if(t.startsWith("!")){let n=t.slice(1);return n.startsWith("/")?`!${e}${n}`:`!${e}/**/${n}`}return t.startsWith("/")?e+t:`${e}/**/${t}`}var RJ=oge(age),Ob=class{blobNameCalc;blobUploader;log=Dt.child({prefix:"IsoGitIndexingService"});onProgress=new fn;orchestrator;filesDiscovered=0;currentWorkspaceRoot;indexingStartTime;apiServer;constructor(e,n=1024*1024,r=$d){this.apiServer=e,this.blobNameCalc=new es(n),this.blobUploader=new ts(this.blobNameCalc,e,r)}dispose(){this.blobUploader.dispose(),this.onProgress.dispose()}async createBlobFilter(e){let n;try{n=await EJ(VJ(e,".gitignore"),"utf-8")}catch{}let r=new jb({gitignoreContent:n});try{let i=await EJ(VJ(e,".augmentignore"),"utf-8");r.setAugmentIgnoreRules(i)}catch{}return r}getIndexingStatus(){let e=this.orchestrator?.filesAccepted??0,n=this.orchestrator?.filesSettled??0;return{itemsInFlight:Math.max(0,e-n),calculateQueueSize:0,probeQueueSize:0,uploadQueueSize:0,probeRetryQueueSize:0,probeRetryBackoffQueueSize:0,calculateErrors:0,probeErrors:0,mtimeCacheHits:0,mtimeCacheMisses:0,pathsAccepted:e,currentFile:void 0,consecutiveNoneIndexedProbes:0,calculateAvgMs:0,calculateMaxMs:0}}emitProgress(e){let n=this.orchestrator?.filesAccepted??0,r=this.orchestrator?.filesSettled??0,i=n>0?Math.round(r/n*100):0;this.onProgress.fire({phase:e,percentage:i,filesDiscovered:this.filesDiscovered,filesTracked:n,filesProcessed:r,filesIndexed:this.orchestrator?.filesIndexed??0,filesFailed:this.orchestrator?.filesFailed??0,filesRejected:this.orchestrator?.filesRejected??0,filesPreIndexed:0,workspaceRoot:this.currentWorkspaceRoot,durationMs:e==="complete"&&this.indexingStartTime?Date.now()-this.indexingStartTime:void 0})}async findIndexedBaseCommit(e,n,r=20){let i=ns(e);if(!i.isGitRepo||!i.dir||!i.gitdir)return this.log.warn(`Not a git repository: ${e}`),null;try{let a=await bJ(i.dir,i.gitdir,n,r,this.log);if(a.length===0)return null;this.log.info(`Looking for indexed checkpoint among ${a.length} ancestor commits`);let o=this.apiServer.getLatestIndexedCommitBlobset({commit_shas:a}),s=null,A,l=new Map;for await(let u of o){u.commit_sha&&!s&&(s=u.commit_sha,A=u.checkpoint_id,this.log.info(`Found indexed base commit: ${s}`+(A?` (checkpoint: ${A})`:"")));for(let h of u.file_infos)l.set(h.file_path,h.blob_name)}return s?(this.log.info(`Collected ${l.size} file mappings from base commit`),{commitSha:s,checkpointId:A,fileInfoMap:l}):(this.log.info("No indexed checkpoint found \u2014 will do full tree walk"),null)}catch(a){return this.log.info("No indexed base commit found",{error:a}),null}}async createGitShaCommitIndex(e,n){this.log.info(`Starting indexing for commit ${n} in repo ${e}`),this.currentWorkspaceRoot=e,this.indexingStartTime=Date.now(),this.emitProgress("qualifying");let r=ns(e);if(!r.isGitRepo||!r.dir||!r.gitdir)throw new Error(`Not a git repository: ${e}`);let{dir:i,gitdir:a,headRef:o}=r,s=n==="HEAD"&&o?o:n,A;try{let{stdout:U}=await RJ("git",["--git-dir",a,"rev-parse",s],{cwd:i});A=U.trim(),this.log.info(`Resolved commit ${n} to OID ${A}`)}catch(U){throw new Error(`Failed to resolve commit ${n}: ${U}`)}let l=await this.createBlobFilter(e),u;try{let{stdout:U}=await RJ("git",["--git-dir",a,"log","-1","--format=%ct",A],{cwd:i});u=Number.parseInt(U.trim(),10)*1e3}catch(U){throw new Error(`Failed to read commit ${A}: ${U}`)}let h=await this.findIndexedBaseCommit(e,A);if(h&&h.commitSha===A)return this.log.info("Target commit already indexed, nothing to do"),this.emitProgress("complete"),{commitSha:A,filesIndexed:0,alreadyIndexed:!0,registered:!1,durationMs:Date.now()-(this.indexingStartTime??Date.now())};this.emitProgress("indexing");let b=new rs(this.apiServer,h?.checkpointId);if(h)if(h.checkpointId)this.log.info(`Resuming from checkpoint ${h.checkpointId} (${h.fileInfoMap.size} base files)`);else{let U=Array.from(h.fileInfoMap.values());this.log.info(`Seeding checkpoint with ${U.length} blobs from base commit`),b.update(U,[]),await b.flushToLimit()}let m=new is(l,this.blobUploader,b);this.orchestrator=m;let f=!1,E=new Promise(U=>{m.onDone(async S=>{if(S.hadFailures)this.log.warn("Not registering commit blobset due to failures");else try{await this.apiServer.registerIndexedCommitBlobset({commit:{commit_sha:A,commit_time:new Date(u).toISOString()},blobs:b.blobs}),f=!0,this.log.info(`Checkpoint registered: commit ${A}, ${m.filesSettled} files indexed`)}catch(W){this.log.error("Failed to register commit blobset",{error:W})}this.emitProgress("complete"),U()})}),R=m.onBlobSettled.event(()=>{this.emitProgress("indexing")}),V=()=>{this.filesDiscovered++,this.filesDiscovered%1e3===0&&this.emitProgress("indexing")},I={async onFileAdded(U,S){let W;try{W=await S()}catch(C){if(C instanceof Vs)return;throw C}await m.addFile(U,W.length,async()=>W),V()},async onFileChanged(U,S,W){let C;try{C=await W()}catch(Y){if(Y instanceof Vs)return;throw Y}await m.addFile(U,C.length,async()=>C),V()},onFileDeleted(U){m.removeFile(U)}},G=AJ({repoRoot:e,targetSha:A,baseSha:h?.commitSha,baseFileInfoMap:h?.fileInfoMap});this.log.info(h?`Incremental walk from base ${h.commitSha} to ${A}`:`Full tree walk for ${A}`),await G.walk(I),this.emitProgress("indexing"),m.traversalCompleted(),await E,R.dispose();let F=m.filesSettled,J=Date.now()-(this.indexingStartTime??Date.now());return this.log.info(`Git SHA indexing complete: ${F} files, ${(J/1e3).toFixed(1)}s, registered=${f}`),{commitSha:A,filesIndexed:F,alreadyIndexed:!1,registered:f,durationMs:J}}};import{readFile as uge}from"node:fs/promises";import{basename as hge}from"node:path";import{readdir as sge,stat as lge}from"node:fs/promises";import{join as cge,relative as pJ}from"node:path";var Age=new Set([".git","node_modules",".hg",".svn","__pycache__",".tox",".mypy_cache",".pytest_cache",".next",".nuxt",".output",".turbo",".vercel",".cache",".parcel-cache","dist","build","out","coverage",".idea",".vscode"]),dge=Dt.child({prefix:"FileWalker"});async function*BJ(t,e,n={}){let r=new Set([...Age,...n.skipDirectories??[]]),i=n.signal,a=n.shouldEnterDirectory;async function*o(s){if(i?.aborted)return;let A;try{A=await sge(s,{withFileTypes:!0})}catch(l){dge.warn(`Cannot read directory ${s}`,{error:l});return}for(let l of A){if(i?.aborted)return;let u=cge(s,l.name);if(l.isDirectory()){if(r.has(l.name))continue;if(a){let h=pJ(t,u);if(!a(h))continue}yield*o(u)}else if(l.isFile())try{let h=await lge(u);yield{relativePath:pJ(t,u),absolutePath:u,size:h.size}}catch{}}}yield*o(e)}var gge=2e5,Pb=class{blobNameCalc;blobUploader;log=Dt.child({prefix:"WorkspaceIndexingService"});onProgress=new fn;onCheckpoint=new fn;orchestrator;walkComplete=!1;filesDiscovered=0;filesPreIndexed=0;currentWorkspaceRoot;indexingStartTime;apiServer;constructor(e,n=1024*1024,r=$d){this.apiServer=e,this.blobNameCalc=new es(n),this.blobUploader=new ts(this.blobNameCalc,e,r)}dispose(){this.blobUploader.dispose(),this.onProgress.dispose()}createBlobFilter(e){let n=new Qc({workspaceRoot:e});return n.loadDirectory(""),n}getIndexingStatus(){let e=this.orchestrator?.filesAccepted??0,n=this.orchestrator?.filesSettled??0;return{itemsInFlight:Math.max(0,e-n),calculateQueueSize:0,probeQueueSize:0,uploadQueueSize:0,probeRetryQueueSize:0,probeRetryBackoffQueueSize:0,calculateErrors:0,probeErrors:0,mtimeCacheHits:0,mtimeCacheMisses:0,pathsAccepted:e,currentFile:void 0,consecutiveNoneIndexedProbes:0,calculateAvgMs:0,calculateMaxMs:0}}emitProgress(e){let n=this.orchestrator?.filesAccepted??0,r=this.orchestrator?.filesSettled??0,i=n+this.filesPreIndexed,a=r+this.filesPreIndexed,o=i>0?Math.round(a/i*100):0;this.onProgress.fire({phase:e,percentage:o,filesDiscovered:this.filesDiscovered,filesTracked:n,filesProcessed:r,filesIndexed:this.orchestrator?.filesIndexed??0,filesFailed:this.orchestrator?.filesFailed??0,filesRejected:this.orchestrator?.filesRejected??0,filesPreIndexed:this.filesPreIndexed,walkComplete:this.walkComplete,workspaceRoot:this.currentWorkspaceRoot,durationMs:e==="complete"&&this.indexingStartTime?Date.now()-this.indexingStartTime:void 0})}async indexWorkspace(e,n,r={}){this.log.info(`Starting workspace indexing for ${n}`+(n===e?"":` (workspace root: ${e})`)),this.walkComplete=!1,this.filesDiscovered=0,this.filesPreIndexed=0,this.currentWorkspaceRoot=e,this.indexingStartTime=Date.now(),this.emitProgress("qualifying");let i=this.createBlobFilter(e),a=r.baselineCheckpointId,o=new rs(this.apiServer,a),s=!1,A=o.onCheckpoint.event(F=>{s=!0,this.onCheckpoint.fire(F)}),l=new is(i,this.blobUploader,o);this.orchestrator=l;let u=r.preIndexedFiles;if(u&&u.size>0)if(o.seed(u),this.filesPreIndexed=u.size,a)this.log.info(`Resuming from checkpoint ${a} (${u.size} pre-indexed files)`),s=!0,this.onCheckpoint.fire({checkpointId:a,addedBlobs:[],deletedBlobs:[]});else{let F=Array.from(u.values());this.log.info(`Seeding checkpoint with ${F.length} pre-indexed blobs`),o.update(F,[]),await o.flushToLimit()}let h=new Promise((F,J)=>{l.onDone(U=>{U.hadFailures&&this.log.warn("Workspace indexing completed with failures"),this.emitProgress("complete"),F()})}),b=l.onBlobSettled.event(()=>{this.emitProgress("indexing")});this.emitProgress("indexing");let m={skipDirectories:r.skipDirectories,signal:r.signal,shouldEnterDirectory:F=>(i.loadDirectory(F),!i.isDirectoryIgnored(F))},f=r.maxTrackableFiles??gge,E=0;for await(let F of BJ(e,n,m)){if(r.signal?.aborted)break;if(l.filesAccepted>=f){this.log.warn(`Workspace exceeds ${f} trackable files, stopping walk`),this.emitProgress("workspace-too-large");break}this.filesDiscovered++;let J=hge(F.relativePath);if(J===".gitignore"||J===".augmentignore"){i.updateIgnoreFile(F.relativePath);continue}if(u?.has(F.relativePath)){E++;continue}await l.addFile(F.relativePath,F.size,()=>uge(F.absolutePath)),this.filesDiscovered%1e3===0&&this.emitProgress("indexing")}E>0&&this.log.info(`Skipped ${E} pre-indexed files (incremental)`),this.walkComplete=!0,this.emitProgress("indexing"),l.traversalCompleted(),await h,b.dispose(),o.flushCheckpointEvent();let R=o.blobs.checkpointId;!s&&R&&this.onCheckpoint.fire({checkpointId:R,addedBlobs:[],deletedBlobs:[]}),A.dispose();let V=Date.now()-(this.indexingStartTime??Date.now()),I=l.filesAccepted,G=l.filesSettled;return this.log.info(`Workspace indexing complete: ${I} tracked, ${G} processed, ${E} skipped (pre-indexed), ${(V/1e3).toFixed(1)}s, checkpoint: ${R??"none"}`),{checkpointId:R,filesTracked:I,filesProcessed:G,filesSkipped:E,durationMs:V,pathToBlobName:l.pathToBlobName}}};import{watch as IJ}from"node:fs";import{readdir as fge,stat as bge}from"node:fs/promises";import{join as ZJ,relative as N0}from"node:path";var mge=500,Ege=new Set([".git","node_modules",".hg",".svn","__pycache__",".tox",".mypy_cache",".pytest_cache",".next",".nuxt",".output",".turbo",".vercel",".cache",".parcel-cache"]),qb=Dt.child({prefix:"FileWatcher"}),$b=class{workspaceRoot;debounceMs;skipDirs;shouldWatch;onChange=new fn;watchers=[];pendingEvents=new Map;debounceTimer=null;running=!1;constructor(e){this.workspaceRoot=e.workspaceRoot,this.debounceMs=e.debounceMs??mge,this.skipDirs=new Set([...Ege,...e.skipDirectories??[]]),this.shouldWatch=e.shouldWatchDirectory}async start(){if(this.running)return;this.running=!0;let e=process.platform==="darwin"||process.platform==="win32";e?this.startRecursive():await this.startPerDirectory(this.workspaceRoot),qb.info(`Started watching ${this.workspaceRoot} (${e?"recursive":"per-directory"}, ${this.watchers.length} watcher(s))`)}stop(){if(this.running){this.running=!1;for(let e of this.watchers)e.close();this.watchers=[],this.flush(),qb.info("Stopped watching")}}startRecursive(){let e=IJ(this.workspaceRoot,{recursive:!0},(n,r)=>{if(!r||!this.running)return;let i=r.toString();this.shouldSkip(i)||this.enqueue(i,n==="rename"?"rename":"change")});e.on("error",n=>{qb.warn("Watcher error",{error:n})}),this.watchers.push(e)}async startPerDirectory(e){if(!this.running)return;let n=N0(this.workspaceRoot,e);if(n&&this.shouldSkipDir(n))return;this.watchSingleDirectory(e);let r;try{r=await fge(e,{withFileTypes:!0})}catch{return}let i=[];for(let a of r)if(a.isDirectory()&&!this.skipDirs.has(a.name)){let o=n?`${n}/${a.name}`:a.name;if(this.shouldWatch&&!this.shouldWatch(o))continue;i.push(this.startPerDirectory(ZJ(e,a.name)))}await Promise.all(i)}watchSingleDirectory(e){let n=IJ(e,(r,i)=>{if(!i||!this.running)return;let a=ZJ(e,i.toString()),o=N0(this.workspaceRoot,a);this.shouldSkip(o)||(this.enqueue(o,r==="rename"?"rename":"change"),r==="rename"&&this.maybeWatchNewDirectory(a))});n.on("error",r=>{qb.warn(`Watcher error on ${e}`,{error:r})}),this.watchers.push(n)}maybeWatchNewDirectory(e){bge(e).then(n=>{if(n.isDirectory()&&this.running){let r=N0(this.workspaceRoot,e);if(this.shouldSkipDir(r)||this.shouldWatch&&!this.shouldWatch(r))return;this.watchSingleDirectory(e)}}).catch(()=>{})}shouldSkip(e){let n=e.split("/");if(n.some(r=>this.skipDirs.has(r)))return!0;if(this.shouldWatch&&n.length>1){let r=n.slice(0,-1).join("/");if(!this.shouldWatch(r))return!0}return!1}shouldSkipDir(e){return e.split("/").some(r=>this.skipDirs.has(r))}enqueue(e,n){this.pendingEvents.set(e,n),this.debounceTimer&&clearTimeout(this.debounceTimer),this.debounceTimer=setTimeout(()=>this.flush(),this.debounceMs)}flush(){if(this.debounceTimer&&(clearTimeout(this.debounceTimer),this.debounceTimer=null),this.pendingEvents.size===0)return;let e=[];for(let[n,r]of this.pendingEvents)e.push({type:r,relativePath:n});this.pendingEvents.clear(),this.onChange.fire(e)}};import{readFile as Vge,stat as Rge}from"node:fs/promises";import{basename as pge,join as Bge}from"node:path";var Ige=new Set([".gitignore",".augmentignore"]),CJ=Dt.child({prefix:"WatcherProcessor"}),em=class{workspaceRoot;blobFilter;orchestrator;subscription=null;onUpdate=new fn;constructor(e){this.workspaceRoot=e.workspaceRoot,this.blobFilter=e.blobFilter,this.orchestrator=e.orchestrator}attach(e){this.detach(),this.subscription=e.onChange.event(n=>{this.processBatch(n).catch(r=>{CJ.error("Failed to process watcher batch",{error:r})})})}detach(){this.subscription?.dispose(),this.subscription=null}dispose(){this.detach(),this.onUpdate.dispose()}async processBatch(e){let n=[],r=[];for(let o of e){let s=pge(o.relativePath);Ige.has(s)?n.push(o):r.push(o)}for(let o of n)this.blobFilter.updateIgnoreFile(o.relativePath);if(r.length===0)return;let i=0,a=0;for(let o of r){let s=Bge(this.workspaceRoot,o.relativePath),A=null;try{A=await Rge(s)}catch{}if(!A?.isFile()){this.orchestrator.removeFile(o.relativePath),a++;continue}await this.orchestrator.addFile(o.relativePath,A.size,()=>Vge(s)),i++}(i>0||a>0)&&(CJ.info(`Processed batch: ${i} added, ${a} deleted`),this.onUpdate.fire({checkpointId:this.orchestrator.checkpointBuilder.blobs.checkpointId}))}};var tm=class{gitService;workspaceService;deltaResolver;apiServer;maxBlobSize;log=Dt.child({prefix:"ContextIndexer"});fileWatcher=null;watcherProcessor=null;onIndexing=new fn;onCheckpoint=new fn;onWatcherUpdate=new fn;constructor(e,n={}){this.apiServer=e,this.maxBlobSize=n.maxBlobSize??1024*1024,this.gitService=new Ob(e,n.maxBlobSize),this.workspaceService=new Pb(e,n.maxBlobSize),this.deltaResolver=new Mf(e,n.deltaResolverOptions),this.workspaceService.onProgress.event(r=>{this.onIndexing.fire(r)}),this.workspaceService.onCheckpoint.event(r=>{this.onCheckpoint.fire(r)})}getGitRepoDetails(e){return ns(e)}async indexWorkspace(e,n,r){let i=WJ(e),a=WJ(n),o=$y(i),s,A,l=null;this.log.info("Looking for previously indexed git SHA...");try{let h=o?await this.deltaResolver.resolveIndexedCommitDelta(o):null;if(h&&o){this.log.info(`Found indexed commit ${h.indexedCommitSha}: ${h.fileInfoMap.size} unchanged, ${h.modifiedFiles.size} modified, ${h.addedFiles.size} added, ${h.deletedFiles.size} deleted, ${h.untrackedFiles.size} untracked`);let b=o===i?"":`${SJ(o,i)}/`,m=a===i?b:`${SJ(o,a)}/`,f=new Map;for(let[E,R]of h.fileInfoMap)E.startsWith(m)&&f.set(E,R);l={commitSha:h.indexedCommitSha,fileCount:f.size},f.size!==h.fileInfoMap.size&&this.log.info(`Filtered delta to ${f.size} files under ${m} (was ${h.fileInfoMap.size})`),s=f,o===i&&a===i&&(A=h.checkpointId)}else this.log.info(o?"No previously indexed commit found \u2014 doing full walk":"Not inside a git repository \u2014 doing full walk")}catch(h){this.log.warn("Delta resolution failed, falling back to full walk",h)}return{...await this.workspaceService.indexWorkspace(e,n,{...r,preIndexedFiles:s,baselineCheckpointId:A}),baseline:l}}createGitShaCommitIndex(e,n){return this.gitService.createGitShaCommitIndex(e,n)}getIndexingStatus(){return this.gitService.getIndexingStatus()}async startWatcher(e,n){this.stopWatcher();let r=new es(this.maxBlobSize),i=new ts(r,this.apiServer),a=new rs(this.apiServer);n&&a.seed(n);let o=new Qc({workspaceRoot:e});o.loadDirectory("");let s=new is(o,i,a);this.fileWatcher=new $b({workspaceRoot:e,shouldWatchDirectory:A=>(o.loadDirectory(A),!o.isDirectoryIgnored(A))}),this.watcherProcessor=new em({workspaceRoot:e,blobFilter:o,orchestrator:s}),this.watcherProcessor.onUpdate.event(A=>{this.onWatcherUpdate.fire(A)}),a.onCheckpoint.event(A=>{this.onCheckpoint.fire(A)}),this.watcherProcessor.attach(this.fileWatcher),await this.fileWatcher.start(),this.log.info(`File watcher started for ${e}`)}stopWatcher(){this.fileWatcher?.stop(),this.watcherProcessor?.dispose(),this.fileWatcher=null,this.watcherProcessor=null}dispose(){this.stopWatcher(),this.gitService.dispose(),this.workspaceService.dispose(),this.onIndexing.dispose(),this.onCheckpoint.dispose(),this.onWatcherUpdate.dispose()}};if(!GJ)throw new Error("indexing-worker.ts must be run as a worker thread");var im=GJ,Y0=new Zge;process.on("uncaughtException",t=>{im.postMessage({type:"fatalError",message:t.message,stack:t.stack,errorType:"uncaughtException"})});process.on("unhandledRejection",t=>{let e=t instanceof Error?t:new Error(String(t));im.postMessage({type:"fatalError",message:e.message,stack:e.stack,errorType:"unhandledRejection"})});var rm=null,Il=new Map,nm=new Map,Pu=new Map,AA=null,Cge=2e3;function Ii(t){im.postMessage(t)}function Zl(t,e,n,r){Ii({type:"log",workspaceId:r??Y0.getStore(),level:t,prefix:e,message:n,timestamp:Date.now()})}function XJ(t){let e=Il.get(t);if(e)return e;if(!rm)throw new Error("Worker not initialized \u2014 send 'configure' first");let n=rm();return n.onIndexing.event(r=>{nm.set(t,r),Ii({type:"progress",workspaceId:t,progress:r})}),n.onCheckpoint.event(r=>{Ii({type:"checkpointUpdate",workspaceId:t,checkpoint:r})}),n.onWatcherUpdate.event(r=>{Ii({type:"watcherUpdate",workspaceId:t,checkpointId:r.checkpointId})}),Il.set(t,n),n}Hy();Dt.on("data",t=>{let e=t.level??"info";Zl(e,t.prefix??"worker",String(t.message??""),Y0.getStore())});function Sge(){AA||(AA=setInterval(()=>{let t=process.memoryUsage();Ii({type:"metrics",heapUsedMb:Math.round(t.heapUsed/1024/1024),rssMb:Math.round(t.rss/1024/1024),activeHandles:process._getActiveHandles?.().length??0,timestamp:Date.now()})},Cge),AA.unref())}function Wge(){AA&&(clearInterval(AA),AA=null)}async function Xge(t){switch(t.type){case"configure":{let{apiToken:e,apiUrl:n,indexerOptions:r}=t.config;Zl("info","IndexingWorker",`Creating API client for ${n}`);let i=mX({apiToken:e,apiUrl:n}),a=new zf(i,n,e);rm=()=>new tm(a,r),Sge(),Zl("info","IndexingWorker","Configured successfully"),Ii({type:"result",requestId:t.requestId});break}case"indexWorkspace":{let e=XJ(t.workspaceId),n=t.skipDirectories?new Set(t.skipDirectories):void 0,r=t.folderToIndex??t.workspaceRoot,i=await e.indexWorkspace(t.workspaceRoot,r,{skipDirectories:n}),{pathToBlobName:a,...o}=i;try{let s=await e.createGitShaCommitIndex(t.workspaceRoot,"HEAD");Zl("info","IndexingWorker",`Git SHA commit index created: ${s.commitSha}`)}catch(s){Zl("warn","IndexingWorker",`Git SHA commit index failed (non-fatal): ${s}`)}Zl("info","IndexingWorker","Starting file watcher"),await e.startWatcher(t.workspaceRoot,a),Ii({type:"result",requestId:t.requestId,data:o});break}case"createGitShaCommitIndex":{let n=await XJ(t.workspaceId).createGitShaCommitIndex(t.repoRoot,t.revision);Ii({type:"result",requestId:t.requestId,data:n});break}case"statusRequest":{let e=Il.get(t.workspaceId)?.getIndexingStatus()??{itemsInFlight:0,calculateQueueSize:0,probeQueueSize:0,uploadQueueSize:0,probeRetryQueueSize:0,probeRetryBackoffQueueSize:0,calculateErrors:0,probeErrors:0,mtimeCacheHits:0,mtimeCacheMisses:0,pathsAccepted:0,currentFile:void 0,consecutiveNoneIndexedProbes:0,calculateAvgMs:0,calculateMaxMs:0};Ii({type:"statusResponse",requestId:t.requestId,workspaceId:t.workspaceId,status:e,progress:nm.get(t.workspaceId)??null});break}case"dispose":{Il.get(t.workspaceId)?.dispose(),Il.delete(t.workspaceId),nm.delete(t.workspaceId),Ii({type:"result",requestId:t.requestId});break}case"shutdown":{for(let e of Il.values())e.dispose();Il.clear(),nm.clear(),Pu.clear(),rm=null,Wge(),Ii({type:"result",requestId:t.requestId}),setTimeout(()=>process.exit(0),100);break}}}im.on("message",t=>{let e="workspaceId"in t?t.workspaceId:void 0,n=async()=>{try{await Xge(t)}catch(a){let o=a instanceof Error?a:new Error(String(a));Ii({type:"error",requestId:t.requestId,message:o.message,stack:o.stack})}};if(!e){n();return}let i=(Pu.get(e)??Promise.resolve()).then(()=>Y0.run(e,n));Pu.set(e,i),i.finally(()=>{Pu.get(e)===i&&Pu.delete(e)})});Ii({type:"ready"});Zl("info","IndexingWorker","single worker indexing enabled");
|
|
242
242
|
/*! Bundled license information:
|
|
243
243
|
|
|
244
244
|
safe-buffer/index.js:
|
package/package.json
CHANGED