@github/copilot-language-server 1.400.0 → 1.401.0

package/dist/tfidfWorker.js

@@ -79,7 +79,7 @@ ${f.join(`
 				`).run(u,o)}this.db.exec("COMMIT"),this.db.prepare(`
 			DELETE FROM ChunkOccurrences
 			WHERE chunkCount < 1;
-		`).run()}get fileCount(){return this.db.prepare("SELECT COUNT(*) as count FROM Documents").get()?.count??0}search(e,t){let r=new Bt(t?.maxResults??1/0,-1/0),i=this.computeEmbeddings(e);if(!i.size)return[];let o=new Map;for(let u of this.getAllChunksWithTerms(Array.from(i.keys()))){let a=this.score(u,i,o);a>0&&r.add(a,u.chunk)}return r.toArray(t?.maxSpread)}computeEmbeddings(e){let t=ar(e);return this.computeTfidf(t)}score(e,t,r){let i=0;for(let[o,u]of t.entries()){let a=e.tf[o];if(!a)continue;let l=r.get(o);typeof l!="number"&&(l=this.idf(o),r.set(o,l));let c=a*l;i+=c*u}return i}idf(e){let t=this.getChunkOccurrences(e)??0;return t>0?Math.log((this.getChunkCount()+1)/t):0}computeTfidf(e){let t=new Map;for(let[r,i]of Object.entries(e)){let o=this.idf(r);o>0&&t.set(r,i*o)}return t}getChunkCount(){return typeof this._cachedChunkCount=="number"?this._cachedChunkCount:this.db.prepare("SELECT COUNT(*) as count FROM Chunks").get()?.count??0}getChunkOccurrences(e){return this.db.prepare("SELECT chunkCount FROM ChunkOccurrences WHERE term = ?").get(e)?.chunkCount??0}async addOrUpdateDocs(e){this._cachedChunkCount=void 0;let t=Object.create(null),r=s(a=>{this.delete(a.map(l=>l.uri)),this.db.exec("BEGIN TRANSACTION");try{for(let{uri:l,doc:c}of a){let f=this.db.prepare("INSERT OR REPLACE INTO Documents (uri, contentVersionId) VALUES (?, ?)").run(l,c.contentVersionId).lastInsertRowid,h=this.db.prepare("INSERT INTO Chunks (documentId, text, startLineNumber, startColumn, endLineNumber, endColumn, isFullFile, termFrequencies) VALUES (?, ?, ?, ?, ?, ?, ?, jsonb(?))");for(let _ of c.chunks){h.run(f,_.chunk.text,_.chunk.range.startLineNumber,_.chunk.range.startColumn,_.chunk.range.endLineNumber,_.chunk.range.endColumn,_.chunk.isFullFile?1:0,JSON.stringify(_.tf));for(let A of Object.keys(_.tf))t[A]=(t[A]??0)+1}}this.db.exec("COMMIT")}catch(l){throw this.db.exec("ROLLBACK"),l}},"processBatch"),i=200,o=[];for(let a of e)o.push({uri:a.uri,doc:await a.getDoc()}),o.length>=i&&(r(o),o.length=0);r(o);let u=this.db.prepare(`
+		`).run()}get fileCount(){return this.db.prepare("SELECT COUNT(*) as count FROM Documents").get()?.count??0}search(e,t){let r=new Bt(t?.maxResults??1/0,-1/0),i=this.computeEmbeddings(e);if(!i.size)return[];let o=new Map;for(let u of this.getAllChunksWithTerms(Array.from(i.keys()))){let a=this.score(u,i,o);a>0&&r.add(a,u.chunk)}return r.toArray(t?.maxSpread)}computeEmbeddings(e){let t=ar(e);return this.computeTfidf(t)}score(e,t,r){let i=0;for(let[o,u]of t.entries()){let a=e.tf[o];if(!a)continue;let l=r.get(o);typeof l!="number"&&(l=this.idf(o),r.set(o,l));let c=a*l;i+=c*u}return i}idf(e){let t=this.getChunkOccurrences(e)??0;return t>0?Math.log((this.getChunkCount()+1)/t):0}computeTfidf(e){let t=new Map;for(let[r,i]of Object.entries(e)){let o=this.idf(r);o>0&&t.set(r,i*o)}return t}getChunkCount(){return typeof this._cachedChunkCount=="number"?this._cachedChunkCount:this.db.prepare("SELECT COUNT(*) as count FROM Chunks").get()?.count??0}getChunkOccurrences(e){return this.db.prepare("SELECT chunkCount FROM ChunkOccurrences WHERE term = ?").get(e)?.chunkCount??0}async addOrUpdateDocs(e){this._cachedChunkCount=void 0;let t=Object.create(null),r=s(a=>{this.delete(a.map(l=>l.uri)),this.db.exec("BEGIN TRANSACTION");try{for(let{uri:l,doc:c}of a){let f=this.db.prepare("INSERT OR REPLACE INTO Documents (uri, contentVersionId) VALUES (?, ?)").run(l,c.contentVersionId).lastInsertRowid,h=this.db.prepare("INSERT INTO Chunks (documentId, text, startLineNumber, startColumn, endLineNumber, endColumn, isFullFile, termFrequencies) VALUES (?, ?, ?, ?, ?, ?, ?, jsonb(?))");for(let _ of c.chunks){h.run(f,_.chunk.text,_.chunk.range.startLineNumber,_.chunk.range.startColumn,_.chunk.range.endLineNumber,_.chunk.range.endColumn,_.chunk.isFullFile?1:0,JSON.stringify(_.tf));for(let A of Object.keys(_.tf))t[A]=(t[A]??0)+1}}this.db.exec("COMMIT")}catch(l){throw this.db.exec("ROLLBACK"),l}},"processBatch"),i=200,o=[];for(let a of e){try{o.push({uri:a.uri,doc:await a.getDoc()})}catch(l){console.warn(`Failed to get document data for ${a.uri}, skip processing it:`,l);continue}o.length>=i&&(r(o),o.length=0)}r(o);let u=this.db.prepare(`
 			INSERT INTO ChunkOccurrences (term, chunkCount)
 			VALUES (?, ?)
 			ON CONFLICT(term) DO UPDATE SET chunkCount = chunkCount + ?;