@aws/lsp-codewhisperer 0.0.14 → 0.0.15

package/out/language-server/utilities/supplementalContextUtil/crossFileContextUtil.js

@@ -0,0 +1,205 @@
+"use strict";
+// Port of implementation in AWS Toolkit for VSCode
+// https://github.com/aws/aws-toolkit-vscode/blob/9d8ddbd85f4533e539a58e76f7c46883d8e50a79/packages/core/src/codewhisperer/util/supplementalContext/crossFileContextUtil.ts
+// Implementation is converted to work with LSP TextDocument instead of vscode APIs.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.supportedLanguageToDialects = void 0;
+exports.fetchSupplementalContextForSrc = fetchSupplementalContextForSrc;
+exports.splitFileToChunks = splitFileToChunks;
+exports.getCrossFileCandidates = getCrossFileCandidates;
+const path = require("path");
+const rankBm25_1 = require("./rankBm25");
+const constants_1 = require("../../models/constants");
+const codeParsingUtil_1 = require("./codeParsingUtil");
+const filesystemUtilities_1 = require("../filesystemUtilities");
+const server_interface_1 = require("@aws/language-server-runtimes/server-interface");
+const supplementalContextUtil_1 = require("./supplementalContextUtil");
+// TODO: ugly, can we make it prettier? like we have to manually type 'java', 'javascriptreact' which is error prone
+// TODO: Move to another config file or constants file
+// Supported language to its corresponding file ext
+exports.supportedLanguageToDialects = {
+    java: new Set(['.java']),
+    python: new Set(['.py']),
+    javascript: new Set(['.js', '.jsx']),
+    javascriptreact: new Set(['.js', '.jsx']),
+    typescript: new Set(['.ts', '.tsx']),
+    typescriptreact: new Set(['.ts', '.tsx']),
+};
+function isCrossFileSupported(languageId) {
+    return Object.keys(exports.supportedLanguageToDialects).includes(languageId);
+}
+async function fetchSupplementalContextForSrc(document, position, workspace, cancellationToken) {
+    const shouldProceed = shouldFetchCrossFileContext(document.languageId);
+    if (!shouldProceed) {
+        return shouldProceed === undefined
+            ? undefined
+            : {
+                supplementalContextItems: [],
+                strategy: 'Empty',
+            };
+    }
+    const codeChunksCalculated = constants_1.crossFileContextConfig.numberOfChunkToFetch;
+    // Step 1: Get relevant cross files to refer
+    const relevantCrossFileCandidates = await getCrossFileCandidates(document, workspace);
+    throwIfCancelled(cancellationToken);
+    // Step 2: Split files to chunks with upper bound on chunkCount
+    // We restrict the total number of chunks to improve on latency.
+    // Chunk linking is required as we want to pass the next chunk value for matched chunk.
+    let chunkList = [];
+    for (const relevantFile of relevantCrossFileCandidates) {
+        throwIfCancelled(cancellationToken);
+        const chunks = splitFileToChunks(relevantFile, constants_1.crossFileContextConfig.numberOfLinesEachChunk);
+        const linkedChunks = linkChunks(chunks);
+        chunkList.push(...linkedChunks);
+        if (chunkList.length >= codeChunksCalculated) {
+            break;
+        }
+    }
+    // it's required since chunkList.push(...) is likely giving us a list of size > 60
+    chunkList = chunkList.slice(0, codeChunksCalculated);
+    // Step 3: Generate Input chunk (10 lines left of cursor position)
+    // and Find Best K chunks w.r.t input chunk using BM25
+    const inputChunk = getInputChunk(document, position, constants_1.crossFileContextConfig.numberOfLinesEachChunk);
+    const bestChunks = findBestKChunkMatches(inputChunk, chunkList, constants_1.crossFileContextConfig.topK);
+    throwIfCancelled(cancellationToken);
+    // Step 4: Transform best chunks to supplemental contexts
+    const supplementalContexts = [];
+    for (const chunk of bestChunks) {
+        throwIfCancelled(cancellationToken);
+        supplementalContexts.push({
+            filePath: chunk.fileName,
+            content: chunk.nextContent,
+            score: chunk.score,
+        });
+    }
+    // DO NOT send code chunk with empty content
+    return {
+        supplementalContextItems: supplementalContexts.filter(item => item.content.trim().length !== 0),
+        strategy: 'OpenTabs_BM25',
+    };
+}
+function findBestKChunkMatches(chunkInput, chunkReferences, k) {
+    const chunkContentList = chunkReferences.map(chunk => chunk.content);
+    //performBM25Scoring returns the output in a sorted order (descending of scores)
+    const top3 = new rankBm25_1.BM25Okapi(chunkContentList).topN(chunkInput.content, constants_1.crossFileContextConfig.topK);
+    return top3.map(doc => {
+        // reference to the original metadata since BM25.top3 will sort the result
+        const chunkIndex = doc.index;
+        const chunkReference = chunkReferences[chunkIndex];
+        return {
+            content: chunkReference.content,
+            fileName: chunkReference.fileName,
+            nextContent: chunkReference.nextContent,
+            score: doc.score,
+        };
+    });
+}
+/* This extract 10 lines to the left of the cursor from trigger file.
+ * This will be the inputquery to bm25 matching against list of cross-file chunks
+ */
+function getInputChunk(document, cursorPosition, chunkSize) {
+    const startLine = Math.max(cursorPosition.line - chunkSize, 0);
+    const endLine = Math.max(cursorPosition.line - 1, 0);
+    const endCharacter = document.getText(server_interface_1.Range.create(endLine, 0, endLine + 1, 0)).trimRight().length;
+    const inputChunkContent = document
+        .getText(server_interface_1.Range.create(startLine, 0, endLine, endCharacter))
+        .replaceAll('\r\n', '\n');
+    const inputChunk = { fileName: path.basename(document.uri), content: inputChunkContent, nextContent: '' };
+    return inputChunk;
+}
+/**
+ * Note: UserGroup is not used in VsCode implementation, keeping param for reference, but not implemented.
+ *
+ * Util to decide if we need to fetch crossfile context since CodeWhisperer CrossFile Context feature is gated by userGroup and language level
+ * @param languageId: LSP TextDocument language Identifier
+ * @param userGroup: CodeWhisperer user group settings, refer to userGroupUtil.ts
+ * @returns specifically returning undefined if the langueage is not supported,
+ * otherwise true/false depending on if the language is fully supported or not belonging to the user group
+ */
+function shouldFetchCrossFileContext(languageId, _userGroup) {
+    if (!isCrossFileSupported(languageId)) {
+        return undefined;
+    }
+    return true;
+}
+/**
+ * This linking is required from science experimentations to pass the next contnet chunk
+ * when a given chunk context passes the match in BM25.
+ * Special handling is needed for last(its next points to its own) and first chunk
+ */
+function linkChunks(chunks) {
+    const updatedChunks = [];
+    // This additional chunk is needed to create a next pointer to chunk 0.
+    const firstChunk = chunks[0];
+    const firstChunkSubContent = firstChunk.content.split('\n').slice(0, 3).join('\n').trimEnd();
+    const newFirstChunk = {
+        fileName: firstChunk.fileName,
+        content: firstChunkSubContent,
+        nextContent: firstChunk.content,
+    };
+    updatedChunks.push(newFirstChunk);
+    const n = chunks.length;
+    for (let i = 0; i < n; i++) {
+        const chunk = chunks[i];
+        const nextChunk = i < n - 1 ? chunks[i + 1] : chunk;
+        chunk.nextContent = nextChunk.content;
+        updatedChunks.push(chunk);
+    }
+    return updatedChunks;
+}
+function splitFileToChunks(document, chunkSize) {
+    const chunks = [];
+    const fileContent = document.getText().trimEnd().replaceAll('\r\n', '\n');
+    const lines = fileContent.split('\n');
+    for (let i = 0; i < lines.length; i += chunkSize) {
+        const chunkContent = lines.slice(i, Math.min(i + chunkSize, lines.length)).join('\n');
+        const chunk = { fileName: new URL(document.uri).pathname, content: chunkContent.trimEnd(), nextContent: '' };
+        chunks.push(chunk);
+    }
+    return chunks;
+}
+/**
+ * This function will return relevant cross files sorted by file distance for the given editor file
+ * by referencing open files, imported files and same package files.
+ */
+async function getCrossFileCandidates(document, workspace) {
+    const targetFile = document.uri;
+    const language = document.languageId;
+    const dialects = exports.supportedLanguageToDialects[language];
+    /**
+     * Consider a file which
+     * 1. is different from the target
+     * 2. has the same file extension or it's one of the dialect of target file (e.g .js vs. .jsx)
+     * 3. is not a test file
+     *
+     * Porting note: this function relies of Workspace feature to get all documents,
+     * managed by this language server, instead of VSCode `vscode.window` API as VSCode toolkit does.
+     */
+    const unsortedCandidates = await workspace.getAllTextDocuments();
+    return unsortedCandidates
+        .filter((candidateFile) => {
+        return !!(targetFile !== candidateFile.uri &&
+            (path.extname(targetFile) === path.extname(candidateFile.uri) ||
+                (dialects && dialects.has(path.extname(candidateFile.uri)))) &&
+            !(0, codeParsingUtil_1.isTestFile)(new URL(candidateFile.uri).pathname, { languageId: language }));
+    })
+        .map((candidate) => {
+        return {
+            file: candidate,
+            // PORT_TODO: port and test getFileDistance to work with LSP's URIs
+            fileDistance: (0, filesystemUtilities_1.getFileDistance)(targetFile, candidate.uri),
+        };
+    })
+        .sort((file1, file2) => {
+        return file1.fileDistance - file2.fileDistance;
+    })
+        .map((fileToDistance) => {
+        return fileToDistance.file;
+    });
+}
+function throwIfCancelled(token) {
+    if (token.isCancellationRequested) {
+        throw new supplementalContextUtil_1.CancellationError();
+    }
+}
+//# sourceMappingURL=crossFileContextUtil.js.map

package/out/language-server/utilities/supplementalContextUtil/crossFileContextUtil.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"crossFileContextUtil.js","sourceRoot":"","sources":["../../../../src/language-server/utilities/supplementalContextUtil/crossFileContextUtil.ts"],"names":[],"mappings":";AAAA,mDAAmD;AACnD,2KAA2K;AAC3K,oFAAoF;;;AAgDpF,wEAgEC;AAkFD,8CAYC;AAWD,wDAqCC;AA5PD,6BAA4B;AAC5B,yCAAoD;AACpD,sDAA+D;AAC/D,uDAA8C;AAC9C,gEAAwD;AAExD,qFAMuD;AACvD,uEAA6D;AAU7D,oHAAoH;AACpH,sDAAsD;AACtD,mDAAmD;AACtC,QAAA,2BAA2B,GAA8D;IAClG,IAAI,EAAE,IAAI,GAAG,CAAS,CAAC,OAAO,CAAC,CAAC;IAChC,MAAM,EAAE,IAAI,GAAG,CAAS,CAAC,KAAK,CAAC,CAAC;IAChC,UAAU,EAAE,IAAI,GAAG,CAAS,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAC5C,eAAe,EAAE,IAAI,GAAG,CAAS,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACjD,UAAU,EAAE,IAAI,GAAG,CAAS,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAC5C,eAAe,EAAE,IAAI,GAAG,CAAS,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;CACpD,CAAA;AAED,SAAS,oBAAoB,CAAC,UAAkB;IAC5C,OAAO,MAAM,CAAC,IAAI,CAAC,mCAA2B,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAA;AACxE,CAAC;AASM,KAAK,UAAU,8BAA8B,CAChD,QAAsB,EACtB,QAAkB,EAClB,SAAoB,EACpB,iBAAoC;IAEpC,MAAM,aAAa,GAAG,2BAA2B,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAA;IAEtE,IAAI,CAAC,aAAa,EAAE,CAAC;QACjB,OAAO,aAAa,KAAK,SAAS;YAC9B,CAAC,CAAC,SAAS;YACX,CAAC,CAAC;gBACI,wBAAwB,EAAE,EAAE;gBAC5B,QAAQ,EAAE,OAAO;aACpB,CAAA;IACX,CAAC;IAED,MAAM,oBAAoB,GAAG,kCAAsB,CAAC,oBAAoB,CAAA;IAExE,4CAA4C;IAC5C,MAAM,2BAA2B,GAAG,MAAM,sBAAsB,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAA;IAErF,gBAAgB,CAAC,iBAAiB,CAAC,CAAA;IAEnC,+DAA+D;IAC/D,gEAAgE;IAChE,uFAAuF;IACvF,IAAI,SAAS,GAAY,EAAE,CAAA;IAC3B,KAAK,MAAM,YAAY,IAAI,2BAA2B,EAAE,CAAC;QACrD,gBAAgB,CAAC,iBAAiB,CAAC,CAAA;QACnC,MAAM,MAAM,GAAY,iBAAiB,CAAC,YAAY,EAAE,kCAAsB,CAAC,sBAAsB,CAAC,CAAA;QACtG,MAAM,YAAY,GAAG,UAAU,CAAC,MAAM,CAAC,CAAA;QACvC,SAAS,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAA;QAC/B,IAAI,SAAS,CAAC,MAAM,IAAI,oBAAoB,EAAE,CAAC;YAC3C,MAAK;QACT,CAAC;IACL,CAAC;IAED,kFAAkF;IAClF,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,oBAAoB,CAAC,CAAA;IAEpD,kEAAkE;IAClE,sDAAsD;IACtD,MAAM,UAAU,GAAU,aAAa,CAAC,QAAQ,EAAE,QAAQ,EAAE,kCAAsB,CAAC,sBAAsB,CAAC,CAAA;IAC1G,MAAM,UAAU,GAAY,qBAAqB,CAAC,UAAU,EAAE,SAAS,EAAE,kCAAsB,CAAC,IAAI,CAAC,CAAA;IACrG,gBAAgB,CAAC,iBAAiB,CAAC,CAAA;IAEnC,yDAAyD;IACzD,MAAM,oBAAoB,GAA2C,EAAE,CAAA;IACvE,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;QAC7B,gBAAgB,CAAC,iBAAiB,CAAC,CAAA;QAEnC,oBAAoB,CAAC,IAAI,CAAC;YACtB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,OAAO,EAAE,KAAK,CAAC,WAAW;YAC1B,KAAK,EAAE,KAAK,CAAC,KAAK;SACrB,CAAC,CAAA;IACN,CAAC;IAED,4CAA4C;IAC5C,OAAO;QACH,wBAAwB,EAAE,oBAAoB,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,CAAC;QAC/F,QAAQ,EAAE,eAAe;KAC5B,CAAA;AACL,CAAC;AAED,SAAS,qBAAqB,CAAC,UAAiB,EAAE,eAAwB,EAAE,CAAS;IACjF,MAAM,gBAAgB,GAAG,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;IAEpE,gFAAgF;IAChF,MAAM,IAAI,GAAmB,IAAI,oBAAS,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,kCAAsB,CAAC,IAAI,CAAC,CAAA;IAElH,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;QAClB,0EAA0E;QAC1E,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,CAAA;QAC5B,MAAM,cAAc,GAAG,eAAe,CAAC,UAAU,CAAC,CAAA;QAClD,OAAO;YACH,OAAO,EAAE,cAAc,CAAC,OAAO;YAC/B,QAAQ,EAAE,cAAc,CAAC,QAAQ;YACjC,WAAW,EAAE,cAAc,CAAC,WAAW;YACvC,KAAK,EAAE,GAAG,CAAC,KAAK;SACnB,CAAA;IACL,CAAC,CAAC,CAAA;AACN,CAAC;AAED;;GAEG;AACH,SAAS,aAAa,CAAC,QAAsB,EAAE,cAAwB,EAAE,SAAiB;IACtF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,GAAG,SAAS,EAAE,CAAC,CAAC,CAAA;IAC9D,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;IACpD,MAAM,YAAY,GAAG,QAAQ,CAAC,OAAO,CAAC,wBAAK,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,EAAE,OAAO,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,MAAM,CAAA;IAClG,MAAM,iBAAiB,GAAG,QAAQ;SAC7B,OAAO,CAAC,wBAAK,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC;SAC1D,UAAU,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;IAC7B,MAAM,UAAU,GAAU,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,OAAO,EAAE,iBAAiB,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;IAChH,OAAO,UAAU,CAAA;AACrB,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,2BAA2B,CAAC,UAAsC,EAAE,UAAgB;IACzF,IAAI,CAAC,oBAAoB,CAAC,UAAU,CAAC,EAAE,CAAC;QACpC,OAAO,SAAS,CAAA;IACpB,CAAC;IAED,OAAO,IAAI,CAAA;AACf,CAAC;AAED;;;;GAIG;AACH,SAAS,UAAU,CAAC,MAAe;IAC/B,MAAM,aAAa,GAAY,EAAE,CAAA;IAEjC,uEAAuE;IACvE,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,CAAA;IAC5B,MAAM,oBAAoB,GAAG,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,CAAA;IAC5F,MAAM,aAAa,GAAG;QAClB,QAAQ,EAAE,UAAU,CAAC,QAAQ;QAC7B,OAAO,EAAE,oBAAoB;QAC7B,WAAW,EAAE,UAAU,CAAC,OAAO;KAClC,CAAA;IACD,aAAa,CAAC,IAAI,CAAC,aAAa,CAAC,CAAA;IAEjC,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAA;IACvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAA;QACvB,MAAM,SAAS,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAA;QAEnD,KAAK,CAAC,WAAW,GAAG,SAAS,CAAC,OAAO,CAAA;QACrC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IAC7B,CAAC;IAED,OAAO,aAAa,CAAA;AACxB,CAAC;AAED,SAAgB,iBAAiB,CAAC,QAAsB,EAAE,SAAiB;IACvE,MAAM,MAAM,GAAY,EAAE,CAAA;IAE1B,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;IACzE,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAErC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;QAC/C,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACrF,MAAM,KAAK,GAAG,EAAE,QAAQ,EAAE,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,OAAO,EAAE,YAAY,CAAC,OAAO,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;QAC5G,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IACtB,CAAC;IACD,OAAO,MAAM,CAAA;AACjB,CAAC;AAOD;;;GAGG;AACI,KAAK,UAAU,sBAAsB,CAAC,QAAsB,EAAE,SAAoB;IACrF,MAAM,UAAU,GAAG,QAAQ,CAAC,GAAG,CAAA;IAC/B,MAAM,QAAQ,GAAG,QAAQ,CAAC,UAAwC,CAAA;IAClE,MAAM,QAAQ,GAAG,mCAA2B,CAAC,QAAQ,CAAC,CAAA;IAEtD;;;;;;;;OAQG;IACH,MAAM,kBAAkB,GAAG,MAAM,SAAS,CAAC,mBAAmB,EAAE,CAAA;IAChE,OAAO,kBAAkB;SACpB,MAAM,CAAC,CAAC,aAA2B,EAAE,EAAE;QACpC,OAAO,CAAC,CAAC,CACL,UAAU,KAAK,aAAa,CAAC,GAAG;YAChC,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,GAAG,CAAC;gBACzD,CAAC,QAAQ,IAAI,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAChE,CAAC,IAAA,4BAAU,EAAC,IAAI,GAAG,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC,CAC7E,CAAA;IACL,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,SAAuB,EAAgB,EAAE;QAC3C,OAAO;YACH,IAAI,EAAE,SAAS;YACf,mEAAmE;YACnE,YAAY,EAAE,IAAA,qCAAe,EAAC,UAAU,EAAE,SAAS,CAAC,GAAG,CAAC;SAC3D,CAAA;IACL,CAAC,CAAC;SACD,IAAI,CAAC,CAAC,KAAmB,EAAE,KAAmB,EAAE,EAAE;QAC/C,OAAO,KAAK,CAAC,YAAY,GAAG,KAAK,CAAC,YAAY,CAAA;IAClD,CAAC,CAAC;SACD,GAAG,CAAC,CAAC,cAA4B,EAAE,EAAE;QAClC,OAAO,cAAc,CAAC,IAAI,CAAA;IAC9B,CAAC,CAAC,CAAA;AACV,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAwB;IAC9C,IAAI,KAAK,CAAC,uBAAuB,EAAE,CAAC;QAChC,MAAM,IAAI,2CAAiB,EAAE,CAAA;IACjC,CAAC;AACL,CAAC"}

package/out/language-server/utilities/supplementalContextUtil/crossFileContextUtil.test.js

@@ -0,0 +1,157 @@
+"use strict";
+// Port of implementation in AWS Toolkit for VSCode
+// https://github.com/aws/aws-toolkit-vscode/blob/24840fda8559a3e3ace3517ad9844db76680dc50/packages/core/src/test/shared/filesystemUtilities.test.ts
+Object.defineProperty(exports, "__esModule", { value: true });
+const assert = require("assert");
+const path = require("path");
+const sinon = require("sinon");
+const crossFile = require("./crossFileContextUtil");
+const vscode_languageserver_textdocument_1 = require("vscode-languageserver-textdocument");
+const testing_1 = require("@aws/language-server-runtimes/testing");
+const testUtils_1 = require("../../testUtils");
+const crossFileContextUtil_1 = require("./crossFileContextUtil");
+const constants_1 = require("../../models/constants");
+describe('crossFileContextUtil', function () {
+    const fakeCancellationToken = {
+        isCancellationRequested: false,
+        onCancellationRequested: sinon.spy(),
+    };
+    let features;
+    beforeEach(() => {
+        features = new testing_1.TestFeatures();
+    });
+    afterEach(() => {
+        features.dispose();
+    });
+    function openDocument(uri, languageId, version = 1, content = '') {
+        const document = vscode_languageserver_textdocument_1.TextDocument.create(uri, languageId, version, content);
+        features.openDocument(document);
+        return document;
+    }
+    describe('fetchSupplementalContextForSrc', function () {
+        describe('should fetch 3 chunks and each chunk should contains 10 lines', async function () {
+            async function assertCorrectCodeChunk() {
+                openDocument('file:///CrossFile.java', 'java', 1, testUtils_1.SAMPLE_FILE_OF_60_LINES_IN_JAVA);
+                const currentDocument = openDocument('file:///TargetFile.java', 'java');
+                const actual = await crossFile.fetchSupplementalContextForSrc(currentDocument, { line: 0, character: 0 }, features.workspace, fakeCancellationToken);
+                assert.ok(actual);
+                assert.ok(actual.supplementalContextItems.length === 3);
+                assert.strictEqual(actual.supplementalContextItems[0].content.split('\n').length, 10);
+                assert.strictEqual(actual.supplementalContextItems[1].content.split('\n').length, 10);
+                assert.strictEqual(actual.supplementalContextItems[2].content.split('\n').length, 10);
+            }
+            it('control group', async function () {
+                await assertCorrectCodeChunk();
+            });
+        });
+    });
+    describe('non supported language should return undefined', function () {
+        it('c++', async function () {
+            const currentDocument = openDocument('file:///TestFile.cpp', 'cpp', 1, 'content');
+            const actual = await crossFile.fetchSupplementalContextForSrc(currentDocument, { line: 0, character: 0 }, features.workspace, fakeCancellationToken);
+            assert.strictEqual(actual, undefined);
+        });
+        it('ruby', async function () {
+            const currentDocument = openDocument('file:///testfile.rb', 'ruby', 1, 'content');
+            const actual = await crossFile.fetchSupplementalContextForSrc(currentDocument, { line: 0, character: 0 }, features.workspace, fakeCancellationToken);
+            assert.strictEqual(actual, undefined);
+        });
+    });
+    describe('getCrossFileCandidate', function () {
+        it('should return opened files, exclude test files and sorted ascendingly by file distance', async function () {
+            const targetFile = path.join('file://', 'src', 'service', 'microService', 'CodeWhispererFileContextProvider.java');
+            const fileWithDistance3 = path.join('file://', 'src', 'service', 'CodewhispererRecommendationService.java');
+            const fileWithDistance5 = path.join('file://', 'src', 'util', 'CodeWhispererConstants.java');
+            const fileWithDistance6 = path.join('file://', 'src', 'ui', 'popup', 'CodeWhispererPopupManager.java');
+            const fileWithDistance7 = path.join('file://', 'src', 'ui', 'popup', 'components', 'CodeWhispererPopup.java');
+            const fileWithDistance8 = path.join('file://', 'src', 'ui', 'popup', 'components', 'actions', 'AcceptRecommendationAction.java');
+            const testFile1 = path.join('file://', 'test', 'service', 'CodeWhispererFileContextProviderTest.java');
+            const testFile2 = path.join('file://', 'test', 'ui', 'CodeWhispererPopupManagerTest.java');
+            const expectedFilePaths = [
+                fileWithDistance3,
+                fileWithDistance5,
+                fileWithDistance6,
+                fileWithDistance7,
+                fileWithDistance8,
+            ];
+            const shuffledFilePaths = (0, testUtils_1.shuffleList)(expectedFilePaths);
+            for (const filePath of shuffledFilePaths) {
+                openDocument(filePath, 'java');
+            }
+            openDocument(testFile1, 'java');
+            openDocument(testFile2, 'java');
+            const targetDocument = openDocument(targetFile, 'java');
+            const actual = await crossFile.getCrossFileCandidates(targetDocument, features.workspace);
+            assert.ok(actual.length === 5);
+            actual.forEach((actualFile, index) => {
+                const expectedFile = expectedFilePaths[index];
+                assert.strictEqual(expectedFile, actualFile.uri);
+            });
+        });
+    });
+    // describe('partial support - control group', function () {
+    // Not implemented, user group filters are not used in VSCode and tests are no-op
+    // https://github.com/aws/aws-toolkit-vscode/blob/a2daf60dca5e5699b8b25f6dc84708295f042c38/packages/amazonq/test/unit/codewhisperer/util/crossFileContextUtil.test.ts#L144
+    // })
+    // describe('partial support - crossfile group', function () {
+    // Not implemented, user group filters are not used in VSCode and tests are no-op
+    // https://github.com/aws/aws-toolkit-vscode/blob/a2daf60dca5e5699b8b25f6dc84708295f042c38/packages/amazonq/test/unit/codewhisperer/util/crossFileContextUtil.test.ts#L144
+    // })
+    describe('full support', function () {
+        const fileExtLists = ['.java', '.js', '.ts', '.py', '.tsx', '.jsx'];
+        function convertToExtensionMap(input) {
+            const result = {};
+            for (const [language, extensions] of Object.entries(input)) {
+                for (const extension of extensions) {
+                    if (!result[extension]) {
+                        result[extension] = [];
+                    }
+                    result[extension].push(language);
+                }
+            }
+            return result;
+        }
+        const extToLanguageIdMap = convertToExtensionMap(crossFileContextUtil_1.supportedLanguageToDialects);
+        fileExtLists.forEach(fileExt => {
+            extToLanguageIdMap[fileExt].forEach(languageId => {
+                it('should be non empty', async function () {
+                    const document = openDocument(`file:///file-1${fileExt}`, languageId, 1, 'content-1');
+                    openDocument(`file:///file-2${fileExt}`, languageId, 1, 'content-2');
+                    openDocument(`file:///file-3${fileExt}`, languageId, 1, 'content-3');
+                    openDocument(`file:///file-4${fileExt}`, languageId, 1, 'content-4');
+                    const actual = await crossFile.fetchSupplementalContextForSrc(document, { line: 0, character: 0 }, features.workspace, fakeCancellationToken);
+                    assert.ok(actual && actual.supplementalContextItems.length !== 0);
+                });
+            });
+        });
+    });
+    describe('splitFileToChunks', function () {
+        it('should split file to a chunk of 2 lines', function () {
+            const document = vscode_languageserver_textdocument_1.TextDocument.create('file:///file.py', 'python', 1, 'line_1\nline_2\nline_3\nline_4\nline_5\nline_6\nline_7');
+            const chunks = crossFile.splitFileToChunks(document, 2);
+            assert.strictEqual(chunks.length, 4);
+            assert.strictEqual(chunks[0].content, 'line_1\nline_2');
+            assert.strictEqual(chunks[1].content, 'line_3\nline_4');
+            assert.strictEqual(chunks[2].content, 'line_5\nline_6');
+            assert.strictEqual(chunks[3].content, 'line_7');
+        });
+        it('should split file to a chunk of 5 lines', function () {
+            const document = vscode_languageserver_textdocument_1.TextDocument.create('file:///file.py', 'python', 1, 'line_1\nline_2\nline_3\nline_4\nline_5\nline_6\nline_7');
+            const chunks = crossFile.splitFileToChunks(document, 5);
+            assert.strictEqual(chunks.length, 2);
+            assert.strictEqual(chunks[0].content, 'line_1\nline_2\nline_3\nline_4\nline_5');
+            assert.strictEqual(chunks[1].content, 'line_6\nline_7');
+        });
+        it('codewhisperer crossfile config should use 10 lines', function () {
+            const document = vscode_languageserver_textdocument_1.TextDocument.create('file:///testfile.java', 'java', 1, testUtils_1.SAMPLE_FILE_OF_60_LINES_IN_JAVA);
+            const chunks = crossFile.splitFileToChunks(document, constants_1.crossFileContextConfig.numberOfLinesEachChunk);
+            assert.strictEqual(chunks.length, 6);
+        });
+        it('should split file with Windows line endings', function () {
+            const document = vscode_languageserver_textdocument_1.TextDocument.create('file:///testfile.java', 'java', 1, testUtils_1.SAMPLE_FILE_OF_60_LINES_IN_JAVA.replaceAll('\n', '\r\n'));
+            const chunks = crossFile.splitFileToChunks(document, constants_1.crossFileContextConfig.numberOfLinesEachChunk);
+            assert.strictEqual(chunks.length, 6);
+        });
+    });
+});
+//# sourceMappingURL=crossFileContextUtil.test.js.map

package/out/language-server/utilities/supplementalContextUtil/crossFileContextUtil.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"crossFileContextUtil.test.js","sourceRoot":"","sources":["../../../../src/language-server/utilities/supplementalContextUtil/crossFileContextUtil.test.ts"],"names":[],"mappings":";AAAA,mDAAmD;AACnD,oJAAoJ;;AAEpJ,iCAAgC;AAChC,6BAA4B;AAC5B,+BAA8B;AAC9B,oDAAmD;AACnD,2FAAiE;AACjE,mEAAoE;AAEpE,+CAA8E;AAC9E,iEAAoE;AACpE,sDAA+D;AAE/D,QAAQ,CAAC,sBAAsB,EAAE;IAC7B,MAAM,qBAAqB,GAAsB;QAC7C,uBAAuB,EAAE,KAAK;QAC9B,uBAAuB,EAAE,KAAK,CAAC,GAAG,EAAE;KACvC,CAAA;IACD,IAAI,QAAsB,CAAA;IAE1B,UAAU,CAAC,GAAG,EAAE;QACZ,QAAQ,GAAG,IAAI,sBAAY,EAAE,CAAA;IACjC,CAAC,CAAC,CAAA;IAEF,SAAS,CAAC,GAAG,EAAE;QACX,QAAQ,CAAC,OAAO,EAAE,CAAA;IACtB,CAAC,CAAC,CAAA;IAEF,SAAS,YAAY,CAAC,GAAW,EAAE,UAAkB,EAAE,OAAO,GAAG,CAAC,EAAE,UAAkB,EAAE;QACpF,MAAM,QAAQ,GAAG,iDAAY,CAAC,MAAM,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,OAAO,CAAC,CAAA;QACvE,QAAQ,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAA;QAE/B,OAAO,QAAQ,CAAA;IACnB,CAAC;IAED,QAAQ,CAAC,gCAAgC,EAAE;QACvC,QAAQ,CAAC,+DAA+D,EAAE,KAAK;YAC3E,KAAK,UAAU,sBAAsB;gBACjC,YAAY,CAAC,wBAAwB,EAAE,MAAM,EAAE,CAAC,EAAE,2CAA+B,CAAC,CAAA;gBAClF,MAAM,eAAe,GAAG,YAAY,CAAC,yBAAyB,EAAE,MAAM,CAAC,CAAA;gBAEvE,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,8BAA8B,CACzD,eAAe,EACf,EAAE,IAAI,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,EACzB,QAAQ,CAAC,SAAS,EAClB,qBAAqB,CACxB,CAAA;gBACD,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,CAAA;gBACjB,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,wBAAwB,CAAC,MAAM,KAAK,CAAC,CAAC,CAAA;gBAEvD,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,wBAAwB,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;gBACrF,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,wBAAwB,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;gBACrF,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,wBAAwB,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;YACzF,CAAC;YAED,EAAE,CAAC,eAAe,EAAE,KAAK;gBACrB,MAAM,sBAAsB,EAAE,CAAA;YAClC,CAAC,CAAC,CAAA;QACN,CAAC,CAAC,CAAA;IACN,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,gDAAgD,EAAE;QACvD,EAAE,CAAC,KAAK,EAAE,KAAK;YACX,MAAM,eAAe,GAAG,YAAY,CAAC,sBAAsB,EAAE,KAAK,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;YACjF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,8BAA8B,CACzD,eAAe,EACf,EAAE,IAAI,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,EACzB,QAAQ,CAAC,SAAS,EAClB,qBAAqB,CACxB,CAAA;YACD,MAAM,CAAC,WAAW,CAAC,MAAM,EAAE,SAAS,CAAC,CAAA;QACzC,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,MAAM,EAAE,KAAK;YACZ,MAAM,eAAe,GAAG,YAAY,CAAC,qBAAqB,EAAE,MAAM,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;YACjF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,8BAA8B,CACzD,eAAe,EACf,EAAE,IAAI,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,EACzB,QAAQ,CAAC,SAAS,EAClB,qBAAqB,CACxB,CAAA;YACD,MAAM,CAAC,WAAW,CAAC,MAAM,EAAE,SAAS,CAAC,CAAA;QACzC,CAAC,CAAC,CAAA;IACN,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,uBAAuB,EAAE;QAC9B,EAAE,CAAC,wFAAwF,EAAE,KAAK;YAC9F,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CACxB,SAAS,EACT,KAAK,EACL,SAAS,EACT,cAAc,EACd,uCAAuC,CAC1C,CAAA;YACD,MAAM,iBAAiB,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,yCAAyC,CAAC,CAAA;YAC3G,MAAM,iBAAiB,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,6BAA6B,CAAC,CAAA;YAC5F,MAAM,iBAAiB,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,gCAAgC,CAAC,CAAA;YACtG,MAAM,iBAAiB,GAAG,IAAI,CAAC,IAAI,CAC/B,SAAS,EACT,KAAK,EACL,IAAI,EACJ,OAAO,EACP,YAAY,EACZ,yBAAyB,CAC5B,CAAA;YACD,MAAM,iBAAiB,GAAG,IAAI,CAAC,IAAI,CAC/B,SAAS,EACT,KAAK,EACL,IAAI,EACJ,OAAO,EACP,YAAY,EACZ,SAAS,EACT,iCAAiC,CACpC,CAAA;YACD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,2CAA2C,CAAC,CAAA;YACtG,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,oCAAoC,CAAC,CAAA;YAE1F,MAAM,iBAAiB,GAAG;gBACtB,iBAAiB;gBACjB,iBAAiB;gBACjB,iBAAiB;gBACjB,iBAAiB;gBACjB,iBAAiB;aACpB,CAAA;YAED,MAAM,iBAAiB,GAAG,IAAA,uBAAW,EAAC,iBAAiB,CAAC,CAAA;YAExD,KAAK,MAAM,QAAQ,IAAI,iBAAiB,EAAE,CAAC;gBACvC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;YAClC,CAAC;YAED,YAAY,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;YAC/B,YAAY,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;YAC/B,MAAM,cAAc,GAAG,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAAA;YAEvD,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,sBAAsB,CAAC,cAAc,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAA;YAEzF,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAA;YAC9B,MAAM,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,KAAK,EAAE,EAAE;gBACjC,MAAM,YAAY,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAA;gBAC7C,MAAM,CAAC,WAAW,CAAC,YAAY,EAAE,UAAU,CAAC,GAAG,CAAC,CAAA;YACpD,CAAC,CAAC,CAAA;QACN,CAAC,CAAC,CAAA;IACN,CAAC,CAAC,CAAA;IAEF,4DAA4D;IAC5D,iFAAiF;IACjF,0KAA0K;IAC1K,KAAK;IAEL,8DAA8D;IAC9D,iFAAiF;IACjF,0KAA0K;IAC1K,KAAK;IAEL,QAAQ,CAAC,cAAc,EAAE;QACrB,MAAM,YAAY,GAAG,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,CAAA;QAEnE,SAAS,qBAAqB,CAAC,KAA4C;YACvE,MAAM,MAAM,GAA6B,EAAE,CAAA;YAE3C,KAAK,MAAM,CAAC,QAAQ,EAAE,UAAU,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;gBACzD,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;oBACjC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;wBACrB,MAAM,CAAC,SAAS,CAAC,GAAG,EAAE,CAAA;oBAC1B,CAAC;oBACD,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;gBACpC,CAAC;YACL,CAAC;YAED,OAAO,MAAM,CAAA;QACjB,CAAC;QACD,MAAM,kBAAkB,GAAG,qBAAqB,CAAC,kDAA2B,CAAC,CAAA;QAE7E,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;YAC3B,kBAAkB,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE;gBAC7C,EAAE,CAAC,qBAAqB,EAAE,KAAK;oBAC3B,MAAM,QAAQ,GAAG,YAAY,CAAC,iBAAiB,OAAO,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,WAAW,CAAC,CAAA;oBACrF,YAAY,CAAC,iBAAiB,OAAO,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,WAAW,CAAC,CAAA;oBACpE,YAAY,CAAC,iBAAiB,OAAO,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,WAAW,CAAC,CAAA;oBACpE,YAAY,CAAC,iBAAiB,OAAO,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,WAAW,CAAC,CAAA;oBAEpE,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,8BAA8B,CACzD,QAAQ,EACR,EAAE,IAAI,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,EACzB,QAAQ,CAAC,SAAS,EAClB,qBAAqB,CACxB,CAAA;oBAED,MAAM,CAAC,EAAE,CAAC,MAAM,IAAI,MAAM,CAAC,wBAAwB,CAAC,MAAM,KAAK,CAAC,CAAC,CAAA;gBACrE,CAAC,CAAC,CAAA;YACN,CAAC,CAAC,CAAA;QACN,CAAC,CAAC,CAAA;IACN,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,mBAAmB,EAAE;QAC1B,EAAE,CAAC,yCAAyC,EAAE;YAC1C,MAAM,QAAQ,GAAG,iDAAY,CAAC,MAAM,CAChC,iBAAiB,EACjB,QAAQ,EACR,CAAC,EACD,wDAAwD,CAC3D,CAAA;YAED,MAAM,MAAM,GAAG,SAAS,CAAC,iBAAiB,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAA;YAEvD,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAA;YACpC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAA;YACvD,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAA;YACvD,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAA;YACvD,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;QACnD,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,yCAAyC,EAAE;YAC1C,MAAM,QAAQ,GAAG,iDAAY,CAAC,MAAM,CAChC,iBAAiB,EACjB,QAAQ,EACR,CAAC,EACD,wDAAwD,CAC3D,CAAA;YAED,MAAM,MAAM,GAAG,SAAS,CAAC,iBAAiB,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAA;YAEvD,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAA;YACpC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,wCAAwC,CAAC,CAAA;YAC/E,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAA;QAC3D,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,oDAAoD,EAAE;YACrD,MAAM,QAAQ,GAAG,iDAAY,CAAC,MAAM,CAAC,uBAAuB,EAAE,MAAM,EAAE,CAAC,EAAE,2CAA+B,CAAC,CAAA;YAEzG,MAAM,MAAM,GAAG,SAAS,CAAC,iBAAiB,CAAC,QAAQ,EAAE,kCAAsB,CAAC,sBAAsB,CAAC,CAAA;YACnG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAA;QACxC,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,6CAA6C,EAAE;YAC9C,MAAM,QAAQ,GAAG,iDAAY,CAAC,MAAM,CAChC,uBAAuB,EACvB,MAAM,EACN,CAAC,EACD,2CAA+B,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,CAC3D,CAAA;YAED,MAAM,MAAM,GAAG,SAAS,CAAC,iBAAiB,CAAC,QAAQ,EAAE,kCAAsB,CAAC,sBAAsB,CAAC,CAAA;YACnG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAA;QACxC,CAAC,CAAC,CAAA;IACN,CAAC,CAAC,CAAA;AACN,CAAC,CAAC,CAAA"}

package/out/language-server/utilities/supplementalContextUtil/rankBm25.d.ts

@@ -0,0 +1,32 @@
+/*!
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+export interface BM25Document {
+    content: string;
+    /** The score that the document recieves. */
+    score: number;
+    index: number;
+}
+export declare abstract class BM25 {
+    protected readonly corpus: string[];
+    protected readonly tokenizer: (str: string) => string[];
+    protected readonly k1: number;
+    protected readonly b: number;
+    protected readonly epsilon: number;
+    protected readonly corpusSize: number;
+    protected readonly avgdl: number;
+    protected readonly idf: Map<string, number>;
+    protected readonly docLen: number[];
+    protected readonly docFreqs: Map<string, number>[];
+    protected readonly nd: Map<string, number>;
+    constructor(corpus: string[], tokenizer: ((str: string) => string[]) | undefined, k1: number, b: number, epsilon: number);
+    abstract calIdf(nd: Map<string, number>): void;
+    abstract score(query: string): BM25Document[];
+    topN(query: string, n: number): BM25Document[];
+}
+export declare class BM25Okapi extends BM25 {
+    constructor(corpus: string[], tokenizer?: (str: string) => string[]);
+    calIdf(nd: Map<string, number>): void;
+    score(query: string): BM25Document[];
+}

package/out/language-server/utilities/supplementalContextUtil/rankBm25.js

@@ -0,0 +1,113 @@
+"use strict";
+/*!
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BM25Okapi = exports.BM25 = void 0;
+class BM25 {
+    corpus;
+    tokenizer;
+    k1;
+    b;
+    epsilon;
+    corpusSize;
+    avgdl;
+    idf = new Map();
+    docLen = [];
+    docFreqs = [];
+    nd = new Map();
+    constructor(corpus, tokenizer = defaultTokenizer, k1, b, epsilon) {
+        this.corpus = corpus;
+        this.tokenizer = tokenizer;
+        this.k1 = k1;
+        this.b = b;
+        this.epsilon = epsilon;
+        this.corpusSize = corpus.length;
+        let numDoc = 0;
+        corpus
+            .map(document => {
+            return tokenizer(document);
+        })
+            .forEach(document => {
+            this.docLen.push(document.length);
+            numDoc += document.length;
+            const frequencies = new Map();
+            document.forEach(word => {
+                frequencies.set(word, (frequencies.get(word) || 0) + 1);
+            });
+            this.docFreqs.push(frequencies);
+            frequencies.forEach((freq, word) => {
+                this.nd.set(word, (this.nd.get(word) || 0) + 1);
+            });
+        });
+        this.avgdl = numDoc / this.corpusSize;
+        this.calIdf(this.nd);
+    }
+    topN(query, n) {
+        const notSorted = this.score(query);
+        const sorted = notSorted.sort((a, b) => b.score - a.score);
+        return sorted.slice(0, Math.min(n, sorted.length));
+    }
+}
+exports.BM25 = BM25;
+class BM25Okapi extends BM25 {
+    constructor(corpus, tokenizer = defaultTokenizer) {
+        super(corpus, tokenizer, 1.5, 0.75, 0.25);
+    }
+    calIdf(nd) {
+        let idfSum = 0;
+        const negativeIdfs = [];
+        for (const [word, freq] of nd) {
+            const idf = Math.log(this.corpusSize - freq + 0.5) - Math.log(freq + 0.5);
+            this.idf.set(word, idf);
+            idfSum += idf;
+            if (idf < 0) {
+                negativeIdfs.push(word);
+            }
+        }
+        const averageIdf = idfSum / this.idf.size;
+        const eps = this.epsilon * averageIdf;
+        for (const word of negativeIdfs) {
+            this.idf.set(word, eps);
+        }
+    }
+    score(query) {
+        const queryWords = defaultTokenizer(query);
+        return this.docFreqs.map((docFreq, index) => {
+            let score = 0;
+            queryWords.forEach((queryWord, _) => {
+                const queryWordFreqForDocument = docFreq.get(queryWord) || 0;
+                const numerator = (this.idf.get(queryWord) || 0.0) * queryWordFreqForDocument * (this.k1 + 1);
+                const denominator = queryWordFreqForDocument + this.k1 * (1 - this.b + (this.b * this.docLen[index]) / this.avgdl);
+                score += numerator / denominator;
+            });
+            return {
+                content: this.corpus[index],
+                score: score,
+                index: index,
+            };
+        });
+    }
+}
+exports.BM25Okapi = BM25Okapi;
+// TODO: This is a very simple tokenizer, we want to replace this by more sophisticated one.
+function defaultTokenizer(content) {
+    const regex = /\w+/g;
+    const words = content.split(' ');
+    const result = [];
+    for (const word of words) {
+        const wordList = findAll(word, regex);
+        result.push(...wordList);
+    }
+    return result;
+}
+function findAll(str, re) {
+    let match;
+    const matches = [];
+    while ((match = re.exec(str)) !== null) {
+        matches.push(match[0]);
+    }
+    return matches;
+}
+//# sourceMappingURL=rankBm25.js.map

package/out/language-server/utilities/supplementalContextUtil/rankBm25.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rankBm25.js","sourceRoot":"","sources":["../../../../src/language-server/utilities/supplementalContextUtil/rankBm25.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAaH,MAAsB,IAAI;IASC;IACA;IACA;IACA;IACA;IAZJ,UAAU,CAAQ;IAClB,KAAK,CAAQ;IACb,GAAG,GAAwB,IAAI,GAAG,EAAE,CAAA;IACpC,MAAM,GAAa,EAAE,CAAA;IACrB,QAAQ,GAA0B,EAAE,CAAA;IACpC,EAAE,GAAwB,IAAI,GAAG,EAAE,CAAA;IAEtD,YACuB,MAAgB,EAChB,YAAuC,gBAAgB,EACvD,EAAU,EACV,CAAS,EACT,OAAe;QAJf,WAAM,GAAN,MAAM,CAAU;QAChB,cAAS,GAAT,SAAS,CAA8C;QACvD,OAAE,GAAF,EAAE,CAAQ;QACV,MAAC,GAAD,CAAC,CAAQ;QACT,YAAO,GAAP,OAAO,CAAQ;QAElC,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,MAAM,CAAA;QAE/B,IAAI,MAAM,GAAG,CAAC,CAAA;QACd,MAAM;aACD,GAAG,CAAC,QAAQ,CAAC,EAAE;YACZ,OAAO,SAAS,CAAC,QAAQ,CAAC,CAAA;QAC9B,CAAC,CAAC;aACD,OAAO,CAAC,QAAQ,CAAC,EAAE;YAChB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAA;YACjC,MAAM,IAAI,QAAQ,CAAC,MAAM,CAAA;YAEzB,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAA;YAC7C,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE;gBACpB,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YAC3D,CAAC,CAAC,CAAA;YACF,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;YAE/B,WAAW,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE;gBAC/B,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YACnD,CAAC,CAAC,CAAA;QACN,CAAC,CAAC,CAAA;QAEN,IAAI,CAAC,KAAK,GAAG,MAAM,GAAG,IAAI,CAAC,UAAU,CAAA;QAErC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IACxB,CAAC;IAMD,IAAI,CAAC,KAAa,EAAE,CAAS;QACzB,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;QACnC,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAA;QAC1D,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAA;IACtD,CAAC;CACJ;AAnDD,oBAmDC;AAED,MAAa,SAAU,SAAQ,IAAI;IAC/B,YAAY,MAAgB,EAAE,YAAuC,gBAAgB;QACjF,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,CAAA;IAC7C,CAAC;IAED,MAAM,CAAC,EAAuB;QAC1B,IAAI,MAAM,GAAG,CAAC,CAAA;QAEd,MAAM,YAAY,GAAa,EAAE,CAAA;QACjC,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,GAAG,IAAI,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,CAAA;YACzE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;YACvB,MAAM,IAAI,GAAG,CAAA;YAEb,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;gBACV,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAC3B,CAAC;QACL,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAA;QACzC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,GAAG,UAAU,CAAA;QACrC,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;YAC9B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;QAC3B,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAa;QACf,MAAM,UAAU,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAA;QAC1C,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE;YACxC,IAAI,KAAK,GAAG,CAAC,CAAA;YACb,UAAU,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,CAAC,EAAE,EAAE;gBAChC,MAAM,wBAAwB,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;gBAC5D,MAAM,SAAS,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,GAAG,wBAAwB,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,CAAA;gBAC7F,MAAM,WAAW,GACb,wBAAwB,GAAG,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAA;gBAElG,KAAK,IAAI,SAAS,GAAG,WAAW,CAAA;YACpC,CAAC,CAAC,CAAA;YAEF,OAAO;gBACH,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;gBAC3B,KAAK,EAAE,KAAK;gBACZ,KAAK,EAAE,KAAK;aACf,CAAA;QACL,CAAC,CAAC,CAAA;IACN,CAAC;CACJ;AA9CD,8BA8CC;AAED,4FAA4F;AAC5F,SAAS,gBAAgB,CAAC,OAAe;IACrC,MAAM,KAAK,GAAG,MAAM,CAAA;IACpB,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IAChC,MAAM,MAAM,GAAG,EAAE,CAAA;IACjB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;QACrC,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAA;IAC5B,CAAC;IAED,OAAO,MAAM,CAAA;AACjB,CAAC;AAED,SAAS,OAAO,CAAC,GAAW,EAAE,EAAU;IACpC,IAAI,KAA6B,CAAA;IACjC,MAAM,OAAO,GAAa,EAAE,CAAA;IAE5B,OAAO,CAAC,KAAK,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACrC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;IAC1B,CAAC;IAED,OAAO,OAAO,CAAA;AAClB,CAAC"}

package/out/language-server/utilities/supplementalContextUtil/supplementalContextUtil.d.ts

@@ -0,0 +1,5 @@
+import { CodeWhispererSupplementalContext } from '../../models/model';
+import { CancellationToken, Logging, Position, TextDocument, Workspace } from '@aws/language-server-runtimes/server-interface';
+export declare class CancellationError extends Error {
+}
+export declare function fetchSupplementalContext(document: TextDocument, position: Position, workspace: Workspace, logging: Logging, cancellationToken: CancellationToken): Promise<CodeWhispererSupplementalContext | undefined>;

package/out/language-server/utilities/supplementalContextUtil/supplementalContextUtil.js

@@ -0,0 +1,57 @@
+"use strict";
+// Port of implementation in AWS Toolkit for VSCode
+// https://github.com/aws/aws-toolkit-vscode/blob/9d8ddbd85f4533e539a58e76f7c46883d8e50a79/packages/core/src/codewhisperer/util/supplementalContext/supplementalContextUtil.ts
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CancellationError = void 0;
+exports.fetchSupplementalContext = fetchSupplementalContext;
+const crossFileContextUtil_1 = require("./crossFileContextUtil");
+const codeParsingUtil_1 = require("./codeParsingUtil");
+class CancellationError extends Error {
+}
+exports.CancellationError = CancellationError;
+async function fetchSupplementalContext(document, position, workspace, logging, cancellationToken) {
+    const timesBeforeFetching = performance.now();
+    const isUtg = (0, codeParsingUtil_1.isTestFile)(document.uri, {
+        languageId: document.languageId,
+        fileContent: document.getText(),
+    });
+    try {
+        let supplementalContextValue;
+        if (isUtg) {
+            return;
+        }
+        else {
+            supplementalContextValue = await (0, crossFileContextUtil_1.fetchSupplementalContextForSrc)(document, position, workspace, cancellationToken);
+        }
+        if (supplementalContextValue) {
+            return {
+                isUtg: isUtg,
+                isProcessTimeout: false,
+                supplementalContextItems: supplementalContextValue.supplementalContextItems,
+                contentsLength: supplementalContextValue.supplementalContextItems.reduce((acc, curr) => acc + curr.content.length, 0),
+                latency: performance.now() - timesBeforeFetching,
+                strategy: supplementalContextValue.strategy,
+            };
+        }
+        else {
+            return undefined;
+        }
+    }
+    catch (err) {
+        if (err instanceof CancellationError) {
+            return {
+                isUtg: isUtg,
+                isProcessTimeout: true,
+                supplementalContextItems: [],
+                contentsLength: 0,
+                latency: performance.now() - timesBeforeFetching,
+                strategy: 'Empty',
+            };
+        }
+        else {
+            logging.log(`Fail to fetch supplemental context for target file ${document.uri}: ${err}`);
+            return undefined;
+        }
+    }
+}
+//# sourceMappingURL=supplementalContextUtil.js.map

package/out/language-server/utilities/supplementalContextUtil/supplementalContextUtil.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"supplementalContextUtil.js","sourceRoot":"","sources":["../../../../src/language-server/utilities/supplementalContextUtil/supplementalContextUtil.ts"],"names":[],"mappings":";AAAA,mDAAmD;AACnD,8KAA8K;;;AAe9K,4DA4DC;AAzED,iEAAuE;AACvE,uDAA8C;AAU9C,MAAa,iBAAkB,SAAQ,KAAK;CAAG;AAA/C,8CAA+C;AAExC,KAAK,UAAU,wBAAwB,CAC1C,QAAsB,EACtB,QAAkB,EAClB,SAAoB,EACpB,OAAgB,EAChB,iBAAoC;IAEpC,MAAM,mBAAmB,GAAG,WAAW,CAAC,GAAG,EAAE,CAAA;IAE7C,MAAM,KAAK,GAAG,IAAA,4BAAU,EAAC,QAAQ,CAAC,GAAG,EAAE;QACnC,UAAU,EAAE,QAAQ,CAAC,UAAU;QAC/B,WAAW,EAAE,QAAQ,CAAC,OAAO,EAAE;KAClC,CAAC,CAAA;IAEF,IAAI,CAAC;QACD,IAAI,wBAEW,CAAA;QAEf,IAAI,KAAK,EAAE,CAAC;YACR,OAAM;QACV,CAAC;aAAM,CAAC;YACJ,wBAAwB,GAAG,MAAM,IAAA,qDAA8B,EAC3D,QAAQ,EACR,QAAQ,EACR,SAAS,EACT,iBAAiB,CACpB,CAAA;QACL,CAAC;QAED,IAAI,wBAAwB,EAAE,CAAC;YAC3B,OAAO;gBACH,KAAK,EAAE,KAAK;gBACZ,gBAAgB,EAAE,KAAK;gBACvB,wBAAwB,EAAE,wBAAwB,CAAC,wBAAwB;gBAC3E,cAAc,EAAE,wBAAwB,CAAC,wBAAwB,CAAC,MAAM,CACpE,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EACxC,CAAC,CACJ;gBACD,OAAO,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,mBAAmB;gBAChD,QAAQ,EAAE,wBAAwB,CAAC,QAAQ;aAC9C,CAAA;QACL,CAAC;aAAM,CAAC;YACJ,OAAO,SAAS,CAAA;QACpB,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACX,IAAI,GAAG,YAAY,iBAAiB,EAAE,CAAC;YACnC,OAAO;gBACH,KAAK,EAAE,KAAK;gBACZ,gBAAgB,EAAE,IAAI;gBACtB,wBAAwB,EAAE,EAAE;gBAC5B,cAAc,EAAE,CAAC;gBACjB,OAAO,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,mBAAmB;gBAChD,QAAQ,EAAE,OAAO;aACpB,CAAA;QACL,CAAC;aAAM,CAAC;YACJ,OAAO,CAAC,GAAG,CAAC,sDAAsD,QAAQ,CAAC,GAAG,KAAK,GAAG,EAAE,CAAC,CAAA;YACzF,OAAO,SAAS,CAAA;QACpB,CAAC;IACL,CAAC;AACL,CAAC"}