@aws/lsp-codewhisperer 0.0.81 → 0.0.83
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +43 -0
- package/out/language-server/agenticChat/agenticChatController.js +123 -28
- package/out/language-server/agenticChat/agenticChatController.js.map +1 -1
- package/out/language-server/agenticChat/constants/constants.d.ts +7 -0
- package/out/language-server/agenticChat/constants/constants.js +13 -1
- package/out/language-server/agenticChat/constants/constants.js.map +1 -1
- package/out/language-server/agenticChat/context/additionalContextProvider.d.ts +1 -1
- package/out/language-server/agenticChat/context/additionalContextProvider.js +50 -3
- package/out/language-server/agenticChat/context/additionalContextProvider.js.map +1 -1
- package/out/language-server/agenticChat/context/memorybank/memoryBankController.d.ts +104 -0
- package/out/language-server/agenticChat/context/memorybank/memoryBankController.js +681 -0
- package/out/language-server/agenticChat/context/memorybank/memoryBankController.js.map +1 -0
- package/out/language-server/agenticChat/context/memorybank/memoryBankPrompts.d.ts +14 -0
- package/out/language-server/agenticChat/context/memorybank/memoryBankPrompts.js +156 -0
- package/out/language-server/agenticChat/context/memorybank/memoryBankPrompts.js.map +1 -0
- package/out/language-server/agenticChat/tools/chatDb/chatDb.d.ts +3 -25
- package/out/language-server/agenticChat/tools/chatDb/chatDb.js +12 -107
- package/out/language-server/agenticChat/tools/chatDb/chatDb.js.map +1 -1
- package/out/language-server/agenticChat/tools/chatDb/util.js +1 -2
- package/out/language-server/agenticChat/tools/chatDb/util.js.map +1 -1
- package/out/language-server/agenticChat/tools/fsRead.d.ts +3 -1
- package/out/language-server/agenticChat/tools/fsRead.js +12 -7
- package/out/language-server/agenticChat/tools/fsRead.js.map +1 -1
- package/out/language-server/agenticChat/tools/qCodeAnalysis/codeReview.js +32 -16
- package/out/language-server/agenticChat/tools/qCodeAnalysis/codeReview.js.map +1 -1
- package/out/language-server/agenticChat/tools/qCodeAnalysis/codeReviewConstants.js +1 -1
- package/out/language-server/agenticChat/tools/qCodeAnalysis/codeReviewConstants.js.map +1 -1
- package/out/language-server/agenticChat/tools/qCodeAnalysis/codeReviewTypes.d.ts +8 -0
- package/out/language-server/chat/chatSessionService.d.ts +1 -1
- package/out/language-server/chat/chatSessionService.js +1 -1
- package/out/language-server/chat/chatSessionService.js.map +1 -1
- package/out/language-server/chat/telemetry/chatTelemetryController.d.ts +2 -2
- package/out/language-server/chat/telemetry/chatTelemetryController.js +3 -2
- package/out/language-server/chat/telemetry/chatTelemetryController.js.map +1 -1
- package/out/language-server/inline-completion/codeWhispererServer.js +13 -11
- package/out/language-server/inline-completion/codeWhispererServer.js.map +1 -1
- package/out/language-server/inline-completion/contants/constants.js.map +1 -0
- package/out/language-server/inline-completion/editCompletionHandler.d.ts +1 -1
- package/out/language-server/inline-completion/editCompletionHandler.js +8 -5
- package/out/language-server/inline-completion/editCompletionHandler.js.map +1 -1
- package/out/language-server/inline-completion/session/sessionManager.d.ts +7 -1
- package/out/language-server/inline-completion/session/sessionManager.js +20 -4
- package/out/language-server/inline-completion/session/sessionManager.js.map +1 -1
- package/out/language-server/inline-completion/{telemetry.d.ts → telemetry/telemetry.d.ts} +2 -2
- package/out/language-server/inline-completion/{telemetry.js → telemetry/telemetry.js} +2 -2
- package/out/language-server/inline-completion/telemetry/telemetry.js.map +1 -0
- package/out/language-server/inline-completion/{codeDiffTracker.d.ts → tracker/codeDiffTracker.d.ts} +2 -2
- package/out/language-server/inline-completion/{codeDiffTracker.js → tracker/codeDiffTracker.js} +1 -1
- package/out/language-server/inline-completion/tracker/codeDiffTracker.js.map +1 -0
- package/out/language-server/inline-completion/tracker/codeEditTracker.js +1 -1
- package/out/language-server/inline-completion/tracker/codeEditTracker.js.map +1 -1
- package/out/language-server/inline-completion/{codePercentage.d.ts → tracker/codePercentageTracker.d.ts} +1 -1
- package/out/language-server/inline-completion/{codePercentage.js → tracker/codePercentageTracker.js} +1 -1
- package/out/language-server/inline-completion/tracker/codePercentageTracker.js.map +1 -0
- package/out/language-server/inline-completion/{diffUtils.d.ts → utils/diffUtils.d.ts} +1 -1
- package/out/language-server/inline-completion/{diffUtils.js → utils/diffUtils.js} +1 -1
- package/out/language-server/inline-completion/utils/diffUtils.js.map +1 -0
- package/out/language-server/inline-completion/{mergeRightUtils.d.ts → utils/mergeRightUtils.d.ts} +3 -11
- package/out/language-server/inline-completion/utils/mergeRightUtils.js +95 -0
- package/out/language-server/inline-completion/utils/mergeRightUtils.js.map +1 -0
- package/out/language-server/netTransform/artifactManager.d.ts +2 -2
- package/out/language-server/netTransform/artifactManager.js +6 -10
- package/out/language-server/netTransform/artifactManager.js.map +1 -1
- package/out/shared/amazonQServiceManager/AmazonQIAMServiceManager.d.ts +1 -1
- package/out/shared/amazonQServiceManager/AmazonQIAMServiceManager.js +8 -2
- package/out/shared/amazonQServiceManager/AmazonQIAMServiceManager.js.map +1 -1
- package/out/shared/amazonQServiceManager/AmazonQTokenServiceManager.js +3 -0
- package/out/shared/amazonQServiceManager/AmazonQTokenServiceManager.js.map +1 -1
- package/out/shared/amazonQServiceManager/BaseAmazonQServiceManager.js +6 -0
- package/out/shared/amazonQServiceManager/BaseAmazonQServiceManager.js.map +1 -1
- package/out/shared/codeWhispererService.d.ts +2 -2
- package/out/shared/codeWhispererService.js +37 -6
- package/out/shared/codeWhispererService.js.map +1 -1
- package/out/shared/streamingClientService.d.ts +1 -0
- package/out/shared/streamingClientService.js +20 -3
- package/out/shared/streamingClientService.js.map +1 -1
- package/out/shared/supplementalContextUtil/supplementalContextUtil.js +4 -4
- package/out/shared/supplementalContextUtil/supplementalContextUtil.js.map +1 -1
- package/out/shared/telemetry/telemetryService.d.ts +1 -0
- package/out/shared/telemetry/telemetryService.js +10 -8
- package/out/shared/telemetry/telemetryService.js.map +1 -1
- package/out/shared/testUtils.d.ts +0 -3
- package/out/shared/testUtils.js +2 -21
- package/out/shared/testUtils.js.map +1 -1
- package/out/shared/utils.d.ts +1 -1
- package/out/shared/utils.js +5 -1
- package/out/shared/utils.js.map +1 -1
- package/package.json +3 -4
- package/out/language-server/inline-completion/codeDiffTracker.js.map +0 -1
- package/out/language-server/inline-completion/codePercentage.js.map +0 -1
- package/out/language-server/inline-completion/constants.js.map +0 -1
- package/out/language-server/inline-completion/diffUtils.js.map +0 -1
- package/out/language-server/inline-completion/mergeRightUtils.js +0 -114
- package/out/language-server/inline-completion/mergeRightUtils.js.map +0 -1
- package/out/language-server/inline-completion/telemetry.js.map +0 -1
- /package/out/language-server/inline-completion/{constants.d.ts → contants/constants.d.ts} +0 -0
- /package/out/language-server/inline-completion/{constants.js → contants/constants.js} +0 -0
|
@@ -0,0 +1,681 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/*!
|
|
3
|
+
* Copyright Amazon.com, Inc. or its affiliates.
|
|
4
|
+
* All Rights Reserved. SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
*/
|
|
6
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
+
exports.MemoryBankController = void 0;
|
|
8
|
+
const memoryBankPrompts_1 = require("./memoryBankPrompts");
|
|
9
|
+
const mcpUtils_1 = require("../../tools/mcp/mcpUtils");
|
|
10
|
+
const constants_1 = require("../../constants/constants");
|
|
11
|
+
const MEMORY_BANK_DIRECTORY = '.amazonq/rules/memory-bank';
|
|
12
|
+
const MEMORY_BANK_FILES = {
|
|
13
|
+
PRODUCT: 'product.md',
|
|
14
|
+
STRUCTURE: 'structure.md',
|
|
15
|
+
TECH: 'tech.md',
|
|
16
|
+
GUIDELINES: 'guidelines.md',
|
|
17
|
+
};
|
|
18
|
+
/**
|
|
19
|
+
* Controller for Memory Bank functionality
|
|
20
|
+
* Handles memory bank creation detection and prompt generation
|
|
21
|
+
*/
|
|
22
|
+
class MemoryBankController {
|
|
23
|
+
features;
|
|
24
|
+
static instance;
|
|
25
|
+
constructor(features) {
|
|
26
|
+
this.features = features;
|
|
27
|
+
}
|
|
28
|
+
static getInstance(features) {
|
|
29
|
+
if (!MemoryBankController.instance) {
|
|
30
|
+
MemoryBankController.instance = new MemoryBankController(features);
|
|
31
|
+
}
|
|
32
|
+
return MemoryBankController.instance;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Check if a prompt is requesting memory bank creation
|
|
36
|
+
* Can be expanded based on feedbacks
|
|
37
|
+
*/
|
|
38
|
+
isMemoryBankCreationRequest(prompt) {
|
|
39
|
+
const normalizedPrompt = prompt.toLowerCase().trim();
|
|
40
|
+
const triggers = [
|
|
41
|
+
'create a memory bank',
|
|
42
|
+
'create memory bank',
|
|
43
|
+
'generate a memory bank',
|
|
44
|
+
'generate memory bank',
|
|
45
|
+
'regenerate memory bank',
|
|
46
|
+
'build memory bank',
|
|
47
|
+
'make memory bank',
|
|
48
|
+
'setup memory bank',
|
|
49
|
+
];
|
|
50
|
+
return triggers.some(trigger => normalizedPrompt.includes(trigger));
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Prepare comprehensive memory bank creation prompt with all necessary input
|
|
54
|
+
* This does all the programmatic work upfront and creates a single comprehensive prompt
|
|
55
|
+
*/
|
|
56
|
+
async prepareComprehensiveMemoryBankPrompt(workspaceFolderUri, llmCallFunction) {
|
|
57
|
+
try {
|
|
58
|
+
this.features.logging.info(`Memory Bank: Starting pre-processing for workspace: "${workspaceFolderUri}"`);
|
|
59
|
+
// Step 1: Clean directory
|
|
60
|
+
await this.cleanMemoryBankDirectory(workspaceFolderUri);
|
|
61
|
+
// Step 2: Execute deterministic analysis (TF-IDF)
|
|
62
|
+
this.features.logging.info(`Memory Bank: running analysis for workspace`);
|
|
63
|
+
const analysisResults = await this.executeGuidelinesGenerationPipeline(workspaceFolderUri);
|
|
64
|
+
// Step 3: Make LLM call for file ranking
|
|
65
|
+
const rankingPrompt = memoryBankPrompts_1.MemoryBankPrompts.getFileRankingPrompt(analysisResults.formattedFilesString, constants_1.MAX_NUMBER_OF_FILES_FOR_MEMORY_BANK_RANKING);
|
|
66
|
+
const rankedFilesResponse = await llmCallFunction(rankingPrompt);
|
|
67
|
+
// Step 4: Parse ranked files
|
|
68
|
+
let rankedFilesList = [];
|
|
69
|
+
try {
|
|
70
|
+
// Clean the response - remove any markdown formatting or extra text
|
|
71
|
+
let cleanResponse = rankedFilesResponse.trim();
|
|
72
|
+
// Extract JSON array if it's wrapped in markdown or other text
|
|
73
|
+
const jsonMatch = cleanResponse.match(/\[.*\]/s);
|
|
74
|
+
if (jsonMatch) {
|
|
75
|
+
cleanResponse = jsonMatch[0];
|
|
76
|
+
}
|
|
77
|
+
else {
|
|
78
|
+
// Handle case where LLM returns comma-separated quoted strings without brackets
|
|
79
|
+
if (cleanResponse.includes('",') && cleanResponse.includes('"')) {
|
|
80
|
+
// Add brackets to make it a valid JSON array
|
|
81
|
+
cleanResponse = `[${cleanResponse}]`;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
rankedFilesList = JSON.parse(cleanResponse);
|
|
85
|
+
if (!Array.isArray(rankedFilesList)) {
|
|
86
|
+
throw new Error('Invalid ranking response format - not an array');
|
|
87
|
+
}
|
|
88
|
+
// Validate that all items are strings (file paths)
|
|
89
|
+
rankedFilesList = rankedFilesList.filter(item => typeof item === 'string' && item.length > 0);
|
|
90
|
+
if (rankedFilesList.length === 0) {
|
|
91
|
+
throw new Error('No valid file paths in ranking response');
|
|
92
|
+
}
|
|
93
|
+
this.features.logging.info(`Memory Bank: parsed ${rankedFilesList.length} ranked files from LLM response`);
|
|
94
|
+
}
|
|
95
|
+
catch (error) {
|
|
96
|
+
this.features.logging.warn(`Memory Bank: failed to parse LLM ranking response, using TF-IDF fallback: ${error}`);
|
|
97
|
+
rankedFilesList = analysisResults.rankedFilesList.slice(0, constants_1.MAX_NUMBER_OF_FILES_FOR_MEMORY_BANK_RANKING);
|
|
98
|
+
}
|
|
99
|
+
this.features.logging.info(`Memory Bank: using ${rankedFilesList.length} files for documentation generation`);
|
|
100
|
+
// Step 5: Create the comprehensive prompt with ranked files and workspace path
|
|
101
|
+
const normalizedWorkspacePath = (0, mcpUtils_1.normalizePathFromUri)(workspaceFolderUri, this.features.logging);
|
|
102
|
+
this.features.logging.info(`Memory Bank: Generating final prompt with path: "${normalizedWorkspacePath}"`);
|
|
103
|
+
const finalPrompt = memoryBankPrompts_1.MemoryBankPrompts.getCompleteMemoryBankPrompt(rankedFilesList, normalizedWorkspacePath);
|
|
104
|
+
return finalPrompt;
|
|
105
|
+
}
|
|
106
|
+
catch (error) {
|
|
107
|
+
this.features.logging.error(`Memory Bank preparation failed: ${error}`);
|
|
108
|
+
throw error;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* Clean and recreate memory bank directory
|
|
113
|
+
*/
|
|
114
|
+
async cleanMemoryBankDirectory(workspaceFolderUri) {
|
|
115
|
+
try {
|
|
116
|
+
const normalizedWorkspacePath = (0, mcpUtils_1.normalizePathFromUri)(workspaceFolderUri, this.features.logging);
|
|
117
|
+
const memoryBankPath = `${normalizedWorkspacePath}/${MEMORY_BANK_DIRECTORY}`;
|
|
118
|
+
// Remove all existing memory bank files to ensure clean recreation
|
|
119
|
+
const filesToRemove = ['product.md', 'structure.md', 'tech.md', 'guidelines.md'];
|
|
120
|
+
let removedCount = 0;
|
|
121
|
+
for (const fileName of filesToRemove) {
|
|
122
|
+
const filePath = `${memoryBankPath}/${fileName}`;
|
|
123
|
+
try {
|
|
124
|
+
const exists = await this.features.workspace.fs.exists(filePath);
|
|
125
|
+
if (exists) {
|
|
126
|
+
await this.features.workspace.fs.rm(filePath);
|
|
127
|
+
removedCount++;
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
catch (error) {
|
|
131
|
+
// Ignore errors when removing files that don't exist
|
|
132
|
+
this.features.logging.error(`Could not remove ${fileName}: ${error}`);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
if (removedCount > 0) {
|
|
136
|
+
this.features.logging.info(`Memory Bank: cleaned ${removedCount} existing files`);
|
|
137
|
+
}
|
|
138
|
+
// Create the directory structure using mkdir with recursive option
|
|
139
|
+
await this.features.workspace.fs.mkdir(memoryBankPath, { recursive: true });
|
|
140
|
+
}
|
|
141
|
+
catch (error) {
|
|
142
|
+
this.features.logging.error(`Memory Bank directory creation failed: ${error}`);
|
|
143
|
+
throw error;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* files discovery
|
|
148
|
+
*/
|
|
149
|
+
async discoverAllSourceFiles(workspaceFolderUri, extensions) {
|
|
150
|
+
try {
|
|
151
|
+
// Recursively discover all source files
|
|
152
|
+
const allWorkspaceFolders = this.features.workspace.getAllWorkspaceFolders();
|
|
153
|
+
const workspaceFolders = allWorkspaceFolders?.map(({ uri }) => {
|
|
154
|
+
return (0, mcpUtils_1.normalizePathFromUri)(uri, this.features.logging);
|
|
155
|
+
}) ?? [(0, mcpUtils_1.normalizePathFromUri)(workspaceFolderUri, this.features.logging)];
|
|
156
|
+
// Collect files from all workspace folders
|
|
157
|
+
let allSourceFiles = [];
|
|
158
|
+
for (const folder of workspaceFolders) {
|
|
159
|
+
const sourceFiles = await this.discoverSourceFiles(folder, extensions);
|
|
160
|
+
this.features.logging.info(`Found ${sourceFiles.length} files in "${folder}"`);
|
|
161
|
+
allSourceFiles.push(...sourceFiles);
|
|
162
|
+
}
|
|
163
|
+
this.features.logging.info(`Total files discovered: ${allSourceFiles.length}`);
|
|
164
|
+
// OPTIMIZATION: Parallel file size calculation with batching
|
|
165
|
+
const batchSize = 10; // Process 10 files at a time
|
|
166
|
+
const files = [];
|
|
167
|
+
for (let i = 0; i < allSourceFiles.length; i += batchSize) {
|
|
168
|
+
const batch = allSourceFiles.slice(i, i + batchSize);
|
|
169
|
+
const batchResults = await Promise.all(batch.map(async (filePath) => ({
|
|
170
|
+
path: filePath,
|
|
171
|
+
size: await this.calculateFileLineCount(filePath),
|
|
172
|
+
})));
|
|
173
|
+
files.push(...batchResults);
|
|
174
|
+
}
|
|
175
|
+
return files;
|
|
176
|
+
}
|
|
177
|
+
catch (error) {
|
|
178
|
+
this.features.logging.error(`Error in getAllFiles: ${error}`);
|
|
179
|
+
return [];
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
/**
|
|
183
|
+
* line counting
|
|
184
|
+
*/
|
|
185
|
+
async calculateFileLineCount(filePath) {
|
|
186
|
+
try {
|
|
187
|
+
const content = await this.features.workspace.fs.readFile(filePath);
|
|
188
|
+
return content.split('\n').length;
|
|
189
|
+
}
|
|
190
|
+
catch (error) {
|
|
191
|
+
this.features.logging.error(`Error reading file ${filePath}: ${error}`);
|
|
192
|
+
return 0;
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
/**
|
|
196
|
+
* lexical dissimilarity calculation
|
|
197
|
+
*/
|
|
198
|
+
async calculateLexicalDissimilarity(files) {
|
|
199
|
+
try {
|
|
200
|
+
// OPTIMIZATION: Parallel file reading with batching
|
|
201
|
+
const batchSize = 20; // Process 20 files at a time to reduce I/O overhead
|
|
202
|
+
const fileContents = [];
|
|
203
|
+
let hasReadErrors = false;
|
|
204
|
+
for (let i = 0; i < files.length; i += batchSize) {
|
|
205
|
+
const batch = files.slice(i, i + batchSize);
|
|
206
|
+
const batchContents = await Promise.all(batch.map(async (file) => {
|
|
207
|
+
try {
|
|
208
|
+
return await this.features.workspace.fs.readFile(file.path);
|
|
209
|
+
}
|
|
210
|
+
catch (error) {
|
|
211
|
+
this.features.logging.warn(`Could not read file for TF-IDF analysis: ${file.path}`);
|
|
212
|
+
hasReadErrors = true;
|
|
213
|
+
return ''; // Empty content for unreadable files
|
|
214
|
+
}
|
|
215
|
+
}));
|
|
216
|
+
fileContents.push(...batchContents);
|
|
217
|
+
}
|
|
218
|
+
// Check if all files are empty (no content to analyze)
|
|
219
|
+
const hasContent = fileContents.some(content => content.trim().length > 0);
|
|
220
|
+
if (!hasContent) {
|
|
221
|
+
// If no files have content due to read errors, log as error
|
|
222
|
+
if (hasReadErrors) {
|
|
223
|
+
this.features.logging.error('All files failed to read or are empty, using fallback dissimilarity values');
|
|
224
|
+
}
|
|
225
|
+
// If no files have content, return fallback values
|
|
226
|
+
return files.map(f => ({ ...f, dissimilarity: 0.85 }));
|
|
227
|
+
}
|
|
228
|
+
// Step 2: Get the TF-IDF vectors for each file (equivalent to sklearn's TfidfVectorizer)
|
|
229
|
+
const tfidfMatrix = this.createTfidfMatrix(fileContents);
|
|
230
|
+
// Step 3: Get the cosine similarity of each file (equivalent to sklearn's cosine_similarity)
|
|
231
|
+
const cosineSimilarities = this.calculateCosineSimilarityMatrix(tfidfMatrix);
|
|
232
|
+
// Step 4: Get the lexical dissimilarity of each file (1 - similarity)
|
|
233
|
+
const lexicalDissimilarities = [];
|
|
234
|
+
for (let i = 0; i < cosineSimilarities.length; i++) {
|
|
235
|
+
// Calculate mean similarity for this file with all files (including itself)
|
|
236
|
+
const meanSimilarity = cosineSimilarities[i].reduce((sum, sim) => sum + sim, 0) / cosineSimilarities[i].length;
|
|
237
|
+
// Dissimilarity = 1 - mean_similarity (exactly like Python code)
|
|
238
|
+
const dissimilarity = 1 - meanSimilarity;
|
|
239
|
+
lexicalDissimilarities.push({
|
|
240
|
+
path: files[i].path,
|
|
241
|
+
size: files[i].size,
|
|
242
|
+
dissimilarity: Math.max(0.0, Math.min(1.0, dissimilarity)), // Ensure bounds [0,1]
|
|
243
|
+
});
|
|
244
|
+
}
|
|
245
|
+
return lexicalDissimilarities;
|
|
246
|
+
}
|
|
247
|
+
catch (error) {
|
|
248
|
+
this.features.logging.error(`Error in calculateLexicalDissimilarity: ${error}`);
|
|
249
|
+
// Fallback to reasonable defaults if TF-IDF calculation fails
|
|
250
|
+
return files.map(f => ({ ...f, dissimilarity: 0.85 }));
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
/**
|
|
254
|
+
* Create TF-IDF matrix, Returns array of TF-IDF vectors, where each vector is a Map<term, tfidf_score>
|
|
255
|
+
*/
|
|
256
|
+
createTfidfMatrix(documents) {
|
|
257
|
+
// Step 1: Tokenize all documents and build vocabulary
|
|
258
|
+
const tokenizedDocs = documents.map(doc => this.tokenizeDocument(doc));
|
|
259
|
+
const vocabulary = new Set();
|
|
260
|
+
tokenizedDocs.forEach(tokens => tokens.forEach(token => vocabulary.add(token)));
|
|
261
|
+
const vocabArray = Array.from(vocabulary);
|
|
262
|
+
const numDocs = documents.length;
|
|
263
|
+
// Step 2: Calculate document frequencies (DF)
|
|
264
|
+
const documentFrequencies = new Map();
|
|
265
|
+
vocabArray.forEach(term => {
|
|
266
|
+
const df = tokenizedDocs.filter(tokens => tokens.includes(term)).length;
|
|
267
|
+
documentFrequencies.set(term, df);
|
|
268
|
+
});
|
|
269
|
+
// Step 3: Calculate TF-IDF for each document
|
|
270
|
+
const tfidfMatrix = [];
|
|
271
|
+
for (let docIndex = 0; docIndex < numDocs; docIndex++) {
|
|
272
|
+
const tokens = tokenizedDocs[docIndex];
|
|
273
|
+
const tfidfVector = new Map();
|
|
274
|
+
// Calculate term frequencies for this document
|
|
275
|
+
const termFrequencies = new Map();
|
|
276
|
+
tokens.forEach(token => {
|
|
277
|
+
termFrequencies.set(token, (termFrequencies.get(token) || 0) + 1);
|
|
278
|
+
});
|
|
279
|
+
// Calculate TF-IDF for each term in vocabulary
|
|
280
|
+
vocabArray.forEach(term => {
|
|
281
|
+
const tf = termFrequencies.get(term) || 0;
|
|
282
|
+
const df = documentFrequencies.get(term) || 1;
|
|
283
|
+
const idf = Math.log(numDocs / df);
|
|
284
|
+
const tfidf = tf * idf;
|
|
285
|
+
tfidfVector.set(term, tfidf);
|
|
286
|
+
});
|
|
287
|
+
tfidfMatrix.push(tfidfVector);
|
|
288
|
+
}
|
|
289
|
+
return tfidfMatrix;
|
|
290
|
+
}
|
|
291
|
+
/**
|
|
292
|
+
* Calculate cosine similarity matrix
|
|
293
|
+
*/
|
|
294
|
+
calculateCosineSimilarityMatrix(tfidfMatrix) {
|
|
295
|
+
const numDocs = tfidfMatrix.length;
|
|
296
|
+
const similarities = [];
|
|
297
|
+
for (let i = 0; i < numDocs; i++) {
|
|
298
|
+
const row = [];
|
|
299
|
+
for (let j = 0; j < numDocs; j++) {
|
|
300
|
+
const similarity = this.calculateCosineSimilarity(tfidfMatrix[i], tfidfMatrix[j]);
|
|
301
|
+
row.push(similarity);
|
|
302
|
+
}
|
|
303
|
+
similarities.push(row);
|
|
304
|
+
}
|
|
305
|
+
return similarities;
|
|
306
|
+
}
|
|
307
|
+
/**
|
|
308
|
+
* Calculate cosine similarity between two TF-IDF vectors
|
|
309
|
+
*/
|
|
310
|
+
calculateCosineSimilarity(vectorA, vectorB) {
|
|
311
|
+
let dotProduct = 0;
|
|
312
|
+
let normA = 0;
|
|
313
|
+
let normB = 0;
|
|
314
|
+
// Get all unique terms from both vectors
|
|
315
|
+
const allTerms = new Set([...vectorA.keys(), ...vectorB.keys()]);
|
|
316
|
+
allTerms.forEach(term => {
|
|
317
|
+
const valueA = vectorA.get(term) || 0;
|
|
318
|
+
const valueB = vectorB.get(term) || 0;
|
|
319
|
+
dotProduct += valueA * valueB;
|
|
320
|
+
normA += valueA * valueA;
|
|
321
|
+
normB += valueB * valueB;
|
|
322
|
+
});
|
|
323
|
+
// Avoid division by zero
|
|
324
|
+
if (normA === 0 || normB === 0) {
|
|
325
|
+
return 0;
|
|
326
|
+
}
|
|
327
|
+
return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
|
|
328
|
+
}
|
|
329
|
+
/**
|
|
330
|
+
* Tokenize document into terms (simple whitespace + punctuation splitting)
|
|
331
|
+
*/
|
|
332
|
+
tokenizeDocument(document) {
|
|
333
|
+
return document
|
|
334
|
+
.toLowerCase()
|
|
335
|
+
.replace(/[^\w\s]/g, ' ') // Replace punctuation with spaces
|
|
336
|
+
.split(/\s+/) // Split on whitespace
|
|
337
|
+
.filter(token => token.length > 2); // Filter out very short tokens
|
|
338
|
+
}
|
|
339
|
+
/**
|
|
340
|
+
* Execute the complete guidelines generation pipeline
|
|
341
|
+
* https://code.amazon.com/packages/QIDEPersonalization/blobs/mainline/--/src/stylefile-gen.ipynb
|
|
342
|
+
*/
|
|
343
|
+
async executeGuidelinesGenerationPipeline(workspaceFolderUri) {
|
|
344
|
+
try {
|
|
345
|
+
// Step 1: Discover all source files
|
|
346
|
+
// OPTIMIZATION: Prioritize common extensions first for faster discovery
|
|
347
|
+
const extensions = [
|
|
348
|
+
'.ts',
|
|
349
|
+
'.js',
|
|
350
|
+
'.tsx',
|
|
351
|
+
'.jsx',
|
|
352
|
+
'.py',
|
|
353
|
+
'.java',
|
|
354
|
+
'.cpp',
|
|
355
|
+
'.c',
|
|
356
|
+
'.h',
|
|
357
|
+
'.cs',
|
|
358
|
+
'.go',
|
|
359
|
+
'.rs',
|
|
360
|
+
'.php',
|
|
361
|
+
'.rb',
|
|
362
|
+
'.swift',
|
|
363
|
+
'.kt',
|
|
364
|
+
'.scala',
|
|
365
|
+
];
|
|
366
|
+
const discoveredFiles = await this.discoverAllSourceFiles(workspaceFolderUri, extensions);
|
|
367
|
+
if (discoveredFiles.length === 0) {
|
|
368
|
+
throw new Error('No source files found in workspace');
|
|
369
|
+
}
|
|
370
|
+
// Filter out very large files to prevent conversation overflow
|
|
371
|
+
const MAX_FILE_SIZE_FOR_MEMORY_BANK = 20000; // 20KB limit
|
|
372
|
+
const reasonableSizedFiles = discoveredFiles.filter(file => file.size <= MAX_FILE_SIZE_FOR_MEMORY_BANK);
|
|
373
|
+
this.features.logging.debug(`Memory Bank analysis: filtered ${discoveredFiles.length - reasonableSizedFiles.length} files over ${MAX_FILE_SIZE_FOR_MEMORY_BANK} characters`);
|
|
374
|
+
// Limit files to prevent memory exhaustion on large projects
|
|
375
|
+
const MAX_FILES_FOR_ANALYSIS = 200;
|
|
376
|
+
let filesToAnalyze;
|
|
377
|
+
if (reasonableSizedFiles.length > MAX_FILES_FOR_ANALYSIS) {
|
|
378
|
+
const shuffled = [...reasonableSizedFiles].sort(() => Math.random() - 0.5);
|
|
379
|
+
filesToAnalyze = shuffled.slice(0, MAX_FILES_FOR_ANALYSIS);
|
|
380
|
+
this.features.logging.info(`Memory Bank analysis: randomly selected ${filesToAnalyze.length} files (from ${reasonableSizedFiles.length} reasonable-sized files for ranking)`);
|
|
381
|
+
}
|
|
382
|
+
else {
|
|
383
|
+
filesToAnalyze = reasonableSizedFiles;
|
|
384
|
+
}
|
|
385
|
+
// Step 2: Calculate lexical dissimilarity using TF-IDF
|
|
386
|
+
const filesWithDissimilarity = await this.calculateLexicalDissimilarity(filesToAnalyze);
|
|
387
|
+
// Step 3: Sort by size
|
|
388
|
+
filesWithDissimilarity.sort((a, b) => b.size - a.size);
|
|
389
|
+
// Step 4: Format files string for LLM ranking
|
|
390
|
+
const formattedFilesString = this.formatFilesForRanking(filesWithDissimilarity);
|
|
391
|
+
// Step 5: Create fallback ranking (deterministic, for when LLM fails)
|
|
392
|
+
const rankedFilesList = filesWithDissimilarity
|
|
393
|
+
.sort((a, b) => b.dissimilarity - a.dissimilarity)
|
|
394
|
+
.slice(0, constants_1.MAX_NUMBER_OF_FILES_FOR_MEMORY_BANK_RANKING)
|
|
395
|
+
.map(f => f.path);
|
|
396
|
+
return {
|
|
397
|
+
discoveredFiles: filesToAnalyze,
|
|
398
|
+
filesWithDissimilarity,
|
|
399
|
+
formattedFilesString,
|
|
400
|
+
rankedFilesList,
|
|
401
|
+
};
|
|
402
|
+
}
|
|
403
|
+
catch (error) {
|
|
404
|
+
this.features.logging.error(`Memory Bank analysis pipeline failed: ${error}`);
|
|
405
|
+
throw error;
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
/**
|
|
409
|
+
* Format files for processing pipeline
|
|
410
|
+
*/
|
|
411
|
+
formatFilesForRanking(files) {
|
|
412
|
+
// Files are already sorted by size in executeGuidelinesGenerationPipeline()
|
|
413
|
+
return files
|
|
414
|
+
.map(f => `${f.path} has ${f.size} lines and a mean lexical dissimilarity of ${f.dissimilarity.toFixed(6)} to the other files`)
|
|
415
|
+
.join('\n');
|
|
416
|
+
}
|
|
417
|
+
/**
|
|
418
|
+
* Recursively discover source files with given extensions
|
|
419
|
+
*/
|
|
420
|
+
async discoverSourceFiles(workspaceFolderUri, extensions) {
|
|
421
|
+
const sourceFiles = [];
|
|
422
|
+
const traverseDirectory = async (dirPath) => {
|
|
423
|
+
try {
|
|
424
|
+
const entries = await this.features.workspace.fs.readdir(dirPath);
|
|
425
|
+
for (const entry of entries) {
|
|
426
|
+
const fullPath = `${dirPath}/${entry.name}`;
|
|
427
|
+
// Skip common directories that don't contain source code
|
|
428
|
+
if (entry.isDirectory() && this.shouldSkipDirectory(entry.name)) {
|
|
429
|
+
continue;
|
|
430
|
+
}
|
|
431
|
+
if (entry.isDirectory()) {
|
|
432
|
+
// Directory - recurse
|
|
433
|
+
await traverseDirectory(fullPath);
|
|
434
|
+
}
|
|
435
|
+
else {
|
|
436
|
+
// File - check if it's a source file
|
|
437
|
+
if (extensions.some(ext => entry.name.endsWith(ext))) {
|
|
438
|
+
sourceFiles.push(fullPath);
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
catch (error) {
|
|
444
|
+
this.features.logging.error(`Could not read directory ${dirPath}: ${error}`);
|
|
445
|
+
}
|
|
446
|
+
};
|
|
447
|
+
await traverseDirectory(workspaceFolderUri);
|
|
448
|
+
return sourceFiles;
|
|
449
|
+
}
|
|
450
|
+
/**
|
|
451
|
+
* Check if a directory should be skipped during source file discovery
|
|
452
|
+
*/
|
|
453
|
+
shouldSkipDirectory(dirName) {
|
|
454
|
+
// Comprehensive language-agnostic directory exclusions
|
|
455
|
+
const skipDirs = [
|
|
456
|
+
// Version Control Systems
|
|
457
|
+
'.git',
|
|
458
|
+
'.svn',
|
|
459
|
+
'.hg',
|
|
460
|
+
'.bzr',
|
|
461
|
+
'.fossil-settings',
|
|
462
|
+
// Package Managers & Dependencies
|
|
463
|
+
'node_modules',
|
|
464
|
+
'bower_components',
|
|
465
|
+
'jspm_packages',
|
|
466
|
+
'vendor',
|
|
467
|
+
'packages',
|
|
468
|
+
'deps',
|
|
469
|
+
'_deps',
|
|
470
|
+
'third_party',
|
|
471
|
+
'external',
|
|
472
|
+
'Pods',
|
|
473
|
+
'Carthage',
|
|
474
|
+
'DerivedData', // iOS/macOS
|
|
475
|
+
'venv',
|
|
476
|
+
'env',
|
|
477
|
+
'.venv',
|
|
478
|
+
'.env',
|
|
479
|
+
'virtualenv',
|
|
480
|
+
'__pycache__',
|
|
481
|
+
'.tox', // Python
|
|
482
|
+
'gems',
|
|
483
|
+
'.bundle', // Ruby
|
|
484
|
+
'composer', // PHP
|
|
485
|
+
'node_modules',
|
|
486
|
+
'elm-stuff', // Elm
|
|
487
|
+
'target',
|
|
488
|
+
'project/target',
|
|
489
|
+
'project/project', // Scala/SBT
|
|
490
|
+
// Build Outputs & Artifacts
|
|
491
|
+
'build',
|
|
492
|
+
'builds',
|
|
493
|
+
'dist',
|
|
494
|
+
'out',
|
|
495
|
+
'output',
|
|
496
|
+
'bin',
|
|
497
|
+
'obj',
|
|
498
|
+
'lib',
|
|
499
|
+
'release',
|
|
500
|
+
'debug',
|
|
501
|
+
'Release',
|
|
502
|
+
'Debug',
|
|
503
|
+
'x64',
|
|
504
|
+
'x86',
|
|
505
|
+
'AnyCPU',
|
|
506
|
+
'.next',
|
|
507
|
+
'.nuxt',
|
|
508
|
+
'.output',
|
|
509
|
+
'.vercel',
|
|
510
|
+
'.netlify', // Web frameworks
|
|
511
|
+
'public/build',
|
|
512
|
+
'static/build',
|
|
513
|
+
'assets/build',
|
|
514
|
+
'cmake-build-debug',
|
|
515
|
+
'cmake-build-release', // CMake
|
|
516
|
+
'_build',
|
|
517
|
+
'ebin',
|
|
518
|
+
'deps', // Erlang/Elixir
|
|
519
|
+
'zig-cache',
|
|
520
|
+
'zig-out', // Zig
|
|
521
|
+
// IDE & Editor Directories
|
|
522
|
+
'.vscode',
|
|
523
|
+
'.idea',
|
|
524
|
+
'.vs',
|
|
525
|
+
'.vscode-test',
|
|
526
|
+
'.eclipse',
|
|
527
|
+
'.metadata',
|
|
528
|
+
'.settings',
|
|
529
|
+
'.project',
|
|
530
|
+
'.classpath',
|
|
531
|
+
'.atom',
|
|
532
|
+
'.sublime-project',
|
|
533
|
+
'.sublime-workspace',
|
|
534
|
+
'__pycache__',
|
|
535
|
+
'.mypy_cache',
|
|
536
|
+
'.dmypy.json', // Python
|
|
537
|
+
'.dart_tool',
|
|
538
|
+
'.flutter-plugins',
|
|
539
|
+
'.flutter-plugins-dependencies', // Dart/Flutter
|
|
540
|
+
// Testing & Coverage
|
|
541
|
+
'coverage',
|
|
542
|
+
'.coverage',
|
|
543
|
+
'.nyc_output',
|
|
544
|
+
'.pytest_cache',
|
|
545
|
+
'.cache',
|
|
546
|
+
'htmlcov',
|
|
547
|
+
'test-results',
|
|
548
|
+
'test-reports',
|
|
549
|
+
'allure-results',
|
|
550
|
+
'junit',
|
|
551
|
+
'xunit',
|
|
552
|
+
'nunit',
|
|
553
|
+
'TestResults',
|
|
554
|
+
'.jest',
|
|
555
|
+
'jest_html_reporters.html',
|
|
556
|
+
// Logs & Temporary Files
|
|
557
|
+
'logs',
|
|
558
|
+
'log',
|
|
559
|
+
'tmp',
|
|
560
|
+
'temp',
|
|
561
|
+
'.tmp',
|
|
562
|
+
'.temp',
|
|
563
|
+
'crash-reports',
|
|
564
|
+
'error-reports',
|
|
565
|
+
// Documentation Build Outputs
|
|
566
|
+
'_site',
|
|
567
|
+
'.jekyll-cache',
|
|
568
|
+
'.jekyll-metadata', // Jekyll
|
|
569
|
+
'docs/_build',
|
|
570
|
+
'doc/_build',
|
|
571
|
+
'documentation/_build', // Sphinx
|
|
572
|
+
'.docusaurus',
|
|
573
|
+
'website/build', // Docusaurus
|
|
574
|
+
'book',
|
|
575
|
+
'_book', // GitBook/mdBook
|
|
576
|
+
// Language-Specific Caches & Artifacts
|
|
577
|
+
'.gradle',
|
|
578
|
+
'gradle', // Gradle
|
|
579
|
+
'.m2',
|
|
580
|
+
'.ivy2', // Maven/Ivy
|
|
581
|
+
'.stack-work',
|
|
582
|
+
'.cabal-sandbox',
|
|
583
|
+
'cabal.sandbox.config', // Haskell
|
|
584
|
+
'_opam',
|
|
585
|
+
'.opam', // OCaml
|
|
586
|
+
'Cargo.lock', // Rust (keep Cargo.toml but skip lock in some cases)
|
|
587
|
+
'.cargo', // Rust cache
|
|
588
|
+
'.mix',
|
|
589
|
+
'_build', // Elixir
|
|
590
|
+
'rebar3.crashdump',
|
|
591
|
+
'_checkouts', // Erlang
|
|
592
|
+
'.rebar',
|
|
593
|
+
'.rebar3',
|
|
594
|
+
'priv/static', // Phoenix framework
|
|
595
|
+
// OS-Specific
|
|
596
|
+
'.DS_Store',
|
|
597
|
+
'Thumbs.db',
|
|
598
|
+
'Desktop.ini',
|
|
599
|
+
'$RECYCLE.BIN',
|
|
600
|
+
'.Trash-*',
|
|
601
|
+
'.fuse_hidden*',
|
|
602
|
+
// Cloud & Deployment
|
|
603
|
+
'.serverless',
|
|
604
|
+
'.aws-sam',
|
|
605
|
+
'.terraform',
|
|
606
|
+
'.pulumi',
|
|
607
|
+
'cdk.out',
|
|
608
|
+
'.cdk.staging',
|
|
609
|
+
'amplify',
|
|
610
|
+
// Mobile Development
|
|
611
|
+
'ios/build',
|
|
612
|
+
'android/build',
|
|
613
|
+
'android/.gradle',
|
|
614
|
+
'ios/Pods',
|
|
615
|
+
'android/app/build',
|
|
616
|
+
// Game Development
|
|
617
|
+
'Library',
|
|
618
|
+
'Temp',
|
|
619
|
+
'Obj',
|
|
620
|
+
'Build',
|
|
621
|
+
'Builds', // Unity
|
|
622
|
+
'Intermediate',
|
|
623
|
+
'Binaries',
|
|
624
|
+
'DerivedDataCache', // Unreal
|
|
625
|
+
// Database
|
|
626
|
+
'*.db-journal',
|
|
627
|
+
'*.sqlite-journal',
|
|
628
|
+
// Backup & Archive
|
|
629
|
+
'backup',
|
|
630
|
+
'backups',
|
|
631
|
+
'.backup',
|
|
632
|
+
'archive',
|
|
633
|
+
'archives',
|
|
634
|
+
];
|
|
635
|
+
// Skip any directory starting with . (hidden directories) except some important ones
|
|
636
|
+
if (dirName.startsWith('.')) {
|
|
637
|
+
const allowedHiddenDirs = ['.github', '.gitlab', '.circleci', '.travis', '.azure', '.devcontainer'];
|
|
638
|
+
return !allowedHiddenDirs.includes(dirName);
|
|
639
|
+
}
|
|
640
|
+
return skipDirs.includes(dirName);
|
|
641
|
+
}
|
|
642
|
+
/**
|
|
643
|
+
* Check if memory bank exists in workspace
|
|
644
|
+
*/
|
|
645
|
+
async memoryBankExists(workspaceFolderUri) {
|
|
646
|
+
try {
|
|
647
|
+
const normalizedWorkspacePath = (0, mcpUtils_1.normalizePathFromUri)(workspaceFolderUri, this.features.logging);
|
|
648
|
+
const memoryBankPath = `${normalizedWorkspacePath}/${MEMORY_BANK_DIRECTORY}`;
|
|
649
|
+
this.features.logging.info(`Memory Bank: Checking existence at path: "${memoryBankPath}"`);
|
|
650
|
+
const exists = await this.features.workspace.fs.exists(memoryBankPath);
|
|
651
|
+
if (!exists) {
|
|
652
|
+
this.features.logging.info(`Memory Bank: Directory does not exist: "${memoryBankPath}"`);
|
|
653
|
+
return false;
|
|
654
|
+
}
|
|
655
|
+
// Check if at least one memory bank file exists
|
|
656
|
+
const files = Object.values(MEMORY_BANK_FILES);
|
|
657
|
+
let foundFiles = 0;
|
|
658
|
+
for (const file of files) {
|
|
659
|
+
const filePath = `${memoryBankPath}/${file}`;
|
|
660
|
+
const fileExists = await this.features.workspace.fs.exists(filePath);
|
|
661
|
+
if (fileExists) {
|
|
662
|
+
foundFiles++;
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
const hasFiles = foundFiles > 0;
|
|
666
|
+
if (hasFiles) {
|
|
667
|
+
this.features.logging.info(`Memory Bank: Found ${foundFiles} existing memory bank files`);
|
|
668
|
+
}
|
|
669
|
+
else {
|
|
670
|
+
this.features.logging.info(`Memory Bank: No existing memory bank files found`);
|
|
671
|
+
}
|
|
672
|
+
return hasFiles;
|
|
673
|
+
}
|
|
674
|
+
catch (error) {
|
|
675
|
+
this.features.logging.error(`Error checking memory bank existence: ${error}`);
|
|
676
|
+
return false;
|
|
677
|
+
}
|
|
678
|
+
}
|
|
679
|
+
}
|
|
680
|
+
exports.MemoryBankController = MemoryBankController;
|
|
681
|
+
//# sourceMappingURL=memoryBankController.js.map
|