@juspay/yama 1.1.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/README.md +1 -1
- package/dist/core/ContextGatherer.d.ts +8 -3
- package/dist/core/ContextGatherer.js +40 -21
- package/dist/core/Guardian.js +5 -1
- package/dist/features/CodeReviewer.d.ts +4 -0
- package/dist/features/CodeReviewer.js +24 -1
- package/dist/types/index.d.ts +6 -0
- package/dist/utils/Cache.d.ts +5 -0
- package/dist/utils/Cache.js +16 -0
- package/dist/utils/ConfigManager.d.ts +4 -0
- package/dist/utils/ConfigManager.js +21 -1
- package/dist/utils/MemoryBankManager.d.ts +73 -0
- package/dist/utils/MemoryBankManager.js +310 -0
- package/dist/utils/ProviderLimits.d.ts +58 -0
- package/dist/utils/ProviderLimits.js +143 -0
- package/package.json +1 -1
- package/yama.config.example.yaml +10 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,10 @@
|
|
|
1
|
+
# [1.2.0](https://github.com/juspay/yama/compare/v1.1.1...v1.2.0) (2025-08-08)
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
### Features
|
|
5
|
+
|
|
6
|
+
* **Memory:** support memory bank path and maxToken from config file ([1bc69d5](https://github.com/juspay/yama/commit/1bc69d5bda3ac5868d7537b881007beaf6916476))
|
|
7
|
+
|
|
1
8
|
## [1.1.1](https://github.com/juspay/yama/compare/v1.1.0...v1.1.1) (2025-07-28)
|
|
2
9
|
|
|
3
10
|
|
package/README.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Unified Context Gatherer - The foundation for all Yama operations
|
|
3
3
|
* Gathers all necessary context once and reuses it across all operations
|
|
4
4
|
*/
|
|
5
|
-
import { PRIdentifier, PRInfo, PRDiff, AIProviderConfig, DiffStrategyConfig } from "../types/index.js";
|
|
5
|
+
import { PRIdentifier, PRInfo, PRDiff, AIProviderConfig, DiffStrategyConfig, MemoryBankConfig } from "../types/index.js";
|
|
6
6
|
import { BitbucketProvider } from "./providers/BitbucketProvider.js";
|
|
7
7
|
export interface ProjectContext {
|
|
8
8
|
memoryBank: {
|
|
@@ -36,8 +36,9 @@ export declare class ContextGatherer {
|
|
|
36
36
|
private neurolink;
|
|
37
37
|
private bitbucketProvider;
|
|
38
38
|
private aiConfig;
|
|
39
|
+
private memoryBankManager;
|
|
39
40
|
private startTime;
|
|
40
|
-
constructor(bitbucketProvider: BitbucketProvider, aiConfig: AIProviderConfig);
|
|
41
|
+
constructor(bitbucketProvider: BitbucketProvider, aiConfig: AIProviderConfig, memoryBankConfig: MemoryBankConfig);
|
|
41
42
|
/**
|
|
42
43
|
* Main context gathering method - used by all operations
|
|
43
44
|
*/
|
|
@@ -56,6 +57,10 @@ export declare class ContextGatherer {
|
|
|
56
57
|
* Step 2: Gather project context (memory bank + clinerules)
|
|
57
58
|
*/
|
|
58
59
|
private gatherProjectContext;
|
|
60
|
+
/**
|
|
61
|
+
* Get safe token limit based on AI provider using shared utility
|
|
62
|
+
*/
|
|
63
|
+
private getSafeTokenLimit;
|
|
59
64
|
/**
|
|
60
65
|
* Parse project context with AI
|
|
61
66
|
*/
|
|
@@ -101,5 +106,5 @@ export declare class ContextGatherer {
|
|
|
101
106
|
*/
|
|
102
107
|
getStats(): any;
|
|
103
108
|
}
|
|
104
|
-
export declare function createContextGatherer(bitbucketProvider: BitbucketProvider, aiConfig: AIProviderConfig): ContextGatherer;
|
|
109
|
+
export declare function createContextGatherer(bitbucketProvider: BitbucketProvider, aiConfig: AIProviderConfig, memoryBankConfig: MemoryBankConfig): ContextGatherer;
|
|
105
110
|
//# sourceMappingURL=ContextGatherer.d.ts.map
|
|
@@ -6,14 +6,18 @@
|
|
|
6
6
|
import { ProviderError, } from "../types/index.js";
|
|
7
7
|
import { logger } from "../utils/Logger.js";
|
|
8
8
|
import { cache, Cache } from "../utils/Cache.js";
|
|
9
|
+
import { createMemoryBankManager } from "../utils/MemoryBankManager.js";
|
|
10
|
+
import { getProviderTokenLimit } from "../utils/ProviderLimits.js";
|
|
9
11
|
export class ContextGatherer {
|
|
10
12
|
neurolink;
|
|
11
13
|
bitbucketProvider;
|
|
12
14
|
aiConfig;
|
|
15
|
+
memoryBankManager;
|
|
13
16
|
startTime = 0;
|
|
14
|
-
constructor(bitbucketProvider, aiConfig) {
|
|
17
|
+
constructor(bitbucketProvider, aiConfig, memoryBankConfig) {
|
|
15
18
|
this.bitbucketProvider = bitbucketProvider;
|
|
16
19
|
this.aiConfig = aiConfig;
|
|
20
|
+
this.memoryBankManager = createMemoryBankManager(memoryBankConfig, bitbucketProvider);
|
|
17
21
|
}
|
|
18
22
|
/**
|
|
19
23
|
* Main context gathering method - used by all operations
|
|
@@ -122,10 +126,10 @@ export class ContextGatherer {
|
|
|
122
126
|
}
|
|
123
127
|
return cache.getOrSet(cacheKey, async () => {
|
|
124
128
|
try {
|
|
125
|
-
//
|
|
126
|
-
const
|
|
127
|
-
if (
|
|
128
|
-
logger.debug("No memory
|
|
129
|
+
// Use MemoryBankManager to get memory bank files
|
|
130
|
+
const memoryBankResult = await this.memoryBankManager.getMemoryBankFiles(identifier, forceRefresh);
|
|
131
|
+
if (memoryBankResult.files.length === 0) {
|
|
132
|
+
logger.debug("No memory bank files found");
|
|
129
133
|
return {
|
|
130
134
|
memoryBank: {
|
|
131
135
|
summary: "No project context available",
|
|
@@ -137,19 +141,12 @@ export class ContextGatherer {
|
|
|
137
141
|
filesProcessed: 0,
|
|
138
142
|
};
|
|
139
143
|
}
|
|
140
|
-
//
|
|
144
|
+
// Convert MemoryBankFile[] to Record<string, string> for AI processing
|
|
141
145
|
const fileContents = {};
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
await this.bitbucketProvider.getFileContent(identifier.workspace, identifier.repository, `memory-bank/${file.name}`, identifier.branch || "main");
|
|
147
|
-
logger.debug(`✓ Got content for: ${file.name}`);
|
|
148
|
-
}
|
|
149
|
-
catch (error) {
|
|
150
|
-
logger.debug(`Could not read file ${file.name}: ${error.message}`);
|
|
151
|
-
}
|
|
152
|
-
}
|
|
146
|
+
memoryBankResult.files.forEach((file) => {
|
|
147
|
+
fileContents[file.name] = file.content;
|
|
148
|
+
});
|
|
149
|
+
logger.debug(`✓ Loaded ${memoryBankResult.files.length} memory bank files from ${memoryBankResult.resolvedPath}${memoryBankResult.fallbackUsed ? " (fallback)" : ""}`);
|
|
153
150
|
// Get .clinerules file
|
|
154
151
|
let clinerules = "";
|
|
155
152
|
try {
|
|
@@ -171,7 +168,7 @@ Standards: ${contextData.standards}`,
|
|
|
171
168
|
standards: contextData.standards,
|
|
172
169
|
},
|
|
173
170
|
clinerules,
|
|
174
|
-
filesProcessed:
|
|
171
|
+
filesProcessed: memoryBankResult.filesProcessed,
|
|
175
172
|
};
|
|
176
173
|
}
|
|
177
174
|
catch (error) {
|
|
@@ -189,6 +186,23 @@ Standards: ${contextData.standards}`,
|
|
|
189
186
|
}
|
|
190
187
|
}, 7200);
|
|
191
188
|
}
|
|
189
|
+
/**
|
|
190
|
+
* Get safe token limit based on AI provider using shared utility
|
|
191
|
+
*/
|
|
192
|
+
getSafeTokenLimit() {
|
|
193
|
+
const provider = this.aiConfig.provider || "auto";
|
|
194
|
+
const configuredTokens = this.aiConfig.maxTokens;
|
|
195
|
+
// Use conservative limits for ContextGatherer (safer for large context processing)
|
|
196
|
+
const providerLimit = getProviderTokenLimit(provider, true);
|
|
197
|
+
// Use the smaller of configured tokens or provider limit
|
|
198
|
+
if (configuredTokens && configuredTokens > 0) {
|
|
199
|
+
const safeLimit = Math.min(configuredTokens, providerLimit);
|
|
200
|
+
logger.debug(`Token limit: configured=${configuredTokens}, provider=${providerLimit}, using=${safeLimit}`);
|
|
201
|
+
return safeLimit;
|
|
202
|
+
}
|
|
203
|
+
logger.debug(`Token limit: using provider default=${providerLimit} for ${provider}`);
|
|
204
|
+
return providerLimit;
|
|
205
|
+
}
|
|
192
206
|
/**
|
|
193
207
|
* Parse project context with AI
|
|
194
208
|
*/
|
|
@@ -219,13 +233,18 @@ Extract and summarize the content and return ONLY this JSON format:
|
|
|
219
233
|
hasClinerules: !!clinerules,
|
|
220
234
|
analysisType: "memory-bank-synthesis",
|
|
221
235
|
};
|
|
236
|
+
// Get safe token limit based on provider
|
|
237
|
+
const safeMaxTokens = this.getSafeTokenLimit();
|
|
238
|
+
logger.debug(`Using AI provider: ${this.aiConfig.provider || "auto"}`);
|
|
239
|
+
logger.debug(`Configured maxTokens: ${this.aiConfig.maxTokens}`);
|
|
240
|
+
logger.debug(`Safe maxTokens limit: ${safeMaxTokens}`);
|
|
222
241
|
const result = await this.neurolink.generate({
|
|
223
242
|
input: { text: prompt },
|
|
224
243
|
systemPrompt: "You are an Expert Project Analyst. Synthesize project context from documentation and configuration files to help AI understand the codebase architecture, patterns, and business domain.",
|
|
225
244
|
provider: this.aiConfig.provider,
|
|
226
245
|
model: this.aiConfig.model,
|
|
227
246
|
temperature: 0.3,
|
|
228
|
-
maxTokens:
|
|
247
|
+
maxTokens: safeMaxTokens, // Use provider-aware safe token limit
|
|
229
248
|
timeout: "10m", // Allow longer processing for quality
|
|
230
249
|
context: aiContext,
|
|
231
250
|
enableAnalytics: this.aiConfig.enableAnalytics || true,
|
|
@@ -445,7 +464,7 @@ Extract and summarize the content and return ONLY this JSON format:
|
|
|
445
464
|
}
|
|
446
465
|
}
|
|
447
466
|
// Export factory function
|
|
448
|
-
export function createContextGatherer(bitbucketProvider, aiConfig) {
|
|
449
|
-
return new ContextGatherer(bitbucketProvider, aiConfig);
|
|
467
|
+
export function createContextGatherer(bitbucketProvider, aiConfig, memoryBankConfig) {
|
|
468
|
+
return new ContextGatherer(bitbucketProvider, aiConfig, memoryBankConfig);
|
|
450
469
|
}
|
|
451
470
|
//# sourceMappingURL=ContextGatherer.js.map
|
package/dist/core/Guardian.js
CHANGED
|
@@ -43,7 +43,11 @@ export class Guardian {
|
|
|
43
43
|
const { NeuroLink } = await import("@juspay/neurolink");
|
|
44
44
|
this.neurolink = new NeuroLink();
|
|
45
45
|
// Initialize core components
|
|
46
|
-
this.contextGatherer = new ContextGatherer(this.bitbucketProvider, this.config.providers.ai
|
|
46
|
+
this.contextGatherer = new ContextGatherer(this.bitbucketProvider, this.config.providers.ai, this.config.memoryBank || {
|
|
47
|
+
enabled: true,
|
|
48
|
+
path: "memory-bank",
|
|
49
|
+
fallbackPaths: ["docs/memory-bank", ".memory-bank"],
|
|
50
|
+
});
|
|
47
51
|
this.codeReviewer = new CodeReviewer(this.bitbucketProvider, this.config.providers.ai, this.config.features.codeReview);
|
|
48
52
|
this.descriptionEnhancer = new DescriptionEnhancer(this.bitbucketProvider, this.config.providers.ai);
|
|
49
53
|
this.initialized = true;
|
|
@@ -51,6 +51,10 @@ export declare class CodeReviewer {
|
|
|
51
51
|
* Legacy method - kept for compatibility but simplified
|
|
52
52
|
*/
|
|
53
53
|
private buildAnalysisPrompt;
|
|
54
|
+
/**
|
|
55
|
+
* Get safe token limit based on AI provider using shared utility
|
|
56
|
+
*/
|
|
57
|
+
private getSafeTokenLimit;
|
|
54
58
|
/**
|
|
55
59
|
* Analyze code with AI using the enhanced prompt
|
|
56
60
|
*/
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
// NeuroLink will be dynamically imported
|
|
6
6
|
import { ProviderError, } from "../types/index.js";
|
|
7
7
|
import { logger } from "../utils/Logger.js";
|
|
8
|
+
import { getProviderTokenLimit } from "../utils/ProviderLimits.js";
|
|
8
9
|
export class CodeReviewer {
|
|
9
10
|
neurolink;
|
|
10
11
|
bitbucketProvider;
|
|
@@ -397,6 +398,23 @@ Return ONLY valid JSON:
|
|
|
397
398
|
// Legacy method - now delegates to new structure
|
|
398
399
|
return this.buildCoreAnalysisPrompt(context);
|
|
399
400
|
}
|
|
401
|
+
/**
|
|
402
|
+
* Get safe token limit based on AI provider using shared utility
|
|
403
|
+
*/
|
|
404
|
+
getSafeTokenLimit() {
|
|
405
|
+
const provider = this.aiConfig.provider || "auto";
|
|
406
|
+
const configuredTokens = this.aiConfig.maxTokens;
|
|
407
|
+
// Use conservative limits for CodeReviewer (safer for large diffs)
|
|
408
|
+
const providerLimit = getProviderTokenLimit(provider, true);
|
|
409
|
+
// Use the smaller of configured tokens or provider limit
|
|
410
|
+
if (configuredTokens && configuredTokens > 0) {
|
|
411
|
+
const safeLimit = Math.min(configuredTokens, providerLimit);
|
|
412
|
+
logger.debug(`Token limit: configured=${configuredTokens}, provider=${providerLimit}, using=${safeLimit}`);
|
|
413
|
+
return safeLimit;
|
|
414
|
+
}
|
|
415
|
+
logger.debug(`Token limit: using provider default=${providerLimit} for ${provider}`);
|
|
416
|
+
return providerLimit;
|
|
417
|
+
}
|
|
400
418
|
/**
|
|
401
419
|
* Analyze code with AI using the enhanced prompt
|
|
402
420
|
*/
|
|
@@ -427,13 +445,18 @@ Return ONLY valid JSON:
|
|
|
427
445
|
};
|
|
428
446
|
// Simplified, focused prompt without context pollution
|
|
429
447
|
const corePrompt = this.buildCoreAnalysisPrompt(context);
|
|
448
|
+
// Get safe token limit based on provider
|
|
449
|
+
const safeMaxTokens = this.getSafeTokenLimit();
|
|
450
|
+
logger.debug(`Using AI provider: ${this.aiConfig.provider || "auto"}`);
|
|
451
|
+
logger.debug(`Configured maxTokens: ${this.aiConfig.maxTokens}`);
|
|
452
|
+
logger.debug(`Safe maxTokens limit: ${safeMaxTokens}`);
|
|
430
453
|
const result = await this.neurolink.generate({
|
|
431
454
|
input: { text: corePrompt },
|
|
432
455
|
systemPrompt: this.getSecurityReviewSystemPrompt(),
|
|
433
456
|
provider: this.aiConfig.provider || "auto", // Auto-select best provider
|
|
434
457
|
model: this.aiConfig.model || "best", // Use most capable model
|
|
435
458
|
temperature: this.aiConfig.temperature || 0.3, // Lower for more focused analysis
|
|
436
|
-
maxTokens:
|
|
459
|
+
maxTokens: safeMaxTokens, // Use provider-aware safe token limit
|
|
437
460
|
timeout: "15m", // Allow plenty of time for thorough analysis
|
|
438
461
|
context: aiContext,
|
|
439
462
|
enableAnalytics: this.aiConfig.enableAnalytics || true,
|
package/dist/types/index.d.ts
CHANGED
|
@@ -233,6 +233,7 @@ export interface GuardianConfig {
|
|
|
233
233
|
securityScan?: SecurityScanConfig;
|
|
234
234
|
analytics?: AnalyticsConfig;
|
|
235
235
|
};
|
|
236
|
+
memoryBank?: MemoryBankConfig;
|
|
236
237
|
cache?: CacheConfig;
|
|
237
238
|
performance?: PerformanceConfig;
|
|
238
239
|
rules?: CustomRulesConfig;
|
|
@@ -277,6 +278,11 @@ export interface AnalyticsConfig {
|
|
|
277
278
|
trackMetrics: boolean;
|
|
278
279
|
exportFormat: "json" | "csv" | "yaml";
|
|
279
280
|
}
|
|
281
|
+
export interface MemoryBankConfig {
|
|
282
|
+
enabled: boolean;
|
|
283
|
+
path: string;
|
|
284
|
+
fallbackPaths?: string[];
|
|
285
|
+
}
|
|
280
286
|
export interface CacheConfig {
|
|
281
287
|
enabled: boolean;
|
|
282
288
|
ttl: string;
|
package/dist/utils/Cache.d.ts
CHANGED
|
@@ -57,6 +57,10 @@ export declare class Cache implements ICache {
|
|
|
57
57
|
* Invalidate all keys with a specific tag
|
|
58
58
|
*/
|
|
59
59
|
invalidateTag(tag: string): number;
|
|
60
|
+
/**
|
|
61
|
+
* Invalidate all keys matching a pattern
|
|
62
|
+
*/
|
|
63
|
+
invalidatePattern(pattern: string): number;
|
|
60
64
|
/**
|
|
61
65
|
* Cache key generators for common patterns
|
|
62
66
|
*/
|
|
@@ -69,6 +73,7 @@ export declare class Cache implements ICache {
|
|
|
69
73
|
aiResponse: (prompt: string, provider: string, model: string) => string;
|
|
70
74
|
projectContext: (workspace: string, repository: string, branch: string) => string;
|
|
71
75
|
reviewResult: (workspace: string, repository: string, prId: string | number, configHash: string) => string;
|
|
76
|
+
memoryBankFiles: (workspace: string, repository: string, branch: string, path: string) => string;
|
|
72
77
|
};
|
|
73
78
|
/**
|
|
74
79
|
* Smart cache warming for common patterns
|
package/dist/utils/Cache.js
CHANGED
|
@@ -166,6 +166,21 @@ export class Cache {
|
|
|
166
166
|
logger.debug(`Invalidated tag "${tag}": ${deleted} keys`);
|
|
167
167
|
return deleted;
|
|
168
168
|
}
|
|
169
|
+
/**
|
|
170
|
+
* Invalidate all keys matching a pattern
|
|
171
|
+
*/
|
|
172
|
+
invalidatePattern(pattern) {
|
|
173
|
+
const regex = new RegExp(pattern.replace(/\*/g, ".*"));
|
|
174
|
+
const allKeys = this.keys();
|
|
175
|
+
let deleted = 0;
|
|
176
|
+
allKeys.forEach((key) => {
|
|
177
|
+
if (regex.test(key)) {
|
|
178
|
+
deleted += this.del(key);
|
|
179
|
+
}
|
|
180
|
+
});
|
|
181
|
+
logger.debug(`Invalidated pattern "${pattern}": ${deleted} keys`);
|
|
182
|
+
return deleted;
|
|
183
|
+
}
|
|
169
184
|
/**
|
|
170
185
|
* Cache key generators for common patterns
|
|
171
186
|
*/
|
|
@@ -182,6 +197,7 @@ export class Cache {
|
|
|
182
197
|
},
|
|
183
198
|
projectContext: (workspace, repository, branch) => `context:${workspace}:${repository}:${branch}`,
|
|
184
199
|
reviewResult: (workspace, repository, prId, configHash) => `review:${workspace}:${repository}:${prId}:${configHash}`,
|
|
200
|
+
memoryBankFiles: (workspace, repository, branch, path) => `memory-bank:${workspace}:${repository}:${branch}:${path}`,
|
|
185
201
|
};
|
|
186
202
|
/**
|
|
187
203
|
* Smart cache warming for common patterns
|
|
@@ -23,6 +23,10 @@ export declare class ConfigManager {
|
|
|
23
23
|
* Load configuration from a specific file
|
|
24
24
|
*/
|
|
25
25
|
private loadConfigFile;
|
|
26
|
+
/**
|
|
27
|
+
* Apply provider-aware token limits using shared utility
|
|
28
|
+
*/
|
|
29
|
+
private applyProviderTokenLimits;
|
|
26
30
|
/**
|
|
27
31
|
* Apply environment variable overrides
|
|
28
32
|
*/
|
|
@@ -8,6 +8,7 @@ import yaml from "yaml";
|
|
|
8
8
|
import { homedir } from "os";
|
|
9
9
|
import { ConfigurationError } from "../types/index.js";
|
|
10
10
|
import { logger } from "./Logger.js";
|
|
11
|
+
import { validateProviderTokenLimit } from "./ProviderLimits.js";
|
|
11
12
|
export class ConfigManager {
|
|
12
13
|
config = null;
|
|
13
14
|
configPaths = [];
|
|
@@ -24,7 +25,7 @@ export class ConfigManager {
|
|
|
24
25
|
timeout: "10m",
|
|
25
26
|
retryAttempts: 3,
|
|
26
27
|
temperature: 0.3,
|
|
27
|
-
maxTokens:
|
|
28
|
+
maxTokens: 65000,
|
|
28
29
|
},
|
|
29
30
|
git: {
|
|
30
31
|
platform: "bitbucket",
|
|
@@ -143,6 +144,11 @@ export class ConfigManager {
|
|
|
143
144
|
exportFormat: "json",
|
|
144
145
|
interval: "5m",
|
|
145
146
|
},
|
|
147
|
+
memoryBank: {
|
|
148
|
+
enabled: true,
|
|
149
|
+
path: "memory-bank",
|
|
150
|
+
fallbackPaths: ["docs/memory-bank", ".memory-bank"],
|
|
151
|
+
},
|
|
146
152
|
};
|
|
147
153
|
constructor() {
|
|
148
154
|
this.setupConfigPaths();
|
|
@@ -205,6 +211,8 @@ export class ConfigManager {
|
|
|
205
211
|
}
|
|
206
212
|
// Override with environment variables
|
|
207
213
|
config = this.applyEnvironmentOverrides(config);
|
|
214
|
+
// Apply provider-aware token limits
|
|
215
|
+
config = this.applyProviderTokenLimits(config);
|
|
208
216
|
// Validate configuration
|
|
209
217
|
this.validateConfig(config);
|
|
210
218
|
this.config = config;
|
|
@@ -232,6 +240,18 @@ export class ConfigManager {
|
|
|
232
240
|
throw new ConfigurationError(`Failed to parse config file ${filePath}: ${error.message}`);
|
|
233
241
|
}
|
|
234
242
|
}
|
|
243
|
+
/**
|
|
244
|
+
* Apply provider-aware token limits using shared utility
|
|
245
|
+
*/
|
|
246
|
+
applyProviderTokenLimits(config) {
|
|
247
|
+
const provider = config.providers.ai.provider || 'auto';
|
|
248
|
+
const configuredTokens = config.providers.ai.maxTokens;
|
|
249
|
+
// Use the shared utility to validate and adjust token limits
|
|
250
|
+
const validatedTokens = validateProviderTokenLimit(provider, configuredTokens, false // Use standard limits for configuration
|
|
251
|
+
);
|
|
252
|
+
config.providers.ai.maxTokens = validatedTokens;
|
|
253
|
+
return config;
|
|
254
|
+
}
|
|
235
255
|
/**
|
|
236
256
|
* Apply environment variable overrides
|
|
237
257
|
*/
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Memory Bank Manager - Handles configurable memory bank operations
|
|
3
|
+
* Provides abstraction for memory bank file access with fallback support
|
|
4
|
+
*/
|
|
5
|
+
import { MemoryBankConfig, PRIdentifier } from "../types/index.js";
|
|
6
|
+
import { BitbucketProvider } from "../core/providers/BitbucketProvider.js";
|
|
7
|
+
export interface MemoryBankFile {
|
|
8
|
+
name: string;
|
|
9
|
+
content: string;
|
|
10
|
+
path: string;
|
|
11
|
+
}
|
|
12
|
+
export interface MemoryBankResult {
|
|
13
|
+
files: MemoryBankFile[];
|
|
14
|
+
resolvedPath: string;
|
|
15
|
+
filesProcessed: number;
|
|
16
|
+
fallbackUsed: boolean;
|
|
17
|
+
}
|
|
18
|
+
export declare class MemoryBankManager {
|
|
19
|
+
private config;
|
|
20
|
+
private bitbucketProvider;
|
|
21
|
+
constructor(config: MemoryBankConfig, bitbucketProvider: BitbucketProvider);
|
|
22
|
+
/**
|
|
23
|
+
* Get memory bank files from the configured path with fallback support
|
|
24
|
+
*/
|
|
25
|
+
getMemoryBankFiles(identifier: PRIdentifier, forceRefresh?: boolean): Promise<MemoryBankResult>;
|
|
26
|
+
/**
|
|
27
|
+
* Try to get files from a specific path
|
|
28
|
+
*/
|
|
29
|
+
private tryGetFilesFromPath;
|
|
30
|
+
/**
|
|
31
|
+
* Get the effective memory bank path (resolved after fallback logic)
|
|
32
|
+
*/
|
|
33
|
+
getEffectiveMemoryBankPath(identifier: PRIdentifier): Promise<string | null>;
|
|
34
|
+
/**
|
|
35
|
+
* Check if memory bank exists at any configured path
|
|
36
|
+
*/
|
|
37
|
+
hasMemoryBank(identifier: PRIdentifier): Promise<boolean>;
|
|
38
|
+
/**
|
|
39
|
+
* Get memory bank configuration
|
|
40
|
+
*/
|
|
41
|
+
getConfig(): MemoryBankConfig;
|
|
42
|
+
/**
|
|
43
|
+
* Update memory bank configuration
|
|
44
|
+
*/
|
|
45
|
+
updateConfig(newConfig: Partial<MemoryBankConfig>): void;
|
|
46
|
+
/**
|
|
47
|
+
* Validates that a path is safe for use as a relative path
|
|
48
|
+
* Protects against path traversal attacks including encoded variants
|
|
49
|
+
*/
|
|
50
|
+
private static isSafeRelativePath;
|
|
51
|
+
/**
|
|
52
|
+
* Validate memory bank configuration
|
|
53
|
+
*/
|
|
54
|
+
private validateConfig;
|
|
55
|
+
/**
|
|
56
|
+
* Clear memory bank cache for a specific repository
|
|
57
|
+
*/
|
|
58
|
+
clearCache(identifier: PRIdentifier): void;
|
|
59
|
+
/**
|
|
60
|
+
* Get memory bank statistics
|
|
61
|
+
*/
|
|
62
|
+
getStats(identifier: PRIdentifier): Promise<{
|
|
63
|
+
enabled: boolean;
|
|
64
|
+
primaryPath: string;
|
|
65
|
+
fallbackPaths: string[];
|
|
66
|
+
hasMemoryBank: boolean;
|
|
67
|
+
resolvedPath: string | null;
|
|
68
|
+
fileCount: number;
|
|
69
|
+
cacheHits: number;
|
|
70
|
+
}>;
|
|
71
|
+
}
|
|
72
|
+
export declare function createMemoryBankManager(config: MemoryBankConfig, bitbucketProvider: BitbucketProvider): MemoryBankManager;
|
|
73
|
+
//# sourceMappingURL=MemoryBankManager.d.ts.map
|
|
@@ -0,0 +1,310 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Memory Bank Manager - Handles configurable memory bank operations
|
|
3
|
+
* Provides abstraction for memory bank file access with fallback support
|
|
4
|
+
*/
|
|
5
|
+
import { ConfigurationError } from "../types/index.js";
|
|
6
|
+
import { logger } from "./Logger.js";
|
|
7
|
+
import { cache, Cache } from "./Cache.js";
|
|
8
|
+
export class MemoryBankManager {
|
|
9
|
+
config;
|
|
10
|
+
bitbucketProvider;
|
|
11
|
+
constructor(config, bitbucketProvider) {
|
|
12
|
+
this.config = config;
|
|
13
|
+
this.bitbucketProvider = bitbucketProvider;
|
|
14
|
+
this.validateConfig();
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Get memory bank files from the configured path with fallback support
|
|
18
|
+
*/
|
|
19
|
+
async getMemoryBankFiles(identifier, forceRefresh = false) {
|
|
20
|
+
if (!this.config.enabled) {
|
|
21
|
+
logger.debug("Memory bank is disabled in configuration");
|
|
22
|
+
return {
|
|
23
|
+
files: [],
|
|
24
|
+
resolvedPath: "",
|
|
25
|
+
filesProcessed: 0,
|
|
26
|
+
fallbackUsed: false,
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
const cacheKey = Cache.keys.memoryBankFiles(identifier.workspace, identifier.repository, identifier.branch || "main", this.config.path);
|
|
30
|
+
if (!forceRefresh && cache.has(cacheKey)) {
|
|
31
|
+
logger.debug("Using cached memory bank files");
|
|
32
|
+
return cache.get(cacheKey);
|
|
33
|
+
}
|
|
34
|
+
logger.debug(`Gathering memory bank files from configured paths...`);
|
|
35
|
+
// Try primary path first
|
|
36
|
+
const primaryResult = await this.tryGetFilesFromPath(identifier, this.config.path);
|
|
37
|
+
if (primaryResult.files.length > 0) {
|
|
38
|
+
const result = {
|
|
39
|
+
...primaryResult,
|
|
40
|
+
resolvedPath: this.config.path,
|
|
41
|
+
fallbackUsed: false,
|
|
42
|
+
};
|
|
43
|
+
// Cache the result
|
|
44
|
+
cache.set(cacheKey, result, 7200); // 2 hours
|
|
45
|
+
return result;
|
|
46
|
+
}
|
|
47
|
+
// Try fallback paths if primary path failed
|
|
48
|
+
if (this.config.fallbackPaths && this.config.fallbackPaths.length > 0) {
|
|
49
|
+
logger.debug(`Primary path '${this.config.path}' not found, trying fallback paths...`);
|
|
50
|
+
for (const fallbackPath of this.config.fallbackPaths) {
|
|
51
|
+
logger.debug(`Trying fallback path: ${fallbackPath}`);
|
|
52
|
+
const fallbackResult = await this.tryGetFilesFromPath(identifier, fallbackPath);
|
|
53
|
+
if (fallbackResult.files.length > 0) {
|
|
54
|
+
logger.info(`Memory bank found at fallback path: ${fallbackPath} (${fallbackResult.files.length} files)`);
|
|
55
|
+
const result = {
|
|
56
|
+
...fallbackResult,
|
|
57
|
+
resolvedPath: fallbackPath,
|
|
58
|
+
fallbackUsed: true,
|
|
59
|
+
};
|
|
60
|
+
// Cache the result
|
|
61
|
+
cache.set(cacheKey, result, 7200); // 2 hours
|
|
62
|
+
return result;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
// No memory bank found anywhere
|
|
67
|
+
logger.debug(`No memory bank found in primary path '${this.config.path}' or fallback paths`);
|
|
68
|
+
const emptyResult = {
|
|
69
|
+
files: [],
|
|
70
|
+
resolvedPath: "",
|
|
71
|
+
filesProcessed: 0,
|
|
72
|
+
fallbackUsed: false,
|
|
73
|
+
};
|
|
74
|
+
// Cache empty result for shorter time to allow for quick retry
|
|
75
|
+
cache.set(cacheKey, emptyResult, 1800); // 30 minutes
|
|
76
|
+
return emptyResult;
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Try to get files from a specific path
|
|
80
|
+
*/
|
|
81
|
+
async tryGetFilesFromPath(identifier, path) {
|
|
82
|
+
try {
|
|
83
|
+
// Get directory listing
|
|
84
|
+
const directoryFiles = await this.bitbucketProvider.listDirectoryContent(identifier.workspace, identifier.repository, path, identifier.branch || "main");
|
|
85
|
+
if (!directoryFiles.length) {
|
|
86
|
+
logger.debug(`No files found in directory: ${path}`);
|
|
87
|
+
return { files: [], filesProcessed: 0 };
|
|
88
|
+
}
|
|
89
|
+
// Filter to only files (not directories)
|
|
90
|
+
const files = directoryFiles.filter((f) => f.type === "file");
|
|
91
|
+
logger.debug(`Found ${files.length} files in ${path}`);
|
|
92
|
+
// Get content of each file
|
|
93
|
+
const memoryBankFiles = [];
|
|
94
|
+
for (const file of files) {
|
|
95
|
+
try {
|
|
96
|
+
const content = await this.bitbucketProvider.getFileContent(identifier.workspace, identifier.repository, `${path}/${file.name}`, identifier.branch || "main");
|
|
97
|
+
memoryBankFiles.push({
|
|
98
|
+
name: file.name,
|
|
99
|
+
content,
|
|
100
|
+
path: `${path}/${file.name}`,
|
|
101
|
+
});
|
|
102
|
+
logger.debug(`✓ Loaded content for: ${file.name}`);
|
|
103
|
+
}
|
|
104
|
+
catch (error) {
|
|
105
|
+
logger.debug(`Could not read file ${file.name}: ${error.message}`);
|
|
106
|
+
// Continue with other files even if one fails
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
return {
|
|
110
|
+
files: memoryBankFiles,
|
|
111
|
+
filesProcessed: memoryBankFiles.length,
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
catch (error) {
|
|
115
|
+
logger.debug(`Failed to access path '${path}': ${error.message}`);
|
|
116
|
+
return { files: [], filesProcessed: 0 };
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* Get the effective memory bank path (resolved after fallback logic)
|
|
121
|
+
*/
|
|
122
|
+
async getEffectiveMemoryBankPath(identifier) {
|
|
123
|
+
const result = await this.getMemoryBankFiles(identifier);
|
|
124
|
+
return result.resolvedPath || null;
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* Check if memory bank exists at any configured path
|
|
128
|
+
*/
|
|
129
|
+
async hasMemoryBank(identifier) {
|
|
130
|
+
const result = await this.getMemoryBankFiles(identifier);
|
|
131
|
+
return result.files.length > 0;
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* Get memory bank configuration
|
|
135
|
+
*/
|
|
136
|
+
getConfig() {
|
|
137
|
+
return { ...this.config };
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Update memory bank configuration
|
|
141
|
+
*/
|
|
142
|
+
updateConfig(newConfig) {
|
|
143
|
+
this.config = { ...this.config, ...newConfig };
|
|
144
|
+
this.validateConfig();
|
|
145
|
+
logger.debug("Memory bank configuration updated");
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* Validates that a path is safe for use as a relative path
|
|
149
|
+
* Protects against path traversal attacks including encoded variants
|
|
150
|
+
*/
|
|
151
|
+
static isSafeRelativePath(path) {
|
|
152
|
+
if (!path || typeof path !== "string") {
|
|
153
|
+
return false;
|
|
154
|
+
}
|
|
155
|
+
// Reject empty or whitespace-only paths
|
|
156
|
+
if (path.trim().length === 0) {
|
|
157
|
+
return false;
|
|
158
|
+
}
|
|
159
|
+
// Reject excessively long paths
|
|
160
|
+
if (path.length > 1000) {
|
|
161
|
+
return false;
|
|
162
|
+
}
|
|
163
|
+
// Reject absolute paths (Unix-style)
|
|
164
|
+
if (path.startsWith("/")) {
|
|
165
|
+
return false;
|
|
166
|
+
}
|
|
167
|
+
// Reject absolute paths (Windows-style)
|
|
168
|
+
if (/^[a-zA-Z]:/.test(path)) {
|
|
169
|
+
return false;
|
|
170
|
+
}
|
|
171
|
+
// Reject UNC paths (Windows network paths)
|
|
172
|
+
if (path.startsWith("\\\\") || path.startsWith("//")) {
|
|
173
|
+
return false;
|
|
174
|
+
}
|
|
175
|
+
// Decode URL-encoded characters to catch encoded traversal attempts
|
|
176
|
+
let decodedPath = path;
|
|
177
|
+
try {
|
|
178
|
+
// Multiple rounds of decoding to catch double-encoded attacks
|
|
179
|
+
for (let i = 0; i < 3; i++) {
|
|
180
|
+
const previousPath = decodedPath;
|
|
181
|
+
decodedPath = decodeURIComponent(decodedPath);
|
|
182
|
+
if (decodedPath === previousPath) {
|
|
183
|
+
break; // No more decoding needed
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
catch {
|
|
188
|
+
// If decoding fails, treat as suspicious
|
|
189
|
+
return false;
|
|
190
|
+
}
|
|
191
|
+
// Normalize Unicode characters
|
|
192
|
+
decodedPath = decodedPath.normalize("NFC");
|
|
193
|
+
// Check for null bytes (can be used to bypass filters)
|
|
194
|
+
if (decodedPath.includes("\0") || decodedPath.includes("%00")) {
|
|
195
|
+
return false;
|
|
196
|
+
}
|
|
197
|
+
// Normalize path separators to forward slashes
|
|
198
|
+
const normalizedPath = decodedPath.replace(/\\/g, "/");
|
|
199
|
+
// Check for path traversal sequences after normalization
|
|
200
|
+
if (normalizedPath.includes("../") || normalizedPath.includes("/..")) {
|
|
201
|
+
return false;
|
|
202
|
+
}
|
|
203
|
+
// Check for path traversal at the beginning or end
|
|
204
|
+
if (normalizedPath.startsWith("..") || normalizedPath.endsWith("..")) {
|
|
205
|
+
return false;
|
|
206
|
+
}
|
|
207
|
+
// Check for hidden traversal patterns
|
|
208
|
+
if (normalizedPath.includes("./..") || normalizedPath.includes("../.")) {
|
|
209
|
+
return false;
|
|
210
|
+
}
|
|
211
|
+
// Split path into segments and validate each
|
|
212
|
+
const segments = normalizedPath.split("/").filter(segment => segment.length > 0);
|
|
213
|
+
for (const segment of segments) {
|
|
214
|
+
// Reject any segment that is exactly ".."
|
|
215
|
+
if (segment === "..") {
|
|
216
|
+
return false;
|
|
217
|
+
}
|
|
218
|
+
// Reject segments that contain ".." anywhere
|
|
219
|
+
if (segment.includes("..")) {
|
|
220
|
+
return false;
|
|
221
|
+
}
|
|
222
|
+
// Allow segments that start with a single dot (like .memory-bank, .config)
|
|
223
|
+
// but reject multiple dots or suspicious patterns
|
|
224
|
+
if (segment.startsWith(".") && segment !== ".") {
|
|
225
|
+
// Allow single dot followed by alphanumeric/dash/underscore
|
|
226
|
+
if (!/^\.[\w-]+$/.test(segment)) {
|
|
227
|
+
return false;
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
// Reject segments with control characters
|
|
231
|
+
// Check for control characters (0x00-0x1F and 0x7F)
|
|
232
|
+
for (let i = 0; i < segment.length; i++) {
|
|
233
|
+
const charCode = segment.charCodeAt(i);
|
|
234
|
+
if ((charCode >= 0 && charCode <= 31) || charCode === 127) {
|
|
235
|
+
return false;
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
// Additional check: ensure the resolved path doesn't escape the base
|
|
240
|
+
// This is a final safety check using path resolution logic
|
|
241
|
+
const pathParts = segments.filter(part => part !== ".");
|
|
242
|
+
let depth = 0;
|
|
243
|
+
for (const part of pathParts) {
|
|
244
|
+
if (part === "..") {
|
|
245
|
+
depth--;
|
|
246
|
+
if (depth < 0) {
|
|
247
|
+
return false; // Would escape the base directory
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
else {
|
|
251
|
+
depth++;
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
return true;
|
|
255
|
+
}
|
|
256
|
+
/**
|
|
257
|
+
* Validate memory bank configuration
|
|
258
|
+
*/
|
|
259
|
+
validateConfig() {
|
|
260
|
+
if (!this.config.path) {
|
|
261
|
+
throw new ConfigurationError("Memory bank path must be specified when memory bank is enabled");
|
|
262
|
+
}
|
|
263
|
+
if (!MemoryBankManager.isSafeRelativePath(this.config.path)) {
|
|
264
|
+
throw new ConfigurationError(`Memory bank path is unsafe or contains path traversal: ${this.config.path}`);
|
|
265
|
+
}
|
|
266
|
+
// Validate fallback paths
|
|
267
|
+
if (this.config.fallbackPaths) {
|
|
268
|
+
for (const fallbackPath of this.config.fallbackPaths) {
|
|
269
|
+
if (!MemoryBankManager.isSafeRelativePath(fallbackPath)) {
|
|
270
|
+
throw new ConfigurationError(`Memory bank fallback path is unsafe or contains path traversal: ${fallbackPath}`);
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
logger.debug("Memory bank configuration validated successfully");
|
|
275
|
+
}
|
|
276
|
+
/**
|
|
277
|
+
* Clear memory bank cache for a specific repository
|
|
278
|
+
*/
|
|
279
|
+
clearCache(identifier) {
|
|
280
|
+
const patterns = [
|
|
281
|
+
`memory-bank:${identifier.workspace}:${identifier.repository}:*`,
|
|
282
|
+
`project-context:${identifier.workspace}:${identifier.repository}:*`,
|
|
283
|
+
];
|
|
284
|
+
patterns.forEach((pattern) => {
|
|
285
|
+
cache.invalidatePattern(pattern);
|
|
286
|
+
});
|
|
287
|
+
logger.debug(`Memory bank cache cleared for ${identifier.workspace}/${identifier.repository}`);
|
|
288
|
+
}
|
|
289
|
+
/**
|
|
290
|
+
* Get memory bank statistics
|
|
291
|
+
*/
|
|
292
|
+
async getStats(identifier) {
|
|
293
|
+
const result = await this.getMemoryBankFiles(identifier);
|
|
294
|
+
const cacheStats = cache.stats();
|
|
295
|
+
return {
|
|
296
|
+
enabled: this.config.enabled,
|
|
297
|
+
primaryPath: this.config.path,
|
|
298
|
+
fallbackPaths: this.config.fallbackPaths || [],
|
|
299
|
+
hasMemoryBank: result.files.length > 0,
|
|
300
|
+
resolvedPath: result.resolvedPath || null,
|
|
301
|
+
fileCount: result.files.length,
|
|
302
|
+
cacheHits: cacheStats.hits,
|
|
303
|
+
};
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
// Export factory function
|
|
307
|
+
export function createMemoryBankManager(config, bitbucketProvider) {
|
|
308
|
+
return new MemoryBankManager(config, bitbucketProvider);
|
|
309
|
+
}
|
|
310
|
+
//# sourceMappingURL=MemoryBankManager.js.map
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Provider Token Limits Utility
|
|
3
|
+
* Centralized management of AI provider token limits and validation
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* AI Provider types supported by the system
|
|
7
|
+
*/
|
|
8
|
+
export type AIProvider = 'vertex' | 'google-ai' | 'gemini' | 'openai' | 'gpt-4' | 'anthropic' | 'claude' | 'azure' | 'bedrock' | 'auto';
|
|
9
|
+
/**
|
|
10
|
+
* Provider token limits configuration
|
|
11
|
+
* These limits are conservative values to avoid API errors
|
|
12
|
+
*/
|
|
13
|
+
export declare const PROVIDER_TOKEN_LIMITS: Record<AIProvider, number>;
|
|
14
|
+
/**
|
|
15
|
+
* Conservative limits used by CodeReviewer for safety
|
|
16
|
+
* These are slightly lower than the actual limits to provide buffer
|
|
17
|
+
*/
|
|
18
|
+
export declare const CONSERVATIVE_PROVIDER_LIMITS: Record<AIProvider, number>;
|
|
19
|
+
/**
|
|
20
|
+
* Get the token limit for a specific provider
|
|
21
|
+
* @param provider - The AI provider name
|
|
22
|
+
* @param conservative - Whether to use conservative limits (default: false)
|
|
23
|
+
* @returns The token limit for the provider
|
|
24
|
+
*/
|
|
25
|
+
export declare function getProviderTokenLimit(provider: string, conservative?: boolean): number;
|
|
26
|
+
/**
|
|
27
|
+
* Validate and adjust token limit for a provider
|
|
28
|
+
* @param provider - The AI provider name
|
|
29
|
+
* @param configuredTokens - The configured token limit
|
|
30
|
+
* @param conservative - Whether to use conservative limits (default: false)
|
|
31
|
+
* @returns The validated and potentially adjusted token limit
|
|
32
|
+
*/
|
|
33
|
+
export declare function validateProviderTokenLimit(provider: string, configuredTokens: number | undefined, conservative?: boolean): number;
|
|
34
|
+
/**
|
|
35
|
+
* Get all supported providers
|
|
36
|
+
* @returns Array of supported provider names
|
|
37
|
+
*/
|
|
38
|
+
export declare function getSupportedProviders(): AIProvider[];
|
|
39
|
+
/**
|
|
40
|
+
* Check if a provider is supported
|
|
41
|
+
* @param provider - The provider name to check
|
|
42
|
+
* @returns True if the provider is supported
|
|
43
|
+
*/
|
|
44
|
+
export declare function isProviderSupported(provider: string): boolean;
|
|
45
|
+
/**
|
|
46
|
+
* Get provider information including limits and support status
|
|
47
|
+
* @param provider - The provider name
|
|
48
|
+
* @param conservative - Whether to use conservative limits
|
|
49
|
+
* @returns Provider information object
|
|
50
|
+
*/
|
|
51
|
+
export declare function getProviderInfo(provider: string, conservative?: boolean): {
|
|
52
|
+
provider: string;
|
|
53
|
+
isSupported: boolean;
|
|
54
|
+
tokenLimit: number;
|
|
55
|
+
conservativeLimit: number;
|
|
56
|
+
standardLimit: number;
|
|
57
|
+
};
|
|
58
|
+
//# sourceMappingURL=ProviderLimits.d.ts.map
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Provider Token Limits Utility
|
|
3
|
+
* Centralized management of AI provider token limits and validation
|
|
4
|
+
*/
|
|
5
|
+
import { logger } from "./Logger.js";
|
|
6
|
+
/**
|
|
7
|
+
* Provider token limits configuration
|
|
8
|
+
* These limits are conservative values to avoid API errors
|
|
9
|
+
*/
|
|
10
|
+
export const PROVIDER_TOKEN_LIMITS = {
|
|
11
|
+
// Google/Vertex AI providers
|
|
12
|
+
'vertex': 65536, // Vertex AI limit is 65537 exclusive = 65536 max
|
|
13
|
+
'google-ai': 65536, // Google AI Studio limit
|
|
14
|
+
'gemini': 65536, // Gemini model limit
|
|
15
|
+
// OpenAI providers
|
|
16
|
+
'openai': 128000, // OpenAI GPT-4 and newer models
|
|
17
|
+
'gpt-4': 128000, // GPT-4 specific limit
|
|
18
|
+
// Anthropic providers
|
|
19
|
+
'anthropic': 200000, // Claude models limit
|
|
20
|
+
'claude': 200000, // Claude specific limit
|
|
21
|
+
// Microsoft Azure
|
|
22
|
+
'azure': 128000, // Azure OpenAI limit
|
|
23
|
+
// AWS Bedrock
|
|
24
|
+
'bedrock': 100000, // AWS Bedrock limit
|
|
25
|
+
// Auto-selection mode (conservative default)
|
|
26
|
+
'auto': 60000, // Conservative default for auto-selection
|
|
27
|
+
};
|
|
28
|
+
/**
|
|
29
|
+
* Conservative limits used by CodeReviewer for safety
|
|
30
|
+
* These are slightly lower than the actual limits to provide buffer
|
|
31
|
+
*/
|
|
32
|
+
export const CONSERVATIVE_PROVIDER_LIMITS = {
|
|
33
|
+
'vertex': 65536,
|
|
34
|
+
'google-ai': 65536,
|
|
35
|
+
'gemini': 65536,
|
|
36
|
+
'openai': 120000, // Slightly lower for safety
|
|
37
|
+
'gpt-4': 120000,
|
|
38
|
+
'anthropic': 190000, // Slightly lower for safety
|
|
39
|
+
'claude': 190000,
|
|
40
|
+
'azure': 120000,
|
|
41
|
+
'bedrock': 95000, // Significantly lower for safety
|
|
42
|
+
'auto': 60000,
|
|
43
|
+
};
|
|
44
|
+
/**
|
|
45
|
+
* Get the token limit for a specific provider
|
|
46
|
+
* @param provider - The AI provider name
|
|
47
|
+
* @param conservative - Whether to use conservative limits (default: false)
|
|
48
|
+
* @returns The token limit for the provider
|
|
49
|
+
*/
|
|
50
|
+
export function getProviderTokenLimit(provider, conservative = false) {
|
|
51
|
+
// Handle null, undefined, or empty string
|
|
52
|
+
if (!provider || typeof provider !== 'string') {
|
|
53
|
+
return conservative ? CONSERVATIVE_PROVIDER_LIMITS.auto : PROVIDER_TOKEN_LIMITS.auto;
|
|
54
|
+
}
|
|
55
|
+
const normalizedProvider = provider.toLowerCase();
|
|
56
|
+
const limits = conservative ? CONSERVATIVE_PROVIDER_LIMITS : PROVIDER_TOKEN_LIMITS;
|
|
57
|
+
// Handle empty string after normalization
|
|
58
|
+
if (normalizedProvider === '') {
|
|
59
|
+
return conservative ? CONSERVATIVE_PROVIDER_LIMITS.auto : PROVIDER_TOKEN_LIMITS.auto;
|
|
60
|
+
}
|
|
61
|
+
// Direct match
|
|
62
|
+
if (normalizedProvider in limits) {
|
|
63
|
+
return limits[normalizedProvider];
|
|
64
|
+
}
|
|
65
|
+
// Partial match - check if provider contains any known provider name
|
|
66
|
+
for (const [key, limit] of Object.entries(limits)) {
|
|
67
|
+
if (normalizedProvider.includes(key) || key.includes(normalizedProvider)) {
|
|
68
|
+
return limit;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
// Default fallback
|
|
72
|
+
return conservative ? CONSERVATIVE_PROVIDER_LIMITS.auto : PROVIDER_TOKEN_LIMITS.auto;
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Validate and adjust token limit for a provider
|
|
76
|
+
* @param provider - The AI provider name
|
|
77
|
+
* @param configuredTokens - The configured token limit
|
|
78
|
+
* @param conservative - Whether to use conservative limits (default: false)
|
|
79
|
+
* @returns The validated and potentially adjusted token limit
|
|
80
|
+
*/
|
|
81
|
+
export function validateProviderTokenLimit(provider, configuredTokens, conservative = false) {
|
|
82
|
+
const providerLimit = getProviderTokenLimit(provider, conservative);
|
|
83
|
+
if (!configuredTokens || configuredTokens <= 0) {
|
|
84
|
+
logger.debug(`No configured tokens for ${provider}, using provider default: ${providerLimit}`);
|
|
85
|
+
return providerLimit;
|
|
86
|
+
}
|
|
87
|
+
if (configuredTokens > providerLimit) {
|
|
88
|
+
logger.warn(`Configured maxTokens (${configuredTokens}) exceeds ${provider} limit (${providerLimit}). Adjusting to ${providerLimit}.`);
|
|
89
|
+
return providerLimit;
|
|
90
|
+
}
|
|
91
|
+
logger.debug(`Token limit validation passed: ${configuredTokens} <= ${providerLimit} for provider ${provider}`);
|
|
92
|
+
return configuredTokens;
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Get all supported providers
|
|
96
|
+
* @returns Array of supported provider names
|
|
97
|
+
*/
|
|
98
|
+
export function getSupportedProviders() {
|
|
99
|
+
return Object.keys(PROVIDER_TOKEN_LIMITS);
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Check if a provider is supported
|
|
103
|
+
* @param provider - The provider name to check
|
|
104
|
+
* @returns True if the provider is supported
|
|
105
|
+
*/
|
|
106
|
+
export function isProviderSupported(provider) {
|
|
107
|
+
// Handle null, undefined, or empty string
|
|
108
|
+
if (!provider || typeof provider !== 'string') {
|
|
109
|
+
return false;
|
|
110
|
+
}
|
|
111
|
+
const normalizedProvider = provider.toLowerCase();
|
|
112
|
+
// Handle empty string after normalization
|
|
113
|
+
if (normalizedProvider === '') {
|
|
114
|
+
return false;
|
|
115
|
+
}
|
|
116
|
+
// Check direct match
|
|
117
|
+
if (normalizedProvider in PROVIDER_TOKEN_LIMITS) {
|
|
118
|
+
return true;
|
|
119
|
+
}
|
|
120
|
+
// Check partial match
|
|
121
|
+
for (const key of Object.keys(PROVIDER_TOKEN_LIMITS)) {
|
|
122
|
+
if (normalizedProvider.includes(key) || key.includes(normalizedProvider)) {
|
|
123
|
+
return true;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
return false;
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* Get provider information including limits and support status
|
|
130
|
+
* @param provider - The provider name
|
|
131
|
+
* @param conservative - Whether to use conservative limits
|
|
132
|
+
* @returns Provider information object
|
|
133
|
+
*/
|
|
134
|
+
export function getProviderInfo(provider, conservative = false) {
|
|
135
|
+
return {
|
|
136
|
+
provider,
|
|
137
|
+
isSupported: isProviderSupported(provider),
|
|
138
|
+
tokenLimit: getProviderTokenLimit(provider, conservative),
|
|
139
|
+
conservativeLimit: getProviderTokenLimit(provider, true),
|
|
140
|
+
standardLimit: getProviderTokenLimit(provider, false),
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
//# sourceMappingURL=ProviderLimits.js.map
|
package/package.json
CHANGED
package/yama.config.example.yaml
CHANGED
|
@@ -7,7 +7,7 @@ providers:
|
|
|
7
7
|
provider: "auto" # Options: auto, google-ai, openai, anthropic, azure, bedrock
|
|
8
8
|
model: "best" # Model name or "best" for auto-selection
|
|
9
9
|
temperature: 0.3 # Lower = more focused (0.0-1.0)
|
|
10
|
-
maxTokens:
|
|
10
|
+
maxTokens: 60000 # Maximum tokens for response (provider-aware limits will be applied automatically)
|
|
11
11
|
timeout: "15m" # Timeout for AI operations
|
|
12
12
|
enableAnalytics: true
|
|
13
13
|
enableEvaluation: false
|
|
@@ -141,3 +141,12 @@ monitoring:
|
|
|
141
141
|
metrics: ["api_calls", "cache_hits", "processing_time"]
|
|
142
142
|
exportFormat: "prometheus"
|
|
143
143
|
interval: "1m"
|
|
144
|
+
|
|
145
|
+
# Memory Bank Configuration
|
|
146
|
+
memoryBank:
|
|
147
|
+
enabled: true
|
|
148
|
+
path: "memory-bank" # Primary path to look for memory bank files
|
|
149
|
+
fallbackPaths: # Optional fallback paths if primary doesn't exist
|
|
150
|
+
- "docs/memory-bank"
|
|
151
|
+
- ".memory-bank"
|
|
152
|
+
- "project-docs/context"
|