@git.zone/tsdoc 1.5.2 → 1.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist_ts/00_commitinfo_data.js +1 -1
- package/dist_ts/context/config-manager.d.ts +22 -1
- package/dist_ts/context/config-manager.js +113 -4
- package/dist_ts/context/context-analyzer.d.ts +73 -0
- package/dist_ts/context/context-analyzer.js +311 -0
- package/dist_ts/context/context-cache.d.ts +73 -0
- package/dist_ts/context/context-cache.js +238 -0
- package/dist_ts/context/context-trimmer.d.ts +8 -0
- package/dist_ts/context/context-trimmer.js +60 -1
- package/dist_ts/context/enhanced-context.d.ts +12 -14
- package/dist_ts/context/enhanced-context.js +130 -127
- package/dist_ts/context/index.d.ts +6 -3
- package/dist_ts/context/index.js +5 -2
- package/dist_ts/context/lazy-file-loader.d.ts +60 -0
- package/dist_ts/context/lazy-file-loader.js +164 -0
- package/dist_ts/context/types.d.ts +142 -0
- package/package.json +1 -1
- package/readme.md +348 -53
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/context/config-manager.ts +146 -15
- package/ts/context/context-analyzer.ts +391 -0
- package/ts/context/context-cache.ts +285 -0
- package/ts/context/context-trimmer.ts +64 -0
- package/ts/context/enhanced-context.ts +169 -180
- package/ts/context/index.ts +31 -5
- package/ts/context/lazy-file-loader.ts +191 -0
- package/ts/context/types.ts +152 -0
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import * as plugins from '../plugins.js';
|
|
2
2
|
import { ContextTrimmer } from './context-trimmer.js';
|
|
3
3
|
import { ConfigManager } from './config-manager.js';
|
|
4
|
+
import { LazyFileLoader } from './lazy-file-loader.js';
|
|
5
|
+
import { ContextCache } from './context-cache.js';
|
|
6
|
+
import { ContextAnalyzer } from './context-analyzer.js';
|
|
4
7
|
/**
|
|
5
8
|
* Enhanced ProjectContext that supports context optimization strategies
|
|
6
9
|
*/
|
|
@@ -23,6 +26,9 @@ export class EnhancedContext {
|
|
|
23
26
|
this.projectDir = projectDirArg;
|
|
24
27
|
this.configManager = ConfigManager.getInstance();
|
|
25
28
|
this.trimmer = new ContextTrimmer(this.configManager.getTrimConfig());
|
|
29
|
+
this.lazyLoader = new LazyFileLoader(projectDirArg);
|
|
30
|
+
this.cache = new ContextCache(projectDirArg, this.configManager.getCacheConfig());
|
|
31
|
+
this.analyzer = new ContextAnalyzer(projectDirArg, this.configManager.getPrioritizationWeights(), this.configManager.getTierConfig());
|
|
26
32
|
}
|
|
27
33
|
/**
|
|
28
34
|
* Initialize the context builder
|
|
@@ -31,6 +37,7 @@ export class EnhancedContext {
|
|
|
31
37
|
await this.configManager.initialize(this.projectDir);
|
|
32
38
|
this.tokenBudget = this.configManager.getMaxTokens();
|
|
33
39
|
this.trimmer.updateConfig(this.configManager.getTrimConfig());
|
|
40
|
+
await this.cache.init();
|
|
34
41
|
}
|
|
35
42
|
/**
|
|
36
43
|
* Set the context mode
|
|
@@ -47,45 +54,13 @@ export class EnhancedContext {
|
|
|
47
54
|
this.tokenBudget = maxTokens;
|
|
48
55
|
}
|
|
49
56
|
/**
|
|
50
|
-
*
|
|
51
|
-
* @param
|
|
52
|
-
* @param
|
|
57
|
+
* Convert files to context with smart analysis and prioritization
|
|
58
|
+
* @param metadata - File metadata to analyze
|
|
59
|
+
* @param taskType - Task type for context-aware prioritization
|
|
60
|
+
* @param mode - Context mode to use
|
|
61
|
+
* @returns Context string
|
|
53
62
|
*/
|
|
54
|
-
async
|
|
55
|
-
const smartfilePackageJSON = await plugins.smartfile.SmartFile.fromFilePath(plugins.path.join(this.projectDir, 'package.json'), this.projectDir);
|
|
56
|
-
const smartfilesReadme = await plugins.smartfile.SmartFile.fromFilePath(plugins.path.join(this.projectDir, 'readme.md'), this.projectDir);
|
|
57
|
-
const smartfilesReadmeHints = await plugins.smartfile.SmartFile.fromFilePath(plugins.path.join(this.projectDir, 'readme.hints.md'), this.projectDir);
|
|
58
|
-
const smartfilesNpmextraJSON = await plugins.smartfile.SmartFile.fromFilePath(plugins.path.join(this.projectDir, 'npmextra.json'), this.projectDir);
|
|
59
|
-
// Use provided include paths or default to all TypeScript files
|
|
60
|
-
const includeGlobs = includePaths?.map(path => `${path}/**/*.ts`) || ['ts*/**/*.ts'];
|
|
61
|
-
// Get TypeScript files
|
|
62
|
-
const smartfilesModPromises = includeGlobs.map(glob => plugins.smartfile.fs.fileTreeToObject(this.projectDir, glob));
|
|
63
|
-
const smartfilesModArrays = await Promise.all(smartfilesModPromises);
|
|
64
|
-
// Flatten the arrays
|
|
65
|
-
const smartfilesMod = [];
|
|
66
|
-
smartfilesModArrays.forEach(array => {
|
|
67
|
-
smartfilesMod.push(...array);
|
|
68
|
-
});
|
|
69
|
-
// Get test files if not excluded
|
|
70
|
-
let smartfilesTest = [];
|
|
71
|
-
if (!excludePaths?.includes('test/')) {
|
|
72
|
-
smartfilesTest = await plugins.smartfile.fs.fileTreeToObject(this.projectDir, 'test/**/*.ts');
|
|
73
|
-
}
|
|
74
|
-
return {
|
|
75
|
-
smartfilePackageJSON,
|
|
76
|
-
smartfilesReadme,
|
|
77
|
-
smartfilesReadmeHints,
|
|
78
|
-
smartfilesNpmextraJSON,
|
|
79
|
-
smartfilesMod,
|
|
80
|
-
smartfilesTest,
|
|
81
|
-
};
|
|
82
|
-
}
|
|
83
|
-
/**
|
|
84
|
-
* Convert files to context string
|
|
85
|
-
* @param files The files to convert
|
|
86
|
-
* @param mode The context mode to use
|
|
87
|
-
*/
|
|
88
|
-
async convertFilesToContext(files, mode = this.contextMode) {
|
|
63
|
+
async convertFilesToContextWithAnalysis(metadata, taskType, mode = this.contextMode) {
|
|
89
64
|
// Reset context result
|
|
90
65
|
this.contextResult = {
|
|
91
66
|
context: '',
|
|
@@ -95,58 +70,102 @@ export class EnhancedContext {
|
|
|
95
70
|
excludedFiles: [],
|
|
96
71
|
tokenSavings: 0
|
|
97
72
|
};
|
|
73
|
+
// Analyze files for smart prioritization
|
|
74
|
+
const analysis = await this.analyzer.analyze(metadata, taskType, []);
|
|
75
|
+
// Sort files by importance score (highest first)
|
|
76
|
+
const sortedAnalysis = [...analysis.files].sort((a, b) => b.importanceScore - a.importanceScore);
|
|
77
|
+
// Filter out excluded tier
|
|
78
|
+
const relevantFiles = sortedAnalysis.filter(f => f.tier !== 'excluded');
|
|
98
79
|
let totalTokenCount = 0;
|
|
99
80
|
let totalOriginalTokens = 0;
|
|
100
|
-
// Sort files by importance (for now just a simple alphabetical sort)
|
|
101
|
-
// Later this could be enhanced with more sophisticated prioritization
|
|
102
|
-
const sortedFiles = [...files].sort((a, b) => a.relative.localeCompare(b.relative));
|
|
103
81
|
const processedFiles = [];
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
82
|
+
// Load files with cache support
|
|
83
|
+
for (const fileAnalysis of relevantFiles) {
|
|
84
|
+
try {
|
|
85
|
+
// Check cache first
|
|
86
|
+
let contents;
|
|
87
|
+
let originalTokenCount;
|
|
88
|
+
const cached = await this.cache.get(fileAnalysis.path);
|
|
89
|
+
if (cached) {
|
|
90
|
+
contents = cached.contents;
|
|
91
|
+
originalTokenCount = cached.tokenCount;
|
|
92
|
+
}
|
|
93
|
+
else {
|
|
94
|
+
// Load file
|
|
95
|
+
const fileData = await plugins.smartfile.fs.toStringSync(fileAnalysis.path);
|
|
96
|
+
contents = fileData;
|
|
97
|
+
originalTokenCount = this.countTokens(contents);
|
|
98
|
+
// Cache it
|
|
99
|
+
await this.cache.set({
|
|
100
|
+
path: fileAnalysis.path,
|
|
101
|
+
contents,
|
|
102
|
+
tokenCount: originalTokenCount,
|
|
103
|
+
mtime: Date.now(),
|
|
104
|
+
cachedAt: Date.now()
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
totalOriginalTokens += originalTokenCount;
|
|
108
|
+
// Apply tier-based trimming
|
|
109
|
+
let processedContent = contents;
|
|
110
|
+
let trimLevel = 'light';
|
|
111
|
+
if (fileAnalysis.tier === 'essential') {
|
|
112
|
+
trimLevel = 'none';
|
|
113
|
+
}
|
|
114
|
+
else if (fileAnalysis.tier === 'important') {
|
|
115
|
+
trimLevel = 'light';
|
|
116
|
+
}
|
|
117
|
+
else if (fileAnalysis.tier === 'optional') {
|
|
118
|
+
trimLevel = 'aggressive';
|
|
119
|
+
}
|
|
120
|
+
// Apply trimming based on mode and tier
|
|
121
|
+
if (mode !== 'full' && trimLevel !== 'none') {
|
|
122
|
+
const relativePath = plugins.path.relative(this.projectDir, fileAnalysis.path);
|
|
123
|
+
processedContent = this.trimmer.trimFileWithLevel(relativePath, contents, trimLevel);
|
|
124
|
+
}
|
|
125
|
+
// Calculate token count
|
|
126
|
+
const processedTokenCount = this.countTokens(processedContent);
|
|
127
|
+
// Check token budget
|
|
128
|
+
if (totalTokenCount + processedTokenCount > this.tokenBudget) {
|
|
129
|
+
// We don't have budget for this file
|
|
130
|
+
const relativePath = plugins.path.relative(this.projectDir, fileAnalysis.path);
|
|
131
|
+
this.contextResult.excludedFiles.push({
|
|
132
|
+
path: fileAnalysis.path,
|
|
133
|
+
contents,
|
|
134
|
+
relativePath,
|
|
135
|
+
tokenCount: originalTokenCount,
|
|
136
|
+
importanceScore: fileAnalysis.importanceScore
|
|
137
|
+
});
|
|
138
|
+
continue;
|
|
139
|
+
}
|
|
140
|
+
// Format the file for context
|
|
141
|
+
const relativePath = plugins.path.relative(this.projectDir, fileAnalysis.path);
|
|
142
|
+
const formattedContent = `
|
|
143
|
+
====== START OF FILE ${relativePath} ======
|
|
144
|
+
|
|
131
145
|
${processedContent}
|
|
132
|
-
|
|
133
|
-
====== END OF FILE ${
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
+
|
|
147
|
+
====== END OF FILE ${relativePath} ======
|
|
148
|
+
`;
|
|
149
|
+
processedFiles.push(formattedContent);
|
|
150
|
+
totalTokenCount += processedTokenCount;
|
|
151
|
+
// Track file in appropriate list
|
|
152
|
+
const fileInfo = {
|
|
153
|
+
path: fileAnalysis.path,
|
|
154
|
+
contents: processedContent,
|
|
155
|
+
relativePath,
|
|
156
|
+
tokenCount: processedTokenCount,
|
|
157
|
+
importanceScore: fileAnalysis.importanceScore
|
|
158
|
+
};
|
|
159
|
+
if (trimLevel === 'none' || processedContent === contents) {
|
|
160
|
+
this.contextResult.includedFiles.push(fileInfo);
|
|
161
|
+
}
|
|
162
|
+
else {
|
|
163
|
+
this.contextResult.trimmedFiles.push(fileInfo);
|
|
164
|
+
this.contextResult.tokenSavings += (originalTokenCount - processedTokenCount);
|
|
165
|
+
}
|
|
146
166
|
}
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
this.contextResult.tokenSavings += (originalTokenCount - processedTokenCount);
|
|
167
|
+
catch (error) {
|
|
168
|
+
console.warn(`Failed to process file ${fileAnalysis.path}:`, error.message);
|
|
150
169
|
}
|
|
151
170
|
}
|
|
152
171
|
// Join all processed files
|
|
@@ -157,54 +176,38 @@ ${processedContent}
|
|
|
157
176
|
return context;
|
|
158
177
|
}
|
|
159
178
|
/**
|
|
160
|
-
* Build context for the project
|
|
161
|
-
* @param taskType
|
|
179
|
+
* Build context for the project using smart analysis
|
|
180
|
+
* @param taskType Task type for context-aware prioritization (defaults to 'description')
|
|
162
181
|
*/
|
|
163
182
|
async buildContext(taskType) {
|
|
164
183
|
// Initialize if needed
|
|
165
184
|
if (this.tokenBudget === 0) {
|
|
166
185
|
await this.initialize();
|
|
167
186
|
}
|
|
168
|
-
//
|
|
169
|
-
if
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
// Gather files
|
|
176
|
-
const taskConfig = taskType ? this.configManager.getTaskConfig(taskType) : undefined;
|
|
177
|
-
const files = await this.gatherFiles(taskConfig?.includePaths, taskConfig?.excludePaths);
|
|
178
|
-
// Convert files to context
|
|
179
|
-
// Create an array of all files to process
|
|
180
|
-
const allFiles = [];
|
|
181
|
-
// Add individual files
|
|
182
|
-
if (files.smartfilePackageJSON)
|
|
183
|
-
allFiles.push(files.smartfilePackageJSON);
|
|
184
|
-
if (files.smartfilesReadme)
|
|
185
|
-
allFiles.push(files.smartfilesReadme);
|
|
186
|
-
if (files.smartfilesReadmeHints)
|
|
187
|
-
allFiles.push(files.smartfilesReadmeHints);
|
|
188
|
-
if (files.smartfilesNpmextraJSON)
|
|
189
|
-
allFiles.push(files.smartfilesNpmextraJSON);
|
|
190
|
-
// Add arrays of files
|
|
191
|
-
if (files.smartfilesMod) {
|
|
192
|
-
if (Array.isArray(files.smartfilesMod)) {
|
|
193
|
-
allFiles.push(...files.smartfilesMod);
|
|
194
|
-
}
|
|
195
|
-
else {
|
|
196
|
-
allFiles.push(files.smartfilesMod);
|
|
197
|
-
}
|
|
198
|
-
}
|
|
199
|
-
if (files.smartfilesTest) {
|
|
200
|
-
if (Array.isArray(files.smartfilesTest)) {
|
|
201
|
-
allFiles.push(...files.smartfilesTest);
|
|
202
|
-
}
|
|
203
|
-
else {
|
|
204
|
-
allFiles.push(files.smartfilesTest);
|
|
205
|
-
}
|
|
187
|
+
// Smart context building always requires a task type for optimal prioritization
|
|
188
|
+
// Default to 'description' if not provided
|
|
189
|
+
const effectiveTaskType = taskType || 'description';
|
|
190
|
+
// Get task-specific configuration
|
|
191
|
+
const taskConfig = this.configManager.getTaskConfig(effectiveTaskType);
|
|
192
|
+
if (taskConfig.mode) {
|
|
193
|
+
this.setContextMode(taskConfig.mode);
|
|
206
194
|
}
|
|
207
|
-
|
|
195
|
+
// Build globs for scanning
|
|
196
|
+
const includeGlobs = taskConfig?.includePaths?.map(p => `${p}/**/*.ts`) || [
|
|
197
|
+
'ts/**/*.ts',
|
|
198
|
+
'ts*/**/*.ts'
|
|
199
|
+
];
|
|
200
|
+
// Add config files
|
|
201
|
+
const configGlobs = [
|
|
202
|
+
'package.json',
|
|
203
|
+
'readme.md',
|
|
204
|
+
'readme.hints.md',
|
|
205
|
+
'npmextra.json'
|
|
206
|
+
];
|
|
207
|
+
// Scan files for metadata (fast, doesn't load contents)
|
|
208
|
+
const metadata = await this.lazyLoader.scanFiles([...configGlobs, ...includeGlobs]);
|
|
209
|
+
// Use smart analyzer to build context with intelligent prioritization
|
|
210
|
+
await this.convertFilesToContextWithAnalysis(metadata, effectiveTaskType, this.contextMode);
|
|
208
211
|
return this.contextResult;
|
|
209
212
|
}
|
|
210
213
|
/**
|
|
@@ -269,4 +272,4 @@ ${gitDiff}
|
|
|
269
272
|
};
|
|
270
273
|
}
|
|
271
274
|
}
|
|
272
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
275
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZW5oYW5jZWQtY29udGV4dC5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uL3RzL2NvbnRleHQvZW5oYW5jZWQtY29udGV4dC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEtBQUssT0FBTyxNQUFNLGVBQWUsQ0FBQztBQUV6QyxPQUFPLEVBQUUsY0FBYyxFQUFFLE1BQU0sc0JBQXNCLENBQUM7QUFDdEQsT0FBTyxFQUFFLGFBQWEsRUFBRSxNQUFNLHFCQUFxQixDQUFDO0FBQ3BELE9BQU8sRUFBRSxjQUFjLEVBQUUsTUFBTSx1QkFBdUIsQ0FBQztBQUN2RCxPQUFPLEVBQUUsWUFBWSxFQUFFLE1BQU0sb0JBQW9CLENBQUM7QUFDbEQsT0FBTyxFQUFFLGVBQWUsRUFBRSxNQUFNLHVCQUF1QixDQUFDO0FBRXhEOztHQUVHO0FBQ0gsTUFBTSxPQUFPLGVBQWU7SUFrQjFCOzs7T0FHRztJQUNILFlBQVksYUFBcUI7UUFmekIsZ0JBQVcsR0FBZ0IsU0FBUyxDQUFDO1FBQ3JDLGdCQUFXLEdBQVcsTUFBTSxDQUFDLENBQUMsc0JBQXNCO1FBQ3BELGtCQUFhLEdBQW1CO1lBQ3RDLE9BQU8sRUFBRSxFQUFFO1lBQ1gsVUFBVSxFQUFFLENBQUM7WUFDYixhQUFhLEVBQUUsRUFBRTtZQUNqQixZQUFZLEVBQUUsRUFBRTtZQUNoQixhQUFhLEVBQUUsRUFBRTtZQUNqQixZQUFZLEVBQUUsQ0FBQztTQUNoQixDQUFDO1FBT0EsSUFBSSxDQUFDLFVBQVUsR0FBRyxhQUFhLENBQUM7UUFDaEMsSUFBSSxDQUFDLGFBQWEsR0FBRyxhQUFhLENBQUMsV0FBVyxFQUFFLENBQUM7UUFDakQsSUFBSSxDQUFDLE9BQU8sR0FBRyxJQUFJLGNBQWMsQ0FBQyxJQUFJLENBQUMsYUFBYSxDQUFDLGFBQWEsRUFBRSxDQUFDLENBQUM7UUFDdEUsSUFBSSxDQUFDLFVBQVUsR0FBRyxJQUFJLGNBQWMsQ0FBQyxhQUFhLENBQUMsQ0FBQztRQUNwRCxJQUFJLENBQUMsS0FBSyxHQUFHLElBQUksWUFBWSxDQUFDLGFBQWEsRUFBRSxJQUFJLENBQUMsYUFBYSxDQUFDLGNBQWMsRUFBRSxDQUFDLENBQUM7UUFDbEYsSUFBSSxDQUFDLFFBQVEsR0FBRyxJQUFJLGVBQWUsQ0FDakMsYUFBYSxFQUNiLElBQUksQ0FBQyxhQUFhLENBQUMsd0JBQXdCLEVBQUUsRUFDN0MsSUFBSSxDQUFDLGFBQWEsQ0FBQyxhQUFhLEVBQUUsQ0FDbkMsQ0FBQztJQUNKLENBQUM7SUFFRDs7T0FFRztJQUNJLEtBQUssQ0FBQyxVQUFVO1FBQ3JCLE1BQU0sSUFBSSxDQUFDLGFBQWEsQ0FBQyxVQUFVLENBQUMsSUFBSSxDQUFDLFVBQVUsQ0FBQyxDQUFDO1FBQ3JELElBQUksQ0FBQyxXQUFXLEdBQUcsSUFBSSxDQUFDLGFBQWEsQ0FBQyxZQUFZLEVBQUUsQ0FBQztRQUNyRCxJQUFJLENBQUMsT0FBTyxDQUFDLFlBQVksQ0FBQyxJQUFJLENBQUMsYUFBYSxDQUFDLGFBQWEsRUFBRSxDQUFDLENBQUM7UUFDOUQsTUFBTSxJQUFJLENBQUMsS0FBSyxDQUFDLElBQUksRUFBRSxDQUFDO0lBQzFCLENBQUM7SUFFRDs7O09BR0c7SUFDSSxjQUFjLENBQUMsSUFBaUI7UUFDckMsSUFBSSxDQUFDLFdBQVcsR0FBRyxJQUFJLENBQUM7SUFDMUIsQ0FBQztJQUVEOzs7T0FHRztJQUNJLGNBQWMsQ0FBQyxTQUFpQjtRQUNyQyxJQUFJLENBQUMsV0FBVyxHQUFHLFNBQVMsQ0FBQztJQUMvQixDQUFDO0lBRUQ7Ozs7OztPQU1HO0lBQ0ksS0FBSyxDQUFDLGlDQUFpQyxDQUM1QyxRQUF5QixFQUN6QixRQUFrQixFQUNsQixPQUFvQixJQUFJLENBQUMsV0FBVztRQUVwQyx1QkFBdUI7UUFDdkIsSUFBSSxDQUFDLGFBQWEsR0FBRztZQUNuQixPQUFPLEVBQUUsRUFBRTtZQUNYLFVBQVUsRUFBRSxDQUFDO1lBQ2IsYUFBYSxFQUFFLEVBQUU7WUFDakIsWUFBWSxFQUFFLEVBQUU7WUFDaEIsYUFBYSxFQUFFLEVBQUU7WUFDakIsWUFBWSxFQUFFLENBQUM7U0FDaEIsQ0FBQztRQUVGLHlDQUF5QztRQUN6QyxNQUFNLFFBQVEsR0FBRyxNQUFNLElBQUksQ0FBQyxRQUFRLENBQUMsT0FBTyxDQUFDLFFBQVEsRUFBRSxRQUFRLEVBQUUsRUFBRSxDQUFDLENBQUM7UUFFckUsaURBQWlEO1FBQ2pELE1BQU0sY0FBYyxHQUFHLENBQUMsR0FBRyxRQUFRLENBQUMsS0FBSyxDQUFDLENBQUMsSUFBSSxDQUM3QyxDQUFDLENBQUMsRUFBRSxDQUFDLEVBQUUsRUFBRSxDQUFDLENBQUMsQ0FBQyxlQUFlLEdBQUcsQ0FBQyxDQUFDLGVBQWUsQ0FDaEQsQ0FBQztRQUVGLDJCQUEyQjtRQUMzQixNQUFNLGFBQWEsR0FBRyxjQUFjLENBQUMsTUFBTSxDQUFDLENBQUMsQ0FBQyxFQUFFLENBQUMsQ0FBQyxDQUFDLElBQUksS0FBSyxVQUFVLENBQUMsQ0FBQztRQUV4RSxJQUFJLGVBQWUsR0FBRyxDQUFDLENBQUM7UUFDeEIsSUFBSSxtQkFBbUIsR0FBRyxDQUFDLENBQUM7UUFDNUIsTUFBTSxjQUFjLEdBQWEsRUFBRSxDQUFDO1FBRXBDLGdDQUFnQztRQUNoQyxLQUFLLE1BQU0sWUFBWSxJQUFJLGFBQWEsRUFBRSxDQUFDO1lBQ3pDLElBQUksQ0FBQztnQkFDSCxvQkFBb0I7Z0JBQ3BCLElBQUksUUFBZ0IsQ0FBQztnQkFDckIsSUFBSSxrQkFBMEIsQ0FBQztnQkFFL0IsTUFBTSxNQUFNLEdBQUcsTUFBTSxJQUFJLENBQUMsS0FBSyxDQUFDLEdBQUcsQ0FBQyxZQUFZLENBQUMsSUFBSSxDQUFDLENBQUM7Z0JBQ3ZELElBQUksTUFBTSxFQUFFLENBQUM7b0JBQ1gsUUFBUSxHQUFHLE1BQU0sQ0FBQyxRQUFRLENBQUM7b0JBQzNCLGtCQUFrQixHQUFHLE1BQU0sQ0FBQyxVQUFVLENBQUM7Z0JBQ3pDLENBQUM7cUJBQU0sQ0FBQztvQkFDTixZQUFZO29CQUNaLE1BQU0sUUFBUSxHQUFHLE1BQU0sT0FBTyxDQUFDLFNBQVMsQ0FBQyxFQUFFLENBQUMsWUFBWSxDQUFDLFlBQVksQ0FBQyxJQUFJLENBQUMsQ0FBQztvQkFDNUUsUUFBUSxHQUFHLFFBQVEsQ0FBQztvQkFDcEIsa0JBQWtCLEdBQUcsSUFBSSxDQUFDLFdBQVcsQ0FBQyxRQUFRLENBQUMsQ0FBQztvQkFFaEQsV0FBVztvQkFDWCxNQUFNLElBQUksQ0FBQyxLQUFLLENBQUMsR0FBRyxDQUFDO3dCQUNuQixJQUFJLEVBQUUsWUFBWSxDQUFDLElBQUk7d0JBQ3ZCLFFBQVE7d0JBQ1IsVUFBVSxFQUFFLGtCQUFrQjt3QkFDOUIsS0FBSyxFQUFFLElBQUksQ0FBQyxHQUFHLEVBQUU7d0JBQ2pCLFFBQVEsRUFBRSxJQUFJLENBQUMsR0FBRyxFQUFFO3FCQUNyQixDQUFDLENBQUM7Z0JBQ0wsQ0FBQztnQkFFRCxtQkFBbUIsSUFBSSxrQkFBa0IsQ0FBQztnQkFFMUMsNEJBQTRCO2dCQUM1QixJQUFJLGdCQUFnQixHQUFHLFFBQVEsQ0FBQztnQkFDaEMsSUFBSSxTQUFTLEdBQW9DLE9BQU8sQ0FBQztnQkFFekQsSUFBSSxZQUFZLENBQUMsSUFBSSxLQUFLLFdBQVcsRUFBRSxDQUFDO29CQUN0QyxTQUFTLEdBQUcsTUFBTSxDQUFDO2dCQUNyQixDQUFDO3FCQUFNLElBQUksWUFBWSxDQUFDLElBQUksS0FBSyxXQUFXLEVBQUUsQ0FBQztvQkFDN0MsU0FBUyxHQUFHLE9BQU8sQ0FBQztnQkFDdEIsQ0FBQztxQkFBTSxJQUFJLFlBQVksQ0FBQyxJQUFJLEtBQUssVUFBVSxFQUFFLENBQUM7b0JBQzVDLFNBQVMsR0FBRyxZQUFZLENBQUM7Z0JBQzNCLENBQUM7Z0JBRUQsd0NBQXdDO2dCQUN4QyxJQUFJLElBQUksS0FBSyxNQUFNLElBQUksU0FBUyxLQUFLLE1BQU0sRUFBRSxDQUFDO29CQUM1QyxNQUFNLFlBQVksR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLFFBQVEsQ0FBQyxJQUFJLENBQUMsVUFBVSxFQUFFLFlBQVksQ0FBQyxJQUFJLENBQUMsQ0FBQztvQkFDL0UsZ0JBQWdCLEdBQUcsSUFBSSxDQUFDLE9BQU8sQ0FBQyxpQkFBaUIsQ0FDL0MsWUFBWSxFQUNaLFFBQVEsRUFDUixTQUFTLENBQ1YsQ0FBQztnQkFDSixDQUFDO2dCQUVELHdCQUF3QjtnQkFDeEIsTUFBTSxtQkFBbUIsR0FBRyxJQUFJLENBQUMsV0FBVyxDQUFDLGdCQUFnQixDQUFDLENBQUM7Z0JBRS9ELHFCQUFxQjtnQkFDckIsSUFBSSxlQUFlLEdBQUcsbUJBQW1CLEdBQUcsSUFBSSxDQUFDLFdBQVcsRUFBRSxDQUFDO29CQUM3RCxxQ0FBcUM7b0JBQ3JDLE1BQU0sWUFBWSxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsUUFBUSxDQUFDLElBQUksQ0FBQyxVQUFVLEVBQUUsWUFBWSxDQUFDLElBQUksQ0FBQyxDQUFDO29CQUMvRSxJQUFJLENBQUMsYUFBYSxDQUFDLGFBQWEsQ0FBQyxJQUFJLENBQUM7d0JBQ3BDLElBQUksRUFBRSxZQUFZLENBQUMsSUFBSTt3QkFDdkIsUUFBUTt3QkFDUixZQUFZO3dCQUNaLFVBQVUsRUFBRSxrQkFBa0I7d0JBQzlCLGVBQWUsRUFBRSxZQUFZLENBQUMsZUFBZTtxQkFDOUMsQ0FBQyxDQUFDO29CQUNILFNBQVM7Z0JBQ1gsQ0FBQztnQkFFRCw4QkFBOEI7Z0JBQzlCLE1BQU0sWUFBWSxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsUUFBUSxDQUFDLElBQUksQ0FBQyxVQUFVLEVBQUUsWUFBWSxDQUFDLElBQUksQ0FBQyxDQUFDO2dCQUMvRSxNQUFNLGdCQUFnQixHQUFHO3VCQUNWLFlBQVk7O0VBRWpDLGdCQUFnQjs7cUJBRUcsWUFBWTtTQUN4QixDQUFDO2dCQUVGLGNBQWMsQ0FBQyxJQUFJLENBQUMsZ0JBQWdCLENBQUMsQ0FBQztnQkFDdEMsZUFBZSxJQUFJLG1CQUFtQixDQUFDO2dCQUV2QyxpQ0FBaUM7Z0JBQ2pDLE1BQU0sUUFBUSxHQUFjO29CQUMxQixJQUFJLEVBQUUsWUFBWSxDQUFDLElBQUk7b0JBQ3ZCLFFBQVEsRUFBRSxnQkFBZ0I7b0JBQzFCLFlBQVk7b0JBQ1osVUFBVSxFQUFFLG1CQUFtQjtvQkFDL0IsZUFBZSxFQUFFLFlBQVksQ0FBQyxlQUFlO2lCQUM5QyxDQUFDO2dCQUVGLElBQUksU0FBUyxLQUFLLE1BQU0sSUFBSSxnQkFBZ0IsS0FBSyxRQUFRLEVBQUUsQ0FBQztvQkFDMUQsSUFBSSxDQUFDLGFBQWEsQ0FBQyxhQUFhLENBQUMsSUFBSSxDQUFDLFFBQVEsQ0FBQyxDQUFDO2dCQUNsRCxDQUFDO3FCQUFNLENBQUM7b0JBQ04sSUFBSSxDQUFDLGFBQWEsQ0FBQyxZQUFZLENBQUMsSUFBSSxDQUFDLFFBQVEsQ0FBQyxDQUFDO29CQUMvQyxJQUFJLENBQUMsYUFBYSxDQUFDLFlBQVksSUFBSSxDQUFDLGtCQUFrQixHQUFHLG1CQUFtQixDQUFDLENBQUM7Z0JBQ2hGLENBQUM7WUFDSCxDQUFDO1lBQUMsT0FBTyxLQUFLLEVBQUUsQ0FBQztnQkFDZixPQUFPLENBQUMsSUFBSSxDQUFDLDBCQUEwQixZQUFZLENBQUMsSUFBSSxHQUFHLEVBQUUsS0FBSyxDQUFDLE9BQU8sQ0FBQyxDQUFDO1lBQzlFLENBQUM7UUFDSCxDQUFDO1FBRUQsMkJBQTJCO1FBQzNCLE1BQU0sT0FBTyxHQUFHLGNBQWMsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLENBQUM7UUFFMUMsd0JBQXdCO1FBQ3hCLElBQUksQ0FBQyxhQUFhLENBQUMsT0FBTyxHQUFHLE9BQU8sQ0FBQztRQUNyQyxJQUFJLENBQUMsYUFBYSxDQUFDLFVBQVUsR0FBRyxlQUFlLENBQUM7UUFFaEQsT0FBTyxPQUFPLENBQUM7SUFDakIsQ0FBQztJQUVEOzs7T0FHRztJQUNJLEtBQUssQ0FBQyxZQUFZLENBQUMsUUFBbUI7UUFDM0MsdUJBQXVCO1FBQ3ZCLElBQUksSUFBSSxDQUFDLFdBQVcsS0FBSyxDQUFDLEVBQUUsQ0FBQztZQUMzQixNQUFNLElBQUksQ0FBQyxVQUFVLEVBQUUsQ0FBQztRQUMxQixDQUFDO1FBRUQsZ0ZBQWdGO1FBQ2hGLDJDQUEyQztRQUMzQyxNQUFNLGlCQUFpQixHQUFHLFFBQVEsSUFBSSxhQUFhLENBQUM7UUFFcEQsa0NBQWtDO1FBQ2xDLE1BQU0sVUFBVSxHQUFHLElBQUksQ0FBQyxhQUFhLENBQUMsYUFBYSxDQUFDLGlCQUFpQixDQUFDLENBQUM7UUFDdkUsSUFBSSxVQUFVLENBQUMsSUFBSSxFQUFFLENBQUM7WUFDcEIsSUFBSSxDQUFDLGNBQWMsQ0FBQyxVQUFVLENBQUMsSUFBSSxDQUFDLENBQUM7UUFDdkMsQ0FBQztRQUVELDJCQUEyQjtRQUMzQixNQUFNLFlBQVksR0FBRyxVQUFVLEVBQUUsWUFBWSxFQUFFLEdBQUcsQ0FBQyxDQUFDLENBQUMsRUFBRSxDQUFDLEdBQUcsQ0FBQyxVQUFVLENBQUMsSUFBSTtZQUN6RSxZQUFZO1lBQ1osYUFBYTtTQUNkLENBQUM7UUFFRixtQkFBbUI7UUFDbkIsTUFBTSxXQUFXLEdBQUc7WUFDbEIsY0FBYztZQUNkLFdBQVc7WUFDWCxpQkFBaUI7WUFDakIsZUFBZTtTQUNoQixDQUFDO1FBRUYsd0RBQXdEO1FBQ3hELE1BQU0sUUFBUSxHQUFHLE1BQU0sSUFBSSxDQUFDLFVBQVUsQ0FBQyxTQUFTLENBQUMsQ0FBQyxHQUFHLFdBQVcsRUFBRSxHQUFHLFlBQVksQ0FBQyxDQUFDLENBQUM7UUFFcEYsc0VBQXNFO1FBQ3RFLE1BQU0sSUFBSSxDQUFDLGlDQUFpQyxDQUFDLFFBQVEsRUFBRSxpQkFBaUIsRUFBRSxJQUFJLENBQUMsV0FBVyxDQUFDLENBQUM7UUFFNUYsT0FBTyxJQUFJLENBQUMsYUFBYSxDQUFDO0lBQzVCLENBQUM7SUFFRDs7O09BR0c7SUFDSSxpQkFBaUIsQ0FBQyxPQUFlO1FBQ3RDLHNEQUFzRDtRQUN0RCxJQUFJLENBQUMsSUFBSSxDQUFDLGFBQWEsQ0FBQyxPQUFPLEVBQUUsQ0FBQztZQUNoQyxPQUFPLElBQUksQ0FBQyxhQUFhLENBQUM7UUFDNUIsQ0FBQztRQUVELDBCQUEwQjtRQUMxQixNQUFNLFdBQVcsR0FBRzs7O0VBR3RCLE9BQU87OztLQUdKLENBQUM7UUFFRixNQUFNLGNBQWMsR0FBRyxJQUFJLENBQUMsV0FBVyxDQUFDLFdBQVcsQ0FBQyxDQUFDO1FBRXJELGlDQUFpQztRQUNqQyxJQUFJLENBQUMsYUFBYSxDQUFDLE9BQU8sSUFBSSxXQUFXLENBQUM7UUFDMUMsSUFBSSxDQUFDLGFBQWEsQ0FBQyxVQUFVLElBQUksY0FBYyxDQUFDO1FBRWhELE9BQU8sSUFBSSxDQUFDLGFBQWEsQ0FBQztJQUM1QixDQUFDO0lBRUQ7Ozs7T0FJRztJQUNJLFdBQVcsQ0FBQyxJQUFZLEVBQUUsUUFBZ0IsZUFBZTtRQUM5RCxJQUFJLENBQUM7WUFDSCxnREFBZ0Q7WUFDaEQsTUFBTSxNQUFNLEdBQUcsT0FBTyxDQUFDLFlBQVksQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUFDLENBQUM7WUFDakQsT0FBTyxNQUFNLENBQUMsTUFBTSxDQUFDO1FBQ3ZCLENBQUM7UUFBQyxPQUFPLEtBQUssRUFBRSxDQUFDO1lBQ2YsT0FBTyxDQUFDLEtBQUssQ0FBQyx3QkFBd0IsRUFBRSxLQUFLLENBQUMsQ0FBQztZQUMvQyxpREFBaUQ7WUFDakQsT0FBTyxJQUFJLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxNQUFNLEdBQUcsQ0FBQyxDQUFDLENBQUM7UUFDcEMsQ0FBQztJQUNILENBQUM7SUFFRDs7T0FFRztJQUNJLGdCQUFnQjtRQUNyQixPQUFPLElBQUksQ0FBQyxhQUFhLENBQUM7SUFDNUIsQ0FBQztJQUVEOztPQUVHO0lBQ0ksYUFBYTtRQUNsQixPQUFPLElBQUksQ0FBQyxhQUFhLENBQUMsVUFBVSxDQUFDO0lBQ3ZDLENBQUM7SUFFRDs7T0FFRztJQUNJLHdCQUF3QjtRQUM3QixPQUFPO1lBQ0wsT0FBTyxFQUFFLElBQUksQ0FBQyxhQUFhLENBQUMsT0FBTztZQUNuQyxVQUFVLEVBQUUsSUFBSSxDQUFDLGFBQWEsQ0FBQyxVQUFVO1NBQzFDLENBQUM7SUFDSixDQUFDO0NBQ0YifQ==
|
|
@@ -2,6 +2,9 @@ import { EnhancedContext } from './enhanced-context.js';
|
|
|
2
2
|
import { TaskContextFactory } from './task-context-factory.js';
|
|
3
3
|
import { ConfigManager } from './config-manager.js';
|
|
4
4
|
import { ContextTrimmer } from './context-trimmer.js';
|
|
5
|
-
import
|
|
6
|
-
|
|
7
|
-
|
|
5
|
+
import { LazyFileLoader } from './lazy-file-loader.js';
|
|
6
|
+
import { ContextCache } from './context-cache.js';
|
|
7
|
+
import { ContextAnalyzer } from './context-analyzer.js';
|
|
8
|
+
import type { ContextMode, IContextConfig, IContextResult, IFileInfo, ITrimConfig, ITaskConfig, TaskType, ICacheConfig, IAnalyzerConfig, IPrioritizationWeights, ITierConfig, ITierSettings, IFileMetadata, ICacheEntry, IFileDependencies, IFileAnalysis, IAnalysisResult } from './types.js';
|
|
9
|
+
export { EnhancedContext, TaskContextFactory, ConfigManager, ContextTrimmer, LazyFileLoader, ContextCache, ContextAnalyzer, };
|
|
10
|
+
export type { ContextMode, IContextConfig, IContextResult, IFileInfo, ITrimConfig, ITaskConfig, TaskType, ICacheConfig, IAnalyzerConfig, IPrioritizationWeights, ITierConfig, ITierSettings, IFileMetadata, ICacheEntry, IFileDependencies, IFileAnalysis, IAnalysisResult };
|
package/dist_ts/context/index.js
CHANGED
|
@@ -2,7 +2,10 @@ import { EnhancedContext } from './enhanced-context.js';
|
|
|
2
2
|
import { TaskContextFactory } from './task-context-factory.js';
|
|
3
3
|
import { ConfigManager } from './config-manager.js';
|
|
4
4
|
import { ContextTrimmer } from './context-trimmer.js';
|
|
5
|
+
import { LazyFileLoader } from './lazy-file-loader.js';
|
|
6
|
+
import { ContextCache } from './context-cache.js';
|
|
7
|
+
import { ContextAnalyzer } from './context-analyzer.js';
|
|
5
8
|
export {
|
|
6
9
|
// Classes
|
|
7
|
-
EnhancedContext, TaskContextFactory, ConfigManager, ContextTrimmer, };
|
|
8
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
10
|
+
EnhancedContext, TaskContextFactory, ConfigManager, ContextTrimmer, LazyFileLoader, ContextCache, ContextAnalyzer, };
|
|
11
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi90cy9jb250ZXh0L2luZGV4LnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxlQUFlLEVBQUUsTUFBTSx1QkFBdUIsQ0FBQztBQUN4RCxPQUFPLEVBQUUsa0JBQWtCLEVBQUUsTUFBTSwyQkFBMkIsQ0FBQztBQUMvRCxPQUFPLEVBQUUsYUFBYSxFQUFFLE1BQU0scUJBQXFCLENBQUM7QUFDcEQsT0FBTyxFQUFFLGNBQWMsRUFBRSxNQUFNLHNCQUFzQixDQUFDO0FBQ3RELE9BQU8sRUFBRSxjQUFjLEVBQUUsTUFBTSx1QkFBdUIsQ0FBQztBQUN2RCxPQUFPLEVBQUUsWUFBWSxFQUFFLE1BQU0sb0JBQW9CLENBQUM7QUFDbEQsT0FBTyxFQUFFLGVBQWUsRUFBRSxNQUFNLHVCQUF1QixDQUFDO0FBcUJ4RCxPQUFPO0FBQ0wsVUFBVTtBQUNWLGVBQWUsRUFDZixrQkFBa0IsRUFDbEIsYUFBYSxFQUNiLGNBQWMsRUFDZCxjQUFjLEVBQ2QsWUFBWSxFQUNaLGVBQWUsR0FDaEIsQ0FBQyJ9
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import type { IFileMetadata, IFileInfo } from './types.js';
|
|
2
|
+
/**
|
|
3
|
+
* LazyFileLoader handles efficient file loading by:
|
|
4
|
+
* - Scanning files for metadata without loading contents
|
|
5
|
+
* - Providing fast file size and token estimates
|
|
6
|
+
* - Loading contents only when requested
|
|
7
|
+
* - Parallel loading of selected files
|
|
8
|
+
*/
|
|
9
|
+
export declare class LazyFileLoader {
|
|
10
|
+
private projectRoot;
|
|
11
|
+
private metadataCache;
|
|
12
|
+
/**
|
|
13
|
+
* Creates a new LazyFileLoader
|
|
14
|
+
* @param projectRoot - Root directory of the project
|
|
15
|
+
*/
|
|
16
|
+
constructor(projectRoot: string);
|
|
17
|
+
/**
|
|
18
|
+
* Scans files in given globs and creates metadata without loading contents
|
|
19
|
+
* @param globs - File patterns to scan (e.g., ['ts/**\/*.ts', 'test/**\/*.ts'])
|
|
20
|
+
* @returns Array of file metadata
|
|
21
|
+
*/
|
|
22
|
+
scanFiles(globs: string[]): Promise<IFileMetadata[]>;
|
|
23
|
+
/**
|
|
24
|
+
* Gets metadata for a single file without loading contents
|
|
25
|
+
* @param filePath - Absolute path to the file
|
|
26
|
+
* @returns File metadata
|
|
27
|
+
*/
|
|
28
|
+
getMetadata(filePath: string): Promise<IFileMetadata>;
|
|
29
|
+
/**
|
|
30
|
+
* Loads file contents for selected files in parallel
|
|
31
|
+
* @param metadata - Array of file metadata to load
|
|
32
|
+
* @param tokenizer - Function to calculate accurate token count
|
|
33
|
+
* @returns Array of complete file info with contents
|
|
34
|
+
*/
|
|
35
|
+
loadFiles(metadata: IFileMetadata[], tokenizer: (content: string) => number): Promise<IFileInfo[]>;
|
|
36
|
+
/**
|
|
37
|
+
* Loads a single file with contents
|
|
38
|
+
* @param filePath - Absolute path to the file
|
|
39
|
+
* @param tokenizer - Function to calculate accurate token count
|
|
40
|
+
* @returns Complete file info with contents
|
|
41
|
+
*/
|
|
42
|
+
loadFile(filePath: string, tokenizer: (content: string) => number): Promise<IFileInfo>;
|
|
43
|
+
/**
|
|
44
|
+
* Updates importance scores for metadata entries
|
|
45
|
+
* @param scores - Map of file paths to importance scores
|
|
46
|
+
*/
|
|
47
|
+
updateImportanceScores(scores: Map<string, number>): void;
|
|
48
|
+
/**
|
|
49
|
+
* Clears the metadata cache
|
|
50
|
+
*/
|
|
51
|
+
clearCache(): void;
|
|
52
|
+
/**
|
|
53
|
+
* Gets total estimated tokens for all cached metadata
|
|
54
|
+
*/
|
|
55
|
+
getTotalEstimatedTokens(): number;
|
|
56
|
+
/**
|
|
57
|
+
* Gets cached metadata entries
|
|
58
|
+
*/
|
|
59
|
+
getCachedMetadata(): IFileMetadata[];
|
|
60
|
+
}
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
import * as plugins from '../plugins.js';
|
|
2
|
+
import * as fs from 'fs';
|
|
3
|
+
/**
|
|
4
|
+
* LazyFileLoader handles efficient file loading by:
|
|
5
|
+
* - Scanning files for metadata without loading contents
|
|
6
|
+
* - Providing fast file size and token estimates
|
|
7
|
+
* - Loading contents only when requested
|
|
8
|
+
* - Parallel loading of selected files
|
|
9
|
+
*/
|
|
10
|
+
export class LazyFileLoader {
|
|
11
|
+
/**
|
|
12
|
+
* Creates a new LazyFileLoader
|
|
13
|
+
* @param projectRoot - Root directory of the project
|
|
14
|
+
*/
|
|
15
|
+
constructor(projectRoot) {
|
|
16
|
+
this.metadataCache = new Map();
|
|
17
|
+
this.projectRoot = projectRoot;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Scans files in given globs and creates metadata without loading contents
|
|
21
|
+
* @param globs - File patterns to scan (e.g., ['ts/**\/*.ts', 'test/**\/*.ts'])
|
|
22
|
+
* @returns Array of file metadata
|
|
23
|
+
*/
|
|
24
|
+
async scanFiles(globs) {
|
|
25
|
+
const metadata = [];
|
|
26
|
+
for (const globPattern of globs) {
|
|
27
|
+
try {
|
|
28
|
+
const smartFiles = await plugins.smartfile.fs.fileTreeToObject(this.projectRoot, globPattern);
|
|
29
|
+
const fileArray = Array.isArray(smartFiles) ? smartFiles : [smartFiles];
|
|
30
|
+
for (const smartFile of fileArray) {
|
|
31
|
+
try {
|
|
32
|
+
const meta = await this.getMetadata(smartFile.path);
|
|
33
|
+
metadata.push(meta);
|
|
34
|
+
}
|
|
35
|
+
catch (error) {
|
|
36
|
+
// Skip files that can't be read
|
|
37
|
+
console.warn(`Failed to get metadata for ${smartFile.path}:`, error.message);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
catch (error) {
|
|
42
|
+
// Skip patterns that don't match any files
|
|
43
|
+
console.warn(`No files found for pattern ${globPattern}`);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
return metadata;
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Gets metadata for a single file without loading contents
|
|
50
|
+
* @param filePath - Absolute path to the file
|
|
51
|
+
* @returns File metadata
|
|
52
|
+
*/
|
|
53
|
+
async getMetadata(filePath) {
|
|
54
|
+
// Check cache first
|
|
55
|
+
if (this.metadataCache.has(filePath)) {
|
|
56
|
+
const cached = this.metadataCache.get(filePath);
|
|
57
|
+
const currentStats = await fs.promises.stat(filePath);
|
|
58
|
+
// Return cached if file hasn't changed
|
|
59
|
+
if (cached.mtime === Math.floor(currentStats.mtimeMs)) {
|
|
60
|
+
return cached;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
// Get file stats
|
|
64
|
+
const stats = await fs.promises.stat(filePath);
|
|
65
|
+
const relativePath = plugins.path.relative(this.projectRoot, filePath);
|
|
66
|
+
// Estimate tokens: rough estimate of ~4 characters per token
|
|
67
|
+
// This is faster than reading and tokenizing the entire file
|
|
68
|
+
const estimatedTokens = Math.ceil(stats.size / 4);
|
|
69
|
+
const metadata = {
|
|
70
|
+
path: filePath,
|
|
71
|
+
relativePath,
|
|
72
|
+
size: stats.size,
|
|
73
|
+
mtime: Math.floor(stats.mtimeMs),
|
|
74
|
+
estimatedTokens,
|
|
75
|
+
};
|
|
76
|
+
// Cache the metadata
|
|
77
|
+
this.metadataCache.set(filePath, metadata);
|
|
78
|
+
return metadata;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Loads file contents for selected files in parallel
|
|
82
|
+
* @param metadata - Array of file metadata to load
|
|
83
|
+
* @param tokenizer - Function to calculate accurate token count
|
|
84
|
+
* @returns Array of complete file info with contents
|
|
85
|
+
*/
|
|
86
|
+
async loadFiles(metadata, tokenizer) {
|
|
87
|
+
// Load files in parallel
|
|
88
|
+
const loadPromises = metadata.map(async (meta) => {
|
|
89
|
+
try {
|
|
90
|
+
const contents = await plugins.smartfile.fs.toStringSync(meta.path);
|
|
91
|
+
const tokenCount = tokenizer(contents);
|
|
92
|
+
const fileInfo = {
|
|
93
|
+
path: meta.path,
|
|
94
|
+
relativePath: meta.relativePath,
|
|
95
|
+
contents,
|
|
96
|
+
tokenCount,
|
|
97
|
+
importanceScore: meta.importanceScore,
|
|
98
|
+
};
|
|
99
|
+
return fileInfo;
|
|
100
|
+
}
|
|
101
|
+
catch (error) {
|
|
102
|
+
console.warn(`Failed to load file ${meta.path}:`, error.message);
|
|
103
|
+
return null;
|
|
104
|
+
}
|
|
105
|
+
});
|
|
106
|
+
// Wait for all loads to complete and filter out failures
|
|
107
|
+
const results = await Promise.all(loadPromises);
|
|
108
|
+
return results.filter((r) => r !== null);
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Loads a single file with contents
|
|
112
|
+
* @param filePath - Absolute path to the file
|
|
113
|
+
* @param tokenizer - Function to calculate accurate token count
|
|
114
|
+
* @returns Complete file info with contents
|
|
115
|
+
*/
|
|
116
|
+
async loadFile(filePath, tokenizer) {
|
|
117
|
+
const meta = await this.getMetadata(filePath);
|
|
118
|
+
const contents = await plugins.smartfile.fs.toStringSync(filePath);
|
|
119
|
+
const tokenCount = tokenizer(contents);
|
|
120
|
+
const relativePath = plugins.path.relative(this.projectRoot, filePath);
|
|
121
|
+
return {
|
|
122
|
+
path: filePath,
|
|
123
|
+
relativePath,
|
|
124
|
+
contents,
|
|
125
|
+
tokenCount,
|
|
126
|
+
importanceScore: meta.importanceScore,
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* Updates importance scores for metadata entries
|
|
131
|
+
* @param scores - Map of file paths to importance scores
|
|
132
|
+
*/
|
|
133
|
+
updateImportanceScores(scores) {
|
|
134
|
+
for (const [path, score] of scores) {
|
|
135
|
+
const meta = this.metadataCache.get(path);
|
|
136
|
+
if (meta) {
|
|
137
|
+
meta.importanceScore = score;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Clears the metadata cache
|
|
143
|
+
*/
|
|
144
|
+
clearCache() {
|
|
145
|
+
this.metadataCache.clear();
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* Gets total estimated tokens for all cached metadata
|
|
149
|
+
*/
|
|
150
|
+
getTotalEstimatedTokens() {
|
|
151
|
+
let total = 0;
|
|
152
|
+
for (const meta of this.metadataCache.values()) {
|
|
153
|
+
total += meta.estimatedTokens;
|
|
154
|
+
}
|
|
155
|
+
return total;
|
|
156
|
+
}
|
|
157
|
+
/**
|
|
158
|
+
* Gets cached metadata entries
|
|
159
|
+
*/
|
|
160
|
+
getCachedMetadata() {
|
|
161
|
+
return Array.from(this.metadataCache.values());
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibGF6eS1maWxlLWxvYWRlci5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uL3RzL2NvbnRleHQvbGF6eS1maWxlLWxvYWRlci50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEtBQUssT0FBTyxNQUFNLGVBQWUsQ0FBQztBQUN6QyxPQUFPLEtBQUssRUFBRSxNQUFNLElBQUksQ0FBQztBQUd6Qjs7Ozs7O0dBTUc7QUFDSCxNQUFNLE9BQU8sY0FBYztJQUl6Qjs7O09BR0c7SUFDSCxZQUFZLFdBQW1CO1FBTnZCLGtCQUFhLEdBQStCLElBQUksR0FBRyxFQUFFLENBQUM7UUFPNUQsSUFBSSxDQUFDLFdBQVcsR0FBRyxXQUFXLENBQUM7SUFDakMsQ0FBQztJQUVEOzs7O09BSUc7SUFDSSxLQUFLLENBQUMsU0FBUyxDQUFDLEtBQWU7UUFDcEMsTUFBTSxRQUFRLEdBQW9CLEVBQUUsQ0FBQztRQUVyQyxLQUFLLE1BQU0sV0FBVyxJQUFJLEtBQUssRUFBRSxDQUFDO1lBQ2hDLElBQUksQ0FBQztnQkFDSCxNQUFNLFVBQVUsR0FBRyxNQUFNLE9BQU8sQ0FBQyxTQUFTLENBQUMsRUFBRSxDQUFDLGdCQUFnQixDQUFDLElBQUksQ0FBQyxXQUFXLEVBQUUsV0FBVyxDQUFDLENBQUM7Z0JBQzlGLE1BQU0sU0FBUyxHQUFHLEtBQUssQ0FBQyxPQUFPLENBQUMsVUFBVSxDQUFDLENBQUMsQ0FBQyxDQUFDLFVBQVUsQ0FBQyxDQUFDLENBQUMsQ0FBQyxVQUFVLENBQUMsQ0FBQztnQkFFeEUsS0FBSyxNQUFNLFNBQVMsSUFBSSxTQUFTLEVBQUUsQ0FBQztvQkFDbEMsSUFBSSxDQUFDO3dCQUNILE1BQU0sSUFBSSxHQUFHLE1BQU0sSUFBSSxDQUFDLFdBQVcsQ0FBQyxTQUFTLENBQUMsSUFBSSxDQUFDLENBQUM7d0JBQ3BELFFBQVEsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLENBQUM7b0JBQ3RCLENBQUM7b0JBQUMsT0FBTyxLQUFLLEVBQUUsQ0FBQzt3QkFDZixnQ0FBZ0M7d0JBQ2hDLE9BQU8sQ0FBQyxJQUFJLENBQUMsOEJBQThCLFNBQVMsQ0FBQyxJQUFJLEdBQUcsRUFBRSxLQUFLLENBQUMsT0FBTyxDQUFDLENBQUM7b0JBQy9FLENBQUM7Z0JBQ0gsQ0FBQztZQUNILENBQUM7WUFBQyxPQUFPLEtBQUssRUFBRSxDQUFDO2dCQUNmLDJDQUEyQztnQkFDM0MsT0FBTyxDQUFDLElBQUksQ0FBQyw4QkFBOEIsV0FBVyxFQUFFLENBQUMsQ0FBQztZQUM1RCxDQUFDO1FBQ0gsQ0FBQztRQUVELE9BQU8sUUFBUSxDQUFDO0lBQ2xCLENBQUM7SUFFRDs7OztPQUlHO0lBQ0ksS0FBSyxDQUFDLFdBQVcsQ0FBQyxRQUFnQjtRQUN2QyxvQkFBb0I7UUFDcEIsSUFBSSxJQUFJLENBQUMsYUFBYSxDQUFDLEdBQUcsQ0FBQyxRQUFRLENBQUMsRUFBRSxDQUFDO1lBQ3JDLE1BQU0sTUFBTSxHQUFHLElBQUksQ0FBQyxhQUFhLENBQUMsR0FBRyxDQUFDLFFBQVEsQ0FBRSxDQUFDO1lBQ2pELE1BQU0sWUFBWSxHQUFHLE1BQU0sRUFBRSxDQUFDLFFBQVEsQ0FBQyxJQUFJLENBQUMsUUFBUSxDQUFDLENBQUM7WUFFdEQsdUNBQXVDO1lBQ3ZDLElBQUksTUFBTSxDQUFDLEtBQUssS0FBSyxJQUFJLENBQUMsS0FBSyxDQUFDLFlBQVksQ0FBQyxPQUFPLENBQUMsRUFBRSxDQUFDO2dCQUN0RCxPQUFPLE1BQU0sQ0FBQztZQUNoQixDQUFDO1FBQ0gsQ0FBQztRQUVELGlCQUFpQjtRQUNqQixNQUFNLEtBQUssR0FBRyxNQUFNLEVBQUUsQ0FBQyxRQUFRLENBQUMsSUFBSSxDQUFDLFFBQVEsQ0FBQyxDQUFDO1FBQy9DLE1BQU0sWUFBWSxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsUUFBUSxDQUFDLElBQUksQ0FBQyxXQUFXLEVBQUUsUUFBUSxDQUFDLENBQUM7UUFFdkUsNkRBQTZEO1FBQzdELDZEQUE2RDtRQUM3RCxNQUFNLGVBQWUsR0FBRyxJQUFJLENBQUMsSUFBSSxDQUFDLEtBQUssQ0FBQyxJQUFJLEdBQUcsQ0FBQyxDQUFDLENBQUM7UUFFbEQsTUFBTSxRQUFRLEdBQWtCO1lBQzlCLElBQUksRUFBRSxRQUFRO1lBQ2QsWUFBWTtZQUNaLElBQUksRUFBRSxLQUFLLENBQUMsSUFBSTtZQUNoQixLQUFLLEVBQUUsSUFBSSxDQUFDLEtBQUssQ0FBQyxLQUFLLENBQUMsT0FBTyxDQUFDO1lBQ2hDLGVBQWU7U0FDaEIsQ0FBQztRQUVGLHFCQUFxQjtRQUNyQixJQUFJLENBQUMsYUFBYSxDQUFDLEdBQUcsQ0FBQyxRQUFRLEVBQUUsUUFBUSxDQUFDLENBQUM7UUFFM0MsT0FBTyxRQUFRLENBQUM7SUFDbEIsQ0FBQztJQUVEOzs7OztPQUtHO0lBQ0ksS0FBSyxDQUFDLFNBQVMsQ0FDcEIsUUFBeUIsRUFDekIsU0FBc0M7UUFFdEMseUJBQXlCO1FBQ3pCLE1BQU0sWUFBWSxHQUFHLFFBQVEsQ0FBQyxHQUFHLENBQUMsS0FBSyxFQUFFLElBQUksRUFBRSxFQUFFO1lBQy9DLElBQUksQ0FBQztnQkFDSCxNQUFNLFFBQVEsR0FBRyxNQUFNLE9BQU8sQ0FBQyxTQUFTLENBQUMsRUFBRSxDQUFDLFlBQVksQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLENBQUM7Z0JBQ3BFLE1BQU0sVUFBVSxHQUFHLFNBQVMsQ0FBQyxRQUFRLENBQUMsQ0FBQztnQkFFdkMsTUFBTSxRQUFRLEdBQWM7b0JBQzFCLElBQUksRUFBRSxJQUFJLENBQUMsSUFBSTtvQkFDZixZQUFZLEVBQUUsSUFBSSxDQUFDLFlBQVk7b0JBQy9CLFFBQVE7b0JBQ1IsVUFBVTtvQkFDVixlQUFlLEVBQUUsSUFBSSxDQUFDLGVBQWU7aUJBQ3RDLENBQUM7Z0JBRUYsT0FBTyxRQUFRLENBQUM7WUFDbEIsQ0FBQztZQUFDLE9BQU8sS0FBSyxFQUFFLENBQUM7Z0JBQ2YsT0FBTyxDQUFDLElBQUksQ0FBQyx1QkFBdUIsSUFBSSxDQUFDLElBQUksR0FBRyxFQUFFLEtBQUssQ0FBQyxPQUFPLENBQUMsQ0FBQztnQkFDakUsT0FBTyxJQUFJLENBQUM7WUFDZCxDQUFDO1FBQ0gsQ0FBQyxDQUFDLENBQUM7UUFFSCx5REFBeUQ7UUFDekQsTUFBTSxPQUFPLEdBQUcsTUFBTSxPQUFPLENBQUMsR0FBRyxDQUFDLFlBQVksQ0FBQyxDQUFDO1FBQ2hELE9BQU8sT0FBTyxDQUFDLE1BQU0sQ0FBQyxDQUFDLENBQUMsRUFBa0IsRUFBRSxDQUFDLENBQUMsS0FBSyxJQUFJLENBQUMsQ0FBQztJQUMzRCxDQUFDO0lBRUQ7Ozs7O09BS0c7SUFDSSxLQUFLLENBQUMsUUFBUSxDQUNuQixRQUFnQixFQUNoQixTQUFzQztRQUV0QyxNQUFNLElBQUksR0FBRyxNQUFNLElBQUksQ0FBQyxXQUFXLENBQUMsUUFBUSxDQUFDLENBQUM7UUFDOUMsTUFBTSxRQUFRLEdBQUcsTUFBTSxPQUFPLENBQUMsU0FBUyxDQUFDLEVBQUUsQ0FBQyxZQUFZLENBQUMsUUFBUSxDQUFDLENBQUM7UUFDbkUsTUFBTSxVQUFVLEdBQUcsU0FBUyxDQUFDLFFBQVEsQ0FBQyxDQUFDO1FBQ3ZDLE1BQU0sWUFBWSxHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsUUFBUSxDQUFDLElBQUksQ0FBQyxXQUFXLEVBQUUsUUFBUSxDQUFDLENBQUM7UUFFdkUsT0FBTztZQUNMLElBQUksRUFBRSxRQUFRO1lBQ2QsWUFBWTtZQUNaLFFBQVE7WUFDUixVQUFVO1lBQ1YsZUFBZSxFQUFFLElBQUksQ0FBQyxlQUFlO1NBQ3RDLENBQUM7SUFDSixDQUFDO0lBRUQ7OztPQUdHO0lBQ0ksc0JBQXNCLENBQUMsTUFBMkI7UUFDdkQsS0FBSyxNQUFNLENBQUMsSUFBSSxFQUFFLEtBQUssQ0FBQyxJQUFJLE1BQU0sRUFBRSxDQUFDO1lBQ25DLE1BQU0sSUFBSSxHQUFHLElBQUksQ0FBQyxhQUFhLENBQUMsR0FBRyxDQUFDLElBQUksQ0FBQyxDQUFDO1lBQzFDLElBQUksSUFBSSxFQUFFLENBQUM7Z0JBQ1QsSUFBSSxDQUFDLGVBQWUsR0FBRyxLQUFLLENBQUM7WUFDL0IsQ0FBQztRQUNILENBQUM7SUFDSCxDQUFDO0lBRUQ7O09BRUc7SUFDSSxVQUFVO1FBQ2YsSUFBSSxDQUFDLGFBQWEsQ0FBQyxLQUFLLEVBQUUsQ0FBQztJQUM3QixDQUFDO0lBRUQ7O09BRUc7SUFDSSx1QkFBdUI7UUFDNUIsSUFBSSxLQUFLLEdBQUcsQ0FBQyxDQUFDO1FBQ2QsS0FBSyxNQUFNLElBQUksSUFBSSxJQUFJLENBQUMsYUFBYSxDQUFDLE1BQU0sRUFBRSxFQUFFLENBQUM7WUFDL0MsS0FBSyxJQUFJLElBQUksQ0FBQyxlQUFlLENBQUM7UUFDaEMsQ0FBQztRQUNELE9BQU8sS0FBSyxDQUFDO0lBQ2YsQ0FBQztJQUVEOztPQUVHO0lBQ0ksaUJBQWlCO1FBQ3RCLE9BQU8sS0FBSyxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsYUFBYSxDQUFDLE1BQU0sRUFBRSxDQUFDLENBQUM7SUFDakQsQ0FBQztDQUNGIn0=
|