agileflow 2.90.6 → 2.91.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/README.md +6 -6
- package/lib/codebase-indexer.js +810 -0
- package/lib/validate-names.js +3 -3
- package/package.json +4 -1
- package/scripts/obtain-context.js +238 -0
- package/scripts/precompact-context.sh +13 -1
- package/scripts/query-codebase.js +430 -0
- package/scripts/tui/blessed/data/watcher.js +175 -0
- package/scripts/tui/blessed/index.js +244 -0
- package/scripts/tui/blessed/panels/output.js +95 -0
- package/scripts/tui/blessed/panels/sessions.js +143 -0
- package/scripts/tui/blessed/panels/trace.js +91 -0
- package/scripts/tui/blessed/ui/help.js +77 -0
- package/scripts/tui/blessed/ui/screen.js +52 -0
- package/scripts/tui/blessed/ui/statusbar.js +51 -0
- package/scripts/tui/blessed/ui/tabbar.js +99 -0
- package/scripts/tui/index.js +38 -32
- package/scripts/tui/simple-tui.js +8 -5
- package/scripts/validators/README.md +143 -0
- package/scripts/validators/component-validator.js +212 -0
- package/scripts/validators/json-schema-validator.js +179 -0
- package/scripts/validators/markdown-validator.js +153 -0
- package/scripts/validators/migration-validator.js +117 -0
- package/scripts/validators/security-validator.js +276 -0
- package/scripts/validators/story-format-validator.js +176 -0
- package/scripts/validators/test-result-validator.js +99 -0
- package/scripts/validators/workflow-validator.js +240 -0
- package/src/core/agents/accessibility.md +6 -0
- package/src/core/agents/adr-writer.md +6 -0
- package/src/core/agents/analytics.md +6 -0
- package/src/core/agents/api.md +6 -0
- package/src/core/agents/ci.md +6 -0
- package/src/core/agents/codebase-query.md +237 -0
- package/src/core/agents/compliance.md +6 -0
- package/src/core/agents/configuration-damage-control.md +6 -0
- package/src/core/agents/configuration-visual-e2e.md +6 -0
- package/src/core/agents/database.md +10 -0
- package/src/core/agents/datamigration.md +6 -0
- package/src/core/agents/design.md +6 -0
- package/src/core/agents/devops.md +6 -0
- package/src/core/agents/documentation.md +6 -0
- package/src/core/agents/epic-planner.md +6 -0
- package/src/core/agents/integrations.md +6 -0
- package/src/core/agents/mentor.md +6 -0
- package/src/core/agents/mobile.md +6 -0
- package/src/core/agents/monitoring.md +6 -0
- package/src/core/agents/multi-expert.md +6 -0
- package/src/core/agents/performance.md +6 -0
- package/src/core/agents/product.md +6 -0
- package/src/core/agents/qa.md +6 -0
- package/src/core/agents/readme-updater.md +6 -0
- package/src/core/agents/refactor.md +6 -0
- package/src/core/agents/research.md +6 -0
- package/src/core/agents/security.md +6 -0
- package/src/core/agents/testing.md +10 -0
- package/src/core/agents/ui.md +6 -0
- package/src/core/commands/audit.md +401 -0
- package/src/core/commands/board.md +1 -0
- package/src/core/commands/epic.md +92 -1
- package/src/core/commands/help.md +1 -0
- package/src/core/commands/metrics.md +1 -0
- package/src/core/commands/research/analyze.md +1 -0
- package/src/core/commands/research/ask.md +2 -0
- package/src/core/commands/research/import.md +1 -0
- package/src/core/commands/research/list.md +2 -0
- package/src/core/commands/research/synthesize.md +584 -0
- package/src/core/commands/research/view.md +2 -0
- package/src/core/commands/status.md +126 -1
- package/src/core/commands/story/list.md +9 -9
- package/src/core/commands/story/view.md +1 -0
- package/src/core/experts/codebase-query/expertise.yaml +190 -0
- package/src/core/experts/codebase-query/question.md +73 -0
- package/src/core/experts/codebase-query/self-improve.md +105 -0
- package/tools/cli/commands/tui.js +40 -271
|
@@ -0,0 +1,810 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Codebase Indexer - Fast index for programmatic codebase queries
|
|
3
|
+
*
|
|
4
|
+
* Features:
|
|
5
|
+
* - Builds index of files with metadata (type, exports, imports, tags)
|
|
6
|
+
* - Incremental updates based on file mtime
|
|
7
|
+
* - LRU cache integration for performance
|
|
8
|
+
* - Persistent storage in .agileflow/cache/codebase-index.json
|
|
9
|
+
* - Configuration via docs/00-meta/agileflow-metadata.json
|
|
10
|
+
*
|
|
11
|
+
* Based on RLM (Recursive Language Models) research:
|
|
12
|
+
* Use programmatic search instead of loading full context.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
const fs = require('fs');
|
|
16
|
+
const path = require('path');
|
|
17
|
+
const { LRUCache } = require('./file-cache');
|
|
18
|
+
const { safeReadJSON, safeWriteJSON, debugLog } = require('./errors');
|
|
19
|
+
|
|
20
|
+
// Debug mode via env var
|
|
21
|
+
const DEBUG = process.env.AGILEFLOW_DEBUG === '1';
|
|
22
|
+
|
|
23
|
+
// Index version for migration support
|
|
24
|
+
const INDEX_VERSION = '1.0.0';
|
|
25
|
+
|
|
26
|
+
// Default configuration (can be overridden via agileflow-metadata.json)
|
|
27
|
+
const DEFAULT_CONFIG = {
|
|
28
|
+
ttlMs: 60000, // 1 minute cache TTL (or ttl_hours * 3600000)
|
|
29
|
+
maxCacheSize: 10,
|
|
30
|
+
excludePatterns: [
|
|
31
|
+
'node_modules/**',
|
|
32
|
+
'.git/**',
|
|
33
|
+
'dist/**',
|
|
34
|
+
'build/**',
|
|
35
|
+
'coverage/**',
|
|
36
|
+
'.agileflow/cache/**',
|
|
37
|
+
'*.log',
|
|
38
|
+
'*.lock',
|
|
39
|
+
],
|
|
40
|
+
includePatterns: [
|
|
41
|
+
'**/*.js',
|
|
42
|
+
'**/*.ts',
|
|
43
|
+
'**/*.tsx',
|
|
44
|
+
'**/*.jsx',
|
|
45
|
+
'**/*.md',
|
|
46
|
+
'**/*.json',
|
|
47
|
+
'**/*.yaml',
|
|
48
|
+
'**/*.yml',
|
|
49
|
+
],
|
|
50
|
+
maxFileSizeKb: 500,
|
|
51
|
+
tokenBudget: 15000,
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Load configuration from agileflow-metadata.json if available
|
|
56
|
+
* @param {string} projectRoot - Project root directory
|
|
57
|
+
* @returns {Object} Merged configuration
|
|
58
|
+
*/
|
|
59
|
+
function loadConfig(projectRoot) {
|
|
60
|
+
const metadataPath = path.join(projectRoot, 'docs/00-meta/agileflow-metadata.json');
|
|
61
|
+
const metadata = safeReadJSON(metadataPath);
|
|
62
|
+
|
|
63
|
+
if (!metadata.ok || !metadata.data?.features?.codebaseIndex) {
|
|
64
|
+
return DEFAULT_CONFIG;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const userConfig = metadata.data.features.codebaseIndex;
|
|
68
|
+
|
|
69
|
+
return {
|
|
70
|
+
...DEFAULT_CONFIG,
|
|
71
|
+
// Convert ttl_hours to ttlMs if provided
|
|
72
|
+
ttlMs: userConfig.ttl_hours
|
|
73
|
+
? userConfig.ttl_hours * 60 * 60 * 1000
|
|
74
|
+
: DEFAULT_CONFIG.ttlMs,
|
|
75
|
+
excludePatterns: userConfig.exclude_patterns || DEFAULT_CONFIG.excludePatterns,
|
|
76
|
+
includePatterns: userConfig.include_patterns || DEFAULT_CONFIG.includePatterns,
|
|
77
|
+
maxFileSizeKb: userConfig.max_file_size_kb || DEFAULT_CONFIG.maxFileSizeKb,
|
|
78
|
+
tokenBudget: userConfig.token_budget || DEFAULT_CONFIG.tokenBudget,
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// Tag patterns for auto-detection from path
|
|
83
|
+
const TAG_PATTERNS = {
|
|
84
|
+
api: /\/(api|routes|endpoints|controllers)\//i,
|
|
85
|
+
ui: /\/(components|ui|views|pages)\//i,
|
|
86
|
+
database: /\/(db|database|models|schema|migrations)\//i,
|
|
87
|
+
auth: /\/(auth|login|session|jwt|oauth)\//i,
|
|
88
|
+
test: /\/(test|tests|__tests__|spec|specs)\//i,
|
|
89
|
+
config: /\/(config|settings|env)\//i,
|
|
90
|
+
lib: /\/(lib|utils|helpers|shared)\//i,
|
|
91
|
+
docs: /\/(docs|documentation)\//i,
|
|
92
|
+
scripts: /\/(scripts|bin|tools)\//i,
|
|
93
|
+
types: /\/(types|typings|interfaces)\//i,
|
|
94
|
+
};
|
|
95
|
+
|
|
96
|
+
// In-memory cache for indices
|
|
97
|
+
const indexCache = new LRUCache({
|
|
98
|
+
maxSize: DEFAULT_CONFIG.maxCacheSize,
|
|
99
|
+
ttlMs: DEFAULT_CONFIG.ttlMs,
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Create empty index structure
|
|
104
|
+
* @returns {Object} Empty index
|
|
105
|
+
*/
|
|
106
|
+
function createEmptyIndex(projectRoot) {
|
|
107
|
+
return {
|
|
108
|
+
version: INDEX_VERSION,
|
|
109
|
+
created_at: new Date().toISOString(),
|
|
110
|
+
updated_at: new Date().toISOString(),
|
|
111
|
+
project_root: projectRoot,
|
|
112
|
+
stats: {
|
|
113
|
+
total_files: 0,
|
|
114
|
+
indexed_files: 0,
|
|
115
|
+
build_time_ms: 0,
|
|
116
|
+
},
|
|
117
|
+
files: {},
|
|
118
|
+
tags: {},
|
|
119
|
+
symbols: {
|
|
120
|
+
functions: {},
|
|
121
|
+
classes: {},
|
|
122
|
+
exports: {},
|
|
123
|
+
},
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* Get file type from extension
|
|
129
|
+
* @param {string} filePath - File path
|
|
130
|
+
* @returns {string} File type
|
|
131
|
+
*/
|
|
132
|
+
function getFileType(filePath) {
|
|
133
|
+
const ext = path.extname(filePath).toLowerCase();
|
|
134
|
+
const typeMap = {
|
|
135
|
+
'.js': 'javascript',
|
|
136
|
+
'.ts': 'typescript',
|
|
137
|
+
'.tsx': 'typescript-react',
|
|
138
|
+
'.jsx': 'javascript-react',
|
|
139
|
+
'.md': 'markdown',
|
|
140
|
+
'.json': 'json',
|
|
141
|
+
'.yaml': 'yaml',
|
|
142
|
+
'.yml': 'yaml',
|
|
143
|
+
'.css': 'css',
|
|
144
|
+
'.scss': 'scss',
|
|
145
|
+
'.html': 'html',
|
|
146
|
+
};
|
|
147
|
+
return typeMap[ext] || 'unknown';
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* Extract exports from JavaScript/TypeScript file content
|
|
152
|
+
* @param {string} content - File content
|
|
153
|
+
* @returns {string[]} List of export names
|
|
154
|
+
*/
|
|
155
|
+
function extractExports(content) {
|
|
156
|
+
const exports = [];
|
|
157
|
+
|
|
158
|
+
// Named exports: export const/let/var/function/class name
|
|
159
|
+
const namedExportRegex = /export\s+(?:const|let|var|function|class|async\s+function)\s+(\w+)/g;
|
|
160
|
+
let match;
|
|
161
|
+
while ((match = namedExportRegex.exec(content)) !== null) {
|
|
162
|
+
exports.push(match[1]);
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
// Export { name1, name2 }
|
|
166
|
+
const bracketExportRegex = /export\s*\{([^}]+)\}/g;
|
|
167
|
+
while ((match = bracketExportRegex.exec(content)) !== null) {
|
|
168
|
+
const names = match[1].split(',').map(n => {
|
|
169
|
+
const parts = n.trim().split(/\s+as\s+/);
|
|
170
|
+
return parts[parts.length - 1].trim();
|
|
171
|
+
});
|
|
172
|
+
exports.push(...names.filter(n => n && n !== 'default'));
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
// module.exports = { name1, name2 } or module.exports.name
|
|
176
|
+
const cjsExportRegex = /module\.exports(?:\.(\w+))?\s*=/g;
|
|
177
|
+
while ((match = cjsExportRegex.exec(content)) !== null) {
|
|
178
|
+
if (match[1]) {
|
|
179
|
+
exports.push(match[1]);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
// module.exports = { ... } - extract object keys (handles shorthand: { foo, bar })
|
|
184
|
+
const cjsObjectRegex = /module\.exports\s*=\s*\{([^}]+)\}/;
|
|
185
|
+
const cjsMatch = content.match(cjsObjectRegex);
|
|
186
|
+
if (cjsMatch) {
|
|
187
|
+
// Split by comma and extract property names (handles "foo," "bar:" "baz")
|
|
188
|
+
const props = cjsMatch[1].split(',');
|
|
189
|
+
for (const prop of props) {
|
|
190
|
+
const trimmed = prop.trim();
|
|
191
|
+
// Extract the key name (before : or the whole thing for shorthand)
|
|
192
|
+
const keyMatch = trimmed.match(/^(\w+)/);
|
|
193
|
+
if (keyMatch) {
|
|
194
|
+
exports.push(keyMatch[1]);
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
return [...new Set(exports)]; // Dedupe
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/**
|
|
203
|
+
* Extract imports from JavaScript/TypeScript file content
|
|
204
|
+
* @param {string} content - File content
|
|
205
|
+
* @returns {string[]} List of import sources
|
|
206
|
+
*/
|
|
207
|
+
function extractImports(content) {
|
|
208
|
+
const imports = [];
|
|
209
|
+
|
|
210
|
+
// ES6 imports: import ... from 'source'
|
|
211
|
+
const es6ImportRegex = /import\s+(?:[\w{},\s*]+\s+from\s+)?['"]([^'"]+)['"]/g;
|
|
212
|
+
let match;
|
|
213
|
+
while ((match = es6ImportRegex.exec(content)) !== null) {
|
|
214
|
+
imports.push(match[1]);
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
// CommonJS requires: require('source')
|
|
218
|
+
const requireRegex = /require\s*\(\s*['"]([^'"]+)['"]\s*\)/g;
|
|
219
|
+
while ((match = requireRegex.exec(content)) !== null) {
|
|
220
|
+
imports.push(match[1]);
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
return [...new Set(imports)]; // Dedupe
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/**
|
|
227
|
+
* Extract function and class names from content
|
|
228
|
+
* @param {string} content - File content
|
|
229
|
+
* @returns {Object} { functions: string[], classes: string[] }
|
|
230
|
+
*/
|
|
231
|
+
function extractSymbols(content) {
|
|
232
|
+
const functions = [];
|
|
233
|
+
const classes = [];
|
|
234
|
+
|
|
235
|
+
// Function declarations: function name() or async function name()
|
|
236
|
+
const funcRegex = /(?:async\s+)?function\s+(\w+)/g;
|
|
237
|
+
let match;
|
|
238
|
+
while ((match = funcRegex.exec(content)) !== null) {
|
|
239
|
+
functions.push(match[1]);
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
// Arrow functions assigned to const: const name = () =>
|
|
243
|
+
const arrowRegex = /(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s*)?\(/g;
|
|
244
|
+
while ((match = arrowRegex.exec(content)) !== null) {
|
|
245
|
+
functions.push(match[1]);
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
// Class declarations: class Name
|
|
249
|
+
const classRegex = /class\s+(\w+)/g;
|
|
250
|
+
while ((match = classRegex.exec(content)) !== null) {
|
|
251
|
+
classes.push(match[1]);
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
return {
|
|
255
|
+
functions: [...new Set(functions)],
|
|
256
|
+
classes: [...new Set(classes)],
|
|
257
|
+
};
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
/**
|
|
261
|
+
* Detect tags for a file based on path and content
|
|
262
|
+
* @param {string} filePath - File path relative to project root
|
|
263
|
+
* @param {string} content - File content
|
|
264
|
+
* @returns {string[]} List of tags
|
|
265
|
+
*/
|
|
266
|
+
function detectTags(filePath, content) {
|
|
267
|
+
const tags = [];
|
|
268
|
+
|
|
269
|
+
// Path-based tags
|
|
270
|
+
for (const [tag, pattern] of Object.entries(TAG_PATTERNS)) {
|
|
271
|
+
if (pattern.test(filePath)) {
|
|
272
|
+
tags.push(tag);
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
// Content-based tags (look for common patterns)
|
|
277
|
+
if (/\bexpress\b|\brouter\b|\bapp\.(get|post|put|delete)\b/i.test(content)) {
|
|
278
|
+
if (!tags.includes('api')) tags.push('api');
|
|
279
|
+
}
|
|
280
|
+
if (/\bReact\b|\buseState\b|\buseEffect\b|\bcomponent\b/i.test(content)) {
|
|
281
|
+
if (!tags.includes('ui')) tags.push('ui');
|
|
282
|
+
}
|
|
283
|
+
if (/\bsequelize\b|\bprisma\b|\bmongodb\b|\bsql\b/i.test(content)) {
|
|
284
|
+
if (!tags.includes('database')) tags.push('database');
|
|
285
|
+
}
|
|
286
|
+
if (/\bjwt\b|\bpassport\b|\bauthenticate\b|\blogin\b/i.test(content)) {
|
|
287
|
+
if (!tags.includes('auth')) tags.push('auth');
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
return [...new Set(tags)];
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
/**
|
|
294
|
+
* Check if file should be included based on patterns
|
|
295
|
+
* @param {string} relativePath - Path relative to project root
|
|
296
|
+
* @param {string[]} excludePatterns - Patterns to exclude
|
|
297
|
+
* @returns {boolean} True if file should be included
|
|
298
|
+
*/
|
|
299
|
+
function shouldIncludeFile(relativePath, excludePatterns) {
|
|
300
|
+
for (const pattern of excludePatterns) {
|
|
301
|
+
// Convert glob to regex
|
|
302
|
+
// Handle ** (matches any path segments including none)
|
|
303
|
+
// Handle * (matches within a single path segment)
|
|
304
|
+
let regexPattern = pattern
|
|
305
|
+
.replace(/\./g, '\\.') // Escape dots
|
|
306
|
+
.replace(/\*\*/g, '<<<GLOB>>>') // Temp placeholder for **
|
|
307
|
+
.replace(/\*/g, '[^/]*') // Single * = any chars except /
|
|
308
|
+
.replace(/<<<GLOB>>>/g, '.*') // ** = any chars including /
|
|
309
|
+
.replace(/\?/g, '.'); // ? = any single char
|
|
310
|
+
|
|
311
|
+
// Support patterns that should match the start of path
|
|
312
|
+
const regex = new RegExp(`^${regexPattern}`);
|
|
313
|
+
if (regex.test(relativePath)) {
|
|
314
|
+
return false;
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
return true;
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
/**
|
|
321
|
+
* Recursively scan directory for files
|
|
322
|
+
* @param {string} dirPath - Directory to scan
|
|
323
|
+
* @param {string} projectRoot - Project root for relative paths
|
|
324
|
+
* @param {string[]} excludePatterns - Patterns to exclude
|
|
325
|
+
* @param {number} maxFileSizeKb - Max file size in KB
|
|
326
|
+
* @returns {Object[]} List of file info objects
|
|
327
|
+
*/
|
|
328
|
+
function scanDirectory(dirPath, projectRoot, excludePatterns, maxFileSizeKb) {
|
|
329
|
+
const files = [];
|
|
330
|
+
|
|
331
|
+
try {
|
|
332
|
+
const entries = fs.readdirSync(dirPath, { withFileTypes: true });
|
|
333
|
+
|
|
334
|
+
for (const entry of entries) {
|
|
335
|
+
const fullPath = path.join(dirPath, entry.name);
|
|
336
|
+
const relativePath = path.relative(projectRoot, fullPath);
|
|
337
|
+
|
|
338
|
+
// Check exclusion
|
|
339
|
+
if (!shouldIncludeFile(relativePath, excludePatterns)) {
|
|
340
|
+
continue;
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
if (entry.isDirectory()) {
|
|
344
|
+
// Recurse into subdirectory
|
|
345
|
+
files.push(...scanDirectory(fullPath, projectRoot, excludePatterns, maxFileSizeKb));
|
|
346
|
+
} else if (entry.isFile()) {
|
|
347
|
+
try {
|
|
348
|
+
const stat = fs.statSync(fullPath);
|
|
349
|
+
const sizeKb = stat.size / 1024;
|
|
350
|
+
|
|
351
|
+
// Skip files that are too large
|
|
352
|
+
if (sizeKb > maxFileSizeKb) {
|
|
353
|
+
if (DEBUG) debugLog(`Skipping large file: ${relativePath} (${sizeKb.toFixed(1)}KB)`);
|
|
354
|
+
continue;
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
files.push({
|
|
358
|
+
path: relativePath,
|
|
359
|
+
fullPath,
|
|
360
|
+
size: stat.size,
|
|
361
|
+
mtime: stat.mtime.getTime(),
|
|
362
|
+
});
|
|
363
|
+
} catch (err) {
|
|
364
|
+
if (DEBUG) debugLog(`Error stat file ${relativePath}: ${err.message}`);
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
} catch (err) {
|
|
369
|
+
if (DEBUG) debugLog(`Error scanning directory ${dirPath}: ${err.message}`);
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
return files;
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
/**
|
|
376
|
+
* Build complete codebase index
|
|
377
|
+
* @param {string} projectRoot - Project root directory
|
|
378
|
+
* @param {Object} options - Configuration options
|
|
379
|
+
* @returns {Object} { ok: boolean, data?: Object, error?: string }
|
|
380
|
+
*/
|
|
381
|
+
function buildIndex(projectRoot, options = {}) {
|
|
382
|
+
const startTime = Date.now();
|
|
383
|
+
|
|
384
|
+
// Load config from metadata, then override with options
|
|
385
|
+
const baseConfig = loadConfig(projectRoot);
|
|
386
|
+
const config = {
|
|
387
|
+
...baseConfig,
|
|
388
|
+
...options,
|
|
389
|
+
};
|
|
390
|
+
|
|
391
|
+
try {
|
|
392
|
+
// Verify project root exists
|
|
393
|
+
if (!fs.existsSync(projectRoot)) {
|
|
394
|
+
return { ok: false, error: `Project root not found: ${projectRoot}` };
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
const index = createEmptyIndex(projectRoot);
|
|
398
|
+
|
|
399
|
+
// Scan for files
|
|
400
|
+
const files = scanDirectory(projectRoot, projectRoot, config.excludePatterns, config.maxFileSizeKb);
|
|
401
|
+
index.stats.total_files = files.length;
|
|
402
|
+
|
|
403
|
+
// Process each file
|
|
404
|
+
for (const fileInfo of files) {
|
|
405
|
+
const { path: relativePath, fullPath, size, mtime } = fileInfo;
|
|
406
|
+
const type = getFileType(relativePath);
|
|
407
|
+
|
|
408
|
+
// Read content for code files
|
|
409
|
+
let content = '';
|
|
410
|
+
let exports = [];
|
|
411
|
+
let imports = [];
|
|
412
|
+
let symbols = { functions: [], classes: [] };
|
|
413
|
+
let tags = [];
|
|
414
|
+
|
|
415
|
+
if (['javascript', 'typescript', 'javascript-react', 'typescript-react'].includes(type)) {
|
|
416
|
+
try {
|
|
417
|
+
content = fs.readFileSync(fullPath, 'utf8');
|
|
418
|
+
exports = extractExports(content);
|
|
419
|
+
imports = extractImports(content);
|
|
420
|
+
symbols = extractSymbols(content);
|
|
421
|
+
tags = detectTags(relativePath, content);
|
|
422
|
+
} catch (err) {
|
|
423
|
+
if (DEBUG) debugLog(`Error reading ${relativePath}: ${err.message}`);
|
|
424
|
+
}
|
|
425
|
+
} else {
|
|
426
|
+
// Just detect tags from path for non-code files
|
|
427
|
+
tags = detectTags(relativePath, '');
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
// Add file to index
|
|
431
|
+
index.files[relativePath] = {
|
|
432
|
+
type,
|
|
433
|
+
size,
|
|
434
|
+
mtime,
|
|
435
|
+
exports,
|
|
436
|
+
imports,
|
|
437
|
+
tags,
|
|
438
|
+
};
|
|
439
|
+
|
|
440
|
+
// Update tag index
|
|
441
|
+
for (const tag of tags) {
|
|
442
|
+
if (!index.tags[tag]) {
|
|
443
|
+
index.tags[tag] = [];
|
|
444
|
+
}
|
|
445
|
+
index.tags[tag].push(relativePath);
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
// Update symbol index
|
|
449
|
+
for (const func of symbols.functions) {
|
|
450
|
+
if (!index.symbols.functions[func]) {
|
|
451
|
+
index.symbols.functions[func] = [];
|
|
452
|
+
}
|
|
453
|
+
index.symbols.functions[func].push(relativePath);
|
|
454
|
+
}
|
|
455
|
+
for (const cls of symbols.classes) {
|
|
456
|
+
if (!index.symbols.classes[cls]) {
|
|
457
|
+
index.symbols.classes[cls] = [];
|
|
458
|
+
}
|
|
459
|
+
index.symbols.classes[cls].push(relativePath);
|
|
460
|
+
}
|
|
461
|
+
for (const exp of exports) {
|
|
462
|
+
if (!index.symbols.exports[exp]) {
|
|
463
|
+
index.symbols.exports[exp] = [];
|
|
464
|
+
}
|
|
465
|
+
index.symbols.exports[exp].push(relativePath);
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
index.stats.indexed_files++;
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
// Update timing
|
|
472
|
+
index.stats.build_time_ms = Date.now() - startTime;
|
|
473
|
+
index.updated_at = new Date().toISOString();
|
|
474
|
+
|
|
475
|
+
// Store in cache
|
|
476
|
+
const cacheKey = `index:${projectRoot}`;
|
|
477
|
+
indexCache.set(cacheKey, index);
|
|
478
|
+
|
|
479
|
+
// Persist to disk
|
|
480
|
+
const cachePath = getCachePath(projectRoot);
|
|
481
|
+
const writeResult = saveIndexToDisk(cachePath, index);
|
|
482
|
+
if (!writeResult.ok && DEBUG) {
|
|
483
|
+
debugLog(`Warning: Could not persist index to disk: ${writeResult.error}`);
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
return { ok: true, data: index };
|
|
487
|
+
} catch (err) {
|
|
488
|
+
return { ok: false, error: err.message };
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
/**
|
|
493
|
+
* Get cache file path for a project
|
|
494
|
+
* @param {string} projectRoot - Project root
|
|
495
|
+
* @returns {string} Cache file path
|
|
496
|
+
*/
|
|
497
|
+
function getCachePath(projectRoot) {
|
|
498
|
+
return path.join(projectRoot, '.agileflow', 'cache', 'codebase-index.json');
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
/**
|
|
502
|
+
* Save index to disk atomically
|
|
503
|
+
* @param {string} cachePath - Path to save to
|
|
504
|
+
* @param {Object} index - Index data
|
|
505
|
+
* @returns {Object} { ok: boolean, error?: string }
|
|
506
|
+
*/
|
|
507
|
+
function saveIndexToDisk(cachePath, index) {
|
|
508
|
+
try {
|
|
509
|
+
const dir = path.dirname(cachePath);
|
|
510
|
+
if (!fs.existsSync(dir)) {
|
|
511
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
// Atomic write via temp file
|
|
515
|
+
const tempPath = `${cachePath}.tmp`;
|
|
516
|
+
fs.writeFileSync(tempPath, JSON.stringify(index, null, 2));
|
|
517
|
+
fs.renameSync(tempPath, cachePath);
|
|
518
|
+
|
|
519
|
+
return { ok: true };
|
|
520
|
+
} catch (err) {
|
|
521
|
+
return { ok: false, error: err.message };
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
/**
|
|
526
|
+
* Load index from disk
|
|
527
|
+
* @param {string} cachePath - Cache file path
|
|
528
|
+
* @returns {Object} { ok: boolean, data?: Object, error?: string }
|
|
529
|
+
*/
|
|
530
|
+
function loadIndexFromDisk(cachePath) {
|
|
531
|
+
return safeReadJSON(cachePath);
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
/**
|
|
535
|
+
* Update index incrementally (only changed files)
|
|
536
|
+
* @param {string} projectRoot - Project root
|
|
537
|
+
* @param {Object} options - Configuration options
|
|
538
|
+
* @returns {Object} { ok: boolean, data?: Object, error?: string }
|
|
539
|
+
*/
|
|
540
|
+
function updateIndex(projectRoot, options = {}) {
|
|
541
|
+
const config = {
|
|
542
|
+
...DEFAULT_CONFIG,
|
|
543
|
+
...options,
|
|
544
|
+
};
|
|
545
|
+
|
|
546
|
+
try {
|
|
547
|
+
// Try to load existing index
|
|
548
|
+
const cachePath = getCachePath(projectRoot);
|
|
549
|
+
const loadResult = loadIndexFromDisk(cachePath);
|
|
550
|
+
|
|
551
|
+
// If no existing index, do full build
|
|
552
|
+
if (!loadResult.ok) {
|
|
553
|
+
return buildIndex(projectRoot, options);
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
const existingIndex = loadResult.data;
|
|
557
|
+
|
|
558
|
+
// Check if version matches
|
|
559
|
+
if (existingIndex.version !== INDEX_VERSION) {
|
|
560
|
+
if (DEBUG) debugLog('Index version mismatch, rebuilding');
|
|
561
|
+
return buildIndex(projectRoot, options);
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
const startTime = Date.now();
|
|
565
|
+
|
|
566
|
+
// Scan for current files
|
|
567
|
+
const currentFiles = scanDirectory(projectRoot, projectRoot, config.excludePatterns, config.maxFileSizeKb);
|
|
568
|
+
const currentFilePaths = new Set(currentFiles.map(f => f.path));
|
|
569
|
+
|
|
570
|
+
// Track changes
|
|
571
|
+
let changedCount = 0;
|
|
572
|
+
let addedCount = 0;
|
|
573
|
+
let removedCount = 0;
|
|
574
|
+
|
|
575
|
+
// Remove deleted files from index
|
|
576
|
+
for (const filePath of Object.keys(existingIndex.files)) {
|
|
577
|
+
if (!currentFilePaths.has(filePath)) {
|
|
578
|
+
delete existingIndex.files[filePath];
|
|
579
|
+
removedCount++;
|
|
580
|
+
}
|
|
581
|
+
}
|
|
582
|
+
|
|
583
|
+
// Check for new or modified files
|
|
584
|
+
for (const fileInfo of currentFiles) {
|
|
585
|
+
const { path: relativePath, fullPath, size, mtime } = fileInfo;
|
|
586
|
+
const existing = existingIndex.files[relativePath];
|
|
587
|
+
|
|
588
|
+
// If new file or modified (mtime changed)
|
|
589
|
+
if (!existing || existing.mtime !== mtime) {
|
|
590
|
+
const type = getFileType(relativePath);
|
|
591
|
+
|
|
592
|
+
let exports = [];
|
|
593
|
+
let imports = [];
|
|
594
|
+
let symbols = { functions: [], classes: [] };
|
|
595
|
+
let tags = [];
|
|
596
|
+
|
|
597
|
+
if (['javascript', 'typescript', 'javascript-react', 'typescript-react'].includes(type)) {
|
|
598
|
+
try {
|
|
599
|
+
const content = fs.readFileSync(fullPath, 'utf8');
|
|
600
|
+
exports = extractExports(content);
|
|
601
|
+
imports = extractImports(content);
|
|
602
|
+
symbols = extractSymbols(content);
|
|
603
|
+
tags = detectTags(relativePath, content);
|
|
604
|
+
} catch (err) {
|
|
605
|
+
if (DEBUG) debugLog(`Error reading ${relativePath}: ${err.message}`);
|
|
606
|
+
}
|
|
607
|
+
} else {
|
|
608
|
+
tags = detectTags(relativePath, '');
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
existingIndex.files[relativePath] = {
|
|
612
|
+
type,
|
|
613
|
+
size,
|
|
614
|
+
mtime,
|
|
615
|
+
exports,
|
|
616
|
+
imports,
|
|
617
|
+
tags,
|
|
618
|
+
};
|
|
619
|
+
|
|
620
|
+
if (existing) {
|
|
621
|
+
changedCount++;
|
|
622
|
+
} else {
|
|
623
|
+
addedCount++;
|
|
624
|
+
}
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
|
|
628
|
+
// Rebuild tag and symbol indices
|
|
629
|
+
existingIndex.tags = {};
|
|
630
|
+
existingIndex.symbols = { functions: {}, classes: {}, exports: {} };
|
|
631
|
+
|
|
632
|
+
for (const [filePath, fileData] of Object.entries(existingIndex.files)) {
|
|
633
|
+
for (const tag of fileData.tags || []) {
|
|
634
|
+
if (!existingIndex.tags[tag]) existingIndex.tags[tag] = [];
|
|
635
|
+
existingIndex.tags[tag].push(filePath);
|
|
636
|
+
}
|
|
637
|
+
for (const exp of fileData.exports || []) {
|
|
638
|
+
if (!existingIndex.symbols.exports[exp]) existingIndex.symbols.exports[exp] = [];
|
|
639
|
+
existingIndex.symbols.exports[exp].push(filePath);
|
|
640
|
+
}
|
|
641
|
+
}
|
|
642
|
+
|
|
643
|
+
// Update stats
|
|
644
|
+
existingIndex.stats.total_files = currentFiles.length;
|
|
645
|
+
existingIndex.stats.indexed_files = Object.keys(existingIndex.files).length;
|
|
646
|
+
existingIndex.stats.build_time_ms = Date.now() - startTime;
|
|
647
|
+
existingIndex.updated_at = new Date().toISOString();
|
|
648
|
+
|
|
649
|
+
// Store in cache
|
|
650
|
+
const cacheKey = `index:${projectRoot}`;
|
|
651
|
+
indexCache.set(cacheKey, existingIndex);
|
|
652
|
+
|
|
653
|
+
// Persist to disk
|
|
654
|
+
saveIndexToDisk(cachePath, existingIndex);
|
|
655
|
+
|
|
656
|
+
if (DEBUG) {
|
|
657
|
+
debugLog(`Index updated: +${addedCount} -${removedCount} ~${changedCount}`);
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
return { ok: true, data: existingIndex };
|
|
661
|
+
} catch (err) {
|
|
662
|
+
return { ok: false, error: err.message };
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
|
|
666
|
+
/**
|
|
667
|
+
* Get index (from cache or disk, or build if needed)
|
|
668
|
+
* @param {string} projectRoot - Project root
|
|
669
|
+
* @param {Object} options - Configuration options
|
|
670
|
+
* @returns {Object} { ok: boolean, data?: Object, error?: string }
|
|
671
|
+
*/
|
|
672
|
+
function getIndex(projectRoot, options = {}) {
|
|
673
|
+
// Check memory cache first
|
|
674
|
+
const cacheKey = `index:${projectRoot}`;
|
|
675
|
+
const cached = indexCache.get(cacheKey);
|
|
676
|
+
if (cached) {
|
|
677
|
+
return { ok: true, data: cached };
|
|
678
|
+
}
|
|
679
|
+
|
|
680
|
+
// Try disk cache
|
|
681
|
+
const cachePath = getCachePath(projectRoot);
|
|
682
|
+
const diskResult = loadIndexFromDisk(cachePath);
|
|
683
|
+
if (diskResult.ok) {
|
|
684
|
+
// Validate version
|
|
685
|
+
if (diskResult.data.version === INDEX_VERSION) {
|
|
686
|
+
// Store in memory cache
|
|
687
|
+
indexCache.set(cacheKey, diskResult.data);
|
|
688
|
+
return { ok: true, data: diskResult.data };
|
|
689
|
+
}
|
|
690
|
+
}
|
|
691
|
+
|
|
692
|
+
// Build fresh index
|
|
693
|
+
return buildIndex(projectRoot, options);
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
/**
|
|
697
|
+
* Invalidate cached index
|
|
698
|
+
* @param {string} projectRoot - Project root
|
|
699
|
+
*/
|
|
700
|
+
function invalidateIndex(projectRoot) {
|
|
701
|
+
const cacheKey = `index:${projectRoot}`;
|
|
702
|
+
indexCache.delete(cacheKey);
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
/**
|
|
706
|
+
* Query files by glob pattern
|
|
707
|
+
* @param {Object} index - Codebase index
|
|
708
|
+
* @param {string} pattern - Glob pattern (e.g., "*.auth*", "src/api/**")
|
|
709
|
+
* @returns {string[]} Matching file paths
|
|
710
|
+
*/
|
|
711
|
+
function queryFiles(index, pattern) {
|
|
712
|
+
// Convert glob to regex
|
|
713
|
+
// Order matters: handle ** before * to avoid double processing
|
|
714
|
+
let regexPattern = pattern
|
|
715
|
+
// First, use placeholders to protect multi-char patterns
|
|
716
|
+
.replace(/\*\*\//g, '<<<GLOBSLASH>>>') // **/ placeholder
|
|
717
|
+
.replace(/\*\*/g, '<<<GLOB>>>') // ** placeholder
|
|
718
|
+
.replace(/\./g, '\\.') // Escape dots
|
|
719
|
+
.replace(/\?/g, '.') // ? = any single char
|
|
720
|
+
.replace(/\*/g, '[^/]*') // Single * = any chars except /
|
|
721
|
+
// Now restore placeholders with actual patterns
|
|
722
|
+
.replace(/<<<GLOBSLASH>>>/g, '(?:.+/)?') // **/ = optionally any path + /
|
|
723
|
+
.replace(/<<<GLOB>>>/g, '.*'); // ** alone = any chars including /
|
|
724
|
+
|
|
725
|
+
const regex = new RegExp(`^${regexPattern}$`, 'i');
|
|
726
|
+
|
|
727
|
+
return Object.keys(index.files).filter(filePath => regex.test(filePath));
|
|
728
|
+
}
|
|
729
|
+
|
|
730
|
+
/**
|
|
731
|
+
* Query files by tag
|
|
732
|
+
* @param {Object} index - Codebase index
|
|
733
|
+
* @param {string} tag - Tag to search for
|
|
734
|
+
* @returns {string[]} Files with this tag
|
|
735
|
+
*/
|
|
736
|
+
function queryByTag(index, tag) {
|
|
737
|
+
return index.tags[tag.toLowerCase()] || [];
|
|
738
|
+
}
|
|
739
|
+
|
|
740
|
+
/**
|
|
741
|
+
* Query files by exported symbol
|
|
742
|
+
* @param {Object} index - Codebase index
|
|
743
|
+
* @param {string} symbolName - Symbol name to find
|
|
744
|
+
* @returns {string[]} Files exporting this symbol
|
|
745
|
+
*/
|
|
746
|
+
function queryByExport(index, symbolName) {
|
|
747
|
+
return index.symbols.exports[symbolName] || [];
|
|
748
|
+
}
|
|
749
|
+
|
|
750
|
+
/**
|
|
751
|
+
* Get dependencies of a file
|
|
752
|
+
* @param {Object} index - Codebase index
|
|
753
|
+
* @param {string} filePath - File path
|
|
754
|
+
* @returns {Object} { imports: string[], importedBy: string[] }
|
|
755
|
+
*/
|
|
756
|
+
function getDependencies(index, filePath) {
|
|
757
|
+
const fileData = index.files[filePath];
|
|
758
|
+
if (!fileData) {
|
|
759
|
+
return { imports: [], importedBy: [] };
|
|
760
|
+
}
|
|
761
|
+
|
|
762
|
+
const imports = fileData.imports || [];
|
|
763
|
+
|
|
764
|
+
// Find files that import this file
|
|
765
|
+
const importedBy = [];
|
|
766
|
+
const baseName = path.basename(filePath).replace(/\.\w+$/, '');
|
|
767
|
+
|
|
768
|
+
for (const [otherPath, otherData] of Object.entries(index.files)) {
|
|
769
|
+
if (otherPath === filePath) continue;
|
|
770
|
+
const otherImports = otherData.imports || [];
|
|
771
|
+
for (const imp of otherImports) {
|
|
772
|
+
if (imp.includes(baseName) || imp.includes(filePath)) {
|
|
773
|
+
importedBy.push(otherPath);
|
|
774
|
+
break;
|
|
775
|
+
}
|
|
776
|
+
}
|
|
777
|
+
}
|
|
778
|
+
|
|
779
|
+
return { imports, importedBy };
|
|
780
|
+
}
|
|
781
|
+
|
|
782
|
+
module.exports = {
|
|
783
|
+
// Core functions
|
|
784
|
+
buildIndex,
|
|
785
|
+
updateIndex,
|
|
786
|
+
getIndex,
|
|
787
|
+
invalidateIndex,
|
|
788
|
+
|
|
789
|
+
// Query functions
|
|
790
|
+
queryFiles,
|
|
791
|
+
queryByTag,
|
|
792
|
+
queryByExport,
|
|
793
|
+
getDependencies,
|
|
794
|
+
|
|
795
|
+
// Configuration
|
|
796
|
+
loadConfig,
|
|
797
|
+
|
|
798
|
+
// Utilities (exposed for testing)
|
|
799
|
+
extractExports,
|
|
800
|
+
extractImports,
|
|
801
|
+
extractSymbols,
|
|
802
|
+
detectTags,
|
|
803
|
+
shouldIncludeFile,
|
|
804
|
+
getFileType,
|
|
805
|
+
|
|
806
|
+
// Constants
|
|
807
|
+
INDEX_VERSION,
|
|
808
|
+
DEFAULT_CONFIG,
|
|
809
|
+
TAG_PATTERNS,
|
|
810
|
+
};
|