minovative-mind-cli 2.1.1 → 2.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,613 +0,0 @@
1
- import { promises as fs } from 'node:fs';
2
- import * as crypto from 'node:crypto';
3
- import path from 'node:path';
4
- import { debugLog } from '../utils/logger.js';
5
- import { getAuthorizedIdToken } from './auth.js';
6
- import { ProxyClient } from './proxyClient.js';
7
- import { EXCLUDED_EXTENSIONS } from '../utils/excludedExtensions.js';
8
- import { GEMINI_MODELS } from '../utils/config.js';
9
- import { readCache, writeCache } from '../utils/projectStorage.js';
10
- // ─── Constants ───────────────────────────────────────────────────────
11
- /** Maximum characters per chunk before splitting */
12
- const MAX_CHUNK_CHARS = 3000;
13
- /** Minimum characters for a chunk to be worth embedding */
14
- const MIN_CHUNK_CHARS = 50;
15
- /** Maximum file size (chars) to index — skip likely generated/vendored files */
16
- const MAX_FILE_CHARS = 50_000;
17
- /** Number of texts to embed per API call */
18
- const EMBED_BATCH_SIZE = 25;
19
- /** Sliding window size for non-AST files */
20
- const SLIDING_WINDOW_CHARS = 800;
21
- /** Sliding window overlap for context continuity */
22
- const SLIDING_WINDOW_OVERLAP = 200;
23
- /** Index file path within .minovativemind */
24
- const INDEX_FILENAME = 'embeddings/index.json';
25
- /** Directories to always skip during indexing */
26
- const IGNORED_DIRS = new Set([
27
- 'node_modules', '.git', 'dist', '.next', '.nuxt', '__pycache__',
28
- '.venv', 'venv', '.cache', 'coverage', '.turbo', '.minovativemind',
29
- '.tmp', 'build', 'out', '.output', '.svelte-kit',
30
- ]);
31
- /** File extensions that support AST-aware chunking via declaration regex */
32
- const AST_EXTENSIONS = new Set([
33
- '.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs',
34
- '.py', '.go', '.rs', '.java', '.kt', '.swift',
35
- '.dart', '.rb', '.php', '.c', '.cpp', '.h', '.hpp',
36
- ]);
37
- // ─── Proxy Client (shared instance) ─────────────────────────────────
38
- const proxyClient = new ProxyClient();
39
- // ─── AST-Aware Chunking (reuses symbolExtractor logic) ──────────────
40
- /**
41
- * Builds a regex that matches declaration-level constructs for a given file extension.
42
- * Adapted from symbolExtractor.ts but generalized to match ANY declaration,
43
- * not a specific named symbol.
44
- */
45
- function buildGenericDeclarationRegex(ext) {
46
- if (ext === '.py') {
47
- return /^\s*(?:async\s+)?(?:def|class)\s+\w+/;
48
- }
49
- if (ext === '.go') {
50
- return /^\s*(?:func(?:\s+\([^)]+\))?\s+\w+|type\s+\w+|var\s+\w+|const\s+\w+)/;
51
- }
52
- if (ext === '.rs') {
53
- return /^\s*(?:(?:pub\s+)?(?:async\s+)?fn\s+\w+|(?:pub\s+)?(?:struct|enum|trait|impl)\s+\w+)/;
54
- }
55
- if (ext === '.rb') {
56
- return /^\s*(?:def\s+\w+|class\s+\w+|module\s+\w+)/;
57
- }
58
- if (ext === '.php') {
59
- return /^\s*(?:(?:public|private|protected|static)\s+)?function\s+\w+|^\s*class\s+\w+/;
60
- }
61
- if (ext === '.java' || ext === '.kt') {
62
- return /^\s*(?:(?:public|private|protected|static|abstract|final|override|suspend)\s+)*(?:class|interface|enum|fun|void|int|String|boolean|object)\s+\w+/;
63
- }
64
- if (ext === '.swift') {
65
- return /^\s*(?:(?:public|private|internal|open|fileprivate|static|class|override)\s+)*(?:func|class|struct|enum|protocol|extension)\s+\w+/;
66
- }
67
- if (ext === '.dart') {
68
- return /^\s*(?:(?:abstract|static)\s+)?(?:class|mixin|extension|void|Future|Stream|int|String|bool|double)\s+\w+/;
69
- }
70
- if (ext === '.c' || ext === '.cpp' || ext === '.h' || ext === '.hpp') {
71
- return /^\s*(?:(?:static|inline|extern|virtual|const)\s+)*(?:void|int|char|float|double|bool|auto|class|struct|enum|namespace|template)\s+\w+/;
72
- }
73
- // Default: JS/TS family
74
- return /^\s*(?:(?:export\s+|default\s+|async\s+|abstract\s+|static\s+|public\s+|private\s+|protected\s+|readonly\s+)*(?:class|function|const|let|var|type|interface|enum)\s+\w+|^\s*(?:async\s+)?\w+\s*[<]?\w*[>]?\s*\()/;
75
- }
76
- /**
77
- * Extracts a human-readable label from a declaration line.
78
- * e.g., "export async function fetchUser(" → "function fetchUser"
79
- */
80
- function extractLabel(line, ext) {
81
- const trimmed = line.trim();
82
- // Try to extract the key parts: keyword + name
83
- const patterns = ext === '.py'
84
- ? [/(?:async\s+)?(def|class)\s+(\w+)/, /(\w+)\s*[:=]/]
85
- : [
86
- /(?:export\s+)?(?:default\s+)?(?:async\s+)?(?:abstract\s+)?(function|class|interface|type|enum|const|let|var)\s+(\w+)/,
87
- /(?:pub\s+)?(?:async\s+)?(fn|struct|enum|trait|impl)\s+(\w+)/,
88
- /(def|class|module)\s+(\w+)/,
89
- /(func|type|var|const)\s+(\w+)/,
90
- ];
91
- for (const pat of patterns) {
92
- const match = trimmed.match(pat);
93
- if (match)
94
- return `${match[1]} ${match[2]}`;
95
- }
96
- // Fallback: first 60 chars of the trimmed line
97
- return trimmed.substring(0, 60);
98
- }
99
- /**
100
- * Finds the end of a brace/indent-delimited block starting at a given line.
101
- * Reuses the same balanced-brace tracking logic from symbolExtractor.ts.
102
- */
103
- function findBlockEndLine(lines, startIdx, ext) {
104
- if (ext === '.py') {
105
- // Python: indentation-based
106
- const baseIndent = lines[startIdx].match(/^(\s*)/)?.[1].length ?? 0;
107
- let endIdx = startIdx;
108
- for (let j = startIdx + 1; j < lines.length; j++) {
109
- const line = lines[j];
110
- if (!line.trim()) {
111
- endIdx = j;
112
- continue;
113
- }
114
- if ((line.match(/^(\s*)/)?.[1].length ?? 0) <= baseIndent)
115
- break;
116
- endIdx = j;
117
- }
118
- return endIdx;
119
- }
120
- // Brace-based languages
121
- let braces = 0;
122
- let foundOpen = false;
123
- let inStr = null;
124
- let inComment = false;
125
- let endIdx = startIdx;
126
- for (let j = startIdx; j < lines.length; j++) {
127
- endIdx = j;
128
- const line = lines[j];
129
- for (let c = 0; c < line.length; c++) {
130
- const ch = line[c];
131
- if (inComment) {
132
- if (ch === '*' && line[c + 1] === '/') {
133
- inComment = false;
134
- c++;
135
- }
136
- continue;
137
- }
138
- if (inStr) {
139
- if (ch === '\\') {
140
- c++;
141
- continue;
142
- }
143
- if (ch === inStr)
144
- inStr = null;
145
- continue;
146
- }
147
- if (ch === '/' && line[c + 1] === '/')
148
- break;
149
- if (ch === '/' && line[c + 1] === '*') {
150
- inComment = true;
151
- c++;
152
- continue;
153
- }
154
- if (ch === '"' || ch === "'" || ch === '`') {
155
- inStr = ch;
156
- continue;
157
- }
158
- if (ch === '{') {
159
- braces++;
160
- foundOpen = true;
161
- }
162
- else if (ch === '}')
163
- braces--;
164
- }
165
- if (foundOpen && braces <= 0)
166
- break;
167
- // One-liners without braces (e.g., `const x = 5;`)
168
- if (!foundOpen && j > startIdx) {
169
- const prevTrimmed = lines[j - 1].trim();
170
- if (!prevTrimmed.endsWith(',') && !prevTrimmed.endsWith('.')) {
171
- endIdx = j - 1;
172
- break;
173
- }
174
- }
175
- }
176
- return endIdx;
177
- }
178
- /**
179
- * Chunks a single source file into semantic units.
180
- * Uses AST-aware declaration detection for supported languages,
181
- * falls back to sliding-window chunking for everything else.
182
- */
183
- function chunkFile(filePath, content) {
184
- const ext = path.extname(filePath).toLowerCase();
185
- const lines = content.split('\n');
186
- const chunks = [];
187
- if (AST_EXTENSIONS.has(ext)) {
188
- // AST-aware chunking: split at declaration boundaries
189
- const declRegex = buildGenericDeclarationRegex(ext);
190
- let i = 0;
191
- while (i < lines.length) {
192
- if (declRegex.test(lines[i])) {
193
- const startIdx = i;
194
- const endIdx = findBlockEndLine(lines, i, ext);
195
- const chunkContent = lines.slice(startIdx, endIdx + 1).join('\n');
196
- if (chunkContent.length >= MIN_CHUNK_CHARS) {
197
- // Split oversized chunks at the MAX_CHUNK_CHARS boundary
198
- if (chunkContent.length > MAX_CHUNK_CHARS) {
199
- const subChunks = splitOversizedChunk(chunkContent, filePath, startIdx);
200
- chunks.push(...subChunks);
201
- }
202
- else {
203
- chunks.push({
204
- filePath,
205
- startLine: startIdx + 1,
206
- endLine: endIdx + 1,
207
- label: extractLabel(lines[startIdx], ext),
208
- content: chunkContent,
209
- });
210
- }
211
- }
212
- i = endIdx + 1;
213
- }
214
- else {
215
- i++;
216
- }
217
- }
218
- // If AST parsing found nothing (e.g., a config file with no declarations),
219
- // fall back to treating the whole file as one chunk
220
- if (chunks.length === 0 && content.length >= MIN_CHUNK_CHARS) {
221
- if (content.length > MAX_CHUNK_CHARS) {
222
- chunks.push(...slidingWindowChunk(filePath, content));
223
- }
224
- else {
225
- chunks.push({
226
- filePath,
227
- startLine: 1,
228
- endLine: lines.length,
229
- label: path.basename(filePath),
230
- content,
231
- });
232
- }
233
- }
234
- }
235
- else {
236
- // Non-AST files: sliding window chunking
237
- if (content.length >= MIN_CHUNK_CHARS) {
238
- if (content.length <= MAX_CHUNK_CHARS) {
239
- chunks.push({
240
- filePath,
241
- startLine: 1,
242
- endLine: lines.length,
243
- label: path.basename(filePath),
244
- content,
245
- });
246
- }
247
- else {
248
- chunks.push(...slidingWindowChunk(filePath, content));
249
- }
250
- }
251
- }
252
- return chunks;
253
- }
254
- /**
255
- * Splits an oversized function/class chunk into smaller pieces.
256
- */
257
- function splitOversizedChunk(content, filePath, globalStartIdx) {
258
- const lines = content.split('\n');
259
- const results = [];
260
- let chunkStart = 0;
261
- while (chunkStart < lines.length) {
262
- let chunkEnd = chunkStart;
263
- let charCount = 0;
264
- while (chunkEnd < lines.length && charCount + lines[chunkEnd].length < MAX_CHUNK_CHARS) {
265
- charCount += lines[chunkEnd].length + 1; // +1 for newline
266
- chunkEnd++;
267
- }
268
- if (chunkEnd === chunkStart)
269
- chunkEnd = chunkStart + 1; // Prevent infinite loop on very long lines
270
- const chunkLines = lines.slice(chunkStart, chunkEnd);
271
- const chunkContent = chunkLines.join('\n');
272
- if (chunkContent.length >= MIN_CHUNK_CHARS) {
273
- results.push({
274
- filePath,
275
- startLine: globalStartIdx + chunkStart + 1,
276
- endLine: globalStartIdx + chunkEnd,
277
- label: `${path.basename(filePath)}:${globalStartIdx + chunkStart + 1}`,
278
- content: chunkContent,
279
- });
280
- }
281
- chunkStart = chunkEnd;
282
- }
283
- return results;
284
- }
285
- /**
286
- * Sliding window chunker for non-AST files (markdown, JSON, HTML, CSS, etc).
287
- * Uses character-level windowing with overlap for context continuity.
288
- */
289
- function slidingWindowChunk(filePath, content) {
290
- const lines = content.split('\n');
291
- const results = [];
292
- let charOffset = 0;
293
- let lineOffset = 0;
294
- while (charOffset < content.length) {
295
- const windowEnd = Math.min(charOffset + SLIDING_WINDOW_CHARS, content.length);
296
- const windowContent = content.substring(charOffset, windowEnd);
297
- // Calculate line numbers for this window
298
- const windowLines = windowContent.split('\n');
299
- const startLine = lineOffset + 1;
300
- const endLine = lineOffset + windowLines.length;
301
- if (windowContent.length >= MIN_CHUNK_CHARS) {
302
- results.push({
303
- filePath,
304
- startLine,
305
- endLine,
306
- label: `${path.basename(filePath)}:${startLine}`,
307
- content: windowContent,
308
- });
309
- }
310
- // Advance by (window - overlap) characters
311
- const advance = SLIDING_WINDOW_CHARS - SLIDING_WINDOW_OVERLAP;
312
- // Count how many lines we advanced through
313
- const advancedContent = content.substring(charOffset, Math.min(charOffset + advance, content.length));
314
- const advancedLines = advancedContent.split('\n').length - 1;
315
- lineOffset += advancedLines;
316
- charOffset += advance;
317
- }
318
- return results;
319
- }
320
- // ─── Math Utilities ──────────────────────────────────────────────────
321
- /**
322
- * Computes cosine similarity between two vectors.
323
- * Pure arithmetic — no dependencies, ~1ms for 10K comparisons.
324
- */
325
- function cosineSimilarity(a, b) {
326
- let dot = 0;
327
- let magA = 0;
328
- let magB = 0;
329
- for (let i = 0; i < a.length; i++) {
330
- dot += a[i] * b[i];
331
- magA += a[i] * a[i];
332
- magB += b[i] * b[i];
333
- }
334
- const denom = Math.sqrt(magA) * Math.sqrt(magB);
335
- return denom === 0 ? 0 : dot / denom;
336
- }
337
- // ─── Embedding Index Class ───────────────────────────────────────────
338
- /**
339
- * Manages a persistent local vector index for semantic code search.
340
- *
341
- * Lifecycle:
342
- * 1. On first `gatherContext` call, if no index exists on disk, `buildIndex()` is called.
343
- * 2. After the Execution Agent modifies files, `updateIndex()` delta-reindexes only changed files.
344
- * 3. `search()` embeds the query (1 API call) and scans the local index via cosine similarity.
345
- * 4. Index is persisted to `.minovativemind/embeddings/index.json` via atomic writes.
346
- */
347
- export class EmbeddingIndex {
348
- chunks = [];
349
- modelVersion = GEMINI_MODELS.EMBEDDING;
350
- /**
351
- * Builds the full index for a workspace by scanning all eligible source files,
352
- * chunking them at function/class boundaries, and batch-embedding them.
353
- *
354
- * @param workspaceRoot - Absolute path to the workspace root directory.
355
- * @param onProgress - Optional callback for user-facing progress messages.
356
- */
357
- async buildIndex(workspaceRoot, onProgress) {
358
- this.chunks = [];
359
- this.modelVersion = GEMINI_MODELS.EMBEDDING;
360
- // Phase 1: Discover and chunk all eligible files
361
- const allChunks = [];
362
- await this.walkAndChunk(workspaceRoot, workspaceRoot, allChunks);
363
- if (allChunks.length === 0) {
364
- debugLog('Embedding index: No eligible files found to index.');
365
- return;
366
- }
367
- if (onProgress)
368
- onProgress(`Indexing ${allChunks.length} code chunks...`);
369
- debugLog(`Embedding index: Found ${allChunks.length} chunks to embed.`);
370
- // Phase 2: Batch-embed all chunks
371
- const idToken = await getAuthorizedIdToken();
372
- if (!idToken) {
373
- debugLog('Embedding index: No auth token available, skipping index build.');
374
- return;
375
- }
376
- const batchCount = Math.ceil(allChunks.length / EMBED_BATCH_SIZE);
377
- for (let b = 0; b < batchCount; b++) {
378
- const batchStart = b * EMBED_BATCH_SIZE;
379
- const batchEnd = Math.min(batchStart + EMBED_BATCH_SIZE, allChunks.length);
380
- const batch = allChunks.slice(batchStart, batchEnd);
381
- if (onProgress) {
382
- onProgress(`Embedding batch ${b + 1}/${batchCount} (${batch.length} chunks)...`);
383
- }
384
- try {
385
- const result = await proxyClient.embedTextsViaProxy(idToken, batch.map((c) => c.content), 'RETRIEVAL_DOCUMENT');
386
- for (let i = 0; i < batch.length; i++) {
387
- if (result.embeddings[i]) {
388
- this.chunks.push({
389
- filePath: batch[i].filePath,
390
- startLine: batch[i].startLine,
391
- endLine: batch[i].endLine,
392
- contentHash: this.hashChunk(batch[i].filePath, batch[i].content),
393
- label: batch[i].label,
394
- preview: batch[i].content.substring(0, 200).replace(/\n/g, ' '),
395
- embedding: result.embeddings[i],
396
- });
397
- }
398
- }
399
- }
400
- catch (e) {
401
- debugLog(`Embedding index: Batch ${b + 1} failed: ${e.message}`);
402
- // Continue with remaining batches — partial index is better than none
403
- }
404
- }
405
- debugLog(`Embedding index: Built index with ${this.chunks.length} chunks.`);
406
- }
407
- /**
408
- * Delta-updates the index for files that have changed.
409
- * Removes stale chunks for modified/deleted files, then re-chunks and
410
- * re-embeds only the affected files.
411
- *
412
- * @param workspaceRoot - Absolute path to the workspace root.
413
- * @param changedFiles - Array of relative file paths that were modified.
414
- */
415
- async updateIndex(workspaceRoot, changedFiles) {
416
- if (changedFiles.length === 0)
417
- return;
418
- // Remove all chunks belonging to changed files
419
- const changedSet = new Set(changedFiles);
420
- this.chunks = this.chunks.filter((c) => !changedSet.has(c.filePath));
421
- // Re-chunk and re-embed the changed files
422
- const newChunks = [];
423
- for (const relPath of changedFiles) {
424
- const absPath = path.join(workspaceRoot, relPath);
425
- if (absPath.toLowerCase().endsWith('.pdf'))
426
- continue;
427
- try {
428
- const content = await fs.readFile(absPath, 'utf-8');
429
- if (content.length > MAX_FILE_CHARS || content.length < MIN_CHUNK_CHARS)
430
- continue;
431
- const fileChunks = chunkFile(relPath, content);
432
- newChunks.push(...fileChunks);
433
- }
434
- catch {
435
- // File was deleted or unreadable — chunks already removed above
436
- }
437
- }
438
- if (newChunks.length === 0)
439
- return;
440
- const idToken = await getAuthorizedIdToken();
441
- if (!idToken)
442
- return;
443
- const batchCount = Math.ceil(newChunks.length / EMBED_BATCH_SIZE);
444
- for (let b = 0; b < batchCount; b++) {
445
- const batchStart = b * EMBED_BATCH_SIZE;
446
- const batchEnd = Math.min(batchStart + EMBED_BATCH_SIZE, newChunks.length);
447
- const batch = newChunks.slice(batchStart, batchEnd);
448
- try {
449
- const result = await proxyClient.embedTextsViaProxy(idToken, batch.map((c) => c.content), 'RETRIEVAL_DOCUMENT');
450
- for (let i = 0; i < batch.length; i++) {
451
- if (result.embeddings[i]) {
452
- this.chunks.push({
453
- filePath: batch[i].filePath,
454
- startLine: batch[i].startLine,
455
- endLine: batch[i].endLine,
456
- contentHash: this.hashChunk(batch[i].filePath, batch[i].content),
457
- label: batch[i].label,
458
- preview: batch[i].content.substring(0, 200).replace(/\n/g, ' '),
459
- embedding: result.embeddings[i],
460
- });
461
- }
462
- }
463
- }
464
- catch (e) {
465
- debugLog(`Embedding index: Delta batch failed: ${e.message}`);
466
- }
467
- }
468
- debugLog(`Embedding index: Delta-updated ${changedFiles.length} file(s). Total chunks: ${this.chunks.length}`);
469
- }
470
- /**
471
- * Executes a semantic search against the local index.
472
- *
473
- * 1. Embeds the query string (single API call).
474
- * 2. Computes cosine similarity against all indexed chunks.
475
- * 3. Returns the top-K results sorted by descending similarity score.
476
- *
477
- * @param query - Natural language description of what to search for.
478
- * @param topK - Number of results to return (default 5, max 15).
479
- * @returns Array of search results with file paths, line ranges, and scores.
480
- */
481
- async search(query, topK = 5) {
482
- if (this.chunks.length === 0)
483
- return [];
484
- const clampedK = Math.min(Math.max(topK, 1), 15);
485
- const idToken = await getAuthorizedIdToken();
486
- if (!idToken)
487
- return [];
488
- // Embed the query (single API call)
489
- let queryEmbedding;
490
- try {
491
- const result = await proxyClient.embedTextsViaProxy(idToken, [query], 'RETRIEVAL_QUERY');
492
- if (!result.embeddings[0])
493
- return [];
494
- queryEmbedding = result.embeddings[0];
495
- }
496
- catch (e) {
497
- debugLog(`Embedding index: Failed to embed query: ${e.message}`);
498
- return [];
499
- }
500
- // Cosine similarity scan (pure math, ~1ms for 10K chunks)
501
- const scored = this.chunks.map((chunk) => ({
502
- filePath: chunk.filePath,
503
- startLine: chunk.startLine,
504
- endLine: chunk.endLine,
505
- label: chunk.label,
506
- preview: chunk.preview,
507
- score: cosineSimilarity(queryEmbedding, chunk.embedding),
508
- }));
509
- // Sort by descending similarity and take top-K
510
- scored.sort((a, b) => b.score - a.score);
511
- return scored.slice(0, clampedK);
512
- }
513
- /**
514
- * Persists the index to disk using the atomic write pattern from projectStorage.
515
- */
516
- async save(workspaceRoot) {
517
- const data = {
518
- modelVersion: this.modelVersion,
519
- chunks: this.chunks,
520
- };
521
- await writeCache(workspaceRoot, INDEX_FILENAME, data);
522
- debugLog(`Embedding index: Saved ${this.chunks.length} chunks to disk.`);
523
- }
524
- /**
525
- * Loads the index from disk. Returns false if no index exists or the model
526
- * version has changed (requiring a full rebuild).
527
- */
528
- async load(workspaceRoot) {
529
- const data = readCache(workspaceRoot, INDEX_FILENAME);
530
- if (!data || !data.chunks)
531
- return false;
532
- // Auto-invalidate if the embedding model has been upgraded
533
- if (data.modelVersion !== GEMINI_MODELS.EMBEDDING) {
534
- debugLog(`Embedding index: Model version mismatch (${data.modelVersion} vs ${GEMINI_MODELS.EMBEDDING}). Rebuilding.`);
535
- return false;
536
- }
537
- this.chunks = data.chunks;
538
- this.modelVersion = data.modelVersion;
539
- debugLog(`Embedding index: Loaded ${this.chunks.length} chunks from disk.`);
540
- return true;
541
- }
542
- /** Check if the index is loaded and contains chunks */
543
- isReady() {
544
- return this.chunks.length > 0;
545
- }
546
- /** Get the number of chunks in the index */
547
- get size() {
548
- return this.chunks.length;
549
- }
550
- // ─── Private Helpers ─────────────────────────────────────────────
551
- /**
552
- * Recursively walks the workspace, reading and chunking eligible files.
553
- * Respects .gitignore patterns and EXCLUDED_EXTENSIONS.
554
- */
555
- async walkAndChunk(rootDir, currentDir, results) {
556
- let entries;
557
- try {
558
- entries = await fs.readdir(currentDir, { withFileTypes: true });
559
- }
560
- catch {
561
- return;
562
- }
563
- for (const entry of entries) {
564
- if (entry.name.startsWith('.') && entry.name !== '.env')
565
- continue;
566
- if (entry.isDirectory()) {
567
- if (IGNORED_DIRS.has(entry.name))
568
- continue;
569
- await this.walkAndChunk(rootDir, path.join(currentDir, entry.name), results);
570
- }
571
- else if (entry.isFile()) {
572
- if (EXCLUDED_EXTENSIONS.some((ext) => entry.name.endsWith(ext.replace('*', ''))))
573
- continue;
574
- if (entry.name.toLowerCase().endsWith('.pdf'))
575
- continue;
576
- const absPath = path.join(currentDir, entry.name);
577
- const relPath = path.relative(rootDir, absPath).replace(/\\/g, '/');
578
- try {
579
- const content = await fs.readFile(absPath, 'utf-8');
580
- if (content.length > MAX_FILE_CHARS || content.length < MIN_CHUNK_CHARS)
581
- continue;
582
- const fileChunks = chunkFile(relPath, content);
583
- results.push(...fileChunks);
584
- }
585
- catch {
586
- // Skip unreadable files (binary detection, permission errors)
587
- }
588
- }
589
- }
590
- }
591
- /**
592
- * Generates a content hash for delta-detection.
593
- * Uses SHA-256 of (filePath + content) to detect changes.
594
- */
595
- hashChunk(filePath, content) {
596
- return crypto
597
- .createHash('sha256')
598
- .update(filePath + content)
599
- .digest('hex');
600
- }
601
- }
602
- // ─── Singleton Instance ──────────────────────────────────────────────
603
- let embeddingIndexInstance = null;
604
- /**
605
- * Returns the global singleton EmbeddingIndex instance.
606
- * Lazily created on first access.
607
- */
608
- export function getEmbeddingIndex() {
609
- if (!embeddingIndexInstance) {
610
- embeddingIndexInstance = new EmbeddingIndex();
611
- }
612
- return embeddingIndexInstance;
613
- }