bluera-knowledge 0.12.10 → 0.13.0

package/src/services/index.service.test.ts

@@ -1952,3 +1952,350 @@ describe('IndexService - Symlink Handling', () => {
     // (on most systems, readdir with withFileTypes shows symlinks as isFile() if target is file)
   });
 });
+
+describe('IndexService - Batch Embedding', () => {
+  let indexService: IndexService;
+  let lanceStore: LanceStore;
+  let embeddingEngine: EmbeddingEngine;
+  let tempDir: string;
+  let testFilesDir: string;
+  const storeId = createStoreId('batch-embed-test');
+
+  beforeAll(async () => {
+    tempDir = await mkdtemp(join(tmpdir(), 'index-batch-embed-test-'));
+    testFilesDir = join(tempDir, 'files');
+    await mkdir(testFilesDir, { recursive: true });
+
+    lanceStore = new LanceStore(tempDir);
+    embeddingEngine = new EmbeddingEngine();
+
+    await embeddingEngine.initialize();
+    await lanceStore.initialize(storeId);
+
+    indexService = new IndexService(lanceStore, embeddingEngine);
+  }, 120000);
+
+  afterAll(async () => {
+    await rm(tempDir, { recursive: true, force: true });
+  });
+
+  beforeEach(async () => {
+    // Clear test directory for fresh state
+    await rm(testFilesDir, { recursive: true, force: true });
+    await mkdir(testFilesDir, { recursive: true });
+  });
+
+  it('calls embedBatch instead of sequential embed for multiple chunks', async () => {
+    // Create a file large enough to produce multiple chunks (>1500 chars)
+    const largeContent = Array(50)
+      .fill('This is a paragraph of text that will be chunked. ')
+      .join('\n\n');
+    await writeFile(join(testFilesDir, 'large.md'), largeContent);
+
+    const embedBatchSpy = vi.spyOn(embeddingEngine, 'embedBatch');
+
+    const store: FileStore = {
+      type: 'file',
+      id: storeId,
+      name: 'Batch Embed Test Store',
+      path: testFilesDir,
+      createdAt: new Date(),
+      updatedAt: new Date(),
+    };
+
+    const result = await indexService.indexStore(store);
+
+    expect(result.success).toBe(true);
+    if (result.success) {
+      // Should have created multiple chunks
+      expect(result.data.chunksCreated).toBeGreaterThan(1);
+    }
+
+    // embedBatch should be called (it internally uses embed via Promise.all)
+    expect(embedBatchSpy).toHaveBeenCalled();
+    // Verify batch was called with multiple items
+    const callArgs = embedBatchSpy.mock.calls[0];
+    expect(callArgs).toBeDefined();
+    expect(callArgs[0].length).toBeGreaterThan(1);
+
+    embedBatchSpy.mockRestore();
+  });
+
+  it('preserves chunk order when using batch embedding', async () => {
+    // Create file with distinct, ordered sections that will produce multiple chunks
+    const sections = Array(10)
+      .fill(null)
+      .map((_, i) => `# Section ${String(i + 1)}\n\n${'Content for section. '.repeat(50)}`)
+      .join('\n\n');
+
+    await writeFile(join(testFilesDir, 'ordered.md'), sections);
+
+    const embedBatchSpy = vi.spyOn(embeddingEngine, 'embedBatch');
+
+    const store: FileStore = {
+      type: 'file',
+      id: storeId,
+      name: 'Order Test Store',
+      path: testFilesDir,
+      createdAt: new Date(),
+      updatedAt: new Date(),
+    };
+
+    const result = await indexService.indexStore(store);
+
+    expect(result.success).toBe(true);
+    if (result.success) {
+      // Verify chunks are in correct order
+      expect(result.data.chunksCreated).toBeGreaterThan(1);
+    }
+
+    // embedBatch should be called with chunks in order
+    expect(embedBatchSpy).toHaveBeenCalled();
+    const callArgs = embedBatchSpy.mock.calls[0];
+    expect(callArgs).toBeDefined();
+
+    // Verify that if content has "Section 1", it comes before "Section 2" in the array
+    const batchedTexts = callArgs[0];
+    const section1Index = batchedTexts.findIndex((t: string) => t.includes('Section 1'));
+    const section2Index = batchedTexts.findIndex((t: string) => t.includes('Section 2'));
+
+    // Section 1 should appear before Section 2 in the batch (or they may be in different chunks)
+    if (section1Index !== -1 && section2Index !== -1) {
+      expect(section1Index).toBeLessThan(section2Index);
+    }
+
+    embedBatchSpy.mockRestore();
+  });
+
+  it('handles single-chunk files correctly', async () => {
+    // Create a small file that won't be chunked
+    await writeFile(join(testFilesDir, 'small.ts'), 'export const x = 42;');
+
+    const embedBatchSpy = vi.spyOn(embeddingEngine, 'embedBatch');
+
+    const store: FileStore = {
+      type: 'file',
+      id: storeId,
+      name: 'Single Chunk Test Store',
+      path: testFilesDir,
+      createdAt: new Date(),
+      updatedAt: new Date(),
+    };
+
+    const result = await indexService.indexStore(store);
+
+    expect(result.success).toBe(true);
+    if (result.success) {
+      // Should have exactly one chunk
+      expect(result.data.chunksCreated).toBe(1);
+    }
+
+    // embedBatch should still be called (with a single item)
+    expect(embedBatchSpy).toHaveBeenCalled();
+
+    embedBatchSpy.mockRestore();
+  });
+
+  it('handles multiple files with batch embedding', async () => {
+    // Create multiple files to verify batch embedding works across files
+    await writeFile(join(testFilesDir, 'file1.ts'), 'export const a = 1;');
+    await writeFile(join(testFilesDir, 'file2.ts'), 'export const b = 2;');
+    await writeFile(join(testFilesDir, 'file3.ts'), 'export const c = 3;');
+
+    const embedBatchSpy = vi.spyOn(embeddingEngine, 'embedBatch');
+
+    const multiStoreId = createStoreId('multi-file-test');
+    await lanceStore.initialize(multiStoreId);
+
+    const multiIndexService = new IndexService(lanceStore, embeddingEngine);
+
+    const store: FileStore = {
+      type: 'file',
+      id: multiStoreId,
+      name: 'Multi File Test Store',
+      path: testFilesDir,
+      createdAt: new Date(),
+      updatedAt: new Date(),
+    };
+
+    const result = await multiIndexService.indexStore(store);
+
+    expect(result.success).toBe(true);
+    if (result.success) {
+      // Should index all 3 files
+      expect(result.data.documentsIndexed).toBe(3);
+    }
+
+    // embedBatch should be called once per file (3 times)
+    expect(embedBatchSpy).toHaveBeenCalledTimes(3);
+
+    embedBatchSpy.mockRestore();
+  });
+});
+
+describe('IndexService - Parallel File Processing', () => {
+  let lanceStore: LanceStore;
+  let embeddingEngine: EmbeddingEngine;
+  let tempDir: string;
+  let testFilesDir: string;
+  const storeId = createStoreId('parallel-test');
+
+  beforeAll(async () => {
+    tempDir = await mkdtemp(join(tmpdir(), 'index-parallel-test-'));
+    testFilesDir = join(tempDir, 'files');
+    await mkdir(testFilesDir, { recursive: true });
+
+    lanceStore = new LanceStore(tempDir);
+    embeddingEngine = new EmbeddingEngine();
+
+    await embeddingEngine.initialize();
+    await lanceStore.initialize(storeId);
+  }, 120000);
+
+  afterAll(async () => {
+    await rm(tempDir, { recursive: true, force: true });
+  });
+
+  beforeEach(async () => {
+    // Clear test directory for fresh state
+    await rm(testFilesDir, { recursive: true, force: true });
+    await mkdir(testFilesDir, { recursive: true });
+  });
+
+  it('uses concurrency option from IndexService constructor', async () => {
+    // Create 10 test files
+    for (let i = 0; i < 10; i++) {
+      await writeFile(
+        join(testFilesDir, `file${String(i)}.ts`),
+        `export const x${String(i)} = ${String(i)};`
+      );
+    }
+
+    // Track when files start being processed
+    const processingTimestamps: number[] = [];
+    const originalEmbedBatch = embeddingEngine.embedBatch.bind(embeddingEngine);
+
+    vi.spyOn(embeddingEngine, 'embedBatch').mockImplementation(async (texts: string[]) => {
+      processingTimestamps.push(Date.now());
+      // Small delay to simulate processing time
+      await new Promise((resolve) => setTimeout(resolve, 50));
+      return originalEmbedBatch(texts);
+    });
+
+    const concurrency = 4;
+    const indexService = new IndexService(lanceStore, embeddingEngine, { concurrency });
+
+    const parallelStoreId = createStoreId('parallel-concurrency-test');
+    await lanceStore.initialize(parallelStoreId);
+
+    const store: FileStore = {
+      type: 'file',
+      id: parallelStoreId,
+      name: 'Parallel Test Store',
+      path: testFilesDir,
+      createdAt: new Date(),
+      updatedAt: new Date(),
+    };
+
+    const result = await indexService.indexStore(store);
+
+    expect(result.success).toBe(true);
+    if (result.success) {
+      expect(result.data.documentsIndexed).toBe(10);
+    }
+
+    vi.restoreAllMocks();
+  });
+
+  it('reports progress correctly with parallel processing', async () => {
+    // Create test files
+    for (let i = 0; i < 5; i++) {
+      await writeFile(
+        join(testFilesDir, `progress${String(i)}.ts`),
+        `export const p${String(i)} = ${String(i)};`
+      );
+    }
+
+    const concurrency = 2;
+    const indexService = new IndexService(lanceStore, embeddingEngine, { concurrency });
+
+    const progressStoreId = createStoreId('progress-test');
+    await lanceStore.initialize(progressStoreId);
+
+    const store: FileStore = {
+      type: 'file',
+      id: progressStoreId,
+      name: 'Progress Test Store',
+      path: testFilesDir,
+      createdAt: new Date(),
+      updatedAt: new Date(),
+    };
+
+    const progressEvents: Array<{ type: string; current: number; total: number }> = [];
+    const onProgress = (event: { type: string; current: number; total: number }): void => {
+      progressEvents.push(event);
+    };
+
+    const result = await indexService.indexStore(store, onProgress);
+
+    expect(result.success).toBe(true);
+
+    // Should have start event
+    expect(progressEvents.some((e) => e.type === 'start')).toBe(true);
+
+    // Should have progress events
+    const progressOnly = progressEvents.filter((e) => e.type === 'progress');
+    expect(progressOnly.length).toBeGreaterThan(0);
+
+    // Should have complete event
+    expect(progressEvents.some((e) => e.type === 'complete')).toBe(true);
+
+    // Current should never exceed total
+    for (const event of progressEvents) {
+      expect(event.current).toBeLessThanOrEqual(event.total);
+    }
+  });
+
+  it('continues processing remaining files if one file fails to read', async () => {
+    // Create valid test files
+    await writeFile(join(testFilesDir, 'valid1.ts'), 'export const a = 1;');
+    await writeFile(join(testFilesDir, 'valid2.ts'), 'export const b = 2;');
+    await writeFile(join(testFilesDir, 'valid3.ts'), 'export const c = 3;');
+
+    // Create a file that will fail to read (remove read permission)
+    const unreadablePath = join(testFilesDir, 'unreadable.ts');
+    await writeFile(unreadablePath, 'export const x = 999;');
+    await chmod(unreadablePath, 0o000);
+
+    const concurrency = 2;
+    const indexService = new IndexService(lanceStore, embeddingEngine, { concurrency });
+
+    const errorStoreId = createStoreId('error-handling-test');
+    await lanceStore.initialize(errorStoreId);
+
+    const store: FileStore = {
+      type: 'file',
+      id: errorStoreId,
+      name: 'Error Handling Test Store',
+      path: testFilesDir,
+      createdAt: new Date(),
+      updatedAt: new Date(),
+    };
+
+    // The indexing should either succeed with partial results or fail gracefully
+    const result = await indexService.indexStore(store);
+
+    // Restore permissions for cleanup
+    await chmod(unreadablePath, 0o644);
+
+    // With current implementation, it may fail completely on the first error
+    // This test documents the current behavior
+    if (result.success) {
+      // If it succeeds, it should have indexed at least the valid files
+      expect(result.data.documentsIndexed).toBeGreaterThanOrEqual(0);
+    } else {
+      // If it fails, it should have an error
+      expect(result.error).toBeDefined();
+    }
+  });
+});

package/src/services/index.service.ts

@@ -25,6 +25,7 @@ interface IndexOptions {
   chunkSize?: number;
   chunkOverlap?: number;
   codeGraphService?: CodeGraphService;
+  concurrency?: number;
 }
 
 const TEXT_EXTENSIONS = new Set([
@@ -62,6 +63,7 @@ export class IndexService {
   private readonly embeddingEngine: EmbeddingEngine;
   private readonly chunker: ChunkingService;
   private readonly codeGraphService: CodeGraphService | undefined;
+  private readonly concurrency: number;
 
   constructor(
     lanceStore: LanceStore,
@@ -75,6 +77,7 @@ export class IndexService {
       chunkOverlap: options.chunkOverlap ?? 100,
     });
     this.codeGraphService = options.codeGraphService;
+    this.concurrency = options.concurrency ?? 4;
   }
 
   async indexStore(store: Store, onProgress?: ProgressCallback): Promise<Result<IndexResult>> {
@@ -123,6 +126,7 @@ export class IndexService {
         storeId: store.id,
         path: store.path,
         fileCount: files.length,
+        concurrency: this.concurrency,
       },
       'Files scanned for indexing'
     );
@@ -138,59 +142,30 @@ export class IndexService {
       message: 'Starting index',
     });
 
-    for (const filePath of files) {
-      const content = await readFile(filePath, 'utf-8');
-      const fileHash = createHash('md5').update(content).digest('hex');
-      // Pass file path for semantic Markdown chunking
-      const chunks = this.chunker.chunk(content, filePath);
+    // Process files in parallel batches
+    for (let i = 0; i < files.length; i += this.concurrency) {
+      const batch = files.slice(i, i + this.concurrency);
 
-      // Determine file type for ranking
-      const ext = extname(filePath).toLowerCase();
-      const fileName = basename(filePath).toLowerCase();
-      const fileType = this.classifyFileType(ext, fileName, filePath);
+      const batchResults = await Promise.all(
+        batch.map((filePath) => this.processFile(filePath, store))
+      );
 
-      // Collect source files for code graph
-      if (['.ts', '.tsx', '.js', '.jsx'].includes(ext)) {
-        sourceFiles.push({ path: filePath, content });
+      // Collect results from batch
+      for (const result of batchResults) {
+        documents.push(...result.documents);
+        if (result.sourceFile !== undefined) {
+          sourceFiles.push(result.sourceFile);
+        }
       }
 
-      for (const chunk of chunks) {
-        const vector = await this.embeddingEngine.embed(chunk.content);
-        const chunkId =
-          chunks.length > 1
-            ? `${store.id}-${fileHash}-${String(chunk.chunkIndex)}`
-            : `${store.id}-${fileHash}`;
-
-        const doc: Document = {
-          id: createDocumentId(chunkId),
-          content: chunk.content,
-          vector,
-          metadata: {
-            type: chunks.length > 1 ? 'chunk' : 'file',
-            storeId: store.id,
-            path: filePath,
-            indexedAt: new Date(),
-            fileHash,
-            chunkIndex: chunk.chunkIndex,
-            totalChunks: chunk.totalChunks,
-            // New metadata for ranking
-            fileType,
-            sectionHeader: chunk.sectionHeader,
-            functionName: chunk.functionName,
-            hasDocComments: /\/\*\*[\s\S]*?\*\//.test(chunk.content),
-            docSummary: chunk.docSummary,
-          },
-        };
-        documents.push(doc);
-      }
-      filesProcessed++;
+      filesProcessed += batch.length;
 
-      // Emit progress event
+      // Emit progress event after each batch
       onProgress?.({
         type: 'progress',
         current: filesProcessed,
         total: files.length,
-        message: `Indexing ${filePath}`,
+        message: `Indexed ${String(filesProcessed)}/${String(files.length)} files`,
       });
     }
 
@@ -235,6 +210,80 @@ export class IndexService {
     });
   }
 
+  /**
+   * Process a single file: read, chunk, embed, and return documents.
+   * Extracted for parallel processing.
+   */
+  private async processFile(
+    filePath: string,
+    store: FileStore | RepoStore
+  ): Promise<{
+    documents: Document[];
+    sourceFile: { path: string; content: string } | undefined;
+  }> {
+    const content = await readFile(filePath, 'utf-8');
+    const fileHash = createHash('md5').update(content).digest('hex');
+    const chunks = this.chunker.chunk(content, filePath);
+
+    const ext = extname(filePath).toLowerCase();
+    const fileName = basename(filePath).toLowerCase();
+    const fileType = this.classifyFileType(ext, fileName, filePath);
+
+    // Track source file for code graph
+    const sourceFile = ['.ts', '.tsx', '.js', '.jsx'].includes(ext)
+      ? { path: filePath, content }
+      : undefined;
+
+    // Skip files with no chunks (empty files)
+    if (chunks.length === 0) {
+      return { documents: [], sourceFile };
+    }
+
+    // Batch embed all chunks from this file
+    const chunkContents = chunks.map((c) => c.content);
+    const vectors = await this.embeddingEngine.embedBatch(chunkContents);
+
+    const documents: Document[] = [];
+    for (let i = 0; i < chunks.length; i++) {
+      const chunk = chunks[i];
+      const vector = vectors[i];
+
+      // Fail fast if chunk/vector mismatch (should never happen)
+      if (chunk === undefined || vector === undefined) {
+        throw new Error(
+          `Chunk/vector mismatch at index ${String(i)}: chunk=${String(chunk !== undefined)}, vector=${String(vector !== undefined)}`
+        );
+      }
+
+      const chunkId =
+        chunks.length > 1
+          ? `${store.id}-${fileHash}-${String(chunk.chunkIndex)}`
+          : `${store.id}-${fileHash}`;
+
+      documents.push({
+        id: createDocumentId(chunkId),
+        content: chunk.content,
+        vector,
+        metadata: {
+          type: chunks.length > 1 ? 'chunk' : 'file',
+          storeId: store.id,
+          path: filePath,
+          indexedAt: new Date(),
+          fileHash,
+          chunkIndex: chunk.chunkIndex,
+          totalChunks: chunk.totalChunks,
+          fileType,
+          sectionHeader: chunk.sectionHeader,
+          functionName: chunk.functionName,
+          hasDocComments: /\/\*\*[\s\S]*?\*\//.test(chunk.content),
+          docSummary: chunk.docSummary,
+        },
+      });
+    }
+
+    return { documents, sourceFile };
+  }
+
   private async scanDirectory(dir: string): Promise<string[]> {
     const files: string[] = [];
     const entries = await readdir(dir, { withFileTypes: true });

package/src/services/job.service.test.ts

@@ -422,6 +422,93 @@ describe('JobService', () => {
     });
   });
 
+  describe('cleanupStalePendingJobs', () => {
+    it('should clean up pending jobs that have been stale for too long', () => {
+      const job = jobService.createJob({
+        type: 'index',
+        details: { storeId: 'test' },
+      });
+      // Job stays in pending status (never started)
+
+      // Make it stale (3 hours old)
+      const jobFile = join(tempDir, 'jobs', `${job.id}.json`);
+      const jobData = JSON.parse(readFileSync(jobFile, 'utf-8'));
+      jobData.updatedAt = new Date(Date.now() - 3 * 60 * 60 * 1000).toISOString();
+      writeFileSync(jobFile, JSON.stringify(jobData), 'utf-8');
+
+      const cleaned = jobService.cleanupStalePendingJobs(2);
+      expect(cleaned).toBe(1);
+      expect(existsSync(jobFile)).toBe(false);
+    });
+
+    it('should mark stale pending jobs as failed before cleanup', () => {
+      const job = jobService.createJob({
+        type: 'index',
+        details: { storeId: 'test' },
+      });
+
+      // Make it stale
+      const jobFile = join(tempDir, 'jobs', `${job.id}.json`);
+      const jobData = JSON.parse(readFileSync(jobFile, 'utf-8'));
+      jobData.updatedAt = new Date(Date.now() - 3 * 60 * 60 * 1000).toISOString();
+      writeFileSync(jobFile, JSON.stringify(jobData), 'utf-8');
+
+      // Should mark as failed (not just delete)
+      jobService.cleanupStalePendingJobs(2, { markAsFailed: true });
+
+      const updated = jobService.getJob(job.id);
+      expect(updated?.status).toBe('failed');
+      expect(updated?.message).toContain('stale');
+    });
+
+    it('should not clean up recently created pending jobs', () => {
+      const job = jobService.createJob({
+        type: 'index',
+        details: { storeId: 'test' },
+      });
+      // Job is pending but recent
+
+      const cleaned = jobService.cleanupStalePendingJobs(2);
+      expect(cleaned).toBe(0);
+
+      const retrieved = jobService.getJob(job.id);
+      expect(retrieved).not.toBeNull();
+    });
+
+    it('should not clean up running jobs even if old', () => {
+      const job = jobService.createJob({
+        type: 'index',
+        details: { storeId: 'test' },
+      });
+      jobService.updateJob(job.id, { status: 'running' });
+
+      // Make it old
+      const jobFile = join(tempDir, 'jobs', `${job.id}.json`);
+      const jobData = JSON.parse(readFileSync(jobFile, 'utf-8'));
+      jobData.updatedAt = new Date(Date.now() - 5 * 60 * 60 * 1000).toISOString();
+      writeFileSync(jobFile, JSON.stringify(jobData), 'utf-8');
+
+      const cleaned = jobService.cleanupStalePendingJobs(2);
+      expect(cleaned).toBe(0);
+    });
+
+    it('should use default 2 hours if not specified', () => {
+      const job = jobService.createJob({
+        type: 'index',
+        details: { storeId: 'test' },
+      });
+
+      // Make it 3 hours old
+      const jobFile = join(tempDir, 'jobs', `${job.id}.json`);
+      const jobData = JSON.parse(readFileSync(jobFile, 'utf-8'));
+      jobData.updatedAt = new Date(Date.now() - 3 * 60 * 60 * 1000).toISOString();
+      writeFileSync(jobFile, JSON.stringify(jobData), 'utf-8');
+
+      const cleaned = jobService.cleanupStalePendingJobs();
+      expect(cleaned).toBe(1);
+    });
+  });
+
   describe('deleteJob', () => {
     it('should delete a job file', () => {
       const job = jobService.createJob({

package/src/services/job.service.ts

@@ -225,6 +225,49 @@ export class JobService {
     return cleaned;
   }
 
+  /**
+   * Clean up stale pending jobs that never started or got stuck
+   *
+   * @param olderThanHours - Consider pending jobs stale after this many hours (default 2)
+   * @param options - Options for cleanup behavior
+   * @param options.markAsFailed - If true, mark jobs as failed instead of deleting
+   * @returns Number of jobs cleaned up or marked as failed
+   */
+  cleanupStalePendingJobs(
+    olderThanHours: number = 2,
+    options: { markAsFailed?: boolean } = {}
+  ): number {
+    const jobs = this.listJobs();
+    const cutoffTime = Date.now() - olderThanHours * 60 * 60 * 1000;
+    let cleaned = 0;
+
+    for (const job of jobs) {
+      if (job.status === 'pending' && new Date(job.updatedAt).getTime() < cutoffTime) {
+        const jobFile = path.join(this.jobsDir, `${job.id}.json`);
+
+        if (options.markAsFailed === true) {
+          // Mark as failed instead of deleting
+          this.updateJob(job.id, {
+            status: 'failed',
+            message: `Job marked as stale - pending for over ${String(olderThanHours)} hours without progress`,
+          });
+        } else {
+          // Delete the job file
+          try {
+            fs.unlinkSync(jobFile);
+          } catch (error) {
+            throw new Error(
+              `Failed to delete stale job ${job.id}: ${error instanceof Error ? error.message : String(error)}`
+            );
+          }
+        }
+        cleaned++;
+      }
+    }
+
+    return cleaned;
+  }
+
   /**
    * Delete a specific job
    */