bluera-knowledge 0.9.32 → 0.9.36

package/src/utils/type-guards.test.ts

@@ -4,7 +4,7 @@ import {
   isDocumentMetadata,
   isPartialAppConfig,
   isFloat32ArrayData,
-  hasDefaultExport
+  hasDefaultExport,
 } from './type-guards.js';
 import type { DocumentMetadata } from '../types/document.js';
 
@@ -21,7 +21,12 @@ describe('TypeGuards - parseJSON', () => {
 
   it('throws on invalid JSON structure', () => {
     const validator = (value: unknown): value is { name: string } => {
-      return typeof value === 'object' && value !== null && 'name' in value && typeof (value as any).name === 'string';
+      return (
+        typeof value === 'object' &&
+        value !== null &&
+        'name' in value &&
+        typeof (value as any).name === 'string'
+      );
     };
 
     expect(() => {
@@ -39,7 +44,7 @@ describe('TypeGuards - parseJSON', () => {
 
   it('handles arrays with validator', () => {
     const validator = (value: unknown): value is string[] => {
-      return Array.isArray(value) && value.every(v => typeof v === 'string');
+      return Array.isArray(value) && value.every((v) => typeof v === 'string');
     };
 
     const result = parseJSON('["a","b","c"]', validator);
@@ -79,7 +84,7 @@ describe('TypeGuards - isDocumentMetadata', () => {
     const metadata: DocumentMetadata = {
       storeId: 'store-123',
       path: '/path/to/file.ts',
-      docType: 'code'
+      docType: 'code',
     };
 
     expect(isDocumentMetadata(metadata)).toBe(true);
@@ -102,7 +107,7 @@ describe('TypeGuards - isDocumentMetadata', () => {
   it('returns false when missing required storeId', () => {
     const invalid = {
       path: '/path/to/file.ts',
-      docType: 'code'
+      docType: 'code',
     };
 
     expect(isDocumentMetadata(invalid)).toBe(false);
@@ -111,7 +116,7 @@ describe('TypeGuards - isDocumentMetadata', () => {
   it('returns false when missing required path', () => {
     const invalid = {
       storeId: 'store-123',
-      docType: 'code'
+      docType: 'code',
     };
 
     expect(isDocumentMetadata(invalid)).toBe(false);
@@ -120,7 +125,7 @@ describe('TypeGuards - isDocumentMetadata', () => {
   it('returns false when missing required docType', () => {
     const invalid = {
       storeId: 'store-123',
-      path: '/path/to/file.ts'
+      path: '/path/to/file.ts',
     };
 
     expect(isDocumentMetadata(invalid)).toBe(false);
@@ -130,7 +135,7 @@ describe('TypeGuards - isDocumentMetadata', () => {
     const invalid = {
       storeId: 123,
       path: '/path/to/file.ts',
-      docType: 'code'
+      docType: 'code',
     };
 
     expect(isDocumentMetadata(invalid)).toBe(false);
@@ -140,7 +145,7 @@ describe('TypeGuards - isDocumentMetadata', () => {
     const invalid = {
       storeId: 'store-123',
       path: 123,
-      docType: 'code'
+      docType: 'code',
     };
 
     expect(isDocumentMetadata(invalid)).toBe(false);
@@ -150,7 +155,7 @@ describe('TypeGuards - isDocumentMetadata', () => {
     const invalid = {
       storeId: 'store-123',
       path: '/path/to/file.ts',
-      docType: 123
+      docType: 123,
     };
 
     expect(isDocumentMetadata(invalid)).toBe(false);
@@ -161,7 +166,7 @@ describe('TypeGuards - isDocumentMetadata', () => {
       storeId: 'store-123',
       path: '/path/to/file.ts',
       docType: 'code',
-      extraProp: 'allowed'
+      extraProp: 'allowed',
     };
 
     expect(isDocumentMetadata(metadata)).toBe(true);
@@ -172,7 +177,7 @@ describe('TypeGuards - isPartialAppConfig', () => {
   it('returns true for valid partial config', () => {
     const config = {
       port: 3000,
-      host: 'localhost'
+      host: 'localhost',
     };
 
     expect(isPartialAppConfig(config)).toBe(true);
@@ -199,7 +204,7 @@ describe('TypeGuards - isPartialAppConfig', () => {
   it('returns true for object with any properties', () => {
     const config = {
       anyProp: 'value',
-      anotherProp: 123
+      anotherProp: 123,
     };
 
     expect(isPartialAppConfig(config)).toBe(true);
@@ -296,7 +301,7 @@ describe('TypeGuards - hasDefaultExport', () => {
   it('returns true for object with default and other properties', () => {
     const module = {
       default: 'value',
-      named: 'export'
+      named: 'export',
     };
 
     expect(hasDefaultExport(module)).toBe(true);
@@ -308,7 +313,7 @@ describe('TypeGuards - Type Narrowing', () => {
     const data: unknown = {
       storeId: 'store-123',
       path: '/file.ts',
-      docType: 'code'
+      docType: 'code',
     };
 
     if (isDocumentMetadata(data)) {

package/src/utils/type-guards.ts

@@ -2,8 +2,8 @@
  * Type guard utilities to replace type assertions
  */
 
-import type { DocumentMetadata } from '../types/document.js';
 import type { AppConfig } from '../types/config.js';
+import type { DocumentMetadata } from '../types/document.js';
 
 /**
  * Safely parse JSON with validation

package/src/workers/background-worker-cli.ts

@@ -1,9 +1,8 @@
 #!/usr/bin/env node
-import fs from 'fs';
-import path from 'path';
-import { JobService } from '../services/job.service.js';
 import { BackgroundWorker } from './background-worker.js';
+import { writePidFile, deletePidFile, buildPidFilePath } from './pid-file.js';
 import { createServices } from '../services/index.js';
+import { JobService } from '../services/job.service.js';
 
 /**
  * Background worker CLI entry point
@@ -27,16 +26,18 @@ async function main(): Promise<void> {
   const jobService = new JobService(dataDir);
   const services = await createServices(undefined, dataDir);
 
-  // Write PID file for job cancellation
-  const pidFile = path.join(
+  // Write PID file for job cancellation - CRITICAL: must succeed or job cannot be cancelled
+  const pidFile = buildPidFilePath(
     jobService['jobsDir'], // Access private field for PID path
-    `${jobId}.pid`
+    jobId
   );
 
   try {
-    fs.writeFileSync(pidFile, process.pid.toString(), 'utf-8');
+    writePidFile(pidFile, process.pid);
   } catch (error) {
-    console.error('Warning: Could not write PID file:', error);
+    // CRITICAL: Cannot proceed without PID file - job would be uncancellable
+    console.error(error instanceof Error ? error.message : String(error));
+    process.exit(1);
   }
 
   // Handle SIGTERM for graceful shutdown
@@ -44,16 +45,15 @@ async function main(): Promise<void> {
     console.log(`[${jobId}] Received SIGTERM, cancelling job...`);
     jobService.updateJob(jobId, {
       status: 'cancelled',
-      message: 'Job cancelled by user'
+      message: 'Job cancelled by user',
     });
 
-    // Clean up PID file
-    try {
-      if (fs.existsSync(pidFile)) {
-        fs.unlinkSync(pidFile);
-      }
-    } catch (error) {
-      console.error('Warning: Could not remove PID file:', error);
+    // Clean up PID file (best-effort - don't block shutdown)
+    const deleteResult = deletePidFile(pidFile, 'sigterm');
+    if (!deleteResult.success && deleteResult.error !== undefined) {
+      console.error(
+        `Warning: Could not remove PID file during SIGTERM: ${deleteResult.error.message}`
+      );
     }
 
     process.exit(0);
@@ -71,13 +71,12 @@ async function main(): Promise<void> {
   try {
     await worker.executeJob(jobId);
 
-    // Clean up PID file on success
-    try {
-      if (fs.existsSync(pidFile)) {
-        fs.unlinkSync(pidFile);
-      }
-    } catch (error) {
-      console.error('Warning: Could not remove PID file:', error);
+    // Clean up PID file on success (best-effort - don't change exit code)
+    const successCleanup = deletePidFile(pidFile, 'success');
+    if (!successCleanup.success && successCleanup.error !== undefined) {
+      console.error(
+        `Warning: Could not remove PID file after success: ${successCleanup.error.message}`
+      );
     }
 
     console.log(`[${jobId}] Job completed successfully`);
@@ -86,13 +85,12 @@ async function main(): Promise<void> {
     // Job service already updated with failure status in BackgroundWorker
     console.error(`[${jobId}] Job failed:`, error);
 
-    // Clean up PID file on failure
-    try {
-      if (fs.existsSync(pidFile)) {
-        fs.unlinkSync(pidFile);
-      }
-    } catch (cleanupError) {
-      console.error('Warning: Could not remove PID file:', cleanupError);
+    // Clean up PID file on failure (best-effort - exit code reflects job failure)
+    const failureCleanup = deletePidFile(pidFile, 'failure');
+    if (!failureCleanup.success && failureCleanup.error !== undefined) {
+      console.error(
+        `Warning: Could not remove PID file after failure: ${failureCleanup.error.message}`
+      );
     }
 
     process.exit(1);

package/src/workers/background-worker.test.ts

@@ -1,5 +1,5 @@
 import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
-import { BackgroundWorker } from './background-worker.js';
+import { BackgroundWorker, calculateIndexProgress } from './background-worker.js';
 import { JobService } from '../services/job.service.js';
 import { StoreService } from '../services/store.service.js';
 import { IndexService } from '../services/index.service.js';
@@ -31,7 +31,13 @@ describe('BackgroundWorker', () => {
     embeddingEngine = {
       embed: vi.fn().mockResolvedValue(new Array(384).fill(0)),
     } as unknown as EmbeddingEngine;
-    worker = new BackgroundWorker(jobService, storeService, indexService, lanceStore, embeddingEngine);
+    worker = new BackgroundWorker(
+      jobService,
+      storeService,
+      indexService,
+      lanceStore,
+      embeddingEngine
+    );
   });
 
   afterEach(() => {
@@ -42,27 +48,23 @@ describe('BackgroundWorker', () => {
 
   describe('executeJob', () => {
     it('should throw error for non-existent job', async () => {
-      await expect(worker.executeJob('non-existent')).rejects.toThrow(
-        'Job non-existent not found'
-      );
+      await expect(worker.executeJob('non-existent')).rejects.toThrow('Job non-existent not found');
     });
 
     it('should throw error for unknown job type', async () => {
       const job = jobService.createJob({
         // @ts-expect-error testing invalid job type
         type: 'unknown',
-        details: { storeId: 'test' }
+        details: { storeId: 'test' },
       });
 
-      await expect(worker.executeJob(job.id)).rejects.toThrow(
-        'Unknown job type: unknown'
-      );
+      await expect(worker.executeJob(job.id)).rejects.toThrow('Unknown job type: unknown');
     });
 
     it('should set job to running status before execution', async () => {
       const job = jobService.createJob({
         type: 'crawl',
-        details: { storeId: 'test' }
+        details: { storeId: 'test' },
       });
 
       try {
@@ -79,12 +81,10 @@ describe('BackgroundWorker', () => {
     it('should update job to failed status on error', async () => {
       const job = jobService.createJob({
         type: 'crawl',
-        details: { storeId: 'test', url: 'https://example.com' }
+        details: { storeId: 'test', url: 'https://example.com' },
       });
 
-      await expect(worker.executeJob(job.id)).rejects.toThrow(
-        'Web store test not found'
-      );
+      await expect(worker.executeJob(job.id)).rejects.toThrow('Web store test not found');
 
       const updated = jobService.getJob(job.id);
       expect(updated?.status).toBe('failed');
@@ -96,48 +96,39 @@ describe('BackgroundWorker', () => {
     it('should throw error for job without storeId', async () => {
       const job = jobService.createJob({
         type: 'index',
-        details: {}
+        details: {},
       });
 
-      await expect(worker.executeJob(job.id)).rejects.toThrow(
-        'Store ID required for index job'
-      );
+      await expect(worker.executeJob(job.id)).rejects.toThrow('Store ID required for index job');
     });
 
     it('should throw error for non-existent store', async () => {
       const job = jobService.createJob({
         type: 'index',
-        details: { storeId: 'non-existent-store' }
+        details: { storeId: 'non-existent-store' },
       });
 
-      await expect(worker.executeJob(job.id)).rejects.toThrow(
-        'Store non-existent-store not found'
-      );
+      await expect(worker.executeJob(job.id)).rejects.toThrow('Store non-existent-store not found');
     });
-
   });
 
   describe('executeCloneJob', () => {
     it('should throw error for job without storeId', async () => {
       const job = jobService.createJob({
         type: 'clone',
-        details: {}
+        details: {},
       });
 
-      await expect(worker.executeJob(job.id)).rejects.toThrow(
-        'Store ID required for clone job'
-      );
+      await expect(worker.executeJob(job.id)).rejects.toThrow('Store ID required for clone job');
     });
 
     it('should throw error for non-existent store', async () => {
       const job = jobService.createJob({
         type: 'clone',
-        details: { storeId: 'non-existent-store' }
+        details: { storeId: 'non-existent-store' },
       });
 
-      await expect(worker.executeJob(job.id)).rejects.toThrow(
-        'Store non-existent-store not found'
-      );
+      await expect(worker.executeJob(job.id)).rejects.toThrow('Store non-existent-store not found');
     });
   });
 
@@ -145,29 +136,25 @@ describe('BackgroundWorker', () => {
     it('should throw error for job without storeId', async () => {
       const job = jobService.createJob({
         type: 'crawl',
-        details: { url: 'https://example.com' }
+        details: { url: 'https://example.com' },
       });
 
-      await expect(worker.executeJob(job.id)).rejects.toThrow(
-        'Store ID required for crawl job'
-      );
+      await expect(worker.executeJob(job.id)).rejects.toThrow('Store ID required for crawl job');
     });
 
     it('should throw error for job without url', async () => {
       const job = jobService.createJob({
         type: 'crawl',
-        details: { storeId: 'test-store' }
+        details: { storeId: 'test-store' },
       });
 
-      await expect(worker.executeJob(job.id)).rejects.toThrow(
-        'URL required for crawl job'
-      );
+      await expect(worker.executeJob(job.id)).rejects.toThrow('URL required for crawl job');
     });
 
     it('should throw error for non-existent store', async () => {
       const job = jobService.createJob({
         type: 'crawl',
-        details: { storeId: 'non-existent-store', url: 'https://example.com' }
+        details: { storeId: 'non-existent-store', url: 'https://example.com' },
       });
 
       await expect(worker.executeJob(job.id)).rejects.toThrow(
@@ -175,4 +162,31 @@ describe('BackgroundWorker', () => {
       );
     });
   });
+
+  describe('calculateIndexProgress', () => {
+    it('handles event.total === 0 without division by zero (NaN)', () => {
+      const result = calculateIndexProgress(0, 0, 70);
+      expect(Number.isNaN(result)).toBe(false);
+      expect(result).toBe(0);
+    });
+
+    it('handles event.total === 0 with scale 100', () => {
+      const result = calculateIndexProgress(0, 0, 100);
+      expect(Number.isNaN(result)).toBe(false);
+      expect(result).toBe(0);
+    });
+
+    it('calculates progress correctly for non-zero total', () => {
+      // 5/10 * 70 = 35
+      expect(calculateIndexProgress(5, 10, 70)).toBe(35);
+      // 5/10 * 100 = 50
+      expect(calculateIndexProgress(5, 10, 100)).toBe(50);
+      // 10/10 * 100 = 100
+      expect(calculateIndexProgress(10, 10, 100)).toBe(100);
+    });
+
+    it('uses default scale of 100', () => {
+      expect(calculateIndexProgress(5, 10)).toBe(50);
+    });
+  });
 });

package/src/workers/background-worker.ts

@@ -1,13 +1,29 @@
 import { createHash } from 'node:crypto';
+import { IntelligentCrawler, type CrawlProgress } from '../crawl/intelligent-crawler.js';
+import { IndexService } from '../services/index.service.js';
 import { JobService } from '../services/job.service.js';
 import { StoreService } from '../services/store.service.js';
-import { IndexService } from '../services/index.service.js';
-import type { LanceStore } from '../db/lance.js';
+import { createStoreId, createDocumentId } from '../types/brands.js';
 import type { EmbeddingEngine } from '../db/embeddings.js';
-import { IntelligentCrawler, type CrawlProgress } from '../crawl/intelligent-crawler.js';
-import type { Job } from '../types/job.js';
+import type { LanceStore } from '../db/lance.js';
 import type { Document } from '../types/document.js';
-import { createStoreId, createDocumentId } from '../types/brands.js';
+import type { Job } from '../types/job.js';
+
+/**
+ * Calculate index progress as a percentage, handling division by zero.
+ * @param current - Current number of items processed
+ * @param total - Total number of items (may be 0)
+ * @param scale - Scale factor for progress (default 100 for 0-100%)
+ * @returns Progress value, or 0 if total is 0
+ */
+export function calculateIndexProgress(
+  current: number,
+  total: number,
+  scale: number = 100
+): number {
+  if (total === 0) return 0;
+  return (current / total) * scale;
+}
 
 export class BackgroundWorker {
   constructor(
@@ -34,7 +50,7 @@ export class BackgroundWorker {
         status: 'running',
         message: `Starting ${job.type} operation...`,
         progress: 0,
-        details: { startedAt: new Date().toISOString() }
+        details: { startedAt: new Date().toISOString() },
       });
 
       // Execute based on job type
@@ -57,12 +73,12 @@ export class BackgroundWorker {
         status: 'completed',
         progress: 100,
         message: `${job.type} operation completed successfully`,
-        details: { completedAt: new Date().toISOString() }
+        details: { completedAt: new Date().toISOString() },
       });
     } catch (error) {
       // Mark as failed
       const errorDetails: Record<string, unknown> = {
-        completedAt: new Date().toISOString()
+        completedAt: new Date().toISOString(),
       };
       if (error instanceof Error && error.stack !== undefined) {
         errorDetails['error'] = error.stack;
@@ -72,7 +88,7 @@ export class BackgroundWorker {
       this.jobService.updateJob(jobId, {
         status: 'failed',
         message: error instanceof Error ? error.message : 'Unknown error',
-        details: errorDetails
+        details: errorDetails,
       });
       throw error;
     }
@@ -101,30 +117,33 @@ export class BackgroundWorker {
     this.jobService.updateJob(job.id, {
       status: 'running',
       message: 'Repository cloned, starting indexing...',
-      progress: 30
+      progress: 30,
     });
 
     // Index the repository with progress updates
-    const result = await this.indexService.indexStore(store, (event: { type: string; current: number; total: number; message: string }) => {
-      // Check if job was cancelled
-      const currentJob = this.jobService.getJob(job.id);
-      if (currentJob?.status === 'cancelled') {
-        throw new Error('Job cancelled by user');
-      }
+    const result = await this.indexService.indexStore(
+      store,
+      (event: { type: string; current: number; total: number; message: string }) => {
+        // Check if job was cancelled
+        const currentJob = this.jobService.getJob(job.id);
+        if (currentJob?.status === 'cancelled') {
+          throw new Error('Job cancelled by user');
+        }
 
-      // Indexing is 70% of total progress (30-100%)
-      const indexProgress = (event.current / event.total) * 70;
-      const totalProgress = 30 + indexProgress;
+        // Indexing is 70% of total progress (30-100%)
+        const indexProgress = calculateIndexProgress(event.current, event.total, 70);
+        const totalProgress = 30 + indexProgress;
 
-      this.jobService.updateJob(job.id, {
-        message: `Indexed ${String(event.current)}/${String(event.total)} files`,
-        progress: Math.min(99, totalProgress), // Cap at 99 until fully complete
-        details: {
-          filesProcessed: event.current,
-          totalFiles: event.total
-        }
-      });
-    });
+        this.jobService.updateJob(job.id, {
+          message: `Indexed ${String(event.current)}/${String(event.total)} files`,
+          progress: Math.min(99, totalProgress), // Cap at 99 until fully complete
+          details: {
+            filesProcessed: event.current,
+            totalFiles: event.total,
+          },
+        });
+      }
+    );
 
     if (!result.success) {
       throw result.error;
@@ -148,24 +167,27 @@ export class BackgroundWorker {
     }
 
     // Index with progress updates
-    const result = await this.indexService.indexStore(store, (event: { type: string; current: number; total: number; message: string }) => {
-      // Check if job was cancelled
-      const currentJob = this.jobService.getJob(job.id);
-      if (currentJob?.status === 'cancelled') {
-        throw new Error('Job cancelled by user');
-      }
+    const result = await this.indexService.indexStore(
+      store,
+      (event: { type: string; current: number; total: number; message: string }) => {
+        // Check if job was cancelled
+        const currentJob = this.jobService.getJob(job.id);
+        if (currentJob?.status === 'cancelled') {
+          throw new Error('Job cancelled by user');
+        }
 
-      const progress = (event.current / event.total) * 100;
+        const progress = calculateIndexProgress(event.current, event.total);
 
-      this.jobService.updateJob(job.id, {
-        message: `Indexed ${String(event.current)}/${String(event.total)} files`,
-        progress: Math.min(99, progress), // Cap at 99 until fully complete
-        details: {
-          filesProcessed: event.current,
-          totalFiles: event.total
-        }
-      });
-    });
+        this.jobService.updateJob(job.id, {
+          message: `Indexed ${String(event.current)}/${String(event.total)} files`,
+          progress: Math.min(99, progress), // Cap at 99 until fully complete
+          details: {
+            filesProcessed: event.current,
+            totalFiles: event.total,
+          },
+        });
+      }
+    );
 
     if (!result.success) {
       throw result.error;
@@ -176,15 +198,8 @@ export class BackgroundWorker {
    * Execute a crawl job (web crawling + indexing)
    */
   private async executeCrawlJob(job: Job): Promise<void> {
-    const {
-      storeId,
-      url,
-      crawlInstruction,
-      extractInstruction,
-      maxPages,
-      simple,
-      useHeadless,
-    } = job.details;
+    const { storeId, url, crawlInstruction, extractInstruction, maxPages, simple, useHeadless } =
+      job.details;
 
     if (storeId === undefined || typeof storeId !== 'string') {
       throw new Error('Store ID required for crawl job');
@@ -195,7 +210,7 @@ export class BackgroundWorker {
 
     // Get the store
     const store = await this.storeService.get(createStoreId(storeId));
-    if (!store || store.type !== 'web') {
+    if (store?.type !== 'web') {
       throw new Error(`Web store ${storeId} not found`);
     }
 
@@ -204,10 +219,9 @@ export class BackgroundWorker {
 
     // Listen for progress events
     crawler.on('progress', (progress: CrawlProgress) => {
-      // Check if job was cancelled
+      // Check if job was cancelled - just return early, for-await loop will throw and finally will cleanup
       const currentJob = this.jobService.getJob(job.id);
       if (currentJob?.status === 'cancelled') {
-        void crawler.stop();
         return;
       }
 
@@ -215,9 +229,11 @@ export class BackgroundWorker {
       const crawlProgress = (progress.pagesVisited / resolvedMaxPages) * 80;
 
       this.jobService.updateJob(job.id, {
-        message: progress.message ?? `Crawling page ${String(progress.pagesVisited)}/${String(resolvedMaxPages)}`,
+        message:
+          progress.message ??
+          `Crawling page ${String(progress.pagesVisited)}/${String(resolvedMaxPages)}`,
         progress: Math.min(80, crawlProgress),
-        details: { pagesCrawled: progress.pagesVisited }
+        details: { pagesCrawled: progress.pagesVisited },
       });
     });
 
@@ -276,7 +292,7 @@ export class BackgroundWorker {
       if (docs.length > 0) {
         this.jobService.updateJob(job.id, {
           message: 'Indexing crawled documents...',
-          progress: 85
+          progress: 85,
         });
 
         await this.lanceStore.addDocuments(store.id, docs);
@@ -285,7 +301,7 @@ export class BackgroundWorker {
       this.jobService.updateJob(job.id, {
         message: `Crawled and indexed ${String(docs.length)} pages`,
         progress: 100,
-        details: { pagesCrawled: docs.length }
+        details: { pagesCrawled: docs.length },
       });
     } finally {
       await crawler.stop();