bluera-knowledge 0.13.3 → 0.14.1

package/src/workers/background-worker.test.ts

@@ -1,222 +0,0 @@
-import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
-import { BackgroundWorker, calculateIndexProgress } from './background-worker.js';
-import { JobService } from '../services/job.service.js';
-import { StoreService } from '../services/store.service.js';
-import { IndexService } from '../services/index.service.js';
-import type { LanceStore } from '../db/lance.js';
-import type { EmbeddingEngine } from '../db/embeddings.js';
-import { mkdtempSync, rmSync, existsSync } from 'fs';
-import { tmpdir } from 'os';
-import { join } from 'path';
-
-describe('BackgroundWorker', () => {
-  let tempDir: string;
-  let jobService: JobService;
-  let storeService: StoreService;
-  let indexService: IndexService;
-  let lanceStore: LanceStore;
-  let embeddingEngine: EmbeddingEngine;
-  let worker: BackgroundWorker;
-
-  beforeEach(async () => {
-    tempDir = mkdtempSync(join(tmpdir(), 'background-worker-test-'));
-    jobService = new JobService(tempDir);
-    storeService = new StoreService(tempDir);
-    indexService = new IndexService(tempDir);
-    // Mock LanceStore and EmbeddingEngine for testing
-    lanceStore = {
-      initialize: vi.fn().mockResolvedValue(undefined),
-      addDocuments: vi.fn().mockResolvedValue(undefined),
-      createFtsIndex: vi.fn().mockResolvedValue(undefined),
-    } as unknown as LanceStore;
-    embeddingEngine = {
-      embed: vi.fn().mockResolvedValue(new Array(384).fill(0)),
-    } as unknown as EmbeddingEngine;
-    worker = new BackgroundWorker(
-      jobService,
-      storeService,
-      indexService,
-      lanceStore,
-      embeddingEngine
-    );
-  });
-
-  afterEach(() => {
-    if (existsSync(tempDir)) {
-      rmSync(tempDir, { recursive: true, force: true });
-    }
-  });
-
-  describe('executeJob', () => {
-    it('should throw error for non-existent job', async () => {
-      await expect(worker.executeJob('non-existent')).rejects.toThrow('Job non-existent not found');
-    });
-
-    it('should throw error for unknown job type', async () => {
-      const job = jobService.createJob({
-        // @ts-expect-error testing invalid job type
-        type: 'unknown',
-        details: { storeId: 'test' },
-      });
-
-      // Zod validation catches invalid job type when reading the job file
-      await expect(worker.executeJob(job.id)).rejects.toThrow(
-        /Invalid option.*clone.*index.*crawl/
-      );
-    });
-
-    it('should set job to running status before execution', async () => {
-      const job = jobService.createJob({
-        type: 'crawl',
-        details: { storeId: 'test' },
-      });
-
-      try {
-        await worker.executeJob(job.id);
-      } catch {
-        // Expected to fail since store doesn't exist
-      }
-
-      const updated = jobService.getJob(job.id);
-      // Should have been set to running before throwing error
-      expect(updated?.status).toBe('failed');
-    });
-
-    it('should update job to failed status on error', async () => {
-      const job = jobService.createJob({
-        type: 'crawl',
-        details: { storeId: 'test', url: 'https://example.com' },
-      });
-
-      await expect(worker.executeJob(job.id)).rejects.toThrow('Web store test not found');
-
-      const updated = jobService.getJob(job.id);
-      expect(updated?.status).toBe('failed');
-      expect(updated?.message).toBe('Web store test not found');
-    });
-  });
-
-  describe('executeIndexJob', () => {
-    it('should throw error for job without storeId', async () => {
-      const job = jobService.createJob({
-        type: 'index',
-        details: {},
-      });
-
-      await expect(worker.executeJob(job.id)).rejects.toThrow('Store ID required for index job');
-    });
-
-    it('should throw error for non-existent store', async () => {
-      const job = jobService.createJob({
-        type: 'index',
-        details: { storeId: 'non-existent-store' },
-      });
-
-      await expect(worker.executeJob(job.id)).rejects.toThrow('Store non-existent-store not found');
-    });
-  });
-
-  describe('executeCloneJob', () => {
-    it('should throw error for job without storeId', async () => {
-      const job = jobService.createJob({
-        type: 'clone',
-        details: {},
-      });
-
-      await expect(worker.executeJob(job.id)).rejects.toThrow('Store ID required for clone job');
-    });
-
-    it('should throw error for non-existent store', async () => {
-      const job = jobService.createJob({
-        type: 'clone',
-        details: { storeId: 'non-existent-store' },
-      });
-
-      await expect(worker.executeJob(job.id)).rejects.toThrow('Store non-existent-store not found');
-    });
-  });
-
-  describe('executeCrawlJob', () => {
-    it('should throw error for job without storeId', async () => {
-      const job = jobService.createJob({
-        type: 'crawl',
-        details: { url: 'https://example.com' },
-      });
-
-      await expect(worker.executeJob(job.id)).rejects.toThrow('Store ID required for crawl job');
-    });
-
-    it('should throw error for job without url', async () => {
-      const job = jobService.createJob({
-        type: 'crawl',
-        details: { storeId: 'test-store' },
-      });
-
-      await expect(worker.executeJob(job.id)).rejects.toThrow('URL required for crawl job');
-    });
-
-    it('should throw error for non-existent store', async () => {
-      const job = jobService.createJob({
-        type: 'crawl',
-        details: { storeId: 'non-existent-store', url: 'https://example.com' },
-      });
-
-      await expect(worker.executeJob(job.id)).rejects.toThrow(
-        'Web store non-existent-store not found'
-      );
-    });
-  });
-
-  describe('calculateIndexProgress', () => {
-    it('handles event.total === 0 without division by zero (NaN)', () => {
-      const result = calculateIndexProgress(0, 0, 70);
-      expect(Number.isNaN(result)).toBe(false);
-      expect(result).toBe(0);
-    });
-
-    it('handles event.total === 0 with scale 100', () => {
-      const result = calculateIndexProgress(0, 0, 100);
-      expect(Number.isNaN(result)).toBe(false);
-      expect(result).toBe(0);
-    });
-
-    it('calculates progress correctly for non-zero total', () => {
-      // 5/10 * 70 = 35
-      expect(calculateIndexProgress(5, 10, 70)).toBe(35);
-      // 5/10 * 100 = 50
-      expect(calculateIndexProgress(5, 10, 100)).toBe(50);
-      // 10/10 * 100 = 100
-      expect(calculateIndexProgress(10, 10, 100)).toBe(100);
-    });
-
-    it('uses default scale of 100', () => {
-      expect(calculateIndexProgress(5, 10)).toBe(50);
-    });
-  });
-});
-
-/**
- * Tests to verify background-worker.ts uses file logging for visibility.
- * This ensures job execution is observable in log files.
- */
-describe('BackgroundWorker Logging', () => {
-  const { readFileSync } = require('fs');
-  const source = readFileSync('src/workers/background-worker.ts', 'utf-8');
-
-  it('imports createLogger from logging module', () => {
-    expect(source).toContain('createLogger');
-    expect(source).toContain("from '../logging/index.js'");
-  });
-
-  it('logs job start with jobId and type', () => {
-    // Verify logger.info is called with job start pattern
-    expect(source).toContain('logger.info');
-    expect(source).toContain('Starting job');
-  });
-
-  it('logs job failure with error details', () => {
-    // Verify logger.error is called for failures
-    expect(source).toContain('logger.error');
-    expect(source).toContain('Job failed');
-  });
-});

package/src/workers/background-worker.ts

@@ -1,322 +0,0 @@
-import { createHash } from 'node:crypto';
-import { IntelligentCrawler, type CrawlProgress } from '../crawl/intelligent-crawler.js';
-import { createLogger } from '../logging/index.js';
-import { IndexService } from '../services/index.service.js';
-import { JobService } from '../services/job.service.js';
-import { StoreService } from '../services/store.service.js';
-import { createStoreId, createDocumentId } from '../types/brands.js';
-import type { EmbeddingEngine } from '../db/embeddings.js';
-import type { LanceStore } from '../db/lance.js';
-import type { Document } from '../types/document.js';
-import type { Job } from '../types/job.js';
-
-const logger = createLogger('background-worker');
-
-/**
- * Calculate index progress as a percentage, handling division by zero.
- * @param current - Current number of items processed
- * @param total - Total number of items (may be 0)
- * @param scale - Scale factor for progress (default 100 for 0-100%)
- * @returns Progress value, or 0 if total is 0
- */
-export function calculateIndexProgress(
-  current: number,
-  total: number,
-  scale: number = 100
-): number {
-  if (total === 0) return 0;
-  return (current / total) * scale;
-}
-
-export class BackgroundWorker {
-  constructor(
-    private readonly jobService: JobService,
-    private readonly storeService: StoreService,
-    private readonly indexService: IndexService,
-    private readonly lanceStore: LanceStore,
-    private readonly embeddingEngine: EmbeddingEngine
-  ) {}
-
-  /**
-   * Execute a job based on its type
-   */
-  async executeJob(jobId: string): Promise<void> {
-    const job = this.jobService.getJob(jobId);
-
-    if (!job) {
-      throw new Error(`Job ${jobId} not found`);
-    }
-
-    try {
-      logger.info({ jobId, type: job.type }, 'Starting job execution');
-
-      // Update to running status
-      this.jobService.updateJob(jobId, {
-        status: 'running',
-        message: `Starting ${job.type} operation...`,
-        progress: 0,
-        details: { startedAt: new Date().toISOString() },
-      });
-
-      // Execute based on job type
-      switch (job.type) {
-        case 'clone':
-          await this.executeCloneJob(job);
-          break;
-        case 'index':
-          await this.executeIndexJob(job);
-          break;
-        case 'crawl':
-          await this.executeCrawlJob(job);
-          break;
-        default:
-          throw new Error(`Unknown job type: ${String(job.type)}`);
-      }
-
-      // Mark as completed
-      this.jobService.updateJob(jobId, {
-        status: 'completed',
-        progress: 100,
-        message: `${job.type} operation completed successfully`,
-        details: { completedAt: new Date().toISOString() },
-      });
-    } catch (error) {
-      logger.error(
-        { jobId, error: error instanceof Error ? error.message : String(error) },
-        'Job failed'
-      );
-
-      // Mark as failed
-      const errorDetails: Record<string, unknown> = {
-        completedAt: new Date().toISOString(),
-      };
-      if (error instanceof Error && error.stack !== undefined) {
-        errorDetails['error'] = error.stack;
-      } else {
-        errorDetails['error'] = String(error);
-      }
-      this.jobService.updateJob(jobId, {
-        status: 'failed',
-        message: error instanceof Error ? error.message : 'Unknown error',
-        details: errorDetails,
-      });
-      throw error;
-    }
-  }
-
-  /**
-   * Execute a clone job (git clone + initial indexing)
-   */
-  private async executeCloneJob(job: Job): Promise<void> {
-    const { storeId } = job.details;
-
-    if (storeId === undefined || typeof storeId !== 'string') {
-      throw new Error('Store ID required for clone job');
-    }
-
-    // Get the store
-    const store = await this.storeService.get(createStoreId(storeId));
-    if (!store) {
-      throw new Error(`Store ${storeId} not found`);
-    }
-
-    // Clone is already done by the time the job is created
-    // (happens in StoreService.create), so we just need to index
-
-    // Update progress - cloning considered done (30%)
-    this.jobService.updateJob(job.id, {
-      status: 'running',
-      message: 'Repository cloned, starting indexing...',
-      progress: 30,
-    });
-
-    // Index the repository with progress updates
-    const result = await this.indexService.indexStore(
-      store,
-      (event: { type: string; current: number; total: number; message: string }) => {
-        // Check if job was cancelled
-        const currentJob = this.jobService.getJob(job.id);
-        if (currentJob?.status === 'cancelled') {
-          throw new Error('Job cancelled by user');
-        }
-
-        // Indexing is 70% of total progress (30-100%)
-        const indexProgress = calculateIndexProgress(event.current, event.total, 70);
-        const totalProgress = 30 + indexProgress;
-
-        this.jobService.updateJob(job.id, {
-          message: `Indexed ${String(event.current)}/${String(event.total)} files`,
-          progress: Math.min(99, totalProgress), // Cap at 99 until fully complete
-          details: {
-            filesProcessed: event.current,
-            totalFiles: event.total,
-          },
-        });
-      }
-    );
-
-    if (!result.success) {
-      throw result.error;
-    }
-  }
-
-  /**
-   * Execute an index job (re-indexing existing store)
-   */
-  private async executeIndexJob(job: Job): Promise<void> {
-    const { storeId } = job.details;
-
-    if (storeId === undefined || typeof storeId !== 'string') {
-      throw new Error('Store ID required for index job');
-    }
-
-    // Get the store
-    const store = await this.storeService.getByIdOrName(createStoreId(storeId));
-    if (!store) {
-      throw new Error(`Store ${storeId} not found`);
-    }
-
-    // Index with progress updates
-    const result = await this.indexService.indexStore(
-      store,
-      (event: { type: string; current: number; total: number; message: string }) => {
-        // Check if job was cancelled
-        const currentJob = this.jobService.getJob(job.id);
-        if (currentJob?.status === 'cancelled') {
-          throw new Error('Job cancelled by user');
-        }
-
-        const progress = calculateIndexProgress(event.current, event.total);
-
-        this.jobService.updateJob(job.id, {
-          message: `Indexed ${String(event.current)}/${String(event.total)} files`,
-          progress: Math.min(99, progress), // Cap at 99 until fully complete
-          details: {
-            filesProcessed: event.current,
-            totalFiles: event.total,
-          },
-        });
-      }
-    );
-
-    if (!result.success) {
-      throw result.error;
-    }
-  }
-
-  /**
-   * Execute a crawl job (web crawling + indexing)
-   */
-  private async executeCrawlJob(job: Job): Promise<void> {
-    const { storeId, url, crawlInstruction, extractInstruction, maxPages, simple, useHeadless } =
-      job.details;
-
-    if (storeId === undefined || typeof storeId !== 'string') {
-      throw new Error('Store ID required for crawl job');
-    }
-    if (url === undefined || typeof url !== 'string') {
-      throw new Error('URL required for crawl job');
-    }
-
-    // Get the store
-    const store = await this.storeService.get(createStoreId(storeId));
-    if (store?.type !== 'web') {
-      throw new Error(`Web store ${storeId} not found`);
-    }
-
-    const resolvedMaxPages = typeof maxPages === 'number' ? maxPages : 50;
-    const crawler = new IntelligentCrawler();
-
-    // Listen for progress events
-    crawler.on('progress', (progress: CrawlProgress) => {
-      // Check if job was cancelled - just return early, for-await loop will throw and finally will cleanup
-      const currentJob = this.jobService.getJob(job.id);
-      if (currentJob?.status === 'cancelled') {
-        return;
-      }
-
-      // Crawling is 80% of total progress (0-80%)
-      const crawlProgress = (progress.pagesVisited / resolvedMaxPages) * 80;
-
-      this.jobService.updateJob(job.id, {
-        message:
-          progress.message ??
-          `Crawling page ${String(progress.pagesVisited)}/${String(resolvedMaxPages)}`,
-        progress: Math.min(80, crawlProgress),
-        details: { pagesCrawled: progress.pagesVisited },
-      });
-    });
-
-    try {
-      await this.lanceStore.initialize(store.id);
-      const docs: Document[] = [];
-
-      // Build crawl options, only including defined values
-      const crawlOptions: {
-        maxPages: number;
-        simple: boolean;
-        useHeadless: boolean;
-        crawlInstruction?: string;
-        extractInstruction?: string;
-      } = {
-        maxPages: resolvedMaxPages,
-        simple: simple ?? false,
-        useHeadless: useHeadless ?? true, // Default to headless for reliability
-      };
-      if (crawlInstruction !== undefined) {
-        crawlOptions.crawlInstruction = crawlInstruction;
-      }
-      if (extractInstruction !== undefined) {
-        crawlOptions.extractInstruction = extractInstruction;
-      }
-
-      // Crawl pages using IntelligentCrawler
-      for await (const result of crawler.crawl(url, crawlOptions)) {
-        // Check cancellation between pages
-        const currentJob = this.jobService.getJob(job.id);
-        if (currentJob?.status === 'cancelled') {
-          throw new Error('Job cancelled by user');
-        }
-
-        // Embed and index the content (use extracted if available, otherwise markdown)
-        const contentToEmbed = result.extracted ?? result.markdown;
-        const vector = await this.embeddingEngine.embed(contentToEmbed);
-
-        docs.push({
-          id: createDocumentId(`${store.id}-${createHash('md5').update(result.url).digest('hex')}`),
-          content: contentToEmbed,
-          vector,
-          metadata: {
-            type: 'web',
-            storeId: store.id,
-            url: result.url,
-            title: result.title,
-            extracted: result.extracted !== undefined,
-            depth: result.depth,
-            indexedAt: new Date(),
-          },
-        });
-      }
-
-      // Index all documents (remaining 20%)
-      if (docs.length > 0) {
-        this.jobService.updateJob(job.id, {
-          message: 'Indexing crawled documents...',
-          progress: 85,
-        });
-
-        await this.lanceStore.addDocuments(store.id, docs);
-        // Create FTS index for full-text search
-        await this.lanceStore.createFtsIndex(store.id);
-      }
-
-      this.jobService.updateJob(job.id, {
-        message: `Crawled and indexed ${String(docs.length)} pages`,
-        progress: 100,
-        details: { pagesCrawled: docs.length },
-      });
-    } finally {
-      await crawler.stop();
-    }
-  }
-}