bluera-knowledge 0.9.21

package/src/workers/background-worker.ts

@@ -0,0 +1,294 @@
+import { createHash } from 'node:crypto';
+import { JobService } from '../services/job.service.js';
+import { StoreService } from '../services/store.service.js';
+import { IndexService } from '../services/index.service.js';
+import type { LanceStore } from '../db/lance.js';
+import type { EmbeddingEngine } from '../db/embeddings.js';
+import { IntelligentCrawler, type CrawlProgress } from '../crawl/intelligent-crawler.js';
+import type { Job } from '../types/job.js';
+import type { Document } from '../types/document.js';
+import { createStoreId, createDocumentId } from '../types/brands.js';
+
+export class BackgroundWorker {
+  constructor(
+    private readonly jobService: JobService,
+    private readonly storeService: StoreService,
+    private readonly indexService: IndexService,
+    private readonly lanceStore: LanceStore,
+    private readonly embeddingEngine: EmbeddingEngine
+  ) {}
+
+  /**
+   * Execute a job based on its type
+   */
+  async executeJob(jobId: string): Promise<void> {
+    const job = this.jobService.getJob(jobId);
+
+    if (!job) {
+      throw new Error(`Job ${jobId} not found`);
+    }
+
+    try {
+      // Update to running status
+      this.jobService.updateJob(jobId, {
+        status: 'running',
+        message: `Starting ${job.type} operation...`,
+        progress: 0,
+        details: { startedAt: new Date().toISOString() }
+      });
+
+      // Execute based on job type
+      switch (job.type) {
+        case 'clone':
+          await this.executeCloneJob(job);
+          break;
+        case 'index':
+          await this.executeIndexJob(job);
+          break;
+        case 'crawl':
+          await this.executeCrawlJob(job);
+          break;
+        default:
+          throw new Error(`Unknown job type: ${String(job.type)}`);
+      }
+
+      // Mark as completed
+      this.jobService.updateJob(jobId, {
+        status: 'completed',
+        progress: 100,
+        message: `${job.type} operation completed successfully`,
+        details: { completedAt: new Date().toISOString() }
+      });
+    } catch (error) {
+      // Mark as failed
+      const errorDetails: Record<string, unknown> = {
+        completedAt: new Date().toISOString()
+      };
+      if (error instanceof Error && error.stack !== undefined) {
+        errorDetails['error'] = error.stack;
+      } else {
+        errorDetails['error'] = String(error);
+      }
+      this.jobService.updateJob(jobId, {
+        status: 'failed',
+        message: error instanceof Error ? error.message : 'Unknown error',
+        details: errorDetails
+      });
+      throw error;
+    }
+  }
+
+  /**
+   * Execute a clone job (git clone + initial indexing)
+   */
+  private async executeCloneJob(job: Job): Promise<void> {
+    const { storeId } = job.details;
+
+    if (storeId === undefined || typeof storeId !== 'string') {
+      throw new Error('Store ID required for clone job');
+    }
+
+    // Get the store
+    const store = await this.storeService.get(createStoreId(storeId));
+    if (!store) {
+      throw new Error(`Store ${storeId} not found`);
+    }
+
+    // Clone is already done by the time the job is created
+    // (happens in StoreService.create), so we just need to index
+
+    // Update progress - cloning considered done (30%)
+    this.jobService.updateJob(job.id, {
+      status: 'running',
+      message: 'Repository cloned, starting indexing...',
+      progress: 30
+    });
+
+    // Index the repository with progress updates
+    const result = await this.indexService.indexStore(store, (event: { type: string; current: number; total: number; message: string }) => {
+      // Check if job was cancelled
+      const currentJob = this.jobService.getJob(job.id);
+      if (currentJob?.status === 'cancelled') {
+        throw new Error('Job cancelled by user');
+      }
+
+      // Indexing is 70% of total progress (30-100%)
+      const indexProgress = (event.current / event.total) * 70;
+      const totalProgress = 30 + indexProgress;
+
+      this.jobService.updateJob(job.id, {
+        message: `Indexed ${String(event.current)}/${String(event.total)} files`,
+        progress: Math.min(99, totalProgress), // Cap at 99 until fully complete
+        details: {
+          filesProcessed: event.current,
+          totalFiles: event.total
+        }
+      });
+    });
+
+    if (!result.success) {
+      throw result.error;
+    }
+  }
+
+  /**
+   * Execute an index job (re-indexing existing store)
+   */
+  private async executeIndexJob(job: Job): Promise<void> {
+    const { storeId } = job.details;
+
+    if (storeId === undefined || typeof storeId !== 'string') {
+      throw new Error('Store ID required for index job');
+    }
+
+    // Get the store
+    const store = await this.storeService.getByIdOrName(createStoreId(storeId));
+    if (!store) {
+      throw new Error(`Store ${storeId} not found`);
+    }
+
+    // Index with progress updates
+    const result = await this.indexService.indexStore(store, (event: { type: string; current: number; total: number; message: string }) => {
+      // Check if job was cancelled
+      const currentJob = this.jobService.getJob(job.id);
+      if (currentJob?.status === 'cancelled') {
+        throw new Error('Job cancelled by user');
+      }
+
+      const progress = (event.current / event.total) * 100;
+
+      this.jobService.updateJob(job.id, {
+        message: `Indexed ${String(event.current)}/${String(event.total)} files`,
+        progress: Math.min(99, progress), // Cap at 99 until fully complete
+        details: {
+          filesProcessed: event.current,
+          totalFiles: event.total
+        }
+      });
+    });
+
+    if (!result.success) {
+      throw result.error;
+    }
+  }
+
+  /**
+   * Execute a crawl job (web crawling + indexing)
+   */
+  private async executeCrawlJob(job: Job): Promise<void> {
+    const {
+      storeId,
+      url,
+      crawlInstruction,
+      extractInstruction,
+      maxPages,
+      simple,
+      useHeadless,
+    } = job.details;
+
+    if (storeId === undefined || typeof storeId !== 'string') {
+      throw new Error('Store ID required for crawl job');
+    }
+    if (url === undefined || typeof url !== 'string') {
+      throw new Error('URL required for crawl job');
+    }
+
+    // Get the store
+    const store = await this.storeService.get(createStoreId(storeId));
+    if (!store || store.type !== 'web') {
+      throw new Error(`Web store ${storeId} not found`);
+    }
+
+    const resolvedMaxPages = typeof maxPages === 'number' ? maxPages : 50;
+    const crawler = new IntelligentCrawler();
+
+    // Listen for progress events
+    crawler.on('progress', (progress: CrawlProgress) => {
+      // Check if job was cancelled
+      const currentJob = this.jobService.getJob(job.id);
+      if (currentJob?.status === 'cancelled') {
+        void crawler.stop();
+        return;
+      }
+
+      // Crawling is 80% of total progress (0-80%)
+      const crawlProgress = (progress.pagesVisited / resolvedMaxPages) * 80;
+
+      this.jobService.updateJob(job.id, {
+        message: progress.message ?? `Crawling page ${String(progress.pagesVisited)}/${String(resolvedMaxPages)}`,
+        progress: Math.min(80, crawlProgress),
+        details: { pagesCrawled: progress.pagesVisited }
+      });
+    });
+
+    try {
+      await this.lanceStore.initialize(store.id);
+      const docs: Document[] = [];
+
+      // Build crawl options, only including defined values
+      const crawlOptions: {
+        maxPages: number;
+        simple: boolean;
+        useHeadless: boolean;
+        crawlInstruction?: string;
+        extractInstruction?: string;
+      } = {
+        maxPages: resolvedMaxPages,
+        simple: simple ?? false,
+        useHeadless: useHeadless ?? false,
+      };
+      if (crawlInstruction !== undefined) {
+        crawlOptions.crawlInstruction = crawlInstruction;
+      }
+      if (extractInstruction !== undefined) {
+        crawlOptions.extractInstruction = extractInstruction;
+      }
+
+      // Crawl pages using IntelligentCrawler
+      for await (const result of crawler.crawl(url, crawlOptions)) {
+        // Check cancellation between pages
+        const currentJob = this.jobService.getJob(job.id);
+        if (currentJob?.status === 'cancelled') {
+          throw new Error('Job cancelled by user');
+        }
+
+        // Embed and index the content (use extracted if available, otherwise markdown)
+        const contentToEmbed = result.extracted ?? result.markdown;
+        const vector = await this.embeddingEngine.embed(contentToEmbed);
+
+        docs.push({
+          id: createDocumentId(`${store.id}-${createHash('md5').update(result.url).digest('hex')}`),
+          content: contentToEmbed,
+          vector,
+          metadata: {
+            type: 'web',
+            storeId: store.id,
+            url: result.url,
+            title: result.title,
+            extracted: result.extracted !== undefined,
+            depth: result.depth,
+            indexedAt: new Date(),
+          },
+        });
+      }
+
+      // Index all documents (remaining 20%)
+      if (docs.length > 0) {
+        this.jobService.updateJob(job.id, {
+          message: 'Indexing crawled documents...',
+          progress: 85
+        });
+
+        await this.lanceStore.addDocuments(store.id, docs);
+      }
+
+      this.jobService.updateJob(job.id, {
+        message: `Crawled and indexed ${String(docs.length)} pages`,
+        progress: 100,
+        details: { pagesCrawled: docs.length }
+      });
+    } finally {
+      await crawler.stop();
+    }
+  }
+}

package/src/workers/spawn-worker.test.ts

@@ -0,0 +1,128 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+
+// Mock child_process before importing spawn-worker
+const mockUnref = vi.fn();
+const mockSpawn = vi.fn(() => ({
+  unref: mockUnref
+}));
+
+vi.mock('child_process', () => ({
+  spawn: mockSpawn
+}));
+
+// Import after mocking
+const { spawnBackgroundWorker } = await import('./spawn-worker.js');
+
+describe('spawnBackgroundWorker', () => {
+  beforeEach(() => {
+    mockSpawn.mockClear();
+    mockUnref.mockClear();
+  });
+
+  it('should use tsx in development mode (src folder)', () => {
+    spawnBackgroundWorker('test-job', '/test/data');
+
+    const callArgs = mockSpawn.mock.calls[0];
+    const command = callArgs?.[0];
+    const args = callArgs?.[1];
+
+    // In development (src folder), should use npx tsx
+    expect(command).toBe('npx');
+    expect(args?.[0]).toBe('tsx');
+  });
+
+  it('should spawn a background worker process', () => {
+    spawnBackgroundWorker('test-job-id', '/test/data/dir');
+
+    expect(mockSpawn).toHaveBeenCalled();
+    expect(mockUnref).toHaveBeenCalled();
+  });
+
+  it('should pass job ID as argument', () => {
+    spawnBackgroundWorker('my-job-123', '/test/data/dir');
+
+    const callArgs = mockSpawn.mock.calls[0];
+    const args = callArgs?.[1];
+
+    expect(args).toContain('my-job-123');
+  });
+
+  it('should spawn detached process with ignored stdio', () => {
+    spawnBackgroundWorker('test-job', '/test/data/dir');
+
+    const callArgs = mockSpawn.mock.calls[0];
+    const options = callArgs?.[2];
+
+    expect(options).toMatchObject({
+      detached: true,
+      stdio: 'ignore'
+    });
+  });
+
+  it('should pass environment variables including BLUERA_DATA_DIR', () => {
+    const dataDir = '/custom/data/directory';
+    spawnBackgroundWorker('test-job', dataDir);
+
+    const callArgs = mockSpawn.mock.calls[0];
+    const options = callArgs?.[2];
+
+    expect(options?.env).toMatchObject({
+      ...process.env,
+      BLUERA_DATA_DIR: dataDir
+    });
+  });
+
+  it('should pass dataDir via BLUERA_DATA_DIR environment variable', () => {
+    const testDataDir = '.bluera/bluera-knowledge/data';
+    spawnBackgroundWorker('job-456', testDataDir);
+
+    const callArgs = mockSpawn.mock.calls[0];
+    const options = callArgs?.[2];
+
+    expect(options?.env?.BLUERA_DATA_DIR).toBe(testDataDir);
+  });
+});
+
+// Test production mode with separate import to get fresh module
+describe('spawnBackgroundWorker (production mode)', () => {
+  const mockUnrefProd = vi.fn();
+  const mockSpawnProd = vi.fn(() => ({
+    unref: mockUnrefProd
+  }));
+
+  beforeEach(() => {
+    vi.resetModules();
+    mockSpawnProd.mockClear();
+    mockUnrefProd.mockClear();
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  it('should use Node.js directly in production mode (dist folder)', async () => {
+    // Mock child_process
+    vi.doMock('child_process', () => ({
+      spawn: mockSpawnProd
+    }));
+
+    // Mock url module to return a path containing /dist/
+    vi.doMock('url', () => ({
+      fileURLToPath: () => '/app/dist/workers/spawn-worker.js'
+    }));
+
+    // Import fresh module with production path
+    const { spawnBackgroundWorker: spawnProd } = await import('./spawn-worker.js');
+
+    spawnProd('test-job', '/test/data');
+
+    const callArgs = mockSpawnProd.mock.calls[0];
+    const command = callArgs?.[0];
+    const args = callArgs?.[1];
+
+    // In production (dist folder), should use Node.js directly
+    expect(command).toBe(process.execPath);
+    expect(args?.[0]).toContain('background-worker-cli.js');
+    expect(args?.[1]).toBe('test-job');
+  });
+});

package/src/workers/spawn-worker.ts

@@ -0,0 +1,49 @@
+import { spawn } from 'child_process';
+import { fileURLToPath } from 'url';
+import path from 'path';
+
+/**
+ * Spawn a background worker process to execute a job
+ *
+ * The worker runs detached from the parent process, allowing the
+ * parent to exit while the worker continues running.
+ *
+ * @param jobId - The ID of the job to execute
+ */
+export function spawnBackgroundWorker(jobId: string, dataDir: string): void {
+  // Determine the worker script path
+  // In production, this will be the compiled dist file
+  // In development, we need to use tsx to run TypeScript
+  const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+  // Check if we're running from dist (production) or src (development)
+  const isProduction = __dirname.includes('/dist/');
+
+  let command: string;
+  let args: string[];
+
+  if (isProduction) {
+    // Production: Use Node.js directly with compiled file
+    const workerScript = path.join(__dirname, 'background-worker-cli.js');
+    command = process.execPath; // Use the same Node.js binary
+    args = [workerScript, jobId];
+  } else {
+    // Development: Use tsx to run TypeScript directly
+    const workerScript = path.join(__dirname, 'background-worker-cli.ts');
+    command = 'npx';
+    args = ['tsx', workerScript, jobId];
+  }
+
+  // Spawn the worker process
+  const worker = spawn(command, args, {
+    detached: true,      // Detach from parent process
+    stdio: 'ignore',     // Don't pipe stdio (fully independent)
+    env: {
+      ...process.env,    // Inherit environment variables
+      BLUERA_DATA_DIR: dataDir  // Pass dataDir to worker
+    }
+  });
+
+  // Unref the worker so the parent can exit
+  worker.unref();
+}

package/tests/analysis/ast-parser.test.ts

@@ -0,0 +1,98 @@
+import { describe, it, expect } from 'vitest';
+import { ASTParser } from '../../src/analysis/ast-parser.js';
+
+describe('ASTParser', () => {
+  it('should extract functions from TypeScript code', () => {
+    const code = `
+export async function validateToken(token: string): Promise<boolean> {
+  return token.length > 0;
+}
+
+function helperFunction() {
+  return true;
+}
+`;
+
+    const parser = new ASTParser();
+    const nodes = parser.parse(code, 'typescript');
+
+    expect(nodes).toHaveLength(2);
+    expect(nodes[0]?.type).toBe('function');
+    expect(nodes[0]?.name).toBe('validateToken');
+    expect(nodes[0]?.exported).toBe(true);
+    expect(nodes[0]?.async).toBe(true);
+    expect(nodes[1]?.name).toBe('helperFunction');
+    expect(nodes[1]?.exported).toBe(false);
+  });
+
+  it('should extract classes with methods', () => {
+    const code = `
+export class UserService {
+  constructor(private repo: UserRepo) {}
+
+  async create(data: CreateUserData): Promise<User> {
+    return this.repo.save(data);
+  }
+
+  delete(id: string): void {
+    this.repo.delete(id);
+  }
+}
+`;
+
+    const parser = new ASTParser();
+    const nodes = parser.parse(code, 'typescript');
+
+    const classNode = nodes.find(n => n.type === 'class');
+    expect(classNode).toBeDefined();
+    expect(classNode?.name).toBe('UserService');
+    expect(classNode?.methods).toHaveLength(3); // constructor + create + delete
+  });
+
+  it('should extract imports', () => {
+    const code = `
+import { User } from './models/user.js';
+import type { Repository } from '../types.js';
+import express from 'express';
+`;
+
+    const parser = new ASTParser();
+    const imports = parser.extractImports(code);
+
+    expect(imports).toHaveLength(3);
+    expect(imports[0]?.source).toBe('./models/user.js');
+    expect(imports[0]?.specifiers).toContain('User');
+    expect(imports[2]?.specifiers).toContain('express');
+  });
+
+  it('should extract correct line numbers', () => {
+    const code = `
+export function foo() {
+  return true;
+}
+
+class Bar {
+  method() {}
+}`;
+
+    const parser = new ASTParser();
+    const nodes = parser.parse(code, 'typescript');
+
+    const fooNode = nodes.find(n => n.name === 'foo');
+    expect(fooNode?.startLine).toBe(2);
+    expect(fooNode?.endLine).toBe(4);
+
+    const barNode = nodes.find(n => n.name === 'Bar');
+    expect(barNode?.startLine).toBe(6);
+    expect(barNode?.endLine).toBe(8);
+  });
+
+  it('should handle malformed code gracefully', () => {
+    const code = 'function broken(';
+
+    const parser = new ASTParser();
+    const nodes = parser.parse(code, 'typescript');
+
+    expect(nodes).toEqual([]);
+  });
+});

package/tests/analysis/code-graph.test.ts

@@ -0,0 +1,60 @@
+import { describe, it, expect } from 'vitest';
+import { CodeGraph } from '../../src/analysis/code-graph.js';
+import type { CodeNode } from '../../src/analysis/ast-parser.js';
+
+describe('CodeGraph', () => {
+  it('should build graph from code nodes', () => {
+    const nodes: CodeNode[] = [
+      {
+        type: 'function',
+        name: 'validateToken',
+        exported: true,
+        startLine: 1,
+        endLine: 5
+      },
+      {
+        type: 'function',
+        name: 'parseToken',
+        exported: false,
+        startLine: 7,
+        endLine: 10
+      }
+    ];
+
+    const graph = new CodeGraph();
+    graph.addNodes(nodes, 'src/auth.ts');
+
+    expect(graph.getNode('src/auth.ts:validateToken')).toBeDefined();
+    expect(graph.getNode('src/auth.ts:parseToken')).toBeDefined();
+  });
+
+  it('should track import relationships', () => {
+    const graph = new CodeGraph();
+
+    graph.addImport('src/controllers/user.ts', './services/user.js', ['UserService']);
+
+    const edges = graph.getEdges('src/controllers/user.ts');
+    expect(edges).toHaveLength(1);
+    expect(edges[0]?.type).toBe('imports');
+    expect(edges[0]?.to).toContain('UserService');
+  });
+
+  it('should detect call relationships from code', () => {
+    const code = `
+export function handler(req) {
+  const valid = validateToken(req.token);
+  if (!valid) return error();
+  return success();
+}
+`;
+
+    const graph = new CodeGraph();
+    graph.analyzeCallRelationships(code, 'src/handler.ts', 'handler');
+
+    const edges = graph.getEdges('src/handler.ts:handler');
+    const callEdges = edges.filter(e => e.type === 'calls');
+
+    expect(callEdges.length).toBeGreaterThan(0);
+    expect(callEdges.some(e => e.to.includes('validateToken'))).toBe(true);
+  });
+});