bluera-knowledge 0.9.26 → 0.9.31

package/src/crawl/intelligent-crawler.ts

@@ -8,6 +8,9 @@ import axios from 'axios';
 import { ClaudeClient, type CrawlStrategy } from './claude-client.js';
 import { convertHtmlToMarkdown } from './article-converter.js';
 import { PythonBridge, type CrawledLink } from './bridge.js';
+import { createLogger, summarizePayload } from '../logging/index.js';
+
+const logger = createLogger('crawler');
 
 export interface CrawlOptions {
   crawlInstruction?: string; // Natural language: what to crawl
@@ -69,6 +72,13 @@ export class IntelligentCrawler extends EventEmitter {
     this.visited.clear();
     this.stopped = false;
 
+    logger.info({
+      seedUrl,
+      maxPages,
+      mode: simple ? 'simple' : (crawlInstruction !== undefined && crawlInstruction !== '' ? 'intelligent' : 'simple'),
+      hasExtractInstruction: extractInstruction !== undefined,
+    }, 'Starting crawl');
+
     const startProgress: CrawlProgress = {
       type: 'start',
       pagesVisited: 0,
@@ -86,6 +96,11 @@ export class IntelligentCrawler extends EventEmitter {
       yield* this.crawlSimple(seedUrl, extractInstruction, maxPages, options.useHeadless ?? false);
     }
 
+    logger.info({
+      seedUrl,
+      pagesVisited: this.visited.size,
+    }, 'Crawl complete');
+
     const completeProgress: CrawlProgress = {
       type: 'complete',
       pagesVisited: this.visited.size,
@@ -220,9 +235,9 @@ export class IntelligentCrawler extends EventEmitter {
             const links = await this.extractLinks(current.url, useHeadless);
 
             if (links.length === 0) {
-              console.warn(`No links found on ${current.url} - page may be a leaf node`);
+              logger.debug({ url: current.url }, 'No links found - page may be a leaf node');
             } else {
-              console.log(`Found ${String(links.length)} links on ${current.url}`);
+              logger.debug({ url: current.url, linkCount: links.length }, 'Links extracted from page');
             }
 
             for (const link of links) {
@@ -283,9 +298,16 @@ export class IntelligentCrawler extends EventEmitter {
     const conversion = await convertHtmlToMarkdown(html, url);
 
     if (!conversion.success) {
+      logger.error({ url, error: conversion.error }, 'HTML to markdown conversion failed');
       throw new Error(`Failed to convert HTML: ${conversion.error ?? 'Unknown error'}`);
     }
 
+    logger.debug({
+      url,
+      title: conversion.title,
+      markdownLength: conversion.markdown.length,
+    }, 'Article converted to markdown');
+
     let extracted: string | undefined;
 
     // Optional: Extract specific information using Claude
@@ -342,13 +364,23 @@ export class IntelligentCrawler extends EventEmitter {
    * Fetch HTML content from a URL
    */
   private async fetchHtml(url: string, useHeadless: boolean = false): Promise<string> {
+    const startTime = Date.now();
+    logger.debug({ url, useHeadless }, 'Fetching HTML');
+
     if (useHeadless) {
       try {
         const result = await this.pythonBridge.fetchHeadless(url);
+        const durationMs = Date.now() - startTime;
+        logger.info({
+          url,
+          useHeadless: true,
+          durationMs,
+          ...summarizePayload(result.html, 'raw-html', url),
+        }, 'Raw HTML fetched');
         return result.html;
       } catch (error) {
         // Fallback to axios if headless fails
-        console.warn(`Headless fetch failed for ${url}, falling back to axios:`, error);
+        logger.warn({ url, error: error instanceof Error ? error.message : String(error) }, 'Headless fetch failed, falling back to axios');
       }
     }
 
@@ -362,8 +394,17 @@ export class IntelligentCrawler extends EventEmitter {
         },
       });
 
+      const durationMs = Date.now() - startTime;
+      logger.info({
+        url,
+        useHeadless: false,
+        durationMs,
+        ...summarizePayload(response.data, 'raw-html', url),
+      }, 'Raw HTML fetched');
+
       return response.data;
     } catch (error) {
+      logger.error({ url, error: error instanceof Error ? error.message : String(error) }, 'Failed to fetch HTML');
       throw new Error(
         `Failed to fetch ${url}: ${error instanceof Error ? error.message : String(error)}`,
       );
@@ -398,7 +439,7 @@ export class IntelligentCrawler extends EventEmitter {
     } catch (error: unknown) {
       // Log the error for debugging
       const errorMessage = error instanceof Error ? error.message : String(error);
-      console.error(`Failed to extract links from ${url}:`, errorMessage);
+      logger.error({ url, error: errorMessage }, 'Failed to extract links');
 
       // Re-throw the error instead of silently swallowing it
       throw new Error(`Link extraction failed for ${url}: ${errorMessage}`);

package/src/db/embeddings.test.ts

@@ -48,6 +48,22 @@ describe('EmbeddingEngine', () => {
     const customEngine = new EmbeddingEngine('Xenova/all-MiniLM-L6-v2', 512);
     expect(customEngine.getDimensions()).toBe(512);
   });
+
+  it('skips initialization when already initialized', async () => {
+    // Engine is already initialized from beforeAll
+    // Calling initialize again should be a no-op
+    await engine.initialize();
+    const embedding = await engine.embed('test');
+    expect(embedding).toHaveLength(384);
+  });
+
+  it('handles large batch with multiple chunks', async () => {
+    // Create enough texts to trigger multiple batch iterations (BATCH_SIZE = 32)
+    const texts = Array.from({ length: 40 }, (_, i) => `Text number ${String(i)}`);
+    const embeddings = await engine.embedBatch(texts);
+    expect(embeddings).toHaveLength(40);
+    expect(embeddings.every((e) => e.length === 384)).toBe(true);
+  }, 60000);
 });
 
 function cosineSimilarity(a: number[], b: number[]): number {

package/src/db/lance.test.ts

@@ -292,6 +292,37 @@ describe('LanceStore', () => {
     });
   });
 
+  describe('close', () => {
+    it('clears tables and connection', async () => {
+      const closeStoreId = createStoreId('close-test-store');
+      const closeStore = new LanceStore(tempDir);
+      await closeStore.initialize(closeStoreId);
+
+      const doc = {
+        id: createDocumentId('close-doc'),
+        content: 'test',
+        vector: new Array(384).fill(0.1),
+        metadata: {
+          type: 'file' as const,
+          storeId: closeStoreId,
+          indexedAt: new Date(),
+        },
+      };
+
+      await closeStore.addDocuments(closeStoreId, [doc]);
+
+      // Close should not throw
+      expect(() => closeStore.close()).not.toThrow();
+    });
+
+    it('handles close when never initialized', () => {
+      const uninitializedStore = new LanceStore(tempDir);
+
+      // Should not throw even when never initialized
+      expect(() => uninitializedStore.close()).not.toThrow();
+    });
+  });
+
   describe('multiple documents operations', () => {
     it('adds multiple documents at once', async () => {
       const multiStoreId = createStoreId('multi-doc-store');

package/src/db/lance.ts

@@ -145,6 +145,14 @@ export class LanceStore {
     }
   }
 
+  close(): void {
+    this.tables.clear();
+    if (this.connection !== null) {
+      this.connection.close();
+      this.connection = null;
+    }
+  }
+
   private getTableName(storeId: StoreId): string {
     return `documents_${storeId}`;
   }

package/src/logging/index.ts

@@ -0,0 +1,29 @@
+/**
+ * Logging module - pino-based file logging with auto-rotation
+ *
+ * @example
+ * import { createLogger, summarizePayload } from './logging/index.js';
+ *
+ * const logger = createLogger('my-module');
+ * logger.info({ data }, 'Something happened');
+ *
+ * // For large payloads:
+ * logger.info({
+ *   ...summarizePayload(html, 'raw-html', url),
+ * }, 'Fetched HTML');
+ */
+
+export {
+  createLogger,
+  shutdownLogger,
+  getCurrentLogLevel,
+  isLevelEnabled,
+  getLogDirectory,
+  type LogLevel,
+} from './logger.js';
+
+export {
+  summarizePayload,
+  truncateForLog,
+  type PayloadSummary,
+} from './payload.js';

package/src/logging/logger.test.ts

@@ -0,0 +1,75 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { getCurrentLogLevel, isLevelEnabled, getLogDirectory } from './logger.js';
+
+describe('logger', () => {
+  const originalEnv = process.env['LOG_LEVEL'];
+
+  beforeEach(() => {
+    delete process.env['LOG_LEVEL'];
+  });
+
+  afterEach(() => {
+    if (originalEnv !== undefined) {
+      process.env['LOG_LEVEL'] = originalEnv;
+    } else {
+      delete process.env['LOG_LEVEL'];
+    }
+  });
+
+  describe('getCurrentLogLevel', () => {
+    it('returns info as default level', () => {
+      expect(getCurrentLogLevel()).toBe('info');
+    });
+
+    it('returns level from environment variable', () => {
+      process.env['LOG_LEVEL'] = 'debug';
+      expect(getCurrentLogLevel()).toBe('debug');
+    });
+
+    it('handles lowercase environment variable', () => {
+      process.env['LOG_LEVEL'] = 'WARN';
+      expect(getCurrentLogLevel()).toBe('warn');
+    });
+
+    it('treats empty string as default', () => {
+      process.env['LOG_LEVEL'] = '';
+      expect(getCurrentLogLevel()).toBe('info');
+    });
+
+    it('throws on invalid log level', () => {
+      process.env['LOG_LEVEL'] = 'invalid';
+      expect(() => getCurrentLogLevel()).toThrow('Invalid LOG_LEVEL: "invalid"');
+    });
+  });
+
+  describe('isLevelEnabled', () => {
+    it('returns true when check level is at or above current level', () => {
+      process.env['LOG_LEVEL'] = 'info';
+      expect(isLevelEnabled('info')).toBe(true);
+      expect(isLevelEnabled('warn')).toBe(true);
+      expect(isLevelEnabled('error')).toBe(true);
+    });
+
+    it('returns false when check level is below current level', () => {
+      process.env['LOG_LEVEL'] = 'warn';
+      expect(isLevelEnabled('debug')).toBe(false);
+      expect(isLevelEnabled('info')).toBe(false);
+    });
+
+    it('enables all levels when set to trace', () => {
+      process.env['LOG_LEVEL'] = 'trace';
+      expect(isLevelEnabled('trace')).toBe(true);
+      expect(isLevelEnabled('debug')).toBe(true);
+      expect(isLevelEnabled('info')).toBe(true);
+    });
+  });
+
+  describe('getLogDirectory', () => {
+    it('returns path under home directory', () => {
+      const logDir = getLogDirectory();
+      expect(logDir).toContain('.bluera');
+      expect(logDir).toContain('bluera-knowledge');
+      expect(logDir).toContain('logs');
+    });
+  });
+});

package/src/logging/logger.ts

@@ -0,0 +1,147 @@
+/**
+ * Core logger factory using pino with file-based rolling logs
+ *
+ * Features:
+ * - File-only output (no console pollution for Claude Code)
+ * - Size-based rotation (10MB, keeps 5 files)
+ * - LOG_LEVEL env var control (trace/debug/info/warn/error/fatal)
+ * - Child loggers per module for context
+ */
+
+import pino, { type Logger, type LoggerOptions } from 'pino';
+import { homedir } from 'node:os';
+import { mkdirSync, existsSync } from 'node:fs';
+import { join } from 'node:path';
+
+/** Valid log levels */
+export type LogLevel = 'trace' | 'debug' | 'info' | 'warn' | 'error' | 'fatal';
+
+const VALID_LEVELS: readonly LogLevel[] = ['trace', 'debug', 'info', 'warn', 'error', 'fatal'];
+const VALID_LEVELS_SET: ReadonlySet<string> = new Set(VALID_LEVELS);
+
+/** Default log directory under user home */
+function getLogDir(): string {
+  return join(homedir(), '.bluera', 'bluera-knowledge', 'logs');
+}
+
+/** Resolve and create log directory - fails fast if cannot create */
+function ensureLogDir(): string {
+  const logDir = getLogDir();
+  if (!existsSync(logDir)) {
+    mkdirSync(logDir, { recursive: true });
+  }
+  return logDir;
+}
+
+/** Check if a string is a valid log level */
+function isValidLogLevel(level: string): level is LogLevel {
+  return VALID_LEVELS_SET.has(level);
+}
+
+/** Get log level from environment - fails fast on invalid value */
+function getLogLevel(): LogLevel {
+  const level = process.env['LOG_LEVEL']?.toLowerCase();
+
+  if (level === undefined || level === '') {
+    return 'info';
+  }
+
+  if (!isValidLogLevel(level)) {
+    throw new Error(
+      `Invalid LOG_LEVEL: "${level}". Valid values: ${VALID_LEVELS.join(', ')}`
+    );
+  }
+
+  return level;
+}
+
+/** Root logger instance - lazily initialized */
+let rootLogger: Logger | null = null;
+
+/** Initialize the root logger with pino-roll transport */
+function initializeLogger(): Logger {
+  if (rootLogger !== null) {
+    return rootLogger;
+  }
+
+  const logDir = ensureLogDir();
+  const logFile = join(logDir, 'app.log');
+  const level = getLogLevel();
+
+  const options: LoggerOptions = {
+    level,
+    timestamp: pino.stdTimeFunctions.isoTime,
+    formatters: {
+      level: (label) => ({ level: label }),
+    },
+    transport: {
+      target: 'pino-roll',
+      options: {
+        file: logFile,
+        size: '10m', // 10MB rotation
+        limit: { count: 5 }, // Keep 5 rotated files
+        mkdir: true,
+      },
+    },
+  };
+
+  rootLogger = pino(options);
+  return rootLogger;
+}
+
+/**
+ * Create a named child logger for a specific module
+ *
+ * @param module - Module name (e.g., 'crawler', 'mcp-server', 'search-service')
+ * @returns Logger instance with module context
+ *
+ * @example
+ * const logger = createLogger('crawler');
+ * logger.info({ url }, 'Fetching page');
+ */
+export function createLogger(module: string): Logger {
+  const root = initializeLogger();
+  return root.child({ module });
+}
+
+/**
+ * Get the current log level
+ */
+export function getCurrentLogLevel(): LogLevel {
+  return getLogLevel();
+}
+
+/**
+ * Check if a specific log level is enabled
+ */
+export function isLevelEnabled(level: LogLevel): boolean {
+  const currentLevel = getLogLevel();
+  const currentIndex = VALID_LEVELS.indexOf(currentLevel);
+  const checkIndex = VALID_LEVELS.indexOf(level);
+  return checkIndex >= currentIndex;
+}
+
+/**
+ * Get the log directory path
+ */
+export function getLogDirectory(): string {
+  return getLogDir();
+}
+
+/**
+ * Flush and shutdown the logger - call before process exit
+ */
+export function shutdownLogger(): Promise<void> {
+  return new Promise((resolve) => {
+    if (rootLogger !== null) {
+      rootLogger.flush();
+      // Give time for async transport to flush
+      setTimeout(() => {
+        rootLogger = null;
+        resolve();
+      }, 100);
+    } else {
+      resolve();
+    }
+  });
+}

package/src/logging/payload.test.ts

@@ -0,0 +1,152 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { mkdtempSync, rmSync, existsSync, readdirSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+
+// Mock logger module before importing payload
+vi.mock('./logger.js', () => ({
+  getLogDirectory: vi.fn(),
+  isLevelEnabled: vi.fn()
+}));
+
+import { summarizePayload, truncateForLog } from './payload.js';
+import { getLogDirectory, isLevelEnabled } from './logger.js';
+
+const mockGetLogDirectory = getLogDirectory as ReturnType<typeof vi.fn>;
+const mockIsLevelEnabled = isLevelEnabled as ReturnType<typeof vi.fn>;
+
+describe('payload utilities', () => {
+  let tempDir: string;
+
+  beforeEach(() => {
+    tempDir = mkdtempSync(join(tmpdir(), 'payload-test-'));
+    mockGetLogDirectory.mockReturnValue(tempDir);
+    mockIsLevelEnabled.mockReturnValue(false);
+  });
+
+  afterEach(() => {
+    rmSync(tempDir, { recursive: true, force: true });
+    vi.clearAllMocks();
+  });
+
+  describe('truncateForLog', () => {
+    it('returns content unchanged when under max length', () => {
+      const content = 'short content';
+      expect(truncateForLog(content, 500)).toBe(content);
+    });
+
+    it('truncates content when over max length', () => {
+      const content = 'a'.repeat(600);
+      const result = truncateForLog(content, 500);
+      expect(result.length).toBe(500 + '... [truncated]'.length);
+      expect(result).toContain('... [truncated]');
+    });
+
+    it('uses default max length of 500', () => {
+      const content = 'a'.repeat(600);
+      const result = truncateForLog(content);
+      expect(result.startsWith('a'.repeat(500))).toBe(true);
+      expect(result).toContain('... [truncated]');
+    });
+
+    it('handles exact max length content', () => {
+      const content = 'a'.repeat(500);
+      expect(truncateForLog(content, 500)).toBe(content);
+    });
+  });
+
+  describe('summarizePayload', () => {
+    it('returns summary with preview, size, and hash', () => {
+      const content = 'test content for summarization';
+      const result = summarizePayload(content, 'test-type', 'test-id');
+
+      expect(result.preview).toBe(content);
+      expect(result.sizeBytes).toBe(Buffer.byteLength(content, 'utf8'));
+      expect(result.hash).toMatch(/^[a-f0-9]{12}$/);
+      expect(result.payloadFile).toBeUndefined();
+    });
+
+    it('truncates preview for large content', () => {
+      const content = 'x'.repeat(1000);
+      const result = summarizePayload(content, 'large', 'large-id');
+
+      expect(result.preview).toContain('... [truncated]');
+      expect(result.preview.length).toBeLessThan(content.length);
+    });
+
+    it('does not dump payload when dumpFull is false', () => {
+      const content = 'x'.repeat(20000); // Above threshold
+      const result = summarizePayload(content, 'type', 'id', false);
+
+      expect(result.payloadFile).toBeUndefined();
+      const payloadDir = join(tempDir, 'payload');
+      expect(existsSync(payloadDir)).toBe(false);
+    });
+
+    it('dumps payload to file when dumpFull is true and above threshold', () => {
+      const content = 'x'.repeat(20000); // Above 10KB threshold
+      const result = summarizePayload(content, 'dump-type', 'dump-id', true);
+
+      expect(result.payloadFile).toBeDefined();
+      expect(result.payloadFile).toContain('dump-type');
+      expect(result.payloadFile).toContain(result.hash);
+
+      const payloadDir = join(tempDir, 'payload');
+      expect(existsSync(payloadDir)).toBe(true);
+
+      const files = readdirSync(payloadDir);
+      expect(files.length).toBe(1);
+      expect(files[0]).toBe(result.payloadFile);
+
+      const fileContent = JSON.parse(readFileSync(join(payloadDir, files[0]), 'utf8'));
+      expect(fileContent.type).toBe('dump-type');
+      expect(fileContent.identifier).toBe('dump-id');
+      expect(fileContent.content).toBe(content);
+      expect(fileContent.sizeBytes).toBe(result.sizeBytes);
+    });
+
+    it('does not dump payload below threshold even with dumpFull true', () => {
+      const content = 'small content'; // Below 10KB threshold
+      const result = summarizePayload(content, 'small-type', 'small-id', true);
+
+      expect(result.payloadFile).toBeUndefined();
+    });
+
+    it('creates payload directory if it does not exist', () => {
+      const content = 'y'.repeat(20000);
+      const payloadDir = join(tempDir, 'payload');
+      expect(existsSync(payloadDir)).toBe(false);
+
+      summarizePayload(content, 'create-dir', 'create-id', true);
+
+      expect(existsSync(payloadDir)).toBe(true);
+    });
+
+    it('sanitizes identifier for filename', () => {
+      const content = 'z'.repeat(20000);
+      const result = summarizePayload(content, 'type', 'https://example.com/path?query=1', true);
+
+      expect(result.payloadFile).toBeDefined();
+      expect(result.payloadFile).not.toContain('://');
+      expect(result.payloadFile).not.toContain('?');
+    });
+
+    it('uses trace level check for dumpFull default', () => {
+      mockIsLevelEnabled.mockReturnValue(true);
+      const content = 'a'.repeat(20000);
+
+      const result = summarizePayload(content, 'trace-type', 'trace-id');
+
+      expect(mockIsLevelEnabled).toHaveBeenCalledWith('trace');
+      expect(result.payloadFile).toBeDefined();
+    });
+
+    it('generates consistent hash for same content', () => {
+      const content = 'consistent content';
+      const result1 = summarizePayload(content, 'type1', 'id1');
+      const result2 = summarizePayload(content, 'type2', 'id2');
+
+      expect(result1.hash).toBe(result2.hash);
+    });
+  });
+});