bluera-knowledge 0.9.26 → 0.9.30

package/src/logging/index.ts

@@ -0,0 +1,29 @@
+/**
+ * Logging module - pino-based file logging with auto-rotation
+ *
+ * @example
+ * import { createLogger, summarizePayload } from './logging/index.js';
+ *
+ * const logger = createLogger('my-module');
+ * logger.info({ data }, 'Something happened');
+ *
+ * // For large payloads:
+ * logger.info({
+ *   ...summarizePayload(html, 'raw-html', url),
+ * }, 'Fetched HTML');
+ */
+
+export {
+  createLogger,
+  shutdownLogger,
+  getCurrentLogLevel,
+  isLevelEnabled,
+  getLogDirectory,
+  type LogLevel,
+} from './logger.js';
+
+export {
+  summarizePayload,
+  truncateForLog,
+  type PayloadSummary,
+} from './payload.js';

package/src/logging/logger.test.ts

@@ -0,0 +1,75 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { getCurrentLogLevel, isLevelEnabled, getLogDirectory } from './logger.js';
+
+describe('logger', () => {
+  const originalEnv = process.env['LOG_LEVEL'];
+
+  beforeEach(() => {
+    delete process.env['LOG_LEVEL'];
+  });
+
+  afterEach(() => {
+    if (originalEnv !== undefined) {
+      process.env['LOG_LEVEL'] = originalEnv;
+    } else {
+      delete process.env['LOG_LEVEL'];
+    }
+  });
+
+  describe('getCurrentLogLevel', () => {
+    it('returns info as default level', () => {
+      expect(getCurrentLogLevel()).toBe('info');
+    });
+
+    it('returns level from environment variable', () => {
+      process.env['LOG_LEVEL'] = 'debug';
+      expect(getCurrentLogLevel()).toBe('debug');
+    });
+
+    it('handles lowercase environment variable', () => {
+      process.env['LOG_LEVEL'] = 'WARN';
+      expect(getCurrentLogLevel()).toBe('warn');
+    });
+
+    it('treats empty string as default', () => {
+      process.env['LOG_LEVEL'] = '';
+      expect(getCurrentLogLevel()).toBe('info');
+    });
+
+    it('throws on invalid log level', () => {
+      process.env['LOG_LEVEL'] = 'invalid';
+      expect(() => getCurrentLogLevel()).toThrow('Invalid LOG_LEVEL: "invalid"');
+    });
+  });
+
+  describe('isLevelEnabled', () => {
+    it('returns true when check level is at or above current level', () => {
+      process.env['LOG_LEVEL'] = 'info';
+      expect(isLevelEnabled('info')).toBe(true);
+      expect(isLevelEnabled('warn')).toBe(true);
+      expect(isLevelEnabled('error')).toBe(true);
+    });
+
+    it('returns false when check level is below current level', () => {
+      process.env['LOG_LEVEL'] = 'warn';
+      expect(isLevelEnabled('debug')).toBe(false);
+      expect(isLevelEnabled('info')).toBe(false);
+    });
+
+    it('enables all levels when set to trace', () => {
+      process.env['LOG_LEVEL'] = 'trace';
+      expect(isLevelEnabled('trace')).toBe(true);
+      expect(isLevelEnabled('debug')).toBe(true);
+      expect(isLevelEnabled('info')).toBe(true);
+    });
+  });
+
+  describe('getLogDirectory', () => {
+    it('returns path under home directory', () => {
+      const logDir = getLogDirectory();
+      expect(logDir).toContain('.bluera');
+      expect(logDir).toContain('bluera-knowledge');
+      expect(logDir).toContain('logs');
+    });
+  });
+});

package/src/logging/logger.ts

@@ -0,0 +1,147 @@
+/**
+ * Core logger factory using pino with file-based rolling logs
+ *
+ * Features:
+ * - File-only output (no console pollution for Claude Code)
+ * - Size-based rotation (10MB, keeps 5 files)
+ * - LOG_LEVEL env var control (trace/debug/info/warn/error/fatal)
+ * - Child loggers per module for context
+ */
+
+import pino, { type Logger, type LoggerOptions } from 'pino';
+import { homedir } from 'node:os';
+import { mkdirSync, existsSync } from 'node:fs';
+import { join } from 'node:path';
+
+/** Valid log levels */
+export type LogLevel = 'trace' | 'debug' | 'info' | 'warn' | 'error' | 'fatal';
+
+const VALID_LEVELS: readonly LogLevel[] = ['trace', 'debug', 'info', 'warn', 'error', 'fatal'];
+const VALID_LEVELS_SET: ReadonlySet<string> = new Set(VALID_LEVELS);
+
+/** Default log directory under user home */
+function getLogDir(): string {
+  return join(homedir(), '.bluera', 'bluera-knowledge', 'logs');
+}
+
+/** Resolve and create log directory - fails fast if cannot create */
+function ensureLogDir(): string {
+  const logDir = getLogDir();
+  if (!existsSync(logDir)) {
+    mkdirSync(logDir, { recursive: true });
+  }
+  return logDir;
+}
+
+/** Check if a string is a valid log level */
+function isValidLogLevel(level: string): level is LogLevel {
+  return VALID_LEVELS_SET.has(level);
+}
+
+/** Get log level from environment - fails fast on invalid value */
+function getLogLevel(): LogLevel {
+  const level = process.env['LOG_LEVEL']?.toLowerCase();
+
+  if (level === undefined || level === '') {
+    return 'info';
+  }
+
+  if (!isValidLogLevel(level)) {
+    throw new Error(
+      `Invalid LOG_LEVEL: "${level}". Valid values: ${VALID_LEVELS.join(', ')}`
+    );
+  }
+
+  return level;
+}
+
+/** Root logger instance - lazily initialized */
+let rootLogger: Logger | null = null;
+
+/** Initialize the root logger with pino-roll transport */
+function initializeLogger(): Logger {
+  if (rootLogger !== null) {
+    return rootLogger;
+  }
+
+  const logDir = ensureLogDir();
+  const logFile = join(logDir, 'app.log');
+  const level = getLogLevel();
+
+  const options: LoggerOptions = {
+    level,
+    timestamp: pino.stdTimeFunctions.isoTime,
+    formatters: {
+      level: (label) => ({ level: label }),
+    },
+    transport: {
+      target: 'pino-roll',
+      options: {
+        file: logFile,
+        size: '10m', // 10MB rotation
+        limit: { count: 5 }, // Keep 5 rotated files
+        mkdir: true,
+      },
+    },
+  };
+
+  rootLogger = pino(options);
+  return rootLogger;
+}
+
+/**
+ * Create a named child logger for a specific module
+ *
+ * @param module - Module name (e.g., 'crawler', 'mcp-server', 'search-service')
+ * @returns Logger instance with module context
+ *
+ * @example
+ * const logger = createLogger('crawler');
+ * logger.info({ url }, 'Fetching page');
+ */
+export function createLogger(module: string): Logger {
+  const root = initializeLogger();
+  return root.child({ module });
+}
+
+/**
+ * Get the current log level
+ */
+export function getCurrentLogLevel(): LogLevel {
+  return getLogLevel();
+}
+
+/**
+ * Check if a specific log level is enabled
+ */
+export function isLevelEnabled(level: LogLevel): boolean {
+  const currentLevel = getLogLevel();
+  const currentIndex = VALID_LEVELS.indexOf(currentLevel);
+  const checkIndex = VALID_LEVELS.indexOf(level);
+  return checkIndex >= currentIndex;
+}
+
+/**
+ * Get the log directory path
+ */
+export function getLogDirectory(): string {
+  return getLogDir();
+}
+
+/**
+ * Flush and shutdown the logger - call before process exit
+ */
+export function shutdownLogger(): Promise<void> {
+  return new Promise((resolve) => {
+    if (rootLogger !== null) {
+      rootLogger.flush();
+      // Give time for async transport to flush
+      setTimeout(() => {
+        rootLogger = null;
+        resolve();
+      }, 100);
+    } else {
+      resolve();
+    }
+  });
+}

package/src/logging/payload.test.ts

@@ -0,0 +1,152 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { mkdtempSync, rmSync, existsSync, readdirSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+
+// Mock logger module before importing payload
+vi.mock('./logger.js', () => ({
+  getLogDirectory: vi.fn(),
+  isLevelEnabled: vi.fn()
+}));
+
+import { summarizePayload, truncateForLog } from './payload.js';
+import { getLogDirectory, isLevelEnabled } from './logger.js';
+
+const mockGetLogDirectory = getLogDirectory as ReturnType<typeof vi.fn>;
+const mockIsLevelEnabled = isLevelEnabled as ReturnType<typeof vi.fn>;
+
+describe('payload utilities', () => {
+  let tempDir: string;
+
+  beforeEach(() => {
+    tempDir = mkdtempSync(join(tmpdir(), 'payload-test-'));
+    mockGetLogDirectory.mockReturnValue(tempDir);
+    mockIsLevelEnabled.mockReturnValue(false);
+  });
+
+  afterEach(() => {
+    rmSync(tempDir, { recursive: true, force: true });
+    vi.clearAllMocks();
+  });
+
+  describe('truncateForLog', () => {
+    it('returns content unchanged when under max length', () => {
+      const content = 'short content';
+      expect(truncateForLog(content, 500)).toBe(content);
+    });
+
+    it('truncates content when over max length', () => {
+      const content = 'a'.repeat(600);
+      const result = truncateForLog(content, 500);
+      expect(result.length).toBe(500 + '... [truncated]'.length);
+      expect(result).toContain('... [truncated]');
+    });
+
+    it('uses default max length of 500', () => {
+      const content = 'a'.repeat(600);
+      const result = truncateForLog(content);
+      expect(result.startsWith('a'.repeat(500))).toBe(true);
+      expect(result).toContain('... [truncated]');
+    });
+
+    it('handles exact max length content', () => {
+      const content = 'a'.repeat(500);
+      expect(truncateForLog(content, 500)).toBe(content);
+    });
+  });
+
+  describe('summarizePayload', () => {
+    it('returns summary with preview, size, and hash', () => {
+      const content = 'test content for summarization';
+      const result = summarizePayload(content, 'test-type', 'test-id');
+
+      expect(result.preview).toBe(content);
+      expect(result.sizeBytes).toBe(Buffer.byteLength(content, 'utf8'));
+      expect(result.hash).toMatch(/^[a-f0-9]{12}$/);
+      expect(result.payloadFile).toBeUndefined();
+    });
+
+    it('truncates preview for large content', () => {
+      const content = 'x'.repeat(1000);
+      const result = summarizePayload(content, 'large', 'large-id');
+
+      expect(result.preview).toContain('... [truncated]');
+      expect(result.preview.length).toBeLessThan(content.length);
+    });
+
+    it('does not dump payload when dumpFull is false', () => {
+      const content = 'x'.repeat(20000); // Above threshold
+      const result = summarizePayload(content, 'type', 'id', false);
+
+      expect(result.payloadFile).toBeUndefined();
+      const payloadDir = join(tempDir, 'payload');
+      expect(existsSync(payloadDir)).toBe(false);
+    });
+
+    it('dumps payload to file when dumpFull is true and above threshold', () => {
+      const content = 'x'.repeat(20000); // Above 10KB threshold
+      const result = summarizePayload(content, 'dump-type', 'dump-id', true);
+
+      expect(result.payloadFile).toBeDefined();
+      expect(result.payloadFile).toContain('dump-type');
+      expect(result.payloadFile).toContain(result.hash);
+
+      const payloadDir = join(tempDir, 'payload');
+      expect(existsSync(payloadDir)).toBe(true);
+
+      const files = readdirSync(payloadDir);
+      expect(files.length).toBe(1);
+      expect(files[0]).toBe(result.payloadFile);
+
+      const fileContent = JSON.parse(readFileSync(join(payloadDir, files[0]), 'utf8'));
+      expect(fileContent.type).toBe('dump-type');
+      expect(fileContent.identifier).toBe('dump-id');
+      expect(fileContent.content).toBe(content);
+      expect(fileContent.sizeBytes).toBe(result.sizeBytes);
+    });
+
+    it('does not dump payload below threshold even with dumpFull true', () => {
+      const content = 'small content'; // Below 10KB threshold
+      const result = summarizePayload(content, 'small-type', 'small-id', true);
+
+      expect(result.payloadFile).toBeUndefined();
+    });
+
+    it('creates payload directory if it does not exist', () => {
+      const content = 'y'.repeat(20000);
+      const payloadDir = join(tempDir, 'payload');
+      expect(existsSync(payloadDir)).toBe(false);
+
+      summarizePayload(content, 'create-dir', 'create-id', true);
+
+      expect(existsSync(payloadDir)).toBe(true);
+    });
+
+    it('sanitizes identifier for filename', () => {
+      const content = 'z'.repeat(20000);
+      const result = summarizePayload(content, 'type', 'https://example.com/path?query=1', true);
+
+      expect(result.payloadFile).toBeDefined();
+      expect(result.payloadFile).not.toContain('://');
+      expect(result.payloadFile).not.toContain('?');
+    });
+
+    it('uses trace level check for dumpFull default', () => {
+      mockIsLevelEnabled.mockReturnValue(true);
+      const content = 'a'.repeat(20000);
+
+      const result = summarizePayload(content, 'trace-type', 'trace-id');
+
+      expect(mockIsLevelEnabled).toHaveBeenCalledWith('trace');
+      expect(result.payloadFile).toBeDefined();
+    });
+
+    it('generates consistent hash for same content', () => {
+      const content = 'consistent content';
+      const result1 = summarizePayload(content, 'type1', 'id1');
+      const result2 = summarizePayload(content, 'type2', 'id2');
+
+      expect(result1.hash).toBe(result2.hash);
+    });
+  });
+});

package/src/logging/payload.ts

@@ -0,0 +1,121 @@
+/**
+ * Large payload handling utilities for logging
+ *
+ * Handles large content (raw HTML, MCP responses) by:
+ * - Truncating to preview in log entries
+ * - Optionally dumping full content to separate files at trace level
+ */
+
+import { writeFileSync, mkdirSync, existsSync } from 'node:fs';
+import { join } from 'node:path';
+import { createHash } from 'node:crypto';
+import { getLogDirectory, isLevelEnabled } from './logger.js';
+
+/** Maximum characters for log preview */
+const MAX_PREVIEW_LENGTH = 500;
+
+/** Minimum size to trigger payload dump (10KB) */
+const PAYLOAD_DUMP_THRESHOLD = 10_000;
+
+/** Summary of a large payload for logging */
+export interface PayloadSummary {
+  /** Truncated preview of content */
+  preview: string;
+  /** Size in bytes */
+  sizeBytes: number;
+  /** Short hash for identification */
+  hash: string;
+  /** Filename if full content was dumped (trace level only) */
+  payloadFile?: string;
+}
+
+/** Get the payload dump directory */
+function getPayloadDir(): string {
+  const dir = join(getLogDirectory(), 'payload');
+  if (!existsSync(dir)) {
+    mkdirSync(dir, { recursive: true });
+  }
+  return dir;
+}
+
+/** Generate a safe filename from an identifier */
+function safeFilename(identifier: string): string {
+  return identifier
+    .replace(/[^a-zA-Z0-9-]/g, '_')
+    .substring(0, 50);
+}
+
+/**
+ * Summarize a large payload for logging
+ *
+ * Creates a summary with:
+ * - Truncated preview (first 500 chars)
+ * - Size in bytes
+ * - Short MD5 hash for identification
+ * - Optional full dump to file at trace level
+ *
+ * @param content - The full content to summarize
+ * @param type - Type identifier (e.g., 'raw-html', 'mcp-response')
+ * @param identifier - Unique identifier (e.g., URL, query)
+ * @param dumpFull - Whether to dump full content to file (default: trace level check)
+ * @returns PayloadSummary for inclusion in log entry
+ *
+ * @example
+ * logger.info({
+ *   url,
+ *   ...summarizePayload(html, 'raw-html', url),
+ * }, 'Fetched HTML');
+ */
+export function summarizePayload(
+  content: string,
+  type: string,
+  identifier: string,
+  dumpFull: boolean = isLevelEnabled('trace')
+): PayloadSummary {
+  const sizeBytes = Buffer.byteLength(content, 'utf8');
+  const hash = createHash('md5').update(content).digest('hex').substring(0, 12);
+  const preview = truncateForLog(content, MAX_PREVIEW_LENGTH);
+
+  const baseSummary = { preview, sizeBytes, hash };
+
+  // Dump full payload to file if enabled and above threshold
+  if (dumpFull && sizeBytes > PAYLOAD_DUMP_THRESHOLD) {
+    const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+    const safeId = safeFilename(identifier);
+    const filename = `${timestamp}-${type}-${safeId}-${hash}.json`;
+    const filepath = join(getPayloadDir(), filename);
+
+    writeFileSync(
+      filepath,
+      JSON.stringify(
+        {
+          timestamp: new Date().toISOString(),
+          type,
+          identifier,
+          sizeBytes,
+          content,
+        },
+        null,
+        2
+      )
+    );
+
+    return { ...baseSummary, payloadFile: filename };
+  }
+
+  return baseSummary;
+}
+
+/**
+ * Truncate content for logging with ellipsis indicator
+ *
+ * @param content - Content to truncate
+ * @param maxLength - Maximum length (default: 500)
+ * @returns Truncated string with '... [truncated]' if needed
+ */
+export function truncateForLog(content: string, maxLength: number = MAX_PREVIEW_LENGTH): string {
+  if (content.length <= maxLength) {
+    return content;
+  }
+  return content.substring(0, maxLength) + '... [truncated]';
+}

package/src/mcp/handlers/search.handler.test.ts

@@ -3,6 +3,20 @@ import { handleSearch, handleGetFullContext, resultCache } from './search.handle
 import type { HandlerContext } from '../types.js';
 import type { ServiceContainer } from '../../services/index.js';
 
+/**
+ * Extract JSON from search response that includes a header line.
+ * Format: "Search: ... | Results: ... | ~X tokens | Xms\n\n{json}"
+ */
+function parseSearchResponse(text: string): { header: string; json: Record<string, unknown> } {
+  const parts = text.split('\n\n');
+  const header = parts[0] ?? '';
+  const jsonStr = parts.slice(1).join('\n\n');
+  return {
+    header,
+    json: JSON.parse(jsonStr || '{}')
+  };
+}
+
 describe('Search Handlers', () => {
   let mockContext: HandlerContext;
   let mockServices: ServiceContainer;
@@ -70,7 +84,10 @@ describe('Search Handlers', () => {
         })
       );
 
-      const response = JSON.parse(result.content[0]?.text ?? '{}');
+      const { header, json: response } = parseSearchResponse(result.content[0]?.text ?? '');
+      expect(header).toContain('Search: "test query"');
+      expect(header).toContain('Results: 1');
+      expect(header).toContain('tokens');
       expect(response.results).toHaveLength(1);
       expect(response.totalResults).toBe(1);
     });
@@ -129,14 +146,15 @@ describe('Search Handlers', () => {
       expect(cached?.id).toBe('doc1');
     });
 
-    it('should calculate estimated tokens', async () => {
+    it('should show token count in header', async () => {
       const result = await handleSearch(
         { query: 'test', detail: 'minimal', limit: 10 },
         mockContext
       );
 
-      const response = JSON.parse(result.content[0]?.text ?? '{}');
-      expect(response.estimatedTokens).toBeGreaterThan(0);
+      const { header } = parseSearchResponse(result.content[0]?.text ?? '');
+      // Header should contain token count (either "~X tokens" or "~X.Xk tokens")
+      expect(header).toMatch(/~\d+\.?\d*k? tokens/);
     });
 
     it('should add repoRoot for repo stores', async () => {
@@ -156,7 +174,7 @@ describe('Search Handlers', () => {
         mockContext
       );
 
-      const response = JSON.parse(result.content[0]?.text ?? '{}');
+      const { json: response } = parseSearchResponse(result.content[0]?.text ?? '');
       expect(response.results[0]?.summary.repoRoot).toBe('/repos/test');
     });
 
@@ -166,7 +184,7 @@ describe('Search Handlers', () => {
         mockContext
       );
 
-      const response = JSON.parse(result.content[0]?.text ?? '{}');
+      const { json: response } = parseSearchResponse(result.content[0]?.text ?? '');
       expect(response.results[0]?.summary.repoRoot).toBeUndefined();
     });
 
@@ -176,7 +194,7 @@ describe('Search Handlers', () => {
         mockContext
       );
 
-      const response = JSON.parse(result.content[0]?.text ?? '{}');
+      const { json: response } = parseSearchResponse(result.content[0]?.text ?? '');
       expect(response.results[0]?.summary.storeName).toBe('Test Store');
     });
 
@@ -186,11 +204,12 @@ describe('Search Handlers', () => {
         mockContext
       );
 
-      const response = JSON.parse(result.content[0]?.text ?? '{}');
+      const { header, json: response } = parseSearchResponse(result.content[0]?.text ?? '');
       expect(response).toHaveProperty('totalResults', 1);
-      expect(response).toHaveProperty('estimatedTokens');
       expect(response).toHaveProperty('mode', 'hybrid');
       expect(response).toHaveProperty('timeMs', 50);
+      // Token count is now in header, not in JSON
+      expect(header).toContain('tokens');
     });
   });