bluera-knowledge 0.13.3 → 0.14.1

package/src/services/services.test.ts

@@ -1,108 +0,0 @@
-import { describe, it, expect, vi, beforeEach } from 'vitest';
-import { destroyServices, type ServiceContainer } from './index.js';
-import type { PythonBridge } from '../crawl/bridge.js';
-import type { EmbeddingEngine } from '../db/embeddings.js';
-import type { LanceStore } from '../db/lance.js';
-
-describe('destroyServices', () => {
-  let mockPythonBridge: { stop: ReturnType<typeof vi.fn> };
-  let mockLance: { closeAsync: ReturnType<typeof vi.fn>; close: ReturnType<typeof vi.fn> };
-  let mockEmbeddings: { dispose: ReturnType<typeof vi.fn> };
-  let mockServices: ServiceContainer;
-
-  beforeEach(() => {
-    mockPythonBridge = {
-      stop: vi.fn().mockResolvedValue(undefined),
-    };
-
-    mockLance = {
-      closeAsync: vi.fn().mockResolvedValue(undefined),
-      close: vi.fn(),
-    };
-
-    mockEmbeddings = {
-      dispose: vi.fn().mockResolvedValue(undefined),
-    };
-
-    mockServices = {
-      pythonBridge: mockPythonBridge as unknown as PythonBridge,
-      lance: mockLance as unknown as LanceStore,
-      embeddings: mockEmbeddings as unknown as EmbeddingEngine,
-    } as unknown as ServiceContainer;
-  });
-
-  it('stops the python bridge', async () => {
-    await destroyServices(mockServices);
-
-    expect(mockPythonBridge.stop).toHaveBeenCalledTimes(1);
-  });
-
-  it('throws on stop errors', async () => {
-    mockPythonBridge.stop.mockRejectedValue(new Error('stop failed'));
-
-    await expect(destroyServices(mockServices)).rejects.toThrow(
-      'Service shutdown failed: stop failed'
-    );
-  });
-
-  it('is idempotent - multiple calls work correctly', async () => {
-    await destroyServices(mockServices);
-    await destroyServices(mockServices);
-
-    expect(mockPythonBridge.stop).toHaveBeenCalledTimes(2);
-  });
-
-  it('calls closeAsync on LanceStore for native cleanup', async () => {
-    await destroyServices(mockServices);
-
-    expect(mockLance.closeAsync).toHaveBeenCalledTimes(1);
-    // Should use async version, not sync
-    expect(mockLance.close).not.toHaveBeenCalled();
-  });
-
-  it('throws on LanceStore closeAsync errors', async () => {
-    mockLance.closeAsync.mockRejectedValue(new Error('closeAsync failed'));
-
-    await expect(destroyServices(mockServices)).rejects.toThrow(
-      'Service shutdown failed: closeAsync failed'
-    );
-  });
-
-  it('attempts all cleanup even if first fails, then throws aggregate', async () => {
-    mockLance.closeAsync.mockRejectedValue(new Error('lance failed'));
-    mockPythonBridge.stop.mockRejectedValue(new Error('bridge failed'));
-
-    await expect(destroyServices(mockServices)).rejects.toThrow();
-
-    // Both should have been called even though first failed
-    expect(mockLance.closeAsync).toHaveBeenCalledTimes(1);
-    expect(mockPythonBridge.stop).toHaveBeenCalledTimes(1);
-  });
-
-  it('waits for LanceStore async cleanup before returning', async () => {
-    let closeCompleted = false;
-    mockLance.closeAsync.mockImplementation(async () => {
-      await new Promise((resolve) => setTimeout(resolve, 50));
-      closeCompleted = true;
-    });
-
-    await destroyServices(mockServices);
-
-    // Should have waited for closeAsync to complete
-    expect(closeCompleted).toBe(true);
-  });
-
-  it('calls dispose on EmbeddingEngine to free ONNX runtime resources', async () => {
-    await destroyServices(mockServices);
-
-    expect(mockEmbeddings.dispose).toHaveBeenCalledTimes(1);
-  });
-
-  it('throws on EmbeddingEngine dispose errors', async () => {
-    mockEmbeddings.dispose.mockRejectedValue(new Error('dispose failed'));
-
-    await expect(destroyServices(mockServices)).rejects.toThrow(
-      'Service shutdown failed: dispose failed'
-    );
-  });
-});

package/src/services/snippet.service.test.ts

@@ -1,213 +0,0 @@
-import { describe, it, expect } from 'vitest';
-import { extractSnippet } from './snippet.service.js';
-
-describe('SnippetService', () => {
-  describe('extractSnippet', () => {
-    it('returns full content when shorter than maxLength', () => {
-      const content = 'Short content';
-      const snippet = extractSnippet(content, 'query');
-      expect(snippet).toBe('Short content');
-    });
-
-    it('truncates long content when no query terms match', () => {
-      const content = 'A'.repeat(300);
-      const snippet = extractSnippet(content, 'nonexistent');
-      expect(snippet.length).toBeLessThanOrEqual(203); // 200 + '...'
-      expect(snippet).toContain('...');
-    });
-
-    it('truncates when query has no terms longer than 2 chars', () => {
-      const content = 'A'.repeat(300);
-      const snippet = extractSnippet(content, 'a b c');
-      expect(snippet.length).toBeLessThanOrEqual(203);
-      expect(snippet).toContain('...');
-    });
-
-    it('extracts snippet around single query term match', () => {
-      const content =
-        'The quick brown fox jumps over the lazy dog. ' +
-        'This is some filler text to make the content longer. ' +
-        'Here is the important keyword we want to find. ' +
-        'More filler text at the end to extend the content.';
-      const snippet = extractSnippet(content, 'keyword');
-      expect(snippet).toContain('keyword');
-      expect(snippet).toContain('important');
-    });
-
-    it('extracts snippet around multiple clustered query terms', () => {
-      const content =
-        'Unrelated text at the start. '.repeat(10) +
-        'Here is the target area with multiple keywords: search query terms matching. ' +
-        'Unrelated text at the end. '.repeat(10);
-      const snippet = extractSnippet(content, 'target keywords search');
-      expect(snippet).toContain('target');
-      expect(snippet).toContain('keywords');
-    });
-
-    it('normalizes whitespace in content', () => {
-      const content = 'Multiple    spaces   and\n\nnewlines\t\ttabs';
-      const snippet = extractSnippet(content, 'query');
-      expect(snippet).toBe('Multiple spaces and newlines tabs');
-    });
-
-    it('filters out query terms shorter than 3 characters', () => {
-      const content = 'The quick brown fox jumps over the lazy dog';
-      const snippet = extractSnippet(content, 'a b the quick fox');
-      // Should only match 'quick' and 'fox', not 'a', 'b', 'the'
-      expect(snippet).toContain('quick');
-      expect(snippet).toContain('fox');
-    });
-
-    it('adds ellipsis at start when snippet is from middle', () => {
-      const content = 'A'.repeat(100) + ' target ' + 'B'.repeat(100);
-      const snippet = extractSnippet(content, 'target', { maxLength: 50 });
-      expect(snippet.startsWith('...')).toBe(true);
-    });
-
-    it('adds ellipsis at end when snippet ends before content end', () => {
-      const content = 'target ' + 'A'.repeat(200);
-      const snippet = extractSnippet(content, 'target', { maxLength: 50 });
-      expect(snippet.endsWith('...')).toBe(true);
-    });
-
-    it('adds ellipsis at both ends when snippet is from middle', () => {
-      const content = 'A'.repeat(200) + ' target ' + 'B'.repeat(200);
-      const snippet = extractSnippet(content, 'target', { maxLength: 100 });
-      expect(snippet.startsWith('...')).toBe(true);
-      expect(snippet.endsWith('...')).toBe(true);
-    });
-
-    it('respects custom maxLength option', () => {
-      const content = 'A'.repeat(500);
-      const snippet = extractSnippet(content, 'nonexistent', { maxLength: 50 });
-      expect(snippet.length).toBeLessThanOrEqual(53); // 50 + '...'
-    });
-
-    it('respects custom contextChars option', () => {
-      const content = 'A'.repeat(100) + ' target ' + 'B'.repeat(100);
-      const snippet = extractSnippet(content, 'target', { contextChars: 20 });
-      expect(snippet).toContain('target');
-    });
-
-    it('tries to break at word boundaries for start', () => {
-      const content = 'This is some preamble text. And here is the target word in context.';
-      const snippet = extractSnippet(content, 'target', { maxLength: 40 });
-      // Should contain target
-      expect(snippet).toContain('target');
-    });
-
-    it('tries to break at word boundaries for end', () => {
-      const content = 'The target word is here. Followed by more text that continues.';
-      const snippet = extractSnippet(content, 'target', { maxLength: 40 });
-      // Should contain target and use ellipsis
-      expect(snippet).toContain('target');
-      expect(snippet.endsWith('...')).toBe(true);
-    });
-
-    it('handles content at exact maxLength', () => {
-      const content = 'A'.repeat(200);
-      const snippet = extractSnippet(content, 'query', { maxLength: 200 });
-      expect(snippet.length).toBeLessThanOrEqual(203);
-    });
-
-    it('handles query with case-insensitive matching', () => {
-      const content = 'The TARGET word is here in different case';
-      const snippet = extractSnippet(content, 'target');
-      expect(snippet).toContain('TARGET');
-    });
-
-    it('finds best position when multiple query terms appear', () => {
-      const content =
-        'First occurrence of term. ' +
-        'A'.repeat(200) +
-        ' Second occurrence with more terms from query nearby. ' +
-        'A'.repeat(200);
-      const snippet = extractSnippet(content, 'term query nearby', { maxLength: 100 });
-      expect(snippet).toContain('Second');
-      expect(snippet).toContain('term');
-    });
-
-    it('handles empty query string', () => {
-      const content = 'A'.repeat(300);
-      const snippet = extractSnippet(content, '');
-      expect(snippet.length).toBeLessThanOrEqual(203);
-      expect(snippet).toContain('...');
-    });
-
-    it('handles content with only whitespace', () => {
-      const content = '    ';
-      const snippet = extractSnippet(content, 'query');
-      expect(snippet).toBe('');
-    });
-
-    it('calculates proximity score correctly', () => {
-      // Create content where terms are closer together in one location
-      const content =
-        'distant word here. ' +
-        'A'.repeat(300) +
-        ' clustered search query terms all together here ' +
-        'A'.repeat(300);
-      const snippet = extractSnippet(content, 'search query terms', { maxLength: 100 });
-      expect(snippet).toContain('clustered');
-      expect(snippet).toContain('search');
-      expect(snippet).toContain('query');
-      expect(snippet).toContain('terms');
-    });
-
-    it('truncates at word boundary when space is found', () => {
-      const content = 'The quick brown fox jumps over the lazy dog and continues with more text';
-      const snippet = extractSnippet(content, 'nonexistent', { maxLength: 30 });
-      // Should truncate and add ellipsis
-      expect(snippet.endsWith('...')).toBe(true);
-      expect(snippet.length).toBeLessThanOrEqual(33);
-    });
-
-    it('uses fallback truncation when no good word boundary', () => {
-      const content = 'A'.repeat(50) + ' X ' + 'B'.repeat(200);
-      const snippet = extractSnippet(content, 'nonexistent', { maxLength: 60 });
-      expect(snippet.length).toBeLessThanOrEqual(63);
-      expect(snippet).toContain('...');
-    });
-
-    it('handles content at start edge with maxLength adjustment', () => {
-      const content = 'start term here ' + 'A'.repeat(200);
-      const snippet = extractSnippet(content, 'start', { maxLength: 50 });
-      expect(snippet.startsWith('...')).toBe(false);
-      expect(snippet.endsWith('...')).toBe(true);
-    });
-
-    it('handles content at end edge with maxLength adjustment', () => {
-      const content = 'A'.repeat(200) + ' end term here';
-      const snippet = extractSnippet(content, 'end', { maxLength: 50 });
-      expect(snippet.startsWith('...')).toBe(true);
-      expect(snippet.endsWith('...')).toBe(false);
-    });
-
-    it('finds position when first term array element exists', () => {
-      const content = 'Simple test with term';
-      const snippet = extractSnippet(content, 'test term');
-      expect(snippet).toContain('test');
-      expect(snippet).toContain('term');
-    });
-
-    it('scores positions based on unique nearby terms', () => {
-      const content =
-        'repeated repeated repeated. ' +
-        'A'.repeat(100) +
-        ' diverse unique different varied terms here ' +
-        'A'.repeat(100);
-      const snippet = extractSnippet(content, 'diverse unique different varied', {
-        maxLength: 100,
-      });
-      expect(snippet).toContain('diverse');
-      expect(snippet).toContain('unique');
-    });
-
-    it('handles very long content efficiently', () => {
-      const content = 'A'.repeat(10000) + ' needle ' + 'B'.repeat(10000);
-      const snippet = extractSnippet(content, 'needle', { maxLength: 100 });
-      expect(snippet).toContain('needle');
-      expect(snippet.length).toBeLessThanOrEqual(106); // 100 + '...' on both sides
-    });
-  });
-});

package/src/services/snippet.service.ts

@@ -1,193 +0,0 @@
-/**
- * Query-aware snippet extraction service.
- * Uses proximity-based extraction for consistent results.
- */
-
-export interface SnippetOptions {
-  /** Maximum length of the snippet in characters */
-  maxLength?: number;
-  /** Number of characters to show before/after the match */
-  contextChars?: number;
-}
-
-const DEFAULT_OPTIONS: Required<SnippetOptions> = {
-  maxLength: 200,
-  contextChars: 80,
-};
-
-/**
- * Extract a query-aware snippet from content.
- * Uses proximity-based extraction to find the area with the most query term matches.
- */
-export function extractSnippet(
-  content: string,
-  query: string,
-  options: SnippetOptions = {}
-): string {
-  const opts = { ...DEFAULT_OPTIONS, ...options };
-
-  // Tokenize query into terms (lowercase for matching)
-  const queryTerms = query
-    .toLowerCase()
-    .split(/\s+/)
-    .filter((t) => t.length > 2); // Skip very short words
-
-  // Normalize content for extraction
-  const normalizedContent = content.replace(/\s+/g, ' ').trim();
-
-  if (normalizedContent.length <= opts.maxLength) {
-    return normalizedContent;
-  }
-
-  if (queryTerms.length === 0) {
-    return truncateWithEllipsis(normalizedContent, opts.maxLength);
-  }
-
-  const bestPosition = findBestMatchPosition(normalizedContent, queryTerms);
-
-  if (bestPosition === -1) {
-    return truncateWithEllipsis(normalizedContent, opts.maxLength);
-  }
-
-  return extractContextWindow(normalizedContent, bestPosition, opts);
-}
-
-/**
- * Find the position in content where the most query terms cluster together.
- * Uses multi-factor scoring:
- * - Query term density (base score)
- * - Sentence completeness bonus
- * - Code example presence bonus
- * - Section header proximity bonus
- */
-function findBestMatchPosition(content: string, queryTerms: string[]): number {
-  const lowerContent = content.toLowerCase();
-
-  // Find all positions where query terms appear
-  const termPositions: Array<{ term: string; position: number }> = [];
-
-  for (const term of queryTerms) {
-    let pos = 0;
-    while ((pos = lowerContent.indexOf(term, pos)) !== -1) {
-      termPositions.push({ term, position: pos });
-      pos += 1;
-    }
-  }
-
-  if (termPositions.length === 0) {
-    return -1;
-  }
-
-  const PROXIMITY_WINDOW = 200;
-  const firstTerm = termPositions[0];
-  if (firstTerm === undefined) {
-    return -1;
-  }
-  let bestPosition = firstTerm.position;
-  let bestScore = 0;
-
-  for (const { position } of termPositions) {
-    // Base score: count unique terms within proximity window
-    const nearbyTerms = new Set<string>();
-    for (const { term, position: otherPos } of termPositions) {
-      if (Math.abs(position - otherPos) <= PROXIMITY_WINDOW) {
-        nearbyTerms.add(term);
-      }
-    }
-    let score = nearbyTerms.size * 10; // Base: 10 points per unique term
-
-    // Extract window around position for bonus scoring
-    const windowStart = Math.max(0, position - PROXIMITY_WINDOW / 2);
-    const windowEnd = Math.min(content.length, position + PROXIMITY_WINDOW / 2);
-    const window = content.slice(windowStart, windowEnd);
-
-    // Bonus: Sentence completeness (contains sentence-ending punctuation)
-    if (/[.!?]/.test(window)) {
-      score += 5;
-    }
-
-    // Bonus: Code example presence (backticks, brackets, common code patterns)
-    if (/[`{}()[\]]|=>|function|const |let |var /.test(window)) {
-      score += 3;
-    }
-
-    // Bonus: Near markdown section header
-    const headerMatch = content
-      .slice(Math.max(0, position - 100), position)
-      .match(/^#{1,3}\s+.+$/m);
-    if (headerMatch) {
-      score += 4;
-    }
-
-    if (score > bestScore) {
-      bestScore = score;
-      bestPosition = position;
-    }
-  }
-
-  return bestPosition;
-}
-
-/**
- * Extract a context window around a position.
- */
-function extractContextWindow(
-  content: string,
-  position: number,
-  opts: Required<SnippetOptions>
-): string {
-  const halfWindow = Math.floor(opts.maxLength / 2);
-
-  // Calculate start and end, trying to center on the match
-  let start = Math.max(0, position - halfWindow);
-  let end = Math.min(content.length, position + halfWindow);
-
-  // Adjust if we're near the edges
-  if (start === 0) {
-    end = Math.min(content.length, opts.maxLength);
-  } else if (end === content.length) {
-    start = Math.max(0, content.length - opts.maxLength);
-  }
-
-  // Try to break at word boundaries
-  if (start > 0) {
-    const spaceIndex = content.indexOf(' ', start);
-    if (spaceIndex !== -1 && spaceIndex < start + 20) {
-      start = spaceIndex + 1;
-    }
-  }
-
-  if (end < content.length) {
-    const spaceIndex = content.lastIndexOf(' ', end);
-    if (spaceIndex !== -1 && spaceIndex > end - 20) {
-      end = spaceIndex;
-    }
-  }
-
-  let snippet = content.slice(start, end).trim();
-
-  // Add ellipsis indicators
-  if (start > 0) {
-    snippet = `...${snippet}`;
-  }
-  if (end < content.length) {
-    snippet = `${snippet}...`;
-  }
-
-  return snippet;
-}
-
-/**
- * Truncate content with ellipsis at word boundary.
- */
-function truncateWithEllipsis(content: string, maxLength: number): string {
-  if (content.length <= maxLength) {
-    return content;
-  }
-
-  // Find last space before maxLength
-  const truncateAt = content.lastIndexOf(' ', maxLength - 3);
-  const cutPoint = truncateAt > maxLength * 0.5 ? truncateAt : maxLength - 3;
-
-  return `${content.slice(0, cutPoint).trim()}...`;
-}