bluera-knowledge 0.14.0 → 0.14.2

package/src/crawl/markdown-utils.ts

@@ -1,225 +0,0 @@
-/**
- * Markdown conversion utilities ported from slurp-ai
- * Source: https://github.com/ratacat/slurp-ai
- *
- * These utilities handle complex documentation site patterns (MkDocs, Sphinx, etc.)
- * and produce clean, well-formatted markdown.
- */
-
-import * as cheerio from 'cheerio';
-
-/**
- * Detect language from code element class names.
- * Handles various class naming patterns from different highlighters.
- */
-function detectLanguageFromClass(className: string | undefined): string {
-  if (className === undefined || className === '') return '';
-
-  // Common patterns: "language-python", "lang-js", "highlight-python", "python", "hljs language-python"
-  const patterns = [
-    /language-(\w+)/i,
-    /lang-(\w+)/i,
-    /highlight-(\w+)/i,
-    /hljs\s+(\w+)/i,
-    /^(\w+)$/i,
-  ];
-
-  for (const pattern of patterns) {
-    const match = className.match(pattern);
-    if (match?.[1] !== undefined) {
-      const lang = match[1].toLowerCase();
-      // Filter out common non-language classes
-      if (!['hljs', 'highlight', 'code', 'pre', 'block', 'inline'].includes(lang)) {
-        return lang;
-      }
-    }
-  }
-
-  return '';
-}
-
-/**
- * Escape HTML special characters for safe embedding in HTML.
- */
-function escapeHtml(text: string): string {
-  return text
-    .replace(/&/g, '&')
-    .replace(/</g, '&lt;')
-    .replace(/>/g, '&gt;')
-    .replace(/"/g, '&quot;')
-    .replace(/'/g, '&#039;');
-}
-
-/**
- * Preprocess HTML to handle MkDocs/Material theme code blocks.
- *
- * MkDocs wraps code in tables for line numbers:
- * <table><tbody><tr><td>line numbers</td><td><pre><code>code</code></pre></td></tr></tbody></table>
- *
- * This function converts them to standard <pre><code> blocks that Turndown handles correctly.
- * Also strips syntax highlighting spans and empty anchors from code.
- */
-export function preprocessHtmlForCodeBlocks(html: string): string {
-  if (!html || typeof html !== 'string') return html;
-
-  const $ = cheerio.load(html);
-
-  // Handle MkDocs/Material table-wrapped code blocks
-  $('table').each((_i, table) => {
-    const $table = $(table);
-
-    // Check if this table contains a code block
-    const $codeCell = $table.find('td pre code, td div pre code');
-
-    if ($codeCell.length > 0) {
-      // This is a code block table - extract the code
-      const $pre = $codeCell.closest('pre');
-      const $code = $codeCell.first();
-
-      // Get language from class
-      let language = detectLanguageFromClass($code.attr('class'));
-      if (!language) {
-        language = detectLanguageFromClass($pre.attr('class'));
-      }
-
-      // Get the text content, stripping all inner HTML tags
-      const codeText = $code.text();
-
-      // Create a clean pre > code block
-      const cleanPre = `<pre><code class="language-${language}">${escapeHtml(codeText)}</code></pre>`;
-
-      // Replace the entire table with the clean code block
-      $table.replaceWith(cleanPre);
-    }
-  });
-
-  // Strip empty anchor tags used for line numbers
-  $('pre a, code a').each((_i, anchor) => {
-    const $anchor = $(anchor);
-    if (!$anchor.text().trim()) {
-      $anchor.remove();
-    }
-  });
-
-  // Strip syntax highlighting spans inside code blocks, keeping only text
-  $('pre span, code span').each((_i, span) => {
-    const $span = $(span);
-    $span.replaceWith($span.text());
-  });
-
-  // Handle standalone pre blocks that might have spans/anchors
-  $('pre').each((_i, pre) => {
-    const $pre = $(pre);
-    // If this pre has a code child, it was already processed
-    if ($pre.find('code').length === 0) {
-      // Direct pre without code - get text content
-      const text = $pre.text();
-      const lang = detectLanguageFromClass($pre.attr('class'));
-      $pre.html(`<code class="language-${lang}">${escapeHtml(text)}</code>`);
-    }
-  });
-
-  return $.html();
-}
-
-/**
- * Apply comprehensive cleanup rules to markdown content.
- *
- * Formatting rules:
- * - Double newlines between paragraphs and headings
- * - Double newlines before lists when preceded by normal text
- * - Single newlines between list items
- * - No blank lines inside code blocks
- */
-export function cleanupMarkdown(markdown: string): string {
-  if (!markdown) return '';
-
-  const trimmed = markdown.trim();
-  if (trimmed === '') return '';
-
-  let result = trimmed;
-
-  // 0. Fix broken headings where ## is on its own line followed by the text
-  // Pattern: "## \n\nSome text" → "## Some text"
-  result = result.replace(/^(#{1,6})\s*\n\n+(\S[^\n]*)/gm, '$1 $2');
-
-  // 0.5. Normalize multiple spaces after heading markers to single space
-  // Pattern: "##  Subtitle" → "## Subtitle"
-  result = result.replace(/(#{1,6})\s{2,}/g, '$1 ');
-
-  // 1. Fix navigation links with excessive whitespace
-  result = result.replace(/\*\s+\[\s*([^\n]+?)\s*\]\(([^)]+)\)/g, '* [$1]($2)');
-
-  // 2. Handle headings with specific newline requirements
-
-  // Text followed by heading should have a single newline between them (no blank line)
-  result = result.replace(/([^\n])\n\n+(#\s)/g, '$1\n$2');
-
-  // Add double newlines between text and next heading
-  result = result.replace(/(Some text\.)\n(##\s)/g, '$1\n\n$2');
-
-  // Double newlines after a heading when followed by text
-  result = result.replace(/(#{1,6}\s[^\n]+)\n([^#\n])/g, '$1\n\n$2');
-
-  // Double newlines between headings
-  result = result.replace(/(#{1,6}\s[^\n]+)\n(#{1,6}\s)/g, '$1\n\n$2');
-
-  // 3. Lists - ensure all list items have single newlines only
-  result = result.replace(/(\* Item 1)\n\n+(\* Item 2)\n\n+(\* Item 3)/g, '$1\n$2\n$3');
-
-  // 3.5. General list item spacing - ensure single newlines between list items
-  result = result.replace(/(^\*\s[^\n]+)\n{2,}(^\*\s)/gm, '$1\n$2');
-
-  // 4. Clean up excessive blank lines (3+ newlines → 2 newlines)
-  result = result.replace(/\n{3,}/g, '\n\n');
-
-  // 5. Code blocks - no blank lines after opening or before closing backticks
-  result = result.replace(/(```[^\n]*)\n\n+/g, '$1\n');
-  result = result.replace(/\n\n+```/g, '\n```');
-
-  // 6. Remove empty list items
-  result = result.replace(/\*\s*\n\s*\*/g, '*');
-
-  // 7. Strip any remaining HTML tags that leaked through (common in MkDocs/Material)
-  // Remove table structure tags
-  result = result.replace(/<\/?table[^>]*>/gi, '');
-  result = result.replace(/<\/?tbody[^>]*>/gi, '');
-  result = result.replace(/<\/?thead[^>]*>/gi, '');
-  result = result.replace(/<\/?tr[^>]*>/gi, '');
-  result = result.replace(/<\/?td[^>]*>/gi, '');
-  result = result.replace(/<\/?th[^>]*>/gi, '');
-
-  // Remove empty anchor tags: <a></a> or <a id="..."></a>
-  result = result.replace(/<a[^>]*><\/a>/gi, '');
-
-  // Remove span tags (syntax highlighting remnants)
-  result = result.replace(/<\/?span[^>]*>/gi, '');
-
-  // Remove div tags
-  result = result.replace(/<\/?div[^>]*>/gi, '');
-
-  // Remove pre/code tags that leaked
-  result = result.replace(/<\/?pre[^>]*>/gi, '');
-  result = result.replace(/<\/?code[^>]*>/gi, '');
-
-  // 8. Remove empty markdown links: [](url) and []()
-  result = result.replace(/\[\]\([^)]*\)/g, '');
-
-  // 9. Remove codelineno references that leaked into content
-  // Pattern: [](_file.md#__codelineno-N-M)
-  result = result.replace(/\[\]\([^)]*#__codelineno-[^)]+\)/g, '');
-
-  // Also clean inline codelineno patterns
-  result = result.replace(/\[?\]?\([^)]*#__codelineno-[^)]*\)/g, '');
-
-  // 10. Clean up any double-escaped HTML entities that might result
-  result = result.replace(/&amp;lt;/g, '&lt;');
-  result = result.replace(/&amp;gt;/g, '&gt;');
-  result = result.replace(/&amp;amp;/g, '&amp;');
-
-  // 11. Final cleanup - normalize excessive whitespace from removed tags
-  result = result.replace(/\n{3,}/g, '\n\n');
-  result = result.replace(/[ \t]+\n/g, '\n');
-
-  return result;
-}

package/src/crawl/schemas.ts

@@ -1,114 +0,0 @@
-import { z } from 'zod';
-
-// Schema for crawl4ai link objects (matches CrawledLink interface)
-// This validates the actual runtime structure from Python
-export const CrawledLinkSchema = z.object({
-  href: z.string(),
-  text: z.string(),
-  title: z.string().optional(),
-  base_domain: z.string().optional(),
-  head_data: z.unknown().optional(),
-  head_extraction_status: z.unknown().optional(),
-  head_extraction_error: z.unknown().optional(),
-  intrinsic_score: z.number().optional(),
-  contextual_score: z.unknown().optional(),
-  total_score: z.unknown().optional(),
-});
-
-// Schema for individual crawl page result
-const CrawlPageSchema = z.object({
-  url: z.string(),
-  title: z.string(),
-  content: z.string(),
-  links: z.array(z.string()),
-  crawledAt: z.string(),
-});
-
-// Schema for full crawl response
-export const CrawlResultSchema = z.object({
-  pages: z.array(CrawlPageSchema),
-});
-
-// Schema for headless fetch response
-// Supports both link objects and plain strings
-export const HeadlessResultSchema = z.object({
-  html: z.string(),
-  markdown: z.string(),
-  links: z.array(z.union([CrawledLinkSchema, z.string()])),
-});
-
-// Type exports derived from schemas (single source of truth)
-export type CrawlResult = z.infer<typeof CrawlResultSchema>;
-export type HeadlessResult = z.infer<typeof HeadlessResultSchema>;
-export type CrawledLink = z.infer<typeof CrawledLinkSchema>;
-
-/**
- * Validates a headless fetch response from Python bridge.
- * Throws ZodError if the response doesn't match the expected schema.
- *
- * @param data - Raw data from Python bridge
- * @returns Validated HeadlessResult
- * @throws {z.ZodError} If validation fails
- */
-export function validateHeadlessResult(data: unknown): HeadlessResult {
-  return HeadlessResultSchema.parse(data);
-}
-
-/**
- * Validates a crawl response from Python bridge.
- * Throws ZodError if the response doesn't match the expected schema.
- *
- * @param data - Raw data from Python bridge
- * @returns Validated CrawlResult
- * @throws {z.ZodError} If validation fails
- */
-export function validateCrawlResult(data: unknown): CrawlResult {
-  return CrawlResultSchema.parse(data);
-}
-
-// Schema for Python AST parsing response
-const MethodInfoSchema = z.object({
-  name: z.string(),
-  async: z.boolean(),
-  signature: z.string(),
-  startLine: z.number(),
-  endLine: z.number(),
-  calls: z.array(z.string()),
-});
-
-const CodeNodeSchema = z.object({
-  type: z.enum(['function', 'class']),
-  name: z.string(),
-  exported: z.boolean(),
-  startLine: z.number(),
-  endLine: z.number(),
-  async: z.boolean().optional(),
-  signature: z.string().optional(),
-  calls: z.array(z.string()).optional(),
-  methods: z.array(MethodInfoSchema).optional(),
-});
-
-const ImportInfoSchema = z.object({
-  source: z.string(),
-  imported: z.string(),
-  alias: z.string().optional().nullable(),
-});
-
-export const ParsePythonResultSchema = z.object({
-  nodes: z.array(CodeNodeSchema),
-  imports: z.array(ImportInfoSchema),
-});
-
-export type ParsePythonResult = z.infer<typeof ParsePythonResultSchema>;
-
-/**
- * Validates a Python AST parsing response from Python bridge.
- * Throws ZodError if the response doesn't match the expected schema.
- *
- * @param data - Raw data from Python bridge
- * @returns Validated ParsePythonResult
- * @throws {z.ZodError} If validation fails
- */
-export function validateParsePythonResult(data: unknown): ParsePythonResult {
-  return ParsePythonResultSchema.parse(data);
-}

package/src/db/embeddings.test.ts

@@ -1,79 +0,0 @@
-import { describe, it, expect, beforeAll } from 'vitest';
-import { EmbeddingEngine } from './embeddings.js';
-
-describe('EmbeddingEngine', () => {
-  let engine: EmbeddingEngine;
-
-  beforeAll(async () => {
-    engine = new EmbeddingEngine();
-    await engine.initialize();
-  }, 60000); // Allow time for model download
-
-  it('generates embeddings for text', async () => {
-    const embedding = await engine.embed('Hello world');
-    expect(embedding).toHaveLength(384);
-    expect(embedding.every((n) => typeof n === 'number')).toBe(true);
-  });
-
-  it('generates batch embeddings', async () => {
-    const texts = ['Hello', 'World', 'Test'];
-    const embeddings = await engine.embedBatch(texts);
-    expect(embeddings).toHaveLength(3);
-    expect(embeddings.every((e) => e.length === 384)).toBe(true);
-  });
-
-  it('produces similar embeddings for similar text', async () => {
-    const emb1 = await engine.embed('The cat sat on the mat');
-    const emb2 = await engine.embed('A cat was sitting on a rug');
-    const emb3 = await engine.embed('Quantum physics is complex');
-
-    const sim12 = cosineSimilarity(emb1, emb2);
-    const sim13 = cosineSimilarity(emb1, emb3);
-
-    expect(sim12).toBeGreaterThan(sim13);
-  });
-
-  it('returns correct dimensions', async () => {
-    expect(engine.getDimensions()).toBe(384);
-  });
-
-  it('handles embed when extractor is not initialized', async () => {
-    // Create new engine without initializing
-    const newEngine = new EmbeddingEngine();
-    const embedding = await newEngine.embed('Test');
-    expect(embedding).toHaveLength(384);
-  });
-
-  it('supports custom model dimensions', async () => {
-    const customEngine = new EmbeddingEngine('Xenova/all-MiniLM-L6-v2', 512);
-    expect(customEngine.getDimensions()).toBe(512);
-  });
-
-  it('skips initialization when already initialized', async () => {
-    // Engine is already initialized from beforeAll
-    // Calling initialize again should be a no-op
-    await engine.initialize();
-    const embedding = await engine.embed('test');
-    expect(embedding).toHaveLength(384);
-  });
-
-  it('handles large batch with multiple chunks', async () => {
-    // Create enough texts to trigger multiple batch iterations (BATCH_SIZE = 32)
-    const texts = Array.from({ length: 40 }, (_, i) => `Text number ${String(i)}`);
-    const embeddings = await engine.embedBatch(texts);
-    expect(embeddings).toHaveLength(40);
-    expect(embeddings.every((e) => e.length === 384)).toBe(true);
-  }, 60000);
-});
-
-function cosineSimilarity(a: number[], b: number[]): number {
-  let dotProduct = 0;
-  let normA = 0;
-  let normB = 0;
-  for (let i = 0; i < a.length; i++) {
-    dotProduct += a[i]! * b[i]!;
-    normA += a[i]! * a[i]!;
-    normB += b[i]! * b[i]!;
-  }
-  return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
-}

package/src/db/embeddings.ts

@@ -1,78 +0,0 @@
-import { homedir } from 'node:os';
-import { join } from 'node:path';
-import { pipeline, env, type FeatureExtractionPipeline } from '@huggingface/transformers';
-
-// Set cache directory to ~/.cache/huggingface-transformers (outside node_modules)
-// This allows CI caching and prevents model re-downloads on each npm install
-env.cacheDir = join(homedir(), '.cache', 'huggingface-transformers');
-
-export class EmbeddingEngine {
-  private extractor: FeatureExtractionPipeline | null = null;
-  private readonly modelName: string;
-  private readonly dimensions: number;
-
-  constructor(modelName = 'Xenova/all-MiniLM-L6-v2', dimensions = 384) {
-    this.modelName = modelName;
-    this.dimensions = dimensions;
-  }
-
-  async initialize(): Promise<void> {
-    if (this.extractor !== null) return;
-    // @ts-expect-error TS2590: TypeScript can't represent the complex union type from pipeline()
-    // This is a known limitation with @huggingface/transformers overloaded signatures
-    this.extractor = await pipeline('feature-extraction', this.modelName, {
-      dtype: 'fp32',
-    });
-  }
-
-  async embed(text: string): Promise<number[]> {
-    if (this.extractor === null) {
-      await this.initialize();
-    }
-    if (this.extractor === null) {
-      throw new Error('Failed to initialize embedding model');
-    }
-    const output = await this.extractor(text, {
-      pooling: 'mean',
-      normalize: true,
-    });
-    const result = Array.from(output.data);
-    return result.map((v) => Number(v));
-  }
-
-  async embedBatch(texts: string[]): Promise<number[][]> {
-    const BATCH_SIZE = 32; // Process 32 chunks in parallel
-    const results: number[][] = [];
-
-    for (let i = 0; i < texts.length; i += BATCH_SIZE) {
-      const batch = texts.slice(i, i + BATCH_SIZE);
-
-      // Process batch in parallel using Promise.all
-      const batchResults = await Promise.all(batch.map((text) => this.embed(text)));
-
-      results.push(...batchResults);
-
-      // Small delay between batches to prevent memory issues
-      if (i + BATCH_SIZE < texts.length) {
-        await new Promise((resolve) => setTimeout(resolve, 100));
-      }
-    }
-
-    return results;
-  }
-
-  getDimensions(): number {
-    return this.dimensions;
-  }
-
-  /**
-   * Dispose the embedding pipeline to free resources.
-   * Should be called before process exit to prevent ONNX runtime cleanup issues on macOS.
-   */
-  async dispose(): Promise<void> {
-    if (this.extractor !== null) {
-      await this.extractor.dispose();
-      this.extractor = null;
-    }
-  }
-}

package/src/db/index.ts

@@ -1,2 +0,0 @@
-export { EmbeddingEngine } from './embeddings.js';
-export { LanceStore } from './lance.js';