bluera-knowledge 0.9.21

package/src/crawl/markdown-utils.ts

@@ -0,0 +1,228 @@
+/**
+ * Markdown conversion utilities ported from slurp-ai
+ * Source: https://github.com/ratacat/slurp-ai
+ *
+ * These utilities handle complex documentation site patterns (MkDocs, Sphinx, etc.)
+ * and produce clean, well-formatted markdown.
+ */
+
+import * as cheerio from 'cheerio';
+
+/**
+ * Detect language from code element class names.
+ * Handles various class naming patterns from different highlighters.
+ */
+function detectLanguageFromClass(className: string | undefined): string {
+  if (className === undefined || className === '') return '';
+
+  // Common patterns: "language-python", "lang-js", "highlight-python", "python", "hljs language-python"
+  const patterns = [
+    /language-(\w+)/i,
+    /lang-(\w+)/i,
+    /highlight-(\w+)/i,
+    /hljs\s+(\w+)/i,
+    /^(\w+)$/i,
+  ];
+
+  for (const pattern of patterns) {
+    const match = className.match(pattern);
+    if (match?.[1] !== undefined) {
+      const lang = match[1].toLowerCase();
+      // Filter out common non-language classes
+      if (!['hljs', 'highlight', 'code', 'pre', 'block', 'inline'].includes(lang)) {
+        return lang;
+      }
+    }
+  }
+
+  return '';
+}
+
+/**
+ * Escape HTML special characters for safe embedding in HTML.
+ */
+function escapeHtml(text: string): string {
+  return text
+    .replace(/&/g, '&')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;')
+    .replace(/'/g, '&#039;');
+}
+
+/**
+ * Preprocess HTML to handle MkDocs/Material theme code blocks.
+ *
+ * MkDocs wraps code in tables for line numbers:
+ * <table><tbody><tr><td>line numbers</td><td><pre><code>code</code></pre></td></tr></tbody></table>
+ *
+ * This function converts them to standard <pre><code> blocks that Turndown handles correctly.
+ * Also strips syntax highlighting spans and empty anchors from code.
+ */
+export function preprocessHtmlForCodeBlocks(html: string): string {
+  if (!html || typeof html !== 'string') return html;
+
+  const $ = cheerio.load(html);
+
+  // Handle MkDocs/Material table-wrapped code blocks
+  $('table').each((_i, table) => {
+    const $table = $(table);
+
+    // Check if this table contains a code block
+    const $codeCell = $table.find('td pre code, td div pre code');
+
+    if ($codeCell.length > 0) {
+      // This is a code block table - extract the code
+      const $pre = $codeCell.closest('pre');
+      const $code = $codeCell.first();
+
+      // Get language from class
+      let language = detectLanguageFromClass($code.attr('class'));
+      if (!language) {
+        language = detectLanguageFromClass($pre.attr('class'));
+      }
+
+      // Get the text content, stripping all inner HTML tags
+      const codeText = $code.text();
+
+      // Create a clean pre > code block
+      const cleanPre = `<pre><code class="language-${language}">${escapeHtml(codeText)}</code></pre>`;
+
+      // Replace the entire table with the clean code block
+      $table.replaceWith(cleanPre);
+    }
+  });
+
+  // Strip empty anchor tags used for line numbers
+  $('pre a, code a').each((_i, anchor) => {
+    const $anchor = $(anchor);
+    if (!$anchor.text().trim()) {
+      $anchor.remove();
+    }
+  });
+
+  // Strip syntax highlighting spans inside code blocks, keeping only text
+  $('pre span, code span').each((_i, span) => {
+    const $span = $(span);
+    $span.replaceWith($span.text());
+  });
+
+  // Handle standalone pre blocks that might have spans/anchors
+  $('pre').each((_i, pre) => {
+    const $pre = $(pre);
+    // If this pre has a code child, it was already processed
+    if ($pre.find('code').length === 0) {
+      // Direct pre without code - get text content
+      const text = $pre.text();
+      const lang = detectLanguageFromClass($pre.attr('class'));
+      $pre.html(`<code class="language-${lang}">${escapeHtml(text)}</code>`);
+    }
+  });
+
+  return $.html();
+}
+
+/**
+ * Apply comprehensive cleanup rules to markdown content.
+ *
+ * Formatting rules:
+ * - Double newlines between paragraphs and headings
+ * - Double newlines before lists when preceded by normal text
+ * - Single newlines between list items
+ * - No blank lines inside code blocks
+ */
+export function cleanupMarkdown(markdown: string): string {
+  if (!markdown) return '';
+
+  const trimmed = markdown.trim();
+  if (trimmed === '') return '';
+
+  let result = trimmed;
+
+  // 0. Fix broken headings where ## is on its own line followed by the text
+  // Pattern: "## \n\nSome text" → "## Some text"
+  result = result.replace(/^(#{1,6})\s*\n\n+(\S[^\n]*)/gm, '$1 $2');
+
+  // 0.5. Normalize multiple spaces after heading markers to single space
+  // Pattern: "##  Subtitle" → "## Subtitle"
+  result = result.replace(/(#{1,6})\s{2,}/g, '$1 ');
+
+  // 1. Fix navigation links with excessive whitespace
+  result = result.replace(/\*\s+\[\s*([^\n]+?)\s*\]\(([^)]+)\)/g, '* [$1]($2)');
+
+  // 2. Handle headings with specific newline requirements
+
+  // Text followed by heading should have a single newline between them (no blank line)
+  result = result.replace(/([^\n])\n\n+(#\s)/g, '$1\n$2');
+
+  // Add double newlines between text and next heading
+  result = result.replace(/(Some text\.)\n(##\s)/g, '$1\n\n$2');
+
+  // Double newlines after a heading when followed by text
+  result = result.replace(/(#{1,6}\s[^\n]+)\n([^#\n])/g, '$1\n\n$2');
+
+  // Double newlines between headings
+  result = result.replace(/(#{1,6}\s[^\n]+)\n(#{1,6}\s)/g, '$1\n\n$2');
+
+  // 3. Lists - ensure all list items have single newlines only
+  result = result.replace(
+    /(\* Item 1)\n\n+(\* Item 2)\n\n+(\* Item 3)/g,
+    '$1\n$2\n$3',
+  );
+
+  // 3.5. General list item spacing - ensure single newlines between list items
+  result = result.replace(/(^\*\s[^\n]+)\n{2,}(^\*\s)/gm, '$1\n$2');
+
+  // 4. Clean up excessive blank lines (3+ newlines → 2 newlines)
+  result = result.replace(/\n{3,}/g, '\n\n');
+
+  // 5. Code blocks - no blank lines after opening or before closing backticks
+  result = result.replace(/(```[^\n]*)\n\n+/g, '$1\n');
+  result = result.replace(/\n\n+```/g, '\n```');
+
+  // 6. Remove empty list items
+  result = result.replace(/\*\s*\n\s*\*/g, '*');
+
+  // 7. Strip any remaining HTML tags that leaked through (common in MkDocs/Material)
+  // Remove table structure tags
+  result = result.replace(/<\/?table[^>]*>/gi, '');
+  result = result.replace(/<\/?tbody[^>]*>/gi, '');
+  result = result.replace(/<\/?thead[^>]*>/gi, '');
+  result = result.replace(/<\/?tr[^>]*>/gi, '');
+  result = result.replace(/<\/?td[^>]*>/gi, '');
+  result = result.replace(/<\/?th[^>]*>/gi, '');
+
+  // Remove empty anchor tags: <a></a> or <a id="..."></a>
+  result = result.replace(/<a[^>]*><\/a>/gi, '');
+
+  // Remove span tags (syntax highlighting remnants)
+  result = result.replace(/<\/?span[^>]*>/gi, '');
+
+  // Remove div tags
+  result = result.replace(/<\/?div[^>]*>/gi, '');
+
+  // Remove pre/code tags that leaked
+  result = result.replace(/<\/?pre[^>]*>/gi, '');
+  result = result.replace(/<\/?code[^>]*>/gi, '');
+
+  // 8. Remove empty markdown links: [](url) and []()
+  result = result.replace(/\[\]\([^)]*\)/g, '');
+
+  // 9. Remove codelineno references that leaked into content
+  // Pattern: [](_file.md#__codelineno-N-M)
+  result = result.replace(/\[\]\([^)]*#__codelineno-[^)]+\)/g, '');
+
+  // Also clean inline codelineno patterns
+  result = result.replace(/\[?\]?\([^)]*#__codelineno-[^)]*\)/g, '');
+
+  // 10. Clean up any double-escaped HTML entities that might result
+  result = result.replace(/&amp;lt;/g, '&lt;');
+  result = result.replace(/&amp;gt;/g, '&gt;');
+  result = result.replace(/&amp;amp;/g, '&amp;');
+
+  // 11. Final cleanup - normalize excessive whitespace from removed tags
+  result = result.replace(/\n{3,}/g, '\n\n');
+  result = result.replace(/[ \t]+\n/g, '\n');
+
+  return result;
+}

package/src/crawl/schemas.ts

@@ -0,0 +1,114 @@
+import { z } from 'zod';
+
+// Schema for crawl4ai link objects (matches CrawledLink interface)
+// This validates the actual runtime structure from Python
+export const CrawledLinkSchema = z.object({
+  href: z.string(),
+  text: z.string(),
+  title: z.string().optional(),
+  base_domain: z.string().optional(),
+  head_data: z.unknown().optional(),
+  head_extraction_status: z.unknown().optional(),
+  head_extraction_error: z.unknown().optional(),
+  intrinsic_score: z.number().optional(),
+  contextual_score: z.unknown().optional(),
+  total_score: z.unknown().optional(),
+});
+
+// Schema for individual crawl page result
+const CrawlPageSchema = z.object({
+  url: z.string(),
+  title: z.string(),
+  content: z.string(),
+  links: z.array(z.string()),
+  crawledAt: z.string(),
+});
+
+// Schema for full crawl response
+export const CrawlResultSchema = z.object({
+  pages: z.array(CrawlPageSchema),
+});
+
+// Schema for headless fetch response
+// Supports both link objects and plain strings
+export const HeadlessResultSchema = z.object({
+  html: z.string(),
+  markdown: z.string(),
+  links: z.array(z.union([CrawledLinkSchema, z.string()])),
+});
+
+// Type exports derived from schemas (single source of truth)
+export type CrawlResult = z.infer<typeof CrawlResultSchema>;
+export type HeadlessResult = z.infer<typeof HeadlessResultSchema>;
+export type CrawledLink = z.infer<typeof CrawledLinkSchema>;
+
+/**
+ * Validates a headless fetch response from Python bridge.
+ * Throws ZodError if the response doesn't match the expected schema.
+ *
+ * @param data - Raw data from Python bridge
+ * @returns Validated HeadlessResult
+ * @throws {z.ZodError} If validation fails
+ */
+export function validateHeadlessResult(data: unknown): HeadlessResult {
+  return HeadlessResultSchema.parse(data);
+}
+
+/**
+ * Validates a crawl response from Python bridge.
+ * Throws ZodError if the response doesn't match the expected schema.
+ *
+ * @param data - Raw data from Python bridge
+ * @returns Validated CrawlResult
+ * @throws {z.ZodError} If validation fails
+ */
+export function validateCrawlResult(data: unknown): CrawlResult {
+  return CrawlResultSchema.parse(data);
+}
+
+// Schema for Python AST parsing response
+const MethodInfoSchema = z.object({
+  name: z.string(),
+  async: z.boolean(),
+  signature: z.string(),
+  startLine: z.number(),
+  endLine: z.number(),
+  calls: z.array(z.string()),
+});
+
+const CodeNodeSchema = z.object({
+  type: z.enum(['function', 'class']),
+  name: z.string(),
+  exported: z.boolean(),
+  startLine: z.number(),
+  endLine: z.number(),
+  async: z.boolean().optional(),
+  signature: z.string().optional(),
+  calls: z.array(z.string()).optional(),
+  methods: z.array(MethodInfoSchema).optional(),
+});
+
+const ImportInfoSchema = z.object({
+  source: z.string(),
+  imported: z.string(),
+  alias: z.string().optional().nullable(),
+});
+
+export const ParsePythonResultSchema = z.object({
+  nodes: z.array(CodeNodeSchema),
+  imports: z.array(ImportInfoSchema),
+});
+
+export type ParsePythonResult = z.infer<typeof ParsePythonResultSchema>;
+
+/**
+ * Validates a Python AST parsing response from Python bridge.
+ * Throws ZodError if the response doesn't match the expected schema.
+ *
+ * @param data - Raw data from Python bridge
+ * @returns Validated ParsePythonResult
+ * @throws {z.ZodError} If validation fails
+ */
+export function validateParsePythonResult(data: unknown): ParsePythonResult {
+  return ParsePythonResultSchema.parse(data);
+}

package/src/db/embeddings.test.ts

@@ -0,0 +1,63 @@
+import { describe, it, expect, beforeAll } from 'vitest';
+import { EmbeddingEngine } from './embeddings.js';
+
+describe('EmbeddingEngine', () => {
+  let engine: EmbeddingEngine;
+
+  beforeAll(async () => {
+    engine = new EmbeddingEngine();
+    await engine.initialize();
+  }, 60000); // Allow time for model download
+
+  it('generates embeddings for text', async () => {
+    const embedding = await engine.embed('Hello world');
+    expect(embedding).toHaveLength(384);
+    expect(embedding.every((n) => typeof n === 'number')).toBe(true);
+  });
+
+  it('generates batch embeddings', async () => {
+    const texts = ['Hello', 'World', 'Test'];
+    const embeddings = await engine.embedBatch(texts);
+    expect(embeddings).toHaveLength(3);
+    expect(embeddings.every((e) => e.length === 384)).toBe(true);
+  });
+
+  it('produces similar embeddings for similar text', async () => {
+    const emb1 = await engine.embed('The cat sat on the mat');
+    const emb2 = await engine.embed('A cat was sitting on a rug');
+    const emb3 = await engine.embed('Quantum physics is complex');
+
+    const sim12 = cosineSimilarity(emb1, emb2);
+    const sim13 = cosineSimilarity(emb1, emb3);
+
+    expect(sim12).toBeGreaterThan(sim13);
+  });
+
+  it('returns correct dimensions', async () => {
+    expect(engine.getDimensions()).toBe(384);
+  });
+
+  it('handles embed when extractor is not initialized', async () => {
+    // Create new engine without initializing
+    const newEngine = new EmbeddingEngine();
+    const embedding = await newEngine.embed('Test');
+    expect(embedding).toHaveLength(384);
+  });
+
+  it('supports custom model dimensions', async () => {
+    const customEngine = new EmbeddingEngine('Xenova/all-MiniLM-L6-v2', 512);
+    expect(customEngine.getDimensions()).toBe(512);
+  });
+});
+
+function cosineSimilarity(a: number[], b: number[]): number {
+  let dotProduct = 0;
+  let normA = 0;
+  let normB = 0;
+  for (let i = 0; i < a.length; i++) {
+    dotProduct += a[i]! * b[i]!;
+    normA += a[i]! * a[i]!;
+    normB += b[i]! * b[i]!;
+  }
+  return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
+}

package/src/db/embeddings.ts

@@ -0,0 +1,69 @@
+import { pipeline, env, type FeatureExtractionPipeline } from '@huggingface/transformers';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+
+// Set cache directory to ~/.cache/huggingface-transformers (outside node_modules)
+// This allows CI caching and prevents model re-downloads on each npm install
+env.cacheDir = join(homedir(), '.cache', 'huggingface-transformers');
+
+export class EmbeddingEngine {
+  private extractor: FeatureExtractionPipeline | null = null;
+  private readonly modelName: string;
+  private readonly dimensions: number;
+
+  constructor(modelName = 'Xenova/all-MiniLM-L6-v2', dimensions = 384) {
+    this.modelName = modelName;
+    this.dimensions = dimensions;
+  }
+
+  async initialize(): Promise<void> {
+    if (this.extractor !== null) return;
+    // @ts-expect-error TS2590: TypeScript can't represent the complex union type from pipeline()
+    // This is a known limitation with @huggingface/transformers overloaded signatures
+    this.extractor = await pipeline('feature-extraction', this.modelName, {
+      dtype: 'fp32',
+    });
+  }
+
+  async embed(text: string): Promise<number[]> {
+    if (this.extractor === null) {
+      await this.initialize();
+    }
+    if (this.extractor === null) {
+      throw new Error('Failed to initialize embedding model');
+    }
+    const output = await this.extractor(text, {
+      pooling: 'mean',
+      normalize: true,
+    });
+    const result = Array.from(output.data);
+    return result.map(v => Number(v));
+  }
+
+  async embedBatch(texts: string[]): Promise<number[][]> {
+    const BATCH_SIZE = 32; // Process 32 chunks in parallel
+    const results: number[][] = [];
+
+    for (let i = 0; i < texts.length; i += BATCH_SIZE) {
+      const batch = texts.slice(i, i + BATCH_SIZE);
+
+      // Process batch in parallel using Promise.all
+      const batchResults = await Promise.all(
+        batch.map(text => this.embed(text))
+      );
+
+      results.push(...batchResults);
+
+      // Small delay between batches to prevent memory issues
+      if (i + BATCH_SIZE < texts.length) {
+        await new Promise(resolve => setTimeout(resolve, 100));
+      }
+    }
+
+    return results;
+  }
+
+  getDimensions(): number {
+    return this.dimensions;
+  }
+}

package/src/db/index.ts

@@ -0,0 +1,2 @@
+export { EmbeddingEngine } from './embeddings.js';
+export { LanceStore } from './lance.js';