bluera-knowledge 0.9.26 → 0.9.30

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bluera-knowledge",
-  "version": "0.9.26",
+  "version": "0.9.30",
   "description": "CLI tool for managing knowledge stores with semantic search",
   "type": "module",
   "bin": {
@@ -92,6 +92,8 @@
     "hono": "^4.11.1",
     "node-addon-api": "^8.5.0",
     "ora": "^9.0.0",
+    "pino": "^9.6.0",
+    "pino-roll": "^1.3.0",
     "slurp-ai": "^1.0.6",
     "tree-sitter": "^0.25.0",
     "tree-sitter-go": "^0.23.1",

package/src/analysis/ast-parser.test.ts

@@ -356,6 +356,52 @@ import type { Props } from "./types";
   });
 
   describe('Edge cases and special scenarios', () => {
+    it('handles default exported class', () => {
+      const code = 'export default class DefaultClass { method() {} }';
+      const nodes = parser.parse(code, 'typescript');
+
+      expect(nodes).toHaveLength(1);
+      expect(nodes[0]?.exported).toBe(true);
+      expect(nodes[0]?.name).toBe('DefaultClass');
+    });
+
+    it('handles anonymous default exported class (no id)', () => {
+      const code = 'export default class { method() {} }';
+      const nodes = parser.parse(code, 'javascript');
+
+      // Anonymous classes don't have an id, should be skipped
+      expect(nodes.filter(n => n.type === 'class')).toHaveLength(0);
+    });
+
+    it('handles class with computed property method (non-identifier key)', () => {
+      const code = `class MyClass {
+  ['computed']() { return 42; }
+  normalMethod() { return 1; }
+}`;
+      const nodes = parser.parse(code, 'javascript');
+
+      // Computed properties have StringLiteral keys, not Identifier
+      // Only normalMethod should be captured
+      expect(nodes[0]?.methods).toHaveLength(1);
+      expect(nodes[0]?.methods?.[0]?.name).toBe('normalMethod');
+    });
+
+    it('handles class with rest parameters in method', () => {
+      const code = `class MyClass {
+  method(...args) { return args; }
+}`;
+      const nodes = parser.parse(code, 'javascript');
+
+      expect(nodes[0]?.methods?.[0]?.signature).toBe('method(param)');
+    });
+
+    it('handles function with rest parameters', () => {
+      const code = 'function spread(...items) { return items; }';
+      const nodes = parser.parse(code, 'javascript');
+
+      expect(nodes[0]?.signature).toBe('spread(param)');
+    });
+
     it('handles functions with complex destructured parameters', () => {
       const code = 'function complex({ a, b }, [c, d]) { return a + b + c + d; }';
       const nodes = parser.parse(code, 'javascript');

package/src/cli/commands/crawl.test.ts

@@ -38,6 +38,7 @@ describe('crawl command execution', () => {
       store: {
         getByIdOrName: vi.fn(),
         list: vi.fn(),
+        create: vi.fn(),
       },
       lance: {
         initialize: vi.fn(),
@@ -291,23 +292,110 @@ describe('crawl command execution', () => {
     });
   });
 
-  describe('error handling', () => {
-    it('exits with code 3 when store not found', async () => {
+  describe('store auto-creation', () => {
+    it('auto-creates web store when store does not exist', async () => {
+      const createdStore: WebStore = {
+        id: createStoreId('new-store-id'),
+        name: 'new-store',
+        type: 'web',
+        url: 'https://example.com',
+        depth: 2,
+        createdAt: new Date(),
+        updatedAt: new Date(),
+      };
+
       mockServices.store.getByIdOrName.mockResolvedValue(undefined);
+      mockServices.store.create.mockResolvedValue({ success: true, data: createdStore });
+      mockServices.lance.initialize.mockResolvedValue(undefined);
+      mockServices.embeddings.embed.mockResolvedValue([0.1, 0.2, 0.3]);
+      mockServices.lance.addDocuments.mockResolvedValue(undefined);
+
+      mockCrawler.crawl.mockReturnValue(
+        (async function* () {
+          yield {
+            url: 'https://example.com/page1',
+            title: 'Page 1',
+            markdown: '# Content',
+            depth: 0,
+          };
+        })()
+      );
 
       const command = createCrawlCommand(getOptions);
       const actionHandler = command._actionHandler;
 
-      await expect(actionHandler(['https://example.com', 'nonexistent-store'])).rejects.toThrow('process.exit: 3');
+      await actionHandler(['https://example.com', 'new-store']);
 
-      expect(consoleErrorSpy).toHaveBeenCalledWith(
-        'Error: Web store not found: nonexistent-store'
+      expect(mockServices.store.create).toHaveBeenCalledWith({
+        name: 'new-store',
+        type: 'web',
+        url: 'https://example.com',
+      });
+      expect(consoleLogSpy).toHaveBeenCalledWith('Created web store: new-store');
+      expect(mockCrawler.crawl).toHaveBeenCalled();
+    });
+
+    it('throws error when store creation fails', async () => {
+      mockServices.store.getByIdOrName.mockResolvedValue(undefined);
+      mockServices.store.create.mockResolvedValue({ success: false, error: new Error('Name already exists') });
+
+      const command = createCrawlCommand(getOptions);
+      const actionHandler = command._actionHandler;
+
+      await expect(actionHandler(['https://example.com', 'bad-store'])).rejects.toThrow(
+        'Failed to create store: Name already exists'
       );
-      expect(processExitSpy).toHaveBeenCalledWith(3);
       expect(mockCrawler.crawl).not.toHaveBeenCalled();
     });
 
-    it('exits with code 3 when store is not a web store', async () => {
+    it('includes storeCreated in JSON output when store was created', async () => {
+      const createdStore: WebStore = {
+        id: createStoreId('new-store-id'),
+        name: 'new-store',
+        type: 'web',
+        url: 'https://example.com',
+        depth: 2,
+        createdAt: new Date(),
+        updatedAt: new Date(),
+      };
+
+      mockServices.store.getByIdOrName.mockResolvedValue(undefined);
+      mockServices.store.create.mockResolvedValue({ success: true, data: createdStore });
+      mockServices.lance.initialize.mockResolvedValue(undefined);
+      mockServices.embeddings.embed.mockResolvedValue([0.1, 0.2, 0.3]);
+      mockServices.lance.addDocuments.mockResolvedValue(undefined);
+
+      mockCrawler.crawl.mockReturnValue(
+        (async function* () {
+          yield {
+            url: 'https://example.com/page1',
+            title: 'Page 1',
+            markdown: '# Content',
+            depth: 0,
+          };
+        })()
+      );
+
+      getOptions = () => ({
+        config: undefined,
+        dataDir: '/tmp/test',
+        quiet: false,
+        format: 'json',
+      });
+
+      const command = createCrawlCommand(getOptions);
+      const actionHandler = command._actionHandler;
+
+      await actionHandler(['https://example.com', 'new-store']);
+
+      expect(consoleLogSpy).toHaveBeenCalledWith(
+        expect.stringContaining('"storeCreated": true')
+      );
+    });
+  });
+
+  describe('error handling', () => {
+    it('throws error when store is not a web store', async () => {
       const mockFileStore = {
         id: createStoreId('store-1'),
         name: 'file-store',
@@ -322,10 +410,9 @@ describe('crawl command execution', () => {
       const command = createCrawlCommand(getOptions);
       const actionHandler = command._actionHandler;
 
-      await expect(actionHandler(['https://example.com', 'file-store'])).rejects.toThrow('process.exit: 3');
-
-      expect(consoleErrorSpy).toHaveBeenCalledWith('Error: Web store not found: file-store');
-      expect(processExitSpy).toHaveBeenCalledWith(3);
+      await expect(actionHandler(['https://example.com', 'file-store'])).rejects.toThrow(
+        'Store "file-store" exists but is not a web store (type: file)'
+      );
     });
 
     it('exits with code 6 when crawling fails', async () => {
@@ -933,7 +1020,7 @@ describe('crawl command execution', () => {
         await actionHandler(['https://example.com', 'test-store']);
 
         expect(consoleLogSpy).toHaveBeenCalledWith('Crawling https://example.com');
-        expect(consoleLogSpy).toHaveBeenCalledWith('Crawled and indexed 1 pages');
+        expect(consoleLogSpy).toHaveBeenCalledWith('Crawled 1 pages, indexed 1 chunks');
       } finally {
         Object.defineProperty(process.stdout, 'isTTY', { value: originalIsTTY, configurable: true });
       }

package/src/cli/commands/crawl.ts

@@ -6,6 +6,9 @@ import { IntelligentCrawler, type CrawlProgress } from '../../crawl/intelligent-
 import { createDocumentId } from '../../types/brands.js';
 import type { GlobalOptions } from '../program.js';
 import type { Document } from '../../types/document.js';
+import type { WebStore } from '../../types/store.js';
+import { ChunkingService } from '../../services/chunking.service.js';
+import { classifyWebContentType } from '../../services/index.service.js';
 
 export function createCrawlCommand(getOptions: () => GlobalOptions): Command {
   return new Command('crawl')
@@ -27,10 +30,38 @@ export function createCrawlCommand(getOptions: () => GlobalOptions): Command {
       const globalOpts = getOptions();
       const services = await createServices(globalOpts.config, globalOpts.dataDir);
 
-      const store = await services.store.getByIdOrName(storeIdOrName);
-      if (!store || store.type !== 'web') {
-        console.error(`Error: Web store not found: ${storeIdOrName}`);
-        process.exit(3);
+      // Look up or auto-create web store
+      let store: WebStore;
+      let storeCreated = false;
+      const existingStore = await services.store.getByIdOrName(storeIdOrName);
+
+      if (!existingStore) {
+        // Auto-create web store
+        const result = await services.store.create({
+          name: storeIdOrName,
+          type: 'web',
+          url,
+        });
+        if (!result.success) {
+          await destroyServices(services);
+          throw new Error(`Failed to create store: ${result.error.message}`);
+        }
+        // Type narrowing: success check above ensures result.data is Store
+        // We know it's a WebStore because we created it with type: 'web'
+        const createdStore = result.data;
+        if (createdStore.type !== 'web') {
+          throw new Error('Unexpected store type after creation');
+        }
+        store = createdStore;
+        storeCreated = true;
+        if (globalOpts.quiet !== true && globalOpts.format !== 'json') {
+          console.log(`Created web store: ${store.name}`);
+        }
+      } else if (existingStore.type !== 'web') {
+        await destroyServices(services);
+        throw new Error(`Store "${storeIdOrName}" exists but is not a web store (type: ${existingStore.type})`);
+      } else {
+        store = existingStore;
       }
 
       const maxPages = cmdOptions.maxPages !== undefined ? parseInt(cmdOptions.maxPages) : 50;
@@ -47,7 +78,10 @@ export function createCrawlCommand(getOptions: () => GlobalOptions): Command {
       }
 
       const crawler = new IntelligentCrawler();
+      // Use web preset for larger prose-friendly chunks
+      const webChunker = ChunkingService.forContentType('web');
       let pagesIndexed = 0;
+      let chunksCreated = 0;
 
       // Listen for progress events
       crawler.on('progress', (progress: CrawlProgress) => {
@@ -78,24 +112,40 @@ export function createCrawlCommand(getOptions: () => GlobalOptions): Command {
           ...(cmdOptions.simple !== undefined && { simple: cmdOptions.simple }),
           useHeadless: cmdOptions.headless ?? false,
         })) {
-          // Embed and index the content (use extracted if available, otherwise markdown)
-          const contentToEmbed = result.extracted !== undefined ? result.extracted : result.markdown;
-          const vector = await services.embeddings.embed(contentToEmbed);
-
-          docs.push({
-            id: createDocumentId(`${store.id}-${createHash('md5').update(result.url).digest('hex')}`),
-            content: contentToEmbed,
-            vector,
-            metadata: {
-              type: 'web',
-              storeId: store.id,
-              url: result.url,
-              title: result.title,
-              extracted: result.extracted !== undefined,
-              depth: result.depth,
-              indexedAt: new Date(),
-            },
-          });
+          // Use extracted content if available, otherwise markdown
+          const contentToProcess = result.extracted !== undefined ? result.extracted : result.markdown;
+
+          // Chunk the content using markdown-aware chunking (web content is converted to markdown)
+          const chunks = webChunker.chunk(contentToProcess, `${result.url}.md`);
+          const fileType = classifyWebContentType(result.url, result.title);
+          const urlHash = createHash('md5').update(result.url).digest('hex');
+
+          for (const chunk of chunks) {
+            const chunkId = chunks.length > 1
+              ? `${store.id}-${urlHash}-${String(chunk.chunkIndex)}`
+              : `${store.id}-${urlHash}`;
+            const vector = await services.embeddings.embed(chunk.content);
+
+            docs.push({
+              id: createDocumentId(chunkId),
+              content: chunk.content,
+              vector,
+              metadata: {
+                type: chunks.length > 1 ? 'chunk' : 'web',
+                storeId: store.id,
+                url: result.url,
+                title: result.title,
+                extracted: result.extracted !== undefined,
+                depth: result.depth,
+                indexedAt: new Date(),
+                fileType,
+                chunkIndex: chunk.chunkIndex,
+                totalChunks: chunk.totalChunks,
+                sectionHeader: chunk.sectionHeader,
+              },
+            });
+            chunksCreated++;
+          }
 
           pagesIndexed++;
         }
@@ -111,8 +161,10 @@ export function createCrawlCommand(getOptions: () => GlobalOptions): Command {
         const crawlResult = {
           success: true,
           store: store.name,
+          storeCreated,
           url,
           pagesCrawled: pagesIndexed,
+          chunksCreated,
           mode: cmdOptions.simple === true ? 'simple' : 'intelligent',
           hadCrawlInstruction: cmdOptions.crawl !== undefined,
           hadExtractInstruction: cmdOptions.extract !== undefined,
@@ -121,9 +173,9 @@ export function createCrawlCommand(getOptions: () => GlobalOptions): Command {
         if (globalOpts.format === 'json') {
           console.log(JSON.stringify(crawlResult, null, 2));
         } else if (spinner !== undefined) {
-          spinner.succeed(`Crawled and indexed ${String(pagesIndexed)} pages`);
+          spinner.succeed(`Crawled ${String(pagesIndexed)} pages, indexed ${String(chunksCreated)} chunks`);
         } else if (globalOpts.quiet !== true) {
-          console.log(`Crawled and indexed ${String(pagesIndexed)} pages`);
+          console.log(`Crawled ${String(pagesIndexed)} pages, indexed ${String(chunksCreated)} chunks`);
         }
       } catch (error) {
         const message = `Crawl failed: ${error instanceof Error ? error.message : String(error)}`;

package/src/crawl/article-converter.ts

@@ -7,6 +7,9 @@ import { extractFromHtml } from '@extractus/article-extractor';
 import TurndownService from 'turndown';
 import { gfm } from 'turndown-plugin-gfm';
 import { preprocessHtmlForCodeBlocks, cleanupMarkdown } from './markdown-utils.js';
+import { createLogger, truncateForLog } from '../logging/index.js';
+
+const logger = createLogger('article-converter');
 
 export interface ConversionResult {
   markdown: string;
@@ -28,6 +31,8 @@ export async function convertHtmlToMarkdown(
   html: string,
   url: string,
 ): Promise<ConversionResult> {
+  logger.debug({ url, htmlLength: html.length }, 'Starting HTML conversion');
+
   try {
     // Step 1: Extract main article content
     let articleHtml: string;
@@ -38,13 +43,25 @@ export async function convertHtmlToMarkdown(
       if (article !== null && article.content !== undefined && article.content !== '') {
         articleHtml = article.content;
         title = article.title !== undefined && article.title !== '' ? article.title : undefined;
+        logger.debug({
+          url,
+          title,
+          extractedLength: articleHtml.length,
+          usedFullHtml: false,
+        }, 'Article content extracted');
       } else {
         // Fallback to full HTML if extraction fails
         articleHtml = html;
+        logger.debug({ url, usedFullHtml: true }, 'Article extraction returned empty, using full HTML');
       }
-    } catch {
+    } catch (extractError) {
       // Fallback to full HTML if extraction fails
       articleHtml = html;
+      logger.debug({
+        url,
+        usedFullHtml: true,
+        error: extractError instanceof Error ? extractError.message : String(extractError),
+      }, 'Article extraction failed, using full HTML');
     }
 
     // Step 2: Preprocess HTML for code blocks
@@ -83,12 +100,30 @@ export async function convertHtmlToMarkdown(
     // Step 4: Cleanup markdown with comprehensive regex patterns
     const markdown = cleanupMarkdown(rawMarkdown);
 
+    logger.debug({
+      url,
+      title,
+      rawMarkdownLength: rawMarkdown.length,
+      finalMarkdownLength: markdown.length,
+    }, 'HTML to markdown conversion complete');
+
+    // Log markdown preview at trace level
+    logger.trace({
+      url,
+      markdownPreview: truncateForLog(markdown, 1000),
+    }, 'Markdown content preview');
+
     return {
       markdown,
       ...(title !== undefined && { title }),
       success: true,
     };
   } catch (error) {
+    logger.error({
+      url,
+      error: error instanceof Error ? error.message : String(error),
+    }, 'HTML to markdown conversion failed');
+
     return {
       markdown: '',
       success: false,

package/src/crawl/bridge.ts

@@ -11,6 +11,9 @@ import {
   validateHeadlessResult,
   validateParsePythonResult,
 } from './schemas.js';
+import { createLogger } from '../logging/index.js';
+
+const logger = createLogger('python-bridge');
 
 // Re-export for backwards compatibility
 export type { CrawledLink, ParsePythonResult };
@@ -32,24 +35,26 @@ export class PythonBridge {
   start(): Promise<void> {
     if (this.process) return Promise.resolve();
 
+    logger.debug('Starting Python bridge process');
+
     this.process = spawn('python3', ['python/crawl_worker.py'], {
       stdio: ['pipe', 'pipe', 'pipe'],
     });
 
     // Add error handler for process spawn errors
     this.process.on('error', (err) => {
-      console.error('Python bridge process error:', err);
+      logger.error({ error: err.message, stack: err.stack }, 'Python bridge process error');
       this.rejectAllPending(new Error(`Process error: ${err.message}`));
     });
 
     // Add exit handler to detect non-zero exits
     this.process.on('exit', (code, signal) => {
       if (code !== 0 && code !== null) {
-        console.error(`Python bridge process exited with code ${String(code)}`);
+        logger.error({ code }, 'Python bridge process exited with non-zero code');
         this.rejectAllPending(new Error(`Process exited with code ${String(code)}`));
       } else if (signal && !this.stoppingIntentionally) {
         // Only log if we didn't intentionally stop the process
-        console.error(`Python bridge process killed with signal ${signal}`);
+        logger.error({ signal }, 'Python bridge process killed with signal');
         this.rejectAllPending(new Error(`Process killed with signal ${signal}`));
       }
       this.process = null;
@@ -60,7 +65,7 @@ export class PythonBridge {
     if (this.process.stderr) {
       const stderrRl = createInterface({ input: this.process.stderr });
       stderrRl.on('line', (line) => {
-        console.error('Python bridge stderr:', line);
+        logger.warn({ stderr: line }, 'Python bridge stderr output');
       });
     }
 
@@ -106,11 +111,14 @@ export class PythonBridge {
             } catch (error: unknown) {
               // Log validation failure with original response for debugging
               if (error instanceof ZodError) {
-                console.error('Python bridge response validation failed:', error.issues);
-                console.error('Original response:', JSON.stringify(response.result));
+                logger.error({
+                  issues: error.issues,
+                  response: JSON.stringify(response.result),
+                }, 'Python bridge response validation failed');
                 pending.reject(new Error(`Invalid response format from Python bridge: ${error.message}`));
               } else {
                 const errorMessage = error instanceof Error ? error.message : String(error);
+                logger.error({ error: errorMessage }, 'Response validation error');
                 pending.reject(new Error(`Response validation error: ${errorMessage}`));
               }
             }
@@ -118,7 +126,10 @@ export class PythonBridge {
           // If neither result nor error, leave pending (will timeout)
         }
       } catch (err) {
-        console.error('Failed to parse JSON response from Python bridge:', err, 'Line:', line);
+        logger.error({
+          error: err instanceof Error ? err.message : String(err),
+          line,
+        }, 'Failed to parse JSON response from Python bridge');
       }
     });
 

package/src/crawl/intelligent-crawler.ts

@@ -8,6 +8,9 @@ import axios from 'axios';
 import { ClaudeClient, type CrawlStrategy } from './claude-client.js';
 import { convertHtmlToMarkdown } from './article-converter.js';
 import { PythonBridge, type CrawledLink } from './bridge.js';
+import { createLogger, summarizePayload } from '../logging/index.js';
+
+const logger = createLogger('crawler');
 
 export interface CrawlOptions {
   crawlInstruction?: string; // Natural language: what to crawl
@@ -69,6 +72,13 @@ export class IntelligentCrawler extends EventEmitter {
     this.visited.clear();
     this.stopped = false;
 
+    logger.info({
+      seedUrl,
+      maxPages,
+      mode: simple ? 'simple' : (crawlInstruction !== undefined && crawlInstruction !== '' ? 'intelligent' : 'simple'),
+      hasExtractInstruction: extractInstruction !== undefined,
+    }, 'Starting crawl');
+
     const startProgress: CrawlProgress = {
       type: 'start',
       pagesVisited: 0,
@@ -86,6 +96,11 @@ export class IntelligentCrawler extends EventEmitter {
       yield* this.crawlSimple(seedUrl, extractInstruction, maxPages, options.useHeadless ?? false);
     }
 
+    logger.info({
+      seedUrl,
+      pagesVisited: this.visited.size,
+    }, 'Crawl complete');
+
     const completeProgress: CrawlProgress = {
       type: 'complete',
       pagesVisited: this.visited.size,
@@ -220,9 +235,9 @@ export class IntelligentCrawler extends EventEmitter {
             const links = await this.extractLinks(current.url, useHeadless);
 
             if (links.length === 0) {
-              console.warn(`No links found on ${current.url} - page may be a leaf node`);
+              logger.debug({ url: current.url }, 'No links found - page may be a leaf node');
             } else {
-              console.log(`Found ${String(links.length)} links on ${current.url}`);
+              logger.debug({ url: current.url, linkCount: links.length }, 'Links extracted from page');
             }
 
             for (const link of links) {
@@ -283,9 +298,16 @@ export class IntelligentCrawler extends EventEmitter {
     const conversion = await convertHtmlToMarkdown(html, url);
 
     if (!conversion.success) {
+      logger.error({ url, error: conversion.error }, 'HTML to markdown conversion failed');
       throw new Error(`Failed to convert HTML: ${conversion.error ?? 'Unknown error'}`);
     }
 
+    logger.debug({
+      url,
+      title: conversion.title,
+      markdownLength: conversion.markdown.length,
+    }, 'Article converted to markdown');
+
     let extracted: string | undefined;
 
     // Optional: Extract specific information using Claude
@@ -342,13 +364,23 @@ export class IntelligentCrawler extends EventEmitter {
    * Fetch HTML content from a URL
    */
   private async fetchHtml(url: string, useHeadless: boolean = false): Promise<string> {
+    const startTime = Date.now();
+    logger.debug({ url, useHeadless }, 'Fetching HTML');
+
     if (useHeadless) {
       try {
         const result = await this.pythonBridge.fetchHeadless(url);
+        const durationMs = Date.now() - startTime;
+        logger.info({
+          url,
+          useHeadless: true,
+          durationMs,
+          ...summarizePayload(result.html, 'raw-html', url),
+        }, 'Raw HTML fetched');
         return result.html;
       } catch (error) {
         // Fallback to axios if headless fails
-        console.warn(`Headless fetch failed for ${url}, falling back to axios:`, error);
+        logger.warn({ url, error: error instanceof Error ? error.message : String(error) }, 'Headless fetch failed, falling back to axios');
       }
     }
 
@@ -362,8 +394,17 @@ export class IntelligentCrawler extends EventEmitter {
         },
       });
 
+      const durationMs = Date.now() - startTime;
+      logger.info({
+        url,
+        useHeadless: false,
+        durationMs,
+        ...summarizePayload(response.data, 'raw-html', url),
+      }, 'Raw HTML fetched');
+
       return response.data;
     } catch (error) {
+      logger.error({ url, error: error instanceof Error ? error.message : String(error) }, 'Failed to fetch HTML');
       throw new Error(
         `Failed to fetch ${url}: ${error instanceof Error ? error.message : String(error)}`,
       );
@@ -398,7 +439,7 @@ export class IntelligentCrawler extends EventEmitter {
     } catch (error: unknown) {
       // Log the error for debugging
       const errorMessage = error instanceof Error ? error.message : String(error);
-      console.error(`Failed to extract links from ${url}:`, errorMessage);
+      logger.error({ url, error: errorMessage }, 'Failed to extract links');
 
       // Re-throw the error instead of silently swallowing it
       throw new Error(`Link extraction failed for ${url}: ${errorMessage}`);

package/src/db/embeddings.test.ts

@@ -48,6 +48,22 @@ describe('EmbeddingEngine', () => {
     const customEngine = new EmbeddingEngine('Xenova/all-MiniLM-L6-v2', 512);
     expect(customEngine.getDimensions()).toBe(512);
   });
+
+  it('skips initialization when already initialized', async () => {
+    // Engine is already initialized from beforeAll
+    // Calling initialize again should be a no-op
+    await engine.initialize();
+    const embedding = await engine.embed('test');
+    expect(embedding).toHaveLength(384);
+  });
+
+  it('handles large batch with multiple chunks', async () => {
+    // Create enough texts to trigger multiple batch iterations (BATCH_SIZE = 32)
+    const texts = Array.from({ length: 40 }, (_, i) => `Text number ${String(i)}`);
+    const embeddings = await engine.embedBatch(texts);
+    expect(embeddings).toHaveLength(40);
+    expect(embeddings.every((e) => e.length === 384)).toBe(true);
+  }, 60000);
 });
 
 function cosineSimilarity(a: number[], b: number[]): number {