bluera-knowledge 0.9.32 → 0.9.36

package/src/analysis/tree-sitter-parser.test.ts

@@ -17,7 +17,7 @@ import {
   queryNodesByType,
   extractImportPath,
   type TreeSitterNode,
-  type TreeSitterTree
+  type TreeSitterTree,
 } from './tree-sitter-parser.js';
 
 describe('tree-sitter-parser', () => {
@@ -252,7 +252,7 @@ describe('tree-sitter-parser', () => {
       // Create a mock node without a name field
       const mockNode: Partial<TreeSitterNode> = {
         children: [],
-        childForFieldName: () => null
+        childForFieldName: () => null,
       };
       const sig = getFunctionSignature(mockNode as TreeSitterNode);
       expect(sig).toBe('');
@@ -288,7 +288,7 @@ describe('tree-sitter-parser', () => {
     it('returns empty string when argument node is missing', () => {
       // Create a mock node without an argument field
       const mockNode: Partial<TreeSitterNode> = {
-        childForFieldName: () => null
+        childForFieldName: () => null,
       };
       const path = extractImportPath(mockNode as TreeSitterNode);
       expect(path).toBe('');

package/src/analysis/tree-sitter-parser.ts

@@ -1,6 +1,6 @@
 import Parser from 'tree-sitter';
-import Rust from 'tree-sitter-rust';
 import Go from 'tree-sitter-go';
+import Rust from 'tree-sitter-rust';
 
 /**
  * Tree-sitter infrastructure for parsing Rust and Go code.
@@ -105,21 +105,15 @@ export function getNodeText(node: TreeSitterNode): string {
 /**
  * Get all children of a specific type
  */
-export function getChildrenOfType(
-  node: TreeSitterNode,
-  type: string
-): TreeSitterNode[] {
-  return node.children.filter(child => child.type === type);
+export function getChildrenOfType(node: TreeSitterNode, type: string): TreeSitterNode[] {
+  return node.children.filter((child) => child.type === type);
 }
 
 /**
  * Get the first child of a specific type
  */
-export function getFirstChildOfType(
-  node: TreeSitterNode,
-  type: string
-): TreeSitterNode | null {
-  return node.children.find(child => child.type === type) ?? null;
+export function getFirstChildOfType(node: TreeSitterNode, type: string): TreeSitterNode | null {
+  return node.children.find((child) => child.type === type) ?? null;
 }
 
 /**
@@ -136,14 +130,14 @@ export function getChildByFieldName(
  * Check if node has a visibility modifier (pub)
  */
 export function hasVisibilityModifier(node: TreeSitterNode): boolean {
-  return node.children.some(child => child.type === 'visibility_modifier');
+  return node.children.some((child) => child.type === 'visibility_modifier');
 }
 
 /**
  * Get visibility modifier text (e.g., "pub", "pub(crate)")
  */
 export function getVisibilityModifier(node: TreeSitterNode): string | null {
-  const visNode = node.children.find(child => child.type === 'visibility_modifier');
+  const visNode = node.children.find((child) => child.type === 'visibility_modifier');
   return visNode !== undefined ? visNode.text : null;
 }
 
@@ -152,14 +146,14 @@ export function getVisibilityModifier(node: TreeSitterNode): string | null {
  */
 export function isAsyncFunction(node: TreeSitterNode): boolean {
   // Check for 'async' keyword in function_item or function_signature_item
-  return node.children.some(child => child.type === 'async' || child.text === 'async');
+  return node.children.some((child) => child.type === 'async' || child.text === 'async');
 }
 
 /**
  * Check if a function is unsafe
  */
 export function isUnsafeFunction(node: TreeSitterNode): boolean {
-  return node.children.some(child => child.type === 'unsafe' || child.text === 'unsafe');
+  return node.children.some((child) => child.type === 'unsafe' || child.text === 'unsafe');
 }
 
 /**
@@ -190,7 +184,7 @@ export function getFunctionSignature(node: TreeSitterNode): string {
 
   // Add return type
   if (returnTypeNode !== null) {
-    signature += ' ' + returnTypeNode.text;
+    signature += ` ${returnTypeNode.text}`;
   }
 
   return signature;

package/src/cli/commands/crawl.test.ts

@@ -57,7 +57,7 @@ describe('crawl command execution', () => {
     };
 
     vi.mocked(createServices).mockResolvedValue(mockServices);
-    vi.mocked(IntelligentCrawler).mockImplementation(function(this: any) {
+    vi.mocked(IntelligentCrawler).mockImplementation(function (this: any) {
       return mockCrawler as any;
     } as any);
 
@@ -278,7 +278,14 @@ describe('crawl command execution', () => {
       );
 
       const command = createCrawlCommand(getOptions);
-      command.parseOptions(['--crawl', 'all Getting Started pages', '--extract', 'code examples', '--max-pages', '100']);
+      command.parseOptions([
+        '--crawl',
+        'all Getting Started pages',
+        '--extract',
+        'code examples',
+        '--max-pages',
+        '100',
+      ]);
       const actionHandler = command._actionHandler;
 
       await actionHandler(['https://example.com', 'test-store']);
@@ -337,7 +344,10 @@ describe('crawl command execution', () => {
 
     it('throws error when store creation fails', async () => {
       mockServices.store.getByIdOrName.mockResolvedValue(undefined);
-      mockServices.store.create.mockResolvedValue({ success: false, error: new Error('Name already exists') });
+      mockServices.store.create.mockResolvedValue({
+        success: false,
+        error: new Error('Name already exists'),
+      });
 
       const command = createCrawlCommand(getOptions);
       const actionHandler = command._actionHandler;
@@ -388,9 +398,7 @@ describe('crawl command execution', () => {
 
       await actionHandler(['https://example.com', 'new-store']);
 
-      expect(consoleLogSpy).toHaveBeenCalledWith(
-        expect.stringContaining('"storeCreated": true')
-      );
+      expect(consoleLogSpy).toHaveBeenCalledWith(expect.stringContaining('"storeCreated": true'));
     });
   });
 
@@ -439,7 +447,9 @@ describe('crawl command execution', () => {
       const command = createCrawlCommand(getOptions);
       const actionHandler = command._actionHandler;
 
-      await expect(actionHandler(['https://example.com', 'test-store'])).rejects.toThrow('process.exit: 6');
+      await expect(actionHandler(['https://example.com', 'test-store'])).rejects.toThrow(
+        'process.exit: 6'
+      );
 
       expect(consoleErrorSpy).toHaveBeenCalledWith('Error: Crawl failed: Network timeout');
       expect(processExitSpy).toHaveBeenCalledWith(6);
@@ -475,7 +485,9 @@ describe('crawl command execution', () => {
       const command = createCrawlCommand(getOptions);
       const actionHandler = command._actionHandler;
 
-      await expect(actionHandler(['https://example.com', 'test-store'])).rejects.toThrow('process.exit: 6');
+      await expect(actionHandler(['https://example.com', 'test-store'])).rejects.toThrow(
+        'process.exit: 6'
+      );
 
       expect(consoleErrorSpy).toHaveBeenCalledWith(
         'Error: Crawl failed: Embedding service unavailable'
@@ -514,7 +526,9 @@ describe('crawl command execution', () => {
       const command = createCrawlCommand(getOptions);
       const actionHandler = command._actionHandler;
 
-      await expect(actionHandler(['https://example.com', 'test-store'])).rejects.toThrow('process.exit: 6');
+      await expect(actionHandler(['https://example.com', 'test-store'])).rejects.toThrow(
+        'process.exit: 6'
+      );
 
       expect(consoleErrorSpy).toHaveBeenCalledWith('Error: Crawl failed: Database write error');
       expect(processExitSpy).toHaveBeenCalledWith(6);
@@ -592,18 +606,10 @@ describe('crawl command execution', () => {
 
       await actionHandler(['https://example.com', 'test-store']);
 
-      expect(consoleLogSpy).toHaveBeenCalledWith(
-        expect.stringContaining('"success": true')
-      );
-      expect(consoleLogSpy).toHaveBeenCalledWith(
-        expect.stringContaining('"store": "test-store"')
-      );
-      expect(consoleLogSpy).toHaveBeenCalledWith(
-        expect.stringContaining('"pagesCrawled": 1')
-      );
-      expect(consoleLogSpy).toHaveBeenCalledWith(
-        expect.stringContaining('"mode": "intelligent"')
-      );
+      expect(consoleLogSpy).toHaveBeenCalledWith(expect.stringContaining('"success": true'));
+      expect(consoleLogSpy).toHaveBeenCalledWith(expect.stringContaining('"store": "test-store"'));
+      expect(consoleLogSpy).toHaveBeenCalledWith(expect.stringContaining('"pagesCrawled": 1'));
+      expect(consoleLogSpy).toHaveBeenCalledWith(expect.stringContaining('"mode": "intelligent"'));
       expect(consoleLogSpy).toHaveBeenCalledWith(
         expect.stringContaining('"hadCrawlInstruction": true')
       );
@@ -950,7 +956,11 @@ describe('crawl command execution', () => {
           if (progressCallback) {
             progressCallback({ type: 'strategy', message: 'Planning crawl...' });
             progressCallback({ type: 'strategy', message: undefined }); // Test fallback
-            progressCallback({ type: 'page', pagesVisited: 0, currentUrl: 'https://example.com/page1' });
+            progressCallback({
+              type: 'page',
+              pagesVisited: 0,
+              currentUrl: 'https://example.com/page1',
+            });
             progressCallback({ type: 'page', pagesVisited: 1, currentUrl: undefined }); // Test fallback
             progressCallback({ type: 'extraction', currentUrl: 'https://example.com/page1' });
             progressCallback({ type: 'extraction', currentUrl: undefined }); // Test fallback
@@ -978,7 +988,10 @@ describe('crawl command execution', () => {
 
         expect(mockCrawler.on).toHaveBeenCalledWith('progress', expect.any(Function));
       } finally {
-        Object.defineProperty(process.stdout, 'isTTY', { value: originalIsTTY, configurable: true });
+        Object.defineProperty(process.stdout, 'isTTY', {
+          value: originalIsTTY,
+          configurable: true,
+        });
       }
     });
 
@@ -1022,7 +1035,10 @@ describe('crawl command execution', () => {
         expect(consoleLogSpy).toHaveBeenCalledWith('Crawling https://example.com');
         expect(consoleLogSpy).toHaveBeenCalledWith('Crawled 1 pages, indexed 1 chunks');
       } finally {
-        Object.defineProperty(process.stdout, 'isTTY', { value: originalIsTTY, configurable: true });
+        Object.defineProperty(process.stdout, 'isTTY', {
+          value: originalIsTTY,
+          configurable: true,
+        });
       }
     });
   });

package/src/cli/commands/crawl.ts

@@ -1,198 +1,218 @@
-import { Command } from 'commander';
 import { createHash } from 'node:crypto';
+import { Command } from 'commander';
 import ora, { type Ora } from 'ora';
-import { createServices, destroyServices } from '../../services/index.js';
 import { IntelligentCrawler, type CrawlProgress } from '../../crawl/intelligent-crawler.js';
+import { ChunkingService } from '../../services/chunking.service.js';
+import { createServices, destroyServices } from '../../services/index.js';
+import { classifyWebContentType } from '../../services/index.service.js';
 import { createDocumentId } from '../../types/brands.js';
-import type { GlobalOptions } from '../program.js';
 import type { Document } from '../../types/document.js';
 import type { WebStore } from '../../types/store.js';
-import { ChunkingService } from '../../services/chunking.service.js';
-import { classifyWebContentType } from '../../services/index.service.js';
+import type { GlobalOptions } from '../program.js';
 
 export function createCrawlCommand(getOptions: () => GlobalOptions): Command {
   return new Command('crawl')
     .description('Crawl web pages with natural language control and index into store')
     .argument('<url>', 'URL to crawl')
     .argument('<store>', 'Target web store to add crawled content to')
-    .option('--crawl <instruction>', 'Natural language instruction for what to crawl (e.g., "all Getting Started pages")')
-    .option('--extract <instruction>', 'Natural language instruction for what to extract (e.g., "extract API references")')
+    .option(
+      '--crawl <instruction>',
+      'Natural language instruction for what to crawl (e.g., "all Getting Started pages")'
+    )
+    .option(
+      '--extract <instruction>',
+      'Natural language instruction for what to extract (e.g., "extract API references")'
+    )
     .option('--simple', 'Use simple BFS mode instead of intelligent crawling')
     .option('--max-pages <number>', 'Maximum number of pages to crawl', '50')
     .option('--headless', 'Use headless browser for JavaScript-rendered sites')
-    .action(async (url: string, storeIdOrName: string, cmdOptions: {
-      crawl?: string;
-      extract?: string;
-      simple?: boolean;
-      maxPages?: string;
-      headless?: boolean;
-    }) => {
-      const globalOpts = getOptions();
-      const services = await createServices(globalOpts.config, globalOpts.dataDir);
-
-      // Look up or auto-create web store
-      let store: WebStore;
-      let storeCreated = false;
-      const existingStore = await services.store.getByIdOrName(storeIdOrName);
-
-      if (!existingStore) {
-        // Auto-create web store
-        const result = await services.store.create({
-          name: storeIdOrName,
-          type: 'web',
-          url,
-        });
-        if (!result.success) {
-          await destroyServices(services);
-          throw new Error(`Failed to create store: ${result.error.message}`);
-        }
-        // Type narrowing: success check above ensures result.data is Store
-        // We know it's a WebStore because we created it with type: 'web'
-        const createdStore = result.data;
-        if (createdStore.type !== 'web') {
-          throw new Error('Unexpected store type after creation');
+    .action(
+      async (
+        url: string,
+        storeIdOrName: string,
+        cmdOptions: {
+          crawl?: string;
+          extract?: string;
+          simple?: boolean;
+          maxPages?: string;
+          headless?: boolean;
         }
-        store = createdStore;
-        storeCreated = true;
-        if (globalOpts.quiet !== true && globalOpts.format !== 'json') {
-          console.log(`Created web store: ${store.name}`);
+      ) => {
+        const globalOpts = getOptions();
+        const services = await createServices(globalOpts.config, globalOpts.dataDir);
+
+        // Look up or auto-create web store
+        let store: WebStore;
+        let storeCreated = false;
+        const existingStore = await services.store.getByIdOrName(storeIdOrName);
+
+        if (!existingStore) {
+          // Auto-create web store
+          const result = await services.store.create({
+            name: storeIdOrName,
+            type: 'web',
+            url,
+          });
+          if (!result.success) {
+            await destroyServices(services);
+            throw new Error(`Failed to create store: ${result.error.message}`);
+          }
+          // Type narrowing: success check above ensures result.data is Store
+          // We know it's a WebStore because we created it with type: 'web'
+          const createdStore = result.data;
+          if (createdStore.type !== 'web') {
+            throw new Error('Unexpected store type after creation');
+          }
+          store = createdStore;
+          storeCreated = true;
+          if (globalOpts.quiet !== true && globalOpts.format !== 'json') {
+            console.log(`Created web store: ${store.name}`);
+          }
+        } else if (existingStore.type !== 'web') {
+          await destroyServices(services);
+          throw new Error(
+            `Store "${storeIdOrName}" exists but is not a web store (type: ${existingStore.type})`
+          );
+        } else {
+          store = existingStore;
         }
-      } else if (existingStore.type !== 'web') {
-        await destroyServices(services);
-        throw new Error(`Store "${storeIdOrName}" exists but is not a web store (type: ${existingStore.type})`);
-      } else {
-        store = existingStore;
-      }
 
-      const maxPages = cmdOptions.maxPages !== undefined ? parseInt(cmdOptions.maxPages) : 50;
+        const maxPages = cmdOptions.maxPages !== undefined ? parseInt(cmdOptions.maxPages) : 50;
 
-      // Use spinner in interactive mode
-      const isInteractive = process.stdout.isTTY && globalOpts.quiet !== true && globalOpts.format !== 'json';
-      let spinner: Ora | undefined;
+        // Use spinner in interactive mode
+        const isInteractive =
+          process.stdout.isTTY && globalOpts.quiet !== true && globalOpts.format !== 'json';
+        let spinner: Ora | undefined;
 
-      if (isInteractive) {
-        const mode = cmdOptions.simple === true ? 'simple' : 'intelligent';
-        spinner = ora(`Crawling ${url} (${mode} mode)`).start();
-      } else if (globalOpts.quiet !== true && globalOpts.format !== 'json') {
-        console.log(`Crawling ${url}`);
-      }
+        if (isInteractive) {
+          const mode = cmdOptions.simple === true ? 'simple' : 'intelligent';
+          spinner = ora(`Crawling ${url} (${mode} mode)`).start();
+        } else if (globalOpts.quiet !== true && globalOpts.format !== 'json') {
+          console.log(`Crawling ${url}`);
+        }
+
+        const crawler = new IntelligentCrawler();
+        // Use web preset for larger prose-friendly chunks
+        const webChunker = ChunkingService.forContentType('web');
+        let pagesIndexed = 0;
+        let chunksCreated = 0;
+        let exitCode = 0;
 
-      const crawler = new IntelligentCrawler();
-      // Use web preset for larger prose-friendly chunks
-      const webChunker = ChunkingService.forContentType('web');
-      let pagesIndexed = 0;
-      let chunksCreated = 0;
-      let exitCode = 0;
-
-      // Listen for progress events
-      crawler.on('progress', (progress: CrawlProgress) => {
-        if (spinner) {
-          if (progress.type === 'strategy') {
-            spinner.text = progress.message !== undefined ? progress.message : 'Analyzing crawl strategy...';
-          } else if (progress.type === 'page') {
-            const url = progress.currentUrl !== undefined ? progress.currentUrl : 'unknown';
-            spinner.text = `Crawling ${String(progress.pagesVisited + 1)}/${String(maxPages)} - ${url}`;
-          } else if (progress.type === 'extraction') {
-            const url = progress.currentUrl !== undefined ? progress.currentUrl : 'unknown';
-            spinner.text = `Extracting from ${url}...`;
-          } else if (progress.type === 'error' && progress.message !== undefined) {
-            spinner.warn(progress.message);
+        // Listen for progress events
+        crawler.on('progress', (progress: CrawlProgress) => {
+          if (spinner) {
+            if (progress.type === 'strategy') {
+              spinner.text = progress.message ?? 'Analyzing crawl strategy...';
+            } else if (progress.type === 'page') {
+              const url = progress.currentUrl ?? 'unknown';
+              spinner.text = `Crawling ${String(progress.pagesVisited + 1)}/${String(maxPages)} - ${url}`;
+            } else if (progress.type === 'extraction') {
+              const url = progress.currentUrl ?? 'unknown';
+              spinner.text = `Extracting from ${url}...`;
+            } else if (progress.type === 'error' && progress.message !== undefined) {
+              spinner.warn(progress.message);
+            }
           }
-        }
-      });
-
-      try {
-        await services.lance.initialize(store.id);
-        const docs: Document[] = [];
-
-        // Crawl pages using IntelligentCrawler
-        for await (const result of crawler.crawl(url, {
-          ...(cmdOptions.crawl !== undefined && { crawlInstruction: cmdOptions.crawl }),
-          ...(cmdOptions.extract !== undefined && { extractInstruction: cmdOptions.extract }),
-          maxPages,
-          ...(cmdOptions.simple !== undefined && { simple: cmdOptions.simple }),
-          useHeadless: cmdOptions.headless ?? false,
-        })) {
-          // Use extracted content if available, otherwise markdown
-          const contentToProcess = result.extracted !== undefined ? result.extracted : result.markdown;
-
-          // Chunk the content using markdown-aware chunking (web content is converted to markdown)
-          const chunks = webChunker.chunk(contentToProcess, `${result.url}.md`);
-          const fileType = classifyWebContentType(result.url, result.title);
-          const urlHash = createHash('md5').update(result.url).digest('hex');
-
-          for (const chunk of chunks) {
-            const chunkId = chunks.length > 1
-              ? `${store.id}-${urlHash}-${String(chunk.chunkIndex)}`
-              : `${store.id}-${urlHash}`;
-            const vector = await services.embeddings.embed(chunk.content);
-
-            docs.push({
-              id: createDocumentId(chunkId),
-              content: chunk.content,
-              vector,
-              metadata: {
-                type: chunks.length > 1 ? 'chunk' : 'web',
-                storeId: store.id,
-                url: result.url,
-                title: result.title,
-                extracted: result.extracted !== undefined,
-                depth: result.depth,
-                indexedAt: new Date(),
-                fileType,
-                chunkIndex: chunk.chunkIndex,
-                totalChunks: chunk.totalChunks,
-                sectionHeader: chunk.sectionHeader,
-              },
-            });
-            chunksCreated++;
+        });
+
+        try {
+          await services.lance.initialize(store.id);
+          const docs: Document[] = [];
+
+          // Crawl pages using IntelligentCrawler
+          for await (const result of crawler.crawl(url, {
+            ...(cmdOptions.crawl !== undefined && { crawlInstruction: cmdOptions.crawl }),
+            ...(cmdOptions.extract !== undefined && { extractInstruction: cmdOptions.extract }),
+            maxPages,
+            ...(cmdOptions.simple !== undefined && { simple: cmdOptions.simple }),
+            useHeadless: cmdOptions.headless ?? false,
+          })) {
+            // Use extracted content if available, otherwise markdown
+            const contentToProcess = result.extracted ?? result.markdown;
+
+            // Chunk the content using markdown-aware chunking (web content is converted to markdown)
+            const chunks = webChunker.chunk(contentToProcess, `${result.url}.md`);
+            const fileType = classifyWebContentType(result.url, result.title);
+            const urlHash = createHash('md5').update(result.url).digest('hex');
+
+            for (const chunk of chunks) {
+              const chunkId =
+                chunks.length > 1
+                  ? `${store.id}-${urlHash}-${String(chunk.chunkIndex)}`
+                  : `${store.id}-${urlHash}`;
+              const vector = await services.embeddings.embed(chunk.content);
+
+              docs.push({
+                id: createDocumentId(chunkId),
+                content: chunk.content,
+                vector,
+                metadata: {
+                  type: chunks.length > 1 ? 'chunk' : 'web',
+                  storeId: store.id,
+                  url: result.url,
+                  title: result.title,
+                  extracted: result.extracted !== undefined,
+                  depth: result.depth,
+                  indexedAt: new Date(),
+                  fileType,
+                  chunkIndex: chunk.chunkIndex,
+                  totalChunks: chunk.totalChunks,
+                  sectionHeader: chunk.sectionHeader,
+                },
+              });
+              chunksCreated++;
+            }
+
+            pagesIndexed++;
           }
 
-          pagesIndexed++;
-        }
+          // Index all documents
+          if (docs.length > 0) {
+            if (spinner) {
+              spinner.text = 'Indexing documents...';
+            }
+            await services.lance.addDocuments(store.id, docs);
+          }
 
-        // Index all documents
-        if (docs.length > 0) {
+          const crawlResult = {
+            success: true,
+            store: store.name,
+            storeCreated,
+            url,
+            pagesCrawled: pagesIndexed,
+            chunksCreated,
+            mode: cmdOptions.simple === true ? 'simple' : 'intelligent',
+            hadCrawlInstruction: cmdOptions.crawl !== undefined,
+            hadExtractInstruction: cmdOptions.extract !== undefined,
+          };
+
+          if (globalOpts.format === 'json') {
+            console.log(JSON.stringify(crawlResult, null, 2));
+          } else if (spinner !== undefined) {
+            spinner.succeed(
+              `Crawled ${String(pagesIndexed)} pages, indexed ${String(chunksCreated)} chunks`
+            );
+          } else if (globalOpts.quiet !== true) {
+            console.log(
+              `Crawled ${String(pagesIndexed)} pages, indexed ${String(chunksCreated)} chunks`
+            );
+          }
+        } catch (error) {
+          const message = `Crawl failed: ${error instanceof Error ? error.message : String(error)}`;
           if (spinner) {
-            spinner.text = 'Indexing documents...';
+            spinner.fail(message);
+          } else {
+            console.error(`Error: ${message}`);
           }
-          await services.lance.addDocuments(store.id, docs);
+          exitCode = 6;
+        } finally {
+          await crawler.stop();
+          await destroyServices(services);
         }
 
-        const crawlResult = {
-          success: true,
-          store: store.name,
-          storeCreated,
-          url,
-          pagesCrawled: pagesIndexed,
-          chunksCreated,
-          mode: cmdOptions.simple === true ? 'simple' : 'intelligent',
-          hadCrawlInstruction: cmdOptions.crawl !== undefined,
-          hadExtractInstruction: cmdOptions.extract !== undefined,
-        };
-
-        if (globalOpts.format === 'json') {
-          console.log(JSON.stringify(crawlResult, null, 2));
-        } else if (spinner !== undefined) {
-          spinner.succeed(`Crawled ${String(pagesIndexed)} pages, indexed ${String(chunksCreated)} chunks`);
-        } else if (globalOpts.quiet !== true) {
-          console.log(`Crawled ${String(pagesIndexed)} pages, indexed ${String(chunksCreated)} chunks`);
+        if (exitCode !== 0) {
+          process.exit(exitCode);
         }
-      } catch (error) {
-        const message = `Crawl failed: ${error instanceof Error ? error.message : String(error)}`;
-        if (spinner) {
-          spinner.fail(message);
-        } else {
-          console.error(`Error: ${message}`);
-        }
-        exitCode = 6;
-      } finally {
-        await crawler.stop();
-        await destroyServices(services);
-      }
-
-      if (exitCode !== 0) {
-        process.exit(exitCode);
       }
-    });
+    );
 }