bluera-knowledge 0.9.26 → 0.9.31

package/eslint.config.js

@@ -42,7 +42,7 @@ export default tseslint.config(
     },
   },
   {
-    ignores: ['dist/**', 'node_modules/**', '*.config.js', '**/*.test.ts', 'tests/**/*.ts'],
+    ignores: ['dist/**', 'node_modules/**', '*.config.js', '*.config.ts', '**/*.test.ts', 'tests/**/*.ts'],
   },
   // Test files: Apply custom skip-comment rule only
   {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bluera-knowledge",
-  "version": "0.9.26",
+  "version": "0.9.31",
   "description": "CLI tool for managing knowledge stores with semantic search",
   "type": "module",
   "bin": {
@@ -92,6 +92,8 @@
     "hono": "^4.11.1",
     "node-addon-api": "^8.5.0",
     "ora": "^9.0.0",
+    "pino": "^9.6.0",
+    "pino-roll": "^1.3.0",
     "slurp-ai": "^1.0.6",
     "tree-sitter": "^0.25.0",
     "tree-sitter-go": "^0.23.1",

package/src/analysis/ast-parser.test.ts

@@ -356,6 +356,52 @@ import type { Props } from "./types";
   });
 
   describe('Edge cases and special scenarios', () => {
+    it('handles default exported class', () => {
+      const code = 'export default class DefaultClass { method() {} }';
+      const nodes = parser.parse(code, 'typescript');
+
+      expect(nodes).toHaveLength(1);
+      expect(nodes[0]?.exported).toBe(true);
+      expect(nodes[0]?.name).toBe('DefaultClass');
+    });
+
+    it('handles anonymous default exported class (no id)', () => {
+      const code = 'export default class { method() {} }';
+      const nodes = parser.parse(code, 'javascript');
+
+      // Anonymous classes don't have an id, should be skipped
+      expect(nodes.filter(n => n.type === 'class')).toHaveLength(0);
+    });
+
+    it('handles class with computed property method (non-identifier key)', () => {
+      const code = `class MyClass {
+  ['computed']() { return 42; }
+  normalMethod() { return 1; }
+}`;
+      const nodes = parser.parse(code, 'javascript');
+
+      // Computed properties have StringLiteral keys, not Identifier
+      // Only normalMethod should be captured
+      expect(nodes[0]?.methods).toHaveLength(1);
+      expect(nodes[0]?.methods?.[0]?.name).toBe('normalMethod');
+    });
+
+    it('handles class with rest parameters in method', () => {
+      const code = `class MyClass {
+  method(...args) { return args; }
+}`;
+      const nodes = parser.parse(code, 'javascript');
+
+      expect(nodes[0]?.methods?.[0]?.signature).toBe('method(param)');
+    });
+
+    it('handles function with rest parameters', () => {
+      const code = 'function spread(...items) { return items; }';
+      const nodes = parser.parse(code, 'javascript');
+
+      expect(nodes[0]?.signature).toBe('spread(param)');
+    });
+
     it('handles functions with complex destructured parameters', () => {
       const code = 'function complex({ a, b }, [c, d]) { return a + b + c + d; }';
       const nodes = parser.parse(code, 'javascript');

package/src/cli/commands/crawl.test.ts

@@ -38,6 +38,7 @@ describe('crawl command execution', () => {
       store: {
         getByIdOrName: vi.fn(),
         list: vi.fn(),
+        create: vi.fn(),
       },
       lance: {
         initialize: vi.fn(),
@@ -291,23 +292,110 @@ describe('crawl command execution', () => {
     });
   });
 
-  describe('error handling', () => {
-    it('exits with code 3 when store not found', async () => {
+  describe('store auto-creation', () => {
+    it('auto-creates web store when store does not exist', async () => {
+      const createdStore: WebStore = {
+        id: createStoreId('new-store-id'),
+        name: 'new-store',
+        type: 'web',
+        url: 'https://example.com',
+        depth: 2,
+        createdAt: new Date(),
+        updatedAt: new Date(),
+      };
+
       mockServices.store.getByIdOrName.mockResolvedValue(undefined);
+      mockServices.store.create.mockResolvedValue({ success: true, data: createdStore });
+      mockServices.lance.initialize.mockResolvedValue(undefined);
+      mockServices.embeddings.embed.mockResolvedValue([0.1, 0.2, 0.3]);
+      mockServices.lance.addDocuments.mockResolvedValue(undefined);
+
+      mockCrawler.crawl.mockReturnValue(
+        (async function* () {
+          yield {
+            url: 'https://example.com/page1',
+            title: 'Page 1',
+            markdown: '# Content',
+            depth: 0,
+          };
+        })()
+      );
 
       const command = createCrawlCommand(getOptions);
       const actionHandler = command._actionHandler;
 
-      await expect(actionHandler(['https://example.com', 'nonexistent-store'])).rejects.toThrow('process.exit: 3');
+      await actionHandler(['https://example.com', 'new-store']);
 
-      expect(consoleErrorSpy).toHaveBeenCalledWith(
-        'Error: Web store not found: nonexistent-store'
+      expect(mockServices.store.create).toHaveBeenCalledWith({
+        name: 'new-store',
+        type: 'web',
+        url: 'https://example.com',
+      });
+      expect(consoleLogSpy).toHaveBeenCalledWith('Created web store: new-store');
+      expect(mockCrawler.crawl).toHaveBeenCalled();
+    });
+
+    it('throws error when store creation fails', async () => {
+      mockServices.store.getByIdOrName.mockResolvedValue(undefined);
+      mockServices.store.create.mockResolvedValue({ success: false, error: new Error('Name already exists') });
+
+      const command = createCrawlCommand(getOptions);
+      const actionHandler = command._actionHandler;
+
+      await expect(actionHandler(['https://example.com', 'bad-store'])).rejects.toThrow(
+        'Failed to create store: Name already exists'
       );
-      expect(processExitSpy).toHaveBeenCalledWith(3);
       expect(mockCrawler.crawl).not.toHaveBeenCalled();
     });
 
-    it('exits with code 3 when store is not a web store', async () => {
+    it('includes storeCreated in JSON output when store was created', async () => {
+      const createdStore: WebStore = {
+        id: createStoreId('new-store-id'),
+        name: 'new-store',
+        type: 'web',
+        url: 'https://example.com',
+        depth: 2,
+        createdAt: new Date(),
+        updatedAt: new Date(),
+      };
+
+      mockServices.store.getByIdOrName.mockResolvedValue(undefined);
+      mockServices.store.create.mockResolvedValue({ success: true, data: createdStore });
+      mockServices.lance.initialize.mockResolvedValue(undefined);
+      mockServices.embeddings.embed.mockResolvedValue([0.1, 0.2, 0.3]);
+      mockServices.lance.addDocuments.mockResolvedValue(undefined);
+
+      mockCrawler.crawl.mockReturnValue(
+        (async function* () {
+          yield {
+            url: 'https://example.com/page1',
+            title: 'Page 1',
+            markdown: '# Content',
+            depth: 0,
+          };
+        })()
+      );
+
+      getOptions = () => ({
+        config: undefined,
+        dataDir: '/tmp/test',
+        quiet: false,
+        format: 'json',
+      });
+
+      const command = createCrawlCommand(getOptions);
+      const actionHandler = command._actionHandler;
+
+      await actionHandler(['https://example.com', 'new-store']);
+
+      expect(consoleLogSpy).toHaveBeenCalledWith(
+        expect.stringContaining('"storeCreated": true')
+      );
+    });
+  });
+
+  describe('error handling', () => {
+    it('throws error when store is not a web store', async () => {
       const mockFileStore = {
         id: createStoreId('store-1'),
         name: 'file-store',
@@ -322,10 +410,9 @@ describe('crawl command execution', () => {
       const command = createCrawlCommand(getOptions);
       const actionHandler = command._actionHandler;
 
-      await expect(actionHandler(['https://example.com', 'file-store'])).rejects.toThrow('process.exit: 3');
-
-      expect(consoleErrorSpy).toHaveBeenCalledWith('Error: Web store not found: file-store');
-      expect(processExitSpy).toHaveBeenCalledWith(3);
+      await expect(actionHandler(['https://example.com', 'file-store'])).rejects.toThrow(
+        'Store "file-store" exists but is not a web store (type: file)'
+      );
     });
 
     it('exits with code 6 when crawling fails', async () => {
@@ -933,7 +1020,7 @@ describe('crawl command execution', () => {
         await actionHandler(['https://example.com', 'test-store']);
 
         expect(consoleLogSpy).toHaveBeenCalledWith('Crawling https://example.com');
-        expect(consoleLogSpy).toHaveBeenCalledWith('Crawled and indexed 1 pages');
+        expect(consoleLogSpy).toHaveBeenCalledWith('Crawled 1 pages, indexed 1 chunks');
       } finally {
         Object.defineProperty(process.stdout, 'isTTY', { value: originalIsTTY, configurable: true });
       }

package/src/cli/commands/crawl.ts

@@ -6,6 +6,9 @@ import { IntelligentCrawler, type CrawlProgress } from '../../crawl/intelligent-
 import { createDocumentId } from '../../types/brands.js';
 import type { GlobalOptions } from '../program.js';
 import type { Document } from '../../types/document.js';
+import type { WebStore } from '../../types/store.js';
+import { ChunkingService } from '../../services/chunking.service.js';
+import { classifyWebContentType } from '../../services/index.service.js';
 
 export function createCrawlCommand(getOptions: () => GlobalOptions): Command {
   return new Command('crawl')
@@ -27,10 +30,38 @@ export function createCrawlCommand(getOptions: () => GlobalOptions): Command {
       const globalOpts = getOptions();
       const services = await createServices(globalOpts.config, globalOpts.dataDir);
 
-      const store = await services.store.getByIdOrName(storeIdOrName);
-      if (!store || store.type !== 'web') {
-        console.error(`Error: Web store not found: ${storeIdOrName}`);
-        process.exit(3);
+      // Look up or auto-create web store
+      let store: WebStore;
+      let storeCreated = false;
+      const existingStore = await services.store.getByIdOrName(storeIdOrName);
+
+      if (!existingStore) {
+        // Auto-create web store
+        const result = await services.store.create({
+          name: storeIdOrName,
+          type: 'web',
+          url,
+        });
+        if (!result.success) {
+          await destroyServices(services);
+          throw new Error(`Failed to create store: ${result.error.message}`);
+        }
+        // Type narrowing: success check above ensures result.data is Store
+        // We know it's a WebStore because we created it with type: 'web'
+        const createdStore = result.data;
+        if (createdStore.type !== 'web') {
+          throw new Error('Unexpected store type after creation');
+        }
+        store = createdStore;
+        storeCreated = true;
+        if (globalOpts.quiet !== true && globalOpts.format !== 'json') {
+          console.log(`Created web store: ${store.name}`);
+        }
+      } else if (existingStore.type !== 'web') {
+        await destroyServices(services);
+        throw new Error(`Store "${storeIdOrName}" exists but is not a web store (type: ${existingStore.type})`);
+      } else {
+        store = existingStore;
       }
 
       const maxPages = cmdOptions.maxPages !== undefined ? parseInt(cmdOptions.maxPages) : 50;
@@ -47,7 +78,10 @@ export function createCrawlCommand(getOptions: () => GlobalOptions): Command {
       }
 
       const crawler = new IntelligentCrawler();
+      // Use web preset for larger prose-friendly chunks
+      const webChunker = ChunkingService.forContentType('web');
       let pagesIndexed = 0;
+      let chunksCreated = 0;
 
       // Listen for progress events
       crawler.on('progress', (progress: CrawlProgress) => {
@@ -78,24 +112,40 @@ export function createCrawlCommand(getOptions: () => GlobalOptions): Command {
           ...(cmdOptions.simple !== undefined && { simple: cmdOptions.simple }),
           useHeadless: cmdOptions.headless ?? false,
         })) {
-          // Embed and index the content (use extracted if available, otherwise markdown)
-          const contentToEmbed = result.extracted !== undefined ? result.extracted : result.markdown;
-          const vector = await services.embeddings.embed(contentToEmbed);
-
-          docs.push({
-            id: createDocumentId(`${store.id}-${createHash('md5').update(result.url).digest('hex')}`),
-            content: contentToEmbed,
-            vector,
-            metadata: {
-              type: 'web',
-              storeId: store.id,
-              url: result.url,
-              title: result.title,
-              extracted: result.extracted !== undefined,
-              depth: result.depth,
-              indexedAt: new Date(),
-            },
-          });
+          // Use extracted content if available, otherwise markdown
+          const contentToProcess = result.extracted !== undefined ? result.extracted : result.markdown;
+
+          // Chunk the content using markdown-aware chunking (web content is converted to markdown)
+          const chunks = webChunker.chunk(contentToProcess, `${result.url}.md`);
+          const fileType = classifyWebContentType(result.url, result.title);
+          const urlHash = createHash('md5').update(result.url).digest('hex');
+
+          for (const chunk of chunks) {
+            const chunkId = chunks.length > 1
+              ? `${store.id}-${urlHash}-${String(chunk.chunkIndex)}`
+              : `${store.id}-${urlHash}`;
+            const vector = await services.embeddings.embed(chunk.content);
+
+            docs.push({
+              id: createDocumentId(chunkId),
+              content: chunk.content,
+              vector,
+              metadata: {
+                type: chunks.length > 1 ? 'chunk' : 'web',
+                storeId: store.id,
+                url: result.url,
+                title: result.title,
+                extracted: result.extracted !== undefined,
+                depth: result.depth,
+                indexedAt: new Date(),
+                fileType,
+                chunkIndex: chunk.chunkIndex,
+                totalChunks: chunk.totalChunks,
+                sectionHeader: chunk.sectionHeader,
+              },
+            });
+            chunksCreated++;
+          }
 
           pagesIndexed++;
         }
@@ -111,8 +161,10 @@ export function createCrawlCommand(getOptions: () => GlobalOptions): Command {
         const crawlResult = {
           success: true,
           store: store.name,
+          storeCreated,
           url,
           pagesCrawled: pagesIndexed,
+          chunksCreated,
           mode: cmdOptions.simple === true ? 'simple' : 'intelligent',
           hadCrawlInstruction: cmdOptions.crawl !== undefined,
           hadExtractInstruction: cmdOptions.extract !== undefined,
@@ -121,9 +173,9 @@ export function createCrawlCommand(getOptions: () => GlobalOptions): Command {
         if (globalOpts.format === 'json') {
           console.log(JSON.stringify(crawlResult, null, 2));
         } else if (spinner !== undefined) {
-          spinner.succeed(`Crawled and indexed ${String(pagesIndexed)} pages`);
+          spinner.succeed(`Crawled ${String(pagesIndexed)} pages, indexed ${String(chunksCreated)} chunks`);
         } else if (globalOpts.quiet !== true) {
-          console.log(`Crawled and indexed ${String(pagesIndexed)} pages`);
+          console.log(`Crawled ${String(pagesIndexed)} pages, indexed ${String(chunksCreated)} chunks`);
         }
       } catch (error) {
         const message = `Crawl failed: ${error instanceof Error ? error.message : String(error)}`;

package/src/cli/commands/store.test.ts

@@ -935,7 +935,7 @@ describe('store command execution', () => {
       });
     });
 
-    it('routes source to path for repo stores', async () => {
+    it('routes source to path for repo stores with local path', async () => {
       const mockStore: RepoStore = {
         id: createStoreId('store-2'),
         name: 'repo-store',
@@ -967,6 +967,73 @@ describe('store command execution', () => {
       });
     });
 
+    it('routes URL source to url for repo stores (Bug #1 fix)', async () => {
+      const mockStore: RepoStore = {
+        id: createStoreId('store-2'),
+        name: 'repo-url-store',
+        type: 'repo',
+        path: '/cloned/repo/path',
+        url: 'https://github.com/user/repo',
+        createdAt: new Date(),
+        updatedAt: new Date(),
+      };
+
+      mockServices.store.create.mockResolvedValue({
+        success: true,
+        data: mockStore,
+      });
+
+      const command = createStoreCommand(getOptions);
+      const createCommand = command.commands.find(c => c.name() === 'create');
+      const actionHandler = createCommand?._actionHandler;
+
+      createCommand.parseOptions(['--type', 'repo', '--source', 'https://github.com/user/repo']);
+      await actionHandler!(['repo-url-store']);
+
+      // URL should be routed to 'url' parameter, not 'path'
+      expect(mockServices.store.create).toHaveBeenCalledWith({
+        name: 'repo-url-store',
+        type: 'repo',
+        path: undefined,
+        url: 'https://github.com/user/repo',
+        description: undefined,
+        tags: undefined,
+      });
+    });
+
+    it('routes http:// URL source to url for repo stores', async () => {
+      const mockStore: RepoStore = {
+        id: createStoreId('store-2'),
+        name: 'repo-http-store',
+        type: 'repo',
+        path: '/cloned/repo/path',
+        url: 'http://internal-git.example.com/repo',
+        createdAt: new Date(),
+        updatedAt: new Date(),
+      };
+
+      mockServices.store.create.mockResolvedValue({
+        success: true,
+        data: mockStore,
+      });
+
+      const command = createStoreCommand(getOptions);
+      const createCommand = command.commands.find(c => c.name() === 'create');
+      const actionHandler = createCommand?._actionHandler;
+
+      createCommand.parseOptions(['--type', 'repo', '--source', 'http://internal-git.example.com/repo']);
+      await actionHandler!(['repo-http-store']);
+
+      expect(mockServices.store.create).toHaveBeenCalledWith({
+        name: 'repo-http-store',
+        type: 'repo',
+        path: undefined,
+        url: 'http://internal-git.example.com/repo',
+        description: undefined,
+        tags: undefined,
+      });
+    });
+
     it('routes source to url for web stores', async () => {
       const mockStore: WebStore = {
         id: createStoreId('store-3'),

package/src/cli/commands/store.ts

@@ -54,12 +54,15 @@ export function createStoreCommand(getOptions: () => GlobalOptions): Command {
     }) => {
       const globalOpts = getOptions();
       const services = await createServices(globalOpts.config, globalOpts.dataDir);
+      let exitCode = 0;
       try {
+        // Detect if source is a URL (for repo stores that should clone from remote)
+        const isUrl = options.source.startsWith('http://') || options.source.startsWith('https://');
         const result = await services.store.create({
           name,
           type: options.type,
-          path: options.type !== 'web' ? options.source : undefined,
-          url: options.type === 'web' ? options.source : undefined,
+          path: options.type === 'file' || (options.type === 'repo' && !isUrl) ? options.source : undefined,
+          url: options.type === 'web' || (options.type === 'repo' && isUrl) ? options.source : undefined,
           description: options.description,
           tags: options.tags?.split(',').map((t) => t.trim()),
         });
@@ -72,11 +75,14 @@ export function createStoreCommand(getOptions: () => GlobalOptions): Command {
           }
         } else {
           console.error(`Error: ${result.error.message}`);
-          process.exit(1);
+          exitCode = 1;
         }
       } finally {
         await destroyServices(services);
       }
+      if (exitCode !== 0) {
+        process.exit(exitCode);
+      }
     });
 
   store

package/src/crawl/article-converter.ts

@@ -7,6 +7,9 @@ import { extractFromHtml } from '@extractus/article-extractor';
 import TurndownService from 'turndown';
 import { gfm } from 'turndown-plugin-gfm';
 import { preprocessHtmlForCodeBlocks, cleanupMarkdown } from './markdown-utils.js';
+import { createLogger, truncateForLog } from '../logging/index.js';
+
+const logger = createLogger('article-converter');
 
 export interface ConversionResult {
   markdown: string;
@@ -28,6 +31,8 @@ export async function convertHtmlToMarkdown(
   html: string,
   url: string,
 ): Promise<ConversionResult> {
+  logger.debug({ url, htmlLength: html.length }, 'Starting HTML conversion');
+
   try {
     // Step 1: Extract main article content
     let articleHtml: string;
@@ -38,13 +43,25 @@ export async function convertHtmlToMarkdown(
       if (article !== null && article.content !== undefined && article.content !== '') {
         articleHtml = article.content;
         title = article.title !== undefined && article.title !== '' ? article.title : undefined;
+        logger.debug({
+          url,
+          title,
+          extractedLength: articleHtml.length,
+          usedFullHtml: false,
+        }, 'Article content extracted');
       } else {
         // Fallback to full HTML if extraction fails
         articleHtml = html;
+        logger.debug({ url, usedFullHtml: true }, 'Article extraction returned empty, using full HTML');
       }
-    } catch {
+    } catch (extractError) {
       // Fallback to full HTML if extraction fails
       articleHtml = html;
+      logger.debug({
+        url,
+        usedFullHtml: true,
+        error: extractError instanceof Error ? extractError.message : String(extractError),
+      }, 'Article extraction failed, using full HTML');
     }
 
     // Step 2: Preprocess HTML for code blocks
@@ -83,12 +100,30 @@ export async function convertHtmlToMarkdown(
     // Step 4: Cleanup markdown with comprehensive regex patterns
     const markdown = cleanupMarkdown(rawMarkdown);
 
+    logger.debug({
+      url,
+      title,
+      rawMarkdownLength: rawMarkdown.length,
+      finalMarkdownLength: markdown.length,
+    }, 'HTML to markdown conversion complete');
+
+    // Log markdown preview at trace level
+    logger.trace({
+      url,
+      markdownPreview: truncateForLog(markdown, 1000),
+    }, 'Markdown content preview');
+
     return {
       markdown,
       ...(title !== undefined && { title }),
       success: true,
     };
   } catch (error) {
+    logger.error({
+      url,
+      error: error instanceof Error ? error.message : String(error),
+    }, 'HTML to markdown conversion failed');
+
     return {
       markdown: '',
       success: false,

package/src/crawl/bridge.ts

@@ -11,6 +11,9 @@ import {
   validateHeadlessResult,
   validateParsePythonResult,
 } from './schemas.js';
+import { createLogger } from '../logging/index.js';
+
+const logger = createLogger('python-bridge');
 
 // Re-export for backwards compatibility
 export type { CrawledLink, ParsePythonResult };
@@ -32,24 +35,26 @@ export class PythonBridge {
   start(): Promise<void> {
     if (this.process) return Promise.resolve();
 
+    logger.debug('Starting Python bridge process');
+
     this.process = spawn('python3', ['python/crawl_worker.py'], {
       stdio: ['pipe', 'pipe', 'pipe'],
     });
 
     // Add error handler for process spawn errors
     this.process.on('error', (err) => {
-      console.error('Python bridge process error:', err);
+      logger.error({ error: err.message, stack: err.stack }, 'Python bridge process error');
       this.rejectAllPending(new Error(`Process error: ${err.message}`));
     });
 
     // Add exit handler to detect non-zero exits
     this.process.on('exit', (code, signal) => {
       if (code !== 0 && code !== null) {
-        console.error(`Python bridge process exited with code ${String(code)}`);
+        logger.error({ code }, 'Python bridge process exited with non-zero code');
         this.rejectAllPending(new Error(`Process exited with code ${String(code)}`));
       } else if (signal && !this.stoppingIntentionally) {
         // Only log if we didn't intentionally stop the process
-        console.error(`Python bridge process killed with signal ${signal}`);
+        logger.error({ signal }, 'Python bridge process killed with signal');
         this.rejectAllPending(new Error(`Process killed with signal ${signal}`));
       }
       this.process = null;
@@ -60,7 +65,7 @@ export class PythonBridge {
     if (this.process.stderr) {
       const stderrRl = createInterface({ input: this.process.stderr });
       stderrRl.on('line', (line) => {
-        console.error('Python bridge stderr:', line);
+        logger.warn({ stderr: line }, 'Python bridge stderr output');
       });
     }
 
@@ -106,11 +111,14 @@ export class PythonBridge {
             } catch (error: unknown) {
               // Log validation failure with original response for debugging
               if (error instanceof ZodError) {
-                console.error('Python bridge response validation failed:', error.issues);
-                console.error('Original response:', JSON.stringify(response.result));
+                logger.error({
+                  issues: error.issues,
+                  response: JSON.stringify(response.result),
+                }, 'Python bridge response validation failed');
                 pending.reject(new Error(`Invalid response format from Python bridge: ${error.message}`));
               } else {
                 const errorMessage = error instanceof Error ? error.message : String(error);
+                logger.error({ error: errorMessage }, 'Response validation error');
                 pending.reject(new Error(`Response validation error: ${errorMessage}`));
               }
             }
@@ -118,7 +126,10 @@ export class PythonBridge {
           // If neither result nor error, leave pending (will timeout)
         }
       } catch (err) {
-        console.error('Failed to parse JSON response from Python bridge:', err, 'Line:', line);
+        logger.error({
+          error: err instanceof Error ? err.message : String(err),
+          line,
+        }, 'Failed to parse JSON response from Python bridge');
       }
     });