doc2vec 1.1.1 → 1.2.0

package/dist/doc2vec.js

@@ -37,23 +37,17 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-const readability_1 = require("@mozilla/readability");
 const axios_1 = __importDefault(require("axios"));
-const cheerio_1 = require("cheerio");
 const crypto_1 = __importDefault(require("crypto"));
-const jsdom_1 = require("jsdom");
-const puppeteer_1 = __importDefault(require("puppeteer"));
-const sanitize_html_1 = __importDefault(require("sanitize-html"));
-const turndown_1 = __importDefault(require("turndown"));
 const yaml = __importStar(require("js-yaml"));
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
-const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
 const openai_1 = require("openai");
 const dotenv = __importStar(require("dotenv"));
-const sqliteVec = __importStar(require("sqlite-vec"));
-const js_client_rest_1 = require("@qdrant/js-client-rest");
 const logger_1 = require("./logger");
+const utils_1 = require("./utils");
+const database_1 = require("./database");
+const content_processor_1 = require("./content-processor");
 const GITHUB_TOKEN = process.env.GITHUB_PERSONAL_ACCESS_TOKEN;
 dotenv.config();
 class Doc2Vec {
@@ -67,11 +61,7 @@ class Doc2Vec {
         this.logger.info('Initializing Doc2Vec');
         this.config = this.loadConfig(configPath);
         this.openai = new openai_1.OpenAI({ apiKey: process.env.OPENAI_API_KEY });
-        this.turndownService = new turndown_1.default({
-            codeBlockStyle: 'fenced',
-            headingStyle: 'atx'
-        });
-        this.setupTurndownRules();
+        this.contentProcessor = new content_processor_1.ContentProcessor(this.logger);
     }
     loadConfig(configPath) {
         try {
@@ -88,89 +78,6 @@ class Doc2Vec {
             process.exit(1);
         }
     }
-    setupTurndownRules() {
-        const logger = this.logger.child('markdown');
-        logger.debug('Setting up Turndown rules for markdown conversion');
-        this.turndownService.addRule('codeBlocks', {
-            filter: (node) => node.nodeName === 'PRE',
-            replacement: (content, node) => {
-                const htmlNode = node;
-                const code = htmlNode.querySelector('code');
-                let codeContent;
-                if (code) {
-                    codeContent = code.textContent || '';
-                }
-                else {
-                    codeContent = htmlNode.textContent || '';
-                }
-                const lines = codeContent.split('\n');
-                let minIndent = Infinity;
-                for (const line of lines) {
-                    if (line.trim() === '')
-                        continue;
-                    const leadingWhitespace = line.match(/^\s*/)?.[0] || '';
-                    minIndent = Math.min(minIndent, leadingWhitespace.length);
-                }
-                const cleanedLines = lines.map(line => {
-                    return line.substring(minIndent);
-                });
-                let cleanContent = cleanedLines.join('\n');
-                cleanContent = cleanContent.replace(/^\s+|\s+$/g, '');
-                cleanContent = cleanContent.replace(/\n{2,}/g, '\n');
-                return `\n\`\`\`\n${cleanContent}\n\`\`\`\n`;
-            }
-        });
-        this.turndownService.addRule('tableCell', {
-            filter: ['th', 'td'],
-            replacement: (content, node) => {
-                const htmlNode = node;
-                let cellContent = '';
-                if (htmlNode.querySelector('p')) {
-                    cellContent = Array.from(htmlNode.querySelectorAll('p'))
-                        .map(p => p.textContent || '')
-                        .join(' ')
-                        .trim();
-                }
-                else {
-                    cellContent = content.trim();
-                }
-                return ` ${cellContent.replace(/\|/g, '\\|')} |`;
-            }
-        });
-        this.turndownService.addRule('tableRow', {
-            filter: 'tr',
-            replacement: (content, node) => {
-                const htmlNode = node;
-                const cells = Array.from(htmlNode.cells);
-                const isHeader = htmlNode.parentNode?.nodeName === 'THEAD';
-                let output = '|' + content.trimEnd();
-                if (isHeader) {
-                    const separator = cells.map(() => '---').join(' | ');
-                    output += '\n|' + separator + '|';
-                }
-                if (!isHeader || !htmlNode.nextElementSibling) {
-                    output += '\n';
-                }
-                return output;
-            }
-        });
-        this.turndownService.addRule('table', {
-            filter: 'table',
-            replacement: (content) => {
-                return '\n' + content.replace(/\n+/g, '\n').trim() + '\n';
-            }
-        });
-        this.turndownService.addRule('preserveTableWhitespace', {
-            filter: (node) => {
-                return ((node.nodeName === 'TD' || node.nodeName === 'TH') &&
-                    (node.textContent?.trim().length === 0));
-            },
-            replacement: () => {
-                return ' |';
-            }
-        });
-        logger.debug('Turndown rules setup complete');
-    }
     async run() {
         this.logger.section('PROCESSING SOURCES');
         for (const sourceConfig of this.config.sources) {
@@ -191,134 +98,14 @@ class Doc2Vec {
         }
         this.logger.section('PROCESSING COMPLETE');
     }
-    getUrlPrefix(url) {
-        try {
-            const parsedUrl = new URL(url);
-            return parsedUrl.origin + parsedUrl.pathname;
-        }
-        catch (error) {
-            return url;
-        }
-    }
-    generateMetadataUUID(repo) {
-        // Simple deterministic approach - hash the repo name and convert to UUID format
-        const hash = crypto_1.default.createHash('md5').update(`metadata_${repo}`).digest('hex');
-        // Format as UUID with version bits set correctly (version 4)
-        return `${hash.substr(0, 8)}-${hash.substr(8, 4)}-4${hash.substr(13, 3)}-${hash.substr(16, 4)}-${hash.substr(20, 12)}`;
-    }
-    async initDatabaseMetadata(dbConnection) {
-        const logger = this.logger.child('metadata');
-        if (dbConnection.type === 'sqlite') {
-            const db = dbConnection.db;
-            logger.debug('Creating metadata table if it doesn\'t exist');
-            db.exec(`
-                CREATE TABLE IF NOT EXISTS vec_metadata (
-                    key TEXT PRIMARY KEY,
-                    value TEXT
-                );
-            `);
-            logger.info('SQLite metadata table initialized');
-        }
-        else if (dbConnection.type === 'qdrant') {
-            // For Qdrant, we'll use the same collection but verify it exists
-            logger.info(`Using existing Qdrant collection for metadata: ${dbConnection.collectionName}`);
-            // Nothing special to initialize as we'll use the same collection
-        }
-    }
-    async getLastRunDate(dbConnection, repo, defaultDate) {
-        const logger = this.logger.child('metadata');
-        const metadataKey = `last_run_${repo.replace('/', '_')}`;
-        try {
-            if (dbConnection.type === 'sqlite') {
-                const stmt = dbConnection.db.prepare('SELECT value FROM vec_metadata WHERE key = ?');
-                const result = stmt.get(metadataKey);
-                if (result) {
-                    logger.info(`Retrieved last run date for ${repo}: ${result.value}`);
-                    return result.value;
-                }
-            }
-            else if (dbConnection.type === 'qdrant') {
-                // Generate a UUID for this repo's metadata
-                const metadataUUID = this.generateMetadataUUID(repo);
-                logger.debug(`Looking up metadata with UUID: ${metadataUUID}`);
-                try {
-                    // Try to retrieve the metadata point for this repo
-                    const response = await dbConnection.client.retrieve(dbConnection.collectionName, {
-                        ids: [metadataUUID],
-                        with_payload: true,
-                        with_vector: false
-                    });
-                    if (response.length > 0 && response[0].payload?.metadata_value) {
-                        const lastRunDate = response[0].payload.metadata_value;
-                        logger.info(`Retrieved last run date for ${repo}: ${lastRunDate}`);
-                        return lastRunDate;
-                    }
-                }
-                catch (error) {
-                    logger.warn(`Failed to retrieve metadata for ${repo}:`, error);
-                }
-            }
-        }
-        catch (error) {
-            logger.warn(`Error retrieving last run date:`, error);
-        }
-        logger.info(`No saved run date found for ${repo}, using default: ${defaultDate}`);
-        return defaultDate;
-    }
-    async updateLastRunDate(dbConnection, repo) {
-        const logger = this.logger.child('metadata');
-        const now = new Date().toISOString();
-        try {
-            if (dbConnection.type === 'sqlite') {
-                const metadataKey = `last_run_${repo.replace('/', '_')}`;
-                const stmt = dbConnection.db.prepare(`
-                    INSERT INTO vec_metadata (key, value) VALUES (?, ?)
-                    ON CONFLICT(key) DO UPDATE SET value = excluded.value
-                `);
-                stmt.run(metadataKey, now);
-                logger.info(`Updated last run date for ${repo} to ${now}`);
-            }
-            else if (dbConnection.type === 'qdrant') {
-                // Generate UUID for this repo's metadata
-                const metadataUUID = this.generateMetadataUUID(repo);
-                const metadataKey = `last_run_${repo.replace('/', '_')}`;
-                logger.debug(`Using UUID: ${metadataUUID} for metadata`);
-                // Generate a dummy embedding (all zeros)
-                const dummyEmbeddingSize = 3072; // Same size as your content embeddings
-                const dummyEmbedding = new Array(dummyEmbeddingSize).fill(0);
-                // Create a point with special metadata payload
-                const metadataPoint = {
-                    id: metadataUUID,
-                    vector: dummyEmbedding,
-                    payload: {
-                        metadata_key: metadataKey,
-                        metadata_value: now,
-                        is_metadata: true, // Flag to identify metadata points
-                        content: `Metadata: Last run date for ${repo}`,
-                        product_name: 'system',
-                        version: 'metadata',
-                        url: 'metadata://' + repo
-                    }
-                };
-                await dbConnection.client.upsert(dbConnection.collectionName, {
-                    wait: true,
-                    points: [metadataPoint]
-                });
-                logger.info(`Updated last run date for ${repo} to ${now}`);
-            }
-        }
-        catch (error) {
-            logger.error(`Failed to update last run date for ${repo}:`, error);
-        }
-    }
     async fetchAndProcessGitHubIssues(repo, sourceConfig, dbConnection, logger) {
         const [owner, repoName] = repo.split('/');
         const GITHUB_API_URL = `https://api.github.com/repos/${owner}/${repoName}/issues`;
         // Initialize metadata storage if needed
-        await this.initDatabaseMetadata(dbConnection);
+        await database_1.DatabaseManager.initDatabaseMetadata(dbConnection, logger);
         // Get the last run date from the database
         const startDate = sourceConfig.start_date || '2025-01-01';
-        const lastRunDate = await this.getLastRunDate(dbConnection, repo, `${startDate}T00:00:00Z`);
+        const lastRunDate = await database_1.DatabaseManager.getLastRunDate(dbConnection, repo, `${startDate}T00:00:00Z`, logger);
         const fetchWithRetry = async (url, params = {}, retries = 5, delay = 5000) => {
             for (let attempt = 0; attempt < retries; attempt++) {
                 try {
@@ -405,14 +192,14 @@ class Doc2Vec {
                 product_name: sourceConfig.product_name || repo,
                 max_size: sourceConfig.max_size || Infinity
             };
-            const chunks = await this.chunkMarkdown(markdown, issueConfig, url);
+            const chunks = await this.contentProcessor.chunkMarkdown(markdown, issueConfig, url);
             logger.info(`Issue #${issueNumber}: Created ${chunks.length} chunks`);
             // Process and store each chunk immediately
             for (const chunk of chunks) {
-                const chunkHash = this.generateHash(chunk.content);
+                const chunkHash = utils_1.Utils.generateHash(chunk.content);
                 const chunkId = chunk.metadata.chunk_id.substring(0, 8) + '...';
                 if (dbConnection.type === 'sqlite') {
-                    const { checkHashStmt } = this.prepareSQLiteStatements(dbConnection.db);
+                    const { checkHashStmt } = database_1.DatabaseManager.prepareSQLiteStatements(dbConnection.db);
                     const existing = checkHashStmt.get(chunk.metadata.chunk_id);
                     if (existing && existing.hash === chunkHash) {
                         logger.info(`Skipping unchanged chunk: ${chunkId}`);
@@ -420,7 +207,7 @@ class Doc2Vec {
                     }
                     const embeddings = await this.createEmbeddings([chunk.content]);
                     if (embeddings.length) {
-                        this.insertVectorsSQLite(dbConnection.db, chunk, embeddings[0], logger, chunkHash);
+                        database_1.DatabaseManager.insertVectorsSQLite(dbConnection.db, chunk, embeddings[0], logger, chunkHash);
                         logger.debug(`Stored chunk ${chunkId} in SQLite`);
                     }
                     else {
@@ -432,8 +219,8 @@ class Doc2Vec {
                         let pointId;
                         try {
                             pointId = chunk.metadata.chunk_id;
-                            if (!this.isValidUuid(pointId)) {
-                                pointId = this.hashToUuid(chunk.metadata.chunk_id);
+                            if (!utils_1.Utils.isValidUuid(pointId)) {
+                                pointId = utils_1.Utils.hashToUuid(chunk.metadata.chunk_id);
                             }
                         }
                         catch (e) {
@@ -450,7 +237,7 @@ class Doc2Vec {
                         }
                         const embeddings = await this.createEmbeddings([chunk.content]);
                         if (embeddings.length) {
-                            await this.storeChunkInQdrant(dbConnection, chunk, embeddings[0], chunkHash);
+                            await database_1.DatabaseManager.storeChunkInQdrant(dbConnection, chunk, embeddings[0], chunkHash);
                             logger.debug(`Stored chunk ${chunkId} in Qdrant (${dbConnection.collectionName})`);
                         }
                         else {
@@ -472,15 +259,15 @@ class Doc2Vec {
             await processIssue(issues[i]);
         }
         // Update the last run date in the database after processing all issues
-        await this.updateLastRunDate(dbConnection, repo);
+        await database_1.DatabaseManager.updateLastRunDate(dbConnection, repo, logger);
         logger.info(`Successfully processed ${issues.length} issues`);
     }
     async processGithubRepo(config, parentLogger) {
         const logger = parentLogger.child('process');
         logger.info(`Starting processing for GitHub repo: ${config.repo}`);
-        const dbConnection = await this.initDatabase(config, logger);
+        const dbConnection = await database_1.DatabaseManager.initDatabase(config, logger);
         // Initialize metadata storage
-        await this.initDatabaseMetadata(dbConnection);
+        await database_1.DatabaseManager.initDatabaseMetadata(dbConnection, logger);
         logger.section('GITHUB ISSUES');
         // Process GitHub issues
         await this.fetchAndProcessGitHubIssues(config.repo, config, dbConnection, logger);
@@ -489,16 +276,16 @@ class Doc2Vec {
     async processWebsite(config, parentLogger) {
         const logger = parentLogger.child('process');
         logger.info(`Starting processing for website: ${config.url}`);
-        const dbConnection = await this.initDatabase(config, logger);
+        const dbConnection = await database_1.DatabaseManager.initDatabase(config, logger);
         const validChunkIds = new Set();
         const visitedUrls = new Set();
-        const urlPrefix = this.getUrlPrefix(config.url);
+        const urlPrefix = utils_1.Utils.getUrlPrefix(config.url);
         logger.section('CRAWL AND EMBEDDING');
-        await this.crawlWebsite(config.url, config, async (url, content) => {
+        const crawlResult = await this.contentProcessor.crawlWebsite(config.url, config, async (url, content) => {
             visitedUrls.add(url);
             logger.info(`Processing content from ${url} (${content.length} chars markdown)`);
             try {
-                const chunks = await this.chunkMarkdown(content, config, url);
+                const chunks = await this.contentProcessor.chunkMarkdown(content, config, url);
                 logger.info(`Created ${chunks.length} chunks`);
                 if (chunks.length > 0) {
                     const chunkProgress = logger.progress(`Embedding chunks for ${url}`, chunks.length);
@@ -507,9 +294,9 @@ class Doc2Vec {
                         validChunkIds.add(chunk.metadata.chunk_id);
                         const chunkId = chunk.metadata.chunk_id.substring(0, 8) + '...';
                         let needsEmbedding = true;
-                        const chunkHash = this.generateHash(chunk.content);
+                        const chunkHash = utils_1.Utils.generateHash(chunk.content);
                         if (dbConnection.type === 'sqlite') {
-                            const { checkHashStmt } = this.prepareSQLiteStatements(dbConnection.db);
+                            const { checkHashStmt } = database_1.DatabaseManager.prepareSQLiteStatements(dbConnection.db);
                             const existing = checkHashStmt.get(chunk.metadata.chunk_id);
                             if (existing && existing.hash === chunkHash) {
                                 needsEmbedding = false;
@@ -522,8 +309,8 @@ class Doc2Vec {
                                 let pointId;
                                 try {
                                     pointId = chunk.metadata.chunk_id;
-                                    if (!this.isValidUuid(pointId)) {
-                                        pointId = this.hashToUuid(chunk.metadata.chunk_id);
+                                    if (!utils_1.Utils.isValidUuid(pointId)) {
+                                        pointId = utils_1.Utils.hashToUuid(chunk.metadata.chunk_id);
                                     }
                                 }
                                 catch (e) {
@@ -549,11 +336,11 @@ class Doc2Vec {
                             if (embeddings.length > 0) {
                                 const embedding = embeddings[0];
                                 if (dbConnection.type === 'sqlite') {
-                                    this.insertVectorsSQLite(dbConnection.db, chunk, embedding, logger, chunkHash);
+                                    database_1.DatabaseManager.insertVectorsSQLite(dbConnection.db, chunk, embedding, logger, chunkHash);
                                     chunkProgress.update(1, `Stored chunk ${chunkId} in SQLite`);
                                 }
                                 else if (dbConnection.type === 'qdrant') {
-                                    await this.storeChunkInQdrant(dbConnection, chunk, embedding, chunkHash);
+                                    await database_1.DatabaseManager.storeChunkInQdrant(dbConnection, chunk, embedding, chunkHash);
                                     chunkProgress.update(1, `Stored chunk ${chunkId} in Qdrant (${dbConnection.collectionName})`);
                                 }
                             }
@@ -572,24 +359,29 @@ class Doc2Vec {
         }, logger, visitedUrls);
         logger.info(`Found ${validChunkIds.size} valid chunks across processed pages for ${config.url}`);
         logger.section('CLEANUP');
-        if (dbConnection.type === 'sqlite') {
-            logger.info(`Running SQLite cleanup for ${urlPrefix}`);
-            this.removeObsoleteChunksSQLite(dbConnection.db, visitedUrls, urlPrefix, logger);
+        if (crawlResult.hasNetworkErrors) {
+            logger.warn('Skipping cleanup due to network errors encountered during crawling. This prevents removal of valid chunks when the site is temporarily unreachable.');
         }
-        else if (dbConnection.type === 'qdrant') {
-            logger.info(`Running Qdrant cleanup for ${urlPrefix} in collection ${dbConnection.collectionName}`);
-            await this.removeObsoleteChunksQdrant(dbConnection, visitedUrls, urlPrefix, logger);
+        else {
+            if (dbConnection.type === 'sqlite') {
+                logger.info(`Running SQLite cleanup for ${urlPrefix}`);
+                database_1.DatabaseManager.removeObsoleteChunksSQLite(dbConnection.db, visitedUrls, urlPrefix, logger);
+            }
+            else if (dbConnection.type === 'qdrant') {
+                logger.info(`Running Qdrant cleanup for ${urlPrefix} in collection ${dbConnection.collectionName}`);
+                await database_1.DatabaseManager.removeObsoleteChunksQdrant(dbConnection, visitedUrls, urlPrefix, logger);
+            }
         }
         logger.info(`Finished processing website: ${config.url}`);
     }
     async processLocalDirectory(config, parentLogger) {
         const logger = parentLogger.child('process');
         logger.info(`Starting processing for local directory: ${config.path}`);
-        const dbConnection = await this.initDatabase(config, logger);
+        const dbConnection = await database_1.DatabaseManager.initDatabase(config, logger);
         const validChunkIds = new Set();
         const processedFiles = new Set();
         logger.section('FILE SCANNING AND EMBEDDING');
-        await this.processDirectory(config.path, config, async (filePath, content) => {
+        await this.contentProcessor.processDirectory(config.path, config, async (filePath, content) => {
             processedFiles.add(filePath);
             logger.info(`Processing content from ${filePath} (${content.length} chars)`);
             try {
@@ -618,7 +410,7 @@ class Doc2Vec {
                     // Use default file:// URL
                     fileUrl = `file://${filePath}`;
                 }
-                const chunks = await this.chunkMarkdown(content, config, fileUrl);
+                const chunks = await this.contentProcessor.chunkMarkdown(content, config, fileUrl);
                 logger.info(`Created ${chunks.length} chunks`);
                 if (chunks.length > 0) {
                     const chunkProgress = logger.progress(`Embedding chunks for ${filePath}`, chunks.length);
@@ -627,9 +419,9 @@ class Doc2Vec {
                         validChunkIds.add(chunk.metadata.chunk_id);
                         const chunkId = chunk.metadata.chunk_id.substring(0, 8) + '...';
                         let needsEmbedding = true;
-                        const chunkHash = this.generateHash(chunk.content);
+                        const chunkHash = utils_1.Utils.generateHash(chunk.content);
                         if (dbConnection.type === 'sqlite') {
-                            const { checkHashStmt } = this.prepareSQLiteStatements(dbConnection.db);
+                            const { checkHashStmt } = database_1.DatabaseManager.prepareSQLiteStatements(dbConnection.db);
                             const existing = checkHashStmt.get(chunk.metadata.chunk_id);
                             if (existing && existing.hash === chunkHash) {
                                 needsEmbedding = false;
@@ -642,8 +434,8 @@ class Doc2Vec {
                                 let pointId;
                                 try {
                                     pointId = chunk.metadata.chunk_id;
-                                    if (!this.isValidUuid(pointId)) {
-                                        pointId = this.hashToUuid(chunk.metadata.chunk_id);
+                                    if (!utils_1.Utils.isValidUuid(pointId)) {
+                                        pointId = utils_1.Utils.hashToUuid(chunk.metadata.chunk_id);
                                     }
                                 }
                                 catch (e) {
@@ -669,11 +461,11 @@ class Doc2Vec {
                             if (embeddings.length > 0) {
                                 const embedding = embeddings[0];
                                 if (dbConnection.type === 'sqlite') {
-                                    this.insertVectorsSQLite(dbConnection.db, chunk, embedding, logger, chunkHash);
+                                    database_1.DatabaseManager.insertVectorsSQLite(dbConnection.db, chunk, embedding, logger, chunkHash);
                                     chunkProgress.update(1, `Stored chunk ${chunkId} in SQLite`);
                                 }
                                 else if (dbConnection.type === 'qdrant') {
-                                    await this.storeChunkInQdrant(dbConnection, chunk, embedding, chunkHash);
+                                    await database_1.DatabaseManager.storeChunkInQdrant(dbConnection, chunk, embedding, chunkHash);
                                     chunkProgress.update(1, `Stored chunk ${chunkId} in Qdrant (${dbConnection.collectionName})`);
                                 }
                             }
@@ -693,767 +485,14 @@ class Doc2Vec {
         logger.section('CLEANUP');
         if (dbConnection.type === 'sqlite') {
             logger.info(`Running SQLite cleanup for local directory ${config.path}`);
-            this.removeObsoleteFilesSQLite(dbConnection.db, processedFiles, config, logger);
+            database_1.DatabaseManager.removeObsoleteFilesSQLite(dbConnection.db, processedFiles, config, logger);
         }
         else if (dbConnection.type === 'qdrant') {
             logger.info(`Running Qdrant cleanup for local directory ${config.path} in collection ${dbConnection.collectionName}`);
-            await this.removeObsoleteFilesQdrant(dbConnection, processedFiles, config, logger);
+            await database_1.DatabaseManager.removeObsoleteFilesQdrant(dbConnection, processedFiles, config, logger);
         }
         logger.info(`Finished processing local directory: ${config.path}`);
     }
-    async processDirectory(dirPath, config, processFileContent, parentLogger, visitedPaths = new Set()) {
-        const logger = parentLogger.child('directory-processor');
-        logger.info(`Processing directory: ${dirPath}`);
-        const recursive = config.recursive !== undefined ? config.recursive : true;
-        const includeExtensions = config.include_extensions || ['.md', '.txt', '.html', '.htm'];
-        const excludeExtensions = config.exclude_extensions || [];
-        const encoding = config.encoding || 'utf8';
-        try {
-            const files = fs.readdirSync(dirPath);
-            let processedFiles = 0;
-            let skippedFiles = 0;
-            for (const file of files) {
-                const filePath = path.join(dirPath, file);
-                const stat = fs.statSync(filePath);
-                // Skip already visited paths
-                if (visitedPaths.has(filePath)) {
-                    logger.debug(`Skipping already visited path: ${filePath}`);
-                    continue;
-                }
-                visitedPaths.add(filePath);
-                if (stat.isDirectory()) {
-                    if (recursive) {
-                        await this.processDirectory(filePath, config, processFileContent, logger, visitedPaths);
-                    }
-                    else {
-                        logger.debug(`Skipping directory ${filePath} (recursive=false)`);
-                    }
-                }
-                else if (stat.isFile()) {
-                    const extension = path.extname(file).toLowerCase();
-                    // Apply extension filters
-                    if (excludeExtensions.includes(extension)) {
-                        logger.debug(`Skipping file with excluded extension: ${filePath}`);
-                        skippedFiles++;
-                        continue;
-                    }
-                    if (includeExtensions.length > 0 && !includeExtensions.includes(extension)) {
-                        logger.debug(`Skipping file with non-included extension: ${filePath}`);
-                        skippedFiles++;
-                        continue;
-                    }
-                    try {
-                        logger.info(`Reading file: ${filePath}`);
-                        const content = fs.readFileSync(filePath, { encoding: encoding });
-                        if (content.length > config.max_size) {
-                            logger.warn(`File content (${content.length} chars) exceeds max size (${config.max_size}). Skipping ${filePath}.`);
-                            skippedFiles++;
-                            continue;
-                        }
-                        // Convert HTML to Markdown if needed
-                        let processedContent;
-                        if (extension === '.html' || extension === '.htm') {
-                            logger.debug(`Converting HTML to Markdown for ${filePath}`);
-                            const cleanHtml = (0, sanitize_html_1.default)(content, {
-                                allowedTags: [
-                                    'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'a', 'ul', 'ol',
-                                    'li', 'b', 'i', 'strong', 'em', 'code', 'pre',
-                                    'div', 'span', 'table', 'thead', 'tbody', 'tr', 'th', 'td'
-                                ],
-                                allowedAttributes: {
-                                    'a': ['href'],
-                                    'pre': ['class', 'data-language'],
-                                    'code': ['class', 'data-language'],
-                                    'div': ['class'],
-                                    'span': ['class']
-                                }
-                            });
-                            processedContent = this.turndownService.turndown(cleanHtml);
-                        }
-                        else {
-                            processedContent = content;
-                        }
-                        await processFileContent(filePath, processedContent);
-                        processedFiles++;
-                    }
-                    catch (error) {
-                        logger.error(`Error processing file ${filePath}:`, error);
-                    }
-                }
-            }
-            logger.info(`Directory processed. Processed: ${processedFiles}, Skipped: ${skippedFiles}`);
-        }
-        catch (error) {
-            logger.error(`Error reading directory ${dirPath}:`, error);
-        }
-    }
-    removeObsoleteFilesSQLite(db, processedFiles, pathConfig, logger) {
-        const getChunksForPathStmt = db.prepare(`
-            SELECT chunk_id, url FROM vec_items
-            WHERE url LIKE ? || '%'
-        `);
-        const deleteChunkStmt = db.prepare(`DELETE FROM vec_items WHERE chunk_id = ?`);
-        // Determine if we're using URL rewriting or direct file paths
-        const isRewriteMode = typeof pathConfig === 'object' && pathConfig.url_rewrite_prefix;
-        // Set up the URL prefix for searching
-        let urlPrefix;
-        if (isRewriteMode) {
-            // Handle URL rewriting case
-            urlPrefix = pathConfig.url_rewrite_prefix || '';
-            urlPrefix = urlPrefix.endsWith('/') ? urlPrefix.slice(0, -1) : urlPrefix;
-        }
-        else {
-            // Handle direct file path case
-            const dirPrefix = typeof pathConfig === 'string' ? pathConfig : pathConfig.path;
-            const cleanedDirPrefix = dirPrefix.replace(/^\.\/+/, '');
-            urlPrefix = `file://${cleanedDirPrefix}`;
-        }
-        logger.debug(`Searching for chunks with URL prefix: ${urlPrefix}`);
-        const existingChunks = getChunksForPathStmt.all(urlPrefix);
-        let deletedCount = 0;
-        const transaction = db.transaction(() => {
-            for (const { chunk_id, url } of existingChunks) {
-                // Skip if it's not from our URL prefix (safety check)
-                if (!url.startsWith(urlPrefix))
-                    continue;
-                let filePath;
-                let shouldDelete = false;
-                if (isRewriteMode) {
-                    // URL rewrite mode: extract relative path and construct full file path
-                    const config = pathConfig;
-                    const relativePath = url.substring(urlPrefix.length + 1); // +1 for the '/'
-                    filePath = path.join(config.path, relativePath);
-                    shouldDelete = !processedFiles.has(filePath);
-                }
-                else {
-                    // Direct file path mode: remove file:// prefix to match with processedFiles
-                    filePath = url.substring(7); // Remove 'file://' prefix
-                    shouldDelete = !processedFiles.has(filePath);
-                }
-                if (shouldDelete) {
-                    logger.debug(`Deleting obsolete chunk from SQLite: ${chunk_id.substring(0, 8)}... (File not processed: ${filePath})`);
-                    deleteChunkStmt.run(chunk_id);
-                    deletedCount++;
-                }
-            }
-        });
-        transaction();
-        logger.info(`Deleted ${deletedCount} obsolete chunks from SQLite for URL prefix ${urlPrefix}`);
-    }
-    async removeObsoleteFilesQdrant(db, processedFiles, pathConfig, logger) {
-        const { client, collectionName } = db;
-        try {
-            // Determine if we're using URL rewriting or direct file paths
-            const isRewriteMode = typeof pathConfig === 'object' && pathConfig.url_rewrite_prefix;
-            // Set up the URL prefix for searching
-            let urlPrefix;
-            if (isRewriteMode) {
-                // Handle URL rewriting case
-                urlPrefix = pathConfig.url_rewrite_prefix || '';
-                urlPrefix = urlPrefix.endsWith('/') ? urlPrefix.slice(0, -1) : urlPrefix;
-            }
-            else {
-                // Handle direct file path case
-                const dirPrefix = typeof pathConfig === 'string' ? pathConfig : pathConfig.path;
-                const cleanedDirPrefix = dirPrefix.replace(/^\.\/+/, '');
-                urlPrefix = `file://${cleanedDirPrefix}`;
-            }
-            logger.debug(`Checking for obsolete chunks with URL prefix: ${urlPrefix}`);
-            const response = await client.scroll(collectionName, {
-                limit: 10000,
-                with_payload: true,
-                with_vector: false,
-                filter: {
-                    must: [
-                        {
-                            key: "url",
-                            match: {
-                                text: urlPrefix
-                            }
-                        }
-                    ],
-                    must_not: [
-                        {
-                            key: "is_metadata",
-                            match: {
-                                value: true
-                            }
-                        }
-                    ]
-                }
-            });
-            const obsoletePointIds = response.points
-                .filter((point) => {
-                const url = point.payload?.url;
-                // Double check it's not a metadata record
-                if (point.payload?.is_metadata === true) {
-                    return false;
-                }
-                if (!url || !url.startsWith(urlPrefix)) {
-                    return false;
-                }
-                let filePath;
-                if (isRewriteMode) {
-                    // URL rewrite mode: extract relative path and construct full file path
-                    const config = pathConfig;
-                    const relativePath = url.substring(urlPrefix.length + 1); // +1 for the '/'
-                    filePath = path.join(config.path, relativePath);
-                }
-                else {
-                    // Direct file path mode: remove file:// prefix to match with processedFiles
-                    filePath = url.startsWith('file://') ? url.substring(7) : '';
-                }
-                return filePath && !processedFiles.has(filePath);
-            })
-                .map((point) => point.id);
-            if (obsoletePointIds.length > 0) {
-                await client.delete(collectionName, {
-                    points: obsoletePointIds,
-                });
-                logger.info(`Deleted ${obsoletePointIds.length} obsolete chunks from Qdrant for URL prefix ${urlPrefix}`);
-            }
-            else {
-                logger.info(`No obsolete chunks to delete from Qdrant for URL prefix ${urlPrefix}`);
-            }
-        }
-        catch (error) {
-            logger.error(`Error removing obsolete chunks from Qdrant:`, error);
-        }
-    }
-    async initDatabase(config, parentLogger) {
-        const logger = parentLogger.child('database');
-        const dbConfig = config.database_config;
-        if (dbConfig.type === 'sqlite') {
-            const params = dbConfig.params;
-            const dbPath = params.db_path || path.join(process.cwd(), `${config.product_name.replace(/\s+/g, '_')}-${config.version}.db`);
-            logger.info(`Opening SQLite database at ${dbPath}`);
-            const db = new better_sqlite3_1.default(dbPath, { allowExtension: true });
-            sqliteVec.load(db);
-            logger.debug(`Creating vec_items table if it doesn't exist`);
-            db.exec(`
-                CREATE VIRTUAL TABLE IF NOT EXISTS vec_items USING vec0(
-                    embedding FLOAT[3072],
-                    product_name TEXT,
-                    version TEXT,
-                    heading_hierarchy TEXT,
-                    section TEXT,
-                    chunk_id TEXT UNIQUE,
-                    content TEXT,
-                    url TEXT,
-                    hash TEXT
-                );
-            `);
-            logger.info(`SQLite database initialized successfully`);
-            return { db, type: 'sqlite' };
-        }
-        else if (dbConfig.type === 'qdrant') {
-            const params = dbConfig.params;
-            const qdrantUrl = params.qdrant_url || 'http://localhost:6333';
-            const qdrantPort = params.qdrant_port || 443;
-            const collectionName = params.collection_name || `${config.product_name.toLowerCase().replace(/\s+/g, '_')}_${config.version}`;
-            logger.info(`Connecting to Qdrant at ${qdrantUrl}:${qdrantPort}, collection: ${collectionName}`);
-            const qdrantClient = new js_client_rest_1.QdrantClient({ url: qdrantUrl, apiKey: process.env.QDRANT_API_KEY, port: qdrantPort });
-            await this.createCollectionQdrant(qdrantClient, collectionName, logger);
-            logger.info(`Qdrant connection established successfully`);
-            return { client: qdrantClient, collectionName, type: 'qdrant' };
-        }
-        else {
-            const errMsg = `Unsupported database type: ${dbConfig.type}`;
-            logger.error(errMsg);
-            throw new Error(errMsg);
-        }
-    }
-    async createCollectionQdrant(qdrantClient, collectionName, logger) {
-        try {
-            logger.debug(`Checking if collection ${collectionName} exists`);
-            const collections = await qdrantClient.getCollections();
-            const collectionExists = collections.collections.some((collection) => collection.name === collectionName);
-            if (collectionExists) {
-                logger.info(`Collection ${collectionName} already exists`);
-                return;
-            }
-            logger.info(`Creating new collection ${collectionName}`);
-            await qdrantClient.createCollection(collectionName, {
-                vectors: {
-                    size: 3072,
-                    distance: "Cosine",
-                },
-            });
-            logger.info(`Collection ${collectionName} created successfully`);
-        }
-        catch (error) {
-            if (error instanceof Error) {
-                const errorMsg = error.message.toLowerCase();
-                const errorString = JSON.stringify(error).toLowerCase();
-                if (errorMsg.includes("already exists") ||
-                    errorString.includes("already exists") ||
-                    error?.status === 409 ||
-                    errorString.includes("conflict")) {
-                    logger.info(`Collection ${collectionName} already exists (from error response)`);
-                    return;
-                }
-            }
-            logger.error(`Error creating Qdrant collection:`, error);
-            logger.warn(`Continuing with existing collection...`);
-        }
-    }
-    prepareSQLiteStatements(db) {
-        return {
-            insertStmt: db.prepare(`
-            INSERT INTO vec_items (embedding, product_name, version, heading_hierarchy, section, chunk_id, content, url, hash)
-            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
-        `),
-            checkHashStmt: db.prepare(`SELECT hash FROM vec_items WHERE chunk_id = ?`),
-            updateStmt: db.prepare(`
-            UPDATE vec_items SET embedding = ?, product_name = ?, version = ?, heading_hierarchy = ?, section = ?, content = ?, url = ?, hash = ?
-            WHERE chunk_id = ?
-        `),
-            getAllChunkIdsStmt: db.prepare(`SELECT chunk_id FROM vec_items`),
-            deleteChunkStmt: db.prepare(`DELETE FROM vec_items WHERE chunk_id = ?`)
-        };
-    }
-    insertVectorsSQLite(db, chunk, embedding, logger, chunkHash) {
-        const { insertStmt, updateStmt } = this.prepareSQLiteStatements(db);
-        const hash = chunkHash || this.generateHash(chunk.content);
-        const transaction = db.transaction(() => {
-            const params = [
-                new Float32Array(embedding),
-                chunk.metadata.product_name,
-                chunk.metadata.version,
-                JSON.stringify(chunk.metadata.heading_hierarchy),
-                chunk.metadata.section,
-                chunk.metadata.chunk_id,
-                chunk.content,
-                chunk.metadata.url,
-                hash
-            ];
-            try {
-                insertStmt.run(params);
-            }
-            catch (error) {
-                updateStmt.run([...params.slice(0, 8), chunk.metadata.chunk_id]);
-            }
-        });
-        transaction();
-    }
-    async storeChunkInQdrant(db, chunk, embedding, chunkHash) {
-        const { client, collectionName } = db;
-        try {
-            let pointId;
-            try {
-                pointId = chunk.metadata.chunk_id;
-                if (!this.isValidUuid(pointId)) {
-                    pointId = this.hashToUuid(chunk.metadata.chunk_id);
-                }
-            }
-            catch (e) {
-                pointId = crypto_1.default.randomUUID();
-            }
-            const hash = chunkHash || this.generateHash(chunk.content);
-            const pointItem = {
-                id: pointId,
-                vector: embedding,
-                payload: {
-                    content: chunk.content,
-                    product_name: chunk.metadata.product_name,
-                    version: chunk.metadata.version,
-                    heading_hierarchy: chunk.metadata.heading_hierarchy,
-                    section: chunk.metadata.section,
-                    url: chunk.metadata.url,
-                    hash: hash,
-                    original_chunk_id: chunk.metadata.chunk_id,
-                },
-            };
-            await client.upsert(collectionName, {
-                wait: true,
-                points: [pointItem],
-            });
-        }
-        catch (error) {
-            this.logger.error("Error storing chunk in Qdrant:", error);
-        }
-    }
-    removeObsoleteChunksSQLite(db, visitedUrls, urlPrefix, logger) {
-        const getChunksForUrlStmt = db.prepare(`
-            SELECT chunk_id, url FROM vec_items
-            WHERE url LIKE ? || '%'
-        `);
-        const deleteChunkStmt = db.prepare(`DELETE FROM vec_items WHERE chunk_id = ?`);
-        const existingChunks = getChunksForUrlStmt.all(urlPrefix);
-        let deletedCount = 0;
-        const transaction = db.transaction(() => {
-            for (const { chunk_id, url } of existingChunks) {
-                if (!visitedUrls.has(url)) {
-                    logger.debug(`Deleting obsolete chunk from SQLite: ${chunk_id.substring(0, 8)}... (URL not visited)`);
-                    deleteChunkStmt.run(chunk_id);
-                    deletedCount++;
-                }
-            }
-        });
-        transaction();
-        logger.info(`Deleted ${deletedCount} obsolete chunks from SQLite for URL ${urlPrefix}`);
-    }
-    isValidUuid(str) {
-        const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
-        return uuidRegex.test(str);
-    }
-    hashToUuid(hash) {
-        const truncatedHash = hash.substring(0, 32);
-        return [
-            truncatedHash.substring(0, 8),
-            truncatedHash.substring(8, 12),
-            '5' + truncatedHash.substring(13, 16),
-            '8' + truncatedHash.substring(17, 20),
-            truncatedHash.substring(20, 32)
-        ].join('-');
-    }
-    async removeObsoleteChunksQdrant(db, visitedUrls, urlPrefix, logger) {
-        const { client, collectionName } = db;
-        try {
-            // Get all points that match the URL prefix but are not metadata points
-            const response = await client.scroll(collectionName, {
-                limit: 10000,
-                with_payload: true,
-                with_vector: false,
-                filter: {
-                    must: [
-                        {
-                            key: "url",
-                            match: {
-                                text: urlPrefix
-                            }
-                        }
-                    ],
-                    must_not: [
-                        {
-                            key: "is_metadata",
-                            match: {
-                                value: true
-                            }
-                        }
-                    ]
-                }
-            });
-            const obsoletePointIds = response.points
-                .filter((point) => {
-                const url = point.payload?.url;
-                // Double check it's not a metadata record
-                if (point.payload?.is_metadata === true) {
-                    return false;
-                }
-                return url && !visitedUrls.has(url);
-            })
-                .map((point) => point.id);
-            if (obsoletePointIds.length > 0) {
-                await client.delete(collectionName, {
-                    points: obsoletePointIds,
-                });
-                logger.info(`Deleted ${obsoletePointIds.length} obsolete chunks from Qdrant for URL ${urlPrefix}`);
-            }
-            else {
-                logger.info(`No obsolete chunks to delete from Qdrant for URL ${urlPrefix}`);
-            }
-        }
-        catch (error) {
-            logger.error(`Error removing obsolete chunks from Qdrant:`, error);
-        }
-    }
-    async parseSitemap(sitemapUrl, logger) {
-        logger.info(`Parsing sitemap from ${sitemapUrl}`);
-        try {
-            const response = await axios_1.default.get(sitemapUrl);
-            const $ = (0, cheerio_1.load)(response.data, { xmlMode: true });
-            const urls = [];
-            // Handle standard sitemaps
-            $('url > loc').each((_, element) => {
-                const url = $(element).text().trim();
-                if (url) {
-                    urls.push(url);
-                }
-            });
-            // Handle sitemap indexes (sitemaps that link to other sitemaps)
-            const sitemapLinks = [];
-            $('sitemap > loc').each((_, element) => {
-                const nestedSitemapUrl = $(element).text().trim();
-                if (nestedSitemapUrl) {
-                    sitemapLinks.push(nestedSitemapUrl);
-                }
-            });
-            // Recursively process nested sitemaps
-            for (const nestedSitemapUrl of sitemapLinks) {
-                logger.debug(`Found nested sitemap: ${nestedSitemapUrl}`);
-                const nestedUrls = await this.parseSitemap(nestedSitemapUrl, logger);
-                urls.push(...nestedUrls);
-            }
-            logger.info(`Found ${urls.length} URLs in sitemap ${sitemapUrl}`);
-            return urls;
-        }
-        catch (error) {
-            logger.error(`Error parsing sitemap at ${sitemapUrl}:`, error);
-            return [];
-        }
-    }
-    async crawlWebsite(baseUrl, sourceConfig, processPageContent, parentLogger, visitedUrls) {
-        const logger = parentLogger.child('crawler');
-        const queue = [baseUrl];
-        // Process sitemap if provided
-        if (sourceConfig.sitemap_url) {
-            logger.section('SITEMAP PROCESSING');
-            const sitemapUrls = await this.parseSitemap(sourceConfig.sitemap_url, logger);
-            // Add sitemap URLs to the queue if they're within the website scope
-            for (const url of sitemapUrls) {
-                if (url.startsWith(sourceConfig.url) && !queue.includes(url)) {
-                    logger.debug(`Adding URL from sitemap to queue: ${url}`);
-                    queue.push(url);
-                }
-            }
-            logger.info(`Added ${queue.length - 1} URLs from sitemap to the crawl queue`);
-        }
-        logger.info(`Starting crawl from ${baseUrl} with ${queue.length} URLs in initial queue`);
-        let processedCount = 0;
-        let skippedCount = 0;
-        let skippedSizeCount = 0;
-        let errorCount = 0;
-        while (queue.length > 0) {
-            const url = queue.shift();
-            if (!url)
-                continue;
-            const normalizedUrl = this.normalizeUrl(url);
-            if (visitedUrls.has(normalizedUrl))
-                continue;
-            visitedUrls.add(normalizedUrl);
-            if (!this.shouldProcessUrl(url)) {
-                logger.debug(`Skipping URL with unsupported extension: ${url}`);
-                skippedCount++;
-                continue;
-            }
-            try {
-                logger.info(`Crawling: ${url}`);
-                const content = await this.processPage(url, sourceConfig);
-                if (content !== null) {
-                    await processPageContent(url, content);
-                    processedCount++;
-                }
-                else {
-                    skippedSizeCount++;
-                }
-                const response = await axios_1.default.get(url);
-                const $ = (0, cheerio_1.load)(response.data);
-                logger.debug(`Finding links on page ${url}`);
-                let newLinksFound = 0;
-                $('a[href]').each((_, element) => {
-                    const href = $(element).attr('href');
-                    if (!href || href.startsWith('#') || href.startsWith('mailto:'))
-                        return;
-                    const fullUrl = this.buildUrl(href, url);
-                    if (fullUrl.startsWith(sourceConfig.url) && !visitedUrls.has(this.normalizeUrl(fullUrl))) {
-                        if (!queue.includes(fullUrl)) {
-                            queue.push(fullUrl);
-                            newLinksFound++;
-                        }
-                    }
-                });
-                logger.debug(`Found ${newLinksFound} new links on ${url}`);
-            }
-            catch (error) {
-                logger.error(`Failed during link discovery or initial fetch for ${url}:`, error);
-                errorCount++;
-            }
-        }
-        logger.info(`Crawl completed. Processed: ${processedCount}, Skipped (Extension): ${skippedCount}, Skipped (Size): ${skippedSizeCount}, Errors: ${errorCount}`);
-    }
-    shouldProcessUrl(url) {
-        const parsedUrl = new URL(url);
-        const pathname = parsedUrl.pathname;
-        const ext = path.extname(pathname);
-        if (!ext)
-            return true;
-        return ['.html', '.htm'].includes(ext.toLowerCase());
-    }
-    normalizeUrl(url) {
-        try {
-            const urlObj = new URL(url);
-            urlObj.hash = '';
-            urlObj.search = '';
-            return urlObj.toString();
-        }
-        catch (error) {
-            return url;
-        }
-    }
-    buildUrl(href, currentUrl) {
-        try {
-            return new URL(href, currentUrl).toString();
-        }
-        catch (error) {
-            this.logger.warn(`Invalid URL found: ${href}`);
-            return '';
-        }
-    }
-    async processPage(url, sourceConfig) {
-        const logger = this.logger.child('page-processor');
-        logger.debug(`Processing page content from ${url}`);
-        let browser = null;
-        try {
-            browser = await puppeteer_1.default.launch({
-                args: ['--no-sandbox', '--disable-setuid-sandbox'],
-            });
-            const page = await browser.newPage();
-            logger.debug(`Navigating to ${url}`);
-            await page.goto(url, { waitUntil: 'networkidle2', timeout: 60000 });
-            const htmlContent = await page.evaluate(() => {
-                const mainContentElement = document.querySelector('div[role="main"].document') || document.querySelector('main') || document.body;
-                return mainContentElement.innerHTML;
-            });
-            if (htmlContent.length > sourceConfig.max_size) {
-                logger.warn(`Raw HTML content (${htmlContent.length} chars) exceeds max size (${sourceConfig.max_size}). Skipping detailed processing for ${url}.`);
-                await browser.close();
-                return null;
-            }
-            logger.debug(`Got HTML content (${htmlContent.length} chars), creating DOM`);
-            const dom = new jsdom_1.JSDOM(htmlContent);
-            const document = dom.window.document;
-            document.querySelectorAll('pre').forEach((pre) => {
-                pre.classList.add('article-content');
-                pre.setAttribute('data-readable-content-score', '100');
-                this.markCodeParents(pre.parentElement);
-            });
-            logger.debug(`Applying Readability to extract main content`);
-            const reader = new readability_1.Readability(document, {
-                charThreshold: 20,
-                classesToPreserve: ['article-content'],
-            });
-            const article = reader.parse();
-            if (!article) {
-                logger.warn(`Failed to parse article content with Readability for ${url}`);
-                await browser.close();
-                return null;
-            }
-            logger.debug(`Sanitizing HTML (${article.content.length} chars)`);
-            const cleanHtml = (0, sanitize_html_1.default)(article.content, {
-                allowedTags: [
-                    'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'a', 'ul', 'ol',
-                    'li', 'b', 'i', 'strong', 'em', 'code', 'pre',
-                    'div', 'span', 'table', 'thead', 'tbody', 'tr', 'th', 'td'
-                ],
-                allowedAttributes: {
-                    'a': ['href'],
-                    'pre': ['class', 'data-language'],
-                    'code': ['class', 'data-language'],
-                    'div': ['class'],
-                    'span': ['class']
-                }
-            });
-            logger.debug(`Converting HTML to Markdown`);
-            const markdown = this.turndownService.turndown(cleanHtml);
-            logger.debug(`Markdown conversion complete (${markdown.length} chars)`);
-            return markdown;
-        }
-        catch (error) {
-            logger.error(`Error processing page ${url}:`, error);
-            return null;
-        }
-        finally {
-            if (browser && browser.isConnected()) {
-                await browser.close();
-                logger.debug(`Browser closed for ${url}`);
-            }
-        }
-    }
-    markCodeParents(node) {
-        if (!node)
-            return;
-        if (node.querySelector('pre, code')) {
-            node.classList.add('article-content');
-            node.setAttribute('data-readable-content-score', '100');
-        }
-        this.markCodeParents(node.parentElement);
-    }
-    async chunkMarkdown(markdown, sourceConfig, url) {
-        const logger = this.logger.child('chunker');
-        logger.debug(`Chunking markdown from ${url} (${markdown.length} chars)`);
-        const MAX_TOKENS = 1000;
-        const chunks = [];
-        const lines = markdown.split("\n");
-        let currentChunk = "";
-        let headingHierarchy = [];
-        const processChunk = () => {
-            if (currentChunk.trim()) {
-                const tokens = this.tokenize(currentChunk);
-                if (tokens.length > MAX_TOKENS) {
-                    logger.debug(`Chunk exceeds max token count (${tokens.length}), splitting into smaller chunks`);
-                    let subChunk = "";
-                    let tokenCount = 0;
-                    const overlapSize = Math.floor(MAX_TOKENS * 0.05);
-                    let lastTokens = [];
-                    for (const token of tokens) {
-                        if (tokenCount + 1 > MAX_TOKENS) {
-                            chunks.push(createDocumentChunk(subChunk, headingHierarchy));
-                            subChunk = lastTokens.join("") + token;
-                            tokenCount = lastTokens.length + 1;
-                            lastTokens = [];
-                        }
-                        else {
-                            subChunk += token;
-                            tokenCount++;
-                            lastTokens.push(token);
-                            if (lastTokens.length > overlapSize) {
-                                lastTokens.shift();
-                            }
-                        }
-                    }
-                    if (subChunk) {
-                        chunks.push(createDocumentChunk(subChunk, headingHierarchy));
-                    }
-                }
-                else {
-                    chunks.push(createDocumentChunk(currentChunk, headingHierarchy));
-                }
-            }
-            currentChunk = "";
-        };
-        const createDocumentChunk = (content, hierarchy) => {
-            const chunkId = this.generateHash(content);
-            logger.debug(`Created chunk ${chunkId.substring(0, 8)}... with ${content.length} chars`);
-            return {
-                content,
-                metadata: {
-                    product_name: sourceConfig.product_name,
-                    version: sourceConfig.version,
-                    heading_hierarchy: [...hierarchy],
-                    section: hierarchy[hierarchy.length - 1] || "Introduction",
-                    chunk_id: chunkId,
-                    url: url,
-                    hash: this.generateHash(content)
-                }
-            };
-        };
-        for (const line of lines) {
-            if (line.startsWith("#")) {
-                processChunk();
-                const levelMatch = line.match(/^(#+)/);
-                let level = levelMatch ? levelMatch[1].length : 1;
-                const heading = line.replace(/^#+\s*/, "").trim();
-                logger.debug(`Found heading (level ${level}): ${heading}`);
-                while (headingHierarchy.length < level - 1) {
-                    headingHierarchy.push("");
-                }
-                if (level <= headingHierarchy.length) {
-                    headingHierarchy = headingHierarchy.slice(0, level - 1);
-                }
-                headingHierarchy[level - 1] = heading;
-            }
-            else {
-                currentChunk += `${line}\n`;
-            }
-        }
-        processChunk();
-        logger.debug(`Chunking complete, created ${chunks.length} chunks`);
-        return chunks;
-    }
-    tokenize(text) {
-        return text.split(/(\s+)/).filter(token => token.length > 0);
-    }
     async createEmbeddings(texts) {
         const logger = this.logger.child('embeddings');
         try {
@@ -1470,9 +509,6 @@ class Doc2Vec {
             return [];
         }
     }
-    generateHash(content) {
-        return crypto_1.default.createHash("sha256").update(content).digest("hex");
-    }
 }
 if (require.main === module) {
     const configPath = process.argv[2] || 'config.yaml';