doc2vec 1.0.2 → 1.0.4

package/doc2vec.js

@@ -1,1805 +0,0 @@
-#!/usr/bin/env ts-node
-"use strict";
-var __assign = (this && this.__assign) || function () {
-    __assign = Object.assign || function(t) {
-        for (var s, i = 1, n = arguments.length; i < n; i++) {
-            s = arguments[i];
-            for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
-                t[p] = s[p];
-        }
-        return t;
-    };
-    return __assign.apply(this, arguments);
-};
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __generator = (this && this.__generator) || function (thisArg, body) {
-    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype);
-    return g.next = verb(0), g["throw"] = verb(1), g["return"] = verb(2), typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
-    function verb(n) { return function (v) { return step([n, v]); }; }
-    function step(op) {
-        if (f) throw new TypeError("Generator is already executing.");
-        while (g && (g = 0, op[0] && (_ = 0)), _) try {
-            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
-            if (y = 0, t) op = [op[0] & 2, t.value];
-            switch (op[0]) {
-                case 0: case 1: t = op; break;
-                case 4: _.label++; return { value: op[1], done: false };
-                case 5: _.label++; y = op[1]; op = [0]; continue;
-                case 7: op = _.ops.pop(); _.trys.pop(); continue;
-                default:
-                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
-                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
-                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
-                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
-                    if (t[2]) _.ops.pop();
-                    _.trys.pop(); continue;
-            }
-            op = body.call(thisArg, _);
-        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
-        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
-    }
-};
-var __spreadArray = (this && this.__spreadArray) || function (to, from, pack) {
-    if (pack || arguments.length === 2) for (var i = 0, l = from.length, ar; i < l; i++) {
-        if (ar || !(i in from)) {
-            if (!ar) ar = Array.prototype.slice.call(from, 0, i);
-            ar[i] = from[i];
-        }
-    }
-    return to.concat(ar || Array.prototype.slice.call(from));
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-var readability_1 = require("@mozilla/readability");
-var axios_1 = require("axios");
-var cheerio_1 = require("cheerio");
-var crypto_1 = require("crypto");
-var jsdom_1 = require("jsdom");
-var puppeteer_1 = require("puppeteer");
-var sanitize_html_1 = require("sanitize-html");
-var turndown_1 = require("turndown");
-var yaml = require("js-yaml");
-var fs = require("fs");
-var path = require("path");
-var better_sqlite3_1 = require("better-sqlite3");
-var openai_1 = require("openai");
-var dotenv = require("dotenv");
-var sqliteVec = require("sqlite-vec");
-var js_client_rest_1 = require("@qdrant/js-client-rest");
-var logger_1 = require("./logger");
-var GITHUB_TOKEN = process.env.GITHUB_PERSONAL_ACCESS_TOKEN;
-dotenv.config();
-var Doc2Vec = /** @class */ (function () {
-    function Doc2Vec(configPath) {
-        this.logger = new logger_1.Logger('Doc2Vec', {
-            level: logger_1.LogLevel.DEBUG,
-            useTimestamp: true,
-            useColor: true,
-            prettyPrint: true
-        });
-        this.logger.info('Initializing Doc2Vec');
-        this.config = this.loadConfig(configPath);
-        this.openai = new openai_1.OpenAI({ apiKey: process.env.OPENAI_API_KEY });
-        this.turndownService = new turndown_1.default({
-            codeBlockStyle: 'fenced',
-            headingStyle: 'atx'
-        });
-        this.setupTurndownRules();
-    }
-    Doc2Vec.prototype.loadConfig = function (configPath) {
-        try {
-            var logger = this.logger.child('config');
-            logger.info("Loading configuration from ".concat(configPath));
-            var configFile = fs.readFileSync(configPath, 'utf8');
-            var config = yaml.load(configFile);
-            var typedConfig = config;
-            logger.info("Configuration loaded successfully, found ".concat(typedConfig.sources.length, " sources"));
-            return typedConfig;
-        }
-        catch (error) {
-            this.logger.error("Failed to load or parse config file at ".concat(configPath, ":"), error);
-            process.exit(1);
-        }
-    };
-    Doc2Vec.prototype.setupTurndownRules = function () {
-        var logger = this.logger.child('markdown');
-        logger.debug('Setting up Turndown rules for markdown conversion');
-        this.turndownService.addRule('codeBlocks', {
-            filter: function (node) { return node.nodeName === 'PRE'; },
-            replacement: function (content, node) {
-                var _a;
-                var htmlNode = node;
-                var code = htmlNode.querySelector('code');
-                var codeContent;
-                if (code) {
-                    codeContent = code.textContent || '';
-                }
-                else {
-                    codeContent = htmlNode.textContent || '';
-                }
-                var lines = codeContent.split('\n');
-                var minIndent = Infinity;
-                for (var _i = 0, lines_1 = lines; _i < lines_1.length; _i++) {
-                    var line = lines_1[_i];
-                    if (line.trim() === '')
-                        continue;
-                    var leadingWhitespace = ((_a = line.match(/^\s*/)) === null || _a === void 0 ? void 0 : _a[0]) || '';
-                    minIndent = Math.min(minIndent, leadingWhitespace.length);
-                }
-                var cleanedLines = lines.map(function (line) {
-                    return line.substring(minIndent);
-                });
-                var cleanContent = cleanedLines.join('\n');
-                cleanContent = cleanContent.replace(/^\s+|\s+$/g, '');
-                cleanContent = cleanContent.replace(/\n{2,}/g, '\n');
-                return "\n```\n".concat(cleanContent, "\n```\n");
-            }
-        });
-        this.turndownService.addRule('tableCell', {
-            filter: ['th', 'td'],
-            replacement: function (content, node) {
-                var htmlNode = node;
-                var cellContent = '';
-                if (htmlNode.querySelector('p')) {
-                    cellContent = Array.from(htmlNode.querySelectorAll('p'))
-                        .map(function (p) { return p.textContent || ''; })
-                        .join(' ')
-                        .trim();
-                }
-                else {
-                    cellContent = content.trim();
-                }
-                return " ".concat(cellContent.replace(/\|/g, '\\|'), " |");
-            }
-        });
-        this.turndownService.addRule('tableRow', {
-            filter: 'tr',
-            replacement: function (content, node) {
-                var _a;
-                var htmlNode = node;
-                var cells = Array.from(htmlNode.cells);
-                var isHeader = ((_a = htmlNode.parentNode) === null || _a === void 0 ? void 0 : _a.nodeName) === 'THEAD';
-                var output = '|' + content.trimEnd();
-                if (isHeader) {
-                    var separator = cells.map(function () { return '---'; }).join(' | ');
-                    output += '\n|' + separator + '|';
-                }
-                if (!isHeader || !htmlNode.nextElementSibling) {
-                    output += '\n';
-                }
-                return output;
-            }
-        });
-        this.turndownService.addRule('table', {
-            filter: 'table',
-            replacement: function (content) {
-                return '\n' + content.replace(/\n+/g, '\n').trim() + '\n';
-            }
-        });
-        this.turndownService.addRule('preserveTableWhitespace', {
-            filter: function (node) {
-                var _a;
-                return ((node.nodeName === 'TD' || node.nodeName === 'TH') &&
-                    (((_a = node.textContent) === null || _a === void 0 ? void 0 : _a.trim().length) === 0));
-            },
-            replacement: function () {
-                return ' |';
-            }
-        });
-        logger.debug('Turndown rules setup complete');
-    };
-    Doc2Vec.prototype.run = function () {
-        return __awaiter(this, void 0, void 0, function () {
-            var _i, _a, sourceConfig, sourceLogger;
-            return __generator(this, function (_b) {
-                switch (_b.label) {
-                    case 0:
-                        this.logger.section('PROCESSING SOURCES');
-                        _i = 0, _a = this.config.sources;
-                        _b.label = 1;
-                    case 1:
-                        if (!(_i < _a.length)) return [3 /*break*/, 9];
-                        sourceConfig = _a[_i];
-                        sourceLogger = this.logger.child("source:".concat(sourceConfig.product_name));
-                        sourceLogger.info("Processing ".concat(sourceConfig.type, " source for ").concat(sourceConfig.product_name, "@").concat(sourceConfig.version));
-                        if (!(sourceConfig.type === 'github')) return [3 /*break*/, 3];
-                        return [4 /*yield*/, this.processGithubRepo(sourceConfig, sourceLogger)];
-                    case 2:
-                        _b.sent();
-                        return [3 /*break*/, 8];
-                    case 3:
-                        if (!(sourceConfig.type === 'website')) return [3 /*break*/, 5];
-                        return [4 /*yield*/, this.processWebsite(sourceConfig, sourceLogger)];
-                    case 4:
-                        _b.sent();
-                        return [3 /*break*/, 8];
-                    case 5:
-                        if (!(sourceConfig.type === 'local_directory')) return [3 /*break*/, 7];
-                        return [4 /*yield*/, this.processLocalDirectory(sourceConfig, sourceLogger)];
-                    case 6:
-                        _b.sent();
-                        return [3 /*break*/, 8];
-                    case 7:
-                        sourceLogger.error("Unknown source type: ".concat(sourceConfig.type));
-                        _b.label = 8;
-                    case 8:
-                        _i++;
-                        return [3 /*break*/, 1];
-                    case 9:
-                        this.logger.section('PROCESSING COMPLETE');
-                        return [2 /*return*/];
-                }
-            });
-        });
-    };
-    Doc2Vec.prototype.getUrlPrefix = function (url) {
-        try {
-            var parsedUrl = new URL(url);
-            return parsedUrl.origin + parsedUrl.pathname;
-        }
-        catch (error) {
-            return url;
-        }
-    };
-    Doc2Vec.prototype.generateMetadataUUID = function (repo) {
-        // Simple deterministic approach - hash the repo name and convert to UUID format
-        var hash = crypto_1.default.createHash('md5').update("metadata_".concat(repo)).digest('hex');
-        // Format as UUID with version bits set correctly (version 4)
-        return "".concat(hash.substr(0, 8), "-").concat(hash.substr(8, 4), "-4").concat(hash.substr(13, 3), "-").concat(hash.substr(16, 4), "-").concat(hash.substr(20, 12));
-    };
-    Doc2Vec.prototype.initDatabaseMetadata = function (dbConnection) {
-        return __awaiter(this, void 0, void 0, function () {
-            var logger, db;
-            return __generator(this, function (_a) {
-                logger = this.logger.child('metadata');
-                if (dbConnection.type === 'sqlite') {
-                    db = dbConnection.db;
-                    logger.debug('Creating metadata table if it doesn\'t exist');
-                    db.exec("\n                CREATE TABLE IF NOT EXISTS vec_metadata (\n                    key TEXT PRIMARY KEY,\n                    value TEXT\n                );\n            ");
-                    logger.info('SQLite metadata table initialized');
-                }
-                else if (dbConnection.type === 'qdrant') {
-                    // For Qdrant, we'll use the same collection but verify it exists
-                    logger.info("Using existing Qdrant collection for metadata: ".concat(dbConnection.collectionName));
-                    // Nothing special to initialize as we'll use the same collection
-                }
-                return [2 /*return*/];
-            });
-        });
-    };
-    Doc2Vec.prototype.getLastRunDate = function (dbConnection, repo, defaultDate) {
-        return __awaiter(this, void 0, void 0, function () {
-            var logger, metadataKey, stmt, result, metadataUUID, response, lastRunDate, error_1, error_2;
-            var _a;
-            return __generator(this, function (_b) {
-                switch (_b.label) {
-                    case 0:
-                        logger = this.logger.child('metadata');
-                        metadataKey = "last_run_".concat(repo.replace('/', '_'));
-                        _b.label = 1;
-                    case 1:
-                        _b.trys.push([1, 7, , 8]);
-                        if (!(dbConnection.type === 'sqlite')) return [3 /*break*/, 2];
-                        stmt = dbConnection.db.prepare('SELECT value FROM vec_metadata WHERE key = ?');
-                        result = stmt.get(metadataKey);
-                        if (result) {
-                            logger.info("Retrieved last run date for ".concat(repo, ": ").concat(result.value));
-                            return [2 /*return*/, result.value];
-                        }
-                        return [3 /*break*/, 6];
-                    case 2:
-                        if (!(dbConnection.type === 'qdrant')) return [3 /*break*/, 6];
-                        metadataUUID = this.generateMetadataUUID(repo);
-                        logger.debug("Looking up metadata with UUID: ".concat(metadataUUID));
-                        _b.label = 3;
-                    case 3:
-                        _b.trys.push([3, 5, , 6]);
-                        return [4 /*yield*/, dbConnection.client.retrieve(dbConnection.collectionName, {
-                                ids: [metadataUUID],
-                                with_payload: true,
-                                with_vector: false
-                            })];
-                    case 4:
-                        response = _b.sent();
-                        if (response.length > 0 && ((_a = response[0].payload) === null || _a === void 0 ? void 0 : _a.metadata_value)) {
-                            lastRunDate = response[0].payload.metadata_value;
-                            logger.info("Retrieved last run date for ".concat(repo, ": ").concat(lastRunDate));
-                            return [2 /*return*/, lastRunDate];
-                        }
-                        return [3 /*break*/, 6];
-                    case 5:
-                        error_1 = _b.sent();
-                        logger.warn("Failed to retrieve metadata for ".concat(repo, ":"), error_1);
-                        return [3 /*break*/, 6];
-                    case 6: return [3 /*break*/, 8];
-                    case 7:
-                        error_2 = _b.sent();
-                        logger.warn("Error retrieving last run date:", error_2);
-                        return [3 /*break*/, 8];
-                    case 8:
-                        logger.info("No saved run date found for ".concat(repo, ", using default: ").concat(defaultDate));
-                        return [2 /*return*/, defaultDate];
-                }
-            });
-        });
-    };
-    Doc2Vec.prototype.updateLastRunDate = function (dbConnection, repo) {
-        return __awaiter(this, void 0, void 0, function () {
-            var logger, now, metadataKey, stmt, metadataUUID, metadataKey, dummyEmbeddingSize, dummyEmbedding, metadataPoint, error_3;
-            return __generator(this, function (_a) {
-                switch (_a.label) {
-                    case 0:
-                        logger = this.logger.child('metadata');
-                        now = new Date().toISOString();
-                        _a.label = 1;
-                    case 1:
-                        _a.trys.push([1, 5, , 6]);
-                        if (!(dbConnection.type === 'sqlite')) return [3 /*break*/, 2];
-                        metadataKey = "last_run_".concat(repo.replace('/', '_'));
-                        stmt = dbConnection.db.prepare("\n                    INSERT INTO vec_metadata (key, value) VALUES (?, ?)\n                    ON CONFLICT(key) DO UPDATE SET value = excluded.value\n                ");
-                        stmt.run(metadataKey, now);
-                        logger.info("Updated last run date for ".concat(repo, " to ").concat(now));
-                        return [3 /*break*/, 4];
-                    case 2:
-                        if (!(dbConnection.type === 'qdrant')) return [3 /*break*/, 4];
-                        metadataUUID = this.generateMetadataUUID(repo);
-                        metadataKey = "last_run_".concat(repo.replace('/', '_'));
-                        logger.debug("Using UUID: ".concat(metadataUUID, " for metadata"));
-                        dummyEmbeddingSize = 3072;
-                        dummyEmbedding = new Array(dummyEmbeddingSize).fill(0);
-                        metadataPoint = {
-                            id: metadataUUID,
-                            vector: dummyEmbedding,
-                            payload: {
-                                metadata_key: metadataKey,
-                                metadata_value: now,
-                                is_metadata: true, // Flag to identify metadata points
-                                content: "Metadata: Last run date for ".concat(repo),
-                                product_name: 'system',
-                                version: 'metadata',
-                                url: 'metadata://' + repo
-                            }
-                        };
-                        return [4 /*yield*/, dbConnection.client.upsert(dbConnection.collectionName, {
-                                wait: true,
-                                points: [metadataPoint]
-                            })];
-                    case 3:
-                        _a.sent();
-                        logger.info("Updated last run date for ".concat(repo, " to ").concat(now));
-                        _a.label = 4;
-                    case 4: return [3 /*break*/, 6];
-                    case 5:
-                        error_3 = _a.sent();
-                        logger.error("Failed to update last run date for ".concat(repo, ":"), error_3);
-                        return [3 /*break*/, 6];
-                    case 6: return [2 /*return*/];
-                }
-            });
-        });
-    };
-    Doc2Vec.prototype.fetchAndProcessGitHubIssues = function (repo, sourceConfig, dbConnection, logger) {
-        return __awaiter(this, void 0, void 0, function () {
-            var _a, owner, repoName, GITHUB_API_URL, startDate, lastRunDate, fetchWithRetry, fetchAllIssues, fetchIssueComments, generateMarkdownForIssue, processIssue, issues, i;
-            var _this = this;
-            return __generator(this, function (_b) {
-                switch (_b.label) {
-                    case 0:
-                        _a = repo.split('/'), owner = _a[0], repoName = _a[1];
-                        GITHUB_API_URL = "https://api.github.com/repos/".concat(owner, "/").concat(repoName, "/issues");
-                        // Initialize metadata storage if needed
-                        return [4 /*yield*/, this.initDatabaseMetadata(dbConnection)];
-                    case 1:
-                        // Initialize metadata storage if needed
-                        _b.sent();
-                        startDate = sourceConfig.start_date || '2025-01-01';
-                        return [4 /*yield*/, this.getLastRunDate(dbConnection, repo, "".concat(startDate, "T00:00:00Z"))];
-                    case 2:
-                        lastRunDate = _b.sent();
-                        fetchWithRetry = function (url_1) {
-                            var args_1 = [];
-                            for (var _i = 1; _i < arguments.length; _i++) {
-                                args_1[_i - 1] = arguments[_i];
-                            }
-                            return __awaiter(_this, __spreadArray([url_1], args_1, true), void 0, function (url, params, retries, delay) {
-                                var _loop_1, attempt, state_1;
-                                if (params === void 0) { params = {}; }
-                                if (retries === void 0) { retries = 5; }
-                                if (delay === void 0) { delay = 5000; }
-                                return __generator(this, function (_a) {
-                                    switch (_a.label) {
-                                        case 0:
-                                            _loop_1 = function (attempt) {
-                                                var response, error_4, resetTime, currentTime, waitTime_1;
-                                                return __generator(this, function (_b) {
-                                                    switch (_b.label) {
-                                                        case 0:
-                                                            _b.trys.push([0, 2, , 6]);
-                                                            return [4 /*yield*/, axios_1.default.get(url, {
-                                                                    headers: {
-                                                                        Authorization: "token ".concat(GITHUB_TOKEN),
-                                                                        Accept: 'application/vnd.github.v3+json',
-                                                                    },
-                                                                    params: params,
-                                                                })];
-                                                        case 1:
-                                                            response = _b.sent();
-                                                            return [2 /*return*/, { value: response.data }];
-                                                        case 2:
-                                                            error_4 = _b.sent();
-                                                            if (!(error_4.response && error_4.response.status === 403)) return [3 /*break*/, 4];
-                                                            resetTime = error_4.response.headers['x-ratelimit-reset'];
-                                                            currentTime = Math.floor(Date.now() / 1000);
-                                                            waitTime_1 = resetTime ? (resetTime - currentTime) * 1000 : delay * 2;
-                                                            logger.warn("GitHub rate limit exceeded. Waiting ".concat(waitTime_1 / 1000, "s"));
-                                                            return [4 /*yield*/, new Promise(function (res) { return setTimeout(res, waitTime_1); })];
-                                                        case 3:
-                                                            _b.sent();
-                                                            return [3 /*break*/, 5];
-                                                        case 4:
-                                                            logger.error("GitHub fetch failed: ".concat(error_4.message));
-                                                            throw error_4;
-                                                        case 5: return [3 /*break*/, 6];
-                                                        case 6: return [2 /*return*/];
-                                                    }
-                                                });
-                                            };
-                                            attempt = 0;
-                                            _a.label = 1;
-                                        case 1:
-                                            if (!(attempt < retries)) return [3 /*break*/, 4];
-                                            return [5 /*yield**/, _loop_1(attempt)];
-                                        case 2:
-                                            state_1 = _a.sent();
-                                            if (typeof state_1 === "object")
-                                                return [2 /*return*/, state_1.value];
-                                            _a.label = 3;
-                                        case 3:
-                                            attempt++;
-                                            return [3 /*break*/, 1];
-                                        case 4: throw new Error('Max retries reached');
-                                    }
-                                });
-                            });
-                        };
-                        fetchAllIssues = function (sinceDate) { return __awaiter(_this, void 0, void 0, function () {
-                            var issues, page, perPage, sinceTimestamp, data, filtered;
-                            return __generator(this, function (_a) {
-                                switch (_a.label) {
-                                    case 0:
-                                        issues = [];
-                                        page = 1;
-                                        perPage = 100;
-                                        sinceTimestamp = new Date(sinceDate);
-                                        _a.label = 1;
-                                    case 1:
-                                        if (!true) return [3 /*break*/, 3];
-                                        return [4 /*yield*/, fetchWithRetry(GITHUB_API_URL, {
-                                                per_page: perPage,
-                                                page: page,
-                                                state: 'all',
-                                                since: sinceDate,
-                                            })];
-                                    case 2:
-                                        data = _a.sent();
-                                        if (data.length === 0)
-                                            return [3 /*break*/, 3];
-                                        filtered = data.filter(function (issue) { return new Date(issue.created_at) >= sinceTimestamp; });
-                                        issues = issues.concat(filtered);
-                                        if (filtered.length < data.length)
-                                            return [3 /*break*/, 3];
-                                        page++;
-                                        return [3 /*break*/, 1];
-                                    case 3: return [2 /*return*/, issues];
-                                }
-                            });
-                        }); };
-                        fetchIssueComments = function (issueNumber) { return __awaiter(_this, void 0, void 0, function () {
-                            var url;
-                            return __generator(this, function (_a) {
-                                switch (_a.label) {
-                                    case 0:
-                                        url = "".concat(GITHUB_API_URL, "/").concat(issueNumber, "/comments");
-                                        return [4 /*yield*/, fetchWithRetry(url)];
-                                    case 1: return [2 /*return*/, _a.sent()];
-                                }
-                            });
-                        }); };
-                        generateMarkdownForIssue = function (issue) { return __awaiter(_this, void 0, void 0, function () {
-                            var comments, md, _i, comments_1, c;
-                            return __generator(this, function (_a) {
-                                switch (_a.label) {
-                                    case 0: return [4 /*yield*/, fetchIssueComments(issue.number)];
-                                    case 1:
-                                        comments = _a.sent();
-                                        md = "# Issue #".concat(issue.number, ": ").concat(issue.title, "\n\n");
-                                        md += "- **Author:** ".concat(issue.user.login, "\n");
-                                        md += "- **State:** ".concat(issue.state, "\n");
-                                        md += "- **Created on:** ".concat(new Date(issue.created_at).toDateString(), "\n");
-                                        md += "- **Updated on:** ".concat(new Date(issue.updated_at).toDateString(), "\n");
-                                        md += "- **Labels:** ".concat(issue.labels.map(function (l) { return "`".concat(l.name, "`"); }).join(', ') || 'None', "\n\n");
-                                        md += "## Description\n\n".concat(issue.body || '_No description._', "\n\n## Comments\n\n");
-                                        if (comments.length === 0) {
-                                            md += '_No comments._\n';
-                                        }
-                                        else {
-                                            for (_i = 0, comments_1 = comments; _i < comments_1.length; _i++) {
-                                                c = comments_1[_i];
-                                                md += "### ".concat(c.user.login, " - ").concat(new Date(c.created_at).toDateString(), "\n\n").concat(c.body, "\n\n---\n\n");
-                                            }
-                                        }
-                                        return [2 /*return*/, md];
-                                }
-                            });
-                        }); };
-                        processIssue = function (issue) { return __awaiter(_this, void 0, void 0, function () {
-                            var issueNumber, url, markdown, issueConfig, chunks, _i, chunks_1, chunk, chunkHash, chunkId, checkHashStmt, existing, embeddings, pointId, existingPoints, embeddings, error_5;
-                            return __generator(this, function (_a) {
-                                switch (_a.label) {
-                                    case 0:
-                                        issueNumber = issue.number;
-                                        url = "https://github.com/".concat(repo, "/issues/").concat(issueNumber);
-                                        logger.info("Processing issue #".concat(issueNumber));
-                                        return [4 /*yield*/, generateMarkdownForIssue(issue)];
-                                    case 1:
-                                        markdown = _a.sent();
-                                        issueConfig = __assign(__assign({}, sourceConfig), { product_name: sourceConfig.product_name || repo, max_size: sourceConfig.max_size || Infinity });
-                                        return [4 /*yield*/, this.chunkMarkdown(markdown, issueConfig, url)];
-                                    case 2:
-                                        chunks = _a.sent();
-                                        logger.info("Issue #".concat(issueNumber, ": Created ").concat(chunks.length, " chunks"));
-                                        _i = 0, chunks_1 = chunks;
-                                        _a.label = 3;
-                                    case 3:
-                                        if (!(_i < chunks_1.length)) return [3 /*break*/, 14];
-                                        chunk = chunks_1[_i];
-                                        chunkHash = this.generateHash(chunk.content);
-                                        chunkId = chunk.metadata.chunk_id.substring(0, 8) + '...';
-                                        if (!(dbConnection.type === 'sqlite')) return [3 /*break*/, 5];
-                                        checkHashStmt = this.prepareSQLiteStatements(dbConnection.db).checkHashStmt;
-                                        existing = checkHashStmt.get(chunk.metadata.chunk_id);
-                                        if (existing && existing.hash === chunkHash) {
-                                            logger.info("Skipping unchanged chunk: ".concat(chunkId));
-                                            return [3 /*break*/, 13];
-                                        }
-                                        return [4 /*yield*/, this.createEmbeddings([chunk.content])];
-                                    case 4:
-                                        embeddings = _a.sent();
-                                        if (embeddings.length) {
-                                            this.insertVectorsSQLite(dbConnection.db, chunk, embeddings[0], logger, chunkHash);
-                                            logger.debug("Stored chunk ".concat(chunkId, " in SQLite"));
-                                        }
-                                        else {
-                                            logger.error("Embedding failed for chunk: ".concat(chunkId));
-                                        }
-                                        return [3 /*break*/, 13];
-                                    case 5:
-                                        if (!(dbConnection.type === 'qdrant')) return [3 /*break*/, 13];
-                                        _a.label = 6;
-                                    case 6:
-                                        _a.trys.push([6, 12, , 13]);
-                                        pointId = void 0;
-                                        try {
-                                            pointId = chunk.metadata.chunk_id;
-                                            if (!this.isValidUuid(pointId)) {
-                                                pointId = this.hashToUuid(chunk.metadata.chunk_id);
-                                            }
-                                        }
-                                        catch (e) {
-                                            pointId = crypto_1.default.randomUUID();
-                                        }
-                                        return [4 /*yield*/, dbConnection.client.retrieve(dbConnection.collectionName, {
-                                                ids: [pointId],
-                                                with_payload: true,
-                                                with_vector: false,
-                                            })];
-                                    case 7:
-                                        existingPoints = _a.sent();
-                                        if (existingPoints.length > 0 && existingPoints[0].payload && existingPoints[0].payload.hash === chunkHash) {
-                                            logger.info("Skipping unchanged chunk: ".concat(chunkId));
-                                            return [3 /*break*/, 13];
-                                        }
-                                        return [4 /*yield*/, this.createEmbeddings([chunk.content])];
-                                    case 8:
-                                        embeddings = _a.sent();
-                                        if (!embeddings.length) return [3 /*break*/, 10];
-                                        return [4 /*yield*/, this.storeChunkInQdrant(dbConnection, chunk, embeddings[0], chunkHash)];
-                                    case 9:
-                                        _a.sent();
-                                        logger.debug("Stored chunk ".concat(chunkId, " in Qdrant (").concat(dbConnection.collectionName, ")"));
-                                        return [3 /*break*/, 11];
-                                    case 10:
-                                        logger.error("Embedding failed for chunk: ".concat(chunkId));
-                                        _a.label = 11;
-                                    case 11: return [3 /*break*/, 13];
-                                    case 12:
-                                        error_5 = _a.sent();
-                                        logger.error("Error processing chunk in Qdrant:", error_5);
-                                        return [3 /*break*/, 13];
-                                    case 13:
-                                        _i++;
-                                        return [3 /*break*/, 3];
-                                    case 14: return [2 /*return*/];
-                                }
-                            });
-                        }); };
-                        logger.info("Fetching GitHub issues for ".concat(repo, " since ").concat(lastRunDate));
-                        return [4 /*yield*/, fetchAllIssues(lastRunDate)];
-                    case 3:
-                        issues = _b.sent();
-                        logger.info("Found ".concat(issues.length, " updated/new issues"));
-                        i = 0;
-                        _b.label = 4;
-                    case 4:
-                        if (!(i < issues.length)) return [3 /*break*/, 7];
-                        logger.info("Processing issue ".concat(i + 1, "/").concat(issues.length));
-                        return [4 /*yield*/, processIssue(issues[i])];
-                    case 5:
-                        _b.sent();
-                        _b.label = 6;
-                    case 6:
-                        i++;
-                        return [3 /*break*/, 4];
-                    case 7: 
-                    // Update the last run date in the database after processing all issues
-                    return [4 /*yield*/, this.updateLastRunDate(dbConnection, repo)];
-                    case 8:
-                        // Update the last run date in the database after processing all issues
-                        _b.sent();
-                        logger.info("Successfully processed ".concat(issues.length, " issues"));
-                        return [2 /*return*/];
-                }
-            });
-        });
-    };
-    Doc2Vec.prototype.processGithubRepo = function (config, parentLogger) {
-        return __awaiter(this, void 0, void 0, function () {
-            var logger, dbConnection;
-            return __generator(this, function (_a) {
-                switch (_a.label) {
-                    case 0:
-                        logger = parentLogger.child('process');
-                        logger.info("Starting processing for GitHub repo: ".concat(config.repo));
-                        return [4 /*yield*/, this.initDatabase(config, logger)];
-                    case 1:
-                        dbConnection = _a.sent();
-                        // Initialize metadata storage
-                        return [4 /*yield*/, this.initDatabaseMetadata(dbConnection)];
-                    case 2:
-                        // Initialize metadata storage
-                        _a.sent();
-                        logger.section('GITHUB ISSUES');
-                        // Process GitHub issues
-                        return [4 /*yield*/, this.fetchAndProcessGitHubIssues(config.repo, config, dbConnection, logger)];
-                    case 3:
-                        // Process GitHub issues
-                        _a.sent();
-                        logger.info("Finished processing GitHub repo: ".concat(config.repo));
-                        return [2 /*return*/];
-                }
-            });
-        });
-    };
-    Doc2Vec.prototype.processWebsite = function (config, parentLogger) {
-        return __awaiter(this, void 0, void 0, function () {
-            var logger, dbConnection, validChunkIds, visitedUrls, urlPrefix;
-            var _this = this;
-            return __generator(this, function (_a) {
-                switch (_a.label) {
-                    case 0:
-                        logger = parentLogger.child('process');
-                        logger.info("Starting processing for website: ".concat(config.url));
-                        return [4 /*yield*/, this.initDatabase(config, logger)];
-                    case 1:
-                        dbConnection = _a.sent();
-                        validChunkIds = new Set();
-                        visitedUrls = new Set();
-                        urlPrefix = this.getUrlPrefix(config.url);
-                        logger.section('CRAWL AND EMBEDDING');
-                        return [4 /*yield*/, this.crawlWebsite(config.url, config, function (url, content) { return __awaiter(_this, void 0, void 0, function () {
-                                var chunks, chunkProgress, i, chunk, chunkId, needsEmbedding, chunkHash, checkHashStmt, existing, pointId, existingPoints, error_6, embeddings, embedding, error_7;
-                                return __generator(this, function (_a) {
-                                    switch (_a.label) {
-                                        case 0:
-                                            visitedUrls.add(url);
-                                            logger.info("Processing content from ".concat(url, " (").concat(content.length, " chars markdown)"));
-                                            _a.label = 1;
-                                        case 1:
-                                            _a.trys.push([1, 17, , 18]);
-                                            return [4 /*yield*/, this.chunkMarkdown(content, config, url)];
-                                        case 2:
-                                            chunks = _a.sent();
-                                            logger.info("Created ".concat(chunks.length, " chunks"));
-                                            if (!(chunks.length > 0)) return [3 /*break*/, 16];
-                                            chunkProgress = logger.progress("Embedding chunks for ".concat(url), chunks.length);
-                                            i = 0;
-                                            _a.label = 3;
-                                        case 3:
-                                            if (!(i < chunks.length)) return [3 /*break*/, 15];
-                                            chunk = chunks[i];
-                                            validChunkIds.add(chunk.metadata.chunk_id);
-                                            chunkId = chunk.metadata.chunk_id.substring(0, 8) + '...';
-                                            needsEmbedding = true;
-                                            chunkHash = this.generateHash(chunk.content);
-                                            if (!(dbConnection.type === 'sqlite')) return [3 /*break*/, 4];
-                                            checkHashStmt = this.prepareSQLiteStatements(dbConnection.db).checkHashStmt;
-                                            existing = checkHashStmt.get(chunk.metadata.chunk_id);
-                                            if (existing && existing.hash === chunkHash) {
-                                                needsEmbedding = false;
-                                                chunkProgress.update(1, "Skipping unchanged chunk ".concat(chunkId));
-                                                logger.info("Skipping unchanged chunk: ".concat(chunkId));
-                                            }
-                                            return [3 /*break*/, 8];
-                                        case 4:
-                                            if (!(dbConnection.type === 'qdrant')) return [3 /*break*/, 8];
-                                            _a.label = 5;
-                                        case 5:
-                                            _a.trys.push([5, 7, , 8]);
-                                            pointId = void 0;
-                                            try {
-                                                pointId = chunk.metadata.chunk_id;
-                                                if (!this.isValidUuid(pointId)) {
-                                                    pointId = this.hashToUuid(chunk.metadata.chunk_id);
-                                                }
-                                            }
-                                            catch (e) {
-                                                pointId = crypto_1.default.randomUUID();
-                                            }
-                                            return [4 /*yield*/, dbConnection.client.retrieve(dbConnection.collectionName, {
-                                                    ids: [pointId],
-                                                    with_payload: true,
-                                                    with_vector: false,
-                                                })];
-                                        case 6:
-                                            existingPoints = _a.sent();
-                                            if (existingPoints.length > 0 && existingPoints[0].payload && existingPoints[0].payload.hash === chunkHash) {
-                                                needsEmbedding = false;
-                                                chunkProgress.update(1, "Skipping unchanged chunk ".concat(chunkId));
-                                                logger.info("Skipping unchanged chunk: ".concat(chunkId));
-                                            }
-                                            return [3 /*break*/, 8];
-                                        case 7:
-                                            error_6 = _a.sent();
-                                            logger.error("Error checking existing point in Qdrant:", error_6);
-                                            return [3 /*break*/, 8];
-                                        case 8:
-                                            if (!needsEmbedding) return [3 /*break*/, 14];
-                                            return [4 /*yield*/, this.createEmbeddings([chunk.content])];
-                                        case 9:
-                                            embeddings = _a.sent();
-                                            if (!(embeddings.length > 0)) return [3 /*break*/, 13];
-                                            embedding = embeddings[0];
-                                            if (!(dbConnection.type === 'sqlite')) return [3 /*break*/, 10];
-                                            this.insertVectorsSQLite(dbConnection.db, chunk, embedding, logger, chunkHash);
-                                            chunkProgress.update(1, "Stored chunk ".concat(chunkId, " in SQLite"));
-                                            return [3 /*break*/, 12];
-                                        case 10:
-                                            if (!(dbConnection.type === 'qdrant')) return [3 /*break*/, 12];
-                                            return [4 /*yield*/, this.storeChunkInQdrant(dbConnection, chunk, embedding, chunkHash)];
-                                        case 11:
-                                            _a.sent();
-                                            chunkProgress.update(1, "Stored chunk ".concat(chunkId, " in Qdrant (").concat(dbConnection.collectionName, ")"));
-                                            _a.label = 12;
-                                        case 12: return [3 /*break*/, 14];
-                                        case 13:
-                                            logger.error("Embedding failed for chunk: ".concat(chunkId));
-                                            chunkProgress.update(1, "Failed to embed chunk ".concat(chunkId));
-                                            _a.label = 14;
-                                        case 14:
-                                            i++;
-                                            return [3 /*break*/, 3];
-                                        case 15:
-                                            chunkProgress.complete();
-                                            _a.label = 16;
-                                        case 16: return [3 /*break*/, 18];
-                                        case 17:
-                                            error_7 = _a.sent();
-                                            logger.error("Error during chunking or embedding for ".concat(url, ":"), error_7);
-                                            return [3 /*break*/, 18];
-                                        case 18: return [2 /*return*/];
-                                    }
-                                });
-                            }); }, logger, visitedUrls)];
-                    case 2:
-                        _a.sent();
-                        logger.info("Found ".concat(validChunkIds.size, " valid chunks across processed pages for ").concat(config.url));
-                        logger.section('CLEANUP');
-                        if (!(dbConnection.type === 'sqlite')) return [3 /*break*/, 3];
-                        logger.info("Running SQLite cleanup for ".concat(urlPrefix));
-                        this.removeObsoleteChunksSQLite(dbConnection.db, visitedUrls, urlPrefix, logger);
-                        return [3 /*break*/, 5];
-                    case 3:
-                        if (!(dbConnection.type === 'qdrant')) return [3 /*break*/, 5];
-                        logger.info("Running Qdrant cleanup for ".concat(urlPrefix, " in collection ").concat(dbConnection.collectionName));
-                        return [4 /*yield*/, this.removeObsoleteChunksQdrant(dbConnection, visitedUrls, urlPrefix, logger)];
-                    case 4:
-                        _a.sent();
-                        _a.label = 5;
-                    case 5:
-                        logger.info("Finished processing website: ".concat(config.url));
-                        return [2 /*return*/];
-                }
-            });
-        });
-    };
-    Doc2Vec.prototype.processLocalDirectory = function (config, parentLogger) {
-        return __awaiter(this, void 0, void 0, function () {
-            var logger, dbConnection, validChunkIds, processedFiles;
-            var _this = this;
-            return __generator(this, function (_a) {
-                switch (_a.label) {
-                    case 0:
-                        logger = parentLogger.child('process');
-                        logger.info("Starting processing for local directory: ".concat(config.path));
-                        return [4 /*yield*/, this.initDatabase(config, logger)];
-                    case 1:
-                        dbConnection = _a.sent();
-                        validChunkIds = new Set();
-                        processedFiles = new Set();
-                        logger.section('FILE SCANNING AND EMBEDDING');
-                        return [4 /*yield*/, this.processDirectory(config.path, config, function (filePath, content) { return __awaiter(_this, void 0, void 0, function () {
-                                var fileUrl, chunks, chunkProgress, i, chunk, chunkId, needsEmbedding, chunkHash, checkHashStmt, existing, pointId, existingPoints, error_8, embeddings, embedding, error_9;
-                                return __generator(this, function (_a) {
-                                    switch (_a.label) {
-                                        case 0:
-                                            processedFiles.add(filePath);
-                                            logger.info("Processing content from ".concat(filePath, " (").concat(content.length, " chars)"));
-                                            _a.label = 1;
-                                        case 1:
-                                            _a.trys.push([1, 17, , 18]);
-                                            fileUrl = "file://".concat(filePath);
-                                            return [4 /*yield*/, this.chunkMarkdown(content, config, fileUrl)];
-                                        case 2:
-                                            chunks = _a.sent();
-                                            logger.info("Created ".concat(chunks.length, " chunks"));
-                                            if (!(chunks.length > 0)) return [3 /*break*/, 16];
-                                            chunkProgress = logger.progress("Embedding chunks for ".concat(filePath), chunks.length);
-                                            i = 0;
-                                            _a.label = 3;
-                                        case 3:
-                                            if (!(i < chunks.length)) return [3 /*break*/, 15];
-                                            chunk = chunks[i];
-                                            validChunkIds.add(chunk.metadata.chunk_id);
-                                            chunkId = chunk.metadata.chunk_id.substring(0, 8) + '...';
-                                            needsEmbedding = true;
-                                            chunkHash = this.generateHash(chunk.content);
-                                            if (!(dbConnection.type === 'sqlite')) return [3 /*break*/, 4];
-                                            checkHashStmt = this.prepareSQLiteStatements(dbConnection.db).checkHashStmt;
-                                            existing = checkHashStmt.get(chunk.metadata.chunk_id);
-                                            if (existing && existing.hash === chunkHash) {
-                                                needsEmbedding = false;
-                                                chunkProgress.update(1, "Skipping unchanged chunk ".concat(chunkId));
-                                                logger.info("Skipping unchanged chunk: ".concat(chunkId));
-                                            }
-                                            return [3 /*break*/, 8];
-                                        case 4:
-                                            if (!(dbConnection.type === 'qdrant')) return [3 /*break*/, 8];
-                                            _a.label = 5;
-                                        case 5:
-                                            _a.trys.push([5, 7, , 8]);
-                                            pointId = void 0;
-                                            try {
-                                                pointId = chunk.metadata.chunk_id;
-                                                if (!this.isValidUuid(pointId)) {
-                                                    pointId = this.hashToUuid(chunk.metadata.chunk_id);
-                                                }
-                                            }
-                                            catch (e) {
-                                                pointId = crypto_1.default.randomUUID();
-                                            }
-                                            return [4 /*yield*/, dbConnection.client.retrieve(dbConnection.collectionName, {
-                                                    ids: [pointId],
-                                                    with_payload: true,
-                                                    with_vector: false,
-                                                })];
-                                        case 6:
-                                            existingPoints = _a.sent();
-                                            if (existingPoints.length > 0 && existingPoints[0].payload && existingPoints[0].payload.hash === chunkHash) {
-                                                needsEmbedding = false;
-                                                chunkProgress.update(1, "Skipping unchanged chunk ".concat(chunkId));
-                                                logger.info("Skipping unchanged chunk: ".concat(chunkId));
-                                            }
-                                            return [3 /*break*/, 8];
-                                        case 7:
-                                            error_8 = _a.sent();
-                                            logger.error("Error checking existing point in Qdrant:", error_8);
-                                            return [3 /*break*/, 8];
-                                        case 8:
-                                            if (!needsEmbedding) return [3 /*break*/, 14];
-                                            return [4 /*yield*/, this.createEmbeddings([chunk.content])];
-                                        case 9:
-                                            embeddings = _a.sent();
-                                            if (!(embeddings.length > 0)) return [3 /*break*/, 13];
-                                            embedding = embeddings[0];
-                                            if (!(dbConnection.type === 'sqlite')) return [3 /*break*/, 10];
-                                            this.insertVectorsSQLite(dbConnection.db, chunk, embedding, logger, chunkHash);
-                                            chunkProgress.update(1, "Stored chunk ".concat(chunkId, " in SQLite"));
-                                            return [3 /*break*/, 12];
-                                        case 10:
-                                            if (!(dbConnection.type === 'qdrant')) return [3 /*break*/, 12];
-                                            return [4 /*yield*/, this.storeChunkInQdrant(dbConnection, chunk, embedding, chunkHash)];
-                                        case 11:
-                                            _a.sent();
-                                            chunkProgress.update(1, "Stored chunk ".concat(chunkId, " in Qdrant (").concat(dbConnection.collectionName, ")"));
-                                            _a.label = 12;
-                                        case 12: return [3 /*break*/, 14];
-                                        case 13:
-                                            logger.error("Embedding failed for chunk: ".concat(chunkId));
-                                            chunkProgress.update(1, "Failed to embed chunk ".concat(chunkId));
-                                            _a.label = 14;
-                                        case 14:
-                                            i++;
-                                            return [3 /*break*/, 3];
-                                        case 15:
-                                            chunkProgress.complete();
-                                            _a.label = 16;
-                                        case 16: return [3 /*break*/, 18];
-                                        case 17:
-                                            error_9 = _a.sent();
-                                            logger.error("Error during chunking or embedding for ".concat(filePath, ":"), error_9);
-                                            return [3 /*break*/, 18];
-                                        case 18: return [2 /*return*/];
-                                    }
-                                });
-                            }); }, logger)];
-                    case 2:
-                        _a.sent();
-                        logger.info("Found ".concat(validChunkIds.size, " valid chunks across processed files for ").concat(config.path));
-                        logger.section('CLEANUP');
-                        if (!(dbConnection.type === 'sqlite')) return [3 /*break*/, 3];
-                        logger.info("Running SQLite cleanup for local directory ".concat(config.path));
-                        this.removeObsoleteFilesSQLite(dbConnection.db, processedFiles, config.path, logger);
-                        return [3 /*break*/, 5];
-                    case 3:
-                        if (!(dbConnection.type === 'qdrant')) return [3 /*break*/, 5];
-                        logger.info("Running Qdrant cleanup for local directory ".concat(config.path, " in collection ").concat(dbConnection.collectionName));
-                        return [4 /*yield*/, this.removeObsoleteFilesQdrant(dbConnection, processedFiles, config.path, logger)];
-                    case 4:
-                        _a.sent();
-                        _a.label = 5;
-                    case 5:
-                        logger.info("Finished processing local directory: ".concat(config.path));
-                        return [2 /*return*/];
-                }
-            });
-        });
-    };
-    Doc2Vec.prototype.processDirectory = function (dirPath_1, config_1, processFileContent_1, parentLogger_1) {
-        return __awaiter(this, arguments, void 0, function (dirPath, config, processFileContent, parentLogger, visitedPaths) {
-            var logger, recursive, includeExtensions, excludeExtensions, encoding, files, processedFiles, skippedFiles, _i, files_1, file, filePath, stat, extension, content, processedContent, cleanHtml, error_10, error_11;
-            if (visitedPaths === void 0) { visitedPaths = new Set(); }
-            return __generator(this, function (_a) {
-                switch (_a.label) {
-                    case 0:
-                        logger = parentLogger.child('directory-processor');
-                        logger.info("Processing directory: ".concat(dirPath));
-                        recursive = config.recursive !== undefined ? config.recursive : true;
-                        includeExtensions = config.include_extensions || ['.md', '.txt', '.html', '.htm'];
-                        excludeExtensions = config.exclude_extensions || [];
-                        encoding = config.encoding || 'utf8';
-                        _a.label = 1;
-                    case 1:
-                        _a.trys.push([1, 12, , 13]);
-                        files = fs.readdirSync(dirPath);
-                        processedFiles = 0;
-                        skippedFiles = 0;
-                        _i = 0, files_1 = files;
-                        _a.label = 2;
-                    case 2:
-                        if (!(_i < files_1.length)) return [3 /*break*/, 11];
-                        file = files_1[_i];
-                        filePath = path.join(dirPath, file);
-                        stat = fs.statSync(filePath);
-                        // Skip already visited paths
-                        if (visitedPaths.has(filePath)) {
-                            logger.debug("Skipping already visited path: ".concat(filePath));
-                            return [3 /*break*/, 10];
-                        }
-                        visitedPaths.add(filePath);
-                        if (!stat.isDirectory()) return [3 /*break*/, 6];
-                        if (!recursive) return [3 /*break*/, 4];
-                        return [4 /*yield*/, this.processDirectory(filePath, config, processFileContent, logger, visitedPaths)];
-                    case 3:
-                        _a.sent();
-                        return [3 /*break*/, 5];
-                    case 4:
-                        logger.debug("Skipping directory ".concat(filePath, " (recursive=false)"));
-                        _a.label = 5;
-                    case 5: return [3 /*break*/, 10];
-                    case 6:
-                        if (!stat.isFile()) return [3 /*break*/, 10];
-                        extension = path.extname(file).toLowerCase();
-                        // Apply extension filters
-                        if (excludeExtensions.includes(extension)) {
-                            logger.debug("Skipping file with excluded extension: ".concat(filePath));
-                            skippedFiles++;
-                            return [3 /*break*/, 10];
-                        }
-                        if (includeExtensions.length > 0 && !includeExtensions.includes(extension)) {
-                            logger.debug("Skipping file with non-included extension: ".concat(filePath));
-                            skippedFiles++;
-                            return [3 /*break*/, 10];
-                        }
-                        _a.label = 7;
-                    case 7:
-                        _a.trys.push([7, 9, , 10]);
-                        logger.info("Reading file: ".concat(filePath));
-                        content = fs.readFileSync(filePath, { encoding: encoding });
-                        if (content.length > config.max_size) {
-                            logger.warn("File content (".concat(content.length, " chars) exceeds max size (").concat(config.max_size, "). Skipping ").concat(filePath, "."));
-                            skippedFiles++;
-                            return [3 /*break*/, 10];
-                        }
-                        processedContent = void 0;
-                        if (extension === '.html' || extension === '.htm') {
-                            logger.debug("Converting HTML to Markdown for ".concat(filePath));
-                            cleanHtml = (0, sanitize_html_1.default)(content, {
-                                allowedTags: [
-                                    'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'a', 'ul', 'ol',
-                                    'li', 'b', 'i', 'strong', 'em', 'code', 'pre',
-                                    'div', 'span', 'table', 'thead', 'tbody', 'tr', 'th', 'td'
-                                ],
-                                allowedAttributes: {
-                                    'a': ['href'],
-                                    'pre': ['class', 'data-language'],
-                                    'code': ['class', 'data-language'],
-                                    'div': ['class'],
-                                    'span': ['class']
-                                }
-                            });
-                            processedContent = this.turndownService.turndown(cleanHtml);
-                        }
-                        else {
-                            processedContent = content;
-                        }
-                        return [4 /*yield*/, processFileContent(filePath, processedContent)];
-                    case 8:
-                        _a.sent();
-                        processedFiles++;
-                        return [3 /*break*/, 10];
-                    case 9:
-                        error_10 = _a.sent();
-                        logger.error("Error processing file ".concat(filePath, ":"), error_10);
-                        return [3 /*break*/, 10];
-                    case 10:
-                        _i++;
-                        return [3 /*break*/, 2];
-                    case 11:
-                        logger.info("Directory processed. Processed: ".concat(processedFiles, ", Skipped: ").concat(skippedFiles));
-                        return [3 /*break*/, 13];
-                    case 12:
-                        error_11 = _a.sent();
-                        logger.error("Error reading directory ".concat(dirPath, ":"), error_11);
-                        return [3 /*break*/, 13];
-                    case 13: return [2 /*return*/];
-                }
-            });
-        });
-    };
-    Doc2Vec.prototype.initDatabase = function (config, parentLogger) {
-        return __awaiter(this, void 0, void 0, function () {
-            var logger, dbConfig, params, dbPath, db, params, qdrantUrl, qdrantPort, collectionName, qdrantClient, errMsg;
-            return __generator(this, function (_a) {
-                switch (_a.label) {
-                    case 0:
-                        logger = parentLogger.child('database');
-                        dbConfig = config.database_config;
-                        if (!(dbConfig.type === 'sqlite')) return [3 /*break*/, 1];
-                        params = dbConfig.params;
-                        dbPath = params.db_path || path.join(process.cwd(), "".concat(config.product_name.replace(/\s+/g, '_'), "-").concat(config.version, ".db"));
-                        logger.info("Opening SQLite database at ".concat(dbPath));
-                        db = new better_sqlite3_1.default(dbPath, { allowExtension: true });
-                        sqliteVec.load(db);
-                        logger.debug("Creating vec_items table if it doesn't exist");
-                        db.exec("\n                CREATE VIRTUAL TABLE IF NOT EXISTS vec_items USING vec0(\n                    embedding FLOAT[3072],\n                    product_name TEXT,\n                    version TEXT,\n                    heading_hierarchy TEXT,\n                    section TEXT,\n                    chunk_id TEXT UNIQUE,\n                    content TEXT,\n                    url TEXT,\n                    hash TEXT\n                );\n            ");
-                        logger.info("SQLite database initialized successfully");
-                        return [2 /*return*/, { db: db, type: 'sqlite' }];
-                    case 1:
-                        if (!(dbConfig.type === 'qdrant')) return [3 /*break*/, 3];
-                        params = dbConfig.params;
-                        qdrantUrl = params.qdrant_url || 'http://localhost:6333';
-                        qdrantPort = params.qdrant_port || 443;
-                        collectionName = params.collection_name || "".concat(config.product_name.toLowerCase().replace(/\s+/g, '_'), "_").concat(config.version);
-                        logger.info("Connecting to Qdrant at ".concat(qdrantUrl, ":").concat(qdrantPort, ", collection: ").concat(collectionName));
-                        qdrantClient = new js_client_rest_1.QdrantClient({ url: qdrantUrl, apiKey: process.env.QDRANT_API_KEY, port: qdrantPort });
-                        return [4 /*yield*/, this.createCollectionQdrant(qdrantClient, collectionName, logger)];
-                    case 2:
-                        _a.sent();
-                        logger.info("Qdrant connection established successfully");
-                        return [2 /*return*/, { client: qdrantClient, collectionName: collectionName, type: 'qdrant' }];
-                    case 3:
-                        errMsg = "Unsupported database type: ".concat(dbConfig.type);
-                        logger.error(errMsg);
-                        throw new Error(errMsg);
-                }
-            });
-        });
-    };
-    Doc2Vec.prototype.createCollectionQdrant = function (qdrantClient, collectionName, logger) {
-        return __awaiter(this, void 0, void 0, function () {
-            var collections, collectionExists, error_12, errorMsg, errorString;
-            return __generator(this, function (_a) {
-                switch (_a.label) {
-                    case 0:
-                        _a.trys.push([0, 3, , 4]);
-                        logger.debug("Checking if collection ".concat(collectionName, " exists"));
-                        return [4 /*yield*/, qdrantClient.getCollections()];
-                    case 1:
-                        collections = _a.sent();
-                        collectionExists = collections.collections.some(function (collection) { return collection.name === collectionName; });
-                        if (collectionExists) {
-                            logger.info("Collection ".concat(collectionName, " already exists"));
-                            return [2 /*return*/];
-                        }
-                        logger.info("Creating new collection ".concat(collectionName));
-                        return [4 /*yield*/, qdrantClient.createCollection(collectionName, {
-                                vectors: {
-                                    size: 3072,
-                                    distance: "Cosine",
-                                },
-                            })];
-                    case 2:
-                        _a.sent();
-                        logger.info("Collection ".concat(collectionName, " created successfully"));
-                        return [3 /*break*/, 4];
-                    case 3:
-                        error_12 = _a.sent();
-                        if (error_12 instanceof Error) {
-                            errorMsg = error_12.message.toLowerCase();
-                            errorString = JSON.stringify(error_12).toLowerCase();
-                            if (errorMsg.includes("already exists") ||
-                                errorString.includes("already exists") ||
-                                (error_12 === null || error_12 === void 0 ? void 0 : error_12.status) === 409 ||
-                                errorString.includes("conflict")) {
-                                logger.info("Collection ".concat(collectionName, " already exists (from error response)"));
-                                return [2 /*return*/];
-                            }
-                        }
-                        logger.error("Error creating Qdrant collection:", error_12);
-                        logger.warn("Continuing with existing collection...");
-                        return [3 /*break*/, 4];
-                    case 4: return [2 /*return*/];
-                }
-            });
-        });
-    };
-    Doc2Vec.prototype.prepareSQLiteStatements = function (db) {
-        return {
-            insertStmt: db.prepare("\n            INSERT INTO vec_items (embedding, product_name, version, heading_hierarchy, section, chunk_id, content, url, hash)\n            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)\n        "),
-            checkHashStmt: db.prepare("SELECT hash FROM vec_items WHERE chunk_id = ?"),
-            updateStmt: db.prepare("\n            UPDATE vec_items SET embedding = ?, product_name = ?, version = ?, heading_hierarchy = ?, section = ?, content = ?, url = ?, hash = ?\n            WHERE chunk_id = ?\n        "),
-            getAllChunkIdsStmt: db.prepare("SELECT chunk_id FROM vec_items"),
-            deleteChunkStmt: db.prepare("DELETE FROM vec_items WHERE chunk_id = ?")
-        };
-    };
-    Doc2Vec.prototype.insertVectorsSQLite = function (db, chunk, embedding, logger, chunkHash) {
-        var _a = this.prepareSQLiteStatements(db), insertStmt = _a.insertStmt, updateStmt = _a.updateStmt;
-        var hash = chunkHash || this.generateHash(chunk.content);
-        var transaction = db.transaction(function () {
-            var params = [
-                new Float32Array(embedding),
-                chunk.metadata.product_name,
-                chunk.metadata.version,
-                JSON.stringify(chunk.metadata.heading_hierarchy),
-                chunk.metadata.section,
-                chunk.metadata.chunk_id,
-                chunk.content,
-                chunk.metadata.url,
-                hash
-            ];
-            try {
-                insertStmt.run(params);
-            }
-            catch (error) {
-                updateStmt.run(__spreadArray(__spreadArray([], params.slice(0, 8), true), [chunk.metadata.chunk_id], false));
-            }
-        });
-        transaction();
-    };
-    Doc2Vec.prototype.storeChunkInQdrant = function (db, chunk, embedding, chunkHash) {
-        return __awaiter(this, void 0, void 0, function () {
-            var client, collectionName, pointId, hash, pointItem, error_13;
-            return __generator(this, function (_a) {
-                switch (_a.label) {
-                    case 0:
-                        client = db.client, collectionName = db.collectionName;
-                        _a.label = 1;
-                    case 1:
-                        _a.trys.push([1, 3, , 4]);
-                        pointId = void 0;
-                        try {
-                            pointId = chunk.metadata.chunk_id;
-                            if (!this.isValidUuid(pointId)) {
-                                pointId = this.hashToUuid(chunk.metadata.chunk_id);
-                            }
-                        }
-                        catch (e) {
-                            pointId = crypto_1.default.randomUUID();
-                        }
-                        hash = chunkHash || this.generateHash(chunk.content);
-                        pointItem = {
-                            id: pointId,
-                            vector: embedding,
-                            payload: {
-                                content: chunk.content,
-                                product_name: chunk.metadata.product_name,
-                                version: chunk.metadata.version,
-                                heading_hierarchy: chunk.metadata.heading_hierarchy,
-                                section: chunk.metadata.section,
-                                url: chunk.metadata.url,
-                                hash: hash,
-                                original_chunk_id: chunk.metadata.chunk_id,
-                            },
-                        };
-                        return [4 /*yield*/, client.upsert(collectionName, {
-                                wait: true,
-                                points: [pointItem],
-                            })];
-                    case 2:
-                        _a.sent();
-                        return [3 /*break*/, 4];
-                    case 3:
-                        error_13 = _a.sent();
-                        this.logger.error("Error storing chunk in Qdrant:", error_13);
-                        return [3 /*break*/, 4];
-                    case 4: return [2 /*return*/];
-                }
-            });
-        });
-    };
-    Doc2Vec.prototype.removeObsoleteChunksSQLite = function (db, visitedUrls, urlPrefix, logger) {
-        var getChunksForUrlStmt = db.prepare("\n            SELECT chunk_id, url FROM vec_items\n            WHERE url LIKE ? || '%'\n        ");
-        var deleteChunkStmt = db.prepare("DELETE FROM vec_items WHERE chunk_id = ?");
-        var existingChunks = getChunksForUrlStmt.all(urlPrefix);
-        var deletedCount = 0;
-        var transaction = db.transaction(function () {
-            for (var _i = 0, existingChunks_1 = existingChunks; _i < existingChunks_1.length; _i++) {
-                var _a = existingChunks_1[_i], chunk_id = _a.chunk_id, url = _a.url;
-                if (!visitedUrls.has(url)) {
-                    logger.debug("Deleting obsolete chunk from SQLite: ".concat(chunk_id.substring(0, 8), "... (URL not visited)"));
-                    deleteChunkStmt.run(chunk_id);
-                    deletedCount++;
-                }
-            }
-        });
-        transaction();
-        logger.info("Deleted ".concat(deletedCount, " obsolete chunks from SQLite for URL ").concat(urlPrefix));
-    };
-    Doc2Vec.prototype.removeObsoleteFilesSQLite = function (db, processedFiles, dirPrefix, logger) {
-        var getChunksForDirStmt = db.prepare("\n            SELECT chunk_id, url FROM vec_items\n            WHERE url LIKE 'file://%' AND url LIKE ?\n        ");
-        var deleteChunkStmt = db.prepare("DELETE FROM vec_items WHERE chunk_id = ?");
-        var cleanedDirPrefix = dirPrefix.replace(/^\.\/+/, '');
-        var filePrefix = "file://".concat(cleanedDirPrefix);
-        var existingChunks = getChunksForDirStmt.all("".concat(filePrefix, "%"));
-        var deletedCount = 0;
-        var transaction = db.transaction(function () {
-            for (var _i = 0, existingChunks_2 = existingChunks; _i < existingChunks_2.length; _i++) {
-                var _a = existingChunks_2[_i], chunk_id = _a.chunk_id, url = _a.url;
-                // Remove file:// prefix to match with processedFiles
-                var filePath = url.substring(7);
-                if (!processedFiles.has(filePath)) {
-                    logger.debug("Deleting obsolete chunk from SQLite: ".concat(chunk_id.substring(0, 8), "... (File not processed)"));
-                    deleteChunkStmt.run(chunk_id);
-                    deletedCount++;
-                }
-            }
-        });
-        transaction();
-        logger.info("Deleted ".concat(deletedCount, " obsolete chunks from SQLite for directory ").concat(dirPrefix));
-    };
-    Doc2Vec.prototype.removeObsoleteFilesQdrant = function (db, processedFiles, dirPrefix, logger) {
-        return __awaiter(this, void 0, void 0, function () {
-            var client, collectionName, cleanedDirPrefix, filePrefix, response, obsoletePointIds, error_14;
-            return __generator(this, function (_a) {
-                switch (_a.label) {
-                    case 0:
-                        client = db.client, collectionName = db.collectionName;
-                        _a.label = 1;
-                    case 1:
-                        _a.trys.push([1, 6, , 7]);
-                        cleanedDirPrefix = dirPrefix.replace(/^\.\/+/, '');
-                        filePrefix = "file://".concat(cleanedDirPrefix);
-                        return [4 /*yield*/, client.scroll(collectionName, {
-                                limit: 10000,
-                                with_payload: true,
-                                with_vector: false,
-                                filter: {
-                                    must: [
-                                        {
-                                            key: "url",
-                                            match: {
-                                                text: "".concat(filePrefix)
-                                            }
-                                        }
-                                    ],
-                                    must_not: [
-                                        {
-                                            key: "is_metadata",
-                                            match: {
-                                                value: true
-                                            }
-                                        }
-                                    ]
-                                }
-                            })];
-                    case 2:
-                        response = _a.sent();
-                        obsoletePointIds = response.points
-                            .filter(function (point) {
-                            var _a, _b;
-                            var url = (_a = point.payload) === null || _a === void 0 ? void 0 : _a.url;
-                            // Double check it's not a metadata record
-                            if (((_b = point.payload) === null || _b === void 0 ? void 0 : _b.is_metadata) === true) {
-                                return false;
-                            }
-                            // Remove file:// prefix to match with processedFiles
-                            var filePath = url && url.startsWith('file://') ? url.substring(7) : '';
-                            return filePath && !processedFiles.has(filePath);
-                        })
-                            .map(function (point) { return point.id; });
-                        if (!(obsoletePointIds.length > 0)) return [3 /*break*/, 4];
-                        return [4 /*yield*/, client.delete(collectionName, {
-                                points: obsoletePointIds,
-                            })];
-                    case 3:
-                        _a.sent();
-                        logger.info("Deleted ".concat(obsoletePointIds.length, " obsolete chunks from Qdrant for directory ").concat(dirPrefix));
-                        return [3 /*break*/, 5];
-                    case 4:
-                        logger.info("No obsolete chunks to delete from Qdrant for directory ".concat(dirPrefix));
-                        _a.label = 5;
-                    case 5: return [3 /*break*/, 7];
-                    case 6:
-                        error_14 = _a.sent();
-                        logger.error("Error removing obsolete chunks from Qdrant:", error_14);
-                        return [3 /*break*/, 7];
-                    case 7: return [2 /*return*/];
-                }
-            });
-        });
-    };
-    Doc2Vec.prototype.isValidUuid = function (str) {
-        var uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
-        return uuidRegex.test(str);
-    };
-    Doc2Vec.prototype.hashToUuid = function (hash) {
-        var truncatedHash = hash.substring(0, 32);
-        return [
-            truncatedHash.substring(0, 8),
-            truncatedHash.substring(8, 12),
-            '5' + truncatedHash.substring(13, 16),
-            '8' + truncatedHash.substring(17, 20),
-            truncatedHash.substring(20, 32)
-        ].join('-');
-    };
-    Doc2Vec.prototype.removeObsoleteChunksQdrant = function (db, visitedUrls, urlPrefix, logger) {
-        return __awaiter(this, void 0, void 0, function () {
-            var client, collectionName, response, obsoletePointIds, error_15;
-            return __generator(this, function (_a) {
-                switch (_a.label) {
-                    case 0:
-                        client = db.client, collectionName = db.collectionName;
-                        _a.label = 1;
-                    case 1:
-                        _a.trys.push([1, 6, , 7]);
-                        return [4 /*yield*/, client.scroll(collectionName, {
-                                limit: 10000,
-                                with_payload: true,
-                                with_vector: false,
-                                filter: {
-                                    must: [
-                                        {
-                                            key: "url",
-                                            match: {
-                                                text: urlPrefix
-                                            }
-                                        }
-                                    ],
-                                    must_not: [
-                                        {
-                                            key: "is_metadata",
-                                            match: {
-                                                value: true
-                                            }
-                                        }
-                                    ]
-                                }
-                            })];
-                    case 2:
-                        response = _a.sent();
-                        obsoletePointIds = response.points
-                            .filter(function (point) {
-                            var _a, _b;
-                            var url = (_a = point.payload) === null || _a === void 0 ? void 0 : _a.url;
-                            // Double check it's not a metadata record
-                            if (((_b = point.payload) === null || _b === void 0 ? void 0 : _b.is_metadata) === true) {
-                                return false;
-                            }
-                            return url && !visitedUrls.has(url);
-                        })
-                            .map(function (point) { return point.id; });
-                        if (!(obsoletePointIds.length > 0)) return [3 /*break*/, 4];
-                        return [4 /*yield*/, client.delete(collectionName, {
-                                points: obsoletePointIds,
-                            })];
-                    case 3:
-                        _a.sent();
-                        logger.info("Deleted ".concat(obsoletePointIds.length, " obsolete chunks from Qdrant for URL ").concat(urlPrefix));
-                        return [3 /*break*/, 5];
-                    case 4:
-                        logger.info("No obsolete chunks to delete from Qdrant for URL ".concat(urlPrefix));
-                        _a.label = 5;
-                    case 5: return [3 /*break*/, 7];
-                    case 6:
-                        error_15 = _a.sent();
-                        logger.error("Error removing obsolete chunks from Qdrant:", error_15);
-                        return [3 /*break*/, 7];
-                    case 7: return [2 /*return*/];
-                }
-            });
-        });
-    };
-    Doc2Vec.prototype.crawlWebsite = function (baseUrl, sourceConfig, processPageContent, parentLogger, visitedUrls) {
-        return __awaiter(this, void 0, void 0, function () {
-            var logger, queue, processedCount, skippedCount, skippedSizeCount, errorCount, _loop_2, this_1;
-            var _this = this;
-            return __generator(this, function (_a) {
-                switch (_a.label) {
-                    case 0:
-                        logger = parentLogger.child('crawler');
-                        queue = [baseUrl];
-                        logger.info("Starting crawl from ".concat(baseUrl));
-                        processedCount = 0;
-                        skippedCount = 0;
-                        skippedSizeCount = 0;
-                        errorCount = 0;
-                        _loop_2 = function () {
-                            var url, normalizedUrl, content, response, $_1, newLinksFound_1, error_16;
-                            return __generator(this, function (_b) {
-                                switch (_b.label) {
-                                    case 0:
-                                        url = queue.shift();
-                                        if (!url)
-                                            return [2 /*return*/, "continue"];
-                                        normalizedUrl = this_1.normalizeUrl(url);
-                                        if (visitedUrls.has(normalizedUrl))
-                                            return [2 /*return*/, "continue"];
-                                        visitedUrls.add(normalizedUrl);
-                                        if (!this_1.shouldProcessUrl(url)) {
-                                            logger.debug("Skipping URL with unsupported extension: ".concat(url));
-                                            skippedCount++;
-                                            return [2 /*return*/, "continue"];
-                                        }
-                                        _b.label = 1;
-                                    case 1:
-                                        _b.trys.push([1, 7, , 8]);
-                                        logger.info("Crawling: ".concat(url));
-                                        return [4 /*yield*/, this_1.processPage(url, sourceConfig)];
-                                    case 2:
-                                        content = _b.sent();
-                                        if (!(content !== null)) return [3 /*break*/, 4];
-                                        return [4 /*yield*/, processPageContent(url, content)];
-                                    case 3:
-                                        _b.sent();
-                                        processedCount++;
-                                        return [3 /*break*/, 5];
-                                    case 4:
-                                        skippedSizeCount++;
-                                        _b.label = 5;
-                                    case 5: return [4 /*yield*/, axios_1.default.get(url)];
-                                    case 6:
-                                        response = _b.sent();
-                                        $_1 = (0, cheerio_1.load)(response.data);
-                                        logger.debug("Finding links on page ".concat(url));
-                                        newLinksFound_1 = 0;
-                                        $_1('a[href]').each(function (_, element) {
-                                            var href = $_1(element).attr('href');
-                                            if (!href || href.startsWith('#') || href.startsWith('mailto:'))
-                                                return;
-                                            var fullUrl = _this.buildUrl(href, url);
-                                            if (fullUrl.startsWith(sourceConfig.url) && !visitedUrls.has(_this.normalizeUrl(fullUrl))) {
-                                                if (!queue.includes(fullUrl)) {
-                                                    queue.push(fullUrl);
-                                                    newLinksFound_1++;
-                                                }
-                                            }
-                                        });
-                                        logger.debug("Found ".concat(newLinksFound_1, " new links on ").concat(url));
-                                        return [3 /*break*/, 8];
-                                    case 7:
-                                        error_16 = _b.sent();
-                                        logger.error("Failed during link discovery or initial fetch for ".concat(url, ":"), error_16);
-                                        errorCount++;
-                                        return [3 /*break*/, 8];
-                                    case 8: return [2 /*return*/];
-                                }
-                            });
-                        };
-                        this_1 = this;
-                        _a.label = 1;
-                    case 1:
-                        if (!(queue.length > 0)) return [3 /*break*/, 3];
-                        return [5 /*yield**/, _loop_2()];
-                    case 2:
-                        _a.sent();
-                        return [3 /*break*/, 1];
-                    case 3:
-                        logger.info("Crawl completed. Processed: ".concat(processedCount, ", Skipped (Extension): ").concat(skippedCount, ", Skipped (Size): ").concat(skippedSizeCount, ", Errors: ").concat(errorCount));
-                        return [2 /*return*/];
-                }
-            });
-        });
-    };
-    Doc2Vec.prototype.shouldProcessUrl = function (url) {
-        var parsedUrl = new URL(url);
-        var pathname = parsedUrl.pathname;
-        var ext = path.extname(pathname);
-        if (!ext)
-            return true;
-        return ['.html', '.htm'].includes(ext.toLowerCase());
-    };
-    Doc2Vec.prototype.normalizeUrl = function (url) {
-        try {
-            var urlObj = new URL(url);
-            urlObj.hash = '';
-            urlObj.search = '';
-            return urlObj.toString();
-        }
-        catch (error) {
-            return url;
-        }
-    };
-    Doc2Vec.prototype.buildUrl = function (href, currentUrl) {
-        try {
-            return new URL(href, currentUrl).toString();
-        }
-        catch (error) {
-            this.logger.warn("Invalid URL found: ".concat(href));
-            return '';
-        }
-    };
-    Doc2Vec.prototype.processPage = function (url, sourceConfig) {
-        return __awaiter(this, void 0, void 0, function () {
-            var logger, browser, page, htmlContent, dom, document_1, reader, article, cleanHtml, markdown, error_17;
-            var _this = this;
-            return __generator(this, function (_a) {
-                switch (_a.label) {
-                    case 0:
-                        logger = this.logger.child('page-processor');
-                        logger.debug("Processing page content from ".concat(url));
-                        return [4 /*yield*/, puppeteer_1.default.launch({
-                                args: ['--no-sandbox', '--disable-setuid-sandbox'],
-                            })];
-                    case 1:
-                        browser = _a.sent();
-                        _a.label = 2;
-                    case 2:
-                        _a.trys.push([2, 10, 11, 14]);
-                        return [4 /*yield*/, browser.newPage()];
-                    case 3:
-                        page = _a.sent();
-                        logger.debug("Navigating to ".concat(url));
-                        return [4 /*yield*/, page.goto(url, { waitUntil: 'networkidle2', timeout: 60000 })];
-                    case 4:
-                        _a.sent();
-                        return [4 /*yield*/, page.evaluate(function () {
-                                var mainContentElement = document_1.querySelector('div[role="main"].document') || document_1.querySelector('main') || document_1.body;
-                                return mainContentElement.innerHTML;
-                            })];
-                    case 5:
-                        htmlContent = _a.sent();
-                        if (!(htmlContent.length > sourceConfig.max_size)) return [3 /*break*/, 7];
-                        logger.warn("Raw HTML content (".concat(htmlContent.length, " chars) exceeds max size (").concat(sourceConfig.max_size, "). Skipping detailed processing for ").concat(url, "."));
-                        return [4 /*yield*/, browser.close()];
-                    case 6:
-                        _a.sent();
-                        return [2 /*return*/, null];
-                    case 7:
-                        logger.debug("Got HTML content (".concat(htmlContent.length, " chars), creating DOM"));
-                        dom = new jsdom_1.JSDOM(htmlContent);
-                        document_1 = dom.window.document;
-                        document_1.querySelectorAll('pre').forEach(function (pre) {
-                            pre.classList.add('article-content');
-                            pre.setAttribute('data-readable-content-score', '100');
-                            _this.markCodeParents(pre.parentElement);
-                        });
-                        logger.debug("Applying Readability to extract main content");
-                        reader = new readability_1.Readability(document_1, {
-                            charThreshold: 20,
-                            classesToPreserve: ['article-content'],
-                        });
-                        article = reader.parse();
-                        if (!!article) return [3 /*break*/, 9];
-                        logger.warn("Failed to parse article content with Readability for ".concat(url));
-                        return [4 /*yield*/, browser.close()];
-                    case 8:
-                        _a.sent();
-                        return [2 /*return*/, null];
-                    case 9:
-                        logger.debug("Sanitizing HTML (".concat(article.content.length, " chars)"));
-                        cleanHtml = (0, sanitize_html_1.default)(article.content, {
-                            allowedTags: [
-                                'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'a', 'ul', 'ol',
-                                'li', 'b', 'i', 'strong', 'em', 'code', 'pre',
-                                'div', 'span', 'table', 'thead', 'tbody', 'tr', 'th', 'td'
-                            ],
-                            allowedAttributes: {
-                                'a': ['href'],
-                                'pre': ['class', 'data-language'],
-                                'code': ['class', 'data-language'],
-                                'div': ['class'],
-                                'span': ['class']
-                            }
-                        });
-                        logger.debug("Converting HTML to Markdown");
-                        markdown = this.turndownService.turndown(cleanHtml);
-                        logger.debug("Markdown conversion complete (".concat(markdown.length, " chars)"));
-                        return [2 /*return*/, markdown];
-                    case 10:
-                        error_17 = _a.sent();
-                        logger.error("Error processing page ".concat(url, ":"), error_17);
-                        return [2 /*return*/, null];
-                    case 11:
-                        if (!(browser && browser.isConnected())) return [3 /*break*/, 13];
-                        return [4 /*yield*/, browser.close()];
-                    case 12:
-                        _a.sent();
-                        logger.debug("Browser closed for ".concat(url));
-                        _a.label = 13;
-                    case 13: return [7 /*endfinally*/];
-                    case 14: return [2 /*return*/];
-                }
-            });
-        });
-    };
-    Doc2Vec.prototype.markCodeParents = function (node) {
-        if (!node)
-            return;
-        if (node.querySelector('pre, code')) {
-            node.classList.add('article-content');
-            node.setAttribute('data-readable-content-score', '100');
-        }
-        this.markCodeParents(node.parentElement);
-    };
-    Doc2Vec.prototype.chunkMarkdown = function (markdown, sourceConfig, url) {
-        return __awaiter(this, void 0, void 0, function () {
-            var logger, MAX_TOKENS, chunks, lines, currentChunk, headingHierarchy, processChunk, createDocumentChunk, _i, lines_2, line, levelMatch, level, heading;
-            var _this = this;
-            return __generator(this, function (_a) {
-                logger = this.logger.child('chunker');
-                logger.debug("Chunking markdown from ".concat(url, " (").concat(markdown.length, " chars)"));
-                MAX_TOKENS = 1000;
-                chunks = [];
-                lines = markdown.split("\n");
-                currentChunk = "";
-                headingHierarchy = [];
-                processChunk = function () {
-                    if (currentChunk.trim()) {
-                        var tokens = _this.tokenize(currentChunk);
-                        if (tokens.length > MAX_TOKENS) {
-                            logger.debug("Chunk exceeds max token count (".concat(tokens.length, "), splitting into smaller chunks"));
-                            var subChunk = "";
-                            var tokenCount = 0;
-                            var overlapSize = Math.floor(MAX_TOKENS * 0.05);
-                            var lastTokens = [];
-                            for (var _i = 0, tokens_1 = tokens; _i < tokens_1.length; _i++) {
-                                var token = tokens_1[_i];
-                                if (tokenCount + 1 > MAX_TOKENS) {
-                                    chunks.push(createDocumentChunk(subChunk, headingHierarchy));
-                                    subChunk = lastTokens.join("") + token;
-                                    tokenCount = lastTokens.length + 1;
-                                    lastTokens = [];
-                                }
-                                else {
-                                    subChunk += token;
-                                    tokenCount++;
-                                    lastTokens.push(token);
-                                    if (lastTokens.length > overlapSize) {
-                                        lastTokens.shift();
-                                    }
-                                }
-                            }
-                            if (subChunk) {
-                                chunks.push(createDocumentChunk(subChunk, headingHierarchy));
-                            }
-                        }
-                        else {
-                            chunks.push(createDocumentChunk(currentChunk, headingHierarchy));
-                        }
-                    }
-                    currentChunk = "";
-                };
-                createDocumentChunk = function (content, hierarchy) {
-                    var chunkId = _this.generateHash(content);
-                    logger.debug("Created chunk ".concat(chunkId.substring(0, 8), "... with ").concat(content.length, " chars"));
-                    return {
-                        content: content,
-                        metadata: {
-                            product_name: sourceConfig.product_name,
-                            version: sourceConfig.version,
-                            heading_hierarchy: __spreadArray([], hierarchy, true),
-                            section: hierarchy[hierarchy.length - 1] || "Introduction",
-                            chunk_id: chunkId,
-                            url: url,
-                            hash: _this.generateHash(content)
-                        }
-                    };
-                };
-                for (_i = 0, lines_2 = lines; _i < lines_2.length; _i++) {
-                    line = lines_2[_i];
-                    if (line.startsWith("#")) {
-                        processChunk();
-                        levelMatch = line.match(/^(#+)/);
-                        level = levelMatch ? levelMatch[1].length : 1;
-                        heading = line.replace(/^#+\s*/, "").trim();
-                        logger.debug("Found heading (level ".concat(level, "): ").concat(heading));
-                        while (headingHierarchy.length < level - 1) {
-                            headingHierarchy.push("");
-                        }
-                        if (level <= headingHierarchy.length) {
-                            headingHierarchy = headingHierarchy.slice(0, level - 1);
-                        }
-                        headingHierarchy[level - 1] = heading;
-                    }
-                    else {
-                        currentChunk += "".concat(line, "\n");
-                    }
-                }
-                processChunk();
-                logger.debug("Chunking complete, created ".concat(chunks.length, " chunks"));
-                return [2 /*return*/, chunks];
-            });
-        });
-    };
-    Doc2Vec.prototype.tokenize = function (text) {
-        return text.split(/(\s+)/).filter(function (token) { return token.length > 0; });
-    };
-    Doc2Vec.prototype.createEmbeddings = function (texts) {
-        return __awaiter(this, void 0, void 0, function () {
-            var logger, response, error_18;
-            return __generator(this, function (_a) {
-                switch (_a.label) {
-                    case 0:
-                        logger = this.logger.child('embeddings');
-                        _a.label = 1;
-                    case 1:
-                        _a.trys.push([1, 3, , 4]);
-                        logger.debug("Creating embeddings for ".concat(texts.length, " texts"));
-                        return [4 /*yield*/, this.openai.embeddings.create({
-                                model: "text-embedding-3-large",
-                                input: texts,
-                            })];
-                    case 2:
-                        response = _a.sent();
-                        logger.debug("Successfully created ".concat(response.data.length, " embeddings"));
-                        return [2 /*return*/, response.data.map(function (d) { return d.embedding; })];
-                    case 3:
-                        error_18 = _a.sent();
-                        logger.error('Failed to create embeddings:', error_18);
-                        return [2 /*return*/, []];
-                    case 4: return [2 /*return*/];
-                }
-            });
-        });
-    };
-    Doc2Vec.prototype.generateHash = function (content) {
-        return crypto_1.default.createHash("sha256").update(content).digest("hex");
-    };
-    return Doc2Vec;
-}());
-if (require.main === module) {
-    var configPath = process.argv[2] || 'config.yaml';
-    if (!fs.existsSync(configPath)) {
-        console.error('Please provide a valid path to a YAML config file.');
-        process.exit(1);
-    }
-    var doc2Vec = new Doc2Vec(configPath);
-    doc2Vec.run().catch(console.error);
-}