doc2vec 2.0.0 → 2.2.0

package/dist/doc2vec.js

@@ -37,11 +37,15 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.Doc2Vec = void 0;
 const axios_1 = __importDefault(require("axios"));
 const crypto_1 = __importDefault(require("crypto"));
 const yaml = __importStar(require("js-yaml"));
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
+const os = __importStar(require("os"));
+const child_process_1 = require("child_process");
+const util_1 = require("util");
 const buffer_1 = require("buffer");
 const openai_1 = require("openai");
 const dotenv = __importStar(require("dotenv"));
@@ -50,9 +54,11 @@ const utils_1 = require("./utils");
 const database_1 = require("./database");
 const content_processor_1 = require("./content-processor");
 const GITHUB_TOKEN = process.env.GITHUB_PERSONAL_ACCESS_TOKEN;
+const execAsync = (0, util_1.promisify)(child_process_1.exec);
 dotenv.config();
 class Doc2Vec {
     constructor(configPath) {
+        this.brokenLinksByWebsite = {};
         this.logger = new logger_1.Logger('Doc2Vec', {
             level: logger_1.LogLevel.DEBUG,
             useTimestamp: true,
@@ -61,7 +67,40 @@ class Doc2Vec {
         });
         this.logger.info('Initializing Doc2Vec');
         this.config = this.loadConfig(configPath);
-        this.openai = new openai_1.OpenAI({ apiKey: process.env.OPENAI_API_KEY });
+        this.configDir = path.dirname(path.resolve(configPath));
+        // Initialize OpenAI or Azure OpenAI based on configuration
+        // Check environment variable if not specified in config
+        const embeddingProvider = this.config.embedding?.provider || process.env.EMBEDDING_PROVIDER || 'openai';
+        const embeddingConfig = this.config.embedding || { provider: embeddingProvider };
+        if (embeddingProvider === 'azure') {
+            const azureApiKey = embeddingConfig.azure?.api_key || process.env.AZURE_OPENAI_KEY;
+            const azureEndpoint = embeddingConfig.azure?.endpoint || process.env.AZURE_OPENAI_ENDPOINT;
+            const azureDeploymentName = embeddingConfig.azure?.deployment_name || process.env.AZURE_OPENAI_DEPLOYMENT_NAME || 'text-embedding-3-large';
+            const azureApiVersion = embeddingConfig.azure?.api_version || process.env.AZURE_OPENAI_API_VERSION || '2024-10-21';
+            if (!azureApiKey || !azureEndpoint) {
+                this.logger.error('Azure OpenAI requires api_key and endpoint to be configured');
+                process.exit(1);
+            }
+            this.openai = new openai_1.AzureOpenAI({
+                apiKey: azureApiKey,
+                endpoint: azureEndpoint,
+                deployment: azureDeploymentName,
+                apiVersion: azureApiVersion,
+            });
+            this.embeddingModel = azureDeploymentName;
+            this.logger.info(`Using Azure OpenAI with deployment: ${azureDeploymentName}`);
+        }
+        else {
+            const openaiApiKey = embeddingConfig.openai?.api_key || process.env.OPENAI_API_KEY;
+            const openaiModel = embeddingConfig.openai?.model || process.env.OPENAI_MODEL || 'text-embedding-3-large';
+            if (!openaiApiKey) {
+                this.logger.error('OpenAI requires api_key to be configured');
+                process.exit(1);
+            }
+            this.openai = new openai_1.OpenAI({ apiKey: openaiApiKey });
+            this.embeddingModel = openaiModel;
+            this.logger.info(`Using OpenAI with model: ${openaiModel}`);
+        }
         this.contentProcessor = new content_processor_1.ContentProcessor(this.logger);
     }
     loadConfig(configPath) {
@@ -81,6 +120,22 @@ class Doc2Vec {
             });
             let config = yaml.load(configFile);
             const typedConfig = config;
+            for (const source of typedConfig.sources) {
+                if (source.type === 'code') {
+                    if (!source.version || String(source.version).trim().length === 0) {
+                        if (source.branch && String(source.branch).trim().length > 0) {
+                            source.version = source.branch;
+                        }
+                        else {
+                            source.version = 'local';
+                        }
+                    }
+                }
+                else if (!source.version || String(source.version).trim().length === 0) {
+                    logger.error(`Missing required version for ${source.type} source: ${source.product_name}`);
+                    process.exit(1);
+                }
+            }
             logger.info(`Configuration loaded successfully, found ${typedConfig.sources.length} sources`);
             return typedConfig;
         }
@@ -103,6 +158,9 @@ class Doc2Vec {
             else if (sourceConfig.type === 'local_directory') {
                 await this.processLocalDirectory(sourceConfig, sourceLogger);
             }
+            else if (sourceConfig.type === 'code') {
+                await this.processCodeSource(sourceConfig, sourceLogger);
+            }
             else if (sourceConfig.type === 'zendesk') {
                 await this.processZendesk(sourceConfig, sourceLogger);
             }
@@ -413,6 +471,8 @@ class Doc2Vec {
                 logger.error(`Error during chunking or embedding for ${url}:`, error);
             }
         }, logger, visitedUrls);
+        this.recordBrokenLinks(config.url, crawlResult.brokenLinks);
+        this.writeBrokenLinksReport();
         logger.info(`Found ${validChunkIds.size} valid chunks across processed pages for ${config.url}`);
         logger.section('CLEANUP');
         if (crawlResult.hasNetworkErrors) {
@@ -430,6 +490,38 @@ class Doc2Vec {
         }
         logger.info(`Finished processing website: ${config.url}`);
     }
+    recordBrokenLinks(baseUrl, brokenLinks) {
+        const uniqueByKey = new Map();
+        for (const link of brokenLinks) {
+            const key = `${link.source} -> ${link.target}`;
+            if (!uniqueByKey.has(key)) {
+                uniqueByKey.set(key, link);
+            }
+        }
+        const unique = Array.from(uniqueByKey.values()).sort((a, b) => {
+            const sourceCompare = a.source.localeCompare(b.source);
+            if (sourceCompare !== 0)
+                return sourceCompare;
+            return a.target.localeCompare(b.target);
+        });
+        this.brokenLinksByWebsite[baseUrl] = unique;
+    }
+    writeBrokenLinksReport() {
+        const reportPath = path.join(this.configDir, '404.yaml');
+        const orderedEntries = Object.entries(this.brokenLinksByWebsite)
+            .sort(([a], [b]) => a.localeCompare(b));
+        const reportPayload = orderedEntries.map(([website, links]) => ({
+            website,
+            'broken-links': links
+        }));
+        try {
+            fs.writeFileSync(reportPath, yaml.dump(reportPayload, { noRefs: true }), 'utf8');
+            this.logger.info(`Wrote broken link report to ${reportPath}`);
+        }
+        catch (error) {
+            this.logger.error(`Failed to write broken link report to ${reportPath}:`, error);
+        }
+    }
     async processLocalDirectory(config, parentLogger) {
         const logger = parentLogger.child('process');
         logger.info(`Starting processing for local directory: ${config.path}`);
@@ -549,6 +641,383 @@ class Doc2Vec {
         }
         logger.info(`Finished processing local directory: ${config.path}`);
     }
+    async processCodeSource(config, parentLogger) {
+        const logger = parentLogger.child('process');
+        logger.info(`Starting processing for code source (${config.source})`);
+        const dbConnection = await database_1.DatabaseManager.initDatabase(config, logger);
+        const validChunkIds = new Set();
+        const processedFiles = new Set();
+        let basePath;
+        let cleanupPathConfig;
+        let tempDir = null;
+        let repoUrlPrefix;
+        let repoBranch;
+        let incrementalMode = false;
+        let deleteUrls = [];
+        let allowedFiles;
+        let mtimeCutoff;
+        let fileListKey;
+        let lastMtimeKey;
+        let trackedFiles;
+        let maxObservedMtime = 0;
+        if (config.source === 'local_directory') {
+            if (!config.path) {
+                logger.error('Code source type local_directory requires a path.');
+                return;
+            }
+            basePath = config.path;
+            cleanupPathConfig = config.url_rewrite_prefix
+                ? { path: basePath, url_rewrite_prefix: config.url_rewrite_prefix }
+                : basePath;
+            const resolvedPath = path.resolve(basePath);
+            const pathKey = resolvedPath.replace(/[^a-zA-Z0-9]+/g, '_');
+            lastMtimeKey = `code_last_mtime_${pathKey}`;
+            fileListKey = `code_filelist_${pathKey}`;
+            await database_1.DatabaseManager.initDatabaseMetadata(dbConnection, logger);
+            const lastMtimeValue = await database_1.DatabaseManager.getMetadataValue(dbConnection, lastMtimeKey, '0', logger);
+            mtimeCutoff = lastMtimeValue ? parseFloat(lastMtimeValue) : 0;
+            trackedFiles = new Set();
+            incrementalMode = true;
+        }
+        else if (config.source === 'github') {
+            if (!config.repo) {
+                logger.error('Code source type github requires a repo in owner/repo format.');
+                return;
+            }
+            const cloneResult = await this.cloneGithubRepo(config, logger);
+            basePath = cloneResult.path;
+            tempDir = cloneResult.path;
+            repoUrlPrefix = cloneResult.urlPrefix;
+            repoBranch = cloneResult.branch;
+            cleanupPathConfig = { path: basePath, url_rewrite_prefix: repoUrlPrefix };
+            await database_1.DatabaseManager.initDatabaseMetadata(dbConnection, logger);
+            const shaKey = this.buildCodeShaMetadataKey(config.repo, repoBranch);
+            const lastSha = await database_1.DatabaseManager.getMetadataValue(dbConnection, shaKey, undefined, logger);
+            const headSha = await this.getRepoHeadSha(basePath, logger);
+            if (lastSha && headSha) {
+                if (headSha === lastSha) {
+                    incrementalMode = true;
+                    allowedFiles = new Set();
+                    deleteUrls = [];
+                }
+                else {
+                    const diffResult = await this.getGitChangedFiles(basePath, lastSha, repoBranch, logger);
+                    if (diffResult.mode === 'incremental') {
+                        incrementalMode = true;
+                        allowedFiles = diffResult.changedFiles;
+                        deleteUrls = diffResult.deletedPaths
+                            .map((relativePath) => this.buildCodeFileUrl(path.join(basePath, relativePath), basePath, config, repoUrlPrefix));
+                    }
+                    else {
+                        logger.warn('Falling back to full scan for GitHub code source.');
+                    }
+                }
+            }
+        }
+        else {
+            logger.error(`Unknown code source: ${config.source}`);
+            return;
+        }
+        logger.section('CODE SCANNING AND EMBEDDING');
+        try {
+            const scanResult = await this.contentProcessor.processCodeDirectory(basePath, config, async (filePath, content) => {
+                processedFiles.add(filePath);
+                const relativePath = path.relative(basePath, filePath).replace(/\\/g, '/');
+                const fileUrl = this.buildCodeFileUrl(filePath, basePath, config, repoUrlPrefix);
+                logger.info(`Processing code from ${relativePath || filePath} (${content.length} chars)`);
+                try {
+                    const chunks = await this.contentProcessor.chunkCode(content, config, fileUrl, relativePath || filePath, repoBranch || config.branch, config.repo);
+                    logger.info(`Created ${chunks.length} chunks`);
+                    if (chunks.length > 0) {
+                        const chunkProgress = logger.progress(`Embedding chunks for ${relativePath || filePath}`, chunks.length);
+                        for (let i = 0; i < chunks.length; i++) {
+                            const chunk = chunks[i];
+                            validChunkIds.add(chunk.metadata.chunk_id);
+                            const chunkId = chunk.metadata.chunk_id.substring(0, 8) + '...';
+                            let needsEmbedding = true;
+                            const chunkHash = utils_1.Utils.generateHash(chunk.content);
+                            if (dbConnection.type === 'sqlite') {
+                                const { checkHashStmt } = database_1.DatabaseManager.prepareSQLiteStatements(dbConnection.db);
+                                const existing = checkHashStmt.get(chunk.metadata.chunk_id);
+                                if (existing && existing.hash === chunkHash) {
+                                    needsEmbedding = false;
+                                    chunkProgress.update(1, `Skipping unchanged chunk ${chunkId}`);
+                                    logger.info(`Skipping unchanged chunk: ${chunkId}`);
+                                }
+                            }
+                            else if (dbConnection.type === 'qdrant') {
+                                try {
+                                    let pointId;
+                                    try {
+                                        pointId = chunk.metadata.chunk_id;
+                                        if (!utils_1.Utils.isValidUuid(pointId)) {
+                                            pointId = utils_1.Utils.hashToUuid(chunk.metadata.chunk_id);
+                                        }
+                                    }
+                                    catch (e) {
+                                        pointId = crypto_1.default.randomUUID();
+                                    }
+                                    const existingPoints = await dbConnection.client.retrieve(dbConnection.collectionName, {
+                                        ids: [pointId],
+                                        with_payload: true,
+                                        with_vector: false,
+                                    });
+                                    if (existingPoints.length > 0 && existingPoints[0].payload && existingPoints[0].payload.hash === chunkHash) {
+                                        needsEmbedding = false;
+                                        chunkProgress.update(1, `Skipping unchanged chunk ${chunkId}`);
+                                        logger.info(`Skipping unchanged chunk: ${chunkId}`);
+                                    }
+                                }
+                                catch (error) {
+                                    logger.error(`Error checking existing point in Qdrant:`, error);
+                                }
+                            }
+                            if (needsEmbedding) {
+                                const embeddings = await this.createEmbeddings([chunk.content]);
+                                if (embeddings.length > 0) {
+                                    const embedding = embeddings[0];
+                                    if (dbConnection.type === 'sqlite') {
+                                        database_1.DatabaseManager.insertVectorsSQLite(dbConnection.db, chunk, embedding, logger, chunkHash);
+                                        chunkProgress.update(1, `Stored chunk ${chunkId} in SQLite`);
+                                    }
+                                    else if (dbConnection.type === 'qdrant') {
+                                        await database_1.DatabaseManager.storeChunkInQdrant(dbConnection, chunk, embedding, chunkHash);
+                                        chunkProgress.update(1, `Stored chunk ${chunkId} in Qdrant (${dbConnection.collectionName})`);
+                                    }
+                                }
+                                else {
+                                    logger.error(`Embedding failed for chunk: ${chunkId}`);
+                                    chunkProgress.update(1, `Failed to embed chunk ${chunkId}`);
+                                }
+                            }
+                        }
+                        chunkProgress.complete();
+                    }
+                }
+                catch (error) {
+                    logger.error(`Error during code chunking or embedding for ${filePath}:`, error);
+                }
+            }, logger, undefined, {
+                allowedFiles,
+                mtimeCutoff,
+                trackFiles: trackedFiles
+            });
+            if (trackedFiles) {
+                maxObservedMtime = scanResult.maxMtime;
+            }
+        }
+        finally {
+            logger.section('CLEANUP');
+            if (incrementalMode) {
+                if (deleteUrls.length > 0) {
+                    logger.info(`Cleaning up ${deleteUrls.length} deleted/renamed files`);
+                    for (const url of deleteUrls) {
+                        if (dbConnection.type === 'sqlite') {
+                            database_1.DatabaseManager.removeChunksByUrlSQLite(dbConnection.db, url, logger);
+                        }
+                        else if (dbConnection.type === 'qdrant') {
+                            await database_1.DatabaseManager.removeChunksByUrlQdrant(dbConnection, url, logger);
+                        }
+                    }
+                }
+                else {
+                    logger.info('No deleted/renamed files to clean up.');
+                }
+                if (trackedFiles && fileListKey) {
+                    const previousListValue = await database_1.DatabaseManager.getMetadataValue(dbConnection, fileListKey, '[]', logger);
+                    const previousList = previousListValue ? JSON.parse(previousListValue) : [];
+                    const currentList = Array.from(trackedFiles);
+                    const deletedFiles = previousList.filter((filePath) => !trackedFiles?.has(filePath));
+                    for (const deletedFile of deletedFiles) {
+                        const url = this.buildCodeFileUrl(deletedFile, basePath, config, repoUrlPrefix);
+                        if (dbConnection.type === 'sqlite') {
+                            database_1.DatabaseManager.removeChunksByUrlSQLite(dbConnection.db, url, logger);
+                        }
+                        else if (dbConnection.type === 'qdrant') {
+                            await database_1.DatabaseManager.removeChunksByUrlQdrant(dbConnection, url, logger);
+                        }
+                    }
+                    await database_1.DatabaseManager.setMetadataValue(dbConnection, fileListKey, JSON.stringify(currentList), logger);
+                    if (lastMtimeKey) {
+                        const nextMtime = maxObservedMtime > 0 ? maxObservedMtime : Date.now();
+                        await database_1.DatabaseManager.setMetadataValue(dbConnection, lastMtimeKey, `${nextMtime}`, logger);
+                    }
+                }
+            }
+            else {
+                if (dbConnection.type === 'sqlite') {
+                    logger.info(`Running SQLite cleanup for code source ${basePath}`);
+                    database_1.DatabaseManager.removeObsoleteFilesSQLite(dbConnection.db, processedFiles, cleanupPathConfig, logger);
+                }
+                else if (dbConnection.type === 'qdrant') {
+                    logger.info(`Running Qdrant cleanup for code source ${basePath} in collection ${dbConnection.collectionName}`);
+                    await database_1.DatabaseManager.removeObsoleteFilesQdrant(dbConnection, processedFiles, cleanupPathConfig, logger);
+                }
+            }
+            if (config.source === 'github' && basePath && repoBranch) {
+                const headSha = await this.getRepoHeadSha(basePath, logger);
+                if (headSha) {
+                    const shaKey = this.buildCodeShaMetadataKey(config.repo, repoBranch);
+                    await database_1.DatabaseManager.setMetadataValue(dbConnection, shaKey, headSha, logger);
+                }
+            }
+            if (tempDir) {
+                try {
+                    fs.rmSync(tempDir, { recursive: true, force: true });
+                    logger.debug(`Removed temporary repo at ${tempDir}`);
+                }
+                catch (error) {
+                    logger.warn(`Failed to remove temporary repo at ${tempDir}:`, error);
+                }
+            }
+        }
+        logger.info(`Finished processing code source (${config.source})`);
+    }
+    buildCodeShaMetadataKey(repo, branch) {
+        const normalizedRepo = repo.replace(/[^a-zA-Z0-9]+/g, '_');
+        const normalizedBranch = branch.replace(/[^a-zA-Z0-9]+/g, '_');
+        return `code_last_sha_${normalizedRepo}_${normalizedBranch}`;
+    }
+    async getRepoHeadSha(repoPath, logger) {
+        try {
+            const { stdout } = await execAsync(`git -C "${repoPath}" rev-parse HEAD`);
+            return stdout.trim() || undefined;
+        }
+        catch (error) {
+            logger.warn(`Failed to resolve HEAD sha for ${repoPath}:`, error);
+            return undefined;
+        }
+    }
+    async getGitChangedFiles(repoPath, lastSha, branch, logger) {
+        const diffCommand = `git -C "${repoPath}" diff --name-status ${lastSha}..HEAD`;
+        const attemptDiff = async () => {
+            const { stdout } = await execAsync(diffCommand);
+            return stdout;
+        };
+        let diffOutput;
+        try {
+            diffOutput = await attemptDiff();
+        }
+        catch (error) {
+            logger.warn(`Failed to diff against ${lastSha}. Fetching more history...`);
+            const fetchDepths = [200, 1000, 5000];
+            let fetched = false;
+            for (const depth of fetchDepths) {
+                try {
+                    logger.info(`Fetching with --depth=${depth}...`);
+                    await execAsync(`git -C "${repoPath}" fetch --depth=${depth} origin "${branch}"`);
+                    diffOutput = await attemptDiff();
+                    fetched = true;
+                    break;
+                }
+                catch (fetchError) {
+                    logger.warn(`Diff still failed at --depth=${depth}.`);
+                }
+            }
+            if (!fetched) {
+                try {
+                    logger.info(`Attempting full unshallow fetch...`);
+                    await execAsync(`git -C "${repoPath}" fetch --unshallow origin "${branch}"`);
+                    diffOutput = await attemptDiff();
+                }
+                catch (unshallowError) {
+                    logger.warn(`Failed to diff even after full unshallow. Falling back to full scan.`, unshallowError);
+                    return { mode: 'full', changedFiles: new Set(), deletedPaths: [] };
+                }
+            }
+        }
+        if (!diffOutput) {
+            logger.warn('No diff output available. Falling back to full scan.');
+            return { mode: 'full', changedFiles: new Set(), deletedPaths: [] };
+        }
+        const changedFiles = new Set();
+        const deletedPaths = [];
+        for (const line of diffOutput.split('\n')) {
+            const trimmed = line.trim();
+            if (!trimmed)
+                continue;
+            const parts = trimmed.split('\t');
+            const status = parts[0];
+            if (status.startsWith('R')) {
+                const oldPath = parts[1];
+                const newPath = parts[2];
+                if (oldPath)
+                    deletedPaths.push(oldPath);
+                if (newPath)
+                    changedFiles.add(path.join(repoPath, newPath));
+            }
+            else if (status === 'D') {
+                const deletedPath = parts[1];
+                if (deletedPath)
+                    deletedPaths.push(deletedPath);
+            }
+            else if (status === 'A' || status === 'M') {
+                const changedPath = parts[1];
+                if (changedPath)
+                    changedFiles.add(path.join(repoPath, changedPath));
+            }
+        }
+        logger.info(`Git diff changes: ${changedFiles.size} modified/added, ${deletedPaths.length} deleted/renamed.`);
+        return { mode: 'incremental', changedFiles, deletedPaths };
+    }
+    buildCodeFileUrl(filePath, basePath, config, repoUrlPrefix) {
+        const relativePath = path.relative(basePath, filePath).replace(/\\/g, '/');
+        if (repoUrlPrefix) {
+            return `${repoUrlPrefix}/${relativePath}`;
+        }
+        if (config.url_rewrite_prefix) {
+            if (relativePath.startsWith('..')) {
+                return `file://${filePath}`;
+            }
+            const prefix = config.url_rewrite_prefix.endsWith('/')
+                ? config.url_rewrite_prefix.slice(0, -1)
+                : config.url_rewrite_prefix;
+            return `${prefix}/${relativePath}`;
+        }
+        return `file://${filePath}`;
+    }
+    async cloneGithubRepo(config, logger) {
+        const repo = config.repo;
+        const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'doc2vec-code-'));
+        const requestedBranch = config.branch;
+        const encodedToken = GITHUB_TOKEN ? encodeURIComponent(GITHUB_TOKEN) : '';
+        const repoUrl = encodedToken
+            ? `https://x-access-token:${encodedToken}@github.com/${repo}.git`
+            : `https://github.com/${repo}.git`;
+        const branchArg = requestedBranch ? `--branch "${requestedBranch}"` : '';
+        logger.info(`Cloning ${repo} to ${tempDir}`);
+        try {
+            await execAsync(`git clone --depth 1 ${branchArg} "${repoUrl}" "${tempDir}"`);
+        }
+        catch (error) {
+            try {
+                fs.rmSync(tempDir, { recursive: true, force: true });
+            }
+            catch (cleanupError) {
+                logger.warn(`Failed to clean up temp dir after clone failure: ${tempDir}`, cleanupError);
+            }
+            logger.error(`Failed to clone repo ${repo}:`, error);
+            throw error;
+        }
+        let resolvedBranch = requestedBranch;
+        if (!resolvedBranch) {
+            resolvedBranch = await this.getRepoBranch(tempDir, logger);
+        }
+        const branch = resolvedBranch || 'main';
+        const urlPrefix = `https://github.com/${repo}/blob/${branch}`;
+        return { path: tempDir, branch, urlPrefix };
+    }
+    async getRepoBranch(repoPath, logger) {
+        try {
+            const { stdout } = await execAsync(`git -C "${repoPath}" symbolic-ref --short HEAD`);
+            const branch = stdout.trim();
+            return branch || undefined;
+        }
+        catch (error) {
+            logger.warn(`Failed to resolve repo branch for ${repoPath}:`, error);
+            return undefined;
+        }
+    }
     async processZendesk(config, parentLogger) {
         const logger = parentLogger.child('process');
         logger.info(`Starting processing for Zendesk: ${config.zendesk_subdomain}.zendesk.com`);
@@ -892,7 +1361,7 @@ class Doc2Vec {
         try {
             logger.debug(`Creating embeddings for ${texts.length} texts`);
             const response = await this.openai.embeddings.create({
-                model: "text-embedding-3-large",
+                model: this.embeddingModel,
                 input: texts,
             });
             logger.debug(`Successfully created ${response.data.length} embeddings`);
@@ -904,6 +1373,7 @@ class Doc2Vec {
         }
     }
 }
+exports.Doc2Vec = Doc2Vec;
 if (require.main === module) {
     const configPath = process.argv[2] || 'config.yaml';
     if (!fs.existsSync(configPath)) {

package/dist/vitest.config.js

@@ -0,0 +1,16 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const config_1 = require("vitest/config");
+exports.default = (0, config_1.defineConfig)({
+    test: {
+        globals: true,
+        environment: 'node',
+        include: ['tests/**/*.test.ts'],
+        testTimeout: 30000,
+        coverage: {
+            provider: 'v8',
+            include: ['utils.ts', 'logger.ts', 'content-processor.ts', 'database.ts', 'code-chunker.ts', 'doc2vec.ts'],
+            exclude: ['mcp/**', 'dist/**', 'tests/**'],
+        },
+    },
+});

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "doc2vec",
-    "version": "2.0.0",
+    "version": "2.2.0",
     "type": "commonjs",
     "description": "",
     "main": "dist/doc2vec.js",
@@ -19,6 +19,9 @@
     "scripts": {
         "build": "tsc && chmod 755 dist/doc2vec.js",
         "start": "node dist/doc2vec.js",
+        "test": "vitest run",
+        "test:watch": "vitest",
+        "test:coverage": "vitest run --coverage",
         "prepublishOnly": "npm run build",
         "prepare": "npm run build"
     },
@@ -26,6 +29,7 @@
     "author": "",
     "license": "ISC",
     "dependencies": {
+        "@chonkiejs/core": "^0.0.7",
         "@mozilla/readability": "^0.4.4",
         "@qdrant/js-client-rest": "^1.13.0",
         "@qdrant/qdrant-js": "^1.13.0",
@@ -42,7 +46,9 @@
         "puppeteer": "^24.1.1",
         "sanitize-html": "^2.11.0",
         "sqlite-vec": "0.1.7-alpha.2",
+        "tree-sitter-wasms": "^0.1.0",
         "turndown": "^7.1.2",
+        "web-tree-sitter": "^0.25.4",
         "word-extractor": "^1.0.4"
     },
     "devDependencies": {
@@ -52,7 +58,9 @@
         "@types/node": "^20.10.0",
         "@types/sanitize-html": "^2.9.5",
         "@types/turndown": "^5.0.4",
+        "@vitest/coverage-v8": "^4.0.18",
         "ts-node": "^10.9.1",
-        "typescript": "^5.3.2"
+        "typescript": "^5.3.2",
+        "vitest": "^4.0.18"
     }
 }