doc2vec 2.3.0 → 2.5.0

package/dist/doc2vec.js

@@ -72,6 +72,7 @@ class Doc2Vec {
         // Check environment variable if not specified in config
         const embeddingProvider = this.config.embedding?.provider || process.env.EMBEDDING_PROVIDER || 'openai';
         const embeddingConfig = this.config.embedding || { provider: embeddingProvider };
+        this.embeddingDimension = this.resolveEmbeddingDimension(embeddingConfig);
         if (embeddingProvider === 'azure') {
             const azureApiKey = embeddingConfig.azure?.api_key || process.env.AZURE_OPENAI_KEY;
             const azureEndpoint = embeddingConfig.azure?.endpoint || process.env.AZURE_OPENAI_ENDPOINT;
@@ -88,7 +89,7 @@ class Doc2Vec {
                 apiVersion: azureApiVersion,
             });
             this.embeddingModel = azureDeploymentName;
-            this.logger.info(`Using Azure OpenAI with deployment: ${azureDeploymentName}`);
+            this.logger.info(`Using Azure OpenAI with deployment: ${azureDeploymentName} (${this.embeddingDimension} dimensions)`);
         }
         else {
             const openaiApiKey = embeddingConfig.openai?.api_key || process.env.OPENAI_API_KEY;
@@ -99,7 +100,7 @@ class Doc2Vec {
             }
             this.openai = new openai_1.OpenAI({ apiKey: openaiApiKey });
             this.embeddingModel = openaiModel;
-            this.logger.info(`Using OpenAI with model: ${openaiModel}`);
+            this.logger.info(`Using OpenAI with model: ${openaiModel} (${this.embeddingDimension} dimensions)`);
         }
         this.contentProcessor = new content_processor_1.ContentProcessor(this.logger);
     }
@@ -144,6 +145,21 @@ class Doc2Vec {
             process.exit(1);
         }
     }
+    resolveEmbeddingDimension(embeddingConfig) {
+        const defaultDimension = 3072;
+        const rawConfigValue = embeddingConfig?.dimension;
+        const rawEnvValue = process.env.EMBEDDING_DIMENSION;
+        const candidate = rawConfigValue ?? (rawEnvValue ? Number(rawEnvValue) : undefined);
+        if (candidate === undefined) {
+            return defaultDimension;
+        }
+        const parsedValue = typeof candidate === 'string' ? Number(candidate) : candidate;
+        if (!Number.isFinite(parsedValue) || parsedValue <= 0 || !Number.isInteger(parsedValue)) {
+            this.logger.warn(`Invalid embedding dimension provided (${candidate}), falling back to ${defaultDimension}`);
+            return defaultDimension;
+        }
+        return parsedValue;
+    }
     async run() {
         this.logger.section('PROCESSING SOURCES');
         for (const sourceConfig of this.config.sources) {
@@ -373,13 +389,13 @@ class Doc2Vec {
             await processIssue(issues[i]);
         }
         // Update the last run date in the database after processing all issues
-        await database_1.DatabaseManager.updateLastRunDate(dbConnection, repo, logger);
+        await database_1.DatabaseManager.updateLastRunDate(dbConnection, repo, logger, this.embeddingDimension);
         logger.info(`Successfully processed ${issues.length} issues`);
     }
     async processGithubRepo(config, parentLogger) {
         const logger = parentLogger.child('process');
         logger.info(`Starting processing for GitHub repo: ${config.repo}`);
-        const dbConnection = await database_1.DatabaseManager.initDatabase(config, logger);
+        const dbConnection = await database_1.DatabaseManager.initDatabase(config, logger, this.embeddingDimension);
         // Initialize metadata storage
         await database_1.DatabaseManager.initDatabaseMetadata(dbConnection, logger);
         logger.section('GITHUB ISSUES');
@@ -390,87 +406,64 @@ class Doc2Vec {
     async processWebsite(config, parentLogger) {
         const logger = parentLogger.child('process');
         logger.info(`Starting processing for website: ${config.url}`);
-        const dbConnection = await database_1.DatabaseManager.initDatabase(config, logger);
+        const dbConnection = await database_1.DatabaseManager.initDatabase(config, logger, this.embeddingDimension);
+        await database_1.DatabaseManager.initDatabaseMetadata(dbConnection, logger);
         const validChunkIds = new Set();
         const visitedUrls = new Set();
         const urlPrefix = utils_1.Utils.getUrlPrefix(config.url);
         logger.section('CRAWL AND EMBEDDING');
+        // Pre-load known URLs from the database so the queue includes pages from
+        // previous runs. This ensures link discovery isn't lost when pages are
+        // skipped via ETag matching.
+        let knownUrls;
+        if (dbConnection.type === 'sqlite') {
+            const urls = database_1.DatabaseManager.getStoredUrlsByPrefixSQLite(dbConnection.db, urlPrefix);
+            if (urls.length > 0) {
+                knownUrls = new Set(urls);
+                logger.info(`Found ${urls.length} known URLs in database for pre-seeding`);
+            }
+        }
+        else if (dbConnection.type === 'qdrant') {
+            const urls = await database_1.DatabaseManager.getStoredUrlsByPrefixQdrant(dbConnection, urlPrefix);
+            if (urls.length > 0) {
+                knownUrls = new Set(urls);
+                logger.info(`Found ${urls.length} known URLs in Qdrant for pre-seeding`);
+            }
+        }
+        // ETag store for caching page ETags across runs
+        const etagStore = {
+            get: async (url) => {
+                return database_1.DatabaseManager.getMetadataValue(dbConnection, `etag:${url}`, undefined, logger);
+            },
+            set: async (url, etag) => {
+                await database_1.DatabaseManager.setMetadataValue(dbConnection, `etag:${url}`, etag, logger, this.embeddingDimension);
+            },
+        };
+        const lastmodStore = {
+            get: async (url) => {
+                return database_1.DatabaseManager.getMetadataValue(dbConnection, `lastmod:${url}`, undefined, logger);
+            },
+            set: async (url, lastmod) => {
+                await database_1.DatabaseManager.setMetadataValue(dbConnection, `lastmod:${url}`, lastmod, logger, this.embeddingDimension);
+            },
+        };
         const crawlResult = await this.contentProcessor.crawlWebsite(config.url, config, async (url, content) => {
             visitedUrls.add(url);
             logger.info(`Processing content from ${url} (${content.length} chars markdown)`);
             try {
                 const chunks = await this.contentProcessor.chunkMarkdown(content, config, url);
                 logger.info(`Created ${chunks.length} chunks`);
-                if (chunks.length > 0) {
-                    const chunkProgress = logger.progress(`Processing chunks for ${url}`, chunks.length);
-                    for (let i = 0; i < chunks.length; i++) {
-                        const chunk = chunks[i];
-                        validChunkIds.add(chunk.metadata.chunk_id);
-                        const chunkId = chunk.metadata.chunk_id.substring(0, 8) + '...';
-                        let needsEmbedding = true;
-                        const chunkHash = utils_1.Utils.generateHash(chunk.content);
-                        if (dbConnection.type === 'sqlite') {
-                            const { checkHashStmt } = database_1.DatabaseManager.prepareSQLiteStatements(dbConnection.db);
-                            const existing = checkHashStmt.get(chunk.metadata.chunk_id);
-                            if (existing && existing.hash === chunkHash) {
-                                needsEmbedding = false;
-                                chunkProgress.update(1, `Skipping unchanged chunk ${chunkId}`);
-                                logger.info(`Skipping unchanged chunk: ${chunkId}`);
-                            }
-                        }
-                        else if (dbConnection.type === 'qdrant') {
-                            try {
-                                let pointId;
-                                try {
-                                    pointId = chunk.metadata.chunk_id;
-                                    if (!utils_1.Utils.isValidUuid(pointId)) {
-                                        pointId = utils_1.Utils.hashToUuid(chunk.metadata.chunk_id);
-                                    }
-                                }
-                                catch (e) {
-                                    pointId = crypto_1.default.randomUUID();
-                                }
-                                const existingPoints = await dbConnection.client.retrieve(dbConnection.collectionName, {
-                                    ids: [pointId],
-                                    with_payload: true,
-                                    with_vector: false,
-                                });
-                                if (existingPoints.length > 0 && existingPoints[0].payload && existingPoints[0].payload.hash === chunkHash) {
-                                    needsEmbedding = false;
-                                    chunkProgress.update(1, `Skipping unchanged chunk ${chunkId}`);
-                                    logger.info(`Skipping unchanged chunk: ${chunkId}`);
-                                }
-                            }
-                            catch (error) {
-                                logger.error(`Error checking existing point in Qdrant:`, error);
-                            }
-                        }
-                        if (needsEmbedding) {
-                            const embeddings = await this.createEmbeddings([chunk.content]);
-                            if (embeddings.length > 0) {
-                                const embedding = embeddings[0];
-                                if (dbConnection.type === 'sqlite') {
-                                    database_1.DatabaseManager.insertVectorsSQLite(dbConnection.db, chunk, embedding, logger, chunkHash);
-                                    chunkProgress.update(1, `Stored chunk ${chunkId} in SQLite`);
-                                }
-                                else if (dbConnection.type === 'qdrant') {
-                                    await database_1.DatabaseManager.storeChunkInQdrant(dbConnection, chunk, embedding, chunkHash);
-                                    chunkProgress.update(1, `Stored chunk ${chunkId} in Qdrant (${dbConnection.collectionName})`);
-                                }
-                            }
-                            else {
-                                logger.error(`Embedding failed for chunk: ${chunkId}`);
-                                chunkProgress.update(1, `Failed to embed chunk ${chunkId}`);
-                            }
-                        }
-                    }
-                    chunkProgress.complete();
+                for (const chunk of chunks) {
+                    validChunkIds.add(chunk.metadata.chunk_id);
                 }
+                await this.processChunksForUrl(chunks, url, dbConnection, logger);
+                return true;
             }
             catch (error) {
                 logger.error(`Error during chunking or embedding for ${url}:`, error);
+                return false;
             }
-        }, logger, visitedUrls);
+        }, logger, visitedUrls, { knownUrls, etagStore, lastmodStore });
         this.recordBrokenLinks(config.url, crawlResult.brokenLinks);
         this.writeBrokenLinksReport();
         logger.info(`Found ${validChunkIds.size} valid chunks across processed pages for ${config.url}`);
@@ -525,7 +518,7 @@ class Doc2Vec {
     async processLocalDirectory(config, parentLogger) {
         const logger = parentLogger.child('process');
         logger.info(`Starting processing for local directory: ${config.path}`);
-        const dbConnection = await database_1.DatabaseManager.initDatabase(config, logger);
+        const dbConnection = await database_1.DatabaseManager.initDatabase(config, logger, this.embeddingDimension);
         const validChunkIds = new Set();
         const processedFiles = new Set();
         logger.section('FILE SCANNING AND EMBEDDING');
@@ -560,71 +553,10 @@ class Doc2Vec {
                 }
                 const chunks = await this.contentProcessor.chunkMarkdown(content, config, fileUrl);
                 logger.info(`Created ${chunks.length} chunks`);
-                if (chunks.length > 0) {
-                    const chunkProgress = logger.progress(`Processing chunks for ${filePath}`, chunks.length);
-                    for (let i = 0; i < chunks.length; i++) {
-                        const chunk = chunks[i];
-                        validChunkIds.add(chunk.metadata.chunk_id);
-                        const chunkId = chunk.metadata.chunk_id.substring(0, 8) + '...';
-                        let needsEmbedding = true;
-                        const chunkHash = utils_1.Utils.generateHash(chunk.content);
-                        if (dbConnection.type === 'sqlite') {
-                            const { checkHashStmt } = database_1.DatabaseManager.prepareSQLiteStatements(dbConnection.db);
-                            const existing = checkHashStmt.get(chunk.metadata.chunk_id);
-                            if (existing && existing.hash === chunkHash) {
-                                needsEmbedding = false;
-                                chunkProgress.update(1, `Skipping unchanged chunk ${chunkId}`);
-                                logger.info(`Skipping unchanged chunk: ${chunkId}`);
-                            }
-                        }
-                        else if (dbConnection.type === 'qdrant') {
-                            try {
-                                let pointId;
-                                try {
-                                    pointId = chunk.metadata.chunk_id;
-                                    if (!utils_1.Utils.isValidUuid(pointId)) {
-                                        pointId = utils_1.Utils.hashToUuid(chunk.metadata.chunk_id);
-                                    }
-                                }
-                                catch (e) {
-                                    pointId = crypto_1.default.randomUUID();
-                                }
-                                const existingPoints = await dbConnection.client.retrieve(dbConnection.collectionName, {
-                                    ids: [pointId],
-                                    with_payload: true,
-                                    with_vector: false,
-                                });
-                                if (existingPoints.length > 0 && existingPoints[0].payload && existingPoints[0].payload.hash === chunkHash) {
-                                    needsEmbedding = false;
-                                    chunkProgress.update(1, `Skipping unchanged chunk ${chunkId}`);
-                                    logger.info(`Skipping unchanged chunk: ${chunkId}`);
-                                }
-                            }
-                            catch (error) {
-                                logger.error(`Error checking existing point in Qdrant:`, error);
-                            }
-                        }
-                        if (needsEmbedding) {
-                            const embeddings = await this.createEmbeddings([chunk.content]);
-                            if (embeddings.length > 0) {
-                                const embedding = embeddings[0];
-                                if (dbConnection.type === 'sqlite') {
-                                    database_1.DatabaseManager.insertVectorsSQLite(dbConnection.db, chunk, embedding, logger, chunkHash);
-                                    chunkProgress.update(1, `Stored chunk ${chunkId} in SQLite`);
-                                }
-                                else if (dbConnection.type === 'qdrant') {
-                                    await database_1.DatabaseManager.storeChunkInQdrant(dbConnection, chunk, embedding, chunkHash);
-                                    chunkProgress.update(1, `Stored chunk ${chunkId} in Qdrant (${dbConnection.collectionName})`);
-                                }
-                            }
-                            else {
-                                logger.error(`Embedding failed for chunk: ${chunkId}`);
-                                chunkProgress.update(1, `Failed to embed chunk ${chunkId}`);
-                            }
-                        }
-                    }
-                    chunkProgress.complete();
+                for (const chunk of chunks) {
+                    validChunkIds.add(chunk.metadata.chunk_id);
                 }
+                await this.processChunksForUrl(chunks, fileUrl, dbConnection, logger);
             }
             catch (error) {
                 logger.error(`Error during chunking or embedding for ${filePath}:`, error);
@@ -644,7 +576,7 @@ class Doc2Vec {
     async processCodeSource(config, parentLogger) {
         const logger = parentLogger.child('process');
         logger.info(`Starting processing for code source (${config.source})`);
-        const dbConnection = await database_1.DatabaseManager.initDatabase(config, logger);
+        const dbConnection = await database_1.DatabaseManager.initDatabase(config, logger, this.embeddingDimension);
         const validChunkIds = new Set();
         const processedFiles = new Set();
         let basePath;
@@ -728,71 +660,10 @@ class Doc2Vec {
                 try {
                     const chunks = await this.contentProcessor.chunkCode(content, config, fileUrl, relativePath || filePath, repoBranch || config.branch, config.repo);
                     logger.info(`Created ${chunks.length} chunks`);
-                    if (chunks.length > 0) {
-                        const chunkProgress = logger.progress(`Processing chunks for ${relativePath || filePath}`, chunks.length);
-                        for (let i = 0; i < chunks.length; i++) {
-                            const chunk = chunks[i];
-                            validChunkIds.add(chunk.metadata.chunk_id);
-                            const chunkId = chunk.metadata.chunk_id.substring(0, 8) + '...';
-                            let needsEmbedding = true;
-                            const chunkHash = utils_1.Utils.generateHash(chunk.content);
-                            if (dbConnection.type === 'sqlite') {
-                                const { checkHashStmt } = database_1.DatabaseManager.prepareSQLiteStatements(dbConnection.db);
-                                const existing = checkHashStmt.get(chunk.metadata.chunk_id);
-                                if (existing && existing.hash === chunkHash) {
-                                    needsEmbedding = false;
-                                    chunkProgress.update(1, `Skipping unchanged chunk ${chunkId}`);
-                                    logger.info(`Skipping unchanged chunk: ${chunkId}`);
-                                }
-                            }
-                            else if (dbConnection.type === 'qdrant') {
-                                try {
-                                    let pointId;
-                                    try {
-                                        pointId = chunk.metadata.chunk_id;
-                                        if (!utils_1.Utils.isValidUuid(pointId)) {
-                                            pointId = utils_1.Utils.hashToUuid(chunk.metadata.chunk_id);
-                                        }
-                                    }
-                                    catch (e) {
-                                        pointId = crypto_1.default.randomUUID();
-                                    }
-                                    const existingPoints = await dbConnection.client.retrieve(dbConnection.collectionName, {
-                                        ids: [pointId],
-                                        with_payload: true,
-                                        with_vector: false,
-                                    });
-                                    if (existingPoints.length > 0 && existingPoints[0].payload && existingPoints[0].payload.hash === chunkHash) {
-                                        needsEmbedding = false;
-                                        chunkProgress.update(1, `Skipping unchanged chunk ${chunkId}`);
-                                        logger.info(`Skipping unchanged chunk: ${chunkId}`);
-                                    }
-                                }
-                                catch (error) {
-                                    logger.error(`Error checking existing point in Qdrant:`, error);
-                                }
-                            }
-                            if (needsEmbedding) {
-                                const embeddings = await this.createEmbeddings([chunk.content]);
-                                if (embeddings.length > 0) {
-                                    const embedding = embeddings[0];
-                                    if (dbConnection.type === 'sqlite') {
-                                        database_1.DatabaseManager.insertVectorsSQLite(dbConnection.db, chunk, embedding, logger, chunkHash);
-                                        chunkProgress.update(1, `Stored chunk ${chunkId} in SQLite`);
-                                    }
-                                    else if (dbConnection.type === 'qdrant') {
-                                        await database_1.DatabaseManager.storeChunkInQdrant(dbConnection, chunk, embedding, chunkHash);
-                                        chunkProgress.update(1, `Stored chunk ${chunkId} in Qdrant (${dbConnection.collectionName})`);
-                                    }
-                                }
-                                else {
-                                    logger.error(`Embedding failed for chunk: ${chunkId}`);
-                                    chunkProgress.update(1, `Failed to embed chunk ${chunkId}`);
-                                }
-                            }
-                        }
-                        chunkProgress.complete();
+                    for (const chunk of chunks) {
+                        validChunkIds.add(chunk.metadata.chunk_id);
                     }
+                    await this.processChunksForUrl(chunks, fileUrl, dbConnection, logger);
                 }
                 catch (error) {
                     logger.error(`Error during code chunking or embedding for ${filePath}:`, error);
@@ -837,10 +708,10 @@ class Doc2Vec {
                             await database_1.DatabaseManager.removeChunksByUrlQdrant(dbConnection, url, logger);
                         }
                     }
-                    await database_1.DatabaseManager.setMetadataValue(dbConnection, fileListKey, JSON.stringify(currentList), logger);
+                    await database_1.DatabaseManager.setMetadataValue(dbConnection, fileListKey, JSON.stringify(currentList), logger, this.embeddingDimension);
                     if (lastMtimeKey) {
                         const nextMtime = maxObservedMtime > 0 ? maxObservedMtime : Date.now();
-                        await database_1.DatabaseManager.setMetadataValue(dbConnection, lastMtimeKey, `${nextMtime}`, logger);
+                        await database_1.DatabaseManager.setMetadataValue(dbConnection, lastMtimeKey, `${nextMtime}`, logger, this.embeddingDimension);
                     }
                 }
             }
@@ -858,7 +729,7 @@ class Doc2Vec {
                 const headSha = await this.getRepoHeadSha(basePath, logger);
                 if (headSha) {
                     const shaKey = this.buildCodeShaMetadataKey(config.repo, repoBranch);
-                    await database_1.DatabaseManager.setMetadataValue(dbConnection, shaKey, headSha, logger);
+                    await database_1.DatabaseManager.setMetadataValue(dbConnection, shaKey, headSha, logger, this.embeddingDimension);
                 }
             }
             if (tempDir) {
@@ -1021,7 +892,7 @@ class Doc2Vec {
     async processZendesk(config, parentLogger) {
         const logger = parentLogger.child('process');
         logger.info(`Starting processing for Zendesk: ${config.zendesk_subdomain}.zendesk.com`);
-        const dbConnection = await database_1.DatabaseManager.initDatabase(config, logger);
+        const dbConnection = await database_1.DatabaseManager.initDatabase(config, logger, this.embeddingDimension);
         // Initialize metadata storage
         await database_1.DatabaseManager.initDatabaseMetadata(dbConnection, logger);
         const fetchTickets = config.fetch_tickets !== false; // default true
@@ -1196,7 +1067,7 @@ class Doc2Vec {
             }
         }
         // Update the last run date in the database
-        await database_1.DatabaseManager.updateLastRunDate(dbConnection, `zendesk_tickets_${config.zendesk_subdomain}`, logger);
+        await database_1.DatabaseManager.updateLastRunDate(dbConnection, `zendesk_tickets_${config.zendesk_subdomain}`, logger, this.embeddingDimension);
         logger.info(`Successfully processed ${totalTickets} tickets`);
     }
     async fetchAndProcessZendeskArticles(config, dbConnection, logger) {
@@ -1271,61 +1142,7 @@ class Doc2Vec {
             };
             const chunks = await this.contentProcessor.chunkMarkdown(markdown, articleConfig, url);
             logger.info(`Article #${articleId}: Created ${chunks.length} chunks`);
-            // Process and store each chunk (similar to ticket processing)
-            for (const chunk of chunks) {
-                const chunkHash = utils_1.Utils.generateHash(chunk.content);
-                const chunkId = chunk.metadata.chunk_id.substring(0, 8) + '...';
-                if (dbConnection.type === 'sqlite') {
-                    const { checkHashStmt } = database_1.DatabaseManager.prepareSQLiteStatements(dbConnection.db);
-                    const existing = checkHashStmt.get(chunk.metadata.chunk_id);
-                    if (existing && existing.hash === chunkHash) {
-                        logger.info(`Skipping unchanged chunk: ${chunkId}`);
-                        continue;
-                    }
-                    const embeddings = await this.createEmbeddings([chunk.content]);
-                    if (embeddings.length) {
-                        database_1.DatabaseManager.insertVectorsSQLite(dbConnection.db, chunk, embeddings[0], logger, chunkHash);
-                        logger.debug(`Stored chunk ${chunkId} in SQLite`);
-                    }
-                    else {
-                        logger.error(`Embedding failed for chunk: ${chunkId}`);
-                    }
-                }
-                else if (dbConnection.type === 'qdrant') {
-                    try {
-                        let pointId;
-                        try {
-                            pointId = chunk.metadata.chunk_id;
-                            if (!utils_1.Utils.isValidUuid(pointId)) {
-                                pointId = utils_1.Utils.hashToUuid(chunk.metadata.chunk_id);
-                            }
-                        }
-                        catch (e) {
-                            pointId = crypto_1.default.randomUUID();
-                        }
-                        const existingPoints = await dbConnection.client.retrieve(dbConnection.collectionName, {
-                            ids: [pointId],
-                            with_payload: true,
-                            with_vector: false,
-                        });
-                        if (existingPoints.length > 0 && existingPoints[0].payload && existingPoints[0].payload.hash === chunkHash) {
-                            logger.info(`Skipping unchanged chunk: ${chunkId}`);
-                            continue;
-                        }
-                        const embeddings = await this.createEmbeddings([chunk.content]);
-                        if (embeddings.length) {
-                            await database_1.DatabaseManager.storeChunkInQdrant(dbConnection, chunk, embeddings[0], chunkHash);
-                            logger.debug(`Stored chunk ${chunkId} in Qdrant (${dbConnection.collectionName})`);
-                        }
-                        else {
-                            logger.error(`Embedding failed for chunk: ${chunkId}`);
-                        }
-                    }
-                    catch (error) {
-                        logger.error(`Error processing chunk in Qdrant:`, error);
-                    }
-                }
-            }
+            await this.processChunksForUrl(chunks, url, dbConnection, logger);
         };
         logger.info(`Fetching Zendesk help center articles updated since ${startDate}`);
         let nextPage = `${baseUrl}/articles.json`;
@@ -1356,13 +1173,98 @@ class Doc2Vec {
         }
         logger.info(`Successfully processed ${processedArticles} of ${totalArticles} articles (filtered by date >= ${startDate})`);
     }
+    /**
+     * Process chunks for a given URL with change detection.
+     *
+     * Compares the new chunk hashes against existing hashes stored in the DB.
+     * - If all hashes match (same content, same count): skip entirely (no embedding, no DB writes).
+     * - If any hash differs: delete all existing chunks for this URL and re-embed/insert all new chunks.
+     *
+     * This ensures chunk_index and total_chunks are always consistent, and no orphaned
+     * chunks are left behind when content shifts (e.g., a paragraph is added in the middle).
+     *
+     * @returns The number of chunks that were embedded (0 if skipped).
+     */
+    async processChunksForUrl(chunks, url, dbConnection, logger) {
+        if (chunks.length === 0)
+            return 0;
+        // 1. Compute hashes for all new chunks
+        const newHashes = chunks.map(c => utils_1.Utils.generateHash(c.content));
+        const newHashesSorted = newHashes.slice().sort();
+        // 2. Fetch existing hashes for this URL from the DB
+        let existingHashesSorted;
+        if (dbConnection.type === 'sqlite') {
+            existingHashesSorted = database_1.DatabaseManager.getChunkHashesByUrlSQLite(dbConnection.db, url);
+        }
+        else {
+            existingHashesSorted = await database_1.DatabaseManager.getChunkHashesByUrlQdrant(dbConnection, url);
+        }
+        // 3. Compare: if identical sorted arrays, content is unchanged — skip
+        const unchanged = newHashesSorted.length === existingHashesSorted.length &&
+            newHashesSorted.every((h, i) => h === existingHashesSorted[i]);
+        if (unchanged) {
+            logger.info(`Skipping unchanged URL (${chunks.length} chunks): ${url}`);
+            return 0;
+        }
+        // 4. Content changed — delete all existing chunks for this URL
+        if (existingHashesSorted.length > 0) {
+            logger.info(`Content changed for ${url} (${existingHashesSorted.length} old chunks → ${chunks.length} new chunks), re-embedding all`);
+            if (dbConnection.type === 'sqlite') {
+                database_1.DatabaseManager.removeChunksByUrlSQLite(dbConnection.db, url, logger);
+            }
+            else {
+                await database_1.DatabaseManager.removeChunksByUrlQdrant(dbConnection, url, logger);
+            }
+        }
+        // 5. Embed and insert all new chunks
+        let embeddedCount = 0;
+        const chunkProgress = logger.progress(`Embedding chunks for ${url}`, chunks.length);
+        for (let i = 0; i < chunks.length; i++) {
+            const chunk = chunks[i];
+            const chunkHash = newHashes[i];
+            const chunkId = chunk.metadata.chunk_id.substring(0, 8) + '...';
+            const embeddings = await this.createEmbeddings([chunk.content]);
+            if (embeddings.length > 0) {
+                const embedding = embeddings[0];
+                if (dbConnection.type === 'sqlite') {
+                    database_1.DatabaseManager.insertVectorsSQLite(dbConnection.db, chunk, embedding, logger, chunkHash);
+                    chunkProgress.update(1, `Stored chunk ${chunkId} in SQLite`);
+                }
+                else if (dbConnection.type === 'qdrant') {
+                    await database_1.DatabaseManager.storeChunkInQdrant(dbConnection, chunk, embedding, chunkHash);
+                    chunkProgress.update(1, `Stored chunk ${chunkId} in Qdrant (${dbConnection.collectionName})`);
+                }
+                embeddedCount++;
+            }
+            else {
+                logger.error(`Embedding failed for chunk: ${chunkId}`);
+                chunkProgress.update(1, `Failed to embed chunk ${chunkId}`);
+            }
+        }
+        chunkProgress.complete();
+        return embeddedCount;
+    }
     async createEmbeddings(texts) {
         const logger = this.logger.child('embeddings');
         try {
-            logger.debug(`Creating embeddings for ${texts.length} texts`);
+            // Truncate any texts that exceed the embedding model's token limit.
+            // This is a safety net for pages with dense content (e.g., large API
+            // reference pages with deeply nested config paths) where the chunker's
+            // whitespace-based token count drastically underestimates BPE tokens.
+            const safeTexts = texts.map(text => {
+                if (text.length > Doc2Vec.MAX_EMBEDDING_CHARS) {
+                    const estimatedTokens = Math.ceil(text.length / Doc2Vec.CHARS_PER_TOKEN);
+                    logger.warn(`Truncating oversized chunk (${text.length} chars, ~${estimatedTokens} tokens) ` +
+                        `to ${Doc2Vec.MAX_EMBEDDING_CHARS} chars (~${Doc2Vec.MAX_EMBEDDING_TOKENS} tokens) ` +
+                        `to fit embedding model limit`);
+                    return text.substring(0, Doc2Vec.MAX_EMBEDDING_CHARS);
+                }
+                return text;
+            });
+            logger.debug(`Creating embeddings for ${safeTexts.length} texts`);
             const response = await this.openai.embeddings.create({
                 model: this.embeddingModel,
-                input: texts,
+                input: safeTexts,
             });
             logger.debug(`Successfully created ${response.data.length} embeddings`);
             return response.data.map(d => d.embedding);
@@ -1374,6 +1276,13 @@ class Doc2Vec {
     }
 }
 exports.Doc2Vec = Doc2Vec;
+// Embedding model token limit and character-based estimate.
+// OpenAI's text-embedding-3-large has an 8,191-token context limit.
+// Using ~4 chars per BPE token as a conservative estimate (actual average
+// is ~3.5 for English, lower for code/URLs/config paths).
+Doc2Vec.MAX_EMBEDDING_TOKENS = 8191;
+Doc2Vec.CHARS_PER_TOKEN = 4;
+Doc2Vec.MAX_EMBEDDING_CHARS = Doc2Vec.MAX_EMBEDDING_TOKENS * Doc2Vec.CHARS_PER_TOKEN;
 if (require.main === module) {
     const configPath = process.argv[2] || 'config.yaml';
     if (!fs.existsSync(configPath)) {

package/dist/utils.js

@@ -69,18 +69,23 @@ class Utils {
             return url;
         }
     }
-    static buildUrl(href, currentUrl) {
+    static buildUrl(href, currentUrl, logger) {
         try {
             return new URL(href, currentUrl).toString();
         }
         catch (error) {
-            console.warn(`Invalid URL found: ${href}`);
+            if (logger) {
+                logger.warn(`Invalid URL found: ${href}`);
+            }
             return '';
         }
     }
     static shouldProcessUrl(url) {
         const parsedUrl = new URL(url);
         const pathname = parsedUrl.pathname;
+        // Paths ending with / are directory-like URLs (e.g., /app/2.1.x/), always process them
+        if (pathname.endsWith('/'))
+            return true;
         const ext = path.extname(pathname);
         if (!ext)
             return true;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "doc2vec",
-    "version": "2.3.0",
+    "version": "2.5.0",
     "type": "commonjs",
     "description": "",
     "main": "dist/doc2vec.js",