doc2vec 2.2.0 → 2.4.0

package/dist/database.js

@@ -374,10 +374,10 @@ class DatabaseManager {
                     chunk.metadata.version
                 ];
                 if (hasBranchColumn) {
-                    insertValues.push(chunk.metadata.branch ?? null);
+                    insertValues.push(chunk.metadata.branch ?? '');
                 }
                 if (hasRepoColumn) {
-                    insertValues.push(chunk.metadata.repo ?? null);
+                    insertValues.push(chunk.metadata.repo ?? '');
                 }
                 insertValues.push(JSON.stringify(chunk.metadata.heading_hierarchy), chunk.metadata.section, chunk.metadata.chunk_id, chunk.content, chunk.metadata.url, hash, chunkIndex, totalChunks);
                 insertStmt.run(...insertValues);
@@ -389,10 +389,10 @@ class DatabaseManager {
                     chunk.metadata.version
                 ];
                 if (hasBranchColumn) {
-                    updateValues.push(chunk.metadata.branch ?? null);
+                    updateValues.push(chunk.metadata.branch ?? '');
                 }
                 if (hasRepoColumn) {
-                    updateValues.push(chunk.metadata.repo ?? null);
+                    updateValues.push(chunk.metadata.repo ?? '');
                 }
                 updateValues.push(JSON.stringify(chunk.metadata.heading_hierarchy), chunk.metadata.section, chunk.content, chunk.metadata.url, hash, chunkIndex, totalChunks, chunk.metadata.chunk_id);
                 updateStmt.run(...updateValues);
@@ -535,7 +535,7 @@ class DatabaseManager {
                         {
                             key: 'url',
                             match: {
-                                text: url
+                                value: url // Exact match — must match getChunkHashesByUrlQdrant's filter
                             }
                         }
                     ],
@@ -555,6 +555,96 @@ class DatabaseManager {
             logger.error(`Error deleting chunks from Qdrant for URL ${url}:`, error);
         }
     }
+    /**
+     * Fetch all distinct URLs stored under a given URL prefix.
+     * Used to pre-seed the crawl queue on re-runs so link discovery isn't lost
+     * when pages are skipped via ETag matching.
+     */
+    static getStoredUrlsByPrefixSQLite(db, urlPrefix) {
+        const stmt = db.prepare('SELECT DISTINCT url FROM vec_items WHERE url LIKE ? || \'%\'');
+        const rows = stmt.all(urlPrefix);
+        return rows.map(r => r.url);
+    }
+    static async getStoredUrlsByPrefixQdrant(db, urlPrefix) {
+        const { client, collectionName } = db;
+        try {
+            const response = await client.scroll(collectionName, {
+                limit: 10000,
+                with_payload: { include: ['url'] },
+                with_vector: false,
+                filter: {
+                    must: [
+                        {
+                            key: 'url',
+                            match: { text: urlPrefix }
+                        }
+                    ],
+                    must_not: [
+                        {
+                            key: 'is_metadata',
+                            match: { value: true }
+                        }
+                    ]
+                }
+            });
+            const urls = new Set();
+            for (const point of response.points) {
+                const url = point.payload?.url;
+                if (url && url.startsWith(urlPrefix)) {
+                    urls.add(url);
+                }
+            }
+            return Array.from(urls);
+        }
+        catch (error) {
+            return [];
+        }
+    }
+    /**
+     * Fetch all stored chunk content hashes for a given URL.
+     * Returns a sorted array of hash strings for comparison.
+     */
+    static getChunkHashesByUrlSQLite(db, url) {
+        const stmt = db.prepare('SELECT hash FROM vec_items WHERE url = ?');
+        const rows = stmt.all(url);
+        return rows.map(r => r.hash).sort();
+    }
+    /**
+     * Fetch all stored chunk content hashes for a given URL from Qdrant.
+     * Returns a sorted array of hash strings for comparison.
+     */
+    static async getChunkHashesByUrlQdrant(db, url) {
+        const { client, collectionName } = db;
+        try {
+            const response = await client.scroll(collectionName, {
+                limit: 10000,
+                with_payload: { include: ['hash'] },
+                with_vector: false,
+                filter: {
+                    must: [
+                        {
+                            key: 'url',
+                            match: { value: url }
+                        }
+                    ],
+                    must_not: [
+                        {
+                            key: 'is_metadata',
+                            match: { value: true }
+                        }
+                    ]
+                }
+            });
+            return response.points
+                .map((point) => point.payload?.hash)
+                .filter(Boolean)
+                .sort();
+        }
+        catch (error) {
+            // On error, return empty array so the caller treats it as "changed" (safe default)
+            return [];
+        }
+    }
     static removeObsoleteFilesSQLite(db, processedFiles, pathConfig, logger) {
         const getChunksForPathStmt = db.prepare(`
             SELECT chunk_id, url FROM vec_items

package/dist/doc2vec.js

@@ -391,86 +391,63 @@ class Doc2Vec {
         const logger = parentLogger.child('process');
         logger.info(`Starting processing for website: ${config.url}`);
         const dbConnection = await database_1.DatabaseManager.initDatabase(config, logger);
+        await database_1.DatabaseManager.initDatabaseMetadata(dbConnection, logger);
         const validChunkIds = new Set();
         const visitedUrls = new Set();
         const urlPrefix = utils_1.Utils.getUrlPrefix(config.url);
         logger.section('CRAWL AND EMBEDDING');
+        // Pre-load known URLs from the database so the queue includes pages from
+        // previous runs. This ensures link discovery isn't lost when pages are
+        // skipped via ETag matching.
+        let knownUrls;
+        if (dbConnection.type === 'sqlite') {
+            const urls = database_1.DatabaseManager.getStoredUrlsByPrefixSQLite(dbConnection.db, urlPrefix);
+            if (urls.length > 0) {
+                knownUrls = new Set(urls);
+                logger.info(`Found ${urls.length} known URLs in database for pre-seeding`);
+            }
+        }
+        else if (dbConnection.type === 'qdrant') {
+            const urls = await database_1.DatabaseManager.getStoredUrlsByPrefixQdrant(dbConnection, urlPrefix);
+            if (urls.length > 0) {
+                knownUrls = new Set(urls);
+                logger.info(`Found ${urls.length} known URLs in Qdrant for pre-seeding`);
+            }
+        }
+        // ETag store for caching page ETags across runs
+        const etagStore = {
+            get: async (url) => {
+                return database_1.DatabaseManager.getMetadataValue(dbConnection, `etag:${url}`, undefined, logger);
+            },
+            set: async (url, etag) => {
+                await database_1.DatabaseManager.setMetadataValue(dbConnection, `etag:${url}`, etag, logger);
+            },
+        };
+        const lastmodStore = {
+            get: async (url) => {
+                return database_1.DatabaseManager.getMetadataValue(dbConnection, `lastmod:${url}`, undefined, logger);
+            },
+            set: async (url, lastmod) => {
+                await database_1.DatabaseManager.setMetadataValue(dbConnection, `lastmod:${url}`, lastmod, logger);
+            },
+        };
         const crawlResult = await this.contentProcessor.crawlWebsite(config.url, config, async (url, content) => {
             visitedUrls.add(url);
             logger.info(`Processing content from ${url} (${content.length} chars markdown)`);
             try {
                 const chunks = await this.contentProcessor.chunkMarkdown(content, config, url);
                 logger.info(`Created ${chunks.length} chunks`);
-                if (chunks.length > 0) {
-                    const chunkProgress = logger.progress(`Embedding chunks for ${url}`, chunks.length);
-                    for (let i = 0; i < chunks.length; i++) {
-                        const chunk = chunks[i];
-                        validChunkIds.add(chunk.metadata.chunk_id);
-                        const chunkId = chunk.metadata.chunk_id.substring(0, 8) + '...';
-                        let needsEmbedding = true;
-                        const chunkHash = utils_1.Utils.generateHash(chunk.content);
-                        if (dbConnection.type === 'sqlite') {
-                            const { checkHashStmt } = database_1.DatabaseManager.prepareSQLiteStatements(dbConnection.db);
-                            const existing = checkHashStmt.get(chunk.metadata.chunk_id);
-                            if (existing && existing.hash === chunkHash) {
-                                needsEmbedding = false;
-                                chunkProgress.update(1, `Skipping unchanged chunk ${chunkId}`);
-                                logger.info(`Skipping unchanged chunk: ${chunkId}`);
-                            }
-                        }
-                        else if (dbConnection.type === 'qdrant') {
-                            try {
-                                let pointId;
-                                try {
-                                    pointId = chunk.metadata.chunk_id;
-                                    if (!utils_1.Utils.isValidUuid(pointId)) {
-                                        pointId = utils_1.Utils.hashToUuid(chunk.metadata.chunk_id);
-                                    }
-                                }
-                                catch (e) {
-                                    pointId = crypto_1.default.randomUUID();
-                                }
-                                const existingPoints = await dbConnection.client.retrieve(dbConnection.collectionName, {
-                                    ids: [pointId],
-                                    with_payload: true,
-                                    with_vector: false,
-                                });
-                                if (existingPoints.length > 0 && existingPoints[0].payload && existingPoints[0].payload.hash === chunkHash) {
-                                    needsEmbedding = false;
-                                    chunkProgress.update(1, `Skipping unchanged chunk ${chunkId}`);
-                                    logger.info(`Skipping unchanged chunk: ${chunkId}`);
-                                }
-                            }
-                            catch (error) {
-                                logger.error(`Error checking existing point in Qdrant:`, error);
-                            }
-                        }
-                        if (needsEmbedding) {
-                            const embeddings = await this.createEmbeddings([chunk.content]);
-                            if (embeddings.length > 0) {
-                                const embedding = embeddings[0];
-                                if (dbConnection.type === 'sqlite') {
-                                    database_1.DatabaseManager.insertVectorsSQLite(dbConnection.db, chunk, embedding, logger, chunkHash);
-                                    chunkProgress.update(1, `Stored chunk ${chunkId} in SQLite`);
-                                }
-                                else if (dbConnection.type === 'qdrant') {
-                                    await database_1.DatabaseManager.storeChunkInQdrant(dbConnection, chunk, embedding, chunkHash);
-                                    chunkProgress.update(1, `Stored chunk ${chunkId} in Qdrant (${dbConnection.collectionName})`);
-                                }
-                            }
-                            else {
-                                logger.error(`Embedding failed for chunk: ${chunkId}`);
-                                chunkProgress.update(1, `Failed to embed chunk ${chunkId}`);
-                            }
-                        }
-                    }
-                    chunkProgress.complete();
+                for (const chunk of chunks) {
+                    validChunkIds.add(chunk.metadata.chunk_id);
                 }
+                await this.processChunksForUrl(chunks, url, dbConnection, logger);
+                return true;
             }
             catch (error) {
                 logger.error(`Error during chunking or embedding for ${url}:`, error);
+                return false;
             }
-        }, logger, visitedUrls);
+        }, logger, visitedUrls, { knownUrls, etagStore, lastmodStore });
         this.recordBrokenLinks(config.url, crawlResult.brokenLinks);
         this.writeBrokenLinksReport();
         logger.info(`Found ${validChunkIds.size} valid chunks across processed pages for ${config.url}`);
@@ -560,71 +537,10 @@ class Doc2Vec {
                 }
                 const chunks = await this.contentProcessor.chunkMarkdown(content, config, fileUrl);
                 logger.info(`Created ${chunks.length} chunks`);
-                if (chunks.length > 0) {
-                    const chunkProgress = logger.progress(`Embedding chunks for ${filePath}`, chunks.length);
-                    for (let i = 0; i < chunks.length; i++) {
-                        const chunk = chunks[i];
-                        validChunkIds.add(chunk.metadata.chunk_id);
-                        const chunkId = chunk.metadata.chunk_id.substring(0, 8) + '...';
-                        let needsEmbedding = true;
-                        const chunkHash = utils_1.Utils.generateHash(chunk.content);
-                        if (dbConnection.type === 'sqlite') {
-                            const { checkHashStmt } = database_1.DatabaseManager.prepareSQLiteStatements(dbConnection.db);
-                            const existing = checkHashStmt.get(chunk.metadata.chunk_id);
-                            if (existing && existing.hash === chunkHash) {
-                                needsEmbedding = false;
-                                chunkProgress.update(1, `Skipping unchanged chunk ${chunkId}`);
-                                logger.info(`Skipping unchanged chunk: ${chunkId}`);
-                            }
-                        }
-                        else if (dbConnection.type === 'qdrant') {
-                            try {
-                                let pointId;
-                                try {
-                                    pointId = chunk.metadata.chunk_id;
-                                    if (!utils_1.Utils.isValidUuid(pointId)) {
-                                        pointId = utils_1.Utils.hashToUuid(chunk.metadata.chunk_id);
-                                    }
-                                }
-                                catch (e) {
-                                    pointId = crypto_1.default.randomUUID();
-                                }
-                                const existingPoints = await dbConnection.client.retrieve(dbConnection.collectionName, {
-                                    ids: [pointId],
-                                    with_payload: true,
-                                    with_vector: false,
-                                });
-                                if (existingPoints.length > 0 && existingPoints[0].payload && existingPoints[0].payload.hash === chunkHash) {
-                                    needsEmbedding = false;
-                                    chunkProgress.update(1, `Skipping unchanged chunk ${chunkId}`);
-                                    logger.info(`Skipping unchanged chunk: ${chunkId}`);
-                                }
-                            }
-                            catch (error) {
-                                logger.error(`Error checking existing point in Qdrant:`, error);
-                            }
-                        }
-                        if (needsEmbedding) {
-                            const embeddings = await this.createEmbeddings([chunk.content]);
-                            if (embeddings.length > 0) {
-                                const embedding = embeddings[0];
-                                if (dbConnection.type === 'sqlite') {
-                                    database_1.DatabaseManager.insertVectorsSQLite(dbConnection.db, chunk, embedding, logger, chunkHash);
-                                    chunkProgress.update(1, `Stored chunk ${chunkId} in SQLite`);
-                                }
-                                else if (dbConnection.type === 'qdrant') {
-                                    await database_1.DatabaseManager.storeChunkInQdrant(dbConnection, chunk, embedding, chunkHash);
-                                    chunkProgress.update(1, `Stored chunk ${chunkId} in Qdrant (${dbConnection.collectionName})`);
-                                }
-                            }
-                            else {
-                                logger.error(`Embedding failed for chunk: ${chunkId}`);
-                                chunkProgress.update(1, `Failed to embed chunk ${chunkId}`);
-                            }
-                        }
-                    }
-                    chunkProgress.complete();
+                for (const chunk of chunks) {
+                    validChunkIds.add(chunk.metadata.chunk_id);
                 }
+                await this.processChunksForUrl(chunks, fileUrl, dbConnection, logger);
             }
             catch (error) {
                 logger.error(`Error during chunking or embedding for ${filePath}:`, error);
@@ -728,71 +644,10 @@ class Doc2Vec {
                 try {
                     const chunks = await this.contentProcessor.chunkCode(content, config, fileUrl, relativePath || filePath, repoBranch || config.branch, config.repo);
                     logger.info(`Created ${chunks.length} chunks`);
-                    if (chunks.length > 0) {
-                        const chunkProgress = logger.progress(`Embedding chunks for ${relativePath || filePath}`, chunks.length);
-                        for (let i = 0; i < chunks.length; i++) {
-                            const chunk = chunks[i];
-                            validChunkIds.add(chunk.metadata.chunk_id);
-                            const chunkId = chunk.metadata.chunk_id.substring(0, 8) + '...';
-                            let needsEmbedding = true;
-                            const chunkHash = utils_1.Utils.generateHash(chunk.content);
-                            if (dbConnection.type === 'sqlite') {
-                                const { checkHashStmt } = database_1.DatabaseManager.prepareSQLiteStatements(dbConnection.db);
-                                const existing = checkHashStmt.get(chunk.metadata.chunk_id);
-                                if (existing && existing.hash === chunkHash) {
-                                    needsEmbedding = false;
-                                    chunkProgress.update(1, `Skipping unchanged chunk ${chunkId}`);
-                                    logger.info(`Skipping unchanged chunk: ${chunkId}`);
-                                }
-                            }
-                            else if (dbConnection.type === 'qdrant') {
-                                try {
-                                    let pointId;
-                                    try {
-                                        pointId = chunk.metadata.chunk_id;
-                                        if (!utils_1.Utils.isValidUuid(pointId)) {
-                                            pointId = utils_1.Utils.hashToUuid(chunk.metadata.chunk_id);
-                                        }
-                                    }
-                                    catch (e) {
-                                        pointId = crypto_1.default.randomUUID();
-                                    }
-                                    const existingPoints = await dbConnection.client.retrieve(dbConnection.collectionName, {
-                                        ids: [pointId],
-                                        with_payload: true,
-                                        with_vector: false,
-                                    });
-                                    if (existingPoints.length > 0 && existingPoints[0].payload && existingPoints[0].payload.hash === chunkHash) {
-                                        needsEmbedding = false;
-                                        chunkProgress.update(1, `Skipping unchanged chunk ${chunkId}`);
-                                        logger.info(`Skipping unchanged chunk: ${chunkId}`);
-                                    }
-                                }
-                                catch (error) {
-                                    logger.error(`Error checking existing point in Qdrant:`, error);
-                                }
-                            }
-                            if (needsEmbedding) {
-                                const embeddings = await this.createEmbeddings([chunk.content]);
-                                if (embeddings.length > 0) {
-                                    const embedding = embeddings[0];
-                                    if (dbConnection.type === 'sqlite') {
-                                        database_1.DatabaseManager.insertVectorsSQLite(dbConnection.db, chunk, embedding, logger, chunkHash);
-                                        chunkProgress.update(1, `Stored chunk ${chunkId} in SQLite`);
-                                    }
-                                    else if (dbConnection.type === 'qdrant') {
-                                        await database_1.DatabaseManager.storeChunkInQdrant(dbConnection, chunk, embedding, chunkHash);
-                                        chunkProgress.update(1, `Stored chunk ${chunkId} in Qdrant (${dbConnection.collectionName})`);
-                                    }
-                                }
-                                else {
-                                    logger.error(`Embedding failed for chunk: ${chunkId}`);
-                                    chunkProgress.update(1, `Failed to embed chunk ${chunkId}`);
-                                }
-                            }
-                        }
-                        chunkProgress.complete();
+                    for (const chunk of chunks) {
+                        validChunkIds.add(chunk.metadata.chunk_id);
                     }
+                    await this.processChunksForUrl(chunks, fileUrl, dbConnection, logger);
                 }
                 catch (error) {
                     logger.error(`Error during code chunking or embedding for ${filePath}:`, error);
@@ -1271,61 +1126,7 @@ class Doc2Vec {
             };
             const chunks = await this.contentProcessor.chunkMarkdown(markdown, articleConfig, url);
             logger.info(`Article #${articleId}: Created ${chunks.length} chunks`);
-            // Process and store each chunk (similar to ticket processing)
-            for (const chunk of chunks) {
-                const chunkHash = utils_1.Utils.generateHash(chunk.content);
-                const chunkId = chunk.metadata.chunk_id.substring(0, 8) + '...';
-                if (dbConnection.type === 'sqlite') {
-                    const { checkHashStmt } = database_1.DatabaseManager.prepareSQLiteStatements(dbConnection.db);
-                    const existing = checkHashStmt.get(chunk.metadata.chunk_id);
-                    if (existing && existing.hash === chunkHash) {
-                        logger.info(`Skipping unchanged chunk: ${chunkId}`);
-                        continue;
-                    }
-                    const embeddings = await this.createEmbeddings([chunk.content]);
-                    if (embeddings.length) {
-                        database_1.DatabaseManager.insertVectorsSQLite(dbConnection.db, chunk, embeddings[0], logger, chunkHash);
-                        logger.debug(`Stored chunk ${chunkId} in SQLite`);
-                    }
-                    else {
-                        logger.error(`Embedding failed for chunk: ${chunkId}`);
-                    }
-                }
-                else if (dbConnection.type === 'qdrant') {
-                    try {
-                        let pointId;
-                        try {
-                            pointId = chunk.metadata.chunk_id;
-                            if (!utils_1.Utils.isValidUuid(pointId)) {
-                                pointId = utils_1.Utils.hashToUuid(chunk.metadata.chunk_id);
-                            }
-                        }
-                        catch (e) {
-                            pointId = crypto_1.default.randomUUID();
-                        }
-                        const existingPoints = await dbConnection.client.retrieve(dbConnection.collectionName, {
-                            ids: [pointId],
-                            with_payload: true,
-                            with_vector: false,
-                        });
-                        if (existingPoints.length > 0 && existingPoints[0].payload && existingPoints[0].payload.hash === chunkHash) {
-                            logger.info(`Skipping unchanged chunk: ${chunkId}`);
-                            continue;
-                        }
-                        const embeddings = await this.createEmbeddings([chunk.content]);
-                        if (embeddings.length) {
-                            await database_1.DatabaseManager.storeChunkInQdrant(dbConnection, chunk, embeddings[0], chunkHash);
-                            logger.debug(`Stored chunk ${chunkId} in Qdrant (${dbConnection.collectionName})`);
-                        }
-                        else {
-                            logger.error(`Embedding failed for chunk: ${chunkId}`);
-                        }
-                    }
-                    catch (error) {
-                        logger.error(`Error processing chunk in Qdrant:`, error);
-                    }
-                }
-            }
+            await this.processChunksForUrl(chunks, url, dbConnection, logger);
         };
         logger.info(`Fetching Zendesk help center articles updated since ${startDate}`);
         let nextPage = `${baseUrl}/articles.json`;
@@ -1356,13 +1157,98 @@ class Doc2Vec {
         }
         logger.info(`Successfully processed ${processedArticles} of ${totalArticles} articles (filtered by date >= ${startDate})`);
     }
+    /**
+     * Process chunks for a given URL with change detection.
+     *
+     * Compares the new chunk hashes against existing hashes stored in the DB.
+     * - If all hashes match (same content, same count): skip entirely (no embedding, no DB writes).
+     * - If any hash differs: delete all existing chunks for this URL and re-embed/insert all new chunks.
+     *
+     * This ensures chunk_index and total_chunks are always consistent, and no orphaned
+     * chunks are left behind when content shifts (e.g., a paragraph is added in the middle).
+     *
+     * @returns The number of chunks that were embedded (0 if skipped).
+     */
+    async processChunksForUrl(chunks, url, dbConnection, logger) {
+        if (chunks.length === 0)
+            return 0;
+        // 1. Compute hashes for all new chunks
+        const newHashes = chunks.map(c => utils_1.Utils.generateHash(c.content));
+        const newHashesSorted = newHashes.slice().sort();
+        // 2. Fetch existing hashes for this URL from the DB
+        let existingHashesSorted;
+        if (dbConnection.type === 'sqlite') {
+            existingHashesSorted = database_1.DatabaseManager.getChunkHashesByUrlSQLite(dbConnection.db, url);
+        }
+        else {
+            existingHashesSorted = await database_1.DatabaseManager.getChunkHashesByUrlQdrant(dbConnection, url);
+        }
+        // 3. Compare: if identical sorted arrays, content is unchanged — skip
+        const unchanged = newHashesSorted.length === existingHashesSorted.length &&
+            newHashesSorted.every((h, i) => h === existingHashesSorted[i]);
+        if (unchanged) {
+            logger.info(`Skipping unchanged URL (${chunks.length} chunks): ${url}`);
+            return 0;
+        }
+        // 4. Content changed — delete all existing chunks for this URL
+        if (existingHashesSorted.length > 0) {
+            logger.info(`Content changed for ${url} (${existingHashesSorted.length} old chunks → ${chunks.length} new chunks), re-embedding all`);
+            if (dbConnection.type === 'sqlite') {
+                database_1.DatabaseManager.removeChunksByUrlSQLite(dbConnection.db, url, logger);
+            }
+            else {
+                await database_1.DatabaseManager.removeChunksByUrlQdrant(dbConnection, url, logger);
+            }
+        }
+        // 5. Embed and insert all new chunks
+        let embeddedCount = 0;
+        const chunkProgress = logger.progress(`Embedding chunks for ${url}`, chunks.length);
+        for (let i = 0; i < chunks.length; i++) {
+            const chunk = chunks[i];
+            const chunkHash = newHashes[i];
+            const chunkId = chunk.metadata.chunk_id.substring(0, 8) + '...';
+            const embeddings = await this.createEmbeddings([chunk.content]);
+            if (embeddings.length > 0) {
+                const embedding = embeddings[0];
+                if (dbConnection.type === 'sqlite') {
+                    database_1.DatabaseManager.insertVectorsSQLite(dbConnection.db, chunk, embedding, logger, chunkHash);
+                    chunkProgress.update(1, `Stored chunk ${chunkId} in SQLite`);
+                }
+                else if (dbConnection.type === 'qdrant') {
+                    await database_1.DatabaseManager.storeChunkInQdrant(dbConnection, chunk, embedding, chunkHash);
+                    chunkProgress.update(1, `Stored chunk ${chunkId} in Qdrant (${dbConnection.collectionName})`);
+                }
+                embeddedCount++;
+            }
+            else {
+                logger.error(`Embedding failed for chunk: ${chunkId}`);
+                chunkProgress.update(1, `Failed to embed chunk ${chunkId}`);
+            }
+        }
+        chunkProgress.complete();
+        return embeddedCount;
+    }
     async createEmbeddings(texts) {
         const logger = this.logger.child('embeddings');
         try {
-            logger.debug(`Creating embeddings for ${texts.length} texts`);
+            // Truncate any texts that exceed the embedding model's token limit.
+            // This is a safety net for pages with dense content (e.g., large API
+            // reference pages with deeply nested config paths) where the chunker's
+            // whitespace-based token count drastically underestimates BPE tokens.
+            const safeTexts = texts.map(text => {
+                if (text.length > Doc2Vec.MAX_EMBEDDING_CHARS) {
+                    const estimatedTokens = Math.ceil(text.length / Doc2Vec.CHARS_PER_TOKEN);
+                    logger.warn(`Truncating oversized chunk (${text.length} chars, ~${estimatedTokens} tokens) ` +
+                        `to ${Doc2Vec.MAX_EMBEDDING_CHARS} chars (~${Doc2Vec.MAX_EMBEDDING_TOKENS} tokens) ` +
+                        `to fit embedding model limit`);
+                    return text.substring(0, Doc2Vec.MAX_EMBEDDING_CHARS);
+                }
+                return text;
+            });
+            logger.debug(`Creating embeddings for ${safeTexts.length} texts`);
             const response = await this.openai.embeddings.create({
                 model: this.embeddingModel,
-                input: texts,
+                input: safeTexts,
             });
             logger.debug(`Successfully created ${response.data.length} embeddings`);
             return response.data.map(d => d.embedding);
@@ -1374,6 +1260,13 @@ class Doc2Vec {
     }
 }
 exports.Doc2Vec = Doc2Vec;
+// Embedding model token limit and character-based estimate.
+// OpenAI's text-embedding-3-large has an 8,191-token context limit.
+// Using ~4 chars per BPE token as a conservative estimate (actual average
+// is ~3.5 for English, lower for code/URLs/config paths).
+Doc2Vec.MAX_EMBEDDING_TOKENS = 8191;
+Doc2Vec.CHARS_PER_TOKEN = 4;
+Doc2Vec.MAX_EMBEDDING_CHARS = Doc2Vec.MAX_EMBEDDING_TOKENS * Doc2Vec.CHARS_PER_TOKEN;
 if (require.main === module) {
     const configPath = process.argv[2] || 'config.yaml';
     if (!fs.existsSync(configPath)) {

package/dist/utils.js

@@ -81,6 +81,9 @@ class Utils {
     static shouldProcessUrl(url) {
         const parsedUrl = new URL(url);
         const pathname = parsedUrl.pathname;
+        // Paths ending with / are directory-like URLs (e.g., /app/2.1.x/), always process them
+        if (pathname.endsWith('/'))
+            return true;
         const ext = path.extname(pathname);
         if (!ext)
             return true;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "doc2vec",
-    "version": "2.2.0",
+    "version": "2.4.0",
     "type": "commonjs",
     "description": "",
     "main": "dist/doc2vec.js",