doc2vec 2.11.0 → 2.13.0

package/dist/database.js

@@ -71,6 +71,7 @@ class DatabaseManager {
                     total_chunks INTEGER
                 );
             `);
+            this.ensureChunkDatesTable(db);
             logger.info(`SQLite database initialized successfully`);
             return { db, type: 'sqlite' };
         }
@@ -307,9 +308,24 @@ class DatabaseManager {
             logger.error(`Failed to update last run date for ${repo}:`, error);
         }
     }
+    /**
+     * Chunk creation timestamps live in a companion table because vec_items is
+     * a vec0 virtual table that cannot be ALTERed to add a column. Rows are
+     * written on every chunk insert/update; stale rows for deleted chunks are
+     * harmless (lookups always join against vec_items).
+     */
+    static ensureChunkDatesTable(db) {
+        db.exec(`
+            CREATE TABLE IF NOT EXISTS vec_chunk_dates (
+                chunk_id   TEXT PRIMARY KEY,
+                created_at TEXT NOT NULL
+            );
+        `);
+    }
     static prepareSQLiteStatements(db) {
         let cached = this.columnCache.get(db);
         if (!cached) {
+            this.ensureChunkDatesTable(db);
             cached = {
                 hasBranch: this.hasColumn(db, 'branch'),
                 hasRepo: this.hasColumn(db, 'repo')
@@ -360,12 +376,16 @@ class DatabaseManager {
             `),
             getAllChunkIdsStmt: db.prepare(`SELECT chunk_id FROM vec_items`),
             deleteChunkStmt: db.prepare(`DELETE FROM vec_items WHERE chunk_id = ?`),
+            touchDateStmt: db.prepare(`
+                INSERT INTO vec_chunk_dates (chunk_id, created_at) VALUES (?, ?)
+                ON CONFLICT(chunk_id) DO UPDATE SET created_at = excluded.created_at
+            `),
             hasBranchColumn,
             hasRepoColumn
         };
     }
     static insertVectorsSQLite(db, chunk, embedding, logger, chunkHash) {
-        const { insertStmt, updateStmt, hasBranchColumn, hasRepoColumn } = this.prepareSQLiteStatements(db);
+        const { insertStmt, updateStmt, touchDateStmt, hasBranchColumn, hasRepoColumn } = this.prepareSQLiteStatements(db);
         const hash = chunkHash || utils_1.Utils.generateHash(chunk.content);
         const transaction = db.transaction(() => {
             // Use BigInt for true integer representation in SQLite vec0
@@ -401,6 +421,7 @@ class DatabaseManager {
                 updateValues.push(JSON.stringify(chunk.metadata.heading_hierarchy), chunk.metadata.section, chunk.content, chunk.metadata.url, hash, chunkIndex, totalChunks, chunk.metadata.chunk_id);
                 updateStmt.run(...updateValues);
             }
+            touchDateStmt.run(chunk.metadata.chunk_id, new Date().toISOString());
         });
         transaction();
     }
@@ -434,6 +455,7 @@ class DatabaseManager {
                     original_chunk_id: chunk.metadata.chunk_id,
                     chunk_index: chunk.metadata.chunk_index,
                     total_chunks: chunk.metadata.total_chunks,
+                    created_at: new Date().toISOString(),
                 },
             };
             await client.upsert(collectionName, {
@@ -457,17 +479,20 @@ class DatabaseManager {
         const deleteChunkStmt = db.prepare(`DELETE FROM vec_items WHERE chunk_id = ?`);
         const existingChunks = getChunksForUrlStmt.all(urlPrefix);
         let deletedCount = 0;
+        const deletedUrls = new Set();
         const transaction = db.transaction(() => {
             for (const { chunk_id, url } of existingChunks) {
                 if (!visitedUrls.has(url)) {
                     logger.debug(`Deleting obsolete chunk from SQLite: ${chunk_id.substring(0, 8)}... (URL not visited)`);
                     deleteChunkStmt.run(chunk_id);
                     deletedCount++;
+                    deletedUrls.add(url);
                 }
             }
         });
         transaction();
         logger.info(`Deleted ${deletedCount} obsolete chunks from SQLite for URL ${urlPrefix}`);
+        return { items: deletedUrls.size, chunks: deletedCount };
     }
     static async removeObsoleteChunksQdrant(db, visitedUrls, urlPrefix, logger) {
         const { client, collectionName } = db;
@@ -496,6 +521,7 @@ class DatabaseManager {
                 ]
             };
             const obsoletePointIds = [];
+            const obsoleteUrls = new Set();
             let offset = undefined;
             do {
                 const response = await client.scroll(collectionName, {
@@ -513,6 +539,7 @@ class DatabaseManager {
                     }
                     if (url && !visitedUrls.has(url)) {
                         obsoletePointIds.push(point.id);
+                        obsoleteUrls.add(url);
                     }
                 }
                 offset = response.next_page_offset;
@@ -526,6 +553,7 @@ class DatabaseManager {
             else {
                 logger.info(`No obsolete chunks to delete from Qdrant for URL ${urlPrefix}`);
             }
+            return { items: obsoleteUrls.size, chunks: obsoletePointIds.length };
         }
         catch (error) {
             // Rethrow so a failed cleanup fails the job instead of silently
@@ -548,31 +576,42 @@ class DatabaseManager {
         const deleteStmt = db.prepare(`DELETE FROM vec_items WHERE url = ?`);
         const result = deleteStmt.run(url);
         logger.info(`Deleted ${result.changes} chunks from SQLite for URL ${url}`);
+        return result.changes;
     }
     static async removeChunksByUrlQdrant(db, url, logger) {
         const { client, collectionName } = db;
         try {
-            await client.delete(collectionName, {
-                filter: {
-                    must: [
-                        {
-                            key: 'url',
-                            match: {
-                                value: url // Exact match — must match getChunkHashesByUrlQdrant's filter
-                            }
+            const filter = {
+                must: [
+                    {
+                        key: 'url',
+                        match: {
+                            value: url // Exact match — must match getChunkHashesByUrlQdrant's filter
                         }
-                    ],
-                    must_not: [
-                        {
-                            key: 'is_metadata',
-                            match: {
-                                value: true
-                            }
+                    }
+                ],
+                must_not: [
+                    {
+                        key: 'is_metadata',
+                        match: {
+                            value: true
                         }
-                    ]
-                }
-            });
+                    }
+                ]
+            };
+            // Count first so callers can report how many chunks the delete removed
+            // (Qdrant's delete-by-filter response doesn't include a count).
+            let deletedCount = 0;
+            try {
+                const countResult = await client.count(collectionName, { filter, exact: true });
+                deletedCount = countResult?.count ?? 0;
+            }
+            catch {
+                // Count is informational only — never fail the delete over it
+            }
+            await client.delete(collectionName, { filter });
             logger.info(`Deleted chunks from Qdrant for URL ${url}`);
+            return deletedCount;
         }
         catch (error) {
             // Rethrow so the failure propagates and the job exits non-zero.
@@ -697,6 +736,7 @@ class DatabaseManager {
         logger.debug(`Searching for chunks with URL prefix: ${urlPrefix}`);
         const existingChunks = getChunksForPathStmt.all(urlPrefix);
         let deletedCount = 0;
+        const deletedUrls = new Set();
         const transaction = db.transaction(() => {
             for (const { chunk_id, url } of existingChunks) {
                 // Skip if it's not from our URL prefix (safety check)
@@ -720,11 +760,13 @@ class DatabaseManager {
                     logger.debug(`Deleting obsolete chunk from SQLite: ${chunk_id.substring(0, 8)}... (File not processed: ${filePath})`);
                     deleteChunkStmt.run(chunk_id);
                     deletedCount++;
+                    deletedUrls.add(url);
                 }
             }
         });
         transaction();
         logger.info(`Deleted ${deletedCount} obsolete chunks from SQLite for URL prefix ${urlPrefix}`);
+        return { items: deletedUrls.size, chunks: deletedCount };
     }
     static async removeObsoleteFilesQdrant(db, processedFiles, pathConfig, logger) {
         const { client, collectionName } = db;
@@ -768,6 +810,7 @@ class DatabaseManager {
                 ]
             };
             const obsoletePointIds = [];
+            const obsoleteUrls = new Set();
             let offset = undefined;
             do {
                 const response = await client.scroll(collectionName, {
@@ -799,6 +842,7 @@ class DatabaseManager {
                     }
                     if (filePath && !processedFiles.has(filePath)) {
                         obsoletePointIds.push(point.id);
+                        obsoleteUrls.add(url);
                     }
                 }
                 offset = response.next_page_offset;
@@ -812,6 +856,7 @@ class DatabaseManager {
             else {
                 logger.info(`No obsolete chunks to delete from Qdrant for URL prefix ${urlPrefix}`);
             }
+            return { items: obsoleteUrls.size, chunks: obsoletePointIds.length };
         }
         catch (error) {
             // Rethrow so a failed cleanup fails the job instead of silently

package/dist/doc2vec.js

@@ -53,6 +53,7 @@ const logger_1 = require("./logger");
 const utils_1 = require("./utils");
 const database_1 = require("./database");
 const content_processor_1 = require("./content-processor");
+const types_1 = require("./types");
 const client_s3_1 = require("@aws-sdk/client-s3");
 const markdown_store_1 = require("./markdown-store");
 const GITHUB_TOKEN = process.env.GITHUB_PERSONAL_ACCESS_TOKEN;
@@ -61,6 +62,9 @@ dotenv.config();
 class Doc2Vec {
     constructor(configPath) {
         this.brokenLinksByWebsite = {};
+        // Change counters for the source currently being processed by run().
+        // Sources are processed sequentially, so a single slot is sufficient.
+        this.counters = (0, types_1.newSourceRunCounters)('items');
         this.logger = new logger_1.Logger('Doc2Vec', {
             level: logger_1.LogLevel.DEBUG,
             useTimestamp: true,
@@ -183,6 +187,15 @@ class Doc2Vec {
             const startTime = Date.now();
             let ok = true;
             let errorMessage;
+            const itemsKindByType = {
+                website: 'pages',
+                github: 'issues',
+                local_directory: 'files',
+                code: 'files',
+                s3: 'objects',
+                zendesk: 'items',
+            };
+            this.counters = (0, types_1.newSourceRunCounters)(itemsKindByType[sourceConfig.type] ?? 'items');
             try {
                 if (sourceConfig.type === 'github') {
                     await this.processGithubRepo(sourceConfig, sourceLogger);
@@ -219,7 +232,8 @@ class Doc2Vec {
                 version: sourceConfig.version,
                 duration_ms: Date.now() - startTime,
                 ok,
-                ...(errorMessage && { error: errorMessage })
+                ...(errorMessage && { error: errorMessage }),
+                counters: this.counters
             });
         }
         // Close the Postgres markdown store connection pool
@@ -240,22 +254,31 @@ class Doc2Vec {
         // Get the last run date from the database
         const startDate = sourceConfig.start_date || '2025-01-01';
         const lastRunDate = await database_1.DatabaseManager.getLastRunDate(dbConnection, repo, `${startDate}T00:00:00Z`, logger);
-        const fetchWithRetry = async (url, params = {}, retries = 5, delay = 5000) => {
+        // Retryable: 5xx, 429, rate-limited 403s, and network/timeout errors.
+        // Everything else is deterministic — the same request fails identically,
+        // so retrying only burns wall-clock. In particular a 422 from deep
+        // page-number pagination used to cost ~75s of backoff before failing.
+        const isRetryableStatus = (status) => status === undefined || status >= 500 || status === 429;
+        const requestWithRetry = async (url, params, retries = 5, delay = 5000) => {
             for (let attempt = 0; attempt < retries; attempt++) {
                 try {
                     // Only log on retries to reduce noise during pagination
                     if (attempt > 0) {
                         logger.debug(`GitHub API retry: ${url} (attempt ${attempt + 1}/${retries})`);
                     }
+                    // Callers need the response headers (Link) for cursor
+                    // pagination, so the whole response is returned.
                     const response = await axios_1.default.get(url, {
                         headers: {
                             Authorization: `token ${GITHUB_TOKEN}`,
                             Accept: 'application/vnd.github.v3+json',
                         },
-                        params,
+                        // Subsequent pages come from a Link URL that already
+                        // carries its query string — adding params would duplicate it
+                        ...(params && { params }),
                         timeout: 30000, // 30 second timeout
                     });
-                    return response.data;
+                    return response;
                 }
                 catch (error) {
                     // Enhanced error logging for debugging
@@ -290,6 +313,13 @@ class Doc2Vec {
                             throw error;
                         }
                     }
+                    else if (!isRetryableStatus(error.response?.status)) {
+                        // Deterministic failure (422, 404, 401, ...) — surface the
+                        // API's own message, which explains what to do instead.
+                        const apiMessage = error.response?.data?.message;
+                        logger.error(`GitHub API returned ${error.response.status} (not retryable): ${apiMessage || error.message}`);
+                        throw error;
+                    }
                     else {
                         // For non-403 errors, wait before retrying (exponential backoff)
                         if (attempt < retries - 1) {
@@ -306,23 +336,45 @@ class Doc2Vec {
             }
             throw new Error('Max retries reached');
         };
+        /**
+         * Walk every issue updated since `sinceDate` by following GitHub's
+         * `Link: rel="next"` cursor URLs.
+         *
+         * Page-number pagination (`?page=N`) cannot be used here: GitHub caps it
+         * on this endpoint and answers HTTP 422 ("Pagination with the page
+         * parameter is not supported for large datasets, please use cursor based
+         * pagination") once the offset grows past its limit. That limit is well
+         * below the size of a full-window rebuild on a busy repo, so page-based
+         * paging aborted the whole source and left the collection empty. The
+         * Link URL already contains the opaque `after=` cursor and is passed
+         * through verbatim.
+         */
         const fetchAllIssues = async (sinceDate) => {
             let issues = [];
-            let page = 1;
-            const perPage = 100;
-            while (true) {
+            // sort=updated&direction=asc makes the walk stable: results are
+            // ordered by the same field `since` filters on, so a page boundary
+            // can't shuffle items in or out mid-walk.
+            let url = GITHUB_API_URL;
+            let params = {
+                per_page: 100,
+                state: 'all',
+                since: sinceDate,
+                sort: 'updated',
+                direction: 'asc',
+            };
+            let page = 0;
+            while (url) {
+                page++;
                 // Log progress every 10 pages to reduce noise
                 if (page === 1 || page % 10 === 0) {
                     logger.debug(`Fetching issues page ${page}... (${issues.length} issues so far)`);
                 }
-                const data = await fetchWithRetry(GITHUB_API_URL, {
-                    per_page: perPage,
-                    page,
-                    state: 'all',
-                    since: sinceDate,
-                });
-                if (data.length === 0)
+                const response = await requestWithRetry(url, params);
+                const data = response.data;
+                if (!Array.isArray(data)) {
+                    logger.warn(`Unexpected non-array response while paginating issues for ${repo}; stopping the walk`);
                     break;
+                }
                 // The `since` param already scopes results to issues whose
                 // updated_at is after sinceDate — i.e. anything created, closed,
                 // reopened, edited, or newly commented since the last run. Do NOT
@@ -330,19 +382,45 @@ class Doc2Vec {
                 // received a new comment has an old created_at and would be
                 // dropped, leaving its status and comments permanently stale.
                 issues = issues.concat(data);
-                if (data.length < perPage)
+                const nextUrl = utils_1.Utils.parseNextLink(response.headers?.link ?? response.headers?.Link);
+                // Guard against a server echoing the same cursor forever
+                if (!nextUrl || nextUrl === url)
                     break;
-                page++;
+                url = nextUrl;
+                params = undefined; // the cursor URL is already fully-formed
             }
+            logger.info(`Fetched ${issues.length} issues/PRs for ${repo} across ${page} page(s)`);
             return issues;
         };
         const fetchIssueComments = async (issueNumber) => {
-            const url = `${GITHUB_API_URL}/${issueNumber}/comments`;
-            return await fetchWithRetry(url);
+            // GitHub returns 30 comments per page by default and paginates the
+            // rest behind Link headers; requesting the first page only silently
+            // truncated long discussions, so the stored chunks were missing the
+            // newest comments.
+            const comments = [];
+            let url = `${GITHUB_API_URL}/${issueNumber}/comments`;
+            let params = { per_page: 100 };
+            while (url) {
+                const response = await requestWithRetry(url, params);
+                if (!Array.isArray(response.data))
+                    break;
+                comments.push(...response.data);
+                const nextUrl = utils_1.Utils.parseNextLink(response.headers?.link ?? response.headers?.Link);
+                if (!nextUrl || nextUrl === url)
+                    break;
+                url = nextUrl;
+                params = undefined;
+            }
+            return comments;
         };
         const generateMarkdownForIssue = async (issue) => {
             const comments = await fetchIssueComments(issue.number);
-            let md = `# Issue #${issue.number}: ${issue.title}\n\n`;
+            // This endpoint returns both issues and pull requests; only PRs carry
+            // a `pull_request` object. Labelling every item "Issue #N" made
+            // retrieved PR chunks read as issues, so use the real kind.
+            const itemLabel = issue.pull_request ? 'PR' : 'Issue';
+            let md = `# ${itemLabel} #${issue.number}: ${issue.title}\n\n`;
+            md += `- **Type:** ${issue.pull_request ? 'Pull request' : 'Issue'}\n`;
             md += `- **Author:** ${issue.user.login}\n`;
             md += `- **State:** ${issue.state}\n`;
             md += `- **Created on:** ${new Date(issue.created_at).toDateString()}\n`;
@@ -359,6 +437,9 @@ class Doc2Vec {
             }
             return md;
         };
+        // Chunks that could not be stored (e.g. a rejected Qdrant upsert).
+        // Tallied across all issues and escalated once the walk completes.
+        let failedChunks = 0;
         // Process a single issue and store its chunks
         const processIssue = async (issue) => {
             const issueNumber = issue.number;
@@ -380,11 +461,19 @@ class Doc2Vec {
             // ones would otherwise linger — leaving stale state ("open") and missing
             // the latest comments in search results. All chunks for an issue share
             // its unique url, so delete-by-url reconciles them exactly.
+            let removedChunks = 0;
             if (dbConnection.type === 'sqlite') {
-                database_1.DatabaseManager.removeChunksByUrlSQLite(dbConnection.db, url, logger);
+                removedChunks = database_1.DatabaseManager.removeChunksByUrlSQLite(dbConnection.db, url, logger);
             }
             else if (dbConnection.type === 'qdrant') {
-                await database_1.DatabaseManager.removeChunksByUrlQdrant(dbConnection, url, logger);
+                removedChunks = await database_1.DatabaseManager.removeChunksByUrlQdrant(dbConnection, url, logger);
+            }
+            if (removedChunks > 0) {
+                this.counters.items_updated++;
+                this.counters.chunks_deleted += removedChunks;
+            }
+            else {
+                this.counters.items_new++;
             }
             // Process and store each chunk immediately
             for (const chunk of chunks) {
@@ -400,6 +489,7 @@ class Doc2Vec {
                     const embeddings = await this.createEmbeddings([chunk.content]);
                     if (embeddings.length) {
                         database_1.DatabaseManager.insertVectorsSQLite(dbConnection.db, chunk, embeddings[0], logger, chunkHash);
+                        this.counters.chunks_added++;
                         logger.debug(`Stored chunk ${chunkId} in SQLite`);
                     }
                     else {
@@ -430,6 +520,7 @@ class Doc2Vec {
                         const embeddings = await this.createEmbeddings([chunk.content]);
                         if (embeddings.length) {
                             await database_1.DatabaseManager.storeChunkInQdrant(dbConnection, chunk, embeddings[0], chunkHash);
+                            this.counters.chunks_added++;
                             logger.debug(`Stored chunk ${chunkId} in Qdrant (${dbConnection.collectionName})`);
                         }
                         else {
@@ -437,7 +528,11 @@ class Doc2Vec {
                         }
                     }
                     catch (error) {
-                        logger.error(`Error processing chunk in Qdrant:`, error);
+                        // Keep going so one bad chunk can't cost the whole repo,
+                        // but remember it: the count is escalated below so the run
+                        // fails and the last-run date is not advanced.
+                        failedChunks++;
+                        logger.error(`Error processing chunk ${chunkId} for ${url} in Qdrant:`, error);
                     }
                 }
             }
@@ -450,6 +545,12 @@ class Doc2Vec {
             logger.info(`Processing issue ${i + 1}/${issues.length}`);
             await processIssue(issues[i]);
         }
+        // A chunk that failed to store is silent data loss, so refuse to advance
+        // the last-run date: the next run reprocesses this window instead of
+        // treating the gap as synced. Everything that did store is kept.
+        if (failedChunks > 0) {
+            throw new Error(`${failedChunks} chunk(s) failed to store for ${repo}; not advancing the last-run date so the next run retries them`);
+        }
         // Update the last run date in the database after processing all issues
         await database_1.DatabaseManager.updateLastRunDate(dbConnection, repo, logger, this.embeddingDimension, syncStartDate);
         logger.info(`Successfully processed ${issues.length} issues`);
@@ -579,11 +680,16 @@ class Doc2Vec {
                 }
                 // Vector DB chunks
                 try {
+                    let removedChunks = 0;
                     if (dbConnection.type === 'sqlite') {
-                        database_1.DatabaseManager.removeChunksByUrlSQLite(dbConnection.db, url, logger);
+                        removedChunks = database_1.DatabaseManager.removeChunksByUrlSQLite(dbConnection.db, url, logger);
                     }
                     else if (dbConnection.type === 'qdrant') {
-                        await database_1.DatabaseManager.removeChunksByUrlQdrant(dbConnection, url, logger);
+                        removedChunks = await database_1.DatabaseManager.removeChunksByUrlQdrant(dbConnection, url, logger);
+                    }
+                    if (removedChunks > 0) {
+                        this.counters.items_deleted++;
+                        this.counters.chunks_deleted += removedChunks;
                     }
                 }
                 catch (dbError) {
@@ -603,14 +709,17 @@ class Doc2Vec {
                 await database_1.DatabaseManager.setMetadataValue(dbConnection, syncCompleteKey, 'true', logger, this.embeddingDimension);
                 logger.info('Full sync completed successfully — subsequent runs will use normal caching');
             }
+            let removed = { items: 0, chunks: 0 };
             if (dbConnection.type === 'sqlite') {
                 logger.info(`Running SQLite cleanup for ${urlPrefix}`);
-                database_1.DatabaseManager.removeObsoleteChunksSQLite(dbConnection.db, visitedUrls, urlPrefix, logger);
+                removed = database_1.DatabaseManager.removeObsoleteChunksSQLite(dbConnection.db, visitedUrls, urlPrefix, logger);
             }
             else if (dbConnection.type === 'qdrant') {
                 logger.info(`Running Qdrant cleanup for ${urlPrefix} in collection ${dbConnection.collectionName}`);
-                await database_1.DatabaseManager.removeObsoleteChunksQdrant(dbConnection, visitedUrls, urlPrefix, logger);
+                removed = await database_1.DatabaseManager.removeObsoleteChunksQdrant(dbConnection, visitedUrls, urlPrefix, logger);
             }
+            this.counters.items_deleted += removed.items;
+            this.counters.chunks_deleted += removed.chunks;
         }
         logger.info(`Finished processing website: ${config.url}`);
     }
@@ -698,14 +807,17 @@ class Doc2Vec {
             }
         }, logger);
         logger.section('CLEANUP');
+        let removed = { items: 0, chunks: 0 };
         if (dbConnection.type === 'sqlite') {
             logger.info(`Running SQLite cleanup for local directory ${config.path}`);
-            database_1.DatabaseManager.removeObsoleteFilesSQLite(dbConnection.db, processedFiles, config, logger);
+            removed = database_1.DatabaseManager.removeObsoleteFilesSQLite(dbConnection.db, processedFiles, config, logger);
         }
         else if (dbConnection.type === 'qdrant') {
             logger.info(`Running Qdrant cleanup for local directory ${config.path} in collection ${dbConnection.collectionName}`);
-            await database_1.DatabaseManager.removeObsoleteFilesQdrant(dbConnection, processedFiles, config, logger);
+            removed = await database_1.DatabaseManager.removeObsoleteFilesQdrant(dbConnection, processedFiles, config, logger);
         }
+        this.counters.items_deleted += removed.items;
+        this.counters.chunks_deleted += removed.chunks;
         logger.info(`Finished processing local directory: ${config.path}`);
     }
     async processS3(config, parentLogger) {
@@ -886,11 +998,16 @@ class Doc2Vec {
                 else {
                     fileUrl = `s3://${config.bucket}/${deletedKey}`;
                 }
+                let removedChunks = 0;
                 if (dbConnection.type === 'sqlite') {
-                    database_1.DatabaseManager.removeChunksByUrlSQLite(dbConnection.db, fileUrl, logger);
+                    removedChunks = database_1.DatabaseManager.removeChunksByUrlSQLite(dbConnection.db, fileUrl, logger);
                 }
                 else if (dbConnection.type === 'qdrant') {
-                    await database_1.DatabaseManager.removeChunksByUrlQdrant(dbConnection, fileUrl, logger);
+                    removedChunks = await database_1.DatabaseManager.removeChunksByUrlQdrant(dbConnection, fileUrl, logger);
+                }
+                if (removedChunks > 0) {
+                    this.counters.items_deleted++;
+                    this.counters.chunks_deleted += removedChunks;
                 }
             }
         }
@@ -1025,11 +1142,16 @@ class Doc2Vec {
                 if (deleteUrls.length > 0) {
                     logger.info(`Cleaning up ${deleteUrls.length} deleted/renamed files`);
                     for (const url of deleteUrls) {
+                        let removedChunks = 0;
                         if (dbConnection.type === 'sqlite') {
-                            database_1.DatabaseManager.removeChunksByUrlSQLite(dbConnection.db, url, logger);
+                            removedChunks = database_1.DatabaseManager.removeChunksByUrlSQLite(dbConnection.db, url, logger);
                         }
                         else if (dbConnection.type === 'qdrant') {
-                            await database_1.DatabaseManager.removeChunksByUrlQdrant(dbConnection, url, logger);
+                            removedChunks = await database_1.DatabaseManager.removeChunksByUrlQdrant(dbConnection, url, logger);
+                        }
+                        if (removedChunks > 0) {
+                            this.counters.items_deleted++;
+                            this.counters.chunks_deleted += removedChunks;
                         }
                     }
                 }
@@ -1043,11 +1165,16 @@ class Doc2Vec {
                     const deletedFiles = previousList.filter((filePath) => !trackedFiles?.has(filePath));
                     for (const deletedFile of deletedFiles) {
                         const url = this.buildCodeFileUrl(deletedFile, basePath, config, repoUrlPrefix);
+                        let removedChunks = 0;
                         if (dbConnection.type === 'sqlite') {
-                            database_1.DatabaseManager.removeChunksByUrlSQLite(dbConnection.db, url, logger);
+                            removedChunks = database_1.DatabaseManager.removeChunksByUrlSQLite(dbConnection.db, url, logger);
                         }
                         else if (dbConnection.type === 'qdrant') {
-                            await database_1.DatabaseManager.removeChunksByUrlQdrant(dbConnection, url, logger);
+                            removedChunks = await database_1.DatabaseManager.removeChunksByUrlQdrant(dbConnection, url, logger);
+                        }
+                        if (removedChunks > 0) {
+                            this.counters.items_deleted++;
+                            this.counters.chunks_deleted += removedChunks;
                         }
                     }
                     await database_1.DatabaseManager.setMetadataValue(dbConnection, fileListKey, JSON.stringify(currentList), logger, this.embeddingDimension);
@@ -1058,14 +1185,17 @@ class Doc2Vec {
                 }
             }
             else {
+                let removed = { items: 0, chunks: 0 };
                 if (dbConnection.type === 'sqlite') {
                     logger.info(`Running SQLite cleanup for code source ${basePath}`);
-                    database_1.DatabaseManager.removeObsoleteFilesSQLite(dbConnection.db, processedFiles, cleanupPathConfig, logger);
+                    removed = database_1.DatabaseManager.removeObsoleteFilesSQLite(dbConnection.db, processedFiles, cleanupPathConfig, logger);
                 }
                 else if (dbConnection.type === 'qdrant') {
                     logger.info(`Running Qdrant cleanup for code source ${basePath} in collection ${dbConnection.collectionName}`);
-                    await database_1.DatabaseManager.removeObsoleteFilesQdrant(dbConnection, processedFiles, cleanupPathConfig, logger);
+                    removed = await database_1.DatabaseManager.removeObsoleteFilesQdrant(dbConnection, processedFiles, cleanupPathConfig, logger);
                 }
+                this.counters.items_deleted += removed.items;
+                this.counters.chunks_deleted += removed.chunks;
             }
             if (config.source === 'github' && basePath && repoBranch) {
                 const headSha = await this.getRepoHeadSha(basePath, logger);
@@ -1333,11 +1463,16 @@ class Doc2Vec {
             // Deleted tickets — remove their chunks and stop
             if (ticket.status === 'deleted') {
                 logger.info(`Ticket #${ticketId} was deleted in Zendesk — removing its chunks`);
+                let removedChunks = 0;
                 if (dbConnection.type === 'sqlite') {
-                    database_1.DatabaseManager.removeChunksByUrlSQLite(dbConnection.db, url, logger);
+                    removedChunks = database_1.DatabaseManager.removeChunksByUrlSQLite(dbConnection.db, url, logger);
                 }
                 else {
-                    await database_1.DatabaseManager.removeChunksByUrlQdrant(dbConnection, url, logger);
+                    removedChunks = await database_1.DatabaseManager.removeChunksByUrlQdrant(dbConnection, url, logger);
+                }
+                if (removedChunks > 0) {
+                    this.counters.items_deleted++;
+                    this.counters.chunks_deleted += removedChunks;
                 }
                 return;
             }
@@ -1599,6 +1734,7 @@ class Doc2Vec {
             newHashesSorted.every((h, i) => h === existingHashesSorted[i]);
         if (unchanged) {
             logger.info(`Skipping unchanged URL (${chunks.length} chunks): ${url}`);
+            this.counters.items_unchanged++;
             return 0;
         }
         // 4. Content changed — delete all existing chunks for this URL
@@ -1610,6 +1746,11 @@ class Doc2Vec {
             else {
                 await database_1.DatabaseManager.removeChunksByUrlQdrant(dbConnection, url, logger);
             }
+            this.counters.items_updated++;
+            this.counters.chunks_deleted += existingHashesSorted.length;
+        }
+        else {
+            this.counters.items_new++;
         }
         // 5. Embed and insert all new chunks
         let embeddedCount = 0;
@@ -1637,6 +1778,7 @@ class Doc2Vec {
             }
         }
         chunkProgress.complete();
+        this.counters.chunks_added += embeddedCount;
         return embeddedCount;
     }
     async createEmbeddings(texts) {