doc2vec 2.10.6 → 2.12.0

package/dist/database.js

@@ -71,6 +71,7 @@ class DatabaseManager {
                     total_chunks INTEGER
                 );
             `);
+            this.ensureChunkDatesTable(db);
             logger.info(`SQLite database initialized successfully`);
             return { db, type: 'sqlite' };
         }
@@ -307,9 +308,24 @@ class DatabaseManager {
             logger.error(`Failed to update last run date for ${repo}:`, error);
         }
     }
+    /**
+     * Chunk creation timestamps live in a companion table because vec_items is
+     * a vec0 virtual table that cannot be ALTERed to add a column. Rows are
+     * written on every chunk insert/update; stale rows for deleted chunks are
+     * harmless (lookups always join against vec_items).
+     */
+    static ensureChunkDatesTable(db) {
+        db.exec(`
+            CREATE TABLE IF NOT EXISTS vec_chunk_dates (
+                chunk_id   TEXT PRIMARY KEY,
+                created_at TEXT NOT NULL
+            );
+        `);
+    }
     static prepareSQLiteStatements(db) {
         let cached = this.columnCache.get(db);
         if (!cached) {
+            this.ensureChunkDatesTable(db);
             cached = {
                 hasBranch: this.hasColumn(db, 'branch'),
                 hasRepo: this.hasColumn(db, 'repo')
@@ -360,12 +376,16 @@ class DatabaseManager {
             `),
             getAllChunkIdsStmt: db.prepare(`SELECT chunk_id FROM vec_items`),
             deleteChunkStmt: db.prepare(`DELETE FROM vec_items WHERE chunk_id = ?`),
+            touchDateStmt: db.prepare(`
+                INSERT INTO vec_chunk_dates (chunk_id, created_at) VALUES (?, ?)
+                ON CONFLICT(chunk_id) DO UPDATE SET created_at = excluded.created_at
+            `),
             hasBranchColumn,
             hasRepoColumn
         };
     }
     static insertVectorsSQLite(db, chunk, embedding, logger, chunkHash) {
-        const { insertStmt, updateStmt, hasBranchColumn, hasRepoColumn } = this.prepareSQLiteStatements(db);
+        const { insertStmt, updateStmt, touchDateStmt, hasBranchColumn, hasRepoColumn } = this.prepareSQLiteStatements(db);
         const hash = chunkHash || utils_1.Utils.generateHash(chunk.content);
         const transaction = db.transaction(() => {
             // Use BigInt for true integer representation in SQLite vec0
@@ -401,6 +421,7 @@ class DatabaseManager {
                 updateValues.push(JSON.stringify(chunk.metadata.heading_hierarchy), chunk.metadata.section, chunk.content, chunk.metadata.url, hash, chunkIndex, totalChunks, chunk.metadata.chunk_id);
                 updateStmt.run(...updateValues);
             }
+            touchDateStmt.run(chunk.metadata.chunk_id, new Date().toISOString());
         });
         transaction();
     }
@@ -434,6 +455,7 @@ class DatabaseManager {
                     original_chunk_id: chunk.metadata.chunk_id,
                     chunk_index: chunk.metadata.chunk_index,
                     total_chunks: chunk.metadata.total_chunks,
+                    created_at: new Date().toISOString(),
                 },
             };
             await client.upsert(collectionName, {
@@ -457,17 +479,20 @@ class DatabaseManager {
         const deleteChunkStmt = db.prepare(`DELETE FROM vec_items WHERE chunk_id = ?`);
         const existingChunks = getChunksForUrlStmt.all(urlPrefix);
         let deletedCount = 0;
+        const deletedUrls = new Set();
         const transaction = db.transaction(() => {
             for (const { chunk_id, url } of existingChunks) {
                 if (!visitedUrls.has(url)) {
                     logger.debug(`Deleting obsolete chunk from SQLite: ${chunk_id.substring(0, 8)}... (URL not visited)`);
                     deleteChunkStmt.run(chunk_id);
                     deletedCount++;
+                    deletedUrls.add(url);
                 }
             }
         });
         transaction();
         logger.info(`Deleted ${deletedCount} obsolete chunks from SQLite for URL ${urlPrefix}`);
+        return { items: deletedUrls.size, chunks: deletedCount };
     }
     static async removeObsoleteChunksQdrant(db, visitedUrls, urlPrefix, logger) {
         const { client, collectionName } = db;
@@ -496,6 +521,7 @@ class DatabaseManager {
                 ]
             };
             const obsoletePointIds = [];
+            const obsoleteUrls = new Set();
             let offset = undefined;
             do {
                 const response = await client.scroll(collectionName, {
@@ -513,6 +539,7 @@ class DatabaseManager {
                     }
                     if (url && !visitedUrls.has(url)) {
                         obsoletePointIds.push(point.id);
+                        obsoleteUrls.add(url);
                     }
                 }
                 offset = response.next_page_offset;
@@ -526,6 +553,7 @@ class DatabaseManager {
             else {
                 logger.info(`No obsolete chunks to delete from Qdrant for URL ${urlPrefix}`);
             }
+            return { items: obsoleteUrls.size, chunks: obsoletePointIds.length };
         }
         catch (error) {
             // Rethrow so a failed cleanup fails the job instead of silently
@@ -548,31 +576,42 @@ class DatabaseManager {
         const deleteStmt = db.prepare(`DELETE FROM vec_items WHERE url = ?`);
         const result = deleteStmt.run(url);
         logger.info(`Deleted ${result.changes} chunks from SQLite for URL ${url}`);
+        return result.changes;
     }
     static async removeChunksByUrlQdrant(db, url, logger) {
         const { client, collectionName } = db;
         try {
-            await client.delete(collectionName, {
-                filter: {
-                    must: [
-                        {
-                            key: 'url',
-                            match: {
-                                value: url // Exact match — must match getChunkHashesByUrlQdrant's filter
-                            }
+            const filter = {
+                must: [
+                    {
+                        key: 'url',
+                        match: {
+                            value: url // Exact match — must match getChunkHashesByUrlQdrant's filter
                         }
-                    ],
-                    must_not: [
-                        {
-                            key: 'is_metadata',
-                            match: {
-                                value: true
-                            }
+                    }
+                ],
+                must_not: [
+                    {
+                        key: 'is_metadata',
+                        match: {
+                            value: true
                         }
-                    ]
-                }
-            });
+                    }
+                ]
+            };
+            // Count first so callers can report how many chunks the delete removed
+            // (Qdrant's delete-by-filter response doesn't include a count).
+            let deletedCount = 0;
+            try {
+                const countResult = await client.count(collectionName, { filter, exact: true });
+                deletedCount = countResult?.count ?? 0;
+            }
+            catch {
+                // Count is informational only — never fail the delete over it
+            }
+            await client.delete(collectionName, { filter });
             logger.info(`Deleted chunks from Qdrant for URL ${url}`);
+            return deletedCount;
         }
         catch (error) {
             // Rethrow so the failure propagates and the job exits non-zero.
@@ -697,6 +736,7 @@ class DatabaseManager {
         logger.debug(`Searching for chunks with URL prefix: ${urlPrefix}`);
         const existingChunks = getChunksForPathStmt.all(urlPrefix);
         let deletedCount = 0;
+        const deletedUrls = new Set();
         const transaction = db.transaction(() => {
             for (const { chunk_id, url } of existingChunks) {
                 // Skip if it's not from our URL prefix (safety check)
@@ -720,11 +760,13 @@ class DatabaseManager {
                     logger.debug(`Deleting obsolete chunk from SQLite: ${chunk_id.substring(0, 8)}... (File not processed: ${filePath})`);
                     deleteChunkStmt.run(chunk_id);
                     deletedCount++;
+                    deletedUrls.add(url);
                 }
             }
         });
         transaction();
         logger.info(`Deleted ${deletedCount} obsolete chunks from SQLite for URL prefix ${urlPrefix}`);
+        return { items: deletedUrls.size, chunks: deletedCount };
     }
     static async removeObsoleteFilesQdrant(db, processedFiles, pathConfig, logger) {
         const { client, collectionName } = db;
@@ -768,6 +810,7 @@ class DatabaseManager {
                 ]
             };
             const obsoletePointIds = [];
+            const obsoleteUrls = new Set();
             let offset = undefined;
             do {
                 const response = await client.scroll(collectionName, {
@@ -799,6 +842,7 @@ class DatabaseManager {
                     }
                     if (filePath && !processedFiles.has(filePath)) {
                         obsoletePointIds.push(point.id);
+                        obsoleteUrls.add(url);
                     }
                 }
                 offset = response.next_page_offset;
@@ -812,6 +856,7 @@ class DatabaseManager {
             else {
                 logger.info(`No obsolete chunks to delete from Qdrant for URL prefix ${urlPrefix}`);
             }
+            return { items: obsoleteUrls.size, chunks: obsoletePointIds.length };
         }
         catch (error) {
             // Rethrow so a failed cleanup fails the job instead of silently

package/dist/doc2vec.js

@@ -53,6 +53,7 @@ const logger_1 = require("./logger");
 const utils_1 = require("./utils");
 const database_1 = require("./database");
 const content_processor_1 = require("./content-processor");
+const types_1 = require("./types");
 const client_s3_1 = require("@aws-sdk/client-s3");
 const markdown_store_1 = require("./markdown-store");
 const GITHUB_TOKEN = process.env.GITHUB_PERSONAL_ACCESS_TOKEN;
@@ -61,6 +62,9 @@ dotenv.config();
 class Doc2Vec {
     constructor(configPath) {
         this.brokenLinksByWebsite = {};
+        // Change counters for the source currently being processed by run().
+        // Sources are processed sequentially, so a single slot is sufficient.
+        this.counters = (0, types_1.newSourceRunCounters)('items');
         this.logger = new logger_1.Logger('Doc2Vec', {
             level: logger_1.LogLevel.DEBUG,
             useTimestamp: true,
@@ -176,36 +180,69 @@ class Doc2Vec {
             await this.markdownStore.init();
         }
         this.logger.section('PROCESSING SOURCES');
+        const runStats = [];
         for (const sourceConfig of this.config.sources) {
             const sourceLogger = this.logger.child(`source:${sourceConfig.product_name}`);
             sourceLogger.info(`Processing ${sourceConfig.type} source for ${sourceConfig.product_name}@${sourceConfig.version}`);
-            if (sourceConfig.type === 'github') {
-                await this.processGithubRepo(sourceConfig, sourceLogger);
-            }
-            else if (sourceConfig.type === 'website') {
-                await this.processWebsite(sourceConfig, sourceLogger);
-            }
-            else if (sourceConfig.type === 'local_directory') {
-                await this.processLocalDirectory(sourceConfig, sourceLogger);
-            }
-            else if (sourceConfig.type === 'code') {
-                await this.processCodeSource(sourceConfig, sourceLogger);
-            }
-            else if (sourceConfig.type === 'zendesk') {
-                await this.processZendesk(sourceConfig, sourceLogger);
-            }
-            else if (sourceConfig.type === 's3') {
-                await this.processS3(sourceConfig, sourceLogger);
-            }
-            else {
-                sourceLogger.error(`Unknown source type: ${sourceConfig.type}`);
+            const startTime = Date.now();
+            let ok = true;
+            let errorMessage;
+            const itemsKindByType = {
+                website: 'pages',
+                github: 'issues',
+                local_directory: 'files',
+                code: 'files',
+                s3: 'objects',
+                zendesk: 'items',
+            };
+            this.counters = (0, types_1.newSourceRunCounters)(itemsKindByType[sourceConfig.type] ?? 'items');
+            try {
+                if (sourceConfig.type === 'github') {
+                    await this.processGithubRepo(sourceConfig, sourceLogger);
+                }
+                else if (sourceConfig.type === 'website') {
+                    await this.processWebsite(sourceConfig, sourceLogger);
+                }
+                else if (sourceConfig.type === 'local_directory') {
+                    await this.processLocalDirectory(sourceConfig, sourceLogger);
+                }
+                else if (sourceConfig.type === 'code') {
+                    await this.processCodeSource(sourceConfig, sourceLogger);
+                }
+                else if (sourceConfig.type === 'zendesk') {
+                    await this.processZendesk(sourceConfig, sourceLogger);
+                }
+                else if (sourceConfig.type === 's3') {
+                    await this.processS3(sourceConfig, sourceLogger);
+                }
+                else {
+                    ok = false;
+                    errorMessage = `Unknown source type: ${sourceConfig.type}`;
+                    sourceLogger.error(errorMessage);
+                }
             }
+            catch (error) {
+                ok = false;
+                errorMessage = error instanceof Error ? error.message : String(error);
+                sourceLogger.error(`Failed to process ${sourceConfig.type} source for ${sourceConfig.product_name}:`, error);
+            }
+            runStats.push({
+                product_name: sourceConfig.product_name,
+                type: sourceConfig.type,
+                version: sourceConfig.version,
+                duration_ms: Date.now() - startTime,
+                ok,
+                ...(errorMessage && { error: errorMessage }),
+                counters: this.counters
+            });
         }
         // Close the Postgres markdown store connection pool
         if (this.markdownStore) {
             await this.markdownStore.close();
         }
         this.logger.section('PROCESSING COMPLETE');
+        this.logger.event('run-summary', { sources: runStats });
+        return runStats;
     }
     async fetchAndProcessGitHubIssues(repo, sourceConfig, dbConnection, logger) {
         const [owner, repoName] = repo.split('/');
@@ -357,11 +394,19 @@ class Doc2Vec {
             // ones would otherwise linger — leaving stale state ("open") and missing
             // the latest comments in search results. All chunks for an issue share
             // its unique url, so delete-by-url reconciles them exactly.
+            let removedChunks = 0;
             if (dbConnection.type === 'sqlite') {
-                database_1.DatabaseManager.removeChunksByUrlSQLite(dbConnection.db, url, logger);
+                removedChunks = database_1.DatabaseManager.removeChunksByUrlSQLite(dbConnection.db, url, logger);
             }
             else if (dbConnection.type === 'qdrant') {
-                await database_1.DatabaseManager.removeChunksByUrlQdrant(dbConnection, url, logger);
+                removedChunks = await database_1.DatabaseManager.removeChunksByUrlQdrant(dbConnection, url, logger);
+            }
+            if (removedChunks > 0) {
+                this.counters.items_updated++;
+                this.counters.chunks_deleted += removedChunks;
+            }
+            else {
+                this.counters.items_new++;
             }
             // Process and store each chunk immediately
             for (const chunk of chunks) {
@@ -377,6 +422,7 @@ class Doc2Vec {
                     const embeddings = await this.createEmbeddings([chunk.content]);
                     if (embeddings.length) {
                         database_1.DatabaseManager.insertVectorsSQLite(dbConnection.db, chunk, embeddings[0], logger, chunkHash);
+                        this.counters.chunks_added++;
                         logger.debug(`Stored chunk ${chunkId} in SQLite`);
                     }
                     else {
@@ -407,6 +453,7 @@ class Doc2Vec {
                         const embeddings = await this.createEmbeddings([chunk.content]);
                         if (embeddings.length) {
                             await database_1.DatabaseManager.storeChunkInQdrant(dbConnection, chunk, embeddings[0], chunkHash);
+                            this.counters.chunks_added++;
                             logger.debug(`Stored chunk ${chunkId} in Qdrant (${dbConnection.collectionName})`);
                         }
                         else {
@@ -556,11 +603,16 @@ class Doc2Vec {
                 }
                 // Vector DB chunks
                 try {
+                    let removedChunks = 0;
                     if (dbConnection.type === 'sqlite') {
-                        database_1.DatabaseManager.removeChunksByUrlSQLite(dbConnection.db, url, logger);
+                        removedChunks = database_1.DatabaseManager.removeChunksByUrlSQLite(dbConnection.db, url, logger);
                     }
                     else if (dbConnection.type === 'qdrant') {
-                        await database_1.DatabaseManager.removeChunksByUrlQdrant(dbConnection, url, logger);
+                        removedChunks = await database_1.DatabaseManager.removeChunksByUrlQdrant(dbConnection, url, logger);
+                    }
+                    if (removedChunks > 0) {
+                        this.counters.items_deleted++;
+                        this.counters.chunks_deleted += removedChunks;
                     }
                 }
                 catch (dbError) {
@@ -580,14 +632,17 @@ class Doc2Vec {
                 await database_1.DatabaseManager.setMetadataValue(dbConnection, syncCompleteKey, 'true', logger, this.embeddingDimension);
                 logger.info('Full sync completed successfully — subsequent runs will use normal caching');
             }
+            let removed = { items: 0, chunks: 0 };
             if (dbConnection.type === 'sqlite') {
                 logger.info(`Running SQLite cleanup for ${urlPrefix}`);
-                database_1.DatabaseManager.removeObsoleteChunksSQLite(dbConnection.db, visitedUrls, urlPrefix, logger);
+                removed = database_1.DatabaseManager.removeObsoleteChunksSQLite(dbConnection.db, visitedUrls, urlPrefix, logger);
             }
             else if (dbConnection.type === 'qdrant') {
                 logger.info(`Running Qdrant cleanup for ${urlPrefix} in collection ${dbConnection.collectionName}`);
-                await database_1.DatabaseManager.removeObsoleteChunksQdrant(dbConnection, visitedUrls, urlPrefix, logger);
+                removed = await database_1.DatabaseManager.removeObsoleteChunksQdrant(dbConnection, visitedUrls, urlPrefix, logger);
             }
+            this.counters.items_deleted += removed.items;
+            this.counters.chunks_deleted += removed.chunks;
         }
         logger.info(`Finished processing website: ${config.url}`);
     }
@@ -608,7 +663,11 @@ class Doc2Vec {
         this.brokenLinksByWebsite[baseUrl] = unique;
     }
     writeBrokenLinksReport() {
-        const reportPath = path.join(this.configDir, '404.yaml');
+        // The default (next to the config file) breaks when the config lives on
+        // a read-only mount (e.g. a Kubernetes ConfigMap in controller mode) —
+        // DOC2VEC_REPORT_DIR points the report somewhere writable instead.
+        const reportDir = process.env.DOC2VEC_REPORT_DIR || this.configDir;
+        const reportPath = path.join(reportDir, '404.yaml');
         const orderedEntries = Object.entries(this.brokenLinksByWebsite)
             .sort(([a], [b]) => a.localeCompare(b));
         const reportPayload = orderedEntries.map(([website, links]) => ({
@@ -671,14 +730,17 @@ class Doc2Vec {
             }
         }, logger);
         logger.section('CLEANUP');
+        let removed = { items: 0, chunks: 0 };
         if (dbConnection.type === 'sqlite') {
             logger.info(`Running SQLite cleanup for local directory ${config.path}`);
-            database_1.DatabaseManager.removeObsoleteFilesSQLite(dbConnection.db, processedFiles, config, logger);
+            removed = database_1.DatabaseManager.removeObsoleteFilesSQLite(dbConnection.db, processedFiles, config, logger);
         }
         else if (dbConnection.type === 'qdrant') {
             logger.info(`Running Qdrant cleanup for local directory ${config.path} in collection ${dbConnection.collectionName}`);
-            await database_1.DatabaseManager.removeObsoleteFilesQdrant(dbConnection, processedFiles, config, logger);
+            removed = await database_1.DatabaseManager.removeObsoleteFilesQdrant(dbConnection, processedFiles, config, logger);
         }
+        this.counters.items_deleted += removed.items;
+        this.counters.chunks_deleted += removed.chunks;
         logger.info(`Finished processing local directory: ${config.path}`);
     }
     async processS3(config, parentLogger) {
@@ -859,11 +921,16 @@ class Doc2Vec {
                 else {
                     fileUrl = `s3://${config.bucket}/${deletedKey}`;
                 }
+                let removedChunks = 0;
                 if (dbConnection.type === 'sqlite') {
-                    database_1.DatabaseManager.removeChunksByUrlSQLite(dbConnection.db, fileUrl, logger);
+                    removedChunks = database_1.DatabaseManager.removeChunksByUrlSQLite(dbConnection.db, fileUrl, logger);
                 }
                 else if (dbConnection.type === 'qdrant') {
-                    await database_1.DatabaseManager.removeChunksByUrlQdrant(dbConnection, fileUrl, logger);
+                    removedChunks = await database_1.DatabaseManager.removeChunksByUrlQdrant(dbConnection, fileUrl, logger);
+                }
+                if (removedChunks > 0) {
+                    this.counters.items_deleted++;
+                    this.counters.chunks_deleted += removedChunks;
                 }
             }
         }
@@ -998,11 +1065,16 @@ class Doc2Vec {
                 if (deleteUrls.length > 0) {
                     logger.info(`Cleaning up ${deleteUrls.length} deleted/renamed files`);
                     for (const url of deleteUrls) {
+                        let removedChunks = 0;
                         if (dbConnection.type === 'sqlite') {
-                            database_1.DatabaseManager.removeChunksByUrlSQLite(dbConnection.db, url, logger);
+                            removedChunks = database_1.DatabaseManager.removeChunksByUrlSQLite(dbConnection.db, url, logger);
                         }
                         else if (dbConnection.type === 'qdrant') {
-                            await database_1.DatabaseManager.removeChunksByUrlQdrant(dbConnection, url, logger);
+                            removedChunks = await database_1.DatabaseManager.removeChunksByUrlQdrant(dbConnection, url, logger);
+                        }
+                        if (removedChunks > 0) {
+                            this.counters.items_deleted++;
+                            this.counters.chunks_deleted += removedChunks;
                         }
                     }
                 }
@@ -1016,11 +1088,16 @@ class Doc2Vec {
                     const deletedFiles = previousList.filter((filePath) => !trackedFiles?.has(filePath));
                     for (const deletedFile of deletedFiles) {
                         const url = this.buildCodeFileUrl(deletedFile, basePath, config, repoUrlPrefix);
+                        let removedChunks = 0;
                         if (dbConnection.type === 'sqlite') {
-                            database_1.DatabaseManager.removeChunksByUrlSQLite(dbConnection.db, url, logger);
+                            removedChunks = database_1.DatabaseManager.removeChunksByUrlSQLite(dbConnection.db, url, logger);
                         }
                         else if (dbConnection.type === 'qdrant') {
-                            await database_1.DatabaseManager.removeChunksByUrlQdrant(dbConnection, url, logger);
+                            removedChunks = await database_1.DatabaseManager.removeChunksByUrlQdrant(dbConnection, url, logger);
+                        }
+                        if (removedChunks > 0) {
+                            this.counters.items_deleted++;
+                            this.counters.chunks_deleted += removedChunks;
                         }
                     }
                     await database_1.DatabaseManager.setMetadataValue(dbConnection, fileListKey, JSON.stringify(currentList), logger, this.embeddingDimension);
@@ -1031,14 +1108,17 @@ class Doc2Vec {
                 }
             }
             else {
+                let removed = { items: 0, chunks: 0 };
                 if (dbConnection.type === 'sqlite') {
                     logger.info(`Running SQLite cleanup for code source ${basePath}`);
-                    database_1.DatabaseManager.removeObsoleteFilesSQLite(dbConnection.db, processedFiles, cleanupPathConfig, logger);
+                    removed = database_1.DatabaseManager.removeObsoleteFilesSQLite(dbConnection.db, processedFiles, cleanupPathConfig, logger);
                 }
                 else if (dbConnection.type === 'qdrant') {
                     logger.info(`Running Qdrant cleanup for code source ${basePath} in collection ${dbConnection.collectionName}`);
-                    await database_1.DatabaseManager.removeObsoleteFilesQdrant(dbConnection, processedFiles, cleanupPathConfig, logger);
+                    removed = await database_1.DatabaseManager.removeObsoleteFilesQdrant(dbConnection, processedFiles, cleanupPathConfig, logger);
                 }
+                this.counters.items_deleted += removed.items;
+                this.counters.chunks_deleted += removed.chunks;
             }
             if (config.source === 'github' && basePath && repoBranch) {
                 const headSha = await this.getRepoHeadSha(basePath, logger);
@@ -1306,11 +1386,16 @@ class Doc2Vec {
             // Deleted tickets — remove their chunks and stop
             if (ticket.status === 'deleted') {
                 logger.info(`Ticket #${ticketId} was deleted in Zendesk — removing its chunks`);
+                let removedChunks = 0;
                 if (dbConnection.type === 'sqlite') {
-                    database_1.DatabaseManager.removeChunksByUrlSQLite(dbConnection.db, url, logger);
+                    removedChunks = database_1.DatabaseManager.removeChunksByUrlSQLite(dbConnection.db, url, logger);
                 }
                 else {
-                    await database_1.DatabaseManager.removeChunksByUrlQdrant(dbConnection, url, logger);
+                    removedChunks = await database_1.DatabaseManager.removeChunksByUrlQdrant(dbConnection, url, logger);
+                }
+                if (removedChunks > 0) {
+                    this.counters.items_deleted++;
+                    this.counters.chunks_deleted += removedChunks;
                 }
                 return;
             }
@@ -1572,6 +1657,7 @@ class Doc2Vec {
             newHashesSorted.every((h, i) => h === existingHashesSorted[i]);
         if (unchanged) {
             logger.info(`Skipping unchanged URL (${chunks.length} chunks): ${url}`);
+            this.counters.items_unchanged++;
             return 0;
         }
         // 4. Content changed — delete all existing chunks for this URL
@@ -1583,6 +1669,11 @@ class Doc2Vec {
             else {
                 await database_1.DatabaseManager.removeChunksByUrlQdrant(dbConnection, url, logger);
             }
+            this.counters.items_updated++;
+            this.counters.chunks_deleted += existingHashesSorted.length;
+        }
+        else {
+            this.counters.items_new++;
         }
         // 5. Embed and insert all new chunks
         let embeddedCount = 0;
@@ -1610,6 +1701,7 @@ class Doc2Vec {
             }
         }
         chunkProgress.complete();
+        this.counters.chunks_added += embeddedCount;
         return embeddedCount;
     }
     async createEmbeddings(texts) {
@@ -1651,14 +1743,64 @@ exports.Doc2Vec = Doc2Vec;
 Doc2Vec.MAX_EMBEDDING_TOKENS = 8191;
 Doc2Vec.CHARS_PER_TOKEN = 4;
 Doc2Vec.MAX_EMBEDDING_CHARS = Doc2Vec.MAX_EMBEDDING_TOKENS * Doc2Vec.CHARS_PER_TOKEN;
-if (require.main === module) {
-    const configPath = process.argv[2] || 'config.yaml';
+function runOneShot(configPath) {
     if (!fs.existsSync(configPath)) {
         console.error('Please provide a valid path to a YAML config file.');
         process.exit(1);
     }
     const doc2Vec = new Doc2Vec(configPath);
     doc2Vec.run()
-        .then(() => process.exit(0))
+        .then((stats) => process.exit(stats.some(s => !s.ok) ? 1 : 0))
         .catch((err) => { console.error(err); process.exit(1); });
 }
+if (require.main === module) {
+    const { Command } = require('commander');
+    const program = new Command();
+    program
+        .name('doc2vec')
+        .description('Crawl documentation sources and store embeddings in vector databases')
+        // Legacy invocation: `doc2vec [config.yaml]` runs a one-shot sync
+        .argument('[config]', 'path to a YAML config file', 'config.yaml')
+        .action((configPath) => runOneShot(configPath));
+    program
+        .command('run <config>')
+        .description('Run a one-shot sync for a config file (what the controller spawns)')
+        .action((configPath) => runOneShot(configPath));
+    program
+        .command('controller [configs...]')
+        .description('Run as a long-lived controller: schedule sync jobs per config file, persist runs in Postgres, serve a web UI')
+        .option('--database-url <url>', 'Postgres connection string (or DATABASE_URL env var)')
+        .option('--port <port>', 'HTTP port for the API/UI (or PORT env var)', (v) => parseInt(v, 10))
+        .option('--read-write', 'allow creating/editing configs from the UI (default: read-only)', false)
+        .option('--config-dir <dir>', 'directory where UI-created configs are written (required with --read-write)')
+        .option('--max-parallel <n>', 'maximum number of sync jobs running at once', (v) => parseInt(v, 10), 1)
+        .option('--reload-interval <seconds>', 'how often to re-poll config files for changes', (v) => parseInt(v, 10), 30)
+        .option('--log-retention-days <days>', 'delete run logs older than this many days', (v) => parseInt(v, 10), 14)
+        .option('--slack-webhook-url <url>', 'Slack incoming webhook for run notifications (or SLACK_WEBHOOK_URL env var)')
+        .option('--slack-notify <mode>', "which runs to notify about: 'all' or 'failures'", 'all')
+        .option('--public-url <url>', 'externally reachable base URL, used for links in notifications (or PUBLIC_URL env var)')
+        .action(async (configs, options) => {
+        // Lazy-require so the one-shot path doesn't load express/pg
+        const { startController } = require('./controller');
+        try {
+            await startController({
+                configArgs: configs,
+                databaseUrl: options.databaseUrl || process.env.DATABASE_URL,
+                port: options.port || (process.env.PORT ? parseInt(process.env.PORT, 10) : 8080),
+                readWrite: options.readWrite,
+                configDir: options.configDir,
+                maxParallel: options.maxParallel,
+                reloadIntervalSec: options.reloadInterval,
+                logRetentionDays: options.logRetentionDays,
+                slackWebhookUrl: options.slackWebhookUrl || process.env.SLACK_WEBHOOK_URL,
+                slackNotify: options.slackNotify === 'failures' ? 'failures' : 'all',
+                publicUrl: options.publicUrl || process.env.PUBLIC_URL,
+            });
+        }
+        catch (err) {
+            console.error(err instanceof Error ? err.message : err);
+            process.exit(1);
+        }
+    });
+    program.parse();
+}