doc2vec 2.11.0 → 2.12.0

package/dist/controller/chunk-inspector.js

@@ -0,0 +1,209 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.lookupChunks = lookupChunks;
+const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
+const path = __importStar(require("path"));
+const sqliteVec = __importStar(require("sqlite-vec"));
+const yaml = __importStar(require("js-yaml"));
+const js_client_rest_1 = require("@qdrant/js-client-rest");
+const types_1 = require("./types");
+const MAX_CHUNKS = 500;
+/**
+ * Read-only lookup of the chunks stored for a URL in a source's vector store.
+ * Resolves the store the same way a sync job would (same defaults as
+ * DatabaseManager.initDatabase, same ${ENV_VAR} substitution as Doc2Vec's
+ * config loader), but never creates tables or collections.
+ */
+async function lookupChunks(configContent, selector, url) {
+    const source = findSource(configContent, selector);
+    const dbConfig = source.database_config ?? {};
+    const params = dbConfig.params ?? {};
+    if (dbConfig.type === 'sqlite') {
+        const dbPath = params.db_path
+            || path.join(process.cwd(), `${String(source.product_name).replace(/\s+/g, '_')}-${source.version}.db`);
+        return {
+            source: { product_name: source.product_name, type: source.type, version: source.version },
+            database: { type: 'sqlite', path: dbPath },
+            chunks: lookupSqlite(dbPath, url),
+        };
+    }
+    if (dbConfig.type === 'qdrant') {
+        const qdrantUrl = params.qdrant_url || 'http://localhost:6333';
+        const qdrantPort = params.qdrant_port || 443;
+        const collectionName = params.collection_name
+            || `${String(source.product_name).toLowerCase().replace(/\s+/g, '_')}_${source.version}`;
+        const client = new js_client_rest_1.QdrantClient({ url: qdrantUrl, apiKey: process.env.QDRANT_API_KEY, port: qdrantPort });
+        return {
+            source: { product_name: source.product_name, type: source.type, version: source.version },
+            database: { type: 'qdrant', url: qdrantUrl, collection: collectionName },
+            chunks: await lookupQdrant(client, collectionName, url),
+        };
+    }
+    throw new types_1.ValidationError(`source has unsupported database type: ${dbConfig.type}`);
+}
+function findSource(configContent, selector) {
+    // Same ${VAR} substitution the sync job applies when it loads the config
+    const substituted = configContent.replace(/\$\{([^}]+)\}/g, (match, varName) => process.env[varName] !== undefined ? process.env[varName] : match);
+    let parsed;
+    try {
+        parsed = yaml.load(substituted);
+    }
+    catch (err) {
+        throw new types_1.ValidationError(`config does not parse: ${err instanceof Error ? err.message : String(err)}`);
+    }
+    const sources = Array.isArray(parsed?.sources) ? parsed.sources : [];
+    const source = sources.find(s => s?.product_name === selector.product_name
+        && (selector.type === undefined || s?.type === selector.type)
+        && (selector.version === undefined || String(effectiveVersion(s)) === selector.version));
+    if (!source) {
+        throw new types_1.NotFoundError(`source '${selector.product_name}' not found in config`);
+    }
+    return { ...source, version: effectiveVersion(source) };
+}
+// Mirrors the version defaulting in Doc2Vec.loadConfig for code sources
+function effectiveVersion(source) {
+    if (source?.version && String(source.version).trim().length > 0)
+        return String(source.version);
+    if (source?.type === 'code') {
+        if (source.branch && String(source.branch).trim().length > 0)
+            return String(source.branch);
+        return 'local';
+    }
+    return String(source?.version ?? '');
+}
+function lookupSqlite(dbPath, url) {
+    let db;
+    try {
+        db = new better_sqlite3_1.default(dbPath, { readonly: true, fileMustExist: true, allowExtension: true });
+    }
+    catch {
+        throw new types_1.NotFoundError(`SQLite database not found at ${dbPath} (has this source ever been synced?)`);
+    }
+    try {
+        sqliteVec.load(db);
+        const hasDates = db
+            .prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'vec_chunk_dates'`)
+            .get() !== undefined;
+        const rows = db.prepare(`
+            SELECT v.chunk_id, v.url, v.product_name, v.version, v.section, v.heading_hierarchy,
+                   v.chunk_index, v.total_chunks, v.hash, v.content
+                   ${hasDates ? ', d.created_at AS created_at' : ', NULL AS created_at'}
+            FROM vec_items v
+            ${hasDates ? 'LEFT JOIN vec_chunk_dates d ON d.chunk_id = v.chunk_id' : ''}
+            WHERE v.url = ?
+            LIMIT ${MAX_CHUNKS}
+        `).all(url);
+        return sortChunks(rows.map(row => ({
+            chunk_id: String(row.chunk_id),
+            url: String(row.url),
+            product_name: row.product_name ?? null,
+            version: row.version ?? null,
+            section: row.section ?? null,
+            heading_hierarchy: parseHierarchy(row.heading_hierarchy),
+            chunk_index: row.chunk_index === null || row.chunk_index === undefined ? null : Number(row.chunk_index),
+            total_chunks: row.total_chunks === null || row.total_chunks === undefined ? null : Number(row.total_chunks),
+            hash: row.hash ?? null,
+            created_at: row.created_at ?? null,
+            content: String(row.content ?? ''),
+        })));
+    }
+    finally {
+        db.close();
+    }
+}
+async function lookupQdrant(client, collectionName, url) {
+    const chunks = [];
+    let offset = undefined;
+    do {
+        const response = await client.scroll(collectionName, {
+            limit: 200,
+            offset,
+            with_payload: true,
+            with_vector: false,
+            filter: {
+                must: [{ key: 'url', match: { value: url } }],
+                must_not: [{ key: 'is_metadata', match: { value: true } }],
+            },
+        });
+        for (const point of response.points) {
+            const p = point.payload ?? {};
+            chunks.push({
+                chunk_id: String(p.original_chunk_id ?? point.id),
+                url: String(p.url ?? url),
+                product_name: p.product_name ?? null,
+                version: p.version ?? null,
+                section: p.section ?? null,
+                heading_hierarchy: parseHierarchy(p.heading_hierarchy),
+                chunk_index: p.chunk_index === null || p.chunk_index === undefined ? null : Number(p.chunk_index),
+                total_chunks: p.total_chunks === null || p.total_chunks === undefined ? null : Number(p.total_chunks),
+                hash: p.hash ?? null,
+                created_at: p.created_at ?? null,
+                content: String(p.content ?? ''),
+            });
+            if (chunks.length >= MAX_CHUNKS)
+                return sortChunks(chunks);
+        }
+        offset = response.next_page_offset;
+    } while (offset !== null && offset !== undefined);
+    return sortChunks(chunks);
+}
+function parseHierarchy(value) {
+    if (Array.isArray(value))
+        return value.map(String);
+    if (typeof value === 'string' && value.trim().startsWith('[')) {
+        try {
+            const parsed = JSON.parse(value);
+            if (Array.isArray(parsed))
+                return parsed.map(String);
+        }
+        catch { /* fall through */ }
+    }
+    return value ? [String(value)] : [];
+}
+// Default order: position within the page. The UI offers per-column sorting
+// (including by creation date) on top of this.
+function sortChunks(chunks) {
+    return chunks.sort((a, b) => {
+        const ai = a.chunk_index ?? Number.MAX_SAFE_INTEGER;
+        const bi = b.chunk_index ?? Number.MAX_SAFE_INTEGER;
+        if (ai !== bi)
+            return ai - bi;
+        return a.chunk_id.localeCompare(b.chunk_id);
+    });
+}

package/dist/controller/server.js

@@ -40,6 +40,7 @@ exports.createServer = createServer;
 const express_1 = __importDefault(require("express"));
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
+const chunk_inspector_1 = require("./chunk-inspector");
 const config_registry_1 = require("./config-registry");
 const types_1 = require("./types");
 const SSE_HEARTBEAT_MS = 15000;
@@ -151,6 +152,34 @@ function createServer(deps) {
         const days = Number(req.query.days) || 30;
         res.json(await store.getConfigStats(id, days));
     });
+    // Inspect the chunks currently stored for a URL in one source's vector store.
+    // This reads the live store, so results reflect the present state, not a
+    // historical run.
+    app.get('/api/configs/:id/chunks', async (req, res) => {
+        const config = registry.get(parseId(String(req.params.id)));
+        if (!config)
+            throw new types_1.NotFoundError('config not found');
+        const url = typeof req.query.url === 'string' ? req.query.url.trim() : '';
+        const productName = typeof req.query.product_name === 'string' ? req.query.product_name : '';
+        if (!url)
+            throw new types_1.ValidationError('url query parameter is required');
+        if (!productName)
+            throw new types_1.ValidationError('product_name query parameter is required');
+        try {
+            res.json(await (0, chunk_inspector_1.lookupChunks)(config.content, {
+                product_name: productName,
+                type: typeof req.query.type === 'string' && req.query.type ? req.query.type : undefined,
+                version: typeof req.query.version === 'string' && req.query.version ? req.query.version : undefined,
+            }, url));
+        }
+        catch (err) {
+            if (err instanceof types_1.NotFoundError || err instanceof types_1.ValidationError)
+                throw err;
+            // Vector store unreachable (e.g. Qdrant down) — surface the reason
+            logger.error('Chunk lookup failed:', err);
+            res.status(502).json({ error: `chunk lookup failed: ${err instanceof Error ? err.message : String(err)}` });
+        }
+    });
     // ------------------------------------------------------------------ runs
     app.get('/api/runs', async (req, res) => {
         res.json(await store.listRuns({

package/dist/database.js

@@ -71,6 +71,7 @@ class DatabaseManager {
                     total_chunks INTEGER
                 );
             `);
+            this.ensureChunkDatesTable(db);
             logger.info(`SQLite database initialized successfully`);
             return { db, type: 'sqlite' };
         }
@@ -307,9 +308,24 @@ class DatabaseManager {
             logger.error(`Failed to update last run date for ${repo}:`, error);
         }
     }
+    /**
+     * Chunk creation timestamps live in a companion table because vec_items is
+     * a vec0 virtual table that cannot be ALTERed to add a column. Rows are
+     * written on every chunk insert/update; stale rows for deleted chunks are
+     * harmless (lookups always join against vec_items).
+     */
+    static ensureChunkDatesTable(db) {
+        db.exec(`
+            CREATE TABLE IF NOT EXISTS vec_chunk_dates (
+                chunk_id   TEXT PRIMARY KEY,
+                created_at TEXT NOT NULL
+            );
+        `);
+    }
     static prepareSQLiteStatements(db) {
         let cached = this.columnCache.get(db);
         if (!cached) {
+            this.ensureChunkDatesTable(db);
             cached = {
                 hasBranch: this.hasColumn(db, 'branch'),
                 hasRepo: this.hasColumn(db, 'repo')
@@ -360,12 +376,16 @@ class DatabaseManager {
             `),
             getAllChunkIdsStmt: db.prepare(`SELECT chunk_id FROM vec_items`),
             deleteChunkStmt: db.prepare(`DELETE FROM vec_items WHERE chunk_id = ?`),
+            touchDateStmt: db.prepare(`
+                INSERT INTO vec_chunk_dates (chunk_id, created_at) VALUES (?, ?)
+                ON CONFLICT(chunk_id) DO UPDATE SET created_at = excluded.created_at
+            `),
             hasBranchColumn,
             hasRepoColumn
         };
     }
     static insertVectorsSQLite(db, chunk, embedding, logger, chunkHash) {
-        const { insertStmt, updateStmt, hasBranchColumn, hasRepoColumn } = this.prepareSQLiteStatements(db);
+        const { insertStmt, updateStmt, touchDateStmt, hasBranchColumn, hasRepoColumn } = this.prepareSQLiteStatements(db);
         const hash = chunkHash || utils_1.Utils.generateHash(chunk.content);
         const transaction = db.transaction(() => {
             // Use BigInt for true integer representation in SQLite vec0
@@ -401,6 +421,7 @@ class DatabaseManager {
                 updateValues.push(JSON.stringify(chunk.metadata.heading_hierarchy), chunk.metadata.section, chunk.content, chunk.metadata.url, hash, chunkIndex, totalChunks, chunk.metadata.chunk_id);
                 updateStmt.run(...updateValues);
             }
+            touchDateStmt.run(chunk.metadata.chunk_id, new Date().toISOString());
         });
         transaction();
     }
@@ -434,6 +455,7 @@ class DatabaseManager {
                     original_chunk_id: chunk.metadata.chunk_id,
                     chunk_index: chunk.metadata.chunk_index,
                     total_chunks: chunk.metadata.total_chunks,
+                    created_at: new Date().toISOString(),
                 },
             };
             await client.upsert(collectionName, {
@@ -457,17 +479,20 @@ class DatabaseManager {
         const deleteChunkStmt = db.prepare(`DELETE FROM vec_items WHERE chunk_id = ?`);
         const existingChunks = getChunksForUrlStmt.all(urlPrefix);
         let deletedCount = 0;
+        const deletedUrls = new Set();
         const transaction = db.transaction(() => {
             for (const { chunk_id, url } of existingChunks) {
                 if (!visitedUrls.has(url)) {
                     logger.debug(`Deleting obsolete chunk from SQLite: ${chunk_id.substring(0, 8)}... (URL not visited)`);
                     deleteChunkStmt.run(chunk_id);
                     deletedCount++;
+                    deletedUrls.add(url);
                 }
             }
         });
         transaction();
         logger.info(`Deleted ${deletedCount} obsolete chunks from SQLite for URL ${urlPrefix}`);
+        return { items: deletedUrls.size, chunks: deletedCount };
     }
     static async removeObsoleteChunksQdrant(db, visitedUrls, urlPrefix, logger) {
         const { client, collectionName } = db;
@@ -496,6 +521,7 @@ class DatabaseManager {
                 ]
             };
             const obsoletePointIds = [];
+            const obsoleteUrls = new Set();
             let offset = undefined;
             do {
                 const response = await client.scroll(collectionName, {
@@ -513,6 +539,7 @@ class DatabaseManager {
                     }
                     if (url && !visitedUrls.has(url)) {
                         obsoletePointIds.push(point.id);
+                        obsoleteUrls.add(url);
                     }
                 }
                 offset = response.next_page_offset;
@@ -526,6 +553,7 @@ class DatabaseManager {
             else {
                 logger.info(`No obsolete chunks to delete from Qdrant for URL ${urlPrefix}`);
             }
+            return { items: obsoleteUrls.size, chunks: obsoletePointIds.length };
         }
         catch (error) {
             // Rethrow so a failed cleanup fails the job instead of silently
@@ -548,31 +576,42 @@ class DatabaseManager {
         const deleteStmt = db.prepare(`DELETE FROM vec_items WHERE url = ?`);
         const result = deleteStmt.run(url);
         logger.info(`Deleted ${result.changes} chunks from SQLite for URL ${url}`);
+        return result.changes;
     }
     static async removeChunksByUrlQdrant(db, url, logger) {
         const { client, collectionName } = db;
         try {
-            await client.delete(collectionName, {
-                filter: {
-                    must: [
-                        {
-                            key: 'url',
-                            match: {
-                                value: url // Exact match — must match getChunkHashesByUrlQdrant's filter
-                            }
+            const filter = {
+                must: [
+                    {
+                        key: 'url',
+                        match: {
+                            value: url // Exact match — must match getChunkHashesByUrlQdrant's filter
                         }
-                    ],
-                    must_not: [
-                        {
-                            key: 'is_metadata',
-                            match: {
-                                value: true
-                            }
+                    }
+                ],
+                must_not: [
+                    {
+                        key: 'is_metadata',
+                        match: {
+                            value: true
                         }
-                    ]
-                }
-            });
+                    }
+                ]
+            };
+            // Count first so callers can report how many chunks the delete removed
+            // (Qdrant's delete-by-filter response doesn't include a count).
+            let deletedCount = 0;
+            try {
+                const countResult = await client.count(collectionName, { filter, exact: true });
+                deletedCount = countResult?.count ?? 0;
+            }
+            catch {
+                // Count is informational only — never fail the delete over it
+            }
+            await client.delete(collectionName, { filter });
             logger.info(`Deleted chunks from Qdrant for URL ${url}`);
+            return deletedCount;
         }
         catch (error) {
             // Rethrow so the failure propagates and the job exits non-zero.
@@ -697,6 +736,7 @@ class DatabaseManager {
         logger.debug(`Searching for chunks with URL prefix: ${urlPrefix}`);
         const existingChunks = getChunksForPathStmt.all(urlPrefix);
         let deletedCount = 0;
+        const deletedUrls = new Set();
         const transaction = db.transaction(() => {
             for (const { chunk_id, url } of existingChunks) {
                 // Skip if it's not from our URL prefix (safety check)
@@ -720,11 +760,13 @@ class DatabaseManager {
                     logger.debug(`Deleting obsolete chunk from SQLite: ${chunk_id.substring(0, 8)}... (File not processed: ${filePath})`);
                     deleteChunkStmt.run(chunk_id);
                     deletedCount++;
+                    deletedUrls.add(url);
                 }
             }
         });
         transaction();
         logger.info(`Deleted ${deletedCount} obsolete chunks from SQLite for URL prefix ${urlPrefix}`);
+        return { items: deletedUrls.size, chunks: deletedCount };
     }
     static async removeObsoleteFilesQdrant(db, processedFiles, pathConfig, logger) {
         const { client, collectionName } = db;
@@ -768,6 +810,7 @@ class DatabaseManager {
                 ]
             };
             const obsoletePointIds = [];
+            const obsoleteUrls = new Set();
             let offset = undefined;
             do {
                 const response = await client.scroll(collectionName, {
@@ -799,6 +842,7 @@ class DatabaseManager {
                     }
                     if (filePath && !processedFiles.has(filePath)) {
                         obsoletePointIds.push(point.id);
+                        obsoleteUrls.add(url);
                     }
                 }
                 offset = response.next_page_offset;
@@ -812,6 +856,7 @@ class DatabaseManager {
             else {
                 logger.info(`No obsolete chunks to delete from Qdrant for URL prefix ${urlPrefix}`);
             }
+            return { items: obsoleteUrls.size, chunks: obsoletePointIds.length };
         }
         catch (error) {
             // Rethrow so a failed cleanup fails the job instead of silently