doc2vec 1.2.0 → 2.0.0

package/dist/doc2vec.js

@@ -42,6 +42,7 @@ const crypto_1 = __importDefault(require("crypto"));
 const yaml = __importStar(require("js-yaml"));
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
+const buffer_1 = require("buffer");
 const openai_1 = require("openai");
 const dotenv = __importStar(require("dotenv"));
 const logger_1 = require("./logger");
@@ -67,7 +68,17 @@ class Doc2Vec {
         try {
             const logger = this.logger.child('config');
             logger.info(`Loading configuration from ${configPath}`);
-            const configFile = fs.readFileSync(configPath, 'utf8');
+            let configFile = fs.readFileSync(configPath, 'utf8');
+            // Substitute environment variables in the format ${VAR_NAME}
+            configFile = configFile.replace(/\$\{([^}]+)\}/g, (match, varName) => {
+                const envValue = process.env[varName];
+                if (envValue === undefined) {
+                    logger.warn(`Environment variable ${varName} not found, keeping placeholder ${match}`);
+                    return match;
+                }
+                logger.debug(`Substituted ${match} with environment variable value`);
+                return envValue;
+            });
             let config = yaml.load(configFile);
             const typedConfig = config;
             logger.info(`Configuration loaded successfully, found ${typedConfig.sources.length} sources`);
@@ -92,6 +103,9 @@ class Doc2Vec {
             else if (sourceConfig.type === 'local_directory') {
                 await this.processLocalDirectory(sourceConfig, sourceLogger);
             }
+            else if (sourceConfig.type === 'zendesk') {
+                await this.processZendesk(sourceConfig, sourceLogger);
+            }
             else {
                 sourceLogger.error(`Unknown source type: ${sourceConfig.type}`);
             }
@@ -109,26 +123,64 @@ class Doc2Vec {
         const fetchWithRetry = async (url, params = {}, retries = 5, delay = 5000) => {
             for (let attempt = 0; attempt < retries; attempt++) {
                 try {
+                    // Only log on retries to reduce noise during pagination
+                    if (attempt > 0) {
+                        logger.debug(`GitHub API retry: ${url} (attempt ${attempt + 1}/${retries})`);
+                    }
                     const response = await axios_1.default.get(url, {
                         headers: {
                             Authorization: `token ${GITHUB_TOKEN}`,
                             Accept: 'application/vnd.github.v3+json',
                         },
                         params,
+                        timeout: 30000, // 30 second timeout
                     });
                     return response.data;
                 }
                 catch (error) {
+                    // Enhanced error logging for debugging
+                    const errorDetails = {
+                        code: error.code,
+                        message: error.message,
+                        status: error.response?.status,
+                        isTimeout: error.code === 'ECONNABORTED' || error.message?.includes('timeout'),
+                        isNetworkError: !error.response && error.code,
+                    };
+                    logger.debug(`GitHub API error details: ${JSON.stringify(errorDetails)}`);
                     if (error.response && error.response.status === 403) {
+                        // Check if this is actually a rate limit error
+                        const rateLimitRemaining = error.response.headers['x-ratelimit-remaining'];
                         const resetTime = error.response.headers['x-ratelimit-reset'];
-                        const currentTime = Math.floor(Date.now() / 1000);
-                        const waitTime = resetTime ? (resetTime - currentTime) * 1000 : delay * 2;
-                        logger.warn(`GitHub rate limit exceeded. Waiting ${waitTime / 1000}s`);
-                        await new Promise(res => setTimeout(res, waitTime));
+                        if (rateLimitRemaining === '0' && resetTime) {
+                            const currentTime = Math.floor(Date.now() / 1000);
+                            const resetTimestamp = parseInt(resetTime, 10);
+                            let waitTime = (resetTimestamp - currentTime) * 1000;
+                            // Ensure waitTime is at least 1 second (in case resetTime is in the past)
+                            if (waitTime < 1000) {
+                                waitTime = 1000;
+                            }
+                            logger.warn(`GitHub rate limit exceeded. Waiting ${Math.ceil(waitTime / 1000)}s (attempt ${attempt + 1}/${retries})`);
+                            await new Promise(res => setTimeout(res, waitTime));
+                            // Retry the request after waiting
+                            continue;
+                        }
+                        else {
+                            // Other 403 errors (e.g., forbidden access)
+                            logger.error(`GitHub API returned 403 (not rate limit): ${error.message}`);
+                            throw error;
+                        }
                     }
                     else {
-                        logger.error(`GitHub fetch failed: ${error.message}`);
-                        throw error;
+                        // For non-403 errors, wait before retrying (exponential backoff)
+                        if (attempt < retries - 1) {
+                            const backoffDelay = delay * Math.pow(2, attempt);
+                            logger.warn(`GitHub fetch failed (attempt ${attempt + 1}/${retries}): ${error.message}. Retrying in ${backoffDelay}ms`);
+                            await new Promise(res => setTimeout(res, backoffDelay));
+                        }
+                        else {
+                            logger.error(`GitHub fetch failed after ${retries} attempts: ${error.message} (code: ${error.code || 'unknown'})`);
+                            throw error;
+                        }
                     }
                 }
             }
@@ -140,6 +192,10 @@ class Doc2Vec {
             const perPage = 100;
             const sinceTimestamp = new Date(sinceDate);
             while (true) {
+                // Log progress every 10 pages to reduce noise
+                if (page === 1 || page % 10 === 0) {
+                    logger.debug(`Fetching issues page ${page}... (${issues.length} issues so far)`);
+                }
                 const data = await fetchWithRetry(GITHUB_API_URL, {
                     per_page: perPage,
                     page,
@@ -493,6 +549,344 @@ class Doc2Vec {
         }
         logger.info(`Finished processing local directory: ${config.path}`);
     }
+    async processZendesk(config, parentLogger) {
+        const logger = parentLogger.child('process');
+        logger.info(`Starting processing for Zendesk: ${config.zendesk_subdomain}.zendesk.com`);
+        const dbConnection = await database_1.DatabaseManager.initDatabase(config, logger);
+        // Initialize metadata storage
+        await database_1.DatabaseManager.initDatabaseMetadata(dbConnection, logger);
+        const fetchTickets = config.fetch_tickets !== false; // default true
+        const fetchArticles = config.fetch_articles !== false; // default true
+        if (fetchTickets) {
+            logger.section('ZENDESK TICKETS');
+            await this.fetchAndProcessZendeskTickets(config, dbConnection, logger);
+        }
+        if (fetchArticles) {
+            logger.section('ZENDESK ARTICLES');
+            await this.fetchAndProcessZendeskArticles(config, dbConnection, logger);
+        }
+        logger.info(`Finished processing Zendesk: ${config.zendesk_subdomain}.zendesk.com`);
+    }
+    async fetchAndProcessZendeskTickets(config, dbConnection, logger) {
+        const baseUrl = `https://${config.zendesk_subdomain}.zendesk.com/api/v2`;
+        const auth = buffer_1.Buffer.from(`${config.email}/token:${config.api_token}`).toString('base64');
+        // Get the last run date from the database
+        const startDate = config.start_date || `${new Date().getFullYear()}-01-01`;
+        const lastRunDate = await database_1.DatabaseManager.getLastRunDate(dbConnection, `zendesk_tickets_${config.zendesk_subdomain}`, `${startDate}T00:00:00Z`, logger);
+        const fetchWithRetry = async (url, retries = 3) => {
+            for (let attempt = 0; attempt < retries; attempt++) {
+                try {
+                    const response = await axios_1.default.get(url, {
+                        headers: {
+                            'Authorization': `Basic ${auth}`,
+                            'Content-Type': 'application/json',
+                        },
+                    });
+                    if (response.status === 429) {
+                        const retryAfter = parseInt(response.headers['retry-after'] || '60');
+                        logger.warn(`Rate limited, waiting ${retryAfter}s before retry`);
+                        await new Promise(res => setTimeout(res, retryAfter * 1000));
+                        continue;
+                    }
+                    return response.data;
+                }
+                catch (error) {
+                    logger.error(`Zendesk API error (attempt ${attempt + 1}):`, error.message);
+                    if (attempt === retries - 1)
+                        throw error;
+                    await new Promise(res => setTimeout(res, 2000 * (attempt + 1)));
+                }
+            }
+        };
+        const generateMarkdownForTicket = (ticket, comments) => {
+            let md = `# Ticket #${ticket.id}: ${ticket.subject}\n\n`;
+            md += `- **Status:** ${ticket.status}\n`;
+            md += `- **Priority:** ${ticket.priority || 'None'}\n`;
+            md += `- **Type:** ${ticket.type || 'None'}\n`;
+            md += `- **Requester:** ${ticket.requester_id}\n`;
+            md += `- **Assignee:** ${ticket.assignee_id || 'Unassigned'}\n`;
+            md += `- **Created:** ${new Date(ticket.created_at).toDateString()}\n`;
+            md += `- **Updated:** ${new Date(ticket.updated_at).toDateString()}\n`;
+            if (ticket.tags && ticket.tags.length > 0) {
+                md += `- **Tags:** ${ticket.tags.map((tag) => `\`${tag}\``).join(', ')}\n`;
+            }
+            // Handle ticket description
+            const description = ticket.description || '';
+            const cleanDescription = description || '_No description._';
+            md += `\n## Description\n\n${cleanDescription}\n\n`;
+            if (comments && comments.length > 0) {
+                md += `## Comments\n\n`;
+                for (const comment of comments) {
+                    if (comment.public) {
+                        md += `### ${comment.author_id} - ${new Date(comment.created_at).toDateString()}\n\n`;
+                        // Handle comment body
+                        const rawBody = comment.plain_body || comment.html_body || comment.body || '';
+                        const commentBody = rawBody.replace(/&nbsp;/g, " ") || '_No content._';
+                        md += `${commentBody}\n\n---\n\n`;
+                    }
+                }
+            }
+            else {
+                md += `## Comments\n\n_No comments._\n`;
+            }
+            return md;
+        };
+        const processTicket = async (ticket) => {
+            const ticketId = ticket.id;
+            const url = `https://${config.zendesk_subdomain}.zendesk.com/agent/tickets/${ticketId}`;
+            logger.info(`Processing ticket #${ticketId}`);
+            // Fetch ticket comments
+            const commentsUrl = `${baseUrl}/tickets/${ticketId}/comments.json`;
+            const commentsData = await fetchWithRetry(commentsUrl);
+            const comments = commentsData?.comments || [];
+            // Generate markdown for the ticket
+            const markdown = generateMarkdownForTicket(ticket, comments);
+            // Chunk the markdown content
+            const ticketConfig = {
+                ...config,
+                product_name: config.product_name || `zendesk_${config.zendesk_subdomain}`,
+                max_size: config.max_size || Infinity
+            };
+            const chunks = await this.contentProcessor.chunkMarkdown(markdown, ticketConfig, url);
+            logger.info(`Ticket #${ticketId}: Created ${chunks.length} chunks`);
+            // Process and store each chunk
+            for (const chunk of chunks) {
+                const chunkHash = utils_1.Utils.generateHash(chunk.content);
+                const chunkId = chunk.metadata.chunk_id.substring(0, 8) + '...';
+                if (dbConnection.type === 'sqlite') {
+                    const { checkHashStmt } = database_1.DatabaseManager.prepareSQLiteStatements(dbConnection.db);
+                    const existing = checkHashStmt.get(chunk.metadata.chunk_id);
+                    if (existing && existing.hash === chunkHash) {
+                        logger.info(`Skipping unchanged chunk: ${chunkId}`);
+                        continue;
+                    }
+                    const embeddings = await this.createEmbeddings([chunk.content]);
+                    if (embeddings.length) {
+                        database_1.DatabaseManager.insertVectorsSQLite(dbConnection.db, chunk, embeddings[0], logger, chunkHash);
+                        logger.debug(`Stored chunk ${chunkId} in SQLite`);
+                    }
+                    else {
+                        logger.error(`Embedding failed for chunk: ${chunkId}`);
+                    }
+                }
+                else if (dbConnection.type === 'qdrant') {
+                    try {
+                        let pointId;
+                        try {
+                            pointId = chunk.metadata.chunk_id;
+                            if (!utils_1.Utils.isValidUuid(pointId)) {
+                                pointId = utils_1.Utils.hashToUuid(chunk.metadata.chunk_id);
+                            }
+                        }
+                        catch (e) {
+                            pointId = crypto_1.default.randomUUID();
+                        }
+                        const existingPoints = await dbConnection.client.retrieve(dbConnection.collectionName, {
+                            ids: [pointId],
+                            with_payload: true,
+                            with_vector: false,
+                        });
+                        if (existingPoints.length > 0 && existingPoints[0].payload && existingPoints[0].payload.hash === chunkHash) {
+                            logger.info(`Skipping unchanged chunk: ${chunkId}`);
+                            continue;
+                        }
+                        const embeddings = await this.createEmbeddings([chunk.content]);
+                        if (embeddings.length) {
+                            await database_1.DatabaseManager.storeChunkInQdrant(dbConnection, chunk, embeddings[0], chunkHash);
+                            logger.debug(`Stored chunk ${chunkId} in Qdrant (${dbConnection.collectionName})`);
+                        }
+                        else {
+                            logger.error(`Embedding failed for chunk: ${chunkId}`);
+                        }
+                    }
+                    catch (error) {
+                        logger.error(`Error processing chunk in Qdrant:`, error);
+                    }
+                }
+            }
+        };
+        logger.info(`Fetching Zendesk tickets updated since ${lastRunDate}`);
+        // Build query parameters
+        const statusFilter = config.ticket_status || ['new', 'open', 'pending', 'hold', 'solved'];
+        const query = `updated>${lastRunDate.split('T')[0]} status:${statusFilter.join(',status:')}`;
+        let nextPage = `${baseUrl}/search.json?query=${encodeURIComponent(query)}&sort_by=updated_at&sort_order=asc`;
+        let totalTickets = 0;
+        while (nextPage) {
+            const data = await fetchWithRetry(nextPage);
+            const tickets = data.results || [];
+            logger.info(`Processing batch of ${tickets.length} tickets`);
+            for (const ticket of tickets) {
+                await processTicket(ticket);
+                totalTickets++;
+            }
+            nextPage = data.next_page;
+            if (nextPage) {
+                logger.debug(`Fetching next page: ${nextPage}`);
+                // Rate limiting: wait between requests
+                await new Promise(res => setTimeout(res, 1000));
+            }
+        }
+        // Update the last run date in the database
+        await database_1.DatabaseManager.updateLastRunDate(dbConnection, `zendesk_tickets_${config.zendesk_subdomain}`, logger);
+        logger.info(`Successfully processed ${totalTickets} tickets`);
+    }
+    async fetchAndProcessZendeskArticles(config, dbConnection, logger) {
+        const baseUrl = `https://${config.zendesk_subdomain}.zendesk.com/api/v2/help_center`;
+        const auth = buffer_1.Buffer.from(`${config.email}/token:${config.api_token}`).toString('base64');
+        // Get the start date for filtering
+        const startDate = config.start_date || `${new Date().getFullYear()}-01-01`;
+        const startDateObj = new Date(startDate);
+        const fetchWithRetry = async (url, retries = 3) => {
+            for (let attempt = 0; attempt < retries; attempt++) {
+                try {
+                    const response = await axios_1.default.get(url, {
+                        headers: {
+                            'Authorization': `Basic ${auth}`,
+                            'Content-Type': 'application/json',
+                        },
+                    });
+                    if (response.status === 429) {
+                        const retryAfter = parseInt(response.headers['retry-after'] || '60');
+                        logger.warn(`Rate limited, waiting ${retryAfter}s before retry`);
+                        await new Promise(res => setTimeout(res, retryAfter * 1000));
+                        continue;
+                    }
+                    return response.data;
+                }
+                catch (error) {
+                    logger.error(`Zendesk API error (attempt ${attempt + 1}):`, error.message);
+                    if (attempt === retries - 1)
+                        throw error;
+                    await new Promise(res => setTimeout(res, 2000 * (attempt + 1)));
+                }
+            }
+        };
+        const generateMarkdownForArticle = (article) => {
+            let md = `# ${article.title}\n\n`;
+            md += `- **Author:** ${article.author_id}\n`;
+            md += `- **Section:** ${article.section_id}\n`;
+            md += `- **Created:** ${new Date(article.created_at).toDateString()}\n`;
+            md += `- **Updated:** ${new Date(article.updated_at).toDateString()}\n`;
+            md += `- **Vote Sum:** ${article.vote_sum || 0}\n`;
+            md += `- **Vote Count:** ${article.vote_count || 0}\n`;
+            if (article.label_names && article.label_names.length > 0) {
+                md += `- **Labels:** ${article.label_names.map((label) => `\`${label}\``).join(', ')}\n`;
+            }
+            // Handle article content - convert HTML to markdown
+            const articleBody = article.body || '';
+            let cleanContent = '_No content._';
+            if (articleBody.trim()) {
+                if (articleBody.includes('<')) {
+                    // HTML content - use ContentProcessor to convert to markdown
+                    cleanContent = this.contentProcessor.convertHtmlToMarkdown(articleBody);
+                }
+                else {
+                    // Plain text content
+                    cleanContent = articleBody;
+                }
+            }
+            md += `\n## Content\n\n${cleanContent}\n`;
+            return md;
+        };
+        const processArticle = async (article) => {
+            const articleId = article.id;
+            const url = article.html_url || `https://${config.zendesk_subdomain}.zendesk.com/hc/articles/${articleId}`;
+            logger.info(`Processing article #${articleId}: ${article.title}`);
+            // Generate markdown for the article
+            const markdown = generateMarkdownForArticle(article);
+            // Chunk the markdown content
+            const articleConfig = {
+                ...config,
+                product_name: config.product_name || `zendesk_${config.zendesk_subdomain}`,
+                max_size: config.max_size || Infinity
+            };
+            const chunks = await this.contentProcessor.chunkMarkdown(markdown, articleConfig, url);
+            logger.info(`Article #${articleId}: Created ${chunks.length} chunks`);
+            // Process and store each chunk (similar to ticket processing)
+            for (const chunk of chunks) {
+                const chunkHash = utils_1.Utils.generateHash(chunk.content);
+                const chunkId = chunk.metadata.chunk_id.substring(0, 8) + '...';
+                if (dbConnection.type === 'sqlite') {
+                    const { checkHashStmt } = database_1.DatabaseManager.prepareSQLiteStatements(dbConnection.db);
+                    const existing = checkHashStmt.get(chunk.metadata.chunk_id);
+                    if (existing && existing.hash === chunkHash) {
+                        logger.info(`Skipping unchanged chunk: ${chunkId}`);
+                        continue;
+                    }
+                    const embeddings = await this.createEmbeddings([chunk.content]);
+                    if (embeddings.length) {
+                        database_1.DatabaseManager.insertVectorsSQLite(dbConnection.db, chunk, embeddings[0], logger, chunkHash);
+                        logger.debug(`Stored chunk ${chunkId} in SQLite`);
+                    }
+                    else {
+                        logger.error(`Embedding failed for chunk: ${chunkId}`);
+                    }
+                }
+                else if (dbConnection.type === 'qdrant') {
+                    try {
+                        let pointId;
+                        try {
+                            pointId = chunk.metadata.chunk_id;
+                            if (!utils_1.Utils.isValidUuid(pointId)) {
+                                pointId = utils_1.Utils.hashToUuid(chunk.metadata.chunk_id);
+                            }
+                        }
+                        catch (e) {
+                            pointId = crypto_1.default.randomUUID();
+                        }
+                        const existingPoints = await dbConnection.client.retrieve(dbConnection.collectionName, {
+                            ids: [pointId],
+                            with_payload: true,
+                            with_vector: false,
+                        });
+                        if (existingPoints.length > 0 && existingPoints[0].payload && existingPoints[0].payload.hash === chunkHash) {
+                            logger.info(`Skipping unchanged chunk: ${chunkId}`);
+                            continue;
+                        }
+                        const embeddings = await this.createEmbeddings([chunk.content]);
+                        if (embeddings.length) {
+                            await database_1.DatabaseManager.storeChunkInQdrant(dbConnection, chunk, embeddings[0], chunkHash);
+                            logger.debug(`Stored chunk ${chunkId} in Qdrant (${dbConnection.collectionName})`);
+                        }
+                        else {
+                            logger.error(`Embedding failed for chunk: ${chunkId}`);
+                        }
+                    }
+                    catch (error) {
+                        logger.error(`Error processing chunk in Qdrant:`, error);
+                    }
+                }
+            }
+        };
+        logger.info(`Fetching Zendesk help center articles updated since ${startDate}`);
+        let nextPage = `${baseUrl}/articles.json`;
+        let totalArticles = 0;
+        let processedArticles = 0;
+        while (nextPage) {
+            const data = await fetchWithRetry(nextPage);
+            const articles = data.articles || [];
+            logger.info(`Processing batch of ${articles.length} articles`);
+            for (const article of articles) {
+                totalArticles++;
+                // Check if article was updated since the start date
+                const updatedAt = new Date(article.updated_at);
+                if (updatedAt >= startDateObj) {
+                    await processArticle(article);
+                    processedArticles++;
+                }
+                else {
+                    logger.debug(`Skipping article #${article.id} (updated ${article.updated_at}, before ${startDate})`);
+                }
+            }
+            nextPage = data.next_page;
+            if (nextPage) {
+                logger.debug(`Fetching next page: ${nextPage}`);
+                // Rate limiting: wait between requests
+                await new Promise(res => setTimeout(res, 1000));
+            }
+        }
+        logger.info(`Successfully processed ${processedArticles} of ${totalArticles} articles (filtered by date >= ${startDate})`);
+    }
     async createEmbeddings(texts) {
         const logger = this.logger.child('embeddings');
         try {

package/dist/electron-app/src/database.js

@@ -0,0 +1,218 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DatabaseManager = void 0;
+const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
+const sqliteVec = __importStar(require("sqlite-vec"));
+class DatabaseManager {
+    constructor(dbPath) {
+        // Cache counts for immediate UI feedback
+        this._trackedFilesCount = 0;
+        this._totalChunksCount = 0;
+        this.db = new better_sqlite3_1.default(dbPath);
+        sqliteVec.load(this.db);
+        this.createTables();
+        // Prepare statements for insert/update (matching doc2vec's approach)
+        this.insertStmt = this.db.prepare(`
+            INSERT INTO vec_items (embedding, version, heading_hierarchy, section, chunk_id, content, url, hash, chunk_index, total_chunks)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        `);
+        this.updateStmt = this.db.prepare(`
+            UPDATE vec_items SET embedding = ?, version = ?, heading_hierarchy = ?, section = ?, content = ?, url = ?, hash = ?, chunk_index = ?, total_chunks = ?
+            WHERE chunk_id = ?
+        `);
+        // Initialize counts from database
+        this._trackedFilesCount = this._queryTrackedFilesCount();
+        this._totalChunksCount = this._queryTotalChunksCount();
+        console.log(`Database initialized at: ${dbPath}`);
+    }
+    _queryTrackedFilesCount() {
+        const stmt = this.db.prepare('SELECT COUNT(*) as count FROM files');
+        const row = stmt.get();
+        return Number(row.count);
+    }
+    _queryTotalChunksCount() {
+        const stmt = this.db.prepare('SELECT COUNT(*) as count FROM vec_items');
+        const row = stmt.get();
+        return Number(row.count);
+    }
+    createTables() {
+        // Create vec0 virtual table (sqlite-vec)
+        this.db.exec(`
+            CREATE VIRTUAL TABLE IF NOT EXISTS vec_items USING vec0(
+                embedding FLOAT[3072],
+                version TEXT,
+                heading_hierarchy TEXT,
+                section TEXT,
+                chunk_id TEXT UNIQUE,
+                content TEXT,
+                url TEXT,
+                hash TEXT,
+                chunk_index INTEGER,
+                total_chunks INTEGER
+            )
+        `);
+        // Create files tracking table
+        this.db.exec(`
+            CREATE TABLE IF NOT EXISTS files (
+                path TEXT PRIMARY KEY,
+                hash TEXT,
+                modified_at TEXT,
+                chunk_count INTEGER
+            )
+        `);
+        console.log('Database tables created');
+    }
+    insertChunk(chunk, embedding) {
+        const embeddingData = embedding ? new Float32Array(embedding) : new Float32Array(3072).fill(0);
+        // Use JSON.stringify for heading_hierarchy to match doc2vec format
+        const headingHierarchyJson = JSON.stringify(chunk.headingHierarchy);
+        try {
+            this.insertStmt.run(embeddingData, chunk.version, headingHierarchyJson, chunk.section, chunk.chunkId, chunk.content, chunk.url, chunk.hash, BigInt(chunk.chunkIndex), BigInt(chunk.totalChunks));
+            this._totalChunksCount++;
+        }
+        catch (error) {
+            // If insert fails due to UNIQUE constraint, update instead
+            if (error.code === 'SQLITE_CONSTRAINT_UNIQUE' || error.message?.includes('UNIQUE constraint failed')) {
+                this.updateStmt.run(embeddingData, chunk.version, headingHierarchyJson, chunk.section, chunk.content, chunk.url, chunk.hash, BigInt(chunk.chunkIndex), BigInt(chunk.totalChunks), chunk.chunkId);
+                // Update doesn't change count
+            }
+            else {
+                throw error;
+            }
+        }
+    }
+    removeChunksForFile(filePath) {
+        const url = `file://${filePath}`;
+        // Get count before delete
+        const countStmt = this.db.prepare('SELECT COUNT(*) as count FROM vec_items WHERE url = ?');
+        const countRow = countStmt.get(url);
+        const deletedCount = Number(countRow.count);
+        const stmt = this.db.prepare('DELETE FROM vec_items WHERE url = ?');
+        stmt.run(url);
+        this._totalChunksCount -= deletedCount;
+    }
+    getExistingChunkHash(chunkId) {
+        const stmt = this.db.prepare('SELECT hash FROM vec_items WHERE chunk_id = ?');
+        const row = stmt.get(chunkId);
+        return row?.hash ?? null;
+    }
+    getTotalChunksCount() {
+        return this._totalChunksCount;
+    }
+    upsertFileInfo(filePath, hash, modifiedAt, chunkCount) {
+        // Check if file already exists
+        const checkStmt = this.db.prepare('SELECT 1 FROM files WHERE path = ?');
+        const exists = checkStmt.get(filePath);
+        const stmt = this.db.prepare(`
+            INSERT OR REPLACE INTO files (path, hash, modified_at, chunk_count)
+            VALUES (?, ?, ?, ?)
+        `);
+        stmt.run(filePath, hash, modifiedAt.toISOString(), chunkCount);
+        // Only increment if this is a new file
+        if (!exists) {
+            this._trackedFilesCount++;
+        }
+    }
+    getFileInfo(filePath) {
+        const stmt = this.db.prepare('SELECT * FROM files WHERE path = ?');
+        const row = stmt.get(filePath);
+        if (!row)
+            return null;
+        return {
+            path: row.path,
+            hash: row.hash,
+            modifiedAt: new Date(row.modified_at),
+            chunkCount: row.chunk_count
+        };
+    }
+    removeFileInfo(filePath) {
+        // Check if file exists before deleting
+        const checkStmt = this.db.prepare('SELECT 1 FROM files WHERE path = ?');
+        const exists = checkStmt.get(filePath);
+        const stmt = this.db.prepare('DELETE FROM files WHERE path = ?');
+        stmt.run(filePath);
+        if (exists) {
+            this._trackedFilesCount--;
+        }
+    }
+    getAllTrackedFiles() {
+        const stmt = this.db.prepare('SELECT path FROM files');
+        const rows = stmt.all();
+        return rows.map(r => r.path);
+    }
+    getAllTrackedFilesWithInfo() {
+        const stmt = this.db.prepare('SELECT path, chunk_count, modified_at FROM files ORDER BY modified_at DESC');
+        const rows = stmt.all();
+        return rows.map(r => ({
+            path: r.path,
+            chunkCount: r.chunk_count,
+            modifiedAt: r.modified_at
+        }));
+    }
+    getChunksForFile(filePath) {
+        const url = `file://${filePath}`;
+        const stmt = this.db.prepare(`
+            SELECT chunk_id, content, section, chunk_index 
+            FROM vec_items 
+            WHERE url = ? 
+            ORDER BY chunk_index
+        `);
+        const rows = stmt.all(url);
+        return rows.map(r => ({
+            chunkId: r.chunk_id,
+            content: r.content,
+            section: r.section,
+            chunkIndex: Number(r.chunk_index)
+        }));
+    }
+    getTrackedFilesCount() {
+        return this._trackedFilesCount;
+    }
+    clearAllData() {
+        this.db.exec('DELETE FROM vec_items');
+        this.db.exec('DELETE FROM files');
+        this._trackedFilesCount = 0;
+        this._totalChunksCount = 0;
+        console.log('Cleared all data');
+    }
+    close() {
+        this.db.close();
+    }
+}
+exports.DatabaseManager = DatabaseManager;