doc2vec 1.3.0 → 2.0.0

package/dist/electron-app/src/database.js

@@ -0,0 +1,218 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DatabaseManager = void 0;
+const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
+const sqliteVec = __importStar(require("sqlite-vec"));
+class DatabaseManager {
+    constructor(dbPath) {
+        // Cache counts for immediate UI feedback
+        this._trackedFilesCount = 0;
+        this._totalChunksCount = 0;
+        this.db = new better_sqlite3_1.default(dbPath);
+        sqliteVec.load(this.db);
+        this.createTables();
+        // Prepare statements for insert/update (matching doc2vec's approach)
+        this.insertStmt = this.db.prepare(`
+            INSERT INTO vec_items (embedding, version, heading_hierarchy, section, chunk_id, content, url, hash, chunk_index, total_chunks)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        `);
+        this.updateStmt = this.db.prepare(`
+            UPDATE vec_items SET embedding = ?, version = ?, heading_hierarchy = ?, section = ?, content = ?, url = ?, hash = ?, chunk_index = ?, total_chunks = ?
+            WHERE chunk_id = ?
+        `);
+        // Initialize counts from database
+        this._trackedFilesCount = this._queryTrackedFilesCount();
+        this._totalChunksCount = this._queryTotalChunksCount();
+        console.log(`Database initialized at: ${dbPath}`);
+    }
+    _queryTrackedFilesCount() {
+        const stmt = this.db.prepare('SELECT COUNT(*) as count FROM files');
+        const row = stmt.get();
+        return Number(row.count);
+    }
+    _queryTotalChunksCount() {
+        const stmt = this.db.prepare('SELECT COUNT(*) as count FROM vec_items');
+        const row = stmt.get();
+        return Number(row.count);
+    }
+    createTables() {
+        // Create vec0 virtual table (sqlite-vec)
+        this.db.exec(`
+            CREATE VIRTUAL TABLE IF NOT EXISTS vec_items USING vec0(
+                embedding FLOAT[3072],
+                version TEXT,
+                heading_hierarchy TEXT,
+                section TEXT,
+                chunk_id TEXT UNIQUE,
+                content TEXT,
+                url TEXT,
+                hash TEXT,
+                chunk_index INTEGER,
+                total_chunks INTEGER
+            )
+        `);
+        // Create files tracking table
+        this.db.exec(`
+            CREATE TABLE IF NOT EXISTS files (
+                path TEXT PRIMARY KEY,
+                hash TEXT,
+                modified_at TEXT,
+                chunk_count INTEGER
+            )
+        `);
+        console.log('Database tables created');
+    }
+    insertChunk(chunk, embedding) {
+        const embeddingData = embedding ? new Float32Array(embedding) : new Float32Array(3072).fill(0);
+        // Use JSON.stringify for heading_hierarchy to match doc2vec format
+        const headingHierarchyJson = JSON.stringify(chunk.headingHierarchy);
+        try {
+            this.insertStmt.run(embeddingData, chunk.version, headingHierarchyJson, chunk.section, chunk.chunkId, chunk.content, chunk.url, chunk.hash, BigInt(chunk.chunkIndex), BigInt(chunk.totalChunks));
+            this._totalChunksCount++;
+        }
+        catch (error) {
+            // If insert fails due to UNIQUE constraint, update instead
+            if (error.code === 'SQLITE_CONSTRAINT_UNIQUE' || error.message?.includes('UNIQUE constraint failed')) {
+                this.updateStmt.run(embeddingData, chunk.version, headingHierarchyJson, chunk.section, chunk.content, chunk.url, chunk.hash, BigInt(chunk.chunkIndex), BigInt(chunk.totalChunks), chunk.chunkId);
+                // Update doesn't change count
+            }
+            else {
+                throw error;
+            }
+        }
+    }
+    removeChunksForFile(filePath) {
+        const url = `file://${filePath}`;
+        // Get count before delete
+        const countStmt = this.db.prepare('SELECT COUNT(*) as count FROM vec_items WHERE url = ?');
+        const countRow = countStmt.get(url);
+        const deletedCount = Number(countRow.count);
+        const stmt = this.db.prepare('DELETE FROM vec_items WHERE url = ?');
+        stmt.run(url);
+        this._totalChunksCount -= deletedCount;
+    }
+    getExistingChunkHash(chunkId) {
+        const stmt = this.db.prepare('SELECT hash FROM vec_items WHERE chunk_id = ?');
+        const row = stmt.get(chunkId);
+        return row?.hash ?? null;
+    }
+    getTotalChunksCount() {
+        return this._totalChunksCount;
+    }
+    upsertFileInfo(filePath, hash, modifiedAt, chunkCount) {
+        // Check if file already exists
+        const checkStmt = this.db.prepare('SELECT 1 FROM files WHERE path = ?');
+        const exists = checkStmt.get(filePath);
+        const stmt = this.db.prepare(`
+            INSERT OR REPLACE INTO files (path, hash, modified_at, chunk_count)
+            VALUES (?, ?, ?, ?)
+        `);
+        stmt.run(filePath, hash, modifiedAt.toISOString(), chunkCount);
+        // Only increment if this is a new file
+        if (!exists) {
+            this._trackedFilesCount++;
+        }
+    }
+    getFileInfo(filePath) {
+        const stmt = this.db.prepare('SELECT * FROM files WHERE path = ?');
+        const row = stmt.get(filePath);
+        if (!row)
+            return null;
+        return {
+            path: row.path,
+            hash: row.hash,
+            modifiedAt: new Date(row.modified_at),
+            chunkCount: row.chunk_count
+        };
+    }
+    removeFileInfo(filePath) {
+        // Check if file exists before deleting
+        const checkStmt = this.db.prepare('SELECT 1 FROM files WHERE path = ?');
+        const exists = checkStmt.get(filePath);
+        const stmt = this.db.prepare('DELETE FROM files WHERE path = ?');
+        stmt.run(filePath);
+        if (exists) {
+            this._trackedFilesCount--;
+        }
+    }
+    getAllTrackedFiles() {
+        const stmt = this.db.prepare('SELECT path FROM files');
+        const rows = stmt.all();
+        return rows.map(r => r.path);
+    }
+    getAllTrackedFilesWithInfo() {
+        const stmt = this.db.prepare('SELECT path, chunk_count, modified_at FROM files ORDER BY modified_at DESC');
+        const rows = stmt.all();
+        return rows.map(r => ({
+            path: r.path,
+            chunkCount: r.chunk_count,
+            modifiedAt: r.modified_at
+        }));
+    }
+    getChunksForFile(filePath) {
+        const url = `file://${filePath}`;
+        const stmt = this.db.prepare(`
+            SELECT chunk_id, content, section, chunk_index 
+            FROM vec_items 
+            WHERE url = ? 
+            ORDER BY chunk_index
+        `);
+        const rows = stmt.all(url);
+        return rows.map(r => ({
+            chunkId: r.chunk_id,
+            content: r.content,
+            section: r.section,
+            chunkIndex: Number(r.chunk_index)
+        }));
+    }
+    getTrackedFilesCount() {
+        return this._trackedFilesCount;
+    }
+    clearAllData() {
+        this.db.exec('DELETE FROM vec_items');
+        this.db.exec('DELETE FROM files');
+        this._trackedFilesCount = 0;
+        this._totalChunksCount = 0;
+        console.log('Cleared all data');
+    }
+    close() {
+        this.db.close();
+    }
+}
+exports.DatabaseManager = DatabaseManager;

package/dist/electron-app/src/embeddings.js

@@ -0,0 +1,80 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EmbeddingService = exports.InvalidApiKeyError = void 0;
+const openai_1 = __importDefault(require("openai"));
+class InvalidApiKeyError extends Error {
+    constructor(message = 'Invalid OpenAI API key') {
+        super(message);
+        this.name = 'InvalidApiKeyError';
+    }
+}
+exports.InvalidApiKeyError = InvalidApiKeyError;
+class EmbeddingService {
+    constructor(apiKey) {
+        this.model = 'text-embedding-3-large';
+        this._isValid = true;
+        this.client = new openai_1.default({ apiKey });
+    }
+    get isValid() {
+        return this._isValid;
+    }
+    async validateApiKey() {
+        try {
+            // Make a minimal API call to validate the key
+            await this.client.embeddings.create({
+                model: this.model,
+                input: 'test'
+            });
+            return true;
+        }
+        catch (error) {
+            if (error?.status === 401 || error?.code === 'invalid_api_key' ||
+                error?.message?.includes('Incorrect API key') ||
+                error?.message?.includes('invalid_api_key')) {
+                this._isValid = false;
+                return false;
+            }
+            // Other errors (rate limit, etc.) - key might still be valid
+            throw error;
+        }
+    }
+    async generateEmbedding(text) {
+        if (!this._isValid) {
+            throw new InvalidApiKeyError();
+        }
+        try {
+            const response = await this.client.embeddings.create({
+                model: this.model,
+                input: text
+            });
+            if (!response.data?.[0]?.embedding) {
+                throw new Error('Failed to get embedding from OpenAI');
+            }
+            return response.data[0].embedding;
+        }
+        catch (error) {
+            // Check for authentication errors (401)
+            if (error?.status === 401 || error?.code === 'invalid_api_key' ||
+                error?.message?.includes('Incorrect API key') ||
+                error?.message?.includes('invalid_api_key')) {
+                this._isValid = false;
+                throw new InvalidApiKeyError(error.message || 'Invalid OpenAI API key');
+            }
+            throw error;
+        }
+    }
+    async generateEmbeddings(texts) {
+        const embeddings = [];
+        for (const text of texts) {
+            const embedding = await this.generateEmbedding(text);
+            embeddings.push(embedding);
+            // Small delay to avoid rate limits
+            await new Promise(resolve => setTimeout(resolve, 100));
+        }
+        return embeddings;
+    }
+}
+exports.EmbeddingService = EmbeddingService;