doc2vec 1.2.0 → 2.0.0

package/dist/electron-app/src/embeddings.js

@@ -0,0 +1,80 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EmbeddingService = exports.InvalidApiKeyError = void 0;
+const openai_1 = __importDefault(require("openai"));
+class InvalidApiKeyError extends Error {
+    constructor(message = 'Invalid OpenAI API key') {
+        super(message);
+        this.name = 'InvalidApiKeyError';
+    }
+}
+exports.InvalidApiKeyError = InvalidApiKeyError;
+class EmbeddingService {
+    constructor(apiKey) {
+        this.model = 'text-embedding-3-large';
+        this._isValid = true;
+        this.client = new openai_1.default({ apiKey });
+    }
+    get isValid() {
+        return this._isValid;
+    }
+    async validateApiKey() {
+        try {
+            // Make a minimal API call to validate the key
+            await this.client.embeddings.create({
+                model: this.model,
+                input: 'test'
+            });
+            return true;
+        }
+        catch (error) {
+            if (error?.status === 401 || error?.code === 'invalid_api_key' ||
+                error?.message?.includes('Incorrect API key') ||
+                error?.message?.includes('invalid_api_key')) {
+                this._isValid = false;
+                return false;
+            }
+            // Other errors (rate limit, etc.) - key might still be valid
+            throw error;
+        }
+    }
+    async generateEmbedding(text) {
+        if (!this._isValid) {
+            throw new InvalidApiKeyError();
+        }
+        try {
+            const response = await this.client.embeddings.create({
+                model: this.model,
+                input: text
+            });
+            if (!response.data?.[0]?.embedding) {
+                throw new Error('Failed to get embedding from OpenAI');
+            }
+            return response.data[0].embedding;
+        }
+        catch (error) {
+            // Check for authentication errors (401)
+            if (error?.status === 401 || error?.code === 'invalid_api_key' ||
+                error?.message?.includes('Incorrect API key') ||
+                error?.message?.includes('invalid_api_key')) {
+                this._isValid = false;
+                throw new InvalidApiKeyError(error.message || 'Invalid OpenAI API key');
+            }
+            throw error;
+        }
+    }
+    async generateEmbeddings(texts) {
+        const embeddings = [];
+        for (const text of texts) {
+            const embedding = await this.generateEmbedding(text);
+            embeddings.push(embedding);
+            // Small delay to avoid rate limits
+            await new Promise(resolve => setTimeout(resolve, 100));
+        }
+        return embeddings;
+    }
+}
+exports.EmbeddingService = EmbeddingService;

package/dist/electron-app/src/main.js

@@ -0,0 +1,627 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const electron_1 = require("electron");
+const path = __importStar(require("path"));
+const electron_store_1 = __importDefault(require("electron-store"));
+const syncer_1 = require("./syncer");
+const database_1 = require("./database");
+const processor_1 = require("./processor");
+const embeddings_1 = require("./embeddings");
+const mcp_server_1 = require("./mcp-server");
+// Store for persistent settings
+const store = new electron_store_1.default({
+    defaults: {
+        watchedFolder: '',
+        databasePath: '',
+        openAIApiKey: '',
+        version: '1.0.0',
+        fileExtensions: '.md,.txt,.html,.pdf,.doc,.docx',
+        recursive: true,
+        mcpServerEnabled: false,
+        mcpServerPort: 3333
+    }
+});
+let mainWindow = null;
+let tray = null;
+let syncer = null;
+let database = null;
+let processor = null;
+let embeddingService = null;
+let mcpServer = null;
+let currentStatus = 'idle'; // 'idle', 'watching', 'processing'
+let processedCount = 0;
+let totalChunks = 0;
+let isQuitting = false;
+function createWindow() {
+    mainWindow = new electron_1.BrowserWindow({
+        width: 600,
+        height: 750,
+        webPreferences: {
+            nodeIntegration: false,
+            contextIsolation: true,
+            preload: path.join(__dirname, 'preload.js')
+        },
+        title: 'Docs4ai',
+        show: false
+    });
+    // Load HTML from src directory (not dist)
+    mainWindow.loadFile(path.join(__dirname, '..', 'src', 'index.html'));
+    mainWindow.once('ready-to-show', () => {
+        mainWindow?.show();
+    });
+    mainWindow.on('close', (event) => {
+        if (process.platform === 'darwin' && !isQuitting) {
+            event.preventDefault();
+            mainWindow?.hide();
+        }
+    });
+}
+// Create tray icons for different states
+function createTrayIcon(state) {
+    const size = 22; // macOS menu bar size
+    const canvas = Buffer.alloc(size * size * 4);
+    // Status colors
+    let statusColor = { r: 128, g: 128, b: 128 }; // Gray for idle
+    if (state === 'syncing') {
+        statusColor = { r: 52, g: 199, b: 89 }; // Green
+    }
+    else if (state === 'processing') {
+        statusColor = { r: 255, g: 149, b: 0 }; // Orange
+    }
+    // Draw a document shape
+    for (let y = 2; y < 20; y++) {
+        for (let x = 4; x < 18; x++) {
+            const idx = (y * size + x) * 4;
+            // Folded corner area (top right)
+            if (x >= 13 && y < 7 && (x - 13) + (7 - y) < 5) {
+                // Folded part - lighter
+                if ((x - 13) + (7 - y) === 4) {
+                    canvas[idx] = 80;
+                    canvas[idx + 1] = 80;
+                    canvas[idx + 2] = 80;
+                    canvas[idx + 3] = 255;
+                }
+            }
+            // Document outline
+            else if (y === 2 || y === 19 || x === 4 || x === 17 || (x === 13 && y < 7) || (y === 7 && x > 13)) {
+                canvas[idx] = 60;
+                canvas[idx + 1] = 60;
+                canvas[idx + 2] = 60;
+                canvas[idx + 3] = 255;
+            }
+            // Document fill
+            else if (y > 2 && y < 19 && x > 4 && x < 17) {
+                canvas[idx] = 40;
+                canvas[idx + 1] = 40;
+                canvas[idx + 2] = 40;
+                canvas[idx + 3] = 180;
+            }
+        }
+    }
+    // Draw status indicator dot (bottom right)
+    const dotX = 15;
+    const dotY = 17;
+    const dotRadius = 3;
+    for (let dy = -dotRadius; dy <= dotRadius; dy++) {
+        for (let dx = -dotRadius; dx <= dotRadius; dx++) {
+            if (dx * dx + dy * dy <= dotRadius * dotRadius) {
+                const px = dotX + dx;
+                const py = dotY + dy;
+                if (px >= 0 && px < size && py >= 0 && py < size) {
+                    const idx = (py * size + px) * 4;
+                    canvas[idx] = statusColor.r;
+                    canvas[idx + 1] = statusColor.g;
+                    canvas[idx + 2] = statusColor.b;
+                    canvas[idx + 3] = 255;
+                }
+            }
+        }
+    }
+    const icon = electron_1.nativeImage.createFromBuffer(canvas, { width: size, height: size });
+    if (process.platform === 'darwin') {
+        icon.setTemplateImage(true);
+    }
+    return icon;
+}
+function updateTray() {
+    if (!tray)
+        return;
+    const isSyncing = syncer?.isSyncing ?? false;
+    const state = currentStatus === 'processing' ? 'processing' : (isSyncing ? 'syncing' : 'idle');
+    tray.setImage(createTrayIcon(state));
+    let tooltip = 'Docs4ai';
+    if (currentStatus === 'processing') {
+        tooltip = `Docs4ai - Processing...`;
+    }
+    else if (isSyncing) {
+        tooltip = `Docs4ai - Syncing (${processedCount} files, ${totalChunks} chunks)`;
+    }
+    else {
+        tooltip = 'Docs4ai - Not syncing';
+    }
+    tray.setToolTip(tooltip);
+    // Update context menu
+    const contextMenu = electron_1.Menu.buildFromTemplate([
+        {
+            label: isSyncing ? '● Syncing' : '○ Not syncing',
+            enabled: false
+        },
+        {
+            label: `Files: ${processedCount} | Chunks: ${totalChunks}`,
+            enabled: false
+        },
+        { type: 'separator' },
+        { label: 'Show Docs4ai', click: () => mainWindow?.show() },
+        { type: 'separator' },
+        {
+            label: isSyncing ? 'Stop Sync' : 'Start Sync',
+            click: async () => {
+                if (isSyncing) {
+                    syncCancelled = true;
+                    syncer?.stop();
+                    syncer = null;
+                    sendStats();
+                    updateTray();
+                }
+                else {
+                    // Start watching using the same logic as IPC handler
+                    await startWatchingInternal();
+                }
+            }
+        },
+        { type: 'separator' },
+        { label: 'Quit', click: () => electron_1.app.quit() }
+    ]);
+    tray.setContextMenu(contextMenu);
+}
+function createTray() {
+    tray = new electron_1.Tray(createTrayIcon('idle'));
+    updateTray();
+    tray.on('click', () => {
+        mainWindow?.show();
+    });
+}
+// IPC Handlers
+function setupIpcHandlers() {
+    // Get settings
+    electron_1.ipcMain.handle('get-settings', () => {
+        return {
+            watchedFolder: store.get('watchedFolder'),
+            databasePath: store.get('databasePath'),
+            openAIApiKey: store.get('openAIApiKey'),
+            version: store.get('version'),
+            fileExtensions: store.get('fileExtensions'),
+            recursive: store.get('recursive'),
+            mcpServerEnabled: store.get('mcpServerEnabled'),
+            mcpServerPort: store.get('mcpServerPort')
+        };
+    });
+    // Save settings
+    electron_1.ipcMain.handle('save-settings', (_event, settings) => {
+        Object.entries(settings).forEach(([key, value]) => {
+            store.set(key, value);
+        });
+        // Update embedding service if API key changed
+        if (settings.openAIApiKey) {
+            embeddingService = new embeddings_1.EmbeddingService(settings.openAIApiKey);
+            // Also update MCP server
+            if (mcpServer) {
+                mcpServer.setApiKey(settings.openAIApiKey);
+            }
+        }
+        return true;
+    });
+    // Start MCP server
+    electron_1.ipcMain.handle('start-mcp-server', async () => {
+        const port = store.get('mcpServerPort');
+        const dbPath = store.get('databasePath');
+        const apiKey = store.get('openAIApiKey');
+        if (!dbPath) {
+            return { success: false, error: 'Database path not configured' };
+        }
+        if (!apiKey) {
+            return { success: false, error: 'OpenAI API key required for MCP server' };
+        }
+        try {
+            if (mcpServer?.isRunning()) {
+                await mcpServer.stop();
+            }
+            mcpServer = new mcp_server_1.McpServer(port);
+            mcpServer.setDatabase(dbPath);
+            mcpServer.setApiKey(apiKey);
+            await mcpServer.start();
+            store.set('mcpServerEnabled', true);
+            return { success: true, port };
+        }
+        catch (error) {
+            return { success: false, error: error.message };
+        }
+    });
+    // Stop MCP server
+    electron_1.ipcMain.handle('stop-mcp-server', async () => {
+        try {
+            if (mcpServer) {
+                await mcpServer.stop();
+                mcpServer = null;
+            }
+            store.set('mcpServerEnabled', false);
+            return { success: true };
+        }
+        catch (error) {
+            return { success: false, error: error.message };
+        }
+    });
+    // Get MCP server status
+    electron_1.ipcMain.handle('get-mcp-status', () => {
+        return {
+            running: mcpServer?.isRunning() ?? false,
+            port: mcpServer?.getPort() ?? store.get('mcpServerPort')
+        };
+    });
+    // Select folder
+    electron_1.ipcMain.handle('select-folder', async () => {
+        const result = await electron_1.dialog.showOpenDialog(mainWindow, {
+            properties: ['openDirectory']
+        });
+        if (!result.canceled && result.filePaths.length > 0) {
+            return result.filePaths[0];
+        }
+        return null;
+    });
+    // Select database file
+    electron_1.ipcMain.handle('select-database', async () => {
+        const result = await electron_1.dialog.showSaveDialog(mainWindow, {
+            defaultPath: 'docsync.db',
+            filters: [{ name: 'SQLite Database', extensions: ['db'] }]
+        });
+        if (!result.canceled && result.filePath) {
+            return result.filePath;
+        }
+        return null;
+    });
+    // Get stats
+    electron_1.ipcMain.handle('get-stats', () => {
+        return {
+            isSyncing: syncer?.isSyncing ?? false,
+            trackedFiles: database?.getTrackedFilesCount() ?? 0,
+            totalChunks: database?.getTotalChunksCount() ?? 0
+        };
+    });
+    // Get all tracked files with info
+    electron_1.ipcMain.handle('get-files', () => {
+        if (!database)
+            return [];
+        return database.getAllTrackedFilesWithInfo();
+    });
+    // Get chunks for a specific file
+    electron_1.ipcMain.handle('get-chunks', (_event, filePath) => {
+        if (!database)
+            return [];
+        return database.getChunksForFile(filePath);
+    });
+    // Start watching
+    electron_1.ipcMain.handle('start-watching', async () => {
+        return await startWatchingInternal();
+    });
+    // Stop watching
+    electron_1.ipcMain.handle('stop-watching', () => {
+        syncCancelled = true; // Cancel any running sync
+        syncer?.stop();
+        syncer = null;
+        sendStats();
+        return { success: true };
+    });
+    // Force full sync
+    electron_1.ipcMain.handle('force-sync', async () => {
+        const version = store.get('version');
+        if (!database) {
+            return { success: false, error: 'Database not initialized' };
+        }
+        database.clearAllData();
+        await performInitialSync(version, true); // Force reprocess all
+        sendStats();
+        return { success: true };
+    });
+}
+// Internal function to start watching - used by both IPC handler and tray menu
+async function startWatchingInternal() {
+    const watchedFolder = store.get('watchedFolder');
+    const databasePath = store.get('databasePath');
+    const openAIApiKey = store.get('openAIApiKey');
+    const version = store.get('version');
+    const fileExtensions = store.get('fileExtensions');
+    const recursive = store.get('recursive');
+    if (!watchedFolder) {
+        return { success: false, error: 'No folder selected' };
+    }
+    if (!databasePath) {
+        return { success: false, error: 'No database path selected' };
+    }
+    if (!openAIApiKey) {
+        return { success: false, error: 'OpenAI API key is required for generating embeddings' };
+    }
+    try {
+        // Initialize database
+        database = new database_1.DatabaseManager(databasePath);
+        // Initialize processor
+        processor = new processor_1.ContentProcessor();
+        // Initialize embedding service (required for semantic search)
+        embeddingService = new embeddings_1.EmbeddingService(openAIApiKey);
+        // Validate API key before starting sync
+        console.log('Validating OpenAI API key...');
+        try {
+            const isValid = await embeddingService.validateApiKey();
+            if (!isValid) {
+                embeddingService = null;
+                return { success: false, error: 'Invalid OpenAI API key' };
+            }
+            console.log('API key validated successfully');
+        }
+        catch (error) {
+            embeddingService = null;
+            return { success: false, error: `Failed to validate API key: ${error.message}` };
+        }
+        // Parse extensions
+        const extensions = fileExtensions.split(',').map(e => e.trim()).filter(e => e);
+        // Create syncer
+        syncer = new syncer_1.FolderSyncer(watchedFolder, {
+            recursive,
+            extensions,
+            onFileAdd: async (filePath) => {
+                await processFile(filePath, version);
+                sendStats();
+            },
+            onFileChange: async (filePath) => {
+                await processFile(filePath, version);
+                sendStats();
+            },
+            onFileDelete: async (filePath) => {
+                database?.removeChunksForFile(filePath);
+                database?.removeFileInfo(filePath);
+                sendStats();
+            }
+        });
+        syncer.start();
+        // Reset sync cancelled flag
+        syncCancelled = false;
+        // Initial sync
+        await performInitialSync(version);
+        sendStats();
+        updateTray();
+        return { success: true };
+    }
+    catch (error) {
+        console.error('Error starting syncer:', error);
+        return { success: false, error: error.message };
+    }
+}
+// Flag to track if sync is in progress and should continue
+let syncCancelled = false;
+async function processFile(filePath, version, forceReprocess = false) {
+    if (!database || !processor)
+        return;
+    // Check if sync was cancelled
+    if (syncCancelled)
+        return;
+    try {
+        // Check if file needs processing (incremental sync)
+        if (!forceReprocess) {
+            const fileInfo = database.getFileInfo(filePath);
+            if (fileInfo) {
+                // Get file's current modification time
+                const fs = require('fs');
+                try {
+                    const stats = fs.statSync(filePath);
+                    const currentModTime = stats.mtime;
+                    // Skip if file hasn't been modified since last sync
+                    if (currentModTime <= fileInfo.modifiedAt) {
+                        console.log(`Skipping (unchanged): ${filePath}`);
+                        return;
+                    }
+                }
+                catch {
+                    // File might not exist, continue processing
+                }
+            }
+        }
+        currentStatus = 'processing';
+        updateTray();
+        const content = await processor.readFile(filePath);
+        if (!content) {
+            currentStatus = syncer?.isSyncing ? 'syncing' : 'idle';
+            updateTray();
+            return;
+        }
+        // Check content hash - skip if content is identical
+        const hash = processor.generateHash(content);
+        if (!forceReprocess) {
+            const fileInfo = database.getFileInfo(filePath);
+            if (fileInfo && fileInfo.hash === hash) {
+                console.log(`Skipping (same hash): ${filePath}`);
+                // Update modification time even if content unchanged
+                database.upsertFileInfo(filePath, hash, new Date(), fileInfo.chunkCount);
+                currentStatus = syncer?.isSyncing ? 'syncing' : 'idle';
+                updateTray();
+                return;
+            }
+        }
+        const chunks = processor.chunkContent(content, filePath, version);
+        // Remove old chunks
+        database.removeChunksForFile(filePath);
+        // Generate embeddings and insert chunks
+        for (const chunk of chunks) {
+            let embedding = null;
+            if (embeddingService) {
+                try {
+                    embedding = await embeddingService.generateEmbedding(chunk.content);
+                }
+                catch (error) {
+                    if (error instanceof embeddings_1.InvalidApiKeyError) {
+                        console.error('Invalid API key detected - stopping sync');
+                        handleInvalidApiKey();
+                        return; // Stop processing this file
+                    }
+                    console.error('Error generating embedding:', error);
+                }
+            }
+            database.insertChunk(chunk, embedding);
+        }
+        // Update file info with current timestamp
+        database.upsertFileInfo(filePath, hash, new Date(), chunks.length);
+        console.log(`Processed: ${filePath} (${chunks.length} chunks)`);
+    }
+    catch (error) {
+        console.error(`Error processing ${filePath}:`, error);
+    }
+    finally {
+        currentStatus = syncer?.isSyncing ? 'syncing' : 'idle';
+        updateTray();
+    }
+}
+async function performInitialSync(version, forceReprocess = false) {
+    if (!syncer || !database)
+        return;
+    const files = syncer.getSyncedFiles();
+    console.log(`Initial sync: ${files.length} files (force=${forceReprocess})`);
+    let processed = 0;
+    let skipped = 0;
+    for (let i = 0; i < files.length; i++) {
+        // Check if sync was cancelled
+        if (syncCancelled) {
+            console.log('Sync cancelled');
+            break;
+        }
+        const filePath = files[i];
+        // Check if file needs processing before logging
+        const fileInfo = database.getFileInfo(filePath);
+        const needsProcessing = forceReprocess || !fileInfo;
+        if (!needsProcessing && fileInfo) {
+            const fs = require('fs');
+            try {
+                const stats = fs.statSync(filePath);
+                if (stats.mtime <= fileInfo.modifiedAt) {
+                    skipped++;
+                    continue; // Skip unchanged files silently
+                }
+            }
+            catch {
+                // Process if we can't stat
+            }
+        }
+        console.log(`Processing ${i + 1}/${files.length}: ${filePath}`);
+        await processFile(filePath, version, forceReprocess);
+        processed++;
+        // Update stats after each file during initial sync
+        sendStats();
+    }
+    if (!syncCancelled) {
+        console.log(`Sync complete: ${processed} processed, ${skipped} skipped (unchanged)`);
+    }
+}
+function sendStats() {
+    processedCount = database?.getTrackedFilesCount() ?? 0;
+    totalChunks = database?.getTotalChunksCount() ?? 0;
+    const stats = {
+        isSyncing: syncer?.isSyncing ?? false,
+        trackedFiles: processedCount,
+        totalChunks: totalChunks
+    };
+    console.log('Sending stats:', stats);
+    if (mainWindow && !mainWindow.isDestroyed()) {
+        try {
+            mainWindow.webContents.send('stats-update', stats);
+        }
+        catch (err) {
+            console.error('Error sending stats:', err);
+        }
+    }
+    updateTray();
+}
+function handleInvalidApiKey() {
+    // Cancel sync and stop watching
+    syncCancelled = true;
+    if (syncer) {
+        syncer.stop();
+        syncer = null;
+    }
+    // Clear the embedding service
+    embeddingService = null;
+    // Notify the UI
+    if (mainWindow && !mainWindow.isDestroyed()) {
+        try {
+            mainWindow.webContents.send('api-key-error', 'Invalid OpenAI API key. Please check your API key and try again.');
+            mainWindow.show(); // Bring window to front
+        }
+        catch (err) {
+            console.error('Error sending API key error:', err);
+        }
+    }
+    sendStats();
+    updateTray();
+}
+electron_1.app.whenReady().then(() => {
+    createWindow();
+    createTray();
+    setupIpcHandlers();
+    electron_1.app.on('activate', () => {
+        if (electron_1.BrowserWindow.getAllWindows().length === 0) {
+            createWindow();
+        }
+        else {
+            mainWindow?.show();
+        }
+    });
+});
+electron_1.app.on('window-all-closed', () => {
+    if (process.platform !== 'darwin') {
+        electron_1.app.quit();
+    }
+});
+electron_1.app.on('before-quit', async () => {
+    isQuitting = true;
+    syncCancelled = true;
+    console.log('Quitting app...');
+    syncer?.stop();
+    if (mcpServer?.isRunning()) {
+        await mcpServer.stop();
+    }
+    database?.close();
+    tray?.destroy();
+    console.log('Cleanup complete');
+});