doc2vec 1.3.0 → 2.2.0

package/dist/electron-app/src/processor.js

@@ -0,0 +1,248 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ContentProcessor = void 0;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const crypto = __importStar(require("crypto"));
+class ContentProcessor {
+    constructor() {
+        this.maxTokens = 1000;
+        this.minTokens = 150;
+        this.overlapPercent = 0.1;
+    }
+    async readFile(filePath) {
+        const ext = path.extname(filePath).toLowerCase();
+        try {
+            switch (ext) {
+                case '.pdf':
+                    return await this.readPdf(filePath);
+                case '.doc':
+                    return await this.readDoc(filePath);
+                case '.docx':
+                    return await this.readDocx(filePath);
+                case '.html':
+                case '.htm':
+                    const html = fs.readFileSync(filePath, 'utf-8');
+                    return this.stripHtml(html);
+                default:
+                    return fs.readFileSync(filePath, 'utf-8');
+            }
+        }
+        catch (error) {
+            console.error(`Error reading file ${filePath}:`, error);
+            return null;
+        }
+    }
+    async readPdf(filePath) {
+        try {
+            const pdfParse = require('pdf-parse');
+            const buffer = fs.readFileSync(filePath);
+            const data = await pdfParse(buffer);
+            return `# ${path.basename(filePath, '.pdf')}\n\n${data.text}`;
+        }
+        catch (error) {
+            if (error.code === 'MODULE_NOT_FOUND') {
+                console.warn(`PDF parsing not available for ${filePath}. Install pdf-parse for PDF support.`);
+                return `# ${path.basename(filePath, '.pdf')}\n\n[PDF content - install pdf-parse for extraction]`;
+            }
+            throw error;
+        }
+    }
+    async readDoc(filePath) {
+        try {
+            const WordExtractor = require('word-extractor');
+            const extractor = new WordExtractor();
+            const extracted = await extractor.extract(filePath);
+            const text = extracted.getBody();
+            // Create content with filename as title
+            let content = `# ${path.basename(filePath, '.doc')}\n\n`;
+            // Clean up the text
+            const cleanedText = text
+                .replace(/\r\n/g, '\n')
+                .replace(/\n{3,}/g, '\n\n')
+                .trim();
+            content += cleanedText;
+            return content;
+        }
+        catch (error) {
+            if (error.code === 'MODULE_NOT_FOUND') {
+                console.warn(`DOC parsing not available for ${filePath}. Install word-extractor for DOC support.`);
+                return `# ${path.basename(filePath, '.doc')}\n\n[DOC content - install word-extractor for extraction]`;
+            }
+            throw error;
+        }
+    }
+    async readDocx(filePath) {
+        try {
+            const mammoth = require('mammoth');
+            const result = await mammoth.convertToHtml({ path: filePath });
+            const html = result.value;
+            // Log any warnings
+            if (result.messages.length > 0) {
+                console.log(`Mammoth warnings for ${filePath}: ${result.messages.map((m) => m.message).join(', ')}`);
+            }
+            // Create content with filename as title
+            let content = `# ${path.basename(filePath, '.docx')}\n\n`;
+            // Strip HTML to get plain text
+            content += this.stripHtml(html);
+            // Clean up excessive line breaks
+            content = content.replace(/\n{3,}/g, '\n\n').trim();
+            return content;
+        }
+        catch (error) {
+            if (error.code === 'MODULE_NOT_FOUND') {
+                console.warn(`DOCX parsing not available for ${filePath}. Install mammoth for DOCX support.`);
+                return `# ${path.basename(filePath, '.docx')}\n\n[DOCX content - install mammoth for extraction]`;
+            }
+            throw error;
+        }
+    }
+    stripHtml(html) {
+        return html
+            .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
+            .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
+            .replace(/<[^>]+>/g, '')
+            .replace(/&nbsp;/g, ' ')
+            .replace(/&amp;/g, '&')
+            .replace(/&lt;/g, '<')
+            .replace(/&gt;/g, '>')
+            .replace(/&quot;/g, '"')
+            .replace(/\s+/g, ' ')
+            .trim();
+    }
+    generateHash(content) {
+        return crypto.createHash('sha256').update(content).digest('hex');
+    }
+    tokenize(text) {
+        return text.split(/(\s+)/).filter(token => token.length > 0);
+    }
+    chunkContent(content, filePath, version) {
+        const chunks = [];
+        const lines = content.split('\n');
+        let buffer = '';
+        let headingHierarchy = [];
+        let bufferHeadings = [];
+        let chunkCounter = 0;
+        const computeTopicHierarchy = () => {
+            if (bufferHeadings.length === 0)
+                return headingHierarchy;
+            const deepestLevel = Math.max(...bufferHeadings.map(h => h.level));
+            const deepestHeadings = bufferHeadings.filter(h => h.level === deepestLevel);
+            if (deepestHeadings.length > 1 && deepestLevel > 1) {
+                return headingHierarchy.slice(0, deepestLevel - 1);
+            }
+            return headingHierarchy;
+        };
+        const createChunk = (content, hierarchy) => {
+            const breadcrumbs = hierarchy.filter(h => h).join(' > ');
+            const contextPrefix = breadcrumbs ? `[Topic: ${breadcrumbs}]\n` : '';
+            const searchableText = contextPrefix + content.trim();
+            const chunkId = this.generateHash(searchableText);
+            return {
+                chunkId,
+                content: searchableText,
+                version,
+                section: hierarchy[hierarchy.length - 1] || 'Introduction',
+                headingHierarchy: hierarchy.filter(h => h),
+                chunkIndex: chunkCounter++,
+                totalChunks: 0,
+                url: `file://${filePath}`,
+                hash: chunkId
+            };
+        };
+        const flushBuffer = (force = false) => {
+            const trimmed = buffer.trim();
+            if (!trimmed)
+                return;
+            const tokenCount = this.tokenize(trimmed).length;
+            if (tokenCount < this.minTokens && !force)
+                return;
+            const hierarchy = computeTopicHierarchy();
+            if (tokenCount > this.maxTokens) {
+                const tokens = this.tokenize(trimmed);
+                const overlapSize = Math.floor(this.maxTokens * this.overlapPercent);
+                for (let i = 0; i < tokens.length; i += (this.maxTokens - overlapSize)) {
+                    const subTokens = tokens.slice(i, i + this.maxTokens);
+                    chunks.push(createChunk(subTokens.join(''), hierarchy));
+                }
+            }
+            else {
+                chunks.push(createChunk(trimmed, hierarchy));
+            }
+            buffer = '';
+            bufferHeadings = [];
+        };
+        for (const line of lines) {
+            const isHeading = line.startsWith('#');
+            if (isHeading) {
+                const match = line.match(/^(#+)/);
+                const level = match ? match[1].length : 1;
+                const headingText = line
+                    .replace(/^#+\s*/, '')
+                    .replace(/\[.*?\]\(#[^)]*\)/g, '')
+                    .trim();
+                const currentTokens = this.tokenize(buffer.trim()).length;
+                const hasContent = currentTokens > 0;
+                const isSmall = currentTokens < this.minTokens;
+                const deepestLevel = bufferHeadings.length > 0
+                    ? Math.max(...bufferHeadings.map(h => h.level))
+                    : 0;
+                const shouldMerge = hasContent && isSmall && bufferHeadings.length > 0 && level >= deepestLevel;
+                if (!shouldMerge && hasContent) {
+                    flushBuffer();
+                }
+                headingHierarchy = headingHierarchy.slice(0, level - 1);
+                headingHierarchy[level - 1] = headingText;
+                bufferHeadings.push({ level, text: headingText });
+                buffer += line + '\n';
+            }
+            else {
+                buffer += line + '\n';
+                if (this.tokenize(buffer).length >= this.maxTokens) {
+                    flushBuffer();
+                }
+            }
+        }
+        flushBuffer(true);
+        // Update total chunks
+        const total = chunks.length;
+        chunks.forEach(chunk => {
+            chunk.totalChunks = total;
+        });
+        return chunks;
+    }
+}
+exports.ContentProcessor = ContentProcessor;

package/dist/electron-app/src/syncer.js

@@ -0,0 +1,146 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FolderSyncer = void 0;
+const chokidar_1 = __importDefault(require("chokidar"));
+const path = __importStar(require("path"));
+const fs = __importStar(require("fs"));
+class FolderSyncer {
+    constructor(folderPath, options) {
+        this.fsWatcher = null;
+        this._isSyncing = false;
+        this.folderPath = folderPath;
+        this.options = options;
+    }
+    get isSyncing() {
+        return this._isSyncing;
+    }
+    start() {
+        if (this._isSyncing)
+            return;
+        const globPattern = this.options.recursive
+            ? path.join(this.folderPath, '**', '*')
+            : path.join(this.folderPath, '*');
+        this.fsWatcher = chokidar_1.default.watch(globPattern, {
+            ignored: /(^|[\/\\])\../, // ignore dotfiles
+            persistent: true,
+            ignoreInitial: true,
+            awaitWriteFinish: {
+                stabilityThreshold: 500,
+                pollInterval: 100
+            }
+        });
+        this.fsWatcher
+            .on('add', async (filePath) => {
+            if (this.shouldProcess(filePath)) {
+                console.log(`File added: ${filePath}`);
+                await this.options.onFileAdd(filePath);
+            }
+        })
+            .on('change', async (filePath) => {
+            if (this.shouldProcess(filePath)) {
+                console.log(`File changed: ${filePath}`);
+                await this.options.onFileChange(filePath);
+            }
+        })
+            .on('unlink', async (filePath) => {
+            // For delete events, only check extension (file no longer exists)
+            if (this.hasValidExtension(filePath)) {
+                console.log(`File deleted: ${filePath}`);
+                await this.options.onFileDelete(filePath);
+            }
+        })
+            .on('error', (error) => {
+            console.error('Syncer error:', error);
+        });
+        this._isSyncing = true;
+        console.log(`Started syncing: ${this.folderPath}`);
+    }
+    stop() {
+        if (this.fsWatcher) {
+            this.fsWatcher.close();
+            this.fsWatcher = null;
+        }
+        this._isSyncing = false;
+        console.log('Stopped syncing');
+    }
+    shouldProcess(filePath) {
+        // Check if it's a file (not directory)
+        try {
+            const stat = fs.statSync(filePath);
+            if (!stat.isFile())
+                return false;
+        }
+        catch {
+            return false;
+        }
+        return this.hasValidExtension(filePath);
+    }
+    hasValidExtension(filePath) {
+        // Check extension
+        if (this.options.extensions.length === 0)
+            return true;
+        const ext = path.extname(filePath).toLowerCase();
+        return this.options.extensions.includes(ext);
+    }
+    getSyncedFiles() {
+        const files = [];
+        const walkDir = (dir) => {
+            try {
+                const entries = fs.readdirSync(dir, { withFileTypes: true });
+                for (const entry of entries) {
+                    if (entry.name.startsWith('.'))
+                        continue;
+                    const fullPath = path.join(dir, entry.name);
+                    if (entry.isDirectory() && this.options.recursive) {
+                        walkDir(fullPath);
+                    }
+                    else if (entry.isFile() && this.shouldProcess(fullPath)) {
+                        files.push(fullPath);
+                    }
+                }
+            }
+            catch (error) {
+                console.error(`Error reading directory ${dir}:`, error);
+            }
+        };
+        walkDir(this.folderPath);
+        return files;
+    }
+}
+exports.FolderSyncer = FolderSyncer;

package/dist/macos-app/migrate-to-vec0.js

@@ -0,0 +1,156 @@
+#!/usr/bin/env npx ts-node
+"use strict";
+/**
+ * Migration script to convert DocSync SQLite database to sqlite-vec format
+ *
+ * This script reads the chunks from the DocSync database (created by the Swift app)
+ * and migrates them to a proper vec0 virtual table that the MCP server can query.
+ *
+ * Usage:
+ *   npx ts-node migrate-to-vec0.ts <source-db> <target-db>
+ *
+ * Example:
+ *   npx ts-node migrate-to-vec0.ts ~/Documents/docsync.db ./my-docs.db
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
+const sqliteVec = __importStar(require("sqlite-vec"));
+const fs = __importStar(require("fs"));
+function migrate(sourceDbPath, targetDbPath) {
+    console.log(`Migrating from ${sourceDbPath} to ${targetDbPath}`);
+    if (!fs.existsSync(sourceDbPath)) {
+        console.error(`Source database not found: ${sourceDbPath}`);
+        process.exit(1);
+    }
+    // Open source database (DocSync format)
+    const sourceDb = new better_sqlite3_1.default(sourceDbPath);
+    // Check if source has vec_items table
+    const tableCheck = sourceDb.prepare(`
+        SELECT name FROM sqlite_master WHERE type='table' AND name='vec_items'
+    `).get();
+    if (!tableCheck) {
+        console.error('Source database does not have vec_items table');
+        process.exit(1);
+    }
+    // Get row count
+    const countResult = sourceDb.prepare('SELECT COUNT(*) as count FROM vec_items').get();
+    console.log(`Found ${countResult.count} chunks to migrate`);
+    if (countResult.count === 0) {
+        console.log('No chunks to migrate');
+        process.exit(0);
+    }
+    // Open/create target database with sqlite-vec
+    const targetDb = new better_sqlite3_1.default(targetDbPath);
+    sqliteVec.load(targetDb);
+    // Create vec0 virtual table
+    targetDb.exec(`
+        CREATE VIRTUAL TABLE IF NOT EXISTS vec_items USING vec0(
+            embedding float[3072],
+            product_name text,
+            version text,
+            heading_hierarchy text,
+            section text,
+            chunk_id text primary key,
+            content text,
+            url text,
+            hash text,
+            chunk_index integer,
+            total_chunks integer
+        )
+    `);
+    console.log('Created vec0 virtual table');
+    // Prepare statements
+    const selectStmt = sourceDb.prepare(`
+        SELECT embedding, product_name, version, heading_hierarchy, section, 
+               chunk_id, content, url, hash, chunk_index, total_chunks
+        FROM vec_items
+    `);
+    const insertStmt = targetDb.prepare(`
+        INSERT OR REPLACE INTO vec_items 
+        (embedding, product_name, version, heading_hierarchy, section, chunk_id, content, url, hash, chunk_index, total_chunks)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `);
+    // Migrate chunks
+    let migratedCount = 0;
+    let skippedCount = 0;
+    const rows = selectStmt.all();
+    for (const row of rows) {
+        try {
+            // The embedding is stored as a blob in the source, convert to Float32Array
+            let embedding;
+            if (row.embedding && row.embedding.length > 0) {
+                // Convert blob to Float32Array
+                const buffer = Buffer.from(row.embedding);
+                embedding = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.length / 4);
+            }
+            else {
+                // Skip rows without embeddings
+                console.log(`Skipping chunk ${row.chunk_id.substring(0, 8)}... (no embedding)`);
+                skippedCount++;
+                continue;
+            }
+            insertStmt.run(embedding, row.product_name, row.version, row.heading_hierarchy, row.section, row.chunk_id, row.content, row.url, row.hash, BigInt(row.chunk_index || 0), BigInt(row.total_chunks || 0));
+            migratedCount++;
+            if (migratedCount % 100 === 0) {
+                console.log(`Migrated ${migratedCount} chunks...`);
+            }
+        }
+        catch (error) {
+            console.error(`Error migrating chunk ${row.chunk_id}:`, error);
+            skippedCount++;
+        }
+    }
+    console.log(`\nMigration complete!`);
+    console.log(`  Migrated: ${migratedCount} chunks`);
+    console.log(`  Skipped: ${skippedCount} chunks`);
+    console.log(`\nTarget database: ${targetDbPath}`);
+    console.log(`You can now use this database with the MCP server.`);
+    sourceDb.close();
+    targetDb.close();
+}
+// Main
+const args = process.argv.slice(2);
+if (args.length < 2) {
+    console.log('Usage: npx ts-node migrate-to-vec0.ts <source-db> <target-db>');
+    console.log('');
+    console.log('Example:');
+    console.log('  npx ts-node migrate-to-vec0.ts ~/Documents/docsync.db ./my-docs.db');
+    process.exit(1);
+}
+migrate(args[0], args[1]);

package/dist/vitest.config.js

@@ -0,0 +1,16 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const config_1 = require("vitest/config");
+exports.default = (0, config_1.defineConfig)({
+    test: {
+        globals: true,
+        environment: 'node',
+        include: ['tests/**/*.test.ts'],
+        testTimeout: 30000,
+        coverage: {
+            provider: 'v8',
+            include: ['utils.ts', 'logger.ts', 'content-processor.ts', 'database.ts', 'code-chunker.ts', 'doc2vec.ts'],
+            exclude: ['mcp/**', 'dist/**', 'tests/**'],
+        },
+    },
+});

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "doc2vec",
-    "version": "1.3.0",
+    "version": "2.2.0",
     "type": "commonjs",
     "description": "",
     "main": "dist/doc2vec.js",
@@ -19,6 +19,9 @@
     "scripts": {
         "build": "tsc && chmod 755 dist/doc2vec.js",
         "start": "node dist/doc2vec.js",
+        "test": "vitest run",
+        "test:watch": "vitest",
+        "test:coverage": "vitest run --coverage",
         "prepublishOnly": "npm run build",
         "prepare": "npm run build"
     },
@@ -26,6 +29,7 @@
     "author": "",
     "license": "ISC",
     "dependencies": {
+        "@chonkiejs/core": "^0.0.7",
         "@mozilla/readability": "^0.4.4",
         "@qdrant/js-client-rest": "^1.13.0",
         "@qdrant/qdrant-js": "^1.13.0",
@@ -36,12 +40,16 @@
         "dotenv": "^16.3.1",
         "js-yaml": "^4.1.0",
         "jsdom": "^26.0.0",
+        "mammoth": "^1.11.0",
         "openai": "^4.20.1",
         "pdfjs-dist": "^5.3.31",
         "puppeteer": "^24.1.1",
         "sanitize-html": "^2.11.0",
         "sqlite-vec": "0.1.7-alpha.2",
-        "turndown": "^7.1.2"
+        "tree-sitter-wasms": "^0.1.0",
+        "turndown": "^7.1.2",
+        "web-tree-sitter": "^0.25.4",
+        "word-extractor": "^1.0.4"
     },
     "devDependencies": {
         "@types/better-sqlite3": "^7.6.12",
@@ -50,7 +58,9 @@
         "@types/node": "^20.10.0",
         "@types/sanitize-html": "^2.9.5",
         "@types/turndown": "^5.0.4",
+        "@vitest/coverage-v8": "^4.0.18",
         "ts-node": "^10.9.1",
-        "typescript": "^5.3.2"
+        "typescript": "^5.3.2",
+        "vitest": "^4.0.18"
     }
 }