ocr-provenance-mcp 1.0.0

package/dist/services/python-pool.js

@@ -0,0 +1,265 @@
+/**
+ * Python Worker Process Pool
+ *
+ * Maintains warm Python processes to amortize import costs (torch, transformers).
+ * Workers accept JSON commands on stdin, return JSON on stdout.
+ * Auto-restart on crash, health checks via heartbeat.
+ *
+ * CRITICAL: NEVER use console.log() - stdout is JSON-RPC protocol.
+ *
+ * @module services/python-pool
+ */
+import { spawn } from 'child_process';
+import { EventEmitter } from 'events';
+// ═══════════════════════════════════════════════════════════════════════════════
+// POOL IMPLEMENTATION
+// ═══════════════════════════════════════════════════════════════════════════════
+export class PythonPool extends EventEmitter {
+    workers = new Map();
+    taskQueue = [];
+    config;
+    healthCheckTimer = null;
+    nextWorkerId = 0;
+    shuttingDown = false;
+    pythonCommand;
+    constructor(config) {
+        super();
+        this.config = {
+            poolSize: config?.poolSize ?? 3,
+            maxTasksPerWorker: config?.maxTasksPerWorker ?? 100,
+            healthCheckIntervalMs: config?.healthCheckIntervalMs ?? 30000,
+            taskTimeoutMs: config?.taskTimeoutMs ?? 120000,
+        };
+        this.pythonCommand = process.platform === 'win32' ? 'python' : 'python3';
+    }
+    /**
+     * Start the pool with the given Python script.
+     *
+     * @param scriptPath - Path to the Python worker script
+     * @throws Error if pool is shutting down
+     */
+    start(scriptPath) {
+        if (this.shuttingDown)
+            throw new Error('Pool is shutting down');
+        for (let i = 0; i < this.config.poolSize; i++) {
+            this.spawnWorker(scriptPath);
+        }
+        this.healthCheckTimer = setInterval(() => this.healthCheck(), this.config.healthCheckIntervalMs);
+        this.healthCheckTimer.unref();
+        console.error(`[PythonPool] Started with ${this.config.poolSize} workers for ${scriptPath}`);
+    }
+    /**
+     * Execute a command on the next available worker.
+     *
+     * If all workers are busy, the task is queued and dispatched when
+     * a worker becomes available.
+     *
+     * @param command - JSON-serializable command object
+     * @returns Promise resolving with the worker's JSON response
+     * @throws Error if pool is shutting down or task times out
+     */
+    async execute(command) {
+        if (this.shuttingDown)
+            throw new Error('Pool is shutting down');
+        return new Promise((resolve, reject) => {
+            const timeoutId = setTimeout(() => {
+                reject(new Error(`Python pool task timed out after ${this.config.taskTimeoutMs}ms`));
+            }, this.config.taskTimeoutMs);
+            const task = { command, resolve, reject, timeoutId };
+            // Try to dispatch immediately to a free worker
+            const worker = this.getAvailableWorker();
+            if (worker) {
+                this.dispatchTask(worker, task);
+            }
+            else {
+                this.taskQueue.push(task);
+            }
+        });
+    }
+    /**
+     * Shut down all workers and reject queued tasks.
+     */
+    async shutdown() {
+        this.shuttingDown = true;
+        if (this.healthCheckTimer) {
+            clearInterval(this.healthCheckTimer);
+            this.healthCheckTimer = null;
+        }
+        // Reject all queued tasks
+        for (const task of this.taskQueue) {
+            clearTimeout(task.timeoutId);
+            task.reject(new Error('Pool shutting down'));
+        }
+        this.taskQueue = [];
+        // Kill all workers
+        for (const [, worker] of this.workers) {
+            try {
+                worker.process.kill('SIGTERM');
+                // Force kill after 5s
+                const forceTimer = setTimeout(() => {
+                    try {
+                        worker.process.kill('SIGKILL');
+                    }
+                    catch {
+                        /* already dead */
+                    }
+                }, 5000);
+                forceTimer.unref();
+            }
+            catch {
+                // Worker already dead
+            }
+        }
+        this.workers.clear();
+        console.error('[PythonPool] Shut down');
+    }
+    /**
+     * Get pool status for diagnostics.
+     */
+    getStatus() {
+        let busy = 0;
+        let idle = 0;
+        for (const w of this.workers.values()) {
+            if (w.busy)
+                busy++;
+            else
+                idle++;
+        }
+        return {
+            total: this.workers.size,
+            busy,
+            idle,
+            queued: this.taskQueue.length,
+        };
+    }
+    // ═════════════════════════════════════════════════════════════════════════════
+    // PRIVATE METHODS
+    // ═════════════════════════════════════════════════════════════════════════════
+    spawnWorker(scriptPath) {
+        const id = this.nextWorkerId++;
+        const proc = spawn(this.pythonCommand, [scriptPath], {
+            stdio: ['pipe', 'pipe', 'pipe'],
+            env: { ...process.env },
+        });
+        const worker = {
+            process: proc,
+            busy: false,
+            taskCount: 0,
+            lastHeartbeat: Date.now(),
+            scriptPath,
+        };
+        // Capture stderr for logging
+        let stderrBuf = '';
+        proc.stderr?.on('data', (data) => {
+            stderrBuf += data.toString();
+            const lines = stderrBuf.split('\n');
+            stderrBuf = lines.pop() ?? '';
+            for (const line of lines) {
+                if (line.trim())
+                    console.error(`[PythonPool:${id}] ${line}`);
+            }
+        });
+        proc.on('exit', (code) => {
+            console.error(`[PythonPool] Worker ${id} exited with code ${code}`);
+            this.workers.delete(id);
+            // Auto-restart if not shutting down
+            if (!this.shuttingDown) {
+                console.error(`[PythonPool] Restarting worker ${id}...`);
+                this.spawnWorker(scriptPath);
+            }
+        });
+        proc.on('error', (err) => {
+            console.error(`[PythonPool] Worker ${id} error: ${err.message}`);
+        });
+        this.workers.set(id, worker);
+    }
+    getAvailableWorker() {
+        for (const [id, worker] of this.workers) {
+            if (!worker.busy)
+                return [id, worker];
+        }
+        return null;
+    }
+    dispatchTask(entry, task) {
+        const [id, worker] = entry;
+        worker.busy = true;
+        worker.taskCount++;
+        let stdoutBuf = '';
+        const onData = (data) => {
+            stdoutBuf += data.toString();
+            // Try to parse complete JSON response (newline-delimited)
+            const newlineIdx = stdoutBuf.indexOf('\n');
+            if (newlineIdx !== -1) {
+                const line = stdoutBuf.substring(0, newlineIdx);
+                stdoutBuf = stdoutBuf.substring(newlineIdx + 1);
+                cleanup();
+                clearTimeout(task.timeoutId);
+                try {
+                    const result = JSON.parse(line);
+                    if (result.error) {
+                        task.reject(new Error(String(result.error)));
+                    }
+                    else {
+                        task.resolve(result);
+                    }
+                }
+                catch {
+                    task.reject(new Error(`Failed to parse worker response: ${line.substring(0, 200)}`));
+                }
+                worker.busy = false;
+                worker.lastHeartbeat = Date.now();
+                // Restart worker if it hit max tasks
+                if (worker.taskCount >= this.config.maxTasksPerWorker) {
+                    console.error(`[PythonPool] Worker ${id} hit max tasks (${this.config.maxTasksPerWorker}), recycling`);
+                    worker.process.kill('SIGTERM');
+                }
+                else {
+                    // Process next task in queue
+                    this.processQueue();
+                }
+            }
+        };
+        const onError = (err) => {
+            cleanup();
+            clearTimeout(task.timeoutId);
+            worker.busy = false;
+            task.reject(err);
+        };
+        const cleanup = () => {
+            worker.process.stdout?.removeListener('data', onData);
+            worker.process.removeListener('error', onError);
+        };
+        worker.process.stdout?.on('data', onData);
+        worker.process.on('error', onError);
+        // Send command as newline-delimited JSON
+        const cmdStr = JSON.stringify(task.command) + '\n';
+        worker.process.stdin?.write(cmdStr, (err) => {
+            if (err) {
+                cleanup();
+                clearTimeout(task.timeoutId);
+                worker.busy = false;
+                task.reject(new Error(`Failed to write to worker: ${err.message}`));
+            }
+        });
+    }
+    processQueue() {
+        while (this.taskQueue.length > 0) {
+            const worker = this.getAvailableWorker();
+            if (!worker)
+                break;
+            const task = this.taskQueue.shift();
+            this.dispatchTask(worker, task);
+        }
+    }
+    healthCheck() {
+        const now = Date.now();
+        for (const [id, worker] of this.workers) {
+            if (worker.busy &&
+                now - worker.lastHeartbeat > this.config.taskTimeoutMs * 2) {
+                console.error(`[PythonPool] Worker ${id} appears hung, killing`);
+                worker.process.kill('SIGKILL');
+            }
+        }
+    }
+}
+//# sourceMappingURL=python-pool.js.map

package/dist/services/python-pool.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"python-pool.js","sourceRoot":"","sources":["../../src/services/python-pool.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,KAAK,EAAqB,MAAM,eAAe,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AAuCtC,kFAAkF;AAClF,sBAAsB;AACtB,kFAAkF;AAElF,MAAM,OAAO,UAAW,SAAQ,YAAY;IAClC,OAAO,GAA8B,IAAI,GAAG,EAAE,CAAC;IAC/C,SAAS,GAAe,EAAE,CAAC;IAClB,MAAM,CAAa;IAC5B,gBAAgB,GAA0B,IAAI,CAAC;IAC/C,YAAY,GAAG,CAAC,CAAC;IACjB,YAAY,GAAG,KAAK,CAAC;IACZ,aAAa,CAAS;IAEvC,YAAY,MAA4B;QACtC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,MAAM,GAAG;YACZ,QAAQ,EAAE,MAAM,EAAE,QAAQ,IAAI,CAAC;YAC/B,iBAAiB,EAAE,MAAM,EAAE,iBAAiB,IAAI,GAAG;YACnD,qBAAqB,EAAE,MAAM,EAAE,qBAAqB,IAAI,KAAK;YAC7D,aAAa,EAAE,MAAM,EAAE,aAAa,IAAI,MAAM;SAC/C,CAAC;QACF,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;IAC3E,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAkB;QACtB,IAAI,IAAI,CAAC,YAAY;YAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;QAChE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YAC9C,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;QAC/B,CAAC;QACD,IAAI,CAAC,gBAAgB,GAAG,WAAW,CACjC,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,EACxB,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAClC,CAAC;QACF,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC;QAC9B,OAAO,CAAC,KAAK,CACX,6BAA6B,IAAI,CAAC,MAAM,CAAC,QAAQ,gBAAgB,UAAU,EAAE,CAC9E,CAAC;IACJ,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,OAAO,CAAC,OAAgC;QAC5C,IAAI,IAAI,CAAC,YAAY;YAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;QAEhE,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE;gBAChC,MAAM,CACJ,IAAI,KAAK,CACP,oCAAoC,IAAI,CAAC,MAAM,CAAC,aAAa,IAAI,CAClE,CACF,CAAC;YACJ,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YAE9B,MAAM,IAAI,GAAa,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;YAE/D,+CAA+C;YAC/C,MAAM,MAAM,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACzC,IAAI,MAAM,EAAE,CAAC;gBACX,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YAClC,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ;QACZ,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1B,aAAa,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;YACrC,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;QAC/B,CAAC;QAED,0BAA0B;QAC1B,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YAClC,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC7B,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC,CAAC;QAC/C,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC;QAEpB,mBAAmB;QACnB,KAAK,MAAM,CAAC,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACtC,IAAI,CAAC;gBACH,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBAC/B,sBAAsB;gBACtB,MAAM,UAAU,GAAG,UAAU,CAAC,GAAG,EAAE;oBACjC,IAAI,CAAC;wBACH,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBACjC,CAAC;oBAAC,MAAM,CAAC;wBACP,kBAAkB;oBACpB,CAAC;gBACH,CAAC,EAAE,IAAI,CAAC,CAAC;gBACT,UAAU,CAAC,KAAK,EAAE,CAAC;YACrB,CAAC;YAAC,MAAM,CAAC;gBACP,sBAAsB;YACxB,CAAC;QACH,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACrB,OAAO,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;IAC1C,CAAC;IAED;;OAEG;IACH,SAAS;QACP,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YACtC,IAAI,CAAC,CAAC,IAAI;gBAAE,IAAI,EAAE,CAAC;;gBACd,IAAI,EAAE,CAAC;QACd,CAAC;QACD,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI;YACxB,IAAI;YACJ,IAAI;YACJ,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM;SAC9B,CAAC;IACJ,CAAC;IAED,gFAAgF;IAChF,kBAAkB;IAClB,gFAAgF;IAExE,WAAW,CAAC,UAAkB;QACpC,MAAM,EAAE,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QAC/B,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,UAAU,CAAC,EAAE;YACnD,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;YAC/B,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE;SACxB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAiB;YAC3B,OAAO,EAAE,IAAI;YACb,IAAI,EAAE,KAAK;YACX,SAAS,EAAE,CAAC;YACZ,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE;YACzB,UAAU;SACX,CAAC;QAEF,6BAA6B;QAC7B,IAAI,SAAS,GAAG,EAAE,CAAC;QACnB,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,IAAY,EAAE,EAAE;YACvC,SAAS,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC7B,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACpC,SAAS,GAAG,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;YAC9B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,IAAI,IAAI,CAAC,IAAI,EAAE;oBAAE,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC;YAC/D,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YACvB,OAAO,CAAC,KAAK,CAAC,uBAAuB,EAAE,qBAAqB,IAAI,EAAE,CAAC,CAAC;YACpE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACxB,oCAAoC;YACpC,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;gBACvB,OAAO,CAAC,KAAK,CAAC,kCAAkC,EAAE,KAAK,CAAC,CAAC;gBACzD,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACvB,OAAO,CAAC,KAAK,CAAC,uBAAuB,EAAE,WAAW,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;QACnE,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IAC/B,CAAC;IAEO,kBAAkB;QACxB,KAAK,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACxC,IAAI,CAAC,MAAM,CAAC,IAAI;gBAAE,OAAO,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QACxC,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,YAAY,CAClB,KAA6B,EAC7B,IAAc;QAEd,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,GAAG,KAAK,CAAC;QAC3B,MAAM,CAAC,IAAI,GAAG,IAAI,CAAC;QACnB,MAAM,CAAC,SAAS,EAAE,CAAC;QAEnB,IAAI,SAAS,GAAG,EAAE,CAAC;QAEnB,MAAM,MAAM,GAAG,CAAC,IAAY,EAAQ,EAAE;YACpC,SAAS,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC7B,0DAA0D;YAC1D,MAAM,UAAU,GAAG,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC3C,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;gBACtB,MAAM,IAAI,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;gBAChD,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;gBAEhD,OAAO,EAAE,CAAC;gBACV,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBAE7B,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAA4B,CAAC;oBAC3D,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;wBACjB,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;oBAC/C,CAAC;yBAAM,CAAC;wBACN,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;oBACvB,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC;oBACP,IAAI,CAAC,MAAM,CACT,IAAI,KAAK,CACP,oCAAoC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAC7D,CACF,CAAC;gBACJ,CAAC;gBAED,MAAM,CAAC,IAAI,GAAG,KAAK,CAAC;gBACpB,MAAM,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBAElC,qCAAqC;gBACrC,IAAI,MAAM,CAAC,SAAS,IAAI,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,CAAC;oBACtD,OAAO,CAAC,KAAK,CACX,uBAAuB,EAAE,mBAAmB,IAAI,CAAC,MAAM,CAAC,iBAAiB,cAAc,CACxF,CAAC;oBACF,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBACjC,CAAC;qBAAM,CAAC;oBACN,6BAA6B;oBAC7B,IAAI,CAAC,YAAY,EAAE,CAAC;gBACtB,CAAC;YACH,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,OAAO,GAAG,CAAC,GAAU,EAAQ,EAAE;YACnC,OAAO,EAAE,CAAC;YACV,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC7B,MAAM,CAAC,IAAI,GAAG,KAAK,CAAC;YACpB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACnB,CAAC,CAAC;QAEF,MAAM,OAAO,GAAG,GAAS,EAAE;YACzB,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;YACtD,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAClD,CAAC,CAAC;QAEF,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC1C,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAEpC,yCAAyC;QACzC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC;QACnD,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE;YAC1C,IAAI,GAAG,EAAE,CAAC;gBACR,OAAO,EAAE,CAAC;gBACV,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBAC7B,MAAM,CAAC,IAAI,GAAG,KAAK,CAAC;gBACpB,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,8BAA8B,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YACtE,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,YAAY;QAClB,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACjC,MAAM,MAAM,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACzC,IAAI,CAAC,MAAM;gBAAE,MAAM;YACnB,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAG,CAAC;YACrC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAEO,WAAW;QACjB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,KAAK,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACxC,IACE,MAAM,CAAC,IAAI;gBACX,GAAG,GAAG,MAAM,CAAC,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,GAAG,CAAC,EAC1D,CAAC;gBACD,OAAO,CAAC,KAAK,CACX,uBAAuB,EAAE,wBAAwB,CAClD,CAAC;gBACF,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACjC,CAAC;QACH,CAAC;IACH,CAAC;CACF"}

package/dist/services/search/bm25.d.ts

@@ -0,0 +1,180 @@
+/**
+ * BM25 Search Service using SQLite FTS5
+ *
+ * FAIL FAST: All errors throw immediately with detailed messages
+ * PROVENANCE: Every result includes provenance_id and content_hash
+ */
+import type Database from 'better-sqlite3';
+interface ChunkFilterSQL {
+    conditions: string[];
+    params: unknown[];
+}
+interface BM25SearchOptions {
+    query: string;
+    limit?: number;
+    phraseSearch?: boolean;
+    documentFilter?: string[];
+    includeHighlight?: boolean;
+    chunkFilter?: ChunkFilterSQL;
+    /** Page range filter applied to VLM/extraction searches (which lack chunk metadata) */
+    pageRangeFilter?: {
+        min_page?: number;
+        max_page?: number;
+    };
+    /**
+     * When true, the query is already a valid FTS5 expression (e.g. OR-joined
+     * terms from expandQuery) and must NOT be re-processed by sanitizeFTS5Query().
+     * Re-processing would insert implicit AND between consecutive non-operator
+     * tokens, corrupting the OR semantics (H-2 fix).
+     */
+    preSanitized?: boolean;
+}
+interface BM25SearchResult {
+    chunk_id: string | null;
+    image_id: string | null;
+    embedding_id: string | null;
+    extraction_id: string | null;
+    document_id: string;
+    original_text: string;
+    bm25_score: number;
+    rank: number;
+    result_type: 'chunk' | 'vlm' | 'extraction';
+    source_file_path: string;
+    source_file_name: string;
+    source_file_hash: string;
+    page_number: number | null;
+    character_start: number;
+    character_end: number;
+    chunk_index: number;
+    provenance_id: string;
+    content_hash: string;
+    highlight?: string;
+    heading_context?: string | null;
+    section_path?: string | null;
+    content_types?: string | null;
+    is_atomic?: boolean;
+    page_range?: string | null;
+    heading_level?: number | null;
+    ocr_quality_score?: number | null;
+    doc_title?: string | null;
+    doc_author?: string | null;
+    doc_subject?: string | null;
+    overlap_previous?: number;
+    overlap_next?: number;
+    chunking_strategy?: string | null;
+    embedding_status?: string;
+    doc_page_count?: number | null;
+    datalab_mode?: string | null;
+    total_chunks?: number;
+}
+export declare class BM25SearchService {
+    private readonly db;
+    constructor(db: Database.Database);
+    private verifyFTSTableExists;
+    search(options: BM25SearchOptions): BM25SearchResult[];
+    /**
+     * Search VLM description embeddings using FTS5
+     * Queries vlm_fts JOIN embeddings JOIN images JOIN documents
+     *
+     * NOTE: VLM results only support page_range_filter from chunk filters
+     * (VLM embeddings don't have heading_context, section_path, etc.)
+     */
+    searchVLM(options: BM25SearchOptions): BM25SearchResult[];
+    /**
+     * Search extraction content using FTS5
+     * Queries extractions_fts JOIN extractions JOIN documents
+     *
+     * NOTE: Extractions don't have page numbers or chunk metadata,
+     * so chunkFilter and pageRangeFilter are not applied here.
+     */
+    searchExtractions(options: BM25SearchOptions): BM25SearchResult[];
+    rebuildIndex(): {
+        chunks_indexed: number;
+        vlm_indexed: number;
+        extractions_indexed: number;
+        duration_ms: number;
+        content_hash: string;
+    };
+    /**
+     * Rebuild VLM FTS index from embeddings where image_id IS NOT NULL
+     */
+    rebuildVLMIndex(): {
+        vlm_indexed: number;
+        duration_ms: number;
+    };
+    /**
+     * Rebuild extractions FTS index
+     */
+    rebuildExtractionIndex(): {
+        extractions_indexed: number;
+        duration_ms: number;
+    };
+    /**
+     * Search document metadata (title, author, subject) using FTS5.
+     * Queries documents_fts table (v30+).
+     *
+     * Returns document IDs and metadata fields matching the query.
+     * Used to find documents by metadata rather than content.
+     */
+    searchDocumentMetadata(options: {
+        query: string;
+        limit?: number;
+        phraseSearch?: boolean;
+    }): Array<{
+        document_id: string;
+        file_name: string;
+        doc_title: string | null;
+        doc_author: string | null;
+        doc_subject: string | null;
+        bm25_score: number;
+        rank: number;
+        result_type: 'document_metadata';
+    }>;
+    /**
+     * Check whether all expected FTS triggers exist for a given set of trigger names.
+     * If all triggers are present, the FTS index is kept in sync atomically and cannot be stale.
+     * If any trigger is missing, the index IS stale (triggers are the sync mechanism).
+     */
+    private checkTriggersExist;
+    getStatus(): {
+        chunks_indexed: number;
+        current_chunk_count: number;
+        index_stale: boolean;
+        last_rebuild_at: string | null;
+        tokenizer: string;
+        content_hash: string | null;
+        vlm_indexed: number;
+        current_vlm_count: number;
+        vlm_index_stale: boolean;
+        vlm_last_rebuild_at: string | null;
+        extractions_indexed: number;
+        current_extraction_count: number;
+        extraction_index_stale: boolean;
+        extraction_last_rebuild_at: string | null;
+    };
+    private computeContentHash;
+}
+/**
+ * Sanitize a user-provided query for safe use in FTS5 MATCH expressions.
+ *
+ * - Preserves FTS5 boolean operators (AND, OR, NOT)
+ * - Treats hyphens as word separators (matching unicode61 tokenizer)
+ * - Strips all FTS5 metacharacters (' " ( ) * : ^ ~ + etc.)
+ * - Inserts implicit AND between consecutive non-operator tokens
+ * - Strips leading/trailing/consecutive operators
+ *
+ * This is the SINGLE authoritative FTS5 sanitizer for the entire codebase.
+ *
+ * @param query - Raw user query string
+ * @returns Sanitized FTS5 query string
+ * @throws Error if query contains no valid tokens after sanitization
+ */
+export declare function sanitizeFTS5Query(query: string): string;
+/**
+ * Compute SHA-256 content hash of all chunk IDs and text_hashes for FTS index integrity verification.
+ * L-10 fix: Uses incremental hashing with iterate() instead of loading all rows into memory.
+ * Used by both BM25SearchService and the v3->v4 migration.
+ */
+export declare function computeFTSContentHash(db: Database.Database): string;
+export {};
+//# sourceMappingURL=bm25.d.ts.map

package/dist/services/search/bm25.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bm25.d.ts","sourceRoot":"","sources":["../../../src/services/search/bm25.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;AAI3C,UAAU,cAAc;IACtB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,MAAM,EAAE,OAAO,EAAE,CAAC;CACnB;AAED,UAAU,iBAAiB;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,WAAW,CAAC,EAAE,cAAc,CAAC;IAC7B,uFAAuF;IACvF,eAAe,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC3D;;;;;OAKG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAED,UAAU,gBAAgB;IACxB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,OAAO,GAAG,KAAK,GAAG,YAAY,CAAC;IAC5C,gBAAgB,EAAE,MAAM,CAAC;IACzB,gBAAgB,EAAE,MAAM,CAAC;IACzB,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAeD,qBAAa,iBAAiB;IAChB,OAAO,CAAC,QAAQ,CAAC,EAAE;gBAAF,EAAE,EAAE,QAAQ,CAAC,QAAQ;IAIlD,OAAO,CAAC,oBAAoB;IAa5B,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,gBAAgB,EAAE;IAuItD;;;;;;OAMG;IACH,SAAS,CAAC,OAAO,EAAE,iBAAiB,GAAG,gBAAgB,EAAE;IAsHzD;;;;;;OAMG;IACH,iBAAiB,CAAC,OAAO,EAAE,iBAAiB,GAAG,gBAAgB,EAAE;IAqGjE,YAAY,IAAI;QACd,cAAc,EAAE,MAAM,CAAC;QACvB,WAAW,EAAE,MAAM,CAAC;QACpB,mBAAmB,EAAE,MAAM,CAAC;QAC5B,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;KACtB;IAuCD;;OAEG;IACH,eAAe,IAAI;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE;IA+C/D;;OAEG;IACH,sBAAsB,IAAI;QAAE,mBAAmB,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE;IAiC9E;;;;;;OAMG;IACH,sBAAsB,CAAC,OAAO,EAAE;QAC9B,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,YAAY,CAAC,EAAE,OAAO,CAAC;KACxB,GAAG,KAAK,CAAC;QACR,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,EAAE,MAAM,CAAC;QAClB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;QACzB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;QAC1B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;QAC3B,UAAU,EAAE,MAAM,CAAC;QACnB,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,mBAAmB,CAAC;KAClC,CAAC;IA4CF;;;;OAIG;IACH,OAAO,CAAC,kBAAkB;IAW1B,SAAS,IAAI;QACX,cAAc,EAAE,MAAM,CAAC;QACvB,mBAAmB,EAAE,MAAM,CAAC;QAC5B,WAAW,EAAE,OAAO,CAAC;QACrB,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;QAC/B,SAAS,EAAE,MAAM,CAAC;QAClB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;QAC5B,WAAW,EAAE,MAAM,CAAC;QACpB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,eAAe,EAAE,OAAO,CAAC;QACzB,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;QACnC,mBAAmB,EAAE,MAAM,CAAC;QAC5B,wBAAwB,EAAE,MAAM,CAAC;QACjC,sBAAsB,EAAE,OAAO,CAAC;QAChC,0BAA0B,EAAE,MAAM,GAAG,IAAI,CAAC;KAC3C;IAyFD,OAAO,CAAC,kBAAkB;CAG3B;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CA2DvD;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,GAAG,MAAM,CAUnE"}