@elizaos/plugin-vision 1.2.1

package/dist/workers/ocr-worker.js

@@ -0,0 +1,354 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype);
+    return g.next = verb(0), g["throw"] = verb(1), g["return"] = verb(2), typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+var worker_threads_1 = require("worker_threads");
+var worker_logger_1 = require("./worker-logger");
+var ocr_service_1 = require("../ocr-service");
+var sharp_1 = require("sharp");
+var OCRWorker = /** @class */ (function () {
+    function OCRWorker(config, sharedBuffer, resultsBuffer) {
+        this.isRunning = true;
+        this.frameCount = 0;
+        this.lastFPSReport = Date.now();
+        this.lastFrameId = -1;
+        // Atomic indices for input buffer
+        this.FRAME_ID_INDEX = 0;
+        this.WRITE_LOCK_INDEX = 1;
+        this.WIDTH_INDEX = 2;
+        this.HEIGHT_INDEX = 3;
+        this.DISPLAY_INDEX = 4;
+        this.TIMESTAMP_INDEX = 5;
+        this.DATA_OFFSET = 24;
+        // Results buffer structure
+        this.RESULTS_HEADER_SIZE = 16;
+        this.MAX_TEXT_LENGTH = 65536; // 64KB for text
+        this.config = config;
+        this.sharedBuffer = sharedBuffer;
+        this.dataView = new DataView(sharedBuffer);
+        this.atomicState = new Int32Array(sharedBuffer, 0, 6);
+        this.resultsBuffer = resultsBuffer;
+        this.resultsView = new DataView(resultsBuffer);
+        this.ocrService = new ocr_service_1.OCRService();
+    }
+    OCRWorker.prototype.initialize = function () {
+        return __awaiter(this, void 0, void 0, function () {
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0: return [4 /*yield*/, this.ocrService.initialize()];
+                    case 1:
+                        _a.sent();
+                        worker_logger_1.logger.info('[OCRWorker] Initialized and ready');
+                        return [2 /*return*/];
+                }
+            });
+        });
+    };
+    OCRWorker.prototype.run = function () {
+        return __awaiter(this, void 0, void 0, function () {
+            var currentFrameId, now, fps, error_1;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0: return [4 /*yield*/, this.initialize()];
+                    case 1:
+                        _a.sent();
+                        worker_logger_1.logger.info('[OCRWorker] Starting OCR loop...');
+                        _a.label = 2;
+                    case 2:
+                        if (!this.isRunning) return [3 /*break*/, 11];
+                        _a.label = 3;
+                    case 3:
+                        _a.trys.push([3, 8, , 10]);
+                        currentFrameId = Atomics.load(this.atomicState, this.FRAME_ID_INDEX);
+                        if (!(currentFrameId > this.lastFrameId)) return [3 /*break*/, 5];
+                        return [4 /*yield*/, this.processFrame()];
+                    case 4:
+                        _a.sent();
+                        this.lastFrameId = currentFrameId;
+                        this.frameCount++;
+                        now = Date.now();
+                        if (now - this.lastFPSReport >= 1000) {
+                            fps = this.frameCount / ((now - this.lastFPSReport) / 1000);
+                            worker_logger_1.logger.info("[OCRWorker] OCR FPS: ".concat(fps.toFixed(2)));
+                            worker_threads_1.parentPort === null || worker_threads_1.parentPort === void 0 ? void 0 : worker_threads_1.parentPort.postMessage({
+                                type: 'fps',
+                                fps: fps,
+                                frameCount: this.frameCount,
+                            });
+                            this.frameCount = 0;
+                            this.lastFPSReport = now;
+                        }
+                        return [3 /*break*/, 7];
+                    case 5: 
+                    // No new frame, brief yield
+                    return [4 /*yield*/, new Promise(function (resolve) { return setImmediate(resolve); })];
+                    case 6:
+                        // No new frame, brief yield
+                        _a.sent();
+                        _a.label = 7;
+                    case 7: return [3 /*break*/, 10];
+                    case 8:
+                        error_1 = _a.sent();
+                        worker_logger_1.logger.error('[OCRWorker] Processing error:', error_1);
+                        return [4 /*yield*/, new Promise(function (resolve) { return setTimeout(resolve, 100); })];
+                    case 9:
+                        _a.sent();
+                        return [3 /*break*/, 10];
+                    case 10: return [3 /*break*/, 2];
+                    case 11: return [2 /*return*/];
+                }
+            });
+        });
+    };
+    OCRWorker.prototype.processFrame = function () {
+        return __awaiter(this, void 0, void 0, function () {
+            var metadata, results, screenBuffer, ocrResult, error_2, _i, _a, region, regionBuffer, ocrResult, error_3, totalText, totalBlocks;
+            return __generator(this, function (_b) {
+                switch (_b.label) {
+                    case 0:
+                        metadata = {
+                            frameId: Atomics.load(this.atomicState, this.FRAME_ID_INDEX),
+                            width: Atomics.load(this.atomicState, this.WIDTH_INDEX),
+                            height: Atomics.load(this.atomicState, this.HEIGHT_INDEX),
+                            displayIndex: Atomics.load(this.atomicState, this.DISPLAY_INDEX),
+                            timestamp: Atomics.load(this.atomicState, this.TIMESTAMP_INDEX),
+                        };
+                        results = [];
+                        if (!this.config.processFullScreen) return [3 /*break*/, 5];
+                        _b.label = 1;
+                    case 1:
+                        _b.trys.push([1, 4, , 5]);
+                        return [4 /*yield*/, this.extractFullScreenBuffer(metadata)];
+                    case 2:
+                        screenBuffer = _b.sent();
+                        return [4 /*yield*/, this.ocrService.extractText(screenBuffer)];
+                    case 3:
+                        ocrResult = _b.sent();
+                        results.push(ocrResult);
+                        worker_logger_1.logger.debug("[OCRWorker] Full screen OCR: ".concat(ocrResult.fullText.length, " chars"));
+                        return [3 /*break*/, 5];
+                    case 4:
+                        error_2 = _b.sent();
+                        worker_logger_1.logger.error('[OCRWorker] Full screen OCR failed:', error_2);
+                        return [3 /*break*/, 5];
+                    case 5:
+                        if (!(this.config.textRegions && this.config.textRegions.length > 0)) return [3 /*break*/, 12];
+                        _i = 0, _a = this.config.textRegions;
+                        _b.label = 6;
+                    case 6:
+                        if (!(_i < _a.length)) return [3 /*break*/, 12];
+                        region = _a[_i];
+                        _b.label = 7;
+                    case 7:
+                        _b.trys.push([7, 10, , 11]);
+                        return [4 /*yield*/, this.extractRegionBuffer(region, metadata)];
+                    case 8:
+                        regionBuffer = _b.sent();
+                        return [4 /*yield*/, this.ocrService.extractText(regionBuffer)];
+                    case 9:
+                        ocrResult = _b.sent();
+                        results.push(ocrResult);
+                        worker_logger_1.logger.debug("[OCRWorker] Region OCR (".concat(region.x, ",").concat(region.y, "): ").concat(ocrResult.fullText.length, " chars"));
+                        return [3 /*break*/, 11];
+                    case 10:
+                        error_3 = _b.sent();
+                        worker_logger_1.logger.error('[OCRWorker] Region OCR failed:', error_3);
+                        return [3 /*break*/, 11];
+                    case 11:
+                        _i++;
+                        return [3 /*break*/, 6];
+                    case 12: 
+                    // Write combined results to buffer
+                    return [4 /*yield*/, this.writeResultsToBuffer(results, metadata.frameId)];
+                    case 13:
+                        // Write combined results to buffer
+                        _b.sent();
+                        totalText = results.map(function (r) { return r.fullText; }).join('\n');
+                        totalBlocks = results.reduce(function (sum, r) { return sum + r.blocks.length; }, 0);
+                        worker_threads_1.parentPort === null || worker_threads_1.parentPort === void 0 ? void 0 : worker_threads_1.parentPort.postMessage({
+                            type: 'ocr_complete',
+                            frameId: metadata.frameId,
+                            displayIndex: metadata.displayIndex,
+                            textLength: totalText.length,
+                            blockCount: totalBlocks,
+                            hasText: totalText.length > 0,
+                        });
+                        return [2 /*return*/];
+                }
+            });
+        });
+    };
+    OCRWorker.prototype.extractFullScreenBuffer = function (metadata) {
+        return __awaiter(this, void 0, void 0, function () {
+            var bytesPerPixel, totalPixels, totalBytes, screenData, i, pngBuffer;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        bytesPerPixel = 4;
+                        totalPixels = metadata.width * metadata.height;
+                        totalBytes = totalPixels * bytesPerPixel;
+                        screenData = Buffer.allocUnsafe(totalBytes);
+                        // Copy all data
+                        for (i = 0; i < totalBytes; i++) {
+                            screenData[i] = this.dataView.getUint8(this.DATA_OFFSET + i);
+                        }
+                        return [4 /*yield*/, (0, sharp_1.default)(screenData, {
+                                raw: {
+                                    width: metadata.width,
+                                    height: metadata.height,
+                                    channels: 4,
+                                },
+                            })
+                                .png()
+                                .toBuffer()];
+                    case 1:
+                        pngBuffer = _a.sent();
+                        return [2 /*return*/, pngBuffer];
+                }
+            });
+        });
+    };
+    OCRWorker.prototype.extractRegionBuffer = function (region, metadata) {
+        return __awaiter(this, void 0, void 0, function () {
+            var bytesPerPixel, rowStride, x, y, width, height, regionData, row, sourceY, sourceOffset, destOffset, i, pngBuffer;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        bytesPerPixel = 4;
+                        rowStride = metadata.width * bytesPerPixel;
+                        x = Math.max(0, Math.min(region.x, metadata.width - 1));
+                        y = Math.max(0, Math.min(region.y, metadata.height - 1));
+                        width = Math.min(region.width, metadata.width - x);
+                        height = Math.min(region.height, metadata.height - y);
+                        regionData = Buffer.allocUnsafe(width * height * bytesPerPixel);
+                        // Copy region data row by row
+                        for (row = 0; row < height; row++) {
+                            sourceY = y + row;
+                            sourceOffset = this.DATA_OFFSET + sourceY * rowStride + x * bytesPerPixel;
+                            destOffset = row * width * bytesPerPixel;
+                            for (i = 0; i < width * bytesPerPixel; i++) {
+                                regionData[destOffset + i] = this.dataView.getUint8(sourceOffset + i);
+                            }
+                        }
+                        return [4 /*yield*/, (0, sharp_1.default)(regionData, {
+                                raw: {
+                                    width: width,
+                                    height: height,
+                                    channels: 4,
+                                },
+                            })
+                                .png()
+                                .toBuffer()];
+                    case 1:
+                        pngBuffer = _a.sent();
+                        return [2 /*return*/, pngBuffer];
+                }
+            });
+        });
+    };
+    OCRWorker.prototype.writeResultsToBuffer = function (results, frameId) {
+        return __awaiter(this, void 0, void 0, function () {
+            var combinedResult, resultJson, resultBytes, offset, dataOffset, i;
+            return __generator(this, function (_a) {
+                combinedResult = {
+                    frameId: frameId,
+                    timestamp: Date.now(),
+                    fullText: results.map(function (r) { return r.fullText; }).join('\n'),
+                    blocks: results.flatMap(function (r) { return r.blocks; }),
+                    regions: results.length,
+                };
+                resultJson = JSON.stringify(combinedResult);
+                resultBytes = Buffer.from(resultJson, 'utf-8');
+                offset = this.RESULTS_HEADER_SIZE;
+                // Write length
+                this.resultsView.setUint32(offset, resultBytes.length, true);
+                // Write frame ID
+                this.resultsView.setUint32(offset + 4, frameId, true);
+                // Write timestamp
+                this.resultsView.setFloat64(offset + 8, Date.now(), true);
+                dataOffset = offset + 16;
+                for (i = 0; i < Math.min(resultBytes.length, this.MAX_TEXT_LENGTH); i++) {
+                    this.resultsView.setUint8(dataOffset + i, resultBytes[i]);
+                }
+                return [2 /*return*/];
+            });
+        });
+    };
+    OCRWorker.prototype.stop = function () {
+        this.isRunning = false;
+    };
+    OCRWorker.prototype.dispose = function () {
+        return __awaiter(this, void 0, void 0, function () {
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0: return [4 /*yield*/, this.ocrService.dispose()];
+                    case 1:
+                        _a.sent();
+                        return [2 /*return*/];
+                }
+            });
+        });
+    };
+    OCRWorker.prototype.updateTextRegions = function (regions) {
+        this.config.textRegions = regions;
+    };
+    return OCRWorker;
+}());
+// Worker entry point
+if (worker_threads_1.parentPort) {
+    var config = worker_threads_1.workerData.config, sharedBuffer = worker_threads_1.workerData.sharedBuffer, resultsBuffer = worker_threads_1.workerData.resultsBuffer;
+    var worker_1 = new OCRWorker(config, sharedBuffer, resultsBuffer);
+    // Handle messages from main thread
+    worker_threads_1.parentPort.on('message', function (msg) {
+        if (msg.type === 'stop') {
+            worker_1.stop();
+            worker_1.dispose().then(function () {
+                worker_threads_1.parentPort === null || worker_threads_1.parentPort === void 0 ? void 0 : worker_threads_1.parentPort.postMessage({ type: 'stopped' });
+            });
+        }
+        else if (msg.type === 'update_regions') {
+            worker_1.updateTextRegions(msg.regions);
+        }
+    });
+    // Run the worker
+    worker_1.run().catch(function (error) {
+        worker_logger_1.logger.error('[OCRWorker] Fatal error:', error);
+        worker_threads_1.parentPort === null || worker_threads_1.parentPort === void 0 ? void 0 : worker_threads_1.parentPort.postMessage({ type: 'error', error: error.message });
+        process.exit(1);
+    });
+}
+//# sourceMappingURL=ocr-worker.js.map

package/dist/workers/ocr-worker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ocr-worker.js","sourceRoot":"","sources":["../../src/workers/ocr-worker.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,iDAAwD;AACxD,iDAAyC;AACzC,8CAA4C;AAE5C,+BAA0B;AAgB1B;IA0BE,mBACE,MAAoB,EACpB,YAA+B,EAC/B,aAAgC;QArB1B,cAAS,GAAG,IAAI,CAAC;QACjB,eAAU,GAAG,CAAC,CAAC;QACf,kBAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3B,gBAAW,GAAG,CAAC,CAAC,CAAC;QAEzB,kCAAkC;QACjB,mBAAc,GAAG,CAAC,CAAC;QACnB,qBAAgB,GAAG,CAAC,CAAC;QACrB,gBAAW,GAAG,CAAC,CAAC;QAChB,iBAAY,GAAG,CAAC,CAAC;QACjB,kBAAa,GAAG,CAAC,CAAC;QAClB,oBAAe,GAAG,CAAC,CAAC;QACpB,gBAAW,GAAG,EAAE,CAAC;QAElC,2BAA2B;QACV,wBAAmB,GAAG,EAAE,CAAC;QACzB,oBAAe,GAAG,KAAK,CAAC,CAAC,gBAAgB;QAOxD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,QAAQ,GAAG,IAAI,QAAQ,CAAC,YAAY,CAAC,CAAC;QAC3C,IAAI,CAAC,WAAW,GAAG,IAAI,UAAU,CAAC,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QACtD,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,WAAW,GAAG,IAAI,QAAQ,CAAC,aAAa,CAAC,CAAC;QAC/C,IAAI,CAAC,UAAU,GAAG,IAAI,wBAAU,EAAE,CAAC;IACrC,CAAC;IAEK,8BAAU,GAAhB;;;;4BACE,qBAAM,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE,EAAA;;wBAAlC,SAAkC,CAAC;wBACnC,sBAAM,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;;;;;KAClD;IAEK,uBAAG,GAAT;;;;;4BACE,qBAAM,IAAI,CAAC,UAAU,EAAE,EAAA;;wBAAvB,SAAuB,CAAC;wBAExB,sBAAM,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;;;6BAEzC,IAAI,CAAC,SAAS;;;;wBAGX,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;6BAEvE,CAAA,cAAc,GAAG,IAAI,CAAC,WAAW,CAAA,EAAjC,wBAAiC;wBACnC,qBAAM,IAAI,CAAC,YAAY,EAAE,EAAA;;wBAAzB,SAAyB,CAAC;wBAC1B,IAAI,CAAC,WAAW,GAAG,cAAc,CAAC;wBAClC,IAAI,CAAC,UAAU,EAAE,CAAC;wBAGZ,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;wBACvB,IAAI,GAAG,GAAG,IAAI,CAAC,aAAa,IAAI,IAAI,EAAE,CAAC;4BAC/B,GAAG,GAAG,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC,CAAC;4BAClE,sBAAM,CAAC,IAAI,CAAC,+BAAwB,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC;4BAEtD,2BAAU,aAAV,2BAAU,uBAAV,2BAAU,CAAE,WAAW,CAAC;gCACtB,IAAI,EAAE,KAAK;gCACX,GAAG,KAAA;gCACH,UAAU,EAAE,IAAI,CAAC,UAAU;6BAC5B,CAAC,CAAC;4BAEH,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC;4BACpB,IAAI,CAAC,aAAa,GAAG,GAAG,CAAC;wBAC3B,CAAC;;;oBAED,4BAA4B;oBAC5B,qBAAM,IAAI,OAAO,CAAC,UAAC,OAAO,IAAK,OAAA,YAAY,CAAC,OAAO,CAAC,EAArB,CAAqB,CAAC,EAAA;;wBADrD,4BAA4B;wBAC5B,SAAqD,CAAC;;;;;wBAGxD,sBAAM,CAAC,KAAK,CAAC,+BAA+B,EAAE,OAAK,CAAC,CAAC;wBACrD,qBAAM,IAAI,OAAO,CAAC,UAAC,OAAO,IAAK,OAAA,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,EAAxB,CAAwB,CAAC,EAAA;;wBAAxD,SAAwD,CAAC;;;;;;;KAG9D;IAEa,gCAAY,GAA1B;;;;;;wBAEQ,QAAQ,GAAmB;4BAC/B,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,cAAc,CAAC;4BAC5D,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,WAAW,CAAC;4BACvD,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,YAAY,CAAC;4BACzD,YAAY,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,aAAa,CAAC;4BAChE,SAAS,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,eAAe,CAAC;yBAChE,CAAC;wBAEI,OAAO,GAAgB,EAAE,CAAC;6BAE5B,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAA7B,wBAA6B;;;;wBAGR,qBAAM,IAAI,CAAC,uBAAuB,CAAC,QAAQ,CAAC,EAAA;;wBAA3D,YAAY,GAAG,SAA4C;wBAC/C,qBAAM,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,YAAY,CAAC,EAAA;;wBAA3D,SAAS,GAAG,SAA+C;wBACjE,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;wBAExB,sBAAM,CAAC,KAAK,CAAC,uCAAgC,SAAS,CAAC,QAAQ,CAAC,MAAM,WAAQ,CAAC,CAAC;;;;wBAEhF,sBAAM,CAAC,KAAK,CAAC,qCAAqC,EAAE,OAAK,CAAC,CAAC;;;6BAK3D,CAAA,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAA,EAA7D,yBAA6D;8BACnB,EAAvB,KAAA,IAAI,CAAC,MAAM,CAAC,WAAW;;;6BAAvB,CAAA,cAAuB,CAAA;wBAAjC,MAAM;;;;wBAEQ,qBAAM,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAA;;wBAA/D,YAAY,GAAG,SAAgD;wBACnD,qBAAM,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,YAAY,CAAC,EAAA;;wBAA3D,SAAS,GAAG,SAA+C;wBACjE,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;wBAExB,sBAAM,CAAC,KAAK,CACV,kCAA2B,MAAM,CAAC,CAAC,cAAI,MAAM,CAAC,CAAC,gBAAM,SAAS,CAAC,QAAQ,CAAC,MAAM,WAAQ,CACvF,CAAC;;;;wBAEF,sBAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,OAAK,CAAC,CAAC;;;wBAVrC,IAAuB,CAAA;;;oBAe9C,mCAAmC;oBACnC,qBAAM,IAAI,CAAC,oBAAoB,CAAC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,EAAA;;wBAD1D,mCAAmC;wBACnC,SAA0D,CAAC;wBAGrD,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,UAAC,CAAC,IAAK,OAAA,CAAC,CAAC,QAAQ,EAAV,CAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;wBACtD,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,UAAC,GAAG,EAAE,CAAC,IAAK,OAAA,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,EAArB,CAAqB,EAAE,CAAC,CAAC,CAAC;wBAEzE,2BAAU,aAAV,2BAAU,uBAAV,2BAAU,CAAE,WAAW,CAAC;4BACtB,IAAI,EAAE,cAAc;4BACpB,OAAO,EAAE,QAAQ,CAAC,OAAO;4BACzB,YAAY,EAAE,QAAQ,CAAC,YAAY;4BACnC,UAAU,EAAE,SAAS,CAAC,MAAM;4BAC5B,UAAU,EAAE,WAAW;4BACvB,OAAO,EAAE,SAAS,CAAC,MAAM,GAAG,CAAC;yBAC9B,CAAC,CAAC;;;;;KACJ;IAEa,2CAAuB,GAArC,UAAsC,QAAwB;;;;;;wBACtD,aAAa,GAAG,CAAC,CAAC;wBAClB,WAAW,GAAG,QAAQ,CAAC,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC;wBAC/C,UAAU,GAAG,WAAW,GAAG,aAAa,CAAC;wBAGzC,UAAU,GAAG,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;wBAElD,gBAAgB;wBAChB,KAAS,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;4BACpC,UAAU,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC;wBAC/D,CAAC;wBAGiB,qBAAM,IAAA,eAAK,EAAC,UAAU,EAAE;gCACxC,GAAG,EAAE;oCACH,KAAK,EAAE,QAAQ,CAAC,KAAK;oCACrB,MAAM,EAAE,QAAQ,CAAC,MAAM;oCACvB,QAAQ,EAAE,CAAC;iCACZ;6BACF,CAAC;iCACC,GAAG,EAAE;iCACL,QAAQ,EAAE,EAAA;;wBARP,SAAS,GAAG,SAQL;wBAEb,sBAAO,SAAS,EAAC;;;;KAClB;IAEa,uCAAmB,GAAjC,UACE,MAA+D,EAC/D,QAAwB;;;;;;wBAElB,aAAa,GAAG,CAAC,CAAC;wBAClB,SAAS,GAAG,QAAQ,CAAC,KAAK,GAAG,aAAa,CAAC;wBAG3C,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;wBACxD,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;wBACzD,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;wBACnD,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;wBAGtD,UAAU,GAAG,MAAM,CAAC,WAAW,CAAC,KAAK,GAAG,MAAM,GAAG,aAAa,CAAC,CAAC;wBAEtE,8BAA8B;wBAC9B,KAAS,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC;4BAChC,OAAO,GAAG,CAAC,GAAG,GAAG,CAAC;4BAClB,YAAY,GAAG,IAAI,CAAC,WAAW,GAAG,OAAO,GAAG,SAAS,GAAG,CAAC,GAAG,aAAa,CAAC;4BAC1E,UAAU,GAAG,GAAG,GAAG,KAAK,GAAG,aAAa,CAAC;4BAE/C,KAAS,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,GAAG,aAAa,EAAE,CAAC,EAAE,EAAE,CAAC;gCAC/C,UAAU,CAAC,UAAU,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC;4BACxE,CAAC;wBACH,CAAC;wBAGiB,qBAAM,IAAA,eAAK,EAAC,UAAU,EAAE;gCACxC,GAAG,EAAE;oCACH,KAAK,OAAA;oCACL,MAAM,QAAA;oCACN,QAAQ,EAAE,CAAC;iCACZ;6BACF,CAAC;iCACC,GAAG,EAAE;iCACL,QAAQ,EAAE,EAAA;;wBARP,SAAS,GAAG,SAQL;wBAEb,sBAAO,SAAS,EAAC;;;;KAClB;IAEa,wCAAoB,GAAlC,UAAmC,OAAoB,EAAE,OAAe;;;;gBAEhE,cAAc,GAAG;oBACrB,OAAO,SAAA;oBACP,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;oBACrB,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,UAAC,CAAC,IAAK,OAAA,CAAC,CAAC,QAAQ,EAAV,CAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;oBACnD,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,UAAC,CAAC,IAAK,OAAA,CAAC,CAAC,MAAM,EAAR,CAAQ,CAAC;oBACxC,OAAO,EAAE,OAAO,CAAC,MAAM;iBACxB,CAAC;gBAEI,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;gBAC5C,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;gBAG/C,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAC;gBAExC,eAAe;gBACf,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;gBAE7D,iBAAiB;gBACjB,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;gBAEtD,kBAAkB;gBAClB,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,CAAC;gBAGpD,UAAU,GAAG,MAAM,GAAG,EAAE,CAAC;gBAC/B,KAAS,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC5E,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,UAAU,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC5D,CAAC;;;;KACF;IAED,wBAAI,GAAJ;QACE,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;IACzB,CAAC;IAEK,2BAAO,GAAb;;;;4BACE,qBAAM,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,EAAA;;wBAA/B,SAA+B,CAAC;;;;;KACjC;IAED,qCAAiB,GAAjB,UAAkB,OAAuE;QACvF,IAAI,CAAC,MAAM,CAAC,WAAW,GAAG,OAAO,CAAC;IACpC,CAAC;IACH,gBAAC;AAAD,CAAC,AAhQD,IAgQC;AAED,qBAAqB;AACrB,IAAI,2BAAU,EAAE,CAAC;IACP,IAAA,MAAM,GAAkC,2BAAU,OAA5C,EAAE,YAAY,GAAoB,2BAAU,aAA9B,EAAE,aAAa,GAAK,2BAAU,cAAf,CAAgB;IAC3D,IAAM,QAAM,GAAG,IAAI,SAAS,CAAC,MAAM,EAAE,YAAY,EAAE,aAAa,CAAC,CAAC;IAElE,mCAAmC;IACnC,2BAAU,CAAC,EAAE,CAAC,SAAS,EAAE,UAAC,GAAG;QAC3B,IAAI,GAAG,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACxB,QAAM,CAAC,IAAI,EAAE,CAAC;YACd,QAAM,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC;gBACpB,2BAAU,aAAV,2BAAU,uBAAV,2BAAU,CAAE,WAAW,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;YAC/C,CAAC,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,GAAG,CAAC,IAAI,KAAK,gBAAgB,EAAE,CAAC;YACzC,QAAM,CAAC,iBAAiB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACxC,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,iBAAiB;IACjB,QAAM,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,UAAC,KAAK;QACvB,sBAAM,CAAC,KAAK,CAAC,0BAA0B,EAAE,KAAK,CAAC,CAAC;QAChD,2BAAU,aAAV,2BAAU,uBAAV,2BAAU,CAAE,WAAW,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACjE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACL,CAAC","sourcesContent":["import { parentPort, workerData } from 'worker_threads';\nimport { logger } from './worker-logger';\nimport { OCRService } from '../ocr-service';\nimport type { OCRResult } from '../types';\nimport sharp from 'sharp';\n\ninterface WorkerConfig {\n  processFullScreen: boolean;\n  tileSize: number;\n  textRegions?: Array<{ x: number; y: number; width: number; height: number }>;\n}\n\ninterface SharedMetadata {\n  frameId: number;\n  width: number;\n  height: number;\n  displayIndex: number;\n  timestamp: number;\n}\n\nclass OCRWorker {\n  private config: WorkerConfig;\n  private sharedBuffer: SharedArrayBuffer;\n  private dataView: DataView;\n  private atomicState: Int32Array;\n  private resultsBuffer: SharedArrayBuffer;\n  private resultsView: DataView;\n  private ocrService: OCRService;\n  private isRunning = true;\n  private frameCount = 0;\n  private lastFPSReport = Date.now();\n  private lastFrameId = -1;\n\n  // Atomic indices for input buffer\n  private readonly FRAME_ID_INDEX = 0;\n  private readonly WRITE_LOCK_INDEX = 1;\n  private readonly WIDTH_INDEX = 2;\n  private readonly HEIGHT_INDEX = 3;\n  private readonly DISPLAY_INDEX = 4;\n  private readonly TIMESTAMP_INDEX = 5;\n  private readonly DATA_OFFSET = 24;\n\n  // Results buffer structure\n  private readonly RESULTS_HEADER_SIZE = 16;\n  private readonly MAX_TEXT_LENGTH = 65536; // 64KB for text\n\n  constructor(\n    config: WorkerConfig,\n    sharedBuffer: SharedArrayBuffer,\n    resultsBuffer: SharedArrayBuffer\n  ) {\n    this.config = config;\n    this.sharedBuffer = sharedBuffer;\n    this.dataView = new DataView(sharedBuffer);\n    this.atomicState = new Int32Array(sharedBuffer, 0, 6);\n    this.resultsBuffer = resultsBuffer;\n    this.resultsView = new DataView(resultsBuffer);\n    this.ocrService = new OCRService();\n  }\n\n  async initialize(): Promise<void> {\n    await this.ocrService.initialize();\n    logger.info('[OCRWorker] Initialized and ready');\n  }\n\n  async run(): Promise<void> {\n    await this.initialize();\n\n    logger.info('[OCRWorker] Starting OCR loop...');\n\n    while (this.isRunning) {\n      try {\n        // Check for new frame\n        const currentFrameId = Atomics.load(this.atomicState, this.FRAME_ID_INDEX);\n\n        if (currentFrameId > this.lastFrameId) {\n          await this.processFrame();\n          this.lastFrameId = currentFrameId;\n          this.frameCount++;\n\n          // Report FPS\n          const now = Date.now();\n          if (now - this.lastFPSReport >= 1000) {\n            const fps = this.frameCount / ((now - this.lastFPSReport) / 1000);\n            logger.info(`[OCRWorker] OCR FPS: ${fps.toFixed(2)}`);\n\n            parentPort?.postMessage({\n              type: 'fps',\n              fps,\n              frameCount: this.frameCount,\n            });\n\n            this.frameCount = 0;\n            this.lastFPSReport = now;\n          }\n        } else {\n          // No new frame, brief yield\n          await new Promise((resolve) => setImmediate(resolve));\n        }\n      } catch (error) {\n        logger.error('[OCRWorker] Processing error:', error);\n        await new Promise((resolve) => setTimeout(resolve, 100));\n      }\n    }\n  }\n\n  private async processFrame(): Promise<void> {\n    // Read metadata atomically\n    const metadata: SharedMetadata = {\n      frameId: Atomics.load(this.atomicState, this.FRAME_ID_INDEX),\n      width: Atomics.load(this.atomicState, this.WIDTH_INDEX),\n      height: Atomics.load(this.atomicState, this.HEIGHT_INDEX),\n      displayIndex: Atomics.load(this.atomicState, this.DISPLAY_INDEX),\n      timestamp: Atomics.load(this.atomicState, this.TIMESTAMP_INDEX),\n    };\n\n    const results: OCRResult[] = [];\n\n    if (this.config.processFullScreen) {\n      // Process entire screen\n      try {\n        const screenBuffer = await this.extractFullScreenBuffer(metadata);\n        const ocrResult = await this.ocrService.extractText(screenBuffer);\n        results.push(ocrResult);\n\n        logger.debug(`[OCRWorker] Full screen OCR: ${ocrResult.fullText.length} chars`);\n      } catch (error) {\n        logger.error('[OCRWorker] Full screen OCR failed:', error);\n      }\n    }\n\n    // Process specific text regions if defined\n    if (this.config.textRegions && this.config.textRegions.length > 0) {\n      for (const region of this.config.textRegions) {\n        try {\n          const regionBuffer = await this.extractRegionBuffer(region, metadata);\n          const ocrResult = await this.ocrService.extractText(regionBuffer);\n          results.push(ocrResult);\n\n          logger.debug(\n            `[OCRWorker] Region OCR (${region.x},${region.y}): ${ocrResult.fullText.length} chars`\n          );\n        } catch (error) {\n          logger.error('[OCRWorker] Region OCR failed:', error);\n        }\n      }\n    }\n\n    // Write combined results to buffer\n    await this.writeResultsToBuffer(results, metadata.frameId);\n\n    // Notify main thread\n    const totalText = results.map((r) => r.fullText).join('\\n');\n    const totalBlocks = results.reduce((sum, r) => sum + r.blocks.length, 0);\n\n    parentPort?.postMessage({\n      type: 'ocr_complete',\n      frameId: metadata.frameId,\n      displayIndex: metadata.displayIndex,\n      textLength: totalText.length,\n      blockCount: totalBlocks,\n      hasText: totalText.length > 0,\n    });\n  }\n\n  private async extractFullScreenBuffer(metadata: SharedMetadata): Promise<Buffer> {\n    const bytesPerPixel = 4; // RGBA\n    const totalPixels = metadata.width * metadata.height;\n    const totalBytes = totalPixels * bytesPerPixel;\n\n    // Create buffer for full screen\n    const screenData = Buffer.allocUnsafe(totalBytes);\n\n    // Copy all data\n    for (let i = 0; i < totalBytes; i++) {\n      screenData[i] = this.dataView.getUint8(this.DATA_OFFSET + i);\n    }\n\n    // Convert to PNG for OCR\n    const pngBuffer = await sharp(screenData, {\n      raw: {\n        width: metadata.width,\n        height: metadata.height,\n        channels: 4,\n      },\n    })\n      .png()\n      .toBuffer();\n\n    return pngBuffer;\n  }\n\n  private async extractRegionBuffer(\n    region: { x: number; y: number; width: number; height: number },\n    metadata: SharedMetadata\n  ): Promise<Buffer> {\n    const bytesPerPixel = 4; // RGBA\n    const rowStride = metadata.width * bytesPerPixel;\n\n    // Clamp region to screen bounds\n    const x = Math.max(0, Math.min(region.x, metadata.width - 1));\n    const y = Math.max(0, Math.min(region.y, metadata.height - 1));\n    const width = Math.min(region.width, metadata.width - x);\n    const height = Math.min(region.height, metadata.height - y);\n\n    // Create buffer for region\n    const regionData = Buffer.allocUnsafe(width * height * bytesPerPixel);\n\n    // Copy region data row by row\n    for (let row = 0; row < height; row++) {\n      const sourceY = y + row;\n      const sourceOffset = this.DATA_OFFSET + sourceY * rowStride + x * bytesPerPixel;\n      const destOffset = row * width * bytesPerPixel;\n\n      for (let i = 0; i < width * bytesPerPixel; i++) {\n        regionData[destOffset + i] = this.dataView.getUint8(sourceOffset + i);\n      }\n    }\n\n    // Convert to PNG for OCR\n    const pngBuffer = await sharp(regionData, {\n      raw: {\n        width,\n        height,\n        channels: 4,\n      },\n    })\n      .png()\n      .toBuffer();\n\n    return pngBuffer;\n  }\n\n  private async writeResultsToBuffer(results: OCRResult[], frameId: number): Promise<void> {\n    // Combine all results\n    const combinedResult = {\n      frameId,\n      timestamp: Date.now(),\n      fullText: results.map((r) => r.fullText).join('\\n'),\n      blocks: results.flatMap((r) => r.blocks),\n      regions: results.length,\n    };\n\n    const resultJson = JSON.stringify(combinedResult);\n    const resultBytes = Buffer.from(resultJson, 'utf-8');\n\n    // Write to results buffer\n    const offset = this.RESULTS_HEADER_SIZE;\n\n    // Write length\n    this.resultsView.setUint32(offset, resultBytes.length, true);\n\n    // Write frame ID\n    this.resultsView.setUint32(offset + 4, frameId, true);\n\n    // Write timestamp\n    this.resultsView.setFloat64(offset + 8, Date.now(), true);\n\n    // Write text data\n    const dataOffset = offset + 16;\n    for (let i = 0; i < Math.min(resultBytes.length, this.MAX_TEXT_LENGTH); i++) {\n      this.resultsView.setUint8(dataOffset + i, resultBytes[i]);\n    }\n  }\n\n  stop(): void {\n    this.isRunning = false;\n  }\n\n  async dispose(): Promise<void> {\n    await this.ocrService.dispose();\n  }\n\n  updateTextRegions(regions: Array<{ x: number; y: number; width: number; height: number }>): void {\n    this.config.textRegions = regions;\n  }\n}\n\n// Worker entry point\nif (parentPort) {\n  const { config, sharedBuffer, resultsBuffer } = workerData;\n  const worker = new OCRWorker(config, sharedBuffer, resultsBuffer);\n\n  // Handle messages from main thread\n  parentPort.on('message', (msg) => {\n    if (msg.type === 'stop') {\n      worker.stop();\n      worker.dispose().then(() => {\n        parentPort?.postMessage({ type: 'stopped' });\n      });\n    } else if (msg.type === 'update_regions') {\n      worker.updateTextRegions(msg.regions);\n    }\n  });\n\n  // Run the worker\n  worker.run().catch((error) => {\n    logger.error('[OCRWorker] Fatal error:', error);\n    parentPort?.postMessage({ type: 'error', error: error.message });\n    process.exit(1);\n  });\n}\n"]}

package/dist/workers/screen-capture-worker.d.ts

@@ -0,0 +1 @@
+export {};