@elizaos/plugin-vision 1.2.1 → 2.0.0-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/build.config.ts +53 -53
- package/dist/index.js +6716 -67
- package/dist/index.js.map +33 -1
- package/dist/workers/florence2-worker.js +111763 -307
- package/dist/workers/florence2-worker.js.map +92 -1
- package/dist/workers/ocr-worker.js +119177 -339
- package/dist/workers/ocr-worker.js.map +137 -1
- package/dist/workers/screen-capture-worker.js +350 -418
- package/dist/workers/screen-capture-worker.js.map +11 -1
- package/package.json +15 -20
- package/README.md +0 -270
- package/dist/action.d.ts +0 -8
- package/dist/action.js +0 -1212
- package/dist/action.js.map +0 -1
- package/dist/audio-capture-stream.d.ts +0 -42
- package/dist/audio-capture-stream.js +0 -516
- package/dist/audio-capture-stream.js.map +0 -1
- package/dist/audio-capture.d.ts +0 -25
- package/dist/audio-capture.js +0 -412
- package/dist/audio-capture.js.map +0 -1
- package/dist/basic.test.d.ts +0 -1
- package/dist/basic.test.js +0 -97
- package/dist/basic.test.js.map +0 -1
- package/dist/config.d.ts +0 -73
- package/dist/config.js +0 -254
- package/dist/config.js.map +0 -1
- package/dist/entity-tracker.d.ts +0 -32
- package/dist/entity-tracker.js +0 -361
- package/dist/entity-tracker.js.map +0 -1
- package/dist/errors.d.ts +0 -67
- package/dist/errors.js +0 -395
- package/dist/errors.js.map +0 -1
- package/dist/face-recognition.d.ts +0 -31
- package/dist/face-recognition.js +0 -332
- package/dist/face-recognition.js.map +0 -1
- package/dist/florence2-local.d.ts +0 -25
- package/dist/florence2-local.js +0 -280
- package/dist/florence2-local.js.map +0 -1
- package/dist/florence2-model.d.ts +0 -36
- package/dist/florence2-model.js +0 -503
- package/dist/florence2-model.js.map +0 -1
- package/dist/index.d.ts +0 -3
- package/dist/ocr-service-real.d.ts +0 -32
- package/dist/ocr-service-real.js +0 -396
- package/dist/ocr-service-real.js.map +0 -1
- package/dist/ocr-service.d.ts +0 -28
- package/dist/ocr-service.js +0 -216
- package/dist/ocr-service.js.map +0 -1
- package/dist/provider.d.ts +0 -2
- package/dist/provider.js +0 -285
- package/dist/provider.js.map +0 -1
- package/dist/screen-capture.d.ts +0 -16
- package/dist/screen-capture.js +0 -302
- package/dist/screen-capture.js.map +0 -1
- package/dist/service.d.ts +0 -73
- package/dist/service.js +0 -1662
- package/dist/service.js.map +0 -1
- package/dist/tests/e2e/index.d.ts +0 -8
- package/dist/tests/e2e/index.js +0 -33
- package/dist/tests/e2e/index.js.map +0 -1
- package/dist/tests/e2e/run-local.d.ts +0 -2
- package/dist/tests/e2e/run-local.js +0 -166
- package/dist/tests/e2e/run-local.js.map +0 -1
- package/dist/tests/e2e/screen-vision.d.ts +0 -11
- package/dist/tests/e2e/screen-vision.js +0 -384
- package/dist/tests/e2e/screen-vision.js.map +0 -1
- package/dist/tests/e2e/vision-autonomy.d.ts +0 -11
- package/dist/tests/e2e/vision-autonomy.js +0 -375
- package/dist/tests/e2e/vision-autonomy.js.map +0 -1
- package/dist/tests/e2e/vision-basic.d.ts +0 -11
- package/dist/tests/e2e/vision-basic.js +0 -434
- package/dist/tests/e2e/vision-basic.js.map +0 -1
- package/dist/tests/e2e/vision-capture-log.d.ts +0 -11
- package/dist/tests/e2e/vision-capture-log.js +0 -302
- package/dist/tests/e2e/vision-capture-log.js.map +0 -1
- package/dist/tests/e2e/vision-runtime.d.ts +0 -11
- package/dist/tests/e2e/vision-runtime.js +0 -357
- package/dist/tests/e2e/vision-runtime.js.map +0 -1
- package/dist/tests/e2e/vision-worker-tests.d.ts +0 -11
- package/dist/tests/e2e/vision-worker-tests.js +0 -466
- package/dist/tests/e2e/vision-worker-tests.js.map +0 -1
- package/dist/tests/test-pattern-generator.d.ts +0 -40
- package/dist/tests/test-pattern-generator.js +0 -191
- package/dist/tests/test-pattern-generator.js.map +0 -1
- package/dist/tests.d.ts +0 -3
- package/dist/tests.js +0 -11
- package/dist/tests.js.map +0 -1
- package/dist/types.d.ts +0 -222
- package/dist/types.js +0 -16
- package/dist/types.js.map +0 -1
- package/dist/vision-models.d.ts +0 -47
- package/dist/vision-models.js +0 -501
- package/dist/vision-models.js.map +0 -1
- package/dist/vision-worker-manager.d.ts +0 -61
- package/dist/vision-worker-manager.js +0 -668
- package/dist/vision-worker-manager.js.map +0 -1
- package/dist/workers/florence2-worker-simple.d.ts +0 -13
- package/dist/workers/florence2-worker-simple.js +0 -121
- package/dist/workers/florence2-worker-simple.js.map +0 -1
- package/dist/workers/florence2-worker.d.ts +0 -1
- package/dist/workers/ocr-worker.d.ts +0 -1
- package/dist/workers/screen-capture-worker.d.ts +0 -1
- package/dist/workers/worker-logger.d.ts +0 -9
- package/dist/workers/worker-logger.js +0 -95
- package/dist/workers/worker-logger.js.map +0 -1
|
@@ -1,668 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
|
-
var __generator = (this && this.__generator) || function (thisArg, body) {
|
|
12
|
-
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype);
|
|
13
|
-
return g.next = verb(0), g["throw"] = verb(1), g["return"] = verb(2), typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
|
|
14
|
-
function verb(n) { return function (v) { return step([n, v]); }; }
|
|
15
|
-
function step(op) {
|
|
16
|
-
if (f) throw new TypeError("Generator is already executing.");
|
|
17
|
-
while (g && (g = 0, op[0] && (_ = 0)), _) try {
|
|
18
|
-
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
|
|
19
|
-
if (y = 0, t) op = [op[0] & 2, t.value];
|
|
20
|
-
switch (op[0]) {
|
|
21
|
-
case 0: case 1: t = op; break;
|
|
22
|
-
case 4: _.label++; return { value: op[1], done: false };
|
|
23
|
-
case 5: _.label++; y = op[1]; op = [0]; continue;
|
|
24
|
-
case 7: op = _.ops.pop(); _.trys.pop(); continue;
|
|
25
|
-
default:
|
|
26
|
-
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
|
|
27
|
-
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
|
|
28
|
-
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
|
|
29
|
-
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
|
|
30
|
-
if (t[2]) _.ops.pop();
|
|
31
|
-
_.trys.pop(); continue;
|
|
32
|
-
}
|
|
33
|
-
op = body.call(thisArg, _);
|
|
34
|
-
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
|
|
35
|
-
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
|
|
36
|
-
}
|
|
37
|
-
};
|
|
38
|
-
var __spreadArray = (this && this.__spreadArray) || function (to, from, pack) {
|
|
39
|
-
if (pack || arguments.length === 2) for (var i = 0, l = from.length, ar; i < l; i++) {
|
|
40
|
-
if (ar || !(i in from)) {
|
|
41
|
-
if (!ar) ar = Array.prototype.slice.call(from, 0, i);
|
|
42
|
-
ar[i] = from[i];
|
|
43
|
-
}
|
|
44
|
-
}
|
|
45
|
-
return to.concat(ar || Array.prototype.slice.call(from));
|
|
46
|
-
};
|
|
47
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
48
|
-
exports.VisionWorkerManager = void 0;
|
|
49
|
-
var worker_threads_1 = require("worker_threads");
|
|
50
|
-
var path = require("path");
|
|
51
|
-
var util_1 = require("util");
|
|
52
|
-
var core_1 = require("@elizaos/core");
|
|
53
|
-
var VisionWorkerManager = /** @class */ (function () {
|
|
54
|
-
function VisionWorkerManager(config) {
|
|
55
|
-
// Workers
|
|
56
|
-
this.screenCaptureWorker = null;
|
|
57
|
-
this.florence2Worker = null;
|
|
58
|
-
this.ocrWorker = null;
|
|
59
|
-
// Buffer sizes
|
|
60
|
-
this.SCREEN_BUFFER_SIZE = 50 * 1024 * 1024; // 50MB for 4K screen
|
|
61
|
-
this.FLORENCE2_RESULTS_SIZE = 10 * 1024 * 1024; // 10MB for results
|
|
62
|
-
this.OCR_RESULTS_SIZE = 5 * 1024 * 1024; // 5MB for OCR text
|
|
63
|
-
// Atomic indices
|
|
64
|
-
this.FRAME_ID_INDEX = 0;
|
|
65
|
-
this.WIDTH_INDEX = 2;
|
|
66
|
-
this.HEIGHT_INDEX = 3;
|
|
67
|
-
this.DISPLAY_INDEX = 4;
|
|
68
|
-
this.TIMESTAMP_INDEX = 5;
|
|
69
|
-
this.DATA_OFFSET = 24;
|
|
70
|
-
// Worker stats
|
|
71
|
-
this.workerStats = new Map();
|
|
72
|
-
// Latest processed data cache
|
|
73
|
-
this.latestScreenCapture = null;
|
|
74
|
-
this.latestFlorence2Results = new Map();
|
|
75
|
-
this.latestOCRResult = null;
|
|
76
|
-
this.lastProcessedFrameId = -1;
|
|
77
|
-
// Worker restart tracking
|
|
78
|
-
this.restartAttempts = new Map();
|
|
79
|
-
this.MAX_RESTART_ATTEMPTS = 3;
|
|
80
|
-
this.config = config;
|
|
81
|
-
// Initialize shared buffers
|
|
82
|
-
this.screenBuffer = new SharedArrayBuffer(this.SCREEN_BUFFER_SIZE);
|
|
83
|
-
this.florence2ResultsBuffer = new SharedArrayBuffer(this.FLORENCE2_RESULTS_SIZE);
|
|
84
|
-
this.ocrResultsBuffer = new SharedArrayBuffer(this.OCR_RESULTS_SIZE);
|
|
85
|
-
// Create views
|
|
86
|
-
this.screenAtomicState = new Int32Array(this.screenBuffer, 0, 6);
|
|
87
|
-
this.screenDataView = new DataView(this.screenBuffer);
|
|
88
|
-
this.florence2ResultsView = new DataView(this.florence2ResultsBuffer);
|
|
89
|
-
this.ocrResultsView = new DataView(this.ocrResultsBuffer);
|
|
90
|
-
}
|
|
91
|
-
VisionWorkerManager.prototype.initialize = function () {
|
|
92
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
93
|
-
var error_1;
|
|
94
|
-
return __generator(this, function (_a) {
|
|
95
|
-
switch (_a.label) {
|
|
96
|
-
case 0:
|
|
97
|
-
core_1.logger.info('[VisionWorkerManager] Initializing worker threads...');
|
|
98
|
-
_a.label = 1;
|
|
99
|
-
case 1:
|
|
100
|
-
_a.trys.push([1, 7, , 8]);
|
|
101
|
-
// Start screen capture worker
|
|
102
|
-
return [4 /*yield*/, this.startScreenCaptureWorker()];
|
|
103
|
-
case 2:
|
|
104
|
-
// Start screen capture worker
|
|
105
|
-
_a.sent();
|
|
106
|
-
if (!this.config.florence2Enabled) return [3 /*break*/, 4];
|
|
107
|
-
return [4 /*yield*/, this.startFlorence2Worker()];
|
|
108
|
-
case 3:
|
|
109
|
-
_a.sent();
|
|
110
|
-
_a.label = 4;
|
|
111
|
-
case 4:
|
|
112
|
-
if (!this.config.ocrEnabled) return [3 /*break*/, 6];
|
|
113
|
-
return [4 /*yield*/, this.startOCRWorker()];
|
|
114
|
-
case 5:
|
|
115
|
-
_a.sent();
|
|
116
|
-
_a.label = 6;
|
|
117
|
-
case 6:
|
|
118
|
-
core_1.logger.info('[VisionWorkerManager] All workers initialized');
|
|
119
|
-
return [3 /*break*/, 8];
|
|
120
|
-
case 7:
|
|
121
|
-
error_1 = _a.sent();
|
|
122
|
-
core_1.logger.error('[VisionWorkerManager] Failed to initialize workers:', error_1);
|
|
123
|
-
throw error_1;
|
|
124
|
-
case 8: return [2 /*return*/];
|
|
125
|
-
}
|
|
126
|
-
});
|
|
127
|
-
});
|
|
128
|
-
};
|
|
129
|
-
VisionWorkerManager.prototype.startScreenCaptureWorker = function () {
|
|
130
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
131
|
-
var workerPath;
|
|
132
|
-
var _this = this;
|
|
133
|
-
return __generator(this, function (_a) {
|
|
134
|
-
workerPath = path.join(__dirname, 'workers', 'screen-capture-worker.js');
|
|
135
|
-
this.screenCaptureWorker = new worker_threads_1.Worker(workerPath, {
|
|
136
|
-
workerData: {
|
|
137
|
-
config: {
|
|
138
|
-
displayIndex: this.config.displayIndex,
|
|
139
|
-
captureAllDisplays: this.config.captureAllDisplays,
|
|
140
|
-
targetFPS: this.config.targetScreenFPS,
|
|
141
|
-
sharedBufferSize: this.SCREEN_BUFFER_SIZE,
|
|
142
|
-
},
|
|
143
|
-
sharedBuffer: this.screenBuffer,
|
|
144
|
-
},
|
|
145
|
-
});
|
|
146
|
-
this.screenCaptureWorker.on('message', function (msg) {
|
|
147
|
-
if (msg.type === 'fps') {
|
|
148
|
-
_this.workerStats.set('screenCapture', {
|
|
149
|
-
fps: msg.fps,
|
|
150
|
-
frameCount: msg.frameCount,
|
|
151
|
-
lastUpdate: Date.now(),
|
|
152
|
-
});
|
|
153
|
-
}
|
|
154
|
-
else if (msg.type === 'error') {
|
|
155
|
-
core_1.logger.error('[ScreenCaptureWorker] Error:', msg.error);
|
|
156
|
-
}
|
|
157
|
-
else if (msg.type === 'log') {
|
|
158
|
-
// Handle worker log messages
|
|
159
|
-
_this.handleWorkerLog('ScreenCaptureWorker', msg);
|
|
160
|
-
}
|
|
161
|
-
});
|
|
162
|
-
this.screenCaptureWorker.on('error', function (error) {
|
|
163
|
-
core_1.logger.error('[ScreenCaptureWorker] Worker error:', error);
|
|
164
|
-
// Attempt to restart worker after error
|
|
165
|
-
setTimeout(function () { return _this.restartScreenCaptureWorker(); }, 1000);
|
|
166
|
-
});
|
|
167
|
-
this.screenCaptureWorker.on('exit', function (code) {
|
|
168
|
-
if (code !== 0) {
|
|
169
|
-
core_1.logger.error("[ScreenCaptureWorker] Worker stopped with exit code ".concat(code));
|
|
170
|
-
// Attempt to restart worker after crash
|
|
171
|
-
setTimeout(function () { return _this.restartScreenCaptureWorker(); }, 1000);
|
|
172
|
-
}
|
|
173
|
-
});
|
|
174
|
-
return [2 /*return*/];
|
|
175
|
-
});
|
|
176
|
-
});
|
|
177
|
-
};
|
|
178
|
-
VisionWorkerManager.prototype.startFlorence2Worker = function () {
|
|
179
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
180
|
-
var workerPath, priorityTiles, tileSize, estimatedCols, centerRow, centerCol, r, c;
|
|
181
|
-
var _this = this;
|
|
182
|
-
return __generator(this, function (_a) {
|
|
183
|
-
workerPath = path.join(__dirname, 'workers', 'florence2-worker.js');
|
|
184
|
-
priorityTiles = [];
|
|
185
|
-
tileSize = this.config.tileSize || 256;
|
|
186
|
-
estimatedCols = Math.ceil(1920 / tileSize);
|
|
187
|
-
centerRow = Math.floor(estimatedCols / 2);
|
|
188
|
-
centerCol = Math.floor(estimatedCols / 2);
|
|
189
|
-
// Add center and adjacent tiles as priority
|
|
190
|
-
for (r = centerRow - 1; r <= centerRow + 1; r++) {
|
|
191
|
-
for (c = centerCol - 1; c <= centerCol + 1; c++) {
|
|
192
|
-
if (r >= 0 && c >= 0) {
|
|
193
|
-
priorityTiles.push(r * estimatedCols + c);
|
|
194
|
-
}
|
|
195
|
-
}
|
|
196
|
-
}
|
|
197
|
-
this.florence2Worker = new worker_threads_1.Worker(workerPath, {
|
|
198
|
-
workerData: {
|
|
199
|
-
config: {
|
|
200
|
-
tileSize: this.config.tileSize || 256,
|
|
201
|
-
priorityTiles: priorityTiles,
|
|
202
|
-
},
|
|
203
|
-
sharedBuffer: this.screenBuffer,
|
|
204
|
-
resultsBuffer: this.florence2ResultsBuffer,
|
|
205
|
-
},
|
|
206
|
-
});
|
|
207
|
-
this.florence2Worker.on('message', function (msg) {
|
|
208
|
-
if (msg.type === 'fps') {
|
|
209
|
-
_this.workerStats.set('florence2', {
|
|
210
|
-
fps: msg.fps,
|
|
211
|
-
frameCount: msg.frameCount,
|
|
212
|
-
lastUpdate: Date.now(),
|
|
213
|
-
});
|
|
214
|
-
}
|
|
215
|
-
else if (msg.type === 'tile_analyzed') {
|
|
216
|
-
// Update latest results cache
|
|
217
|
-
_this.updateFlorence2Cache(msg);
|
|
218
|
-
}
|
|
219
|
-
else if (msg.type === 'error') {
|
|
220
|
-
core_1.logger.error('[Florence2Worker] Error:', msg.error);
|
|
221
|
-
}
|
|
222
|
-
else if (msg.type === 'log') {
|
|
223
|
-
// Handle worker log messages
|
|
224
|
-
_this.handleWorkerLog('Florence2Worker', msg);
|
|
225
|
-
}
|
|
226
|
-
});
|
|
227
|
-
this.florence2Worker.on('error', function (error) {
|
|
228
|
-
core_1.logger.error('[Florence2Worker] Worker error:', error);
|
|
229
|
-
// Attempt to restart worker after error
|
|
230
|
-
setTimeout(function () { return _this.restartFlorence2Worker(); }, 1000);
|
|
231
|
-
});
|
|
232
|
-
this.florence2Worker.on('exit', function (code) {
|
|
233
|
-
if (code !== 0) {
|
|
234
|
-
core_1.logger.error("[Florence2Worker] Worker stopped with exit code ".concat(code));
|
|
235
|
-
// Attempt to restart worker after crash
|
|
236
|
-
setTimeout(function () { return _this.restartFlorence2Worker(); }, 1000);
|
|
237
|
-
}
|
|
238
|
-
});
|
|
239
|
-
return [2 /*return*/];
|
|
240
|
-
});
|
|
241
|
-
});
|
|
242
|
-
};
|
|
243
|
-
VisionWorkerManager.prototype.startOCRWorker = function () {
|
|
244
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
245
|
-
var workerPath;
|
|
246
|
-
var _this = this;
|
|
247
|
-
return __generator(this, function (_a) {
|
|
248
|
-
workerPath = path.join(__dirname, 'workers', 'ocr-worker.js');
|
|
249
|
-
this.ocrWorker = new worker_threads_1.Worker(workerPath, {
|
|
250
|
-
workerData: {
|
|
251
|
-
config: {
|
|
252
|
-
processFullScreen: true,
|
|
253
|
-
tileSize: this.config.tileSize || 256,
|
|
254
|
-
textRegions: this.config.textRegions,
|
|
255
|
-
},
|
|
256
|
-
sharedBuffer: this.screenBuffer,
|
|
257
|
-
resultsBuffer: this.ocrResultsBuffer,
|
|
258
|
-
},
|
|
259
|
-
});
|
|
260
|
-
this.ocrWorker.on('message', function (msg) {
|
|
261
|
-
if (msg.type === 'fps') {
|
|
262
|
-
_this.workerStats.set('ocr', {
|
|
263
|
-
fps: msg.fps,
|
|
264
|
-
frameCount: msg.frameCount,
|
|
265
|
-
lastUpdate: Date.now(),
|
|
266
|
-
});
|
|
267
|
-
}
|
|
268
|
-
else if (msg.type === 'ocr_complete') {
|
|
269
|
-
// Update latest OCR cache
|
|
270
|
-
_this.updateOCRCache(msg);
|
|
271
|
-
}
|
|
272
|
-
else if (msg.type === 'error') {
|
|
273
|
-
core_1.logger.error('[OCRWorker] Error:', msg.error);
|
|
274
|
-
}
|
|
275
|
-
else if (msg.type === 'log') {
|
|
276
|
-
// Handle worker log messages
|
|
277
|
-
_this.handleWorkerLog('OCRWorker', msg);
|
|
278
|
-
}
|
|
279
|
-
});
|
|
280
|
-
this.ocrWorker.on('error', function (error) {
|
|
281
|
-
core_1.logger.error('[OCRWorker] Worker error:', error);
|
|
282
|
-
// Attempt to restart worker after error
|
|
283
|
-
setTimeout(function () { return _this.restartOCRWorker(); }, 1000);
|
|
284
|
-
});
|
|
285
|
-
this.ocrWorker.on('exit', function (code) {
|
|
286
|
-
if (code !== 0) {
|
|
287
|
-
core_1.logger.error("[OCRWorker] Worker stopped with exit code ".concat(code));
|
|
288
|
-
// Attempt to restart worker after crash
|
|
289
|
-
setTimeout(function () { return _this.restartOCRWorker(); }, 1000);
|
|
290
|
-
}
|
|
291
|
-
});
|
|
292
|
-
return [2 /*return*/];
|
|
293
|
-
});
|
|
294
|
-
});
|
|
295
|
-
};
|
|
296
|
-
VisionWorkerManager.prototype.updateFlorence2Cache = function (msg) {
|
|
297
|
-
// Read result from shared buffer
|
|
298
|
-
try {
|
|
299
|
-
var tileId = msg.tileId;
|
|
300
|
-
var result = this.readFlorence2Result(tileId);
|
|
301
|
-
if (result) {
|
|
302
|
-
this.latestFlorence2Results.set(tileId, result);
|
|
303
|
-
}
|
|
304
|
-
}
|
|
305
|
-
catch (error) {
|
|
306
|
-
core_1.logger.error('[VisionWorkerManager] Failed to update Florence2 cache:', error);
|
|
307
|
-
}
|
|
308
|
-
};
|
|
309
|
-
VisionWorkerManager.prototype.updateOCRCache = function (_msg) {
|
|
310
|
-
// Read OCR result from shared buffer
|
|
311
|
-
try {
|
|
312
|
-
var result = this.readOCRResult();
|
|
313
|
-
if (result) {
|
|
314
|
-
this.latestOCRResult = result;
|
|
315
|
-
}
|
|
316
|
-
}
|
|
317
|
-
catch (error) {
|
|
318
|
-
core_1.logger.error('[VisionWorkerManager] Failed to update OCR cache:', error);
|
|
319
|
-
}
|
|
320
|
-
};
|
|
321
|
-
VisionWorkerManager.prototype.readFlorence2Result = function (tileId) {
|
|
322
|
-
try {
|
|
323
|
-
// Calculate tile index from ID
|
|
324
|
-
var match = tileId.match(/tile-(\d+)-(\d+)/);
|
|
325
|
-
if (!match) {
|
|
326
|
-
return null;
|
|
327
|
-
}
|
|
328
|
-
var row = parseInt(match[1], 10);
|
|
329
|
-
var col = parseInt(match[2], 10);
|
|
330
|
-
var tileIndex = row * 10 + col; // Assuming max 10 columns
|
|
331
|
-
var RESULTS_HEADER_SIZE = 16;
|
|
332
|
-
var MAX_RESULT_SIZE = 4096;
|
|
333
|
-
var offset = RESULTS_HEADER_SIZE + tileIndex * MAX_RESULT_SIZE;
|
|
334
|
-
// Read length
|
|
335
|
-
var length_1 = this.florence2ResultsView.getUint32(offset, true);
|
|
336
|
-
if (length_1 === 0 || length_1 > MAX_RESULT_SIZE - 4) {
|
|
337
|
-
return null;
|
|
338
|
-
}
|
|
339
|
-
// Read JSON data
|
|
340
|
-
var bytes = new Uint8Array(length_1);
|
|
341
|
-
for (var i = 0; i < length_1; i++) {
|
|
342
|
-
bytes[i] = this.florence2ResultsView.getUint8(offset + 4 + i);
|
|
343
|
-
}
|
|
344
|
-
var json = new util_1.TextDecoder().decode(bytes);
|
|
345
|
-
return JSON.parse(json);
|
|
346
|
-
}
|
|
347
|
-
catch (error) {
|
|
348
|
-
core_1.logger.error('[VisionWorkerManager] Failed to read Florence2 result:', error);
|
|
349
|
-
return null;
|
|
350
|
-
}
|
|
351
|
-
};
|
|
352
|
-
VisionWorkerManager.prototype.readOCRResult = function () {
|
|
353
|
-
try {
|
|
354
|
-
var RESULTS_HEADER_SIZE = 16;
|
|
355
|
-
var offset = RESULTS_HEADER_SIZE;
|
|
356
|
-
// Read length
|
|
357
|
-
var length_2 = this.ocrResultsView.getUint32(offset, true);
|
|
358
|
-
if (length_2 === 0) {
|
|
359
|
-
return null;
|
|
360
|
-
}
|
|
361
|
-
// Read frame ID and timestamp
|
|
362
|
-
var _frameId = this.ocrResultsView.getUint32(offset + 4, true);
|
|
363
|
-
var _timestamp = this.ocrResultsView.getFloat64(offset + 8, true);
|
|
364
|
-
// Read JSON data
|
|
365
|
-
var dataOffset = offset + 16;
|
|
366
|
-
var bytes = new Uint8Array(Math.min(length_2, 65536));
|
|
367
|
-
for (var i = 0; i < bytes.length; i++) {
|
|
368
|
-
bytes[i] = this.ocrResultsView.getUint8(dataOffset + i);
|
|
369
|
-
}
|
|
370
|
-
var json = new util_1.TextDecoder().decode(bytes);
|
|
371
|
-
return JSON.parse(json);
|
|
372
|
-
}
|
|
373
|
-
catch (error) {
|
|
374
|
-
core_1.logger.error('[VisionWorkerManager] Failed to read OCR result:', error);
|
|
375
|
-
return null;
|
|
376
|
-
}
|
|
377
|
-
};
|
|
378
|
-
// Public API - Non-blocking access to latest data
|
|
379
|
-
VisionWorkerManager.prototype.getLatestScreenCapture = function () {
|
|
380
|
-
// Read current frame metadata
|
|
381
|
-
var frameId = Atomics.load(this.screenAtomicState, this.FRAME_ID_INDEX);
|
|
382
|
-
if (frameId <= this.lastProcessedFrameId) {
|
|
383
|
-
return this.latestScreenCapture;
|
|
384
|
-
}
|
|
385
|
-
try {
|
|
386
|
-
var width = Atomics.load(this.screenAtomicState, this.WIDTH_INDEX);
|
|
387
|
-
var height = Atomics.load(this.screenAtomicState, this.HEIGHT_INDEX);
|
|
388
|
-
var _displayIndex = Atomics.load(this.screenAtomicState, this.DISPLAY_INDEX);
|
|
389
|
-
var timestamp = Atomics.load(this.screenAtomicState, this.TIMESTAMP_INDEX);
|
|
390
|
-
// Create screen capture object
|
|
391
|
-
this.latestScreenCapture = {
|
|
392
|
-
timestamp: timestamp,
|
|
393
|
-
width: width,
|
|
394
|
-
height: height,
|
|
395
|
-
data: Buffer.alloc(0), // We don't copy the full data for performance
|
|
396
|
-
tiles: this.generateTiles(width, height),
|
|
397
|
-
};
|
|
398
|
-
this.lastProcessedFrameId = frameId;
|
|
399
|
-
}
|
|
400
|
-
catch (error) {
|
|
401
|
-
core_1.logger.error('[VisionWorkerManager] Failed to read screen capture:', error);
|
|
402
|
-
}
|
|
403
|
-
return this.latestScreenCapture;
|
|
404
|
-
};
|
|
405
|
-
VisionWorkerManager.prototype.getLatestEnhancedScene = function () {
|
|
406
|
-
var _a;
|
|
407
|
-
var screenCapture = this.getLatestScreenCapture();
|
|
408
|
-
// Combine all Florence-2 results
|
|
409
|
-
var florence2Captions = [];
|
|
410
|
-
var allObjects = [];
|
|
411
|
-
var allTags = new Set();
|
|
412
|
-
this.latestFlorence2Results.forEach(function (result) {
|
|
413
|
-
if (result.caption) {
|
|
414
|
-
florence2Captions.push(result.caption);
|
|
415
|
-
}
|
|
416
|
-
if (result.objects) {
|
|
417
|
-
allObjects.push.apply(allObjects, result.objects);
|
|
418
|
-
}
|
|
419
|
-
if (result.tags) {
|
|
420
|
-
result.tags.forEach(function (tag) { return allTags.add(tag); });
|
|
421
|
-
}
|
|
422
|
-
});
|
|
423
|
-
var _tiles = this.generateTiles((screenCapture === null || screenCapture === void 0 ? void 0 : screenCapture.width) || 0, (screenCapture === null || screenCapture === void 0 ? void 0 : screenCapture.height) || 0);
|
|
424
|
-
return {
|
|
425
|
-
timestamp: Date.now(),
|
|
426
|
-
description: florence2Captions.join('. '),
|
|
427
|
-
objects: allObjects,
|
|
428
|
-
people: [], // Could be populated by TensorFlow worker
|
|
429
|
-
sceneChanged: true,
|
|
430
|
-
changePercentage: 100,
|
|
431
|
-
screenCapture: this.latestScreenCapture || undefined,
|
|
432
|
-
screenAnalysis: {
|
|
433
|
-
fullScreenOCR: (_a = this.latestOCRResult) === null || _a === void 0 ? void 0 : _a.fullText,
|
|
434
|
-
activeTile: {
|
|
435
|
-
timestamp: Date.now(),
|
|
436
|
-
florence2: this.latestFlorence2Results.values().next().value,
|
|
437
|
-
ocr: this.latestOCRResult || undefined,
|
|
438
|
-
},
|
|
439
|
-
gridSummary: "".concat((screenCapture === null || screenCapture === void 0 ? void 0 : screenCapture.tiles.length) || 0, " tiles analyzed"),
|
|
440
|
-
uiElements: allObjects.map(function (obj) { return ({
|
|
441
|
-
type: obj.label,
|
|
442
|
-
text: '',
|
|
443
|
-
position: obj.bbox,
|
|
444
|
-
}); }),
|
|
445
|
-
},
|
|
446
|
-
};
|
|
447
|
-
};
|
|
448
|
-
VisionWorkerManager.prototype.generateTiles = function (width, height) {
|
|
449
|
-
var tileSize = this.config.tileSize || 256;
|
|
450
|
-
var tiles = [];
|
|
451
|
-
for (var row = 0; row < Math.ceil(height / tileSize); row++) {
|
|
452
|
-
for (var col = 0; col < Math.ceil(width / tileSize); col++) {
|
|
453
|
-
var x = col * tileSize;
|
|
454
|
-
var y = row * tileSize;
|
|
455
|
-
tiles.push({
|
|
456
|
-
id: "tile-".concat(row, "-").concat(col),
|
|
457
|
-
row: row,
|
|
458
|
-
col: col,
|
|
459
|
-
x: x,
|
|
460
|
-
y: y,
|
|
461
|
-
width: Math.min(tileSize, width - x),
|
|
462
|
-
height: Math.min(tileSize, height - y),
|
|
463
|
-
});
|
|
464
|
-
}
|
|
465
|
-
}
|
|
466
|
-
return tiles;
|
|
467
|
-
};
|
|
468
|
-
VisionWorkerManager.prototype.getWorkerStats = function () {
|
|
469
|
-
return new Map(this.workerStats);
|
|
470
|
-
};
|
|
471
|
-
VisionWorkerManager.prototype.setDisplayIndex = function (index) {
|
|
472
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
473
|
-
return __generator(this, function (_a) {
|
|
474
|
-
if (this.screenCaptureWorker) {
|
|
475
|
-
this.screenCaptureWorker.postMessage({
|
|
476
|
-
type: 'set_display',
|
|
477
|
-
displayIndex: index,
|
|
478
|
-
});
|
|
479
|
-
}
|
|
480
|
-
return [2 /*return*/];
|
|
481
|
-
});
|
|
482
|
-
});
|
|
483
|
-
};
|
|
484
|
-
VisionWorkerManager.prototype.setTextRegions = function (regions) {
|
|
485
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
486
|
-
return __generator(this, function (_a) {
|
|
487
|
-
if (this.ocrWorker) {
|
|
488
|
-
this.ocrWorker.postMessage({
|
|
489
|
-
type: 'update_regions',
|
|
490
|
-
regions: regions,
|
|
491
|
-
});
|
|
492
|
-
}
|
|
493
|
-
return [2 /*return*/];
|
|
494
|
-
});
|
|
495
|
-
});
|
|
496
|
-
};
|
|
497
|
-
VisionWorkerManager.prototype.stop = function () {
|
|
498
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
499
|
-
var stopPromises;
|
|
500
|
-
var _this = this;
|
|
501
|
-
return __generator(this, function (_a) {
|
|
502
|
-
switch (_a.label) {
|
|
503
|
-
case 0:
|
|
504
|
-
core_1.logger.info('[VisionWorkerManager] Stopping all workers...');
|
|
505
|
-
stopPromises = [];
|
|
506
|
-
if (this.screenCaptureWorker) {
|
|
507
|
-
stopPromises.push(new Promise(function (resolve) {
|
|
508
|
-
_this.screenCaptureWorker.once('exit', function () { return resolve(); });
|
|
509
|
-
_this.screenCaptureWorker.postMessage({ type: 'stop' });
|
|
510
|
-
}));
|
|
511
|
-
}
|
|
512
|
-
if (this.florence2Worker) {
|
|
513
|
-
stopPromises.push(new Promise(function (resolve) {
|
|
514
|
-
_this.florence2Worker.once('exit', function () { return resolve(); });
|
|
515
|
-
_this.florence2Worker.postMessage({ type: 'stop' });
|
|
516
|
-
}));
|
|
517
|
-
}
|
|
518
|
-
if (this.ocrWorker) {
|
|
519
|
-
stopPromises.push(new Promise(function (resolve) {
|
|
520
|
-
_this.ocrWorker.once('exit', function () { return resolve(); });
|
|
521
|
-
_this.ocrWorker.postMessage({ type: 'stop' });
|
|
522
|
-
}));
|
|
523
|
-
}
|
|
524
|
-
return [4 /*yield*/, Promise.all(stopPromises)];
|
|
525
|
-
case 1:
|
|
526
|
-
_a.sent();
|
|
527
|
-
core_1.logger.info('[VisionWorkerManager] All workers stopped');
|
|
528
|
-
return [2 /*return*/];
|
|
529
|
-
}
|
|
530
|
-
});
|
|
531
|
-
});
|
|
532
|
-
};
|
|
533
|
-
VisionWorkerManager.prototype.handleWorkerLog = function (workerName, msg) {
|
|
534
|
-
var level = msg.level, message = msg.message, args = msg.args;
|
|
535
|
-
var formattedMessage = "[".concat(workerName, "] ").concat(message);
|
|
536
|
-
switch (level) {
|
|
537
|
-
case 'info':
|
|
538
|
-
core_1.logger.info.apply(core_1.logger, __spreadArray([formattedMessage], args, false));
|
|
539
|
-
break;
|
|
540
|
-
case 'warn':
|
|
541
|
-
core_1.logger.warn.apply(core_1.logger, __spreadArray([formattedMessage], args, false));
|
|
542
|
-
break;
|
|
543
|
-
case 'error':
|
|
544
|
-
core_1.logger.error.apply(core_1.logger, __spreadArray([formattedMessage], args, false));
|
|
545
|
-
break;
|
|
546
|
-
case 'debug':
|
|
547
|
-
core_1.logger.debug.apply(core_1.logger, __spreadArray([formattedMessage], args, false));
|
|
548
|
-
break;
|
|
549
|
-
}
|
|
550
|
-
};
|
|
551
|
-
VisionWorkerManager.prototype.restartScreenCaptureWorker = function () {
|
|
552
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
553
|
-
var attempts, error_2;
|
|
554
|
-
return __generator(this, function (_a) {
|
|
555
|
-
switch (_a.label) {
|
|
556
|
-
case 0:
|
|
557
|
-
attempts = this.restartAttempts.get('screenCapture') || 0;
|
|
558
|
-
if (attempts >= this.MAX_RESTART_ATTEMPTS) {
|
|
559
|
-
core_1.logger.error('[VisionWorkerManager] Max restart attempts reached for screen capture worker');
|
|
560
|
-
return [2 /*return*/];
|
|
561
|
-
}
|
|
562
|
-
this.restartAttempts.set('screenCapture', attempts + 1);
|
|
563
|
-
core_1.logger.info("[VisionWorkerManager] Restarting screen capture worker (attempt ".concat(attempts + 1, ")"));
|
|
564
|
-
_a.label = 1;
|
|
565
|
-
case 1:
|
|
566
|
-
_a.trys.push([1, 3, , 4]);
|
|
567
|
-
// Clean up existing worker
|
|
568
|
-
if (this.screenCaptureWorker) {
|
|
569
|
-
this.screenCaptureWorker.removeAllListeners();
|
|
570
|
-
this.screenCaptureWorker = null;
|
|
571
|
-
}
|
|
572
|
-
// Start new worker
|
|
573
|
-
return [4 /*yield*/, this.startScreenCaptureWorker()];
|
|
574
|
-
case 2:
|
|
575
|
-
// Start new worker
|
|
576
|
-
_a.sent();
|
|
577
|
-
// Reset restart counter on successful start
|
|
578
|
-
this.restartAttempts.set('screenCapture', 0);
|
|
579
|
-
return [3 /*break*/, 4];
|
|
580
|
-
case 3:
|
|
581
|
-
error_2 = _a.sent();
|
|
582
|
-
core_1.logger.error('[VisionWorkerManager] Failed to restart screen capture worker:', error_2);
|
|
583
|
-
return [3 /*break*/, 4];
|
|
584
|
-
case 4: return [2 /*return*/];
|
|
585
|
-
}
|
|
586
|
-
});
|
|
587
|
-
});
|
|
588
|
-
};
|
|
589
|
-
VisionWorkerManager.prototype.restartFlorence2Worker = function () {
|
|
590
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
591
|
-
var attempts, error_3;
|
|
592
|
-
return __generator(this, function (_a) {
|
|
593
|
-
switch (_a.label) {
|
|
594
|
-
case 0:
|
|
595
|
-
attempts = this.restartAttempts.get('florence2') || 0;
|
|
596
|
-
if (attempts >= this.MAX_RESTART_ATTEMPTS) {
|
|
597
|
-
core_1.logger.error('[VisionWorkerManager] Max restart attempts reached for Florence2 worker');
|
|
598
|
-
return [2 /*return*/];
|
|
599
|
-
}
|
|
600
|
-
this.restartAttempts.set('florence2', attempts + 1);
|
|
601
|
-
core_1.logger.info("[VisionWorkerManager] Restarting Florence2 worker (attempt ".concat(attempts + 1, ")"));
|
|
602
|
-
_a.label = 1;
|
|
603
|
-
case 1:
|
|
604
|
-
_a.trys.push([1, 3, , 4]);
|
|
605
|
-
// Clean up existing worker
|
|
606
|
-
if (this.florence2Worker) {
|
|
607
|
-
this.florence2Worker.removeAllListeners();
|
|
608
|
-
this.florence2Worker = null;
|
|
609
|
-
}
|
|
610
|
-
// Start new worker
|
|
611
|
-
return [4 /*yield*/, this.startFlorence2Worker()];
|
|
612
|
-
case 2:
|
|
613
|
-
// Start new worker
|
|
614
|
-
_a.sent();
|
|
615
|
-
// Reset restart counter on successful start
|
|
616
|
-
this.restartAttempts.set('florence2', 0);
|
|
617
|
-
return [3 /*break*/, 4];
|
|
618
|
-
case 3:
|
|
619
|
-
error_3 = _a.sent();
|
|
620
|
-
core_1.logger.error('[VisionWorkerManager] Failed to restart Florence2 worker:', error_3);
|
|
621
|
-
return [3 /*break*/, 4];
|
|
622
|
-
case 4: return [2 /*return*/];
|
|
623
|
-
}
|
|
624
|
-
});
|
|
625
|
-
});
|
|
626
|
-
};
|
|
627
|
-
VisionWorkerManager.prototype.restartOCRWorker = function () {
|
|
628
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
629
|
-
var attempts, error_4;
|
|
630
|
-
return __generator(this, function (_a) {
|
|
631
|
-
switch (_a.label) {
|
|
632
|
-
case 0:
|
|
633
|
-
attempts = this.restartAttempts.get('ocr') || 0;
|
|
634
|
-
if (attempts >= this.MAX_RESTART_ATTEMPTS) {
|
|
635
|
-
core_1.logger.error('[VisionWorkerManager] Max restart attempts reached for OCR worker');
|
|
636
|
-
return [2 /*return*/];
|
|
637
|
-
}
|
|
638
|
-
this.restartAttempts.set('ocr', attempts + 1);
|
|
639
|
-
core_1.logger.info("[VisionWorkerManager] Restarting OCR worker (attempt ".concat(attempts + 1, ")"));
|
|
640
|
-
_a.label = 1;
|
|
641
|
-
case 1:
|
|
642
|
-
_a.trys.push([1, 3, , 4]);
|
|
643
|
-
// Clean up existing worker
|
|
644
|
-
if (this.ocrWorker) {
|
|
645
|
-
this.ocrWorker.removeAllListeners();
|
|
646
|
-
this.ocrWorker = null;
|
|
647
|
-
}
|
|
648
|
-
// Start new worker
|
|
649
|
-
return [4 /*yield*/, this.startOCRWorker()];
|
|
650
|
-
case 2:
|
|
651
|
-
// Start new worker
|
|
652
|
-
_a.sent();
|
|
653
|
-
// Reset restart counter on successful start
|
|
654
|
-
this.restartAttempts.set('ocr', 0);
|
|
655
|
-
return [3 /*break*/, 4];
|
|
656
|
-
case 3:
|
|
657
|
-
error_4 = _a.sent();
|
|
658
|
-
core_1.logger.error('[VisionWorkerManager] Failed to restart OCR worker:', error_4);
|
|
659
|
-
return [3 /*break*/, 4];
|
|
660
|
-
case 4: return [2 /*return*/];
|
|
661
|
-
}
|
|
662
|
-
});
|
|
663
|
-
});
|
|
664
|
-
};
|
|
665
|
-
return VisionWorkerManager;
|
|
666
|
-
}());
|
|
667
|
-
exports.VisionWorkerManager = VisionWorkerManager;
|
|
668
|
-
//# sourceMappingURL=vision-worker-manager.js.map
|