@resolveio/server-lib 22.0.2 → 22.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/managers/method.manager.d.ts +12 -0
- package/managers/method.manager.js +144 -21
- package/managers/method.manager.js.map +1 -1
- package/managers/subscription.manager.js +44 -5
- package/managers/subscription.manager.js.map +1 -1
- package/managers/worker-server.manager.js +32 -17
- package/managers/worker-server.manager.js.map +1 -1
- package/package.json +1 -1
|
@@ -88,9 +88,15 @@ var WorkerServerManager = /** @class */ (function () {
|
|
|
88
88
|
return workerServerManager;
|
|
89
89
|
};
|
|
90
90
|
WorkerServerManager.prototype.initialize = function (methodManager, serverConfig) {
|
|
91
|
+
var _this = this;
|
|
91
92
|
this._methodManager = methodManager;
|
|
92
93
|
this._serverConfig = serverConfig;
|
|
93
|
-
this.
|
|
94
|
+
if (this._methodManager && typeof this._methodManager.onReady === 'function') {
|
|
95
|
+
this._methodManager.onReady(function () { return _this.startWorkerInstance(); });
|
|
96
|
+
}
|
|
97
|
+
else {
|
|
98
|
+
this.startWorkerInstance();
|
|
99
|
+
}
|
|
94
100
|
};
|
|
95
101
|
WorkerServerManager.prototype.startWorkerInstance = function () {
|
|
96
102
|
var _this = this;
|
|
@@ -246,7 +252,7 @@ var WorkerServerManager = /** @class */ (function () {
|
|
|
246
252
|
};
|
|
247
253
|
WorkerServerManager.prototype.handleIncomingTask = function (ws, data) {
|
|
248
254
|
return __awaiter(this, void 0, void 0, function () {
|
|
249
|
-
var taskId, messageId, method, params, userContext, receivedAt, queuedAt, dispatchedAt, queueWaitMs, dispatchDelayMs, timedOut, timeoutHandle, managerThis, result, packedResult, meta, publicationPayload, publicationMeta, isPublicationResponse, payload, err_1, errorAt_1, runMs_1, totalMs_1, errorAt, runMs, totalMs;
|
|
255
|
+
var taskId, messageId, method, params, userContext, receivedAt, queuedAt, dispatchedAt, queueWaitMs, dispatchDelayMs, missingMethod, error_1, timedOut, timeoutHandle, managerThis, result, packedResult, meta, publicationPayload, publicationMeta, isPublicationResponse, payload, err_1, errorAt_1, runMs_1, totalMs_1, errorAt, runMs, totalMs;
|
|
250
256
|
var _a;
|
|
251
257
|
var _this = this;
|
|
252
258
|
var _b;
|
|
@@ -260,14 +266,23 @@ var WorkerServerManager = /** @class */ (function () {
|
|
|
260
266
|
queueWaitMs = queuedAt && dispatchedAt ? (0, common_1.round)(dispatchedAt - queuedAt) : (queuedAt ? (0, common_1.round)(receivedAt - queuedAt) : null);
|
|
261
267
|
dispatchDelayMs = dispatchedAt ? (0, common_1.round)(receivedAt - dispatchedAt) : null;
|
|
262
268
|
this._runningTasks.push(taskId);
|
|
263
|
-
|
|
264
|
-
if (
|
|
269
|
+
missingMethod = !method || !this._methodManager.getMethod(method);
|
|
270
|
+
if (!(!taskId || !method || missingMethod)) return [3 /*break*/, 5];
|
|
271
|
+
if (!missingMethod) return [3 /*break*/, 4];
|
|
265
272
|
console.error(new Date(), 'No method in method manager for handleIncomingTask worker server', method);
|
|
266
|
-
|
|
273
|
+
if (!this._methodManager.getMethod('insertErrorLog')) return [3 /*break*/, 4];
|
|
274
|
+
_c.label = 1;
|
|
267
275
|
case 1:
|
|
268
|
-
_c.
|
|
269
|
-
|
|
276
|
+
_c.trys.push([1, 3, , 4]);
|
|
277
|
+
return [4 /*yield*/, this._methodManager.callMethod('insertErrorLog', "No Method in worker server handleIncomingTask: ".concat(method, " - ").concat(JSON.stringify(data, null, 2)))];
|
|
270
278
|
case 2:
|
|
279
|
+
_c.sent();
|
|
280
|
+
return [3 /*break*/, 4];
|
|
281
|
+
case 3:
|
|
282
|
+
error_1 = _c.sent();
|
|
283
|
+
console.error(new Date(), 'insertErrorLog failed in worker-server', error_1);
|
|
284
|
+
return [3 /*break*/, 4];
|
|
285
|
+
case 4:
|
|
271
286
|
// console.log('Invalid task message received', data);
|
|
272
287
|
this.logWorkerEvent('workerTaskInvalid', {
|
|
273
288
|
taskId: taskId,
|
|
@@ -286,7 +301,7 @@ var WorkerServerManager = /** @class */ (function () {
|
|
|
286
301
|
result: 'Invalid task'
|
|
287
302
|
});
|
|
288
303
|
return [2 /*return*/];
|
|
289
|
-
case
|
|
304
|
+
case 5:
|
|
290
305
|
timedOut = false;
|
|
291
306
|
timeoutHandle = setTimeout(function () { return __awaiter(_this, void 0, void 0, function () {
|
|
292
307
|
var timeoutAt, runMs, totalMs;
|
|
@@ -323,9 +338,9 @@ var WorkerServerManager = /** @class */ (function () {
|
|
|
323
338
|
}
|
|
324
339
|
});
|
|
325
340
|
}); }, ((_b = this._methodManager.getMethod(method)) === null || _b === void 0 ? void 0 : _b.timeoutOverride) || (1000 * 60 * 2));
|
|
326
|
-
_c.label =
|
|
327
|
-
case
|
|
328
|
-
_c.trys.push([
|
|
341
|
+
_c.label = 6;
|
|
342
|
+
case 6:
|
|
343
|
+
_c.trys.push([6, 8, , 10]);
|
|
329
344
|
managerThis = Object.assign({}, this._methodManager, method_manager_1.MethodManager.prototype, {
|
|
330
345
|
id_user: (userContext === null || userContext === void 0 ? void 0 : userContext.id_user) || '',
|
|
331
346
|
user: (userContext === null || userContext === void 0 ? void 0 : userContext.user) || '',
|
|
@@ -335,7 +350,7 @@ var WorkerServerManager = /** @class */ (function () {
|
|
|
335
350
|
console.log(new Date(), 'Running method', method);
|
|
336
351
|
}
|
|
337
352
|
return [4 /*yield*/, (_a = this._methodManager.callMethod).call.apply(_a, __spreadArray([managerThis, method], __read(params), false))];
|
|
338
|
-
case
|
|
353
|
+
case 7:
|
|
339
354
|
result = _c.sent();
|
|
340
355
|
packedResult = null;
|
|
341
356
|
meta = null;
|
|
@@ -390,8 +405,8 @@ var WorkerServerManager = /** @class */ (function () {
|
|
|
390
405
|
if (this._methodManager.getEnableDebug()) {
|
|
391
406
|
console.log(new Date(), 'Done with Task', JSON.stringify(this._runningTasks, null, 2));
|
|
392
407
|
}
|
|
393
|
-
return [3 /*break*/,
|
|
394
|
-
case
|
|
408
|
+
return [3 /*break*/, 10];
|
|
409
|
+
case 8:
|
|
395
410
|
err_1 = _c.sent();
|
|
396
411
|
if (!timedOut) {
|
|
397
412
|
clearTimeout(timeoutHandle);
|
|
@@ -438,11 +453,11 @@ var WorkerServerManager = /** @class */ (function () {
|
|
|
438
453
|
});
|
|
439
454
|
console.error(new Date(), "Error in Method: ".concat(method, " - ").concat(JSON.stringify(data, null, 2)));
|
|
440
455
|
return [4 /*yield*/, this._methodManager.callMethod('insertErrorLog', "Error in Method: ".concat(method, " - ").concat(JSON.stringify(data, null, 2)))];
|
|
441
|
-
case
|
|
456
|
+
case 9:
|
|
442
457
|
_c.sent();
|
|
443
458
|
this._runningTasks = this._runningTasks.filter(function (a) { return a !== taskId; });
|
|
444
|
-
return [3 /*break*/,
|
|
445
|
-
case
|
|
459
|
+
return [3 /*break*/, 10];
|
|
460
|
+
case 10: return [2 /*return*/];
|
|
446
461
|
}
|
|
447
462
|
});
|
|
448
463
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/managers/worker-server.manager.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,qCAAwC;AACxC,8BAAgC;AAEhC,yCAAoD;AACpD,mDAAiD;AAEjD;IAMI;QAHQ,kBAAa,GAAG,EAAE,CAAC;QACtB,wBAAmB,GAAG,GAAG,CAAC;IAEhB,CAAC;IAET,0BAAM,GAAb,UAAc,aAA4B,EAAE,YAAY;QACpD,IAAM,mBAAmB,GAAG,IAAI,mBAAmB,EAAE,CAAC;QACtD,mBAAmB,CAAC,UAAU,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;QAC5D,OAAO,mBAAmB,CAAC;IAC/B,CAAC;IAEM,wCAAU,GAAjB,UAAkB,aAA4B,EAAE,YAAY;QACxD,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;QACpC,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC;QAClC,IAAI,CAAC,mBAAmB,EAAE,CAAC;IAC/B,CAAC;IAEO,iDAAmB,GAA3B;QAAA,iBAqKC;QApKG,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,qEAAqE,CAAC,CAAC;QAE/F,IAAM,WAAW,GAAG,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,CAAC;QAC/E,IAAM,cAAc,GAAG,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAC,CAAC;QACvF,IAAI,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,GAAG,yBAAyB,GAAG,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,GAAG,eAAe,GAAG,WAAW,GAAG,kBAAkB,GAAG,cAAc,CAAC;QACpL,IAAM,EAAE,GAAG,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC;QAEhC,IAAI,QAAQ,GAAG,IAAI,IAAI,EAAE,CAAC;QAC1B,IAAI,QAAQ,GAAG,IAAI,CAAC;QACpB,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,IAAM,mBAAmB,GAAG,KAAK,CAAC;QAClC,IAAM,cAAc,GAAG,CAAC,CAAC;QACzB,IAAM,YAAY,GAAG,mBAAmB,GAAG,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC;QAChE,IAAI,WAAW,GAAG,IAAI,CAAC;QACvB,IAAI,MAAM,GAAG,KAAK,CAAC;QAEnB,wCAAwC;QACxC,WAAW,GAAG,UAAU,CAAC;YACrB,IAAI,CAAC,MAAM,EAAE,CAAC;gBACV,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,EAAE,2CAA2C,CAAC,CAAC;gBACvE,EAAE,CAAC,SAAS,EAAE,CAAC,CAAC,kCAAkC;YACtD,CAAC;QACL,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,qBAAqB;QAEhC,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE;YACV,MAAM,GAAG,IAAI,CAAC;YACd,YAAY,CAAC,WAAW,CAAC,CAAC;YAE1B,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,oCAAoC,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;YACvH,KAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;YAEpC,QAAQ,GAAG,WAAW,CAAC;gBACnB,IAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACvB,IAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC/C,IAAM,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC;gBACvD,IAAI,SAAS,GAAG,YAAY,IAAI,WAAW,GAAG,cAAc,EAAE,CAAC;oBAC3D,EAAE,CAAC,KAAK,EAAE,CAAC;oBACX,OAAO;gBACX,CAAC;gBACD,WAAW,IAAI,CAAC,CAAC;gBACjB,KAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;YACxC,CAAC,EAAE,mBAAmB,CAAC,CAAC;QAC5B,CAAC,CAAC,CAAC;QAET,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,UAAO,OAA0B;;;;;wBACjD,QAAQ,GAAG,IAAI,IAAI,EAAE,CAAC;6BAClB,CAAA,OAAO,OAAO,KAAK,QAAQ,CAAA,EAA3B,wBAA2B;wBAC9B,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;4BAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC;wBAClD,CAAC;6BAEG,CAAA,OAAO,KAAK,MAAM,CAAA,EAAlB,wBAAkB;wBACrB,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;4BAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,yBAAyB,CAAC,CAAC;wBACpD,CAAC;wBAED,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;;;6BAE5B,CAAA,OAAO,KAAK,MAAM,CAAA,EAAlB,wBAAkB;wBAC1B,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;4BAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,WAAW,CAAC,CAAC;wBACtC,CAAC;wBACD,WAAW,GAAG,CAAC,CAAC;;;wBAIhB,IAAI,CAAC;4BACJ,KAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,oBAAW,CAAC,CAAC;wBACxC,CAAC;wBACD,OAAO,CAAC,EAAE,CAAC;4BACV,OAAO,CAAC,KAAK,CAAC,oBAAoB,EAAE,CAAC,CAAC,CAAC;4BACvC,sBAAO;wBACR,CAAC;6BAEG,CAAA,KAAG,CAAC,IAAI,KAAK,MAAM,CAAA,EAAnB,wBAAmB;wBACtB,qBAAM,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,KAAG,CAAC,EAAA;;wBAAtC,SAAsC,CAAC;;4BAIzC,sBAAO;;wBAKR,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;4BAC9B,aAAa,GAAG,OAAO,CAAC;wBACzB,CAAC;6BACI,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;4BAC3B,MAAM,GAAG,OAA+C,CAAC;4BAC/D,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;wBACvC,CAAC;6BACI,IAAI,OAAO,YAAY,WAAW,EAAE,CAAC;4BACzC,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;wBACtC,CAAC;6BACI,IAAI,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;4BAChC,IAAI,GAAG,OAAiC,CAAC;4BAC/C,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;wBAC5E,CAAC;6BACI,CAAC;4BACL,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,OAAc,CAAC,CAAC;wBAC7C,CAAC;wBAED,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;4BAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,uBAAuB,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;wBACxE,CAAC;wBAED,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;4BAC5B,SAAS,GAAG,aAAa,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;4BAE/C,IAAI,SAAS,KAAK,MAAM,EAAE,CAAC;gCAC1B,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;oCAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,kCAAkC,CAAC,CAAC;gCAC7D,CAAC;gCAED,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;gCACpC,sBAAO;4BACR,CAAC;iCACI,IAAI,SAAS,KAAK,MAAM,EAAE,CAAC;gCAC/B,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;oCAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,oBAAoB,CAAC,CAAC;gCAC/C,CAAC;gCACD,WAAW,GAAG,CAAC,CAAC;gCAChB,sBAAO;4BACR,CAAC;wBACF,CAAC;wBAID,IAAI,CAAC;4BACJ,GAAG,GAAG,IAAA,iBAAM,EAAC,aAAa,CAAC,CAAC;wBAC7B,CAAC;wBACD,OAAO,CAAC,EAAE,CAAC;4BACV,OAAO,CAAC,KAAK,CAAC,2BAA2B,EAAE,CAAC,CAAC,CAAC;4BAC9C,sBAAO;wBACR,CAAC;6BAEG,CAAA,GAAG,CAAC,IAAI,KAAK,MAAM,CAAA,EAAnB,wBAAmB;wBACtB,qBAAM,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,GAAG,CAAC,EAAA;;wBAAtC,SAAsC,CAAC;;;;;aAExC,CAAC,CAAC;QAEG,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE;YACX,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,sDAAsD,CAAC,CAAC;YAChF,UAAU,CAAC;gBACP,KAAI,CAAC,mBAAmB,EAAE,CAAC;YAC/B,CAAC,EAAE,IAAI,CAAC,CAAC;YAET,IAAI,QAAQ,EAAE,CAAC;gBACX,aAAa,CAAC,QAAQ,CAAC,CAAC;YAC5B,CAAC;YACD,YAAY,CAAC,WAAW,CAAC,CAAC;QAC9B,CAAC,CAAC,CAAC;QAET,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,UAAC,GAAG;YAClB,IAAM,OAAO,GAAG,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,OAAO,KAAI,EAAE,CAAC;YACnC,IAAM,cAAc,GAAG,OAAO,CAAC,QAAQ,CAAC,iCAAiC,CAAC,CAAC;YAC3E,IAAI,CAAC,cAAc,EAAE,CAAC;gBACrB,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,EAAE,kBAAkB,EAAE,GAAG,CAAC,CAAC;YACpD,CAAC;iBACI,IAAI,KAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;gBAC/C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,2CAA2C,EAAE,OAAO,CAAC,CAAC;YAC/E,CAAC;YACD,EAAE,CAAC,KAAK,EAAE,CAAC;QACZ,CAAC,CAAC,CAAC;IACD,CAAC;IAEa,gDAAkB,GAAhC,UAAiC,EAAuB,EAAE,IAAiB;;;;;;;;;wBACjE,MAAM,GAA6C,IAAI,OAAjD,EAAE,SAAS,GAAkC,IAAI,UAAtC,EAAE,MAAM,GAA0B,IAAI,OAA9B,EAAE,MAAM,GAAkB,IAAI,OAAtB,EAAE,WAAW,GAAK,IAAI,YAAT,CAAU;wBAC9D,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;wBACxB,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;wBACzB,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;wBACjC,WAAW,GAAG,QAAQ,IAAI,YAAY,CAAC,CAAC,CAAC,IAAA,cAAK,EAAC,YAAY,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAA,cAAK,EAAC,UAAU,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;wBAC3H,eAAe,GAAG,YAAY,CAAC,CAAC,CAAC,IAAA,cAAK,EAAC,UAAU,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;wBACzE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;6BAC5B,CAAA,CAAC,MAAM,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA,EAA5D,wBAA4D;6BACxD,CAAC,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,MAAM,CAAC,EAAtC,wBAAsC;wBACtC,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,EAAE,kEAAkE,EAAE,MAAM,CAAC,CAAC;wBACtG,qBAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,gBAAgB,EAAE,yDAAkD,MAAM,gBAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAE,CAAC,EAAA;;wBAArJ,SAAqJ,CAAC;;;wBAG1J,sDAAsD;wBAC/D,IAAI,CAAC,cAAc,CAAC,mBAAmB,EAAE;4BACxC,MAAM,QAAA;4BACN,MAAM,QAAA;4BACN,SAAS,WAAA;4BACT,WAAW,aAAA;4BACX,eAAe,iBAAA;4BACf,WAAW,aAAA;4BACX,MAAM,QAAA;yBACN,CAAC,CAAC;wBACM,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE;4BACxB,IAAI,EAAE,cAAc;4BACpB,MAAM,QAAA;4BACN,SAAS,WAAA;4BACT,KAAK,EAAE,IAAI;4BACX,MAAM,EAAE,cAAc;yBACzB,CAAC,CAAC;wBAEH,sBAAO;;wBAGP,QAAQ,GAAG,KAAK,CAAC;wBACjB,aAAa,GAAG,UAAU,CAAC;;;;;wCAC3B,QAAQ,GAAG,IAAI,CAAC;wCACnB,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;wCACvB,KAAK,GAAG,IAAA,cAAK,EAAC,SAAS,GAAG,UAAU,CAAC,CAAC;wCACtC,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAA,cAAK,EAAC,SAAS,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;wCACrD,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,EAAE,2BAA2B,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;wCAE3F,IAAI,CAAC,cAAc,CAAC,mBAAmB,EAAE;4CACxC,MAAM,QAAA;4CACN,MAAM,QAAA;4CACN,SAAS,WAAA;4CACT,KAAK,OAAA;4CACL,WAAW,aAAA;4CACX,eAAe,iBAAA;4CACf,OAAO,SAAA;4CACP,WAAW,aAAA;4CACX,MAAM,QAAA;yCACN,CAAC,CAAC;wCAEM,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE;4CACxB,IAAI,EAAE,cAAc;4CACpB,MAAM,QAAA;4CACN,SAAS,WAAA;4CACT,KAAK,EAAE,IAAI;4CACX,MAAM,EAAE,gBAAgB;yCAC3B,CAAC,CAAC;wCAEH,qBAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,gBAAgB,EAAE,6BAAsB,MAAM,gBAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAE,CAAC,EAAA;;wCAAzH,SAAyH,CAAC;;;;6BAC7H,EAAE,CAAA,MAAA,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,MAAM,CAAC,0CAAE,eAAe,KAAI,CAAC,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;;;;wBAGtE,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,cAAc,EAAE,8BAAa,CAAC,SAAS,EAAE;4BAC9E,OAAO,EAAE,CAAA,WAAW,aAAX,WAAW,uBAAX,WAAW,CAAE,OAAO,KAAI,EAAE;4BACnC,IAAI,EAAE,CAAA,WAAW,aAAX,WAAW,uBAAX,WAAW,CAAE,IAAI,KAAI,EAAE;4BAC7B,KAAK,EAAE,CAAA,WAAW,aAAX,WAAW,uBAAX,WAAW,CAAE,KAAK,KAAI,EAAE;yBAClC,CAAC,CAAC;wBAEH,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;4BACvC,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,gBAAgB,EAAE,MAAM,CAAC,CAAC;wBACtD,CAAC;wBAEG,qBAAM,CAAA,KAAA,IAAI,CAAC,cAAc,CAAC,UAAU,CAAA,CAAC,IAAI,0BAAC,WAAW,EAAE,MAAM,UAAK,MAAM,YAAC;;wBAAlF,MAAM,GAAG,SAAyE;wBAClF,YAAY,GAAwB,IAAI,CAAC;wBACzC,IAAI,GAAwB,IAAI,CAAC;wBACjC,kBAAkB,GAAQ,SAAS,CAAC;wBACpC,eAAe,GAAwB,IAAI,CAAC;wBAC1C,qBAAqB,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,MAAM,CAAC,qBAAqB,CAAC,CAAC;wBAEzE,IAAI,CAAC;4BACJ,IAAI,qBAAqB,EAAE,CAAC;gCAC3B,kBAAkB,GAAG,MAAM,CAAC,OAAO,CAAC;gCACpC,eAAe,GAAG;oCACjB,IAAI,EAAE,aAAa;oCACnB,WAAW,EAAE,MAAM,CAAC,WAAW;oCAC/B,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;oCACzC,YAAY,EAAE,MAAM,CAAC,YAAY;oCACjC,QAAQ,EAAE,MAAM,CAAC,QAAQ;iCACzB,CAAC;4BACH,CAAC;4BAED,IAAI,qBAAqB,EAAE,CAAC;gCAC3B,IAAI,GAAG,eAAe,CAAC;gCACvB,YAAY,GAAW,IAAA,eAAI,EAAC,kBAAkB,CAAC,CAAC;gCAChD,MAAM,GAAG,IAAI,CAAC;4BACf,CAAC;iCACI,CAAC;gCACL,YAAY,GAAW,IAAA,eAAI,EAAC,MAAM,CAAC,CAAC;4BACrC,CAAC;wBACF,CAAC;wBACD,OAAO,OAAO,EAAE,CAAC;4BAChB,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,EAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC;4BACxD,IAAI,qBAAqB,EAAE,CAAC;gCAC3B,IAAI,GAAG,eAAe,CAAC;gCACvB,MAAM,GAAG,kBAAkB,CAAC;4BAC7B,CAAC;wBACF,CAAC;wBAGD,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;4BAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,iBAAiB,EAAE,MAAM,CAAC,CAAC;wBACpD,CAAC;wBAED,IAAI,CAAC,QAAQ,EAAE,CAAC;4BACf,YAAY,CAAC,aAAa,CAAC,CAAC;4BACtB,OAAO,GAAiB;gCAC7B,IAAI,EAAE,cAAc;gCACpB,MAAM,QAAA;gCACN,SAAS,WAAA;gCACT,KAAK,EAAE,KAAK;gCACZ,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM;gCACpC,YAAY,cAAA;gCACZ,QAAQ,EAAE,SAAS;gCACnB,IAAI,MAAA;6BACJ,CAAC;4BAEF,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;wBACtC,CAAC;wBAEQ,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,UAAA,CAAC,IAAI,OAAA,CAAC,KAAK,MAAM,EAAZ,CAAY,CAAC,CAAC;wBAElE,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;4BACvC,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;wBAC3F,CAAC;;;;wBAGD,IAAI,CAAC,QAAQ,EAAE,CAAC;4BACZ,YAAY,CAAC,aAAa,CAAC,CAAC;4BAClC,YAAU,IAAI,CAAC,GAAG,EAAE,CAAC;4BACrB,UAAQ,IAAA,cAAK,EAAC,SAAO,GAAG,UAAU,CAAC,CAAC;4BACpC,YAAU,QAAQ,CAAC,CAAC,CAAC,IAAA,cAAK,EAAC,SAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;4BAChD,OAAO,CAAC,KAAK,CAAC,qBAAqB,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,KAAG,CAAC,CAAC;4BACrE,KAAG,CAAC,OAAO,GAAG,qBAAqB,GAAG,MAAM,GAAG,UAAU,GAAG,MAAM,GAAG,KAAK,GAAG,KAAG,CAAC,OAAO,CAAC;4BACrG,IAAI,CAAC,cAAc,CAAC,iBAAiB,EAAE;gCACtC,MAAM,QAAA;gCACN,MAAM,QAAA;gCACN,SAAS,WAAA;gCACT,KAAK,SAAA;gCACL,WAAW,aAAA;gCACX,eAAe,iBAAA;gCACf,OAAO,WAAA;gCACP,WAAW,aAAA;gCACX,MAAM,QAAA;gCACN,KAAK,EAAE,KAAG;6BACV,CAAC,CAAC;4BACS,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE;gCACxB,IAAI,EAAE,cAAc;gCACpB,MAAM,QAAA;gCACN,SAAS,WAAA;gCACT,KAAK,EAAE,IAAI;gCACX,MAAM,EAAE,KAAG,IAAI,eAAe;6BACjC,CAAC,CAAC;4BACH,MAAM,KAAG,CAAC;wBACd,CAAC;wBAEJ,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;wBACrB,KAAK,GAAG,IAAA,cAAK,EAAC,OAAO,GAAG,UAAU,CAAC,CAAC;wBACpC,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAA,cAAK,EAAC,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;wBAC5D,IAAI,CAAC,cAAc,CAAC,iBAAiB,EAAE;4BACtC,MAAM,QAAA;4BACN,MAAM,QAAA;4BACN,SAAS,WAAA;4BACT,KAAK,OAAA;4BACL,WAAW,aAAA;4BACX,eAAe,iBAAA;4BACf,OAAO,SAAA;4BACP,WAAW,aAAA;4BACX,MAAM,QAAA;4BACN,KAAK,EAAE,KAAG;yBACV,CAAC,CAAC;wBAEM,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,EAAG,2BAAoB,MAAM,gBAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAE,CAAC,CAAC;wBAE5F,qBAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,gBAAgB,EAAE,2BAAoB,MAAM,gBAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAE,CAAC,EAAA;;wBAAvH,SAAuH,CAAC;wBAExH,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,UAAA,CAAC,IAAI,OAAA,CAAC,KAAK,MAAM,EAAZ,CAAY,CAAC,CAAC;;;;;;KAEzE;IAEI,4CAAc,GAAtB,UAAuB,KAAa,EAAE,OAA4B;QACjE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,YAC7B,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAC5B,KAAK,OAAA,IACF,OAAO,EACT,CAAC,CAAC;IACL,CAAC;IAEO,2CAAa,GAArB,UAAsB,KAAU;QAC/B,IAAI,CAAC;YACJ,IAAM,MAAI,GAAG,IAAI,OAAO,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,UAAC,GAAG,EAAE,GAAG;gBACrC,IAAI,GAAG,KAAK,WAAW,EAAE,CAAC;oBACzB,OAAO,SAAS,CAAC;gBAClB,CAAC;gBACD,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;oBAC7B,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;gBACvB,CAAC;gBACD,IAAI,GAAG,YAAY,KAAK,EAAE,CAAC;oBAC1B,OAAO;wBACN,IAAI,EAAE,GAAG,CAAC,IAAI;wBACd,OAAO,EAAE,GAAG,CAAC,OAAO;wBACpB,KAAK,EAAE,GAAG,CAAC,KAAK;qBAChB,CAAC;gBACH,CAAC;gBACD,IAAI,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;oBACpC,IAAI,MAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;wBACnB,OAAO,YAAY,CAAC;oBACrB,CAAC;oBACD,MAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBACf,CAAC;gBACD,OAAO,GAAG,CAAC;YACZ,CAAC,CAAC,CAAC;QACJ,CAAC;QACD,OAAO,GAAG,EAAE,CAAC;YACZ,OAAO,IAAI,CAAC,SAAS,CAAC;gBACrB,KAAK,EAAE,qBAAqB;gBAC5B,OAAO,EAAE,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,OAAO,KAAI,MAAM,CAAC,GAAG,CAAC;aACpC,CAAC,CAAC;QACJ,CAAC;IACF,CAAC;IAEM,gDAAkB,GAAzB,UAA0B,EAAE,EAAE,OAA8B;QAC3D,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;YAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,SAAS,EAAE,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;QACxF,CAAC;QAED,IAAI,EAAE,IAAI,EAAE,CAAC,UAAU,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;YAC5C,IAAI,CAAC;gBACJ,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;oBACjC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAClB,CAAC;qBACI,CAAC;oBACL,EAAE,CAAC,IAAI,CAAC,IAAA,eAAI,EAAC,OAAO,CAAC,CAAC,CAAC;gBACxB,CAAC;YACF,CAAC;YACD,OAAO,GAAG,EAAE,CAAC;gBACZ,OAAO,CAAC,KAAK,CAAC,iCAAiC,EAAE,GAAG,CAAC,CAAC;YACvD,CAAC;QACF,CAAC;aACI,IAAI,EAAE,EAAE,CAAC;YACb,EAAE,CAAC,KAAK,EAAE,CAAC;QACZ,CAAC;IACF,CAAC;IACF,0BAAC;AAAD,CA7bA,AA6bC,IAAA;AA7bY,kDAAmB","file":"worker-server.manager.js","sourcesContent":["import { pack, unpack } from 'msgpackr';\nimport * as WebSocket from 'ws';\nimport { TaskPayload, TaskResponse } from '../models/server-message.model';\nimport { dateReviver, round } from '../util/common';\nimport { MethodManager } from './method.manager';\n\nexport class WorkerServerManager {\n private _methodManager: MethodManager;\n private _serverConfig;\n private _runningTasks = [];\n\tprivate _taskLogThresholdMs = 200;\n\n constructor() {}\n \n static create(methodManager: MethodManager, serverConfig) {\n const workerServerManager = new WorkerServerManager();\n workerServerManager.initialize(methodManager, serverConfig);\n return workerServerManager;\n }\n\n public initialize(methodManager: MethodManager, serverConfig) {\n this._methodManager = methodManager;\n this._serverConfig = serverConfig;\n this.startWorkerInstance();\n }\n\n private startWorkerInstance() {\n console.log(new Date(), 'Worker instance started, connecting to main server via WebSocket...');\n \n const workerIndex = encodeURIComponent(String(process.env.WORKER_INDEX || ''));\n const workerInstance = encodeURIComponent(String(process.env.NODE_APP_INSTANCE || ''));\n let wsUrl = this._serverConfig['SERVER_URL'] + '/websocket?workerToken=' + this._serverConfig['WORKER_TOKEN'] + '&workerIndex=' + workerIndex + '&workerInstance=' + workerInstance;\n const ws = new WebSocket(wsUrl);\n \n let lastComm = new Date();\n let interval = null;\n let missedPongs = 0;\n const heartbeatIntervalMs = 15000;\n const maxMissedPongs = 2;\n const maxSilenceMs = heartbeatIntervalMs * (maxMissedPongs + 1);\n let openTimeout = null;\n let opened = false;\n \n // Set timeout if the socket never opens\n openTimeout = setTimeout(() => {\n if (!opened) {\n console.error(new Date(), 'WebSocket connection timeout. Retrying...');\n ws.terminate(); // force close if still connecting\n }\n }, 10000); // 10 seconds timeout\n \n ws.on('open', () => {\n opened = true;\n clearTimeout(openTimeout);\n \n console.log(new Date(), 'Connected to main server as worker', process.env.WORKER_INDEX, process.env.NODE_APP_INSTANCE);\n this.sendWorkerResponse(ws, 'ping');\n \n interval = setInterval(() => {\n const now = Date.now();\n const last = lastComm ? lastComm.getTime() : 0;\n const silenceMs = last ? now - last : maxSilenceMs + 1;\n if (silenceMs > maxSilenceMs || missedPongs > maxMissedPongs) {\n ws.close();\n return;\n }\n missedPongs += 1;\n this.sendWorkerResponse(ws, 'ping');\n }, heartbeatIntervalMs);\n });\n \n\t\tws.on('message', async (rawData: WebSocket.RawData) => {\n\t\t\tlastComm = new Date();\n\t\t\tif (typeof rawData === 'string') {\n\t\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\t\tconsole.log(new Date(), 'Message Recv', rawData);\n\t\t\t\t}\n\n\t\t\t\tif (rawData === 'ping') {\n\t\t\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\t\t\tconsole.log(new Date(), 'Recv Ping, Sending Pong');\n\t\t\t\t\t}\n\t\t\t\t\t\n\t\t\t\t\tthis.sendWorkerResponse(ws, 'pong');\n\t\t\t\t}\n\t\t\t\telse if (rawData === 'pong') {\n\t\t\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\t\t\tconsole.log(new Date(), 'Recv Pong');\n\t\t\t\t\t}\n\t\t\t\t\tmissedPongs = 0;\n\t\t\t\t}\n\t\t\t\telse {\n\t\t\t\t\tlet msg: any;\n\t\t\t\t\ttry {\n\t\t\t\t\t\tmsg = JSON.parse(rawData, dateReviver);\n\t\t\t\t\t}\n\t\t\t\t\tcatch (e) {\n\t\t\t\t\t\tconsole.error('Worker parse error', e);\n\t\t\t\t\t\treturn;\n\t\t\t\t\t}\n\t\n\t\t\t\t\tif (msg.type === 'task') {\n\t\t\t\t\t\tawait this.handleIncomingTask(ws, msg);\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\treturn;\n\t\t\t}\n\n\t\t\tlet messageBuffer: Buffer;\n\n\t\t\tif (Buffer.isBuffer(rawData)) {\n\t\t\t\tmessageBuffer = rawData;\n\t\t\t}\n\t\t\telse if (Array.isArray(rawData)) {\n\t\t\t\tconst chunks = rawData as unknown as ReadonlyArray<Uint8Array>;\n\t\t\t\tmessageBuffer = Buffer.concat(chunks);\n\t\t\t}\n\t\t\telse if (rawData instanceof ArrayBuffer) {\n\t\t\t\tmessageBuffer = Buffer.from(rawData);\n\t\t\t}\n\t\t\telse if (ArrayBuffer.isView(rawData)) {\n\t\t\t\tconst view = rawData as NodeJS.ArrayBufferView;\n\t\t\t\tmessageBuffer = Buffer.from(view.buffer, view.byteOffset, view.byteLength);\n\t\t\t}\n\t\t\telse {\n\t\t\t\tmessageBuffer = Buffer.from(rawData as any);\n\t\t\t}\n\n\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\tconsole.log(new Date(), 'Message Recv (binary)', messageBuffer.length);\n\t\t\t}\n\n\t\t\tif (messageBuffer.length === 4) {\n\t\t\t\tlet heartbeat = messageBuffer.toString('utf8');\n\n\t\t\t\tif (heartbeat === 'ping') {\n\t\t\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\t\t\tconsole.log(new Date(), 'Recv Ping (binary), Sending Pong');\n\t\t\t\t\t}\n\n\t\t\t\t\tthis.sendWorkerResponse(ws, 'pong');\n\t\t\t\t\treturn;\n\t\t\t\t}\n\t\t\t\telse if (heartbeat === 'pong') {\n\t\t\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\t\t\tconsole.log(new Date(), 'Recv Pong (binary)');\n\t\t\t\t\t}\n\t\t\t\t\tmissedPongs = 0;\n\t\t\t\t\treturn;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tlet msg: TaskPayload;\n\n\t\t\ttry {\n\t\t\t\tmsg = unpack(messageBuffer);\n\t\t\t}\n\t\t\tcatch (e) {\n\t\t\t\tconsole.error('Worker binary parse error', e);\n\t\t\t\treturn;\n\t\t\t}\n\n\t\t\tif (msg.type === 'task') {\n\t\t\t\tawait this.handleIncomingTask(ws, msg);\n\t\t\t}\n\t\t});\n \n ws.on('close', () => {\n console.log(new Date(), 'Disconnected from main server. Reconnecting in 1s...');\n setTimeout(() => {\n this.startWorkerInstance();\n }, 1000);\n \n if (interval) {\n clearInterval(interval);\n }\n clearTimeout(openTimeout);\n });\n \n\t\tws.on('error', (err) => {\n\t\t\tconst message = err?.message || '';\n\t\t\tconst isRestartNoise = message.includes('Unexpected server response: 502');\n\t\t\tif (!isRestartNoise) {\n\t\t\t\tconsole.error(new Date(), 'Worker WS error:', err);\n\t\t\t}\n\t\t\telse if (this._methodManager.getEnableDebug()) {\n\t\t\t\tconsole.log(new Date(), 'Worker WS reconnecting after main restart', message);\n\t\t\t}\n\t\t\tws.close();\n\t\t});\n }\n\n private async handleIncomingTask(ws: WebSocket.WebSocket, data: TaskPayload) {\n let { taskId, messageId, method, params, userContext } = data;\n\t\tconst receivedAt = Date.now();\n\t\tconst queuedAt = data.queuedAt;\n\t\tconst dispatchedAt = data.dispatchedAt;\n\t\tconst queueWaitMs = queuedAt && dispatchedAt ? round(dispatchedAt - queuedAt) : (queuedAt ? round(receivedAt - queuedAt) : null);\n\t\tconst dispatchDelayMs = dispatchedAt ? round(receivedAt - dispatchedAt) : null;\n this._runningTasks.push(taskId);\n if (!taskId || !method || !this._methodManager.getMethod(method)) {\n if (!this._methodManager.getMethod(method)) {\n console.error(new Date(), 'No method in method manager for handleIncomingTask worker server', method);\n await this._methodManager.callMethod('insertErrorLog', `No Method in worker server handleIncomingTask: ${method} - ${JSON.stringify(data, null, 2)}`);\n }\n\n // console.log('Invalid task message received', data);\n\t\t\tthis.logWorkerEvent('workerTaskInvalid', {\n\t\t\t\ttaskId,\n\t\t\t\tmethod,\n\t\t\t\tmessageId,\n\t\t\t\tqueueWaitMs,\n\t\t\t\tdispatchDelayMs,\n\t\t\t\tuserContext,\n\t\t\t\tparams\n\t\t\t});\n this.sendWorkerResponse(ws, {\n type: 'taskComplete',\n taskId,\n messageId,\n error: true,\n result: 'Invalid task'\n });\n\n return;\n }\n\n let timedOut = false;\n let timeoutHandle = setTimeout(async () => {\n timedOut = true;\n\t\t\tconst timeoutAt = Date.now();\n\t\t\tconst runMs = round(timeoutAt - receivedAt);\n\t\t\tconst totalMs = queuedAt ? round(timeoutAt - queuedAt) : null;\n console.error(new Date(), 'Worker timed out on task:', taskId, 'Method:', method);\n\n\t\t\tthis.logWorkerEvent('workerTaskTimeout', {\n\t\t\t\ttaskId,\n\t\t\t\tmethod,\n\t\t\t\tmessageId,\n\t\t\t\trunMs,\n\t\t\t\tqueueWaitMs,\n\t\t\t\tdispatchDelayMs,\n\t\t\t\ttotalMs,\n\t\t\t\tuserContext,\n\t\t\t\tparams\n\t\t\t});\n\n this.sendWorkerResponse(ws, {\n type: 'taskComplete',\n taskId,\n messageId,\n error: true,\n result: 'Task timed out'\n });\n\n await this._methodManager.callMethod('insertErrorLog', `Timeout in Method: ${method} - ${JSON.stringify(data, null, 2)}`);\n }, this._methodManager.getMethod(method)?.timeoutOverride || (1000 * 60 * 2));\n\n try {\n let managerThis = Object.assign({}, this._methodManager, MethodManager.prototype, {\n id_user: userContext?.id_user || '',\n user: userContext?.user || '',\n id_ws: userContext?.id_ws || ''\n });\n\n if (this._methodManager.getEnableDebug()) {\n console.log(new Date(), 'Running method', method);\n }\n\n\t\t\tlet result = await this._methodManager.callMethod.call(managerThis, method, ...params);\n\t\t\tlet packedResult: Uint8Array | Buffer = null;\n\t\t\tlet meta: Record<string, any> = null;\n\t\t\tlet publicationPayload: any = undefined;\n\t\t\tlet publicationMeta: Record<string, any> = null;\n\t\t\tconst isPublicationResponse = !!(result && result.__publicationResponse);\n\n\t\t\ttry {\n\t\t\t\tif (isPublicationResponse) {\n\t\t\t\t\tpublicationPayload = result.payload;\n\t\t\t\t\tpublicationMeta = {\n\t\t\t\t\t\ttype: 'publication',\n\t\t\t\t\t\tpublication: result.publication,\n\t\t\t\t\t\tsubscriptionData: result.subscriptionData,\n\t\t\t\t\t\tuserSpecific: result.userSpecific,\n\t\t\t\t\t\tsnapshot: result.snapshot\n\t\t\t\t\t};\n\t\t\t\t}\n\n\t\t\t\tif (isPublicationResponse) {\n\t\t\t\t\tmeta = publicationMeta;\n\t\t\t\t\tpackedResult = <Buffer>pack(publicationPayload);\n\t\t\t\t\tresult = null;\n\t\t\t\t}\n\t\t\t\telse {\n\t\t\t\t\tpackedResult = <Buffer>pack(result);\n\t\t\t\t}\n\t\t\t}\n\t\t\tcatch (packErr) {\n\t\t\t\tconsole.error(new Date(), 'Worker pack error', packErr);\n\t\t\t\tif (isPublicationResponse) {\n\t\t\t\t\tmeta = publicationMeta;\n\t\t\t\t\tresult = publicationPayload;\n\t\t\t\t}\n\t\t\t}\n\n\n\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\tconsole.log(new Date(), 'Finished method', method);\n\t\t\t}\n\n\t\t\tif (!timedOut) {\n\t\t\t\tclearTimeout(timeoutHandle);\n\t\t\t\tconst payload: TaskResponse = {\n\t\t\t\t\ttype: 'taskComplete',\n\t\t\t\t\ttaskId,\n\t\t\t\t\tmessageId,\n\t\t\t\t\terror: false,\n\t\t\t\t\tresult: packedResult ? null : result,\n\t\t\t\t\tpackedResult,\n\t\t\t\t\tencoding: 'msgpack',\n\t\t\t\t\tmeta\n\t\t\t\t};\n\n\t\t\t\tthis.sendWorkerResponse(ws, payload);\n\t\t\t}\n\n this._runningTasks = this._runningTasks.filter(a => a !== taskId);\n \n if (this._methodManager.getEnableDebug()) {\n console.log(new Date(), 'Done with Task', JSON.stringify(this._runningTasks, null, 2));\n }\n }\n catch (err) {\n if (!timedOut) {\n clearTimeout(timeoutHandle);\n\t\t\t\tconst errorAt = Date.now();\n\t\t\t\tconst runMs = round(errorAt - receivedAt);\n\t\t\t\tconst totalMs = queuedAt ? round(errorAt - queuedAt) : null;\n console.error('Worker failed task:', taskId, 'Method:', method, err);\n err.message = 'Worker failed task:' + taskId + ' Method:' + method + ' - ' + err.message;\n\t\t\t\tthis.logWorkerEvent('workerTaskError', {\n\t\t\t\t\ttaskId,\n\t\t\t\t\tmethod,\n\t\t\t\t\tmessageId,\n\t\t\t\t\trunMs,\n\t\t\t\t\tqueueWaitMs,\n\t\t\t\t\tdispatchDelayMs,\n\t\t\t\t\ttotalMs,\n\t\t\t\t\tuserContext,\n\t\t\t\t\tparams,\n\t\t\t\t\terror: err\n\t\t\t\t});\n this.sendWorkerResponse(ws, {\n type: 'taskComplete',\n taskId,\n messageId,\n error: true,\n result: err || 'Unknown error'\n });\n throw err;\n }\n\n\t\t\tconst errorAt = Date.now();\n\t\t\tconst runMs = round(errorAt - receivedAt);\n\t\t\tconst totalMs = queuedAt ? round(errorAt - queuedAt) : null;\n\t\t\tthis.logWorkerEvent('workerTaskError', {\n\t\t\t\ttaskId,\n\t\t\t\tmethod,\n\t\t\t\tmessageId,\n\t\t\t\trunMs,\n\t\t\t\tqueueWaitMs,\n\t\t\t\tdispatchDelayMs,\n\t\t\t\ttotalMs,\n\t\t\t\tuserContext,\n\t\t\t\tparams,\n\t\t\t\terror: err\n\t\t\t});\n\n console.error(new Date(), `Error in Method: ${method} - ${JSON.stringify(data, null, 2)}`);\n \n await this._methodManager.callMethod('insertErrorLog', `Error in Method: ${method} - ${JSON.stringify(data, null, 2)}`);\n \n this._runningTasks = this._runningTasks.filter(a => a !== taskId);\n }\n }\n\n\tprivate logWorkerEvent(event: string, details: Record<string, any>) {\n\t\tconsole.log(this.safeStringify({\n\t\t\tts: new Date().toISOString(),\n\t\t\tevent,\n\t\t\t...details\n\t\t}));\n\t}\n\n\tprivate safeStringify(value: any): string {\n\t\ttry {\n\t\t\tconst seen = new WeakSet();\n\t\t\treturn JSON.stringify(value, (key, val) => {\n\t\t\t\tif (key === '__proto__') {\n\t\t\t\t\treturn undefined;\n\t\t\t\t}\n\t\t\t\tif (typeof val === 'bigint') {\n\t\t\t\t\treturn val.toString();\n\t\t\t\t}\n\t\t\t\tif (val instanceof Error) {\n\t\t\t\t\treturn {\n\t\t\t\t\t\tname: val.name,\n\t\t\t\t\t\tmessage: val.message,\n\t\t\t\t\t\tstack: val.stack\n\t\t\t\t\t};\n\t\t\t\t}\n\t\t\t\tif (val && typeof val === 'object') {\n\t\t\t\t\tif (seen.has(val)) {\n\t\t\t\t\t\treturn '[Circular]';\n\t\t\t\t\t}\n\t\t\t\t\tseen.add(val);\n\t\t\t\t}\n\t\t\t\treturn val;\n\t\t\t});\n\t\t}\n\t\tcatch (err) {\n\t\t\treturn JSON.stringify({\n\t\t\t\terror: 'safeStringifyFailed',\n\t\t\t\tmessage: err?.message || String(err)\n\t\t\t});\n\t\t}\n\t}\n\n\tpublic sendWorkerResponse(ws, payload: TaskResponse | string) {\n\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\tconsole.log(new Date(), 'Sending', typeof payload === 'string' ? payload : '[binary]');\n\t\t}\n\n\t\tif (ws && ws.readyState === WebSocket.OPEN) {\n\t\t\ttry {\n\t\t\t\tif (typeof payload === 'string') {\n\t\t\t\t\tws.send(payload);\n\t\t\t\t}\n\t\t\t\telse {\n\t\t\t\t\tws.send(pack(payload));\n\t\t\t\t}\n\t\t\t}\n\t\t\tcatch (err) {\n\t\t\t\tconsole.error('Failed to send worker response:', err);\n\t\t\t}\n\t\t}\n\t\telse if (ws) {\n\t\t\tws.close();\n\t\t}\n\t}\n}\n"]}
|
|
1
|
+
{"version":3,"sources":["../../src/managers/worker-server.manager.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,qCAAwC;AACxC,8BAAgC;AAEhC,yCAAoD;AACpD,mDAAiD;AAEjD;IAMI;QAHQ,kBAAa,GAAG,EAAE,CAAC;QACtB,wBAAmB,GAAG,GAAG,CAAC;IAEhB,CAAC;IAET,0BAAM,GAAb,UAAc,aAA4B,EAAE,YAAY;QACpD,IAAM,mBAAmB,GAAG,IAAI,mBAAmB,EAAE,CAAC;QACtD,mBAAmB,CAAC,UAAU,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;QAC5D,OAAO,mBAAmB,CAAC;IAC/B,CAAC;IAEM,wCAAU,GAAjB,UAAkB,aAA4B,EAAE,YAAY;QAA5D,iBASC;QARG,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;QACpC,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC;QACxC,IAAI,IAAI,CAAC,cAAc,IAAI,OAAO,IAAI,CAAC,cAAc,CAAC,OAAO,KAAK,UAAU,EAAE,CAAC;YAC9E,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,cAAM,OAAA,KAAI,CAAC,mBAAmB,EAAE,EAA1B,CAA0B,CAAC,CAAC;QAC/D,CAAC;aACI,CAAC;YACL,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC5B,CAAC;IACC,CAAC;IAEO,iDAAmB,GAA3B;QAAA,iBAqKC;QApKG,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,qEAAqE,CAAC,CAAC;QAE/F,IAAM,WAAW,GAAG,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,CAAC;QAC/E,IAAM,cAAc,GAAG,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAC,CAAC;QACvF,IAAI,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,GAAG,yBAAyB,GAAG,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,GAAG,eAAe,GAAG,WAAW,GAAG,kBAAkB,GAAG,cAAc,CAAC;QACpL,IAAM,EAAE,GAAG,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC;QAEhC,IAAI,QAAQ,GAAG,IAAI,IAAI,EAAE,CAAC;QAC1B,IAAI,QAAQ,GAAG,IAAI,CAAC;QACpB,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,IAAM,mBAAmB,GAAG,KAAK,CAAC;QAClC,IAAM,cAAc,GAAG,CAAC,CAAC;QACzB,IAAM,YAAY,GAAG,mBAAmB,GAAG,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC;QAChE,IAAI,WAAW,GAAG,IAAI,CAAC;QACvB,IAAI,MAAM,GAAG,KAAK,CAAC;QAEnB,wCAAwC;QACxC,WAAW,GAAG,UAAU,CAAC;YACrB,IAAI,CAAC,MAAM,EAAE,CAAC;gBACV,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,EAAE,2CAA2C,CAAC,CAAC;gBACvE,EAAE,CAAC,SAAS,EAAE,CAAC,CAAC,kCAAkC;YACtD,CAAC;QACL,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,qBAAqB;QAEhC,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE;YACV,MAAM,GAAG,IAAI,CAAC;YACd,YAAY,CAAC,WAAW,CAAC,CAAC;YAE1B,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,oCAAoC,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;YACvH,KAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;YAEpC,QAAQ,GAAG,WAAW,CAAC;gBACnB,IAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACvB,IAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC/C,IAAM,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC;gBACvD,IAAI,SAAS,GAAG,YAAY,IAAI,WAAW,GAAG,cAAc,EAAE,CAAC;oBAC3D,EAAE,CAAC,KAAK,EAAE,CAAC;oBACX,OAAO;gBACX,CAAC;gBACD,WAAW,IAAI,CAAC,CAAC;gBACjB,KAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;YACxC,CAAC,EAAE,mBAAmB,CAAC,CAAC;QAC5B,CAAC,CAAC,CAAC;QAET,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,UAAO,OAA0B;;;;;wBACjD,QAAQ,GAAG,IAAI,IAAI,EAAE,CAAC;6BAClB,CAAA,OAAO,OAAO,KAAK,QAAQ,CAAA,EAA3B,wBAA2B;wBAC9B,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;4BAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC;wBAClD,CAAC;6BAEG,CAAA,OAAO,KAAK,MAAM,CAAA,EAAlB,wBAAkB;wBACrB,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;4BAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,yBAAyB,CAAC,CAAC;wBACpD,CAAC;wBAED,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;;;6BAE5B,CAAA,OAAO,KAAK,MAAM,CAAA,EAAlB,wBAAkB;wBAC1B,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;4BAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,WAAW,CAAC,CAAC;wBACtC,CAAC;wBACD,WAAW,GAAG,CAAC,CAAC;;;wBAIhB,IAAI,CAAC;4BACJ,KAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,oBAAW,CAAC,CAAC;wBACxC,CAAC;wBACD,OAAO,CAAC,EAAE,CAAC;4BACV,OAAO,CAAC,KAAK,CAAC,oBAAoB,EAAE,CAAC,CAAC,CAAC;4BACvC,sBAAO;wBACR,CAAC;6BAEG,CAAA,KAAG,CAAC,IAAI,KAAK,MAAM,CAAA,EAAnB,wBAAmB;wBACtB,qBAAM,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,KAAG,CAAC,EAAA;;wBAAtC,SAAsC,CAAC;;4BAIzC,sBAAO;;wBAKR,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;4BAC9B,aAAa,GAAG,OAAO,CAAC;wBACzB,CAAC;6BACI,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;4BAC3B,MAAM,GAAG,OAA+C,CAAC;4BAC/D,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;wBACvC,CAAC;6BACI,IAAI,OAAO,YAAY,WAAW,EAAE,CAAC;4BACzC,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;wBACtC,CAAC;6BACI,IAAI,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;4BAChC,IAAI,GAAG,OAAiC,CAAC;4BAC/C,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;wBAC5E,CAAC;6BACI,CAAC;4BACL,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,OAAc,CAAC,CAAC;wBAC7C,CAAC;wBAED,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;4BAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,uBAAuB,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;wBACxE,CAAC;wBAED,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;4BAC5B,SAAS,GAAG,aAAa,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;4BAE/C,IAAI,SAAS,KAAK,MAAM,EAAE,CAAC;gCAC1B,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;oCAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,kCAAkC,CAAC,CAAC;gCAC7D,CAAC;gCAED,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;gCACpC,sBAAO;4BACR,CAAC;iCACI,IAAI,SAAS,KAAK,MAAM,EAAE,CAAC;gCAC/B,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;oCAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,oBAAoB,CAAC,CAAC;gCAC/C,CAAC;gCACD,WAAW,GAAG,CAAC,CAAC;gCAChB,sBAAO;4BACR,CAAC;wBACF,CAAC;wBAID,IAAI,CAAC;4BACJ,GAAG,GAAG,IAAA,iBAAM,EAAC,aAAa,CAAC,CAAC;wBAC7B,CAAC;wBACD,OAAO,CAAC,EAAE,CAAC;4BACV,OAAO,CAAC,KAAK,CAAC,2BAA2B,EAAE,CAAC,CAAC,CAAC;4BAC9C,sBAAO;wBACR,CAAC;6BAEG,CAAA,GAAG,CAAC,IAAI,KAAK,MAAM,CAAA,EAAnB,wBAAmB;wBACtB,qBAAM,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,GAAG,CAAC,EAAA;;wBAAtC,SAAsC,CAAC;;;;;aAExC,CAAC,CAAC;QAEG,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE;YACX,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,sDAAsD,CAAC,CAAC;YAChF,UAAU,CAAC;gBACP,KAAI,CAAC,mBAAmB,EAAE,CAAC;YAC/B,CAAC,EAAE,IAAI,CAAC,CAAC;YAET,IAAI,QAAQ,EAAE,CAAC;gBACX,aAAa,CAAC,QAAQ,CAAC,CAAC;YAC5B,CAAC;YACD,YAAY,CAAC,WAAW,CAAC,CAAC;QAC9B,CAAC,CAAC,CAAC;QAET,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,UAAC,GAAG;YAClB,IAAM,OAAO,GAAG,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,OAAO,KAAI,EAAE,CAAC;YACnC,IAAM,cAAc,GAAG,OAAO,CAAC,QAAQ,CAAC,iCAAiC,CAAC,CAAC;YAC3E,IAAI,CAAC,cAAc,EAAE,CAAC;gBACrB,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,EAAE,kBAAkB,EAAE,GAAG,CAAC,CAAC;YACpD,CAAC;iBACI,IAAI,KAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;gBAC/C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,2CAA2C,EAAE,OAAO,CAAC,CAAC;YAC/E,CAAC;YACD,EAAE,CAAC,KAAK,EAAE,CAAC;QACZ,CAAC,CAAC,CAAC;IACD,CAAC;IAEa,gDAAkB,GAAhC,UAAiC,EAAuB,EAAE,IAAiB;;;;;;;;;wBACjE,MAAM,GAA6C,IAAI,OAAjD,EAAE,SAAS,GAAkC,IAAI,UAAtC,EAAE,MAAM,GAA0B,IAAI,OAA9B,EAAE,MAAM,GAAkB,IAAI,OAAtB,EAAE,WAAW,GAAK,IAAI,YAAT,CAAU;wBAC9D,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;wBACxB,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;wBACzB,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;wBACjC,WAAW,GAAG,QAAQ,IAAI,YAAY,CAAC,CAAC,CAAC,IAAA,cAAK,EAAC,YAAY,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAA,cAAK,EAAC,UAAU,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;wBAC3H,eAAe,GAAG,YAAY,CAAC,CAAC,CAAC,IAAA,cAAK,EAAC,UAAU,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;wBACzE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;wBAC1B,aAAa,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;6BACpE,CAAA,CAAC,MAAM,IAAI,CAAC,MAAM,IAAI,aAAa,CAAA,EAAnC,wBAAmC;6BAC/B,aAAa,EAAb,wBAAa;wBACb,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,EAAE,kEAAkE,EAAE,MAAM,CAAC,CAAC;6BAC9G,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,gBAAgB,CAAC,EAA/C,wBAA+C;;;;wBAEjD,qBAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,gBAAgB,EAAE,yDAAkD,MAAM,gBAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAE,CAAC,EAAA;;wBAArJ,SAAqJ,CAAC;;;;wBAGtJ,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,EAAE,wCAAwC,EAAE,OAAK,CAAC,CAAC;;;wBAKrE,sDAAsD;wBAC/D,IAAI,CAAC,cAAc,CAAC,mBAAmB,EAAE;4BACxC,MAAM,QAAA;4BACN,MAAM,QAAA;4BACN,SAAS,WAAA;4BACT,WAAW,aAAA;4BACX,eAAe,iBAAA;4BACf,WAAW,aAAA;4BACX,MAAM,QAAA;yBACN,CAAC,CAAC;wBACM,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE;4BACxB,IAAI,EAAE,cAAc;4BACpB,MAAM,QAAA;4BACN,SAAS,WAAA;4BACT,KAAK,EAAE,IAAI;4BACX,MAAM,EAAE,cAAc;yBACzB,CAAC,CAAC;wBAEH,sBAAO;;wBAGP,QAAQ,GAAG,KAAK,CAAC;wBACjB,aAAa,GAAG,UAAU,CAAC;;;;;wCAC3B,QAAQ,GAAG,IAAI,CAAC;wCACnB,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;wCACvB,KAAK,GAAG,IAAA,cAAK,EAAC,SAAS,GAAG,UAAU,CAAC,CAAC;wCACtC,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAA,cAAK,EAAC,SAAS,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;wCACrD,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,EAAE,2BAA2B,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;wCAE3F,IAAI,CAAC,cAAc,CAAC,mBAAmB,EAAE;4CACxC,MAAM,QAAA;4CACN,MAAM,QAAA;4CACN,SAAS,WAAA;4CACT,KAAK,OAAA;4CACL,WAAW,aAAA;4CACX,eAAe,iBAAA;4CACf,OAAO,SAAA;4CACP,WAAW,aAAA;4CACX,MAAM,QAAA;yCACN,CAAC,CAAC;wCAEM,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE;4CACxB,IAAI,EAAE,cAAc;4CACpB,MAAM,QAAA;4CACN,SAAS,WAAA;4CACT,KAAK,EAAE,IAAI;4CACX,MAAM,EAAE,gBAAgB;yCAC3B,CAAC,CAAC;wCAEH,qBAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,gBAAgB,EAAE,6BAAsB,MAAM,gBAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAE,CAAC,EAAA;;wCAAzH,SAAyH,CAAC;;;;6BAC7H,EAAE,CAAA,MAAA,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,MAAM,CAAC,0CAAE,eAAe,KAAI,CAAC,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;;;;wBAGtE,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,cAAc,EAAE,8BAAa,CAAC,SAAS,EAAE;4BAC9E,OAAO,EAAE,CAAA,WAAW,aAAX,WAAW,uBAAX,WAAW,CAAE,OAAO,KAAI,EAAE;4BACnC,IAAI,EAAE,CAAA,WAAW,aAAX,WAAW,uBAAX,WAAW,CAAE,IAAI,KAAI,EAAE;4BAC7B,KAAK,EAAE,CAAA,WAAW,aAAX,WAAW,uBAAX,WAAW,CAAE,KAAK,KAAI,EAAE;yBAClC,CAAC,CAAC;wBAEH,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;4BACvC,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,gBAAgB,EAAE,MAAM,CAAC,CAAC;wBACtD,CAAC;wBAEG,qBAAM,CAAA,KAAA,IAAI,CAAC,cAAc,CAAC,UAAU,CAAA,CAAC,IAAI,0BAAC,WAAW,EAAE,MAAM,UAAK,MAAM,YAAC;;wBAAlF,MAAM,GAAG,SAAyE;wBAClF,YAAY,GAAwB,IAAI,CAAC;wBACzC,IAAI,GAAwB,IAAI,CAAC;wBACjC,kBAAkB,GAAQ,SAAS,CAAC;wBACpC,eAAe,GAAwB,IAAI,CAAC;wBAC1C,qBAAqB,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,MAAM,CAAC,qBAAqB,CAAC,CAAC;wBAEzE,IAAI,CAAC;4BACJ,IAAI,qBAAqB,EAAE,CAAC;gCAC3B,kBAAkB,GAAG,MAAM,CAAC,OAAO,CAAC;gCACpC,eAAe,GAAG;oCACjB,IAAI,EAAE,aAAa;oCACnB,WAAW,EAAE,MAAM,CAAC,WAAW;oCAC/B,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;oCACzC,YAAY,EAAE,MAAM,CAAC,YAAY;oCACjC,QAAQ,EAAE,MAAM,CAAC,QAAQ;iCACzB,CAAC;4BACH,CAAC;4BAED,IAAI,qBAAqB,EAAE,CAAC;gCAC3B,IAAI,GAAG,eAAe,CAAC;gCACvB,YAAY,GAAW,IAAA,eAAI,EAAC,kBAAkB,CAAC,CAAC;gCAChD,MAAM,GAAG,IAAI,CAAC;4BACf,CAAC;iCACI,CAAC;gCACL,YAAY,GAAW,IAAA,eAAI,EAAC,MAAM,CAAC,CAAC;4BACrC,CAAC;wBACF,CAAC;wBACD,OAAO,OAAO,EAAE,CAAC;4BAChB,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,EAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC;4BACxD,IAAI,qBAAqB,EAAE,CAAC;gCAC3B,IAAI,GAAG,eAAe,CAAC;gCACvB,MAAM,GAAG,kBAAkB,CAAC;4BAC7B,CAAC;wBACF,CAAC;wBAGD,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;4BAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,iBAAiB,EAAE,MAAM,CAAC,CAAC;wBACpD,CAAC;wBAED,IAAI,CAAC,QAAQ,EAAE,CAAC;4BACf,YAAY,CAAC,aAAa,CAAC,CAAC;4BACtB,OAAO,GAAiB;gCAC7B,IAAI,EAAE,cAAc;gCACpB,MAAM,QAAA;gCACN,SAAS,WAAA;gCACT,KAAK,EAAE,KAAK;gCACZ,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM;gCACpC,YAAY,cAAA;gCACZ,QAAQ,EAAE,SAAS;gCACnB,IAAI,MAAA;6BACJ,CAAC;4BAEF,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;wBACtC,CAAC;wBAEQ,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,UAAA,CAAC,IAAI,OAAA,CAAC,KAAK,MAAM,EAAZ,CAAY,CAAC,CAAC;wBAElE,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;4BACvC,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;wBAC3F,CAAC;;;;wBAGD,IAAI,CAAC,QAAQ,EAAE,CAAC;4BACZ,YAAY,CAAC,aAAa,CAAC,CAAC;4BAClC,YAAU,IAAI,CAAC,GAAG,EAAE,CAAC;4BACrB,UAAQ,IAAA,cAAK,EAAC,SAAO,GAAG,UAAU,CAAC,CAAC;4BACpC,YAAU,QAAQ,CAAC,CAAC,CAAC,IAAA,cAAK,EAAC,SAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;4BAChD,OAAO,CAAC,KAAK,CAAC,qBAAqB,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,KAAG,CAAC,CAAC;4BACrE,KAAG,CAAC,OAAO,GAAG,qBAAqB,GAAG,MAAM,GAAG,UAAU,GAAG,MAAM,GAAG,KAAK,GAAG,KAAG,CAAC,OAAO,CAAC;4BACrG,IAAI,CAAC,cAAc,CAAC,iBAAiB,EAAE;gCACtC,MAAM,QAAA;gCACN,MAAM,QAAA;gCACN,SAAS,WAAA;gCACT,KAAK,SAAA;gCACL,WAAW,aAAA;gCACX,eAAe,iBAAA;gCACf,OAAO,WAAA;gCACP,WAAW,aAAA;gCACX,MAAM,QAAA;gCACN,KAAK,EAAE,KAAG;6BACV,CAAC,CAAC;4BACS,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE;gCACxB,IAAI,EAAE,cAAc;gCACpB,MAAM,QAAA;gCACN,SAAS,WAAA;gCACT,KAAK,EAAE,IAAI;gCACX,MAAM,EAAE,KAAG,IAAI,eAAe;6BACjC,CAAC,CAAC;4BACH,MAAM,KAAG,CAAC;wBACd,CAAC;wBAEJ,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;wBACrB,KAAK,GAAG,IAAA,cAAK,EAAC,OAAO,GAAG,UAAU,CAAC,CAAC;wBACpC,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAA,cAAK,EAAC,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;wBAC5D,IAAI,CAAC,cAAc,CAAC,iBAAiB,EAAE;4BACtC,MAAM,QAAA;4BACN,MAAM,QAAA;4BACN,SAAS,WAAA;4BACT,KAAK,OAAA;4BACL,WAAW,aAAA;4BACX,eAAe,iBAAA;4BACf,OAAO,SAAA;4BACP,WAAW,aAAA;4BACX,MAAM,QAAA;4BACN,KAAK,EAAE,KAAG;yBACV,CAAC,CAAC;wBAEM,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,EAAG,2BAAoB,MAAM,gBAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAE,CAAC,CAAC;wBAE5F,qBAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,gBAAgB,EAAE,2BAAoB,MAAM,gBAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAE,CAAC,EAAA;;wBAAvH,SAAuH,CAAC;wBAExH,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,UAAA,CAAC,IAAI,OAAA,CAAC,KAAK,MAAM,EAAZ,CAAY,CAAC,CAAC;;;;;;KAEzE;IAEI,4CAAc,GAAtB,UAAuB,KAAa,EAAE,OAA4B;QACjE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,YAC7B,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAC5B,KAAK,OAAA,IACF,OAAO,EACT,CAAC,CAAC;IACL,CAAC;IAEO,2CAAa,GAArB,UAAsB,KAAU;QAC/B,IAAI,CAAC;YACJ,IAAM,MAAI,GAAG,IAAI,OAAO,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,UAAC,GAAG,EAAE,GAAG;gBACrC,IAAI,GAAG,KAAK,WAAW,EAAE,CAAC;oBACzB,OAAO,SAAS,CAAC;gBAClB,CAAC;gBACD,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;oBAC7B,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;gBACvB,CAAC;gBACD,IAAI,GAAG,YAAY,KAAK,EAAE,CAAC;oBAC1B,OAAO;wBACN,IAAI,EAAE,GAAG,CAAC,IAAI;wBACd,OAAO,EAAE,GAAG,CAAC,OAAO;wBACpB,KAAK,EAAE,GAAG,CAAC,KAAK;qBAChB,CAAC;gBACH,CAAC;gBACD,IAAI,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;oBACpC,IAAI,MAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;wBACnB,OAAO,YAAY,CAAC;oBACrB,CAAC;oBACD,MAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBACf,CAAC;gBACD,OAAO,GAAG,CAAC;YACZ,CAAC,CAAC,CAAC;QACJ,CAAC;QACD,OAAO,GAAG,EAAE,CAAC;YACZ,OAAO,IAAI,CAAC,SAAS,CAAC;gBACrB,KAAK,EAAE,qBAAqB;gBAC5B,OAAO,EAAE,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,OAAO,KAAI,MAAM,CAAC,GAAG,CAAC;aACpC,CAAC,CAAC;QACJ,CAAC;IACF,CAAC;IAEM,gDAAkB,GAAzB,UAA0B,EAAE,EAAE,OAA8B;QAC3D,IAAI,IAAI,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC;YAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,SAAS,EAAE,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;QACxF,CAAC;QAED,IAAI,EAAE,IAAI,EAAE,CAAC,UAAU,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;YAC5C,IAAI,CAAC;gBACJ,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;oBACjC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAClB,CAAC;qBACI,CAAC;oBACL,EAAE,CAAC,IAAI,CAAC,IAAA,eAAI,EAAC,OAAO,CAAC,CAAC,CAAC;gBACxB,CAAC;YACF,CAAC;YACD,OAAO,GAAG,EAAE,CAAC;gBACZ,OAAO,CAAC,KAAK,CAAC,iCAAiC,EAAE,GAAG,CAAC,CAAC;YACvD,CAAC;QACF,CAAC;aACI,IAAI,EAAE,EAAE,CAAC;YACb,EAAE,CAAC,KAAK,EAAE,CAAC;QACZ,CAAC;IACF,CAAC;IACF,0BAAC;AAAD,CA1cA,AA0cC,IAAA;AA1cY,kDAAmB","file":"worker-server.manager.js","sourcesContent":["import { pack, unpack } from 'msgpackr';\nimport * as WebSocket from 'ws';\nimport { TaskPayload, TaskResponse } from '../models/server-message.model';\nimport { dateReviver, round } from '../util/common';\nimport { MethodManager } from './method.manager';\n\nexport class WorkerServerManager {\n private _methodManager: MethodManager;\n private _serverConfig;\n private _runningTasks = [];\n\tprivate _taskLogThresholdMs = 200;\n\n constructor() {}\n \n static create(methodManager: MethodManager, serverConfig) {\n const workerServerManager = new WorkerServerManager();\n workerServerManager.initialize(methodManager, serverConfig);\n return workerServerManager;\n }\n\n public initialize(methodManager: MethodManager, serverConfig) {\n this._methodManager = methodManager;\n this._serverConfig = serverConfig;\n\t\tif (this._methodManager && typeof this._methodManager.onReady === 'function') {\n\t\t\tthis._methodManager.onReady(() => this.startWorkerInstance());\n\t\t}\n\t\telse {\n\t\t\tthis.startWorkerInstance();\n\t\t}\n }\n\n private startWorkerInstance() {\n console.log(new Date(), 'Worker instance started, connecting to main server via WebSocket...');\n \n const workerIndex = encodeURIComponent(String(process.env.WORKER_INDEX || ''));\n const workerInstance = encodeURIComponent(String(process.env.NODE_APP_INSTANCE || ''));\n let wsUrl = this._serverConfig['SERVER_URL'] + '/websocket?workerToken=' + this._serverConfig['WORKER_TOKEN'] + '&workerIndex=' + workerIndex + '&workerInstance=' + workerInstance;\n const ws = new WebSocket(wsUrl);\n \n let lastComm = new Date();\n let interval = null;\n let missedPongs = 0;\n const heartbeatIntervalMs = 15000;\n const maxMissedPongs = 2;\n const maxSilenceMs = heartbeatIntervalMs * (maxMissedPongs + 1);\n let openTimeout = null;\n let opened = false;\n \n // Set timeout if the socket never opens\n openTimeout = setTimeout(() => {\n if (!opened) {\n console.error(new Date(), 'WebSocket connection timeout. Retrying...');\n ws.terminate(); // force close if still connecting\n }\n }, 10000); // 10 seconds timeout\n \n ws.on('open', () => {\n opened = true;\n clearTimeout(openTimeout);\n \n console.log(new Date(), 'Connected to main server as worker', process.env.WORKER_INDEX, process.env.NODE_APP_INSTANCE);\n this.sendWorkerResponse(ws, 'ping');\n \n interval = setInterval(() => {\n const now = Date.now();\n const last = lastComm ? lastComm.getTime() : 0;\n const silenceMs = last ? now - last : maxSilenceMs + 1;\n if (silenceMs > maxSilenceMs || missedPongs > maxMissedPongs) {\n ws.close();\n return;\n }\n missedPongs += 1;\n this.sendWorkerResponse(ws, 'ping');\n }, heartbeatIntervalMs);\n });\n \n\t\tws.on('message', async (rawData: WebSocket.RawData) => {\n\t\t\tlastComm = new Date();\n\t\t\tif (typeof rawData === 'string') {\n\t\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\t\tconsole.log(new Date(), 'Message Recv', rawData);\n\t\t\t\t}\n\n\t\t\t\tif (rawData === 'ping') {\n\t\t\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\t\t\tconsole.log(new Date(), 'Recv Ping, Sending Pong');\n\t\t\t\t\t}\n\t\t\t\t\t\n\t\t\t\t\tthis.sendWorkerResponse(ws, 'pong');\n\t\t\t\t}\n\t\t\t\telse if (rawData === 'pong') {\n\t\t\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\t\t\tconsole.log(new Date(), 'Recv Pong');\n\t\t\t\t\t}\n\t\t\t\t\tmissedPongs = 0;\n\t\t\t\t}\n\t\t\t\telse {\n\t\t\t\t\tlet msg: any;\n\t\t\t\t\ttry {\n\t\t\t\t\t\tmsg = JSON.parse(rawData, dateReviver);\n\t\t\t\t\t}\n\t\t\t\t\tcatch (e) {\n\t\t\t\t\t\tconsole.error('Worker parse error', e);\n\t\t\t\t\t\treturn;\n\t\t\t\t\t}\n\t\n\t\t\t\t\tif (msg.type === 'task') {\n\t\t\t\t\t\tawait this.handleIncomingTask(ws, msg);\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\treturn;\n\t\t\t}\n\n\t\t\tlet messageBuffer: Buffer;\n\n\t\t\tif (Buffer.isBuffer(rawData)) {\n\t\t\t\tmessageBuffer = rawData;\n\t\t\t}\n\t\t\telse if (Array.isArray(rawData)) {\n\t\t\t\tconst chunks = rawData as unknown as ReadonlyArray<Uint8Array>;\n\t\t\t\tmessageBuffer = Buffer.concat(chunks);\n\t\t\t}\n\t\t\telse if (rawData instanceof ArrayBuffer) {\n\t\t\t\tmessageBuffer = Buffer.from(rawData);\n\t\t\t}\n\t\t\telse if (ArrayBuffer.isView(rawData)) {\n\t\t\t\tconst view = rawData as NodeJS.ArrayBufferView;\n\t\t\t\tmessageBuffer = Buffer.from(view.buffer, view.byteOffset, view.byteLength);\n\t\t\t}\n\t\t\telse {\n\t\t\t\tmessageBuffer = Buffer.from(rawData as any);\n\t\t\t}\n\n\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\tconsole.log(new Date(), 'Message Recv (binary)', messageBuffer.length);\n\t\t\t}\n\n\t\t\tif (messageBuffer.length === 4) {\n\t\t\t\tlet heartbeat = messageBuffer.toString('utf8');\n\n\t\t\t\tif (heartbeat === 'ping') {\n\t\t\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\t\t\tconsole.log(new Date(), 'Recv Ping (binary), Sending Pong');\n\t\t\t\t\t}\n\n\t\t\t\t\tthis.sendWorkerResponse(ws, 'pong');\n\t\t\t\t\treturn;\n\t\t\t\t}\n\t\t\t\telse if (heartbeat === 'pong') {\n\t\t\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\t\t\tconsole.log(new Date(), 'Recv Pong (binary)');\n\t\t\t\t\t}\n\t\t\t\t\tmissedPongs = 0;\n\t\t\t\t\treturn;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tlet msg: TaskPayload;\n\n\t\t\ttry {\n\t\t\t\tmsg = unpack(messageBuffer);\n\t\t\t}\n\t\t\tcatch (e) {\n\t\t\t\tconsole.error('Worker binary parse error', e);\n\t\t\t\treturn;\n\t\t\t}\n\n\t\t\tif (msg.type === 'task') {\n\t\t\t\tawait this.handleIncomingTask(ws, msg);\n\t\t\t}\n\t\t});\n \n ws.on('close', () => {\n console.log(new Date(), 'Disconnected from main server. Reconnecting in 1s...');\n setTimeout(() => {\n this.startWorkerInstance();\n }, 1000);\n \n if (interval) {\n clearInterval(interval);\n }\n clearTimeout(openTimeout);\n });\n \n\t\tws.on('error', (err) => {\n\t\t\tconst message = err?.message || '';\n\t\t\tconst isRestartNoise = message.includes('Unexpected server response: 502');\n\t\t\tif (!isRestartNoise) {\n\t\t\t\tconsole.error(new Date(), 'Worker WS error:', err);\n\t\t\t}\n\t\t\telse if (this._methodManager.getEnableDebug()) {\n\t\t\t\tconsole.log(new Date(), 'Worker WS reconnecting after main restart', message);\n\t\t\t}\n\t\t\tws.close();\n\t\t});\n }\n\n private async handleIncomingTask(ws: WebSocket.WebSocket, data: TaskPayload) {\n let { taskId, messageId, method, params, userContext } = data;\n\t\tconst receivedAt = Date.now();\n\t\tconst queuedAt = data.queuedAt;\n\t\tconst dispatchedAt = data.dispatchedAt;\n\t\tconst queueWaitMs = queuedAt && dispatchedAt ? round(dispatchedAt - queuedAt) : (queuedAt ? round(receivedAt - queuedAt) : null);\n\t\tconst dispatchDelayMs = dispatchedAt ? round(receivedAt - dispatchedAt) : null;\n this._runningTasks.push(taskId);\n const missingMethod = !method || !this._methodManager.getMethod(method);\n if (!taskId || !method || missingMethod) {\n if (missingMethod) {\n console.error(new Date(), 'No method in method manager for handleIncomingTask worker server', method);\n\t\t\t\tif (this._methodManager.getMethod('insertErrorLog')) {\n\t\t\t\t\ttry {\n\t\t\t\t\t\tawait this._methodManager.callMethod('insertErrorLog', `No Method in worker server handleIncomingTask: ${method} - ${JSON.stringify(data, null, 2)}`);\n\t\t\t\t\t}\n\t\t\t\t\tcatch (error) {\n\t\t\t\t\t\tconsole.error(new Date(), 'insertErrorLog failed in worker-server', error);\n\t\t\t\t\t}\n\t\t\t\t}\n }\n\n // console.log('Invalid task message received', data);\n\t\t\tthis.logWorkerEvent('workerTaskInvalid', {\n\t\t\t\ttaskId,\n\t\t\t\tmethod,\n\t\t\t\tmessageId,\n\t\t\t\tqueueWaitMs,\n\t\t\t\tdispatchDelayMs,\n\t\t\t\tuserContext,\n\t\t\t\tparams\n\t\t\t});\n this.sendWorkerResponse(ws, {\n type: 'taskComplete',\n taskId,\n messageId,\n error: true,\n result: 'Invalid task'\n });\n\n return;\n }\n\n let timedOut = false;\n let timeoutHandle = setTimeout(async () => {\n timedOut = true;\n\t\t\tconst timeoutAt = Date.now();\n\t\t\tconst runMs = round(timeoutAt - receivedAt);\n\t\t\tconst totalMs = queuedAt ? round(timeoutAt - queuedAt) : null;\n console.error(new Date(), 'Worker timed out on task:', taskId, 'Method:', method);\n\n\t\t\tthis.logWorkerEvent('workerTaskTimeout', {\n\t\t\t\ttaskId,\n\t\t\t\tmethod,\n\t\t\t\tmessageId,\n\t\t\t\trunMs,\n\t\t\t\tqueueWaitMs,\n\t\t\t\tdispatchDelayMs,\n\t\t\t\ttotalMs,\n\t\t\t\tuserContext,\n\t\t\t\tparams\n\t\t\t});\n\n this.sendWorkerResponse(ws, {\n type: 'taskComplete',\n taskId,\n messageId,\n error: true,\n result: 'Task timed out'\n });\n\n await this._methodManager.callMethod('insertErrorLog', `Timeout in Method: ${method} - ${JSON.stringify(data, null, 2)}`);\n }, this._methodManager.getMethod(method)?.timeoutOverride || (1000 * 60 * 2));\n\n try {\n let managerThis = Object.assign({}, this._methodManager, MethodManager.prototype, {\n id_user: userContext?.id_user || '',\n user: userContext?.user || '',\n id_ws: userContext?.id_ws || ''\n });\n\n if (this._methodManager.getEnableDebug()) {\n console.log(new Date(), 'Running method', method);\n }\n\n\t\t\tlet result = await this._methodManager.callMethod.call(managerThis, method, ...params);\n\t\t\tlet packedResult: Uint8Array | Buffer = null;\n\t\t\tlet meta: Record<string, any> = null;\n\t\t\tlet publicationPayload: any = undefined;\n\t\t\tlet publicationMeta: Record<string, any> = null;\n\t\t\tconst isPublicationResponse = !!(result && result.__publicationResponse);\n\n\t\t\ttry {\n\t\t\t\tif (isPublicationResponse) {\n\t\t\t\t\tpublicationPayload = result.payload;\n\t\t\t\t\tpublicationMeta = {\n\t\t\t\t\t\ttype: 'publication',\n\t\t\t\t\t\tpublication: result.publication,\n\t\t\t\t\t\tsubscriptionData: result.subscriptionData,\n\t\t\t\t\t\tuserSpecific: result.userSpecific,\n\t\t\t\t\t\tsnapshot: result.snapshot\n\t\t\t\t\t};\n\t\t\t\t}\n\n\t\t\t\tif (isPublicationResponse) {\n\t\t\t\t\tmeta = publicationMeta;\n\t\t\t\t\tpackedResult = <Buffer>pack(publicationPayload);\n\t\t\t\t\tresult = null;\n\t\t\t\t}\n\t\t\t\telse {\n\t\t\t\t\tpackedResult = <Buffer>pack(result);\n\t\t\t\t}\n\t\t\t}\n\t\t\tcatch (packErr) {\n\t\t\t\tconsole.error(new Date(), 'Worker pack error', packErr);\n\t\t\t\tif (isPublicationResponse) {\n\t\t\t\t\tmeta = publicationMeta;\n\t\t\t\t\tresult = publicationPayload;\n\t\t\t\t}\n\t\t\t}\n\n\n\t\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\t\tconsole.log(new Date(), 'Finished method', method);\n\t\t\t}\n\n\t\t\tif (!timedOut) {\n\t\t\t\tclearTimeout(timeoutHandle);\n\t\t\t\tconst payload: TaskResponse = {\n\t\t\t\t\ttype: 'taskComplete',\n\t\t\t\t\ttaskId,\n\t\t\t\t\tmessageId,\n\t\t\t\t\terror: false,\n\t\t\t\t\tresult: packedResult ? null : result,\n\t\t\t\t\tpackedResult,\n\t\t\t\t\tencoding: 'msgpack',\n\t\t\t\t\tmeta\n\t\t\t\t};\n\n\t\t\t\tthis.sendWorkerResponse(ws, payload);\n\t\t\t}\n\n this._runningTasks = this._runningTasks.filter(a => a !== taskId);\n \n if (this._methodManager.getEnableDebug()) {\n console.log(new Date(), 'Done with Task', JSON.stringify(this._runningTasks, null, 2));\n }\n }\n catch (err) {\n if (!timedOut) {\n clearTimeout(timeoutHandle);\n\t\t\t\tconst errorAt = Date.now();\n\t\t\t\tconst runMs = round(errorAt - receivedAt);\n\t\t\t\tconst totalMs = queuedAt ? round(errorAt - queuedAt) : null;\n console.error('Worker failed task:', taskId, 'Method:', method, err);\n err.message = 'Worker failed task:' + taskId + ' Method:' + method + ' - ' + err.message;\n\t\t\t\tthis.logWorkerEvent('workerTaskError', {\n\t\t\t\t\ttaskId,\n\t\t\t\t\tmethod,\n\t\t\t\t\tmessageId,\n\t\t\t\t\trunMs,\n\t\t\t\t\tqueueWaitMs,\n\t\t\t\t\tdispatchDelayMs,\n\t\t\t\t\ttotalMs,\n\t\t\t\t\tuserContext,\n\t\t\t\t\tparams,\n\t\t\t\t\terror: err\n\t\t\t\t});\n this.sendWorkerResponse(ws, {\n type: 'taskComplete',\n taskId,\n messageId,\n error: true,\n result: err || 'Unknown error'\n });\n throw err;\n }\n\n\t\t\tconst errorAt = Date.now();\n\t\t\tconst runMs = round(errorAt - receivedAt);\n\t\t\tconst totalMs = queuedAt ? round(errorAt - queuedAt) : null;\n\t\t\tthis.logWorkerEvent('workerTaskError', {\n\t\t\t\ttaskId,\n\t\t\t\tmethod,\n\t\t\t\tmessageId,\n\t\t\t\trunMs,\n\t\t\t\tqueueWaitMs,\n\t\t\t\tdispatchDelayMs,\n\t\t\t\ttotalMs,\n\t\t\t\tuserContext,\n\t\t\t\tparams,\n\t\t\t\terror: err\n\t\t\t});\n\n console.error(new Date(), `Error in Method: ${method} - ${JSON.stringify(data, null, 2)}`);\n \n await this._methodManager.callMethod('insertErrorLog', `Error in Method: ${method} - ${JSON.stringify(data, null, 2)}`);\n \n this._runningTasks = this._runningTasks.filter(a => a !== taskId);\n }\n }\n\n\tprivate logWorkerEvent(event: string, details: Record<string, any>) {\n\t\tconsole.log(this.safeStringify({\n\t\t\tts: new Date().toISOString(),\n\t\t\tevent,\n\t\t\t...details\n\t\t}));\n\t}\n\n\tprivate safeStringify(value: any): string {\n\t\ttry {\n\t\t\tconst seen = new WeakSet();\n\t\t\treturn JSON.stringify(value, (key, val) => {\n\t\t\t\tif (key === '__proto__') {\n\t\t\t\t\treturn undefined;\n\t\t\t\t}\n\t\t\t\tif (typeof val === 'bigint') {\n\t\t\t\t\treturn val.toString();\n\t\t\t\t}\n\t\t\t\tif (val instanceof Error) {\n\t\t\t\t\treturn {\n\t\t\t\t\t\tname: val.name,\n\t\t\t\t\t\tmessage: val.message,\n\t\t\t\t\t\tstack: val.stack\n\t\t\t\t\t};\n\t\t\t\t}\n\t\t\t\tif (val && typeof val === 'object') {\n\t\t\t\t\tif (seen.has(val)) {\n\t\t\t\t\t\treturn '[Circular]';\n\t\t\t\t\t}\n\t\t\t\t\tseen.add(val);\n\t\t\t\t}\n\t\t\t\treturn val;\n\t\t\t});\n\t\t}\n\t\tcatch (err) {\n\t\t\treturn JSON.stringify({\n\t\t\t\terror: 'safeStringifyFailed',\n\t\t\t\tmessage: err?.message || String(err)\n\t\t\t});\n\t\t}\n\t}\n\n\tpublic sendWorkerResponse(ws, payload: TaskResponse | string) {\n\t\tif (this._methodManager.getEnableDebug()) {\n\t\t\tconsole.log(new Date(), 'Sending', typeof payload === 'string' ? payload : '[binary]');\n\t\t}\n\n\t\tif (ws && ws.readyState === WebSocket.OPEN) {\n\t\t\ttry {\n\t\t\t\tif (typeof payload === 'string') {\n\t\t\t\t\tws.send(payload);\n\t\t\t\t}\n\t\t\t\telse {\n\t\t\t\t\tws.send(pack(payload));\n\t\t\t\t}\n\t\t\t}\n\t\t\tcatch (err) {\n\t\t\t\tconsole.error('Failed to send worker response:', err);\n\t\t\t}\n\t\t}\n\t\telse if (ws) {\n\t\t\tws.close();\n\t\t}\n\t}\n}\n"]}
|