@resolveio/server-lib 22.2.34 → 22.2.36

package/managers/worker-dispatcher.manager.js

@@ -0,0 +1,1210 @@
+"use strict";
+var __assign = (this && this.__assign) || function () {
+    __assign = Object.assign || function(t) {
+        for (var s, i = 1, n = arguments.length; i < n; i++) {
+            s = arguments[i];
+            for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
+                t[p] = s[p];
+        }
+        return t;
+    };
+    return __assign.apply(this, arguments);
+};
+var __values = (this && this.__values) || function(o) {
+    var s = typeof Symbol === "function" && Symbol.iterator, m = s && o[s], i = 0;
+    if (m) return m.call(o);
+    if (o && typeof o.length === "number") return {
+        next: function () {
+            if (o && i >= o.length) o = void 0;
+            return { value: o && o[i++], done: !o };
+        }
+    };
+    throw new TypeError(s ? "Object is not iterable." : "Symbol.iterator is not defined.");
+};
+var __read = (this && this.__read) || function (o, n) {
+    var m = typeof Symbol === "function" && o[Symbol.iterator];
+    if (!m) return o;
+    var i = m.call(o), r, ar = [], e;
+    try {
+        while ((n === void 0 || n-- > 0) && !(r = i.next()).done) ar.push(r.value);
+    }
+    catch (error) { e = { error: error }; }
+    finally {
+        try {
+            if (r && !r.done && (m = i["return"])) m.call(i);
+        }
+        finally { if (e) throw e.error; }
+    }
+    return ar;
+};
+var __spreadArray = (this && this.__spreadArray) || function (to, from, pack) {
+    if (pack || arguments.length === 2) for (var i = 0, l = from.length, ar; i < l; i++) {
+        if (ar || !(i in from)) {
+            if (!ar) ar = Array.prototype.slice.call(from, 0, i);
+            ar[i] = from[i];
+        }
+    }
+    return to.concat(ar || Array.prototype.slice.call(from));
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WorkerDispatcherManager = void 0;
+var WebSocket = require("ws");
+var common_1 = require("../util/common");
+var msgpackr_1 = require("msgpackr");
+var WorkerDispatcherManager = /** @class */ (function () {
+    function WorkerDispatcherManager() {
+        this._workers = [];
+        this._taskQueue = [];
+        this._clientRequests = {};
+        this._pendingTasks = new Map();
+        this._debugNoWorkerTaskIds = new Set();
+        this._debugCodexTaskIds = new Set();
+        this._taskTimings = new Map();
+        this._dispatchRetryTimer = null;
+        this._dispatchRetryDelayMs = 25;
+        this._nonCodexReserve = 0;
+        this._publicationWorkerIndexes = null;
+        this._publicationWorkerInstances = null;
+        this._codexWorkerIndexes = null;
+        this._codexWorkerInstances = null;
+        this.MAX_CONCURRENCY = 10;
+        this.WORKER_TASK_LOG_THRESHOLD_MS = 200;
+        this.WORKER_PUBLICATION_SLOW_MS = 20000;
+        this._workerTaskDebug = false;
+        this._aiWorkerDebug = false;
+    }
+    WorkerDispatcherManager.create = function (websocketManager, methodManager) {
+        var workerDispatcherManager = new WorkerDispatcherManager();
+        workerDispatcherManager.initialize(websocketManager, methodManager);
+        return workerDispatcherManager;
+    };
+    WorkerDispatcherManager.prototype.initialize = function (websocketManager, methodManager) {
+        var _a, _b, _c;
+        this._websocketManager = websocketManager;
+        this._methodManager = methodManager;
+        this._workerTaskDebug = this.parseDebugFlag((_a = process.env.WORKER_TASK_DEBUG) !== null && _a !== void 0 ? _a : process.env.WORKER_DISPATCH_DEBUG);
+        this._aiWorkerDebug = this.parseDebugFlag(process.env.AI_ASSISTANT_WORKER_DEBUG);
+        var slowPublicationMs = parseInt((_b = process.env.WORKER_PUBLICATION_SLOW_MS) !== null && _b !== void 0 ? _b : '', 10);
+        if (!Number.isNaN(slowPublicationMs) && slowPublicationMs > 0) {
+            this.WORKER_PUBLICATION_SLOW_MS = slowPublicationMs;
+        }
+        this._nonCodexReserve = this.parsePositiveInt((_c = process.env.WORKER_NON_CODEX_RESERVE) !== null && _c !== void 0 ? _c : process.env.WORKER_RESERVED_SLOTS, 0);
+        this._publicationWorkerIndexes = this.parseWorkerSelector(process.env.PUBLICATION_WORKER_INDEX
+            || process.env.SUBSCRIPTION_WORKER_INDEX
+            || process.env.WORKER_PUBLICATION_INDEX);
+        this._publicationWorkerInstances = this.parseWorkerSelector(process.env.PUBLICATION_WORKER_INSTANCE
+            || process.env.SUBSCRIPTION_WORKER_INSTANCE
+            || process.env.WORKER_PUBLICATION_INSTANCE);
+        this._codexWorkerIndexes = this.parseWorkerSelector(process.env.AI_ASSISTANT_CODEX_WORKER_INDEX
+            || process.env.CODEX_WORKER_INDEX
+            || process.env.WORKER_CODEX_INDEX);
+        this._codexWorkerInstances = this.parseWorkerSelector(process.env.AI_ASSISTANT_CODEX_WORKER_INSTANCE
+            || process.env.CODEX_WORKER_INSTANCE
+            || process.env.WORKER_CODEX_INSTANCE);
+    };
+    WorkerDispatcherManager.prototype.parseDebugFlag = function (value) {
+        if (value === true) {
+            return true;
+        }
+        if (value === false || value === null || value === undefined) {
+            return false;
+        }
+        if (typeof value === 'number') {
+            return value === 1;
+        }
+        if (typeof value === 'string') {
+            var normalized = value.trim().toLowerCase();
+            return ['1', 'true', 'yes', 'y', 'on'].includes(normalized);
+        }
+        return false;
+    };
+    WorkerDispatcherManager.prototype.parsePositiveInt = function (value, fallback) {
+        var parsed = parseInt(value !== null && value !== void 0 ? value : '', 10);
+        if (Number.isNaN(parsed) || parsed < 0) {
+            return fallback;
+        }
+        return parsed;
+    };
+    WorkerDispatcherManager.prototype.parseWorkerSelector = function (value) {
+        var e_1, _a, e_2, _b, e_3, _c;
+        if (value === null || value === undefined) {
+            return null;
+        }
+        var parts = [];
+        if (Array.isArray(value)) {
+            try {
+                for (var value_1 = __values(value), value_1_1 = value_1.next(); !value_1_1.done; value_1_1 = value_1.next()) {
+                    var item = value_1_1.value;
+                    var entry = String(item !== null && item !== void 0 ? item : '').split(',');
+                    try {
+                        for (var entry_1 = (e_2 = void 0, __values(entry)), entry_1_1 = entry_1.next(); !entry_1_1.done; entry_1_1 = entry_1.next()) {
+                            var chunk = entry_1_1.value;
+                            var trimmed = String(chunk !== null && chunk !== void 0 ? chunk : '').trim();
+                            if (trimmed) {
+                                parts.push(trimmed);
+                            }
+                        }
+                    }
+                    catch (e_2_1) { e_2 = { error: e_2_1 }; }
+                    finally {
+                        try {
+                            if (entry_1_1 && !entry_1_1.done && (_b = entry_1.return)) _b.call(entry_1);
+                        }
+                        finally { if (e_2) throw e_2.error; }
+                    }
+                }
+            }
+            catch (e_1_1) { e_1 = { error: e_1_1 }; }
+            finally {
+                try {
+                    if (value_1_1 && !value_1_1.done && (_a = value_1.return)) _a.call(value_1);
+                }
+                finally { if (e_1) throw e_1.error; }
+            }
+        }
+        else {
+            var entry = String(value !== null && value !== void 0 ? value : '').split(',');
+            try {
+                for (var entry_2 = __values(entry), entry_2_1 = entry_2.next(); !entry_2_1.done; entry_2_1 = entry_2.next()) {
+                    var chunk = entry_2_1.value;
+                    var trimmed = String(chunk !== null && chunk !== void 0 ? chunk : '').trim();
+                    if (trimmed) {
+                        parts.push(trimmed);
+                    }
+                }
+            }
+            catch (e_3_1) { e_3 = { error: e_3_1 }; }
+            finally {
+                try {
+                    if (entry_2_1 && !entry_2_1.done && (_c = entry_2.return)) _c.call(entry_2);
+                }
+                finally { if (e_3) throw e_3.error; }
+            }
+        }
+        if (!parts.length) {
+            return null;
+        }
+        return new Set(parts);
+    };
+    WorkerDispatcherManager.prototype.selectorToArray = function (value) {
+        if (!value) {
+            return null;
+        }
+        return Array.from(value.values());
+    };
+    WorkerDispatcherManager.prototype.hasWorkerSelector = function (indexes, instances) {
+        return !!(indexes && indexes.size) || !!(instances && instances.size);
+    };
+    WorkerDispatcherManager.prototype.workerMatchesSelector = function (worker, indexes, instances) {
+        if (!this.hasWorkerSelector(indexes, instances)) {
+            return false;
+        }
+        var normalizedIndex = this.normalizeWorkerIndex(worker.workerIndex);
+        var normalizedInstance = this.normalizeWorkerIndex(worker.workerInstance);
+        var indexMatch = indexes ? (normalizedIndex ? indexes.has(normalizedIndex) : false) : true;
+        var instanceMatch = instances ? (normalizedInstance ? instances.has(normalizedInstance) : false) : true;
+        return indexMatch && instanceMatch;
+    };
+    WorkerDispatcherManager.prototype.resolveTaskQueue = function (methodName) {
+        var _a;
+        var method = (_a = this._methodManager) === null || _a === void 0 ? void 0 : _a.getMethod(methodName);
+        var queue = method === null || method === void 0 ? void 0 : method.workerQueue;
+        if (queue === 'publication' || queue === 'codex') {
+            return queue;
+        }
+        return 'default';
+    };
+    WorkerDispatcherManager.prototype.summarizePublicationValue = function (value) {
+        if (value === null || value === undefined) {
+            return value;
+        }
+        var valueType = typeof value;
+        if (valueType === 'string' || valueType === 'number' || valueType === 'boolean') {
+            return value;
+        }
+        if (Array.isArray(value)) {
+            return "[Array(".concat(value.length, ")]");
+        }
+        if (valueType === 'object') {
+            return "[Object keys=".concat(Object.keys(value).length, "]");
+        }
+        return "[".concat(valueType, "]");
+    };
+    WorkerDispatcherManager.prototype.getPublicationLogInfo = function (method, params) {
+        var _this = this;
+        if (method !== 'runPublication' || !Array.isArray(params)) {
+            return null;
+        }
+        var publication = typeof params[0] === 'string' ? params[0] : null;
+        var subscriptionData = Array.isArray(params[1]) ? params[1] : [];
+        var userId = typeof params[2] === 'string' ? params[2] : null;
+        var subscriptionDataPreview = subscriptionData.slice(0, 3).map(function (item) { return _this.summarizePublicationValue(item); });
+        return {
+            publication: publication,
+            subscriptionDataCount: subscriptionData.length,
+            subscriptionDataPreview: subscriptionDataPreview,
+            userId: userId
+        };
+    };
+    WorkerDispatcherManager.prototype.isSafeShutdown = function () {
+        return !this._taskQueue.length && !this._pendingTasks.size;
+    };
+    WorkerDispatcherManager.prototype.addWorker = function (ws) {
+        var _this = this;
+        this._workers.push({
+            id: ws['id_worker'],
+            ws: ws,
+            activeTasks: [],
+            workerIndex: ws['workerIndex'],
+            workerInstance: ws['workerInstance']
+        });
+        setInterval(function () {
+            if (_this._methodManager.getEnableDebug()) {
+                console.log(JSON.stringify(_this._workers.map(function (a) {
+                    return {
+                        id: a.id,
+                        activeTasks: a.activeTasks
+                    };
+                }), null, 2));
+            }
+        }, 5000);
+        if (this._taskQueue.length) {
+            this.dispatchQueue();
+        }
+    };
+    WorkerDispatcherManager.prototype.disconnectWorker = function (workerId) {
+        this._workers = this._workers.filter(function (w) { return w.id !== workerId; });
+    };
+    WorkerDispatcherManager.prototype.hasWorkers = function () {
+        return this._workers.length > 0;
+    };
+    WorkerDispatcherManager.prototype.getQueueSnapshot = function () {
+        var activeTasks = this._workers.reduce(function (total, worker) { return total + (worker.activeTasks ? worker.activeTasks.length : 0); }, 0);
+        return {
+            queueDepth: this._taskQueue.length,
+            pendingTasks: this._pendingTasks.size,
+            workerCount: this._workers.length,
+            activeTasks: activeTasks,
+            workers: this._workers.map(function (worker) { return ({
+                id: worker.id,
+                index: worker.workerIndex,
+                instance: worker.workerInstance,
+                activeTasks: worker.activeTasks ? worker.activeTasks.length : 0
+            }); })
+        };
+    };
+    WorkerDispatcherManager.prototype.hasWorkersForMethod = function (methodName) {
+        var _this = this;
+        if (!this._workers.length) {
+            return false;
+        }
+        var method = this._methodManager.getMethod(methodName);
+        var taskQueue = this.resolveTaskQueue(methodName);
+        var targetWorkerIndex = this.normalizeWorkerIndex(method === null || method === void 0 ? void 0 : method.targetWorkerIndex);
+        var targetWorkerInstance = this.normalizeWorkerIndex(method === null || method === void 0 ? void 0 : method.targetWorkerInstance);
+        var hasExplicitTarget = !!targetWorkerIndex || !!targetWorkerInstance;
+        var candidates = __spreadArray([], __read(this._workers), false);
+        var hasPublicationSelector = this.hasWorkerSelector(this._publicationWorkerIndexes, this._publicationWorkerInstances);
+        if (hasPublicationSelector && !hasExplicitTarget) {
+            if (taskQueue === 'publication') {
+                candidates = candidates.filter(function (worker) {
+                    return _this.workerMatchesSelector(worker, _this._publicationWorkerIndexes, _this._publicationWorkerInstances);
+                });
+            }
+            else {
+                candidates = candidates.filter(function (worker) {
+                    return !_this.workerMatchesSelector(worker, _this._publicationWorkerIndexes, _this._publicationWorkerInstances);
+                });
+            }
+        }
+        var hasCodexSelector = this.hasWorkerSelector(this._codexWorkerIndexes, this._codexWorkerInstances);
+        if (hasCodexSelector) {
+            if (taskQueue === 'codex') {
+                candidates = candidates.filter(function (worker) {
+                    return _this.workerMatchesSelector(worker, _this._codexWorkerIndexes, _this._codexWorkerInstances);
+                });
+            }
+            else if (!hasExplicitTarget) {
+                candidates = candidates.filter(function (worker) {
+                    return !_this.workerMatchesSelector(worker, _this._codexWorkerIndexes, _this._codexWorkerInstances);
+                });
+            }
+        }
+        if (targetWorkerIndex) {
+            candidates = candidates.filter(function (worker) { return _this.normalizeWorkerIndex(worker.workerIndex) === targetWorkerIndex; });
+        }
+        if (targetWorkerInstance) {
+            candidates = candidates.filter(function (worker) { return _this.normalizeWorkerIndex(worker.workerInstance) === targetWorkerInstance; });
+        }
+        return candidates.length > 0;
+    };
+    /**
+     * Add a new task to our in-memory queue and try to dispatch.
+     */
+    WorkerDispatcherManager.prototype.sendClientTask = function (messageId, method, params, userContext) {
+        var taskId = 'task-' + (0, common_1.objectIdHexString)();
+        var queuedAt = Date.now();
+        this._taskQueue.push({
+            type: 'task',
+            taskId: taskId,
+            messageId: messageId,
+            method: method,
+            params: params,
+            userContext: userContext,
+            queuedAt: queuedAt
+        });
+        this._taskTimings.set(taskId, {
+            queuedAt: queuedAt,
+            method: method,
+            messageId: messageId,
+            userContext: userContext,
+            params: params
+        });
+        if (userContext && userContext.id_ws) {
+            this._clientRequests[taskId] = userContext.id_ws;
+        }
+        if (this.shouldDebug()) {
+            var publicationInfo = this.getPublicationLogInfo(method, params);
+            var logPayload = {
+                taskId: taskId,
+                method: method,
+                messageId: messageId,
+                priority: method === 'runPublication' ? 'publication' : 'method',
+                queueDepth: this._taskQueue.length,
+                userContext: userContext
+            };
+            if (publicationInfo) {
+                Object.assign(logPayload, publicationInfo);
+            }
+            this.logWorkerEvent('workerTaskQueued', logPayload);
+        }
+        this.dispatchQueue();
+    };
+    /**
+     * Same as sendInternalTask but returns a Promise so you can `await` it.
+     */
+    WorkerDispatcherManager.prototype.sendInternalPromise = function (method, params) {
+        var _this = this;
+        if (params === void 0) { params = []; }
+        if (!Array.isArray(params)) {
+            params = [params];
+        }
+        // eslint-disable-next-line no-restricted-syntax
+        return new Promise(function (resolve, reject) {
+            var taskId = 'task-' + (0, common_1.objectIdHexString)();
+            var queuedAt = Date.now();
+            _this._taskQueue.push({
+                type: 'task',
+                taskId: taskId,
+                method: method,
+                params: params,
+                messageId: 0,
+                userContext: {
+                    user: 'Internal System'
+                },
+                queuedAt: queuedAt
+            });
+            _this._taskTimings.set(taskId, {
+                queuedAt: queuedAt,
+                method: method,
+                messageId: 0,
+                userContext: { user: 'Internal System' },
+                params: params
+            });
+            _this._pendingTasks.set(taskId, {
+                timeout: null,
+                promise: { resolve: resolve, reject: reject }
+            });
+            if (_this._methodManager.getEnableDebug()) {
+                console.log(new Date(), 'Send Internal Promise', _this._taskQueue);
+            }
+            if (_this.shouldDebug()) {
+                var publicationInfo = _this.getPublicationLogInfo(method, params);
+                var logPayload = {
+                    taskId: taskId,
+                    method: method,
+                    messageId: 0,
+                    priority: method === 'runPublication' ? 'publication' : 'method',
+                    queueDepth: _this._taskQueue.length,
+                    userContext: { user: 'Internal System' }
+                };
+                if (publicationInfo) {
+                    Object.assign(logPayload, publicationInfo);
+                }
+                _this.logWorkerEvent('workerTaskQueued', logPayload);
+            }
+            _this.dispatchQueue();
+        });
+    };
+    /**
+     * Send a task internally and resolve with the raw TaskResponse (includes packedResult/meta).
+     */
+    WorkerDispatcherManager.prototype.sendInternalPromiseRaw = function (method, params) {
+        var _this = this;
+        if (params === void 0) { params = []; }
+        if (!Array.isArray(params)) {
+            params = [params];
+        }
+        // eslint-disable-next-line no-restricted-syntax
+        return new Promise(function (resolve, reject) {
+            var taskId = 'task-' + (0, common_1.objectIdHexString)();
+            var queuedAt = Date.now();
+            _this._taskQueue.push({
+                type: 'task',
+                taskId: taskId,
+                method: method,
+                params: params,
+                messageId: 0,
+                userContext: {
+                    user: 'Internal System'
+                },
+                queuedAt: queuedAt
+            });
+            _this._taskTimings.set(taskId, {
+                queuedAt: queuedAt,
+                method: method,
+                messageId: 0,
+                userContext: { user: 'Internal System' },
+                params: params
+            });
+            _this._pendingTasks.set(taskId, {
+                timeout: null,
+                promise: { resolve: resolve, reject: reject },
+                resolveRaw: true
+            });
+            if (_this._methodManager.getEnableDebug()) {
+                console.log(new Date(), 'Send Internal Raw Promise', _this._taskQueue);
+            }
+            if (_this.shouldDebug()) {
+                var publicationInfo = _this.getPublicationLogInfo(method, params);
+                var logPayload = {
+                    taskId: taskId,
+                    method: method,
+                    messageId: 0,
+                    priority: method === 'runPublication' ? 'publication' : 'method',
+                    queueDepth: _this._taskQueue.length,
+                    userContext: { user: 'Internal System' }
+                };
+                if (publicationInfo) {
+                    Object.assign(logPayload, publicationInfo);
+                }
+                _this.logWorkerEvent('workerTaskQueued', logPayload);
+            }
+            _this.dispatchQueue();
+        });
+    };
+    /**
+     * Send a task internally without returning a promise.
+     */
+    WorkerDispatcherManager.prototype.sendInternalTask = function (method, params) {
+        if (params === void 0) { params = []; }
+        if (!Array.isArray(params)) {
+            params = [params];
+        }
+        var taskId = 'task-' + (0, common_1.objectIdHexString)();
+        var queuedAt = Date.now();
+        this._taskQueue.push({
+            type: 'task',
+            taskId: taskId,
+            method: method,
+            params: params,
+            messageId: 0,
+            userContext: {
+                user: 'Internal System'
+            },
+            queuedAt: queuedAt
+        });
+        this._taskTimings.set(taskId, {
+            queuedAt: queuedAt,
+            method: method,
+            messageId: 0,
+            userContext: { user: 'Internal System' },
+            params: params
+        });
+        if (this._methodManager.getEnableDebug()) {
+            console.log(new Date(), 'Send Internal Task', this._taskQueue);
+        }
+        if (this.shouldDebug()) {
+            var publicationInfo = this.getPublicationLogInfo(method, params);
+            var logPayload = {
+                taskId: taskId,
+                method: method,
+                messageId: 0,
+                priority: method === 'runPublication' ? 'publication' : 'method',
+                queueDepth: this._taskQueue.length,
+                userContext: { user: 'Internal System' }
+            };
+            if (publicationInfo) {
+                Object.assign(logPayload, publicationInfo);
+            }
+            this.logWorkerEvent('workerTaskQueued', logPayload);
+        }
+        this.dispatchQueue();
+    };
+    /**
+     * The main loop that assigns tasks from _taskQueue to any worker that has capacity.
+     */
+    WorkerDispatcherManager.prototype.dispatchQueue = function () {
+        var e_4, _a;
+        var _this = this;
+        if (!this._taskQueue.length) {
+            return;
+        }
+        if (!this._workers.length) {
+            return;
+        }
+        var priorityMatchers = [
+            function (task) { return _this.resolveTaskQueue(task.method) === 'publication'; },
+            function (task) { return _this.resolveTaskQueue(task.method) === 'default'; },
+            function (task) { return _this.resolveTaskQueue(task.method) === 'codex'; }
+        ];
+        var tryAssignTask = function (matcher) {
+            for (var i = 0; i < _this._taskQueue.length; i++) {
+                var task = _this._taskQueue[i];
+                if (!matcher(task)) {
+                    continue;
+                }
+                var targetWorker = _this.findWorkerForTask(task);
+                if (targetWorker) {
+                    _this._taskQueue.splice(i, 1);
+                    _this.assignTaskToWorker(targetWorker, task);
+                    return true;
+                }
+            }
+            return false;
+        };
+        while (this._taskQueue.length > 0) {
+            var assigned = false;
+            try {
+                for (var priorityMatchers_1 = (e_4 = void 0, __values(priorityMatchers)), priorityMatchers_1_1 = priorityMatchers_1.next(); !priorityMatchers_1_1.done; priorityMatchers_1_1 = priorityMatchers_1.next()) {
+                    var matcher = priorityMatchers_1_1.value;
+                    if (tryAssignTask(matcher)) {
+                        assigned = true;
+                        break;
+                    }
+                }
+            }
+            catch (e_4_1) { e_4 = { error: e_4_1 }; }
+            finally {
+                try {
+                    if (priorityMatchers_1_1 && !priorityMatchers_1_1.done && (_a = priorityMatchers_1.return)) _a.call(priorityMatchers_1);
+                }
+                finally { if (e_4) throw e_4.error; }
+            }
+            if (!assigned) {
+                this.logNoWorkerAvailability();
+                this.scheduleDispatchRetry();
+                if (this._methodManager.getEnableDebug()) {
+                    console.log(new Date(), 'No Worker Available', JSON.stringify(this._workers, null, 2));
+                }
+                return; // nothing can run right now due to per-worker limits
+            }
+        }
+    };
+    WorkerDispatcherManager.prototype.scheduleDispatchRetry = function (delayMs) {
+        var _this = this;
+        if (delayMs === void 0) { delayMs = this._dispatchRetryDelayMs; }
+        if (this._dispatchRetryTimer) {
+            return;
+        }
+        this._dispatchRetryTimer = setTimeout(function () {
+            _this._dispatchRetryTimer = null;
+            _this.dispatchQueue();
+        }, delayMs);
+    };
+    /**
+     * Returns the worker with the fewest activeTasks that is under maxConcurrency and per-method limit.
+     */
+    WorkerDispatcherManager.prototype.findWorkerForTask = function (task) {
+        var _this = this;
+        var method = this._methodManager.getMethod(task.method);
+        var taskQueue = this.resolveTaskQueue(task.method);
+        var methodLimit = method && method.maxConcurrency && method.maxConcurrency > 0 ? method.maxConcurrency : null;
+        var methodLimitPerInstance = method && method.maxConcurrencyPerInstance && method.maxConcurrencyPerInstance > 0
+            ? method.maxConcurrencyPerInstance
+            : null;
+        var targetWorkerIndex = this.normalizeWorkerIndex(method === null || method === void 0 ? void 0 : method.targetWorkerIndex);
+        var targetWorkerInstance = this.normalizeWorkerIndex(method === null || method === void 0 ? void 0 : method.targetWorkerInstance);
+        var hasExplicitTarget = !!targetWorkerIndex || !!targetWorkerInstance;
+        var preferNonZero = !targetWorkerIndex;
+        var debugCodex = this._aiWorkerDebug && taskQueue === 'codex';
+        var debugState = debugCodex ? {
+            taskId: task.taskId,
+            method: task.method,
+            queueDepth: this._taskQueue.length,
+            targetWorkerIndex: targetWorkerIndex || null,
+            targetWorkerInstance: targetWorkerInstance || null,
+            workerCount: this._workers.length,
+            publicationSelector: {
+                indexes: this.selectorToArray(this._publicationWorkerIndexes),
+                instances: this.selectorToArray(this._publicationWorkerInstances)
+            },
+            codexSelector: {
+                indexes: this.selectorToArray(this._codexWorkerIndexes),
+                instances: this.selectorToArray(this._codexWorkerInstances)
+            },
+            maxConcurrency: this.MAX_CONCURRENCY,
+            nonCodexReserve: this._nonCodexReserve
+        } : null;
+        var candidates = this._workers.filter(function (x) { return x.activeTasks.length < _this.MAX_CONCURRENCY; });
+        if (debugState) {
+            debugState.candidatesInitial = candidates.length;
+        }
+        var hasPublicationSelector = this.hasWorkerSelector(this._publicationWorkerIndexes, this._publicationWorkerInstances);
+        if (hasPublicationSelector && !hasExplicitTarget) {
+            if (taskQueue === 'publication') {
+                candidates = candidates.filter(function (worker) {
+                    return _this.workerMatchesSelector(worker, _this._publicationWorkerIndexes, _this._publicationWorkerInstances);
+                });
+            }
+            else {
+                candidates = candidates.filter(function (worker) {
+                    return !_this.workerMatchesSelector(worker, _this._publicationWorkerIndexes, _this._publicationWorkerInstances);
+                });
+            }
+        }
+        if (debugState) {
+            debugState.candidatesAfterPublicationFilter = candidates.length;
+            debugState.hasPublicationSelector = hasPublicationSelector;
+        }
+        var hasCodexSelector = this.hasWorkerSelector(this._codexWorkerIndexes, this._codexWorkerInstances);
+        if (hasCodexSelector) {
+            if (taskQueue === 'codex') {
+                candidates = candidates.filter(function (worker) {
+                    return _this.workerMatchesSelector(worker, _this._codexWorkerIndexes, _this._codexWorkerInstances);
+                });
+            }
+            else if (!hasExplicitTarget) {
+                candidates = candidates.filter(function (worker) {
+                    return !_this.workerMatchesSelector(worker, _this._codexWorkerIndexes, _this._codexWorkerInstances);
+                });
+            }
+        }
+        if (debugState) {
+            debugState.candidatesAfterCodexFilter = candidates.length;
+            debugState.hasCodexSelector = hasCodexSelector;
+        }
+        if (targetWorkerIndex) {
+            candidates = candidates.filter(function (worker) { return _this.normalizeWorkerIndex(worker.workerIndex) === targetWorkerIndex; });
+        }
+        if (targetWorkerInstance) {
+            candidates = candidates.filter(function (worker) { return _this.normalizeWorkerIndex(worker.workerInstance) === targetWorkerInstance; });
+        }
+        if (debugState) {
+            debugState.candidatesAfterTargetFilter = candidates.length;
+        }
+        if (!candidates.length) {
+            this.logCodexSelection(debugState, 'noCandidates');
+            return null;
+        }
+        var eligible = candidates.filter(function (worker) {
+            if (!methodLimit) {
+                if (!methodLimitPerInstance) {
+                    return true;
+                }
+            }
+            if (methodLimitPerInstance) {
+                var currentInstance = worker.activeTasks.filter(function (a) { return a.method === task.method; }).length;
+                if (currentInstance >= methodLimitPerInstance) {
+                    return false;
+                }
+            }
+            if (!methodLimit) {
+                return true;
+            }
+            var idx = worker.workerIndex || 'unknown';
+            var current = 0;
+            _this._workers.forEach(function (w) {
+                if ((w.workerIndex || 'unknown') === idx) {
+                    current += w.activeTasks.filter(function (a) { return a.method === task.method; }).length;
+                }
+            });
+            return current < methodLimit;
+        });
+        if (debugState) {
+            debugState.eligibleAfterLimits = eligible.length;
+        }
+        if (!eligible.length) {
+            this.logCodexSelection(debugState, 'limitsExceeded');
+            return null;
+        }
+        if (taskQueue === 'codex' && this._nonCodexReserve > 0) {
+            eligible = eligible.filter(function (worker) { return (_this.MAX_CONCURRENCY - worker.activeTasks.length) > _this._nonCodexReserve; });
+        }
+        if (debugState) {
+            debugState.eligibleAfterReserve = eligible.length;
+        }
+        if (!eligible.length) {
+            this.logCodexSelection(debugState, 'reserveBlocked');
+            return null;
+        }
+        eligible.sort(function (x, y) {
+            var totalX = _this.getWorkerLoad(x);
+            var totalY = _this.getWorkerLoad(y);
+            if (totalX !== totalY) {
+                return totalX - totalY;
+            }
+            if (preferNonZero) {
+                var xZero = _this.isWorkerZero(x) ? 1 : 0;
+                var yZero = _this.isWorkerZero(y) ? 1 : 0;
+                if (xZero !== yZero) {
+                    return xZero - yZero;
+                }
+            }
+            var idxX = _this.normalizeWorkerIndex(x.workerIndex) || '';
+            var idxY = _this.normalizeWorkerIndex(y.workerIndex) || '';
+            var idxCompare = idxX.localeCompare(idxY, undefined, { numeric: true, sensitivity: 'base' });
+            if (idxCompare !== 0) {
+                return idxCompare;
+            }
+            var instX = _this.normalizeWorkerIndex(x.workerInstance) || '';
+            var instY = _this.normalizeWorkerIndex(y.workerInstance) || '';
+            return instX.localeCompare(instY, undefined, { numeric: true, sensitivity: 'base' });
+        });
+        var selected = eligible[0];
+        if (this._aiWorkerDebug && taskQueue === 'codex') {
+            console.log(new Date(), '[AI Worker Debug] AI worker selected', {
+                taskId: task.taskId,
+                method: task.method,
+                workerIndex: (selected === null || selected === void 0 ? void 0 : selected.workerIndex) || null,
+                workerInstance: (selected === null || selected === void 0 ? void 0 : selected.workerInstance) || null,
+                activeTasks: (selected === null || selected === void 0 ? void 0 : selected.activeTasks) ? selected.activeTasks.length : 0,
+                queueDepth: this._taskQueue.length
+            });
+        }
+        return selected;
+    };
+    WorkerDispatcherManager.prototype.logCodexSelection = function (debugState, reason) {
+        if (!debugState || !this._aiWorkerDebug) {
+            return;
+        }
+        var taskId = String(debugState.taskId || '');
+        if (!taskId || this._debugCodexTaskIds.has(taskId)) {
+            return;
+        }
+        this._debugCodexTaskIds.add(taskId);
+        console.warn(new Date(), '[AI Worker Debug] AI worker selection issue', __assign({ reason: reason }, debugState));
+    };
+    WorkerDispatcherManager.prototype.normalizeWorkerIndex = function (value) {
+        if (value === null || value === undefined) {
+            return null;
+        }
+        var normalized = String(value).trim();
+        return normalized.length ? normalized : null;
+    };
+    WorkerDispatcherManager.prototype.getWorkerLoad = function (worker) {
+        return worker.activeTasks.reduce(function (sum, task) { return sum + task.weight; }, 0);
+    };
+    WorkerDispatcherManager.prototype.isWorkerZero = function (worker) {
+        return this.normalizeWorkerIndex(worker.workerIndex) === '0';
+    };
+    WorkerDispatcherManager.prototype.assignTaskToWorker = function (worker, task) {
+        var _this = this;
+        var method = this._methodManager.getMethod(task.method);
+        if (!method) {
+            console.error('Failed to send task to worker - Could not find method:', task.method);
+            this._taskTimings.delete(task.taskId);
+            return;
+        }
+        var dispatchedAt = Date.now();
+        var timing = this._taskTimings.get(task.taskId) || { queuedAt: dispatchedAt };
+        timing.dispatchedAt = dispatchedAt;
+        timing.method = task.method;
+        timing.messageId = task.messageId;
+        timing.userContext = task.userContext;
+        if (!timing.params) {
+            timing.params = task.params;
+        }
+        this._taskTimings.set(task.taskId, timing);
+        worker.activeTasks.push({
+            taskId: task.taskId,
+            weight: method && method.workerTaskWeight ? method.workerTaskWeight : 1,
+            method: task.method,
+            startedAt: dispatchedAt,
+            messageId: task.messageId,
+            userContext: task.userContext
+        });
+        this._debugNoWorkerTaskIds.delete(task.taskId);
+        var payload = {
+            type: 'task',
+            taskId: task.taskId,
+            messageId: task.messageId,
+            method: task.method,
+            params: task.params,
+            userContext: task.userContext || {},
+            queuedAt: timing.queuedAt,
+            dispatchedAt: dispatchedAt
+        };
+        if (this.shouldDebug()) {
+            var publicationInfo = this.getPublicationLogInfo(task.method, task.params);
+            var logPayload = {
+                taskId: task.taskId,
+                method: task.method,
+                messageId: task.messageId,
+                priority: task.method === 'runPublication' ? 'publication' : 'method',
+                worker: {
+                    id: worker.id,
+                    index: worker.workerIndex,
+                    instance: worker.workerInstance
+                },
+                queueDepth: this._taskQueue.length
+            };
+            if (publicationInfo) {
+                Object.assign(logPayload, publicationInfo);
+            }
+            this.logWorkerEvent('workerTaskAssigned', logPayload);
+        }
+        var timeoutHandle = setTimeout(function () {
+            var pending = _this._pendingTasks.get(task.taskId);
+            var timeoutAt = Date.now();
+            var queuedAt = timing.queuedAt;
+            var queueWaitMs = queuedAt ? (0, common_1.round)(dispatchedAt - queuedAt) : null;
+            var dispatchToTimeoutMs = (0, common_1.round)(timeoutAt - dispatchedAt);
+            if (pending) {
+                if (pending.promise) {
+                    pending.promise.reject('Task timed out after 2m in WorkerDispatcherManager for method: ' + task.method);
+                }
+                _this._pendingTasks.delete(task.taskId);
+            }
+            var publicationInfo = _this.getPublicationLogInfo(task.method, task.params);
+            var logPayload = {
+                taskId: task.taskId,
+                method: task.method,
+                messageId: task.messageId,
+                queueWaitMs: queueWaitMs,
+                dispatchToTimeoutMs: dispatchToTimeoutMs,
+                userContext: task.userContext,
+                worker: {
+                    id: worker.id,
+                    index: worker.workerIndex,
+                    instance: worker.workerInstance
+                },
+                workers: _this._workers.map(function (a) { return a.activeTasks; })
+            };
+            if (publicationInfo) {
+                Object.assign(logPayload, publicationInfo);
+            }
+            _this.logWorkerEvent('workerTaskTimeout', logPayload);
+            worker.activeTasks = worker.activeTasks.filter(function (a) { return a.taskId !== task.taskId; });
+            _this._taskTimings.delete(task.taskId);
+            if (task.userContext && task.userContext.id_ws) {
+                var timeoutRes = {
+                    messageId: task.messageId,
+                    hasError: true,
+                    data: 'Task timed out after 2m in WorkerDispatcherManager for method: ' + task.method
+                };
+                var clientWS = _this._websocketManager.getWebSocket(task.userContext.id_ws);
+                if (clientWS) {
+                    _this._websocketManager.send(clientWS, timeoutRes);
+                }
+            }
+        }, method.timeoutOverride || (1000 * 60 * 2));
+        // If we already stored a promise for this task (from sendInternalPromise), add the timeout now
+        var existing = this._pendingTasks.get(task.taskId);
+        if (!existing) {
+            existing = { timeout: null, method: task.method };
+        }
+        else {
+            existing.method = task.method;
+        }
+        existing.timeout = timeoutHandle;
+        this._pendingTasks.set(task.taskId, existing);
+        try {
+            this.sendWorkerPayload(worker.ws, payload);
+        }
+        catch (err) {
+            console.error('Failed to send task to worker:', err);
+            clearTimeout(timeoutHandle);
+            worker.activeTasks = worker.activeTasks.filter(function (a) { return a.taskId !== task.taskId; });
+            // Put task back in front
+            this._taskQueue.unshift(task);
+            this.dispatchQueue();
+        }
+    };
+    WorkerDispatcherManager.prototype.logNoWorkerAvailability = function () {
+        var e_5, _a;
+        var _this = this;
+        if (!this.shouldDebug()) {
+            return;
+        }
+        var workers = this._workers.map(function (worker) { return ({
+            id: worker.id,
+            index: _this.normalizeWorkerIndex(worker.workerIndex),
+            instance: _this.normalizeWorkerIndex(worker.workerInstance),
+            activeTasks: worker.activeTasks.length
+        }); });
+        var _loop_1 = function (task) {
+            var method = this_1._methodManager.getMethod(task.method);
+            var targetWorkerIndex = this_1.normalizeWorkerIndex(method === null || method === void 0 ? void 0 : method.targetWorkerIndex);
+            var targetWorkerInstance = this_1.normalizeWorkerIndex(method === null || method === void 0 ? void 0 : method.targetWorkerInstance);
+            if (!targetWorkerIndex && !targetWorkerInstance) {
+                return "continue";
+            }
+            if (this_1._debugNoWorkerTaskIds.has(task.taskId)) {
+                return "continue";
+            }
+            var hasMatch = this_1._workers.some(function (worker) {
+                return (!targetWorkerIndex || _this.normalizeWorkerIndex(worker.workerIndex) === targetWorkerIndex) &&
+                    (!targetWorkerInstance || _this.normalizeWorkerIndex(worker.workerInstance) === targetWorkerInstance);
+            });
+            if (!hasMatch) {
+                this_1._debugNoWorkerTaskIds.add(task.taskId);
+                console.warn(new Date(), '[WorkerDispatcher] No matching worker for task', {
+                    taskId: task.taskId,
+                    method: task.method,
+                    targetWorkerIndex: targetWorkerIndex,
+                    targetWorkerInstance: targetWorkerInstance,
+                    workers: workers
+                });
+            }
+        };
+        var this_1 = this;
+        try {
+            for (var _b = __values(this._taskQueue), _c = _b.next(); !_c.done; _c = _b.next()) {
+                var task = _c.value;
+                _loop_1(task);
+            }
+        }
+        catch (e_5_1) { e_5 = { error: e_5_1 }; }
+        finally {
+            try {
+                if (_c && !_c.done && (_a = _b.return)) _a.call(_b);
+            }
+            finally { if (e_5) throw e_5.error; }
+        }
+    };
+    WorkerDispatcherManager.prototype.shouldDebug = function () {
+        var _a;
+        if (this._methodManager.getEnableDebug()) {
+            return true;
+        }
+        if (this._workerTaskDebug) {
+            return true;
+        }
+        return this.parseDebugFlag((_a = process.env.WORKER_TASK_DEBUG) !== null && _a !== void 0 ? _a : process.env.WORKER_DISPATCH_DEBUG);
+    };
+    WorkerDispatcherManager.prototype.logWorkerEvent = function (event, details) {
+        console.log(this.safeStringify(__assign({ ts: new Date().toISOString(), event: event }, details)));
+    };
+    WorkerDispatcherManager.prototype.safeStringify = function (value) {
+        try {
+            var seen_1 = new WeakSet();
+            return JSON.stringify(value, function (key, val) {
+                if (key === '__proto__') {
+                    return undefined;
+                }
+                if (typeof val === 'bigint') {
+                    return val.toString();
+                }
+                if (val instanceof Error) {
+                    return {
+                        name: val.name,
+                        message: val.message,
+                        stack: val.stack
+                    };
+                }
+                if (val && typeof val === 'object') {
+                    if (seen_1.has(val)) {
+                        return '[Circular]';
+                    }
+                    seen_1.add(val);
+                }
+                return val;
+            });
+        }
+        catch (err) {
+            return JSON.stringify({
+                error: 'safeStringifyFailed',
+                message: (err === null || err === void 0 ? void 0 : err.message) || String(err)
+            });
+        }
+    };
+    /**
+     * Handle messages coming back from a worker (like 'taskComplete').
+     */
+    WorkerDispatcherManager.prototype.handleWorkerMessage = function (workerId, rawMessage) {
+        var data;
+        try {
+            if (typeof rawMessage === 'string') {
+                data = JSON.parse(rawMessage, common_1.dateReviver);
+            }
+            else if (Buffer.isBuffer(rawMessage)) {
+                data = (0, msgpackr_1.unpack)(rawMessage);
+            }
+            else if (Array.isArray(rawMessage)) {
+                var chunks = rawMessage;
+                data = (0, msgpackr_1.unpack)(Buffer.concat(chunks));
+            }
+            else if (rawMessage instanceof ArrayBuffer) {
+                data = (0, msgpackr_1.unpack)(Buffer.from(rawMessage));
+            }
+            else if (ArrayBuffer.isView(rawMessage)) {
+                var view = rawMessage;
+                data = (0, msgpackr_1.unpack)(Buffer.from(view.buffer, view.byteOffset, view.byteLength));
+            }
+            else {
+                console.error('Unsupported worker message type received:', typeof rawMessage);
+                return;
+            }
+        }
+        catch (err) {
+            console.error('Failed to parse worker message:', err);
+            return;
+        }
+        if (data.type === 'taskComplete') {
+            var worker = this._workers.find(function (x) { return x.id === workerId; });
+            if (!worker) {
+                console.error('Unknown worker for taskComplete:', workerId);
+                return;
+            }
+            var taskId_1 = data.taskId, messageId = data.messageId, error = data.error, result = data.result;
+            var timing = this._taskTimings.get(taskId_1);
+            var methodName = timing === null || timing === void 0 ? void 0 : timing.method;
+            var isAiMethod = !!methodName && methodName.startsWith('ai');
+            var unpackedResult = result;
+            var pendingTask = this._pendingTasks.get(taskId_1);
+            var resolveRaw = !!(pendingTask === null || pendingTask === void 0 ? void 0 : pendingTask.resolveRaw);
+            if (!error && !resolveRaw && unpackedResult === null && data['packedResult']) {
+                try {
+                    unpackedResult = (0, msgpackr_1.unpack)(data['packedResult']);
+                }
+                catch (err) {
+                    console.error('Failed to unpack worker result', err);
+                }
+            }
+            worker.activeTasks = worker.activeTasks.filter(function (a) { return a.taskId !== taskId_1; });
+            this._taskTimings.delete(taskId_1);
+            if (this._aiWorkerDebug && isAiMethod) {
+                var packedBytes = data['packedResult'] && data['packedResult'].byteLength
+                    ? data['packedResult'].byteLength
+                    : null;
+                var resultBytes = null;
+                if (!packedBytes) {
+                    try {
+                        resultBytes = Buffer.byteLength(JSON.stringify(result));
+                    }
+                    catch (_a) {
+                        resultBytes = null;
+                    }
+                }
+                console.log(new Date(), '[AI Worker Debug] worker result', {
+                    taskId: taskId_1,
+                    method: methodName,
+                    messageId: messageId,
+                    hasError: !!error,
+                    packedBytes: packedBytes,
+                    resultBytes: resultBytes,
+                    workerIndex: (worker === null || worker === void 0 ? void 0 : worker.workerIndex) || null,
+                    workerInstance: (worker === null || worker === void 0 ? void 0 : worker.workerInstance) || null,
+                    queueDepth: this._taskQueue.length
+                });
+            }
+            if (pendingTask) {
+                if (pendingTask.promise) {
+                    if (error) {
+                        pendingTask.promise.reject(unpackedResult);
+                    }
+                    else {
+                        pendingTask.promise.resolve(resolveRaw ? data : unpackedResult);
+                    }
+                }
+                clearTimeout(pendingTask.timeout);
+                this._pendingTasks.delete(taskId_1);
+            }
+            // Look up original request if it came from a client
+            var clientReqId = this._clientRequests[taskId_1];
+            if (clientReqId) {
+                var res = {
+                    messageId: messageId,
+                    hasError: false,
+                    data: result
+                };
+                if (error) {
+                    res.hasError = true;
+                    res.data = result;
+                }
+                var clientReqWS = this._websocketManager.getWebSocket(clientReqId);
+                if (clientReqWS) {
+                    if (!error && data['packedResult']) {
+                        this._websocketManager.sendPackedBuffer(clientReqWS, messageId, false, data['packedResult'], data['encoding'] || 'msgpack', { passThrough: true });
+                    }
+                    else {
+                        this._websocketManager.send(clientReqWS, res);
+                    }
+                }
+                if (this._clientRequests[taskId_1]) {
+                    delete this._clientRequests[taskId_1];
+                }
+            }
+            // Try to dispatch more from the queue
+            if (this._taskQueue.length) {
+                this.dispatchQueue();
+            }
+        }
+    };
+    WorkerDispatcherManager.prototype.sendWorkerPayload = function (ws, payload) {
+        if (!ws) {
+            return;
+        }
+        if (ws.readyState !== WebSocket.OPEN) {
+            if (typeof payload !== 'string') {
+                var task_1 = payload;
+                this._taskQueue.unshift(task_1);
+                this.dispatchQueue();
+                this.disconnectWorker(ws['id_worker']);
+                ws.close();
+                var pendingTask = this._pendingTasks.get(task_1.taskId);
+                if (pendingTask) {
+                    clearTimeout(pendingTask.timeout);
+                    this._pendingTasks.delete(task_1.taskId);
+                    if (pendingTask.promise) {
+                        pendingTask.promise.reject('Worker socket not open.');
+                    }
+                }
+            }
+            else {
+                this.disconnectWorker(ws['id_worker']);
+                ws.close();
+            }
+            return;
+        }
+        if (typeof payload === 'string') {
+            try {
+                ws.send(payload);
+            }
+            catch (err) {
+                this.disconnectWorker(ws['id_worker']);
+                ws.close();
+                console.error('Failed to send worker payload:', err);
+            }
+            return;
+        }
+        var task = payload;
+        var payloadBuffer = (0, msgpackr_1.pack)(task);
+        try {
+            ws.send(payloadBuffer);
+        }
+        catch (err) {
+            this._taskQueue.unshift(task);
+            this.dispatchQueue();
+            if (this._methodManager.getEnableDebug()) {
+                console.log(new Date(), 'Sending to Server', task);
+            }
+            this.disconnectWorker(ws['id_worker']);
+            ws.close();
+            console.error('Failed to send worker response:', err);
+            var pendingTask = this._pendingTasks.get(task.taskId);
+            if (pendingTask) {
+                clearTimeout(pendingTask.timeout);
+                this._pendingTasks.delete(task.taskId);
+                if (pendingTask.promise) {
+                    pendingTask.promise.reject('Failed to send worker response: ' + (err === null || err === void 0 ? void 0 : err.toString()));
+                }
+            }
+        }
+    };
+    return WorkerDispatcherManager;
+}());
+exports.WorkerDispatcherManager = WorkerDispatcherManager;
+
+//# sourceMappingURL=worker-dispatcher.manager.js.map