@resolveio/server-lib 20.15.9 → 22.0.0

package/managers/worker-dispatcher.manager.js

@@ -33,13 +33,20 @@ var WorkerDispatcherManager = /** @class */ (function () {
         this._clientRequests = {};
         this._pendingTasks = new Map();
         this._debugNoWorkerTaskIds = new Set();
+        this._debugCodexTaskIds = new Set();
         this._taskTimings = new Map();
         this._dispatchRetryTimer = null;
         this._dispatchRetryDelayMs = 25;
+        this._nonCodexReserve = 0;
+        this._publicationWorkerIndexes = null;
+        this._publicationWorkerInstances = null;
+        this._codexWorkerIndexes = null;
+        this._codexWorkerInstances = null;
         this.MAX_CONCURRENCY = 10;
         this.WORKER_TASK_LOG_THRESHOLD_MS = 200;
         this.WORKER_PUBLICATION_SLOW_MS = 20000;
         this._workerTaskDebug = false;
+        this._aiWorkerDebug = false;
     }
     WorkerDispatcherManager.create = function (websocketManager, methodManager) {
         var workerDispatcherManager = new WorkerDispatcherManager();
@@ -47,14 +54,28 @@ var WorkerDispatcherManager = /** @class */ (function () {
         return workerDispatcherManager;
     };
     WorkerDispatcherManager.prototype.initialize = function (websocketManager, methodManager) {
-        var _a, _b;
+        var _a, _b, _c;
         this._websocketManager = websocketManager;
         this._methodManager = methodManager;
         this._workerTaskDebug = this.parseDebugFlag((_a = process.env.WORKER_TASK_DEBUG) !== null && _a !== void 0 ? _a : process.env.WORKER_DISPATCH_DEBUG);
+        this._aiWorkerDebug = this.parseDebugFlag(process.env.AI_ASSISTANT_WORKER_DEBUG);
         var slowPublicationMs = parseInt((_b = process.env.WORKER_PUBLICATION_SLOW_MS) !== null && _b !== void 0 ? _b : '', 10);
         if (!Number.isNaN(slowPublicationMs) && slowPublicationMs > 0) {
             this.WORKER_PUBLICATION_SLOW_MS = slowPublicationMs;
         }
+        this._nonCodexReserve = this.parsePositiveInt((_c = process.env.WORKER_NON_CODEX_RESERVE) !== null && _c !== void 0 ? _c : process.env.WORKER_RESERVED_SLOTS, 0);
+        this._publicationWorkerIndexes = this.parseWorkerSelector(process.env.PUBLICATION_WORKER_INDEX
+            || process.env.SUBSCRIPTION_WORKER_INDEX
+            || process.env.WORKER_PUBLICATION_INDEX);
+        this._publicationWorkerInstances = this.parseWorkerSelector(process.env.PUBLICATION_WORKER_INSTANCE
+            || process.env.SUBSCRIPTION_WORKER_INSTANCE
+            || process.env.WORKER_PUBLICATION_INSTANCE);
+        this._codexWorkerIndexes = this.parseWorkerSelector(process.env.AI_ASSISTANT_CODEX_WORKER_INDEX
+            || process.env.CODEX_WORKER_INDEX
+            || process.env.WORKER_CODEX_INDEX);
+        this._codexWorkerInstances = this.parseWorkerSelector(process.env.AI_ASSISTANT_CODEX_WORKER_INSTANCE
+            || process.env.CODEX_WORKER_INSTANCE
+            || process.env.WORKER_CODEX_INSTANCE);
     };
     WorkerDispatcherManager.prototype.parseDebugFlag = function (value) {
         if (value === true) {
@@ -72,6 +93,102 @@ var WorkerDispatcherManager = /** @class */ (function () {
         }
         return false;
     };
+    WorkerDispatcherManager.prototype.parsePositiveInt = function (value, fallback) {
+        var parsed = parseInt(value !== null && value !== void 0 ? value : '', 10);
+        if (Number.isNaN(parsed) || parsed < 0) {
+            return fallback;
+        }
+        return parsed;
+    };
+    WorkerDispatcherManager.prototype.parseWorkerSelector = function (value) {
+        var e_1, _a, e_2, _b, e_3, _c;
+        if (value === null || value === undefined) {
+            return null;
+        }
+        var parts = [];
+        if (Array.isArray(value)) {
+            try {
+                for (var value_1 = __values(value), value_1_1 = value_1.next(); !value_1_1.done; value_1_1 = value_1.next()) {
+                    var item = value_1_1.value;
+                    var entry = String(item !== null && item !== void 0 ? item : '').split(',');
+                    try {
+                        for (var entry_1 = (e_2 = void 0, __values(entry)), entry_1_1 = entry_1.next(); !entry_1_1.done; entry_1_1 = entry_1.next()) {
+                            var chunk = entry_1_1.value;
+                            var trimmed = String(chunk !== null && chunk !== void 0 ? chunk : '').trim();
+                            if (trimmed) {
+                                parts.push(trimmed);
+                            }
+                        }
+                    }
+                    catch (e_2_1) { e_2 = { error: e_2_1 }; }
+                    finally {
+                        try {
+                            if (entry_1_1 && !entry_1_1.done && (_b = entry_1.return)) _b.call(entry_1);
+                        }
+                        finally { if (e_2) throw e_2.error; }
+                    }
+                }
+            }
+            catch (e_1_1) { e_1 = { error: e_1_1 }; }
+            finally {
+                try {
+                    if (value_1_1 && !value_1_1.done && (_a = value_1.return)) _a.call(value_1);
+                }
+                finally { if (e_1) throw e_1.error; }
+            }
+        }
+        else {
+            var entry = String(value !== null && value !== void 0 ? value : '').split(',');
+            try {
+                for (var entry_2 = __values(entry), entry_2_1 = entry_2.next(); !entry_2_1.done; entry_2_1 = entry_2.next()) {
+                    var chunk = entry_2_1.value;
+                    var trimmed = String(chunk !== null && chunk !== void 0 ? chunk : '').trim();
+                    if (trimmed) {
+                        parts.push(trimmed);
+                    }
+                }
+            }
+            catch (e_3_1) { e_3 = { error: e_3_1 }; }
+            finally {
+                try {
+                    if (entry_2_1 && !entry_2_1.done && (_c = entry_2.return)) _c.call(entry_2);
+                }
+                finally { if (e_3) throw e_3.error; }
+            }
+        }
+        if (!parts.length) {
+            return null;
+        }
+        return new Set(parts);
+    };
+    WorkerDispatcherManager.prototype.selectorToArray = function (value) {
+        if (!value) {
+            return null;
+        }
+        return Array.from(value.values());
+    };
+    WorkerDispatcherManager.prototype.hasWorkerSelector = function (indexes, instances) {
+        return !!(indexes && indexes.size) || !!(instances && instances.size);
+    };
+    WorkerDispatcherManager.prototype.workerMatchesSelector = function (worker, indexes, instances) {
+        if (!this.hasWorkerSelector(indexes, instances)) {
+            return false;
+        }
+        var normalizedIndex = this.normalizeWorkerIndex(worker.workerIndex);
+        var normalizedInstance = this.normalizeWorkerIndex(worker.workerInstance);
+        var indexMatch = indexes ? (normalizedIndex ? indexes.has(normalizedIndex) : false) : true;
+        var instanceMatch = instances ? (normalizedInstance ? instances.has(normalizedInstance) : false) : true;
+        return indexMatch && instanceMatch;
+    };
+    WorkerDispatcherManager.prototype.resolveTaskQueue = function (methodName) {
+        var _a;
+        var method = (_a = this._methodManager) === null || _a === void 0 ? void 0 : _a.getMethod(methodName);
+        var queue = method === null || method === void 0 ? void 0 : method.workerQueue;
+        if (queue === 'publication' || queue === 'codex') {
+            return queue;
+        }
+        return 'default';
+    };
     WorkerDispatcherManager.prototype.summarizePublicationValue = function (value) {
         if (value === null || value === undefined) {
             return value;
@@ -374,41 +491,51 @@ var WorkerDispatcherManager = /** @class */ (function () {
      * The main loop that assigns tasks from _taskQueue to any worker that has capacity.
      */
     WorkerDispatcherManager.prototype.dispatchQueue = function () {
+        var e_4, _a;
+        var _this = this;
         if (!this._taskQueue.length) {
             return;
         }
         if (!this._workers.length) {
             return;
         }
+        var priorityMatchers = [
+            function (task) { return _this.resolveTaskQueue(task.method) === 'publication'; },
+            function (task) { return _this.resolveTaskQueue(task.method) === 'default'; },
+            function (task) { return _this.resolveTaskQueue(task.method) === 'codex'; }
+        ];
+        var tryAssignTask = function (matcher) {
+            for (var i = 0; i < _this._taskQueue.length; i++) {
+                var task = _this._taskQueue[i];
+                if (!matcher(task)) {
+                    continue;
+                }
+                var targetWorker = _this.findWorkerForTask(task);
+                if (targetWorker) {
+                    _this._taskQueue.splice(i, 1);
+                    _this.assignTaskToWorker(targetWorker, task);
+                    return true;
+                }
+            }
+            return false;
+        };
         while (this._taskQueue.length > 0) {
             var assigned = false;
-            var hasPublicationTask = this._taskQueue.some(function (task) { return task.method === 'runPublication'; });
-            if (hasPublicationTask) {
-                for (var i = 0; i < this._taskQueue.length; i++) {
-                    var task = this._taskQueue[i];
-                    if (task.method !== 'runPublication') {
-                        continue;
-                    }
-                    var targetWorker = this.findWorkerForTask(task);
-                    if (targetWorker) {
-                        this._taskQueue.splice(i, 1);
-                        this.assignTaskToWorker(targetWorker, task);
+            try {
+                for (var priorityMatchers_1 = (e_4 = void 0, __values(priorityMatchers)), priorityMatchers_1_1 = priorityMatchers_1.next(); !priorityMatchers_1_1.done; priorityMatchers_1_1 = priorityMatchers_1.next()) {
+                    var matcher = priorityMatchers_1_1.value;
+                    if (tryAssignTask(matcher)) {
                         assigned = true;
                         break;
                     }
                 }
             }
-            if (!assigned) {
-                for (var i = 0; i < this._taskQueue.length; i++) {
-                    var task = this._taskQueue[i];
-                    var targetWorker = this.findWorkerForTask(task);
-                    if (targetWorker) {
-                        this._taskQueue.splice(i, 1);
-                        this.assignTaskToWorker(targetWorker, task);
-                        assigned = true;
-                        break;
-                    }
+            catch (e_4_1) { e_4 = { error: e_4_1 }; }
+            finally {
+                try {
+                    if (priorityMatchers_1_1 && !priorityMatchers_1_1.done && (_a = priorityMatchers_1.return)) _a.call(priorityMatchers_1);
                 }
+                finally { if (e_4) throw e_4.error; }
             }
             if (!assigned) {
                 this.logNoWorkerAvailability();
@@ -437,21 +564,83 @@ var WorkerDispatcherManager = /** @class */ (function () {
     WorkerDispatcherManager.prototype.findWorkerForTask = function (task) {
         var _this = this;
         var method = this._methodManager.getMethod(task.method);
+        var taskQueue = this.resolveTaskQueue(task.method);
         var methodLimit = method && method.maxConcurrency && method.maxConcurrency > 0 ? method.maxConcurrency : null;
         var methodLimitPerInstance = method && method.maxConcurrencyPerInstance && method.maxConcurrencyPerInstance > 0
             ? method.maxConcurrencyPerInstance
             : null;
         var targetWorkerIndex = this.normalizeWorkerIndex(method === null || method === void 0 ? void 0 : method.targetWorkerIndex);
         var targetWorkerInstance = this.normalizeWorkerIndex(method === null || method === void 0 ? void 0 : method.targetWorkerInstance);
+        var hasExplicitTarget = !!targetWorkerIndex || !!targetWorkerInstance;
         var preferNonZero = !targetWorkerIndex;
+        var debugCodex = this._aiWorkerDebug && taskQueue === 'codex';
+        var debugState = debugCodex ? {
+            taskId: task.taskId,
+            method: task.method,
+            queueDepth: this._taskQueue.length,
+            targetWorkerIndex: targetWorkerIndex || null,
+            targetWorkerInstance: targetWorkerInstance || null,
+            workerCount: this._workers.length,
+            publicationSelector: {
+                indexes: this.selectorToArray(this._publicationWorkerIndexes),
+                instances: this.selectorToArray(this._publicationWorkerInstances)
+            },
+            codexSelector: {
+                indexes: this.selectorToArray(this._codexWorkerIndexes),
+                instances: this.selectorToArray(this._codexWorkerInstances)
+            },
+            maxConcurrency: this.MAX_CONCURRENCY,
+            nonCodexReserve: this._nonCodexReserve
+        } : null;
         var candidates = this._workers.filter(function (x) { return x.activeTasks.length < _this.MAX_CONCURRENCY; });
+        if (debugState) {
+            debugState.candidatesInitial = candidates.length;
+        }
+        var hasPublicationSelector = this.hasWorkerSelector(this._publicationWorkerIndexes, this._publicationWorkerInstances);
+        if (hasPublicationSelector && !hasExplicitTarget) {
+            if (taskQueue === 'publication') {
+                candidates = candidates.filter(function (worker) {
+                    return _this.workerMatchesSelector(worker, _this._publicationWorkerIndexes, _this._publicationWorkerInstances);
+                });
+            }
+            else {
+                candidates = candidates.filter(function (worker) {
+                    return !_this.workerMatchesSelector(worker, _this._publicationWorkerIndexes, _this._publicationWorkerInstances);
+                });
+            }
+        }
+        if (debugState) {
+            debugState.candidatesAfterPublicationFilter = candidates.length;
+            debugState.hasPublicationSelector = hasPublicationSelector;
+        }
+        var hasCodexSelector = this.hasWorkerSelector(this._codexWorkerIndexes, this._codexWorkerInstances);
+        if (hasCodexSelector) {
+            if (taskQueue === 'codex') {
+                candidates = candidates.filter(function (worker) {
+                    return _this.workerMatchesSelector(worker, _this._codexWorkerIndexes, _this._codexWorkerInstances);
+                });
+            }
+            else if (!hasExplicitTarget) {
+                candidates = candidates.filter(function (worker) {
+                    return !_this.workerMatchesSelector(worker, _this._codexWorkerIndexes, _this._codexWorkerInstances);
+                });
+            }
+        }
+        if (debugState) {
+            debugState.candidatesAfterCodexFilter = candidates.length;
+            debugState.hasCodexSelector = hasCodexSelector;
+        }
         if (targetWorkerIndex) {
             candidates = candidates.filter(function (worker) { return _this.normalizeWorkerIndex(worker.workerIndex) === targetWorkerIndex; });
         }
         if (targetWorkerInstance) {
             candidates = candidates.filter(function (worker) { return _this.normalizeWorkerIndex(worker.workerInstance) === targetWorkerInstance; });
         }
+        if (debugState) {
+            debugState.candidatesAfterTargetFilter = candidates.length;
+        }
         if (!candidates.length) {
+            this.logCodexSelection(debugState, 'noCandidates');
             return null;
         }
         var eligible = candidates.filter(function (worker) {
@@ -478,7 +667,21 @@ var WorkerDispatcherManager = /** @class */ (function () {
             });
             return current < methodLimit;
         });
+        if (debugState) {
+            debugState.eligibleAfterLimits = eligible.length;
+        }
+        if (!eligible.length) {
+            this.logCodexSelection(debugState, 'limitsExceeded');
+            return null;
+        }
+        if (taskQueue === 'codex' && this._nonCodexReserve > 0) {
+            eligible = eligible.filter(function (worker) { return (_this.MAX_CONCURRENCY - worker.activeTasks.length) > _this._nonCodexReserve; });
+        }
+        if (debugState) {
+            debugState.eligibleAfterReserve = eligible.length;
+        }
         if (!eligible.length) {
+            this.logCodexSelection(debugState, 'reserveBlocked');
             return null;
         }
         eligible.sort(function (x, y) {
@@ -504,7 +707,29 @@ var WorkerDispatcherManager = /** @class */ (function () {
             var instY = _this.normalizeWorkerIndex(y.workerInstance) || '';
             return instX.localeCompare(instY, undefined, { numeric: true, sensitivity: 'base' });
         });
-        return eligible[0];
+        var selected = eligible[0];
+        if (this._aiWorkerDebug && taskQueue === 'codex') {
+            console.log(new Date(), '[AI Worker Debug] codex worker selected', {
+                taskId: task.taskId,
+                method: task.method,
+                workerIndex: (selected === null || selected === void 0 ? void 0 : selected.workerIndex) || null,
+                workerInstance: (selected === null || selected === void 0 ? void 0 : selected.workerInstance) || null,
+                activeTasks: (selected === null || selected === void 0 ? void 0 : selected.activeTasks) ? selected.activeTasks.length : 0,
+                queueDepth: this._taskQueue.length
+            });
+        }
+        return selected;
+    };
+    WorkerDispatcherManager.prototype.logCodexSelection = function (debugState, reason) {
+        if (!debugState || !this._aiWorkerDebug) {
+            return;
+        }
+        var taskId = String(debugState.taskId || '');
+        if (!taskId || this._debugCodexTaskIds.has(taskId)) {
+            return;
+        }
+        this._debugCodexTaskIds.add(taskId);
+        console.warn(new Date(), '[AI Worker Debug] codex worker selection issue', __assign({ reason: reason }, debugState));
     };
     WorkerDispatcherManager.prototype.normalizeWorkerIndex = function (value) {
         if (value === null || value === undefined) {
@@ -643,7 +868,7 @@ var WorkerDispatcherManager = /** @class */ (function () {
         }
     };
     WorkerDispatcherManager.prototype.logNoWorkerAvailability = function () {
-        var e_1, _a;
+        var e_5, _a;
         var _this = this;
         if (!this.shouldDebug()) {
             return;
@@ -686,12 +911,12 @@ var WorkerDispatcherManager = /** @class */ (function () {
                 _loop_1(task);
             }
         }
-        catch (e_1_1) { e_1 = { error: e_1_1 }; }
+        catch (e_5_1) { e_5 = { error: e_5_1 }; }
         finally {
             try {
                 if (_c && !_c.done && (_a = _b.return)) _a.call(_b);
             }
-            finally { if (e_1) throw e_1.error; }
+            finally { if (e_5) throw e_5.error; }
         }
     };
     WorkerDispatcherManager.prototype.shouldDebug = function () {