@resolveio/server-lib 22.3.34 → 22.3.35

package/src/managers/worker-dispatcher.manager.ts

@@ -0,0 +1,1360 @@
+import * as WebSocket from 'ws';
+import { ServerResponseModel, TaskPayload, TaskResponse } from '../models/server-message.model';
+import { dateReviver, objectIdHexString, round } from '../util/common';
+import { MethodManager } from './method.manager';
+import { WebSocketManager } from './websocket.manager';
+import { pack, unpack } from 'msgpackr';
+
+export interface WorkerConnection {
+	id: string;
+	ws: WebSocket.WebSocket;
+	activeTasks: {
+		taskId: string;
+		weight: number;
+		method?: string;
+		startedAt?: number;
+		messageId?: number;
+		userContext?: { id_user?: string; user?: string; id_ws?: string };
+	}[];
+	workerIndex?: string;
+	workerInstance?: string;
+}
+
+interface PendingTask {
+	timeout: NodeJS.Timeout;
+	promise?: {
+		// eslint-disable-next-line no-unused-vars
+		resolve: (value: any) => void;
+		// eslint-disable-next-line no-unused-vars
+		reject: (reason?: any) => void;
+	};
+	method?: string;
+	resolveRaw?: boolean;
+}
+
+// eslint-disable-next-line no-unused-vars
+type TaskMatcher = (task: TaskPayload) => boolean;
+
+export class WorkerDispatcherManager {
+	private _websocketManager: WebSocketManager;
+	private _methodManager: MethodManager;
+	private _workers: WorkerConnection[] = [];
+	private _taskQueue: TaskPayload[] = [];
+	private _clientRequests: { [taskId: string]: string } = {};
+	private _pendingTasks: Map<string, PendingTask> = new Map();
+	private _debugNoWorkerTaskIds: Set<string> = new Set();
+	private _debugCodexTaskIds: Set<string> = new Set();
+	private _taskTimings: Map<string, {
+		queuedAt: number;
+		dispatchedAt?: number;
+		method?: string;
+		messageId?: number;
+		userContext?: { id_user?: string; user?: string; id_ws?: string };
+		params?: any[];
+	}> = new Map();
+	private _dispatchRetryTimer: NodeJS.Timeout | null = null;
+	private _dispatchRetryDelayMs = 25;
+	private _nonCodexReserve = 0;
+	private _publicationWorkerIndexes: Set<string> | null = null;
+	private _publicationWorkerInstances: Set<string> | null = null;
+	private _codexWorkerIndexes: Set<string> | null = null;
+	private _codexWorkerInstances: Set<string> | null = null;
+
+	private MAX_CONCURRENCY = 10;
+	private WORKER_TASK_LOG_THRESHOLD_MS = 200;
+	private WORKER_PUBLICATION_SLOW_MS = 20000;
+	private _workerTaskDebug = false;
+	private _aiWorkerDebug = false;
+
+	constructor() {}
+
+	static create(websocketManager: WebSocketManager, methodManager: MethodManager) {
+		const workerDispatcherManager = new WorkerDispatcherManager();
+		workerDispatcherManager.initialize(websocketManager, methodManager);
+		return workerDispatcherManager;
+	}
+
+	public initialize(websocketManager: WebSocketManager, methodManager: MethodManager) {
+		this._websocketManager = websocketManager;
+		this._methodManager = methodManager;
+		this._workerTaskDebug = this.parseDebugFlag(process.env.WORKER_TASK_DEBUG ?? process.env.WORKER_DISPATCH_DEBUG);
+		this._aiWorkerDebug = this.parseDebugFlag(process.env.AI_ASSISTANT_WORKER_DEBUG);
+		const slowPublicationMs = parseInt(process.env.WORKER_PUBLICATION_SLOW_MS ?? '', 10);
+		if (!Number.isNaN(slowPublicationMs) && slowPublicationMs > 0) {
+			this.WORKER_PUBLICATION_SLOW_MS = slowPublicationMs;
+		}
+		this._nonCodexReserve = this.parsePositiveInt(
+			process.env.WORKER_NON_CODEX_RESERVE ?? process.env.WORKER_RESERVED_SLOTS,
+			0
+		);
+		this._publicationWorkerIndexes = this.parseWorkerSelector(
+			process.env.PUBLICATION_WORKER_INDEX
+			|| process.env.SUBSCRIPTION_WORKER_INDEX
+			|| process.env.WORKER_PUBLICATION_INDEX
+		);
+		this._publicationWorkerInstances = this.parseWorkerSelector(
+			process.env.PUBLICATION_WORKER_INSTANCE
+			|| process.env.SUBSCRIPTION_WORKER_INSTANCE
+			|| process.env.WORKER_PUBLICATION_INSTANCE
+		);
+		this._codexWorkerIndexes = this.parseWorkerSelector(
+			process.env.AI_ASSISTANT_CODEX_WORKER_INDEX
+			|| process.env.CODEX_WORKER_INDEX
+			|| process.env.WORKER_CODEX_INDEX
+		);
+		this._codexWorkerInstances = this.parseWorkerSelector(
+			process.env.AI_ASSISTANT_CODEX_WORKER_INSTANCE
+			|| process.env.CODEX_WORKER_INSTANCE
+			|| process.env.WORKER_CODEX_INSTANCE
+		);
+	}
+
+	private parseDebugFlag(value: any): boolean {
+		if (value === true) {
+			return true;
+		}
+
+		if (value === false || value === null || value === undefined) {
+			return false;
+		}
+
+		if (typeof value === 'number') {
+			return value === 1;
+		}
+
+		if (typeof value === 'string') {
+			const normalized = value.trim().toLowerCase();
+			return ['1', 'true', 'yes', 'y', 'on'].includes(normalized);
+		}
+
+		return false;
+	}
+
+	private parsePositiveInt(value: any, fallback: number): number {
+		const parsed = parseInt(value ?? '', 10);
+		if (Number.isNaN(parsed) || parsed < 0) {
+			return fallback;
+		}
+		return parsed;
+	}
+
+	private parseWorkerSelector(value: any): Set<string> | null {
+		if (value === null || value === undefined) {
+			return null;
+		}
+
+		const parts: string[] = [];
+
+		if (Array.isArray(value)) {
+			for (const item of value) {
+				const entry = String(item ?? '').split(',');
+				for (const chunk of entry) {
+					const trimmed = String(chunk ?? '').trim();
+					if (trimmed) {
+						parts.push(trimmed);
+					}
+				}
+			}
+		}
+		else {
+			const entry = String(value ?? '').split(',');
+			for (const chunk of entry) {
+				const trimmed = String(chunk ?? '').trim();
+				if (trimmed) {
+					parts.push(trimmed);
+				}
+			}
+		}
+
+		if (!parts.length) {
+			return null;
+		}
+
+		return new Set(parts);
+	}
+
+	private selectorToArray(value: Set<string> | null): string[] | null {
+		if (!value) {
+			return null;
+		}
+		return Array.from(value.values());
+	}
+
+	private hasWorkerSelector(indexes: Set<string> | null, instances: Set<string> | null): boolean {
+		return !!(indexes && indexes.size) || !!(instances && instances.size);
+	}
+
+	private workerMatchesSelector(worker: WorkerConnection, indexes: Set<string> | null, instances: Set<string> | null): boolean {
+		if (!this.hasWorkerSelector(indexes, instances)) {
+			return false;
+		}
+
+		const normalizedIndex = this.normalizeWorkerIndex(worker.workerIndex);
+		const normalizedInstance = this.normalizeWorkerIndex(worker.workerInstance);
+		const indexMatch = indexes ? (normalizedIndex ? indexes.has(normalizedIndex) : false) : true;
+		const instanceMatch = instances ? (normalizedInstance ? instances.has(normalizedInstance) : false) : true;
+
+		return indexMatch && instanceMatch;
+	}
+
+	private resolveTaskQueue(methodName: string): 'publication' | 'codex' | 'default' {
+		const method = this._methodManager?.getMethod(methodName);
+		const queue = method?.workerQueue;
+		if (queue === 'publication' || queue === 'codex') {
+			return queue;
+		}
+		return 'default';
+	}
+
+	private summarizePublicationValue(value: any): any {
+		if (value === null || value === undefined) {
+			return value;
+		}
+
+		const valueType = typeof value;
+		if (valueType === 'string' || valueType === 'number' || valueType === 'boolean') {
+			return value;
+		}
+
+		if (Array.isArray(value)) {
+			return `[Array(${value.length})]`;
+		}
+
+		if (valueType === 'object') {
+			return `[Object keys=${Object.keys(value).length}]`;
+		}
+
+		return `[${valueType}]`;
+	}
+
+	private getPublicationLogInfo(method?: string, params?: any[]): Record<string, any> | null {
+		if (method !== 'runPublication' || !Array.isArray(params)) {
+			return null;
+		}
+
+		const publication = typeof params[0] === 'string' ? params[0] : null;
+		const subscriptionData = Array.isArray(params[1]) ? params[1] : [];
+		const userId = typeof params[2] === 'string' ? params[2] : null;
+		const subscriptionDataPreview = subscriptionData.slice(0, 3).map((item) => this.summarizePublicationValue(item));
+
+		return {
+			publication,
+			subscriptionDataCount: subscriptionData.length,
+			subscriptionDataPreview,
+			userId
+		};
+	}
+
+	public isSafeShutdown() {
+		return !this._taskQueue.length && !this._pendingTasks.size;
+	}
+
+	public addWorker(ws: WebSocket.WebSocket) {
+		this._workers.push({
+			id: ws['id_worker'],
+			ws: ws,
+			activeTasks: [],
+			workerIndex: ws['workerIndex'],
+			workerInstance: ws['workerInstance']
+		});
+
+		setInterval(() => {
+			if (this._methodManager.getEnableDebug()) {
+				console.log(JSON.stringify(this._workers.map(a => {
+					return {
+						id: a.id,
+						activeTasks: a.activeTasks
+					}
+				}), null, 2));
+			}
+		}, 5000);
+
+		if (this._taskQueue.length) {
+			this.dispatchQueue();
+		}
+	}
+
+	private failActiveWorkerTask(task: WorkerConnection['activeTasks'][number], reason: string) {
+		if (!task || !task.taskId) {
+			return;
+		}
+
+		let pendingTask = this._pendingTasks.get(task.taskId);
+		if (pendingTask) {
+			clearTimeout(pendingTask.timeout);
+			if (pendingTask.promise) {
+				pendingTask.promise.reject(reason);
+			}
+			this._pendingTasks.delete(task.taskId);
+		}
+
+		this._taskTimings.delete(task.taskId);
+
+		if (this._clientRequests[task.taskId]) {
+			delete this._clientRequests[task.taskId];
+		}
+
+		if (!task.userContext?.id_ws || task.messageId === undefined || task.messageId === null) {
+			return;
+		}
+
+		let clientWS = this._websocketManager ? this._websocketManager.getWebSocket(task.userContext.id_ws) : null;
+		if (!clientWS) {
+			return;
+		}
+
+		let response: ServerResponseModel = {
+			messageId: task.messageId,
+			hasError: true,
+			data: reason
+		};
+
+		this._websocketManager.send(clientWS, response);
+	}
+
+	public disconnectWorker(workerId: string, reason = 'Worker disconnected.') {
+		let worker = this._workers.find(w => w.id === workerId);
+		this._workers = this._workers.filter(w => w.id !== workerId);
+
+		if (!worker || !worker.activeTasks?.length) {
+			return;
+		}
+
+		let activeTasks = [...worker.activeTasks];
+		worker.activeTasks = [];
+
+		for (const activeTask of activeTasks) {
+			this.failActiveWorkerTask(activeTask, reason);
+		}
+	}
+
+	public hasWorkers() {
+		return this._workers.length > 0;
+	}
+
+	public getQueueSnapshot() {
+		const activeTasks = this._workers.reduce((total, worker) => total + (worker.activeTasks ? worker.activeTasks.length : 0), 0);
+		return {
+			queueDepth: this._taskQueue.length,
+			pendingTasks: this._pendingTasks.size,
+			workerCount: this._workers.length,
+			activeTasks,
+			workers: this._workers.map(worker => ({
+				id: worker.id,
+				index: worker.workerIndex,
+				instance: worker.workerInstance,
+				activeTasks: worker.activeTasks ? worker.activeTasks.length : 0
+			}))
+		};
+	}
+
+	public hasWorkersForMethod(methodName: string) {
+		if (!this._workers.length) {
+			return false;
+		}
+
+		const method = this._methodManager.getMethod(methodName);
+		const taskQueue = this.resolveTaskQueue(methodName);
+		const targetWorkerIndex = this.normalizeWorkerIndex(method?.targetWorkerIndex);
+		const targetWorkerInstance = this.normalizeWorkerIndex(method?.targetWorkerInstance);
+		const hasExplicitTarget = !!targetWorkerIndex || !!targetWorkerInstance;
+
+		let candidates = this.filterGeneralPurposeWorkers([...this._workers], hasExplicitTarget);
+
+		const hasPublicationSelector = this.hasWorkerSelector(this._publicationWorkerIndexes, this._publicationWorkerInstances);
+		if (hasPublicationSelector && !hasExplicitTarget) {
+			if (taskQueue === 'publication') {
+				candidates = candidates.filter(worker =>
+					this.workerMatchesSelector(worker, this._publicationWorkerIndexes, this._publicationWorkerInstances)
+				);
+			}
+			else {
+				candidates = candidates.filter(worker =>
+					!this.workerMatchesSelector(worker, this._publicationWorkerIndexes, this._publicationWorkerInstances)
+				);
+			}
+		}
+
+		const hasCodexSelector = this.hasWorkerSelector(this._codexWorkerIndexes, this._codexWorkerInstances);
+		if (hasCodexSelector) {
+			if (taskQueue === 'codex') {
+				candidates = candidates.filter(worker =>
+					this.workerMatchesSelector(worker, this._codexWorkerIndexes, this._codexWorkerInstances)
+				);
+			}
+			else if (!hasExplicitTarget) {
+				candidates = candidates.filter(worker =>
+					!this.workerMatchesSelector(worker, this._codexWorkerIndexes, this._codexWorkerInstances)
+				);
+			}
+		}
+
+		if (targetWorkerIndex) {
+			candidates = candidates.filter(worker => this.normalizeWorkerIndex(worker.workerIndex) === targetWorkerIndex);
+		}
+		if (targetWorkerInstance) {
+			candidates = candidates.filter(worker => this.normalizeWorkerIndex(worker.workerInstance) === targetWorkerInstance);
+		}
+
+		return candidates.length > 0;
+	}
+
+	/**
+	 * Add a new task to our in-memory queue and try to dispatch.
+	 */
+	public sendClientTask(messageId: number, method: string, params: any[], userContext?: { id_user?: string; user?: string; id_ws?: string }) {
+		let taskId = 'task-' + objectIdHexString();
+		const queuedAt = Date.now();
+
+		this._taskQueue.push({
+			type: 'task',
+			taskId,
+			messageId,
+			method,
+			params,
+			userContext,
+			queuedAt
+		});
+
+		this._taskTimings.set(taskId, {
+			queuedAt,
+			method,
+			messageId,
+			userContext,
+			params
+		});
+
+		if (userContext && userContext.id_ws) {
+			this._clientRequests[taskId] = userContext.id_ws;
+		}
+
+		if (this.shouldDebug()) {
+			const publicationInfo = this.getPublicationLogInfo(method, params);
+			const logPayload: Record<string, any> = {
+				taskId,
+				method,
+				messageId,
+				priority: method === 'runPublication' ? 'publication' : 'method',
+				queueDepth: this._taskQueue.length,
+				userContext
+			};
+			if (publicationInfo) {
+				Object.assign(logPayload, publicationInfo);
+			}
+			this.logWorkerEvent('workerTaskQueued', logPayload);
+		}
+
+		this.dispatchQueue();
+	}
+
+	/**
+	 * Same as sendInternalTask but returns a Promise so you can `await` it.
+	 */
+	public sendInternalPromise(method: string, params: any[] = []): Promise<any> {
+		if (!Array.isArray(params)) {
+			params = [params];
+		}
+
+		// eslint-disable-next-line no-restricted-syntax
+		return new Promise((resolve, reject) => {
+			let taskId = 'task-' + objectIdHexString();
+			const queuedAt = Date.now();
+
+			this._taskQueue.push({
+				type: 'task',
+				taskId,
+				method,
+				params,
+				messageId: 0,
+				userContext: {
+					user: 'Internal System'
+				},
+				queuedAt
+			});
+
+			this._taskTimings.set(taskId, {
+				queuedAt,
+				method,
+				messageId: 0,
+				userContext: { user: 'Internal System' },
+				params
+			});
+
+			this._pendingTasks.set(taskId, {
+				timeout: null,
+				promise: { resolve, reject }
+			});
+
+			if (this._methodManager.getEnableDebug()) {
+				console.log(new Date(), 'Send Internal Promise', this._taskQueue);
+			}
+
+			if (this.shouldDebug()) {
+				const publicationInfo = this.getPublicationLogInfo(method, params);
+				const logPayload: Record<string, any> = {
+					taskId,
+					method,
+					messageId: 0,
+					priority: method === 'runPublication' ? 'publication' : 'method',
+					queueDepth: this._taskQueue.length,
+					userContext: { user: 'Internal System' }
+				};
+				if (publicationInfo) {
+					Object.assign(logPayload, publicationInfo);
+				}
+				this.logWorkerEvent('workerTaskQueued', logPayload);
+			}
+
+			this.dispatchQueue();
+		});
+	}
+
+	/**
+	 * Send a task internally and resolve with the raw TaskResponse (includes packedResult/meta).
+	 */
+	public sendInternalPromiseRaw(method: string, params: any[] = []): Promise<TaskResponse> {
+		if (!Array.isArray(params)) {
+			params = [params];
+		}
+
+		// eslint-disable-next-line no-restricted-syntax
+		return new Promise((resolve, reject) => {
+			let taskId = 'task-' + objectIdHexString();
+			const queuedAt = Date.now();
+
+			this._taskQueue.push({
+				type: 'task',
+				taskId,
+				method,
+				params,
+				messageId: 0,
+				userContext: {
+					user: 'Internal System'
+				},
+				queuedAt
+			});
+
+			this._taskTimings.set(taskId, {
+				queuedAt,
+				method,
+				messageId: 0,
+				userContext: { user: 'Internal System' },
+				params
+			});
+
+			this._pendingTasks.set(taskId, {
+				timeout: null,
+				promise: { resolve, reject },
+				resolveRaw: true
+			});
+
+			if (this._methodManager.getEnableDebug()) {
+				console.log(new Date(), 'Send Internal Raw Promise', this._taskQueue);
+			}
+
+			if (this.shouldDebug()) {
+				const publicationInfo = this.getPublicationLogInfo(method, params);
+				const logPayload: Record<string, any> = {
+					taskId,
+					method,
+					messageId: 0,
+					priority: method === 'runPublication' ? 'publication' : 'method',
+					queueDepth: this._taskQueue.length,
+					userContext: { user: 'Internal System' }
+				};
+				if (publicationInfo) {
+					Object.assign(logPayload, publicationInfo);
+				}
+				this.logWorkerEvent('workerTaskQueued', logPayload);
+			}
+
+			this.dispatchQueue();
+		});
+	}
+
+	/**
+	 * Send a task internally without returning a promise.
+	 */
+	public sendInternalTask(method: string, params = []) {
+		if (!Array.isArray(params)) {
+			params = [params];
+		}
+
+		let taskId = 'task-' + objectIdHexString();
+		const queuedAt = Date.now();
+
+		this._taskQueue.push({
+			type: 'task',
+			taskId,
+			method,
+			params,
+			messageId: 0,
+			userContext: {
+				user: 'Internal System'
+			},
+			queuedAt
+		});
+
+		this._taskTimings.set(taskId, {
+			queuedAt,
+			method,
+			messageId: 0,
+			userContext: { user: 'Internal System' },
+			params
+		});
+
+		if (this._methodManager.getEnableDebug()) {
+			console.log(new Date(), 'Send Internal Task', this._taskQueue);
+		}
+
+		if (this.shouldDebug()) {
+			const publicationInfo = this.getPublicationLogInfo(method, params);
+			const logPayload: Record<string, any> = {
+				taskId,
+				method,
+				messageId: 0,
+				priority: method === 'runPublication' ? 'publication' : 'method',
+				queueDepth: this._taskQueue.length,
+				userContext: { user: 'Internal System' }
+			};
+			if (publicationInfo) {
+				Object.assign(logPayload, publicationInfo);
+			}
+			this.logWorkerEvent('workerTaskQueued', logPayload);
+		}
+
+		this.dispatchQueue();
+	}
+
+	/**
+	 * The main loop that assigns tasks from _taskQueue to any worker that has capacity.
+	 */
+	private dispatchQueue() {
+		if (!this._taskQueue.length) {
+			return;
+		}
+		if (!this._workers.length) {
+			return;
+		}
+
+		const priorityMatchers: TaskMatcher[] = [
+			(task) => this.resolveTaskQueue(task.method) === 'publication',
+			(task) => this.resolveTaskQueue(task.method) === 'default',
+			(task) => this.resolveTaskQueue(task.method) === 'codex'
+		];
+
+		const tryAssignTask = (matcher: TaskMatcher): boolean => {
+			for (let i = 0; i < this._taskQueue.length; i++) {
+				let task = this._taskQueue[i];
+				if (!matcher(task)) {
+					continue;
+				}
+				let targetWorker = this.findWorkerForTask(task);
+
+				if (targetWorker) {
+					this._taskQueue.splice(i, 1);
+					this.assignTaskToWorker(targetWorker, task);
+					return true;
+				}
+			}
+			return false;
+		};
+
+		while (this._taskQueue.length > 0) {
+			let assigned = false;
+			for (const matcher of priorityMatchers) {
+				if (tryAssignTask(matcher)) {
+					assigned = true;
+					break;
+				}
+			}
+
+			if (!assigned) {
+				this.logNoWorkerAvailability();
+				this.scheduleDispatchRetry();
+
+				if (this._methodManager.getEnableDebug()) {
+					console.log(new Date(), 'No Worker Available', JSON.stringify(this._workers, null, 2));
+				}
+
+				return; // nothing can run right now due to per-worker limits
+			}
+		}
+	}
+
+	private scheduleDispatchRetry(delayMs: number = this._dispatchRetryDelayMs) {
+		if (this._dispatchRetryTimer) {
+			return;
+		}
+
+		this._dispatchRetryTimer = setTimeout(() => {
+			this._dispatchRetryTimer = null;
+			this.dispatchQueue();
+		}, delayMs);
+	}
+
+	/**
+	 * Returns the worker with the fewest activeTasks that is under maxConcurrency and per-method limit.
+	 */
+	private findWorkerForTask(task: TaskPayload): WorkerConnection | null {
+		let method = this._methodManager.getMethod(task.method);
+		const taskQueue = this.resolveTaskQueue(task.method);
+		let methodLimit = method && method.maxConcurrency && method.maxConcurrency > 0 ? method.maxConcurrency : null;
+		let methodLimitPerInstance = method && method.maxConcurrencyPerInstance && method.maxConcurrencyPerInstance > 0
+			? method.maxConcurrencyPerInstance
+			: null;
+		let targetWorkerIndex = this.normalizeWorkerIndex(method?.targetWorkerIndex);
+		let targetWorkerInstance = this.normalizeWorkerIndex(method?.targetWorkerInstance);
+		const hasExplicitTarget = !!targetWorkerIndex || !!targetWorkerInstance;
+		const preferNonZero = !targetWorkerIndex;
+		const debugCodex = this._aiWorkerDebug && taskQueue === 'codex';
+		const debugState: Record<string, any> | null = debugCodex ? {
+			taskId: task.taskId,
+			method: task.method,
+			queueDepth: this._taskQueue.length,
+			targetWorkerIndex: targetWorkerIndex || null,
+			targetWorkerInstance: targetWorkerInstance || null,
+			workerCount: this._workers.length,
+			publicationSelector: {
+				indexes: this.selectorToArray(this._publicationWorkerIndexes),
+				instances: this.selectorToArray(this._publicationWorkerInstances)
+			},
+			codexSelector: {
+				indexes: this.selectorToArray(this._codexWorkerIndexes),
+				instances: this.selectorToArray(this._codexWorkerInstances)
+			},
+			maxConcurrency: this.MAX_CONCURRENCY,
+			nonCodexReserve: this._nonCodexReserve
+		} : null;
+
+		let candidates = this._workers.filter(x => x.activeTasks.length < this.MAX_CONCURRENCY);
+		candidates = this.filterGeneralPurposeWorkers(candidates, hasExplicitTarget);
+		if (debugState) {
+			debugState.candidatesInitial = candidates.length;
+		}
+		const hasPublicationSelector = this.hasWorkerSelector(this._publicationWorkerIndexes, this._publicationWorkerInstances);
+		if (hasPublicationSelector && !hasExplicitTarget) {
+			if (taskQueue === 'publication') {
+				candidates = candidates.filter(worker =>
+					this.workerMatchesSelector(worker, this._publicationWorkerIndexes, this._publicationWorkerInstances)
+				);
+			}
+			else {
+				candidates = candidates.filter(worker =>
+					!this.workerMatchesSelector(worker, this._publicationWorkerIndexes, this._publicationWorkerInstances)
+				);
+			}
+		}
+		if (debugState) {
+			debugState.candidatesAfterPublicationFilter = candidates.length;
+			debugState.hasPublicationSelector = hasPublicationSelector;
+		}
+
+		const hasCodexSelector = this.hasWorkerSelector(this._codexWorkerIndexes, this._codexWorkerInstances);
+		if (hasCodexSelector) {
+			if (taskQueue === 'codex') {
+				candidates = candidates.filter(worker =>
+					this.workerMatchesSelector(worker, this._codexWorkerIndexes, this._codexWorkerInstances)
+				);
+			}
+			else if (!hasExplicitTarget) {
+				candidates = candidates.filter(worker =>
+					!this.workerMatchesSelector(worker, this._codexWorkerIndexes, this._codexWorkerInstances)
+				);
+			}
+		}
+		if (debugState) {
+			debugState.candidatesAfterCodexFilter = candidates.length;
+			debugState.hasCodexSelector = hasCodexSelector;
+		}
+		if (targetWorkerIndex) {
+			candidates = candidates.filter(worker => this.normalizeWorkerIndex(worker.workerIndex) === targetWorkerIndex);
+		}
+		if (targetWorkerInstance) {
+			candidates = candidates.filter(worker => this.normalizeWorkerIndex(worker.workerInstance) === targetWorkerInstance);
+		}
+		if (debugState) {
+			debugState.candidatesAfterTargetFilter = candidates.length;
+		}
+		if (!candidates.length) {
+			this.logCodexSelection(debugState, 'noCandidates');
+			return null;
+		}
+
+		let eligible = candidates.filter((worker) => {
+			if (!methodLimit) {
+				if (!methodLimitPerInstance) {
+					return true;
+				}
+			}
+
+			if (methodLimitPerInstance) {
+				const currentInstance = worker.activeTasks.filter(a => a.method === task.method).length;
+				if (currentInstance >= methodLimitPerInstance) {
+					return false;
+				}
+			}
+
+			if (!methodLimit) {
+				return true;
+			}
+
+			let idx = worker.workerIndex || 'unknown';
+			let current = 0;
+
+			this._workers.forEach((w) => {
+				if ((w.workerIndex || 'unknown') === idx) {
+					current += w.activeTasks.filter(a => a.method === task.method).length;
+				}
+			});
+
+			return current < methodLimit;
+		});
+
+		if (debugState) {
+			debugState.eligibleAfterLimits = eligible.length;
+		}
+		if (!eligible.length) {
+			this.logCodexSelection(debugState, 'limitsExceeded');
+			return null;
+		}
+
+		if (taskQueue === 'codex' && this._nonCodexReserve > 0) {
+			eligible = eligible.filter(worker => (this.MAX_CONCURRENCY - worker.activeTasks.length) > this._nonCodexReserve);
+		}
+
+		if (debugState) {
+			debugState.eligibleAfterReserve = eligible.length;
+		}
+		if (!eligible.length) {
+			this.logCodexSelection(debugState, 'reserveBlocked');
+			return null;
+		}
+
+		eligible.sort((x, y) => {
+			const totalX = this.getWorkerLoad(x);
+			const totalY = this.getWorkerLoad(y);
+			if (totalX !== totalY) {
+				return totalX - totalY;
+			}
+
+			if (preferNonZero) {
+				const xZero = this.isWorkerZero(x) ? 1 : 0;
+				const yZero = this.isWorkerZero(y) ? 1 : 0;
+				if (xZero !== yZero) {
+					return xZero - yZero;
+				}
+			}
+
+			const idxX = this.normalizeWorkerIndex(x.workerIndex) || '';
+			const idxY = this.normalizeWorkerIndex(y.workerIndex) || '';
+			const idxCompare = idxX.localeCompare(idxY, undefined, { numeric: true, sensitivity: 'base' });
+			if (idxCompare !== 0) {
+				return idxCompare;
+			}
+
+			const instX = this.normalizeWorkerIndex(x.workerInstance) || '';
+			const instY = this.normalizeWorkerIndex(y.workerInstance) || '';
+			return instX.localeCompare(instY, undefined, { numeric: true, sensitivity: 'base' });
+		});
+
+			const selected = eligible[0];
+			if (this._aiWorkerDebug && taskQueue === 'codex') {
+				console.log(new Date(), '[AI Worker Debug] AI worker selected', {
+					taskId: task.taskId,
+					method: task.method,
+					workerIndex: selected?.workerIndex || null,
+					workerInstance: selected?.workerInstance || null,
+				activeTasks: selected?.activeTasks ? selected.activeTasks.length : 0,
+				queueDepth: this._taskQueue.length
+			});
+		}
+		return selected;
+	}
+
+	private logCodexSelection(debugState: Record<string, any> | null, reason: string): void {
+		if (!debugState || !this._aiWorkerDebug) {
+			return;
+		}
+		const taskId = String(debugState.taskId || '');
+		if (!taskId || this._debugCodexTaskIds.has(taskId)) {
+			return;
+			}
+			this._debugCodexTaskIds.add(taskId);
+			console.warn(new Date(), '[AI Worker Debug] AI worker selection issue', {
+				reason,
+				...debugState
+			});
+		}
+
+	private normalizeWorkerIndex(value?: string | number | null): string | null {
+		if (value === null || value === undefined) {
+			return null;
+		}
+
+		const normalized = String(value).trim();
+		return normalized.length ? normalized : null;
+	}
+
+	private isGeneralPurposeWorker(worker: WorkerConnection): boolean {
+		const normalizedIndex = this.normalizeWorkerIndex(worker?.workerIndex);
+		const normalizedInstance = this.normalizeWorkerIndex(worker?.workerInstance);
+		if (!normalizedIndex || !normalizedInstance) {
+			return false;
+		}
+
+		return /^[0-9]+$/.test(normalizedIndex) && /^[0-9]+$/.test(normalizedInstance);
+	}
+
+	private filterGeneralPurposeWorkers(candidates: WorkerConnection[], hasExplicitTarget: boolean): WorkerConnection[] {
+		if (hasExplicitTarget) {
+			return candidates;
+		}
+
+		return candidates.filter(worker => this.isGeneralPurposeWorker(worker));
+	}
+
+	private getWorkerLoad(worker: WorkerConnection): number {
+		return worker.activeTasks.reduce((sum, task) => sum + task.weight, 0);
+	}
+
+	private isWorkerZero(worker: WorkerConnection): boolean {
+		return this.normalizeWorkerIndex(worker.workerIndex) === '0';
+	}
+
+	private assignTaskToWorker(worker: WorkerConnection, task: TaskPayload) {
+		let method = this._methodManager.getMethod(task.method);
+
+		if (!method) {
+			console.error('Failed to send task to worker - Could not find method:', task.method);
+			this._taskTimings.delete(task.taskId);
+			return;
+		}
+
+		const dispatchedAt = Date.now();
+		const timing = this._taskTimings.get(task.taskId) || { queuedAt: dispatchedAt };
+		timing.dispatchedAt = dispatchedAt;
+		timing.method = task.method;
+		timing.messageId = task.messageId;
+		timing.userContext = task.userContext;
+		if (!timing.params) {
+			timing.params = task.params;
+		}
+		this._taskTimings.set(task.taskId, timing);
+
+		worker.activeTasks.push({
+			taskId: task.taskId,
+			weight: method && method.workerTaskWeight ? method.workerTaskWeight : 1,
+			method: task.method,
+			startedAt: dispatchedAt,
+			messageId: task.messageId,
+			userContext: task.userContext
+		});
+		this._debugNoWorkerTaskIds.delete(task.taskId);
+
+		let payload: TaskPayload = {
+			type: 'task',
+			taskId: task.taskId,
+			messageId: task.messageId,
+			method: task.method,
+			params: task.params,
+			userContext: task.userContext || {},
+			queuedAt: timing.queuedAt,
+			dispatchedAt
+		};
+
+		if (this.shouldDebug()) {
+			const publicationInfo = this.getPublicationLogInfo(task.method, task.params);
+			const logPayload: Record<string, any> = {
+				taskId: task.taskId,
+				method: task.method,
+				messageId: task.messageId,
+				priority: task.method === 'runPublication' ? 'publication' : 'method',
+				worker: {
+					id: worker.id,
+					index: worker.workerIndex,
+					instance: worker.workerInstance
+				},
+				queueDepth: this._taskQueue.length
+			};
+			if (publicationInfo) {
+				Object.assign(logPayload, publicationInfo);
+			}
+			this.logWorkerEvent('workerTaskAssigned', logPayload);
+		}
+
+		let timeoutHandle = setTimeout(() => {
+			const timeoutAt = Date.now();
+			const queuedAt = timing.queuedAt;
+			const queueWaitMs = queuedAt ? round(dispatchedAt - queuedAt) : null;
+			const dispatchToTimeoutMs = round(timeoutAt - dispatchedAt);
+			const timeoutMessage = 'Task timed out after 2m in WorkerDispatcherManager for method: ' + task.method;
+
+			const publicationInfo = this.getPublicationLogInfo(task.method, task.params);
+			const logPayload: Record<string, any> = {
+				taskId: task.taskId,
+				method: task.method,
+				messageId: task.messageId,
+				queueWaitMs,
+				dispatchToTimeoutMs,
+				userContext: task.userContext,
+				worker: {
+					id: worker.id,
+					index: worker.workerIndex,
+					instance: worker.workerInstance
+				},
+				workers: this._workers.map(a => a.activeTasks)
+			};
+			if (publicationInfo) {
+				Object.assign(logPayload, publicationInfo);
+			}
+			this.logWorkerEvent('workerTaskTimeout', logPayload);
+
+			this.disconnectWorker(worker.id, timeoutMessage);
+			if (worker.ws && worker.ws.readyState !== WebSocket.CLOSED) {
+				try {
+					worker.ws.close();
+				}
+				catch {
+					try {
+						worker.ws.terminate();
+					}
+					catch {
+						// Ignore socket cleanup errors after timeout.
+					}
+				}
+			}
+		}, method.timeoutOverride || (1000 * 60 * 2));
+
+		// If we already stored a promise for this task (from sendInternalPromise), add the timeout now
+		let existing = this._pendingTasks.get(task.taskId);
+		if (!existing) {
+			existing = { timeout: null, method: task.method };
+		}
+		else {
+			existing.method = task.method;
+		}
+
+		existing.timeout = timeoutHandle;
+		this._pendingTasks.set(task.taskId, existing);
+
+		try {
+			this.sendWorkerPayload(worker.ws, payload);
+		}
+		catch (err) {
+			console.error('Failed to send task to worker:', err);
+
+			clearTimeout(timeoutHandle);
+			worker.activeTasks = worker.activeTasks.filter(a => a.taskId !== task.taskId);
+
+			// Put task back in front
+			this._taskQueue.unshift(task);
+			this.dispatchQueue();
+		}
+	}
+
+	private logNoWorkerAvailability() {
+		if (!this.shouldDebug()) {
+			return;
+		}
+
+		const workers = this._workers.map(worker => ({
+			id: worker.id,
+			index: this.normalizeWorkerIndex(worker.workerIndex),
+			instance: this.normalizeWorkerIndex(worker.workerInstance),
+			activeTasks: worker.activeTasks.length
+		}));
+
+		for (const task of this._taskQueue) {
+			const method = this._methodManager.getMethod(task.method);
+			const targetWorkerIndex = this.normalizeWorkerIndex(method?.targetWorkerIndex);
+			const targetWorkerInstance = this.normalizeWorkerIndex(method?.targetWorkerInstance);
+
+			if (!targetWorkerIndex && !targetWorkerInstance) {
+				continue;
+			}
+
+			if (this._debugNoWorkerTaskIds.has(task.taskId)) {
+				continue;
+			}
+
+			const hasMatch = this._workers.some(worker =>
+				(!targetWorkerIndex || this.normalizeWorkerIndex(worker.workerIndex) === targetWorkerIndex) &&
+				(!targetWorkerInstance || this.normalizeWorkerIndex(worker.workerInstance) === targetWorkerInstance)
+			);
+
+			if (!hasMatch) {
+				this._debugNoWorkerTaskIds.add(task.taskId);
+				console.warn(new Date(), '[WorkerDispatcher] No matching worker for task', {
+					taskId: task.taskId,
+					method: task.method,
+					targetWorkerIndex,
+					targetWorkerInstance,
+					workers
+				});
+			}
+		}
+	}
+
+	private shouldDebug(): boolean {
+		if (this._methodManager.getEnableDebug()) {
+			return true;
+		}
+
+		if (this._workerTaskDebug) {
+			return true;
+		}
+
+		return this.parseDebugFlag(process.env.WORKER_TASK_DEBUG ?? process.env.WORKER_DISPATCH_DEBUG);
+	}
+
+	private logWorkerEvent(event: string, details: Record<string, any>) {
+		console.log(this.safeStringify({
+			ts: new Date().toISOString(),
+			event,
+			...details
+		}));
+	}
+
+	private safeStringify(value: any): string {
+		try {
+			const seen = new WeakSet();
+			return JSON.stringify(value, (key, val) => {
+				if (key === '__proto__') {
+					return undefined;
+				}
+				if (typeof val === 'bigint') {
+					return val.toString();
+				}
+				if (val instanceof Error) {
+					return {
+						name: val.name,
+						message: val.message,
+						stack: val.stack
+					};
+				}
+				if (val && typeof val === 'object') {
+					if (seen.has(val)) {
+						return '[Circular]';
+					}
+					seen.add(val);
+				}
+				return val;
+			});
+		}
+		catch (err) {
+			return JSON.stringify({
+				error: 'safeStringifyFailed',
+				message: err?.message || String(err)
+			});
+		}
+	}
+
+	/**
+	 * Handle messages coming back from a worker (like 'taskComplete').
+	 */
+	public handleWorkerMessage(workerId: string, rawMessage: WebSocket.RawData) {
+		let data: TaskResponse;
+
+		try {
+			if (typeof rawMessage === 'string') {
+				data = JSON.parse(rawMessage, dateReviver);
+			}
+			else if (Buffer.isBuffer(rawMessage)) {
+				data = unpack(rawMessage);
+			}
+			else if (Array.isArray(rawMessage)) {
+				const chunks = rawMessage as unknown as ReadonlyArray<Uint8Array>;
+				data = unpack(Buffer.concat(chunks));
+			}
+			else if (rawMessage instanceof ArrayBuffer) {
+				data = unpack(Buffer.from(rawMessage));
+			}
+			else if (ArrayBuffer.isView(rawMessage)) {
+				const view = rawMessage as NodeJS.ArrayBufferView;
+				data = unpack(Buffer.from(view.buffer, view.byteOffset, view.byteLength));
+			}
+			else {
+				console.error('Unsupported worker message type received:', typeof rawMessage);
+				return;
+			}
+		}
+		catch (err) {
+			console.error('Failed to parse worker message:', err);
+			return;
+		}
+
+		if (data.type === 'taskComplete') {
+			let worker = this._workers.find(x => x.id === workerId);
+
+			if (!worker) {
+				console.error('Unknown worker for taskComplete:', workerId);
+				return;
+			}
+
+			let { taskId, messageId, error, result } = data;
+			const timing = this._taskTimings.get(taskId);
+			const methodName = timing?.method;
+			const isAiMethod = !!methodName && methodName.startsWith('ai');
+			let unpackedResult = result;
+			let pendingTask = this._pendingTasks.get(taskId);
+			const resolveRaw = !!pendingTask?.resolveRaw;
+
+			if (!error && !resolveRaw && unpackedResult === null && data['packedResult']) {
+				try {
+					unpackedResult = unpack(data['packedResult']);
+				}
+				catch (err) {
+					console.error('Failed to unpack worker result', err);
+				}
+			}
+			worker.activeTasks = worker.activeTasks.filter(a => a.taskId !== taskId);
+			this._taskTimings.delete(taskId);
+
+			if (this._aiWorkerDebug && isAiMethod) {
+				const packedBytes = data['packedResult'] && data['packedResult'].byteLength
+					? data['packedResult'].byteLength
+					: null;
+				let resultBytes: number | null = null;
+				if (!packedBytes) {
+					try {
+						resultBytes = Buffer.byteLength(JSON.stringify(result));
+					}
+					catch {
+						resultBytes = null;
+					}
+				}
+				console.log(new Date(), '[AI Worker Debug] worker result', {
+					taskId,
+					method: methodName,
+					messageId,
+					hasError: !!error,
+					packedBytes,
+					resultBytes,
+					workerIndex: worker?.workerIndex || null,
+					workerInstance: worker?.workerInstance || null,
+					queueDepth: this._taskQueue.length
+				});
+			}
+
+
+			if (pendingTask) {
+				if (pendingTask.promise) {
+					if (error) {
+						pendingTask.promise.reject(unpackedResult);
+					}
+					else {
+						pendingTask.promise.resolve(resolveRaw ? data : unpackedResult);
+					}
+				}
+
+				clearTimeout(pendingTask.timeout);
+				this._pendingTasks.delete(taskId);
+			}
+
+			// Look up original request if it came from a client
+			let clientReqId = this._clientRequests[taskId];
+
+			if (clientReqId) {
+				let res: ServerResponseModel = {
+					messageId: messageId,
+					hasError: false,
+					data: result
+				};
+
+				if (error) {
+					res.hasError = true;
+					res.data = result;
+				}
+
+				let clientReqWS = this._websocketManager.getWebSocket(clientReqId);
+
+				if (clientReqWS) {
+					if (!error && data['packedResult']) {
+						this._websocketManager.sendPackedBuffer(
+							clientReqWS,
+							messageId,
+							false,
+							data['packedResult'],
+							data['encoding'] || 'msgpack',
+							{ passThrough: true }
+						);
+					}
+					else {
+						this._websocketManager.send(clientReqWS, res);
+					}
+				}
+
+				if (this._clientRequests[taskId]) {
+					delete this._clientRequests[taskId];
+				}
+			}
+
+			// Try to dispatch more from the queue
+			if (this._taskQueue.length) {
+				this.dispatchQueue();
+			}
+		}
+	}
+
+	public sendWorkerPayload(ws: WebSocket.WebSocket, payload: TaskPayload | string) {
+		if (!ws) {
+			return;
+		}
+
+		if (ws.readyState !== WebSocket.OPEN) {
+			if (typeof payload !== 'string') {
+				const task = payload;
+				this.disconnectWorker(ws['id_worker'], 'Worker socket not open for method: ' + task.method);
+				try {
+					ws.close();
+				}
+				catch {}
+			}
+			else {
+				this.disconnectWorker(ws['id_worker'], 'Worker socket not open.');
+				try {
+					ws.close();
+				}
+				catch {}
+			}
+
+			return;
+		}
+
+		if (typeof payload === 'string') {
+			try {
+				ws.send(payload);
+			}
+			catch (err) {
+				this.disconnectWorker(ws['id_worker'], 'Failed to send worker payload: ' + err?.toString());
+				try {
+					ws.close();
+				}
+				catch {}
+				console.error('Failed to send worker payload:', err);
+			}
+
+			return;
+		}
+
+		const task = payload;
+		const payloadBuffer = pack(task);
+
+		try {
+			ws.send(payloadBuffer);
+		}
+		catch (err) {
+			if (this._methodManager.getEnableDebug()) {
+				console.log(new Date(), 'Sending to Server', task);
+			}
+
+			this.disconnectWorker(ws['id_worker'], 'Failed to send worker response for method: ' + task.method);
+			try {
+				ws.close();
+			}
+			catch {}
+			console.error('Failed to send worker response:', err);
+		}
+	}
+}