@resolveio/server-lib 22.3.11 → 22.3.13

package/src/managers/worker-dispatcher.manager.ts

@@ -1,1360 +0,0 @@
-import * as WebSocket from 'ws';
-import { ServerResponseModel, TaskPayload, TaskResponse } from '../models/server-message.model';
-import { dateReviver, objectIdHexString, round } from '../util/common';
-import { MethodManager } from './method.manager';
-import { WebSocketManager } from './websocket.manager';
-import { pack, unpack } from 'msgpackr';
-
-export interface WorkerConnection {
-	id: string;
-	ws: WebSocket.WebSocket;
-	activeTasks: {
-		taskId: string;
-		weight: number;
-		method?: string;
-		startedAt?: number;
-		messageId?: number;
-		userContext?: { id_user?: string; user?: string; id_ws?: string };
-	}[];
-	workerIndex?: string;
-	workerInstance?: string;
-}
-
-interface PendingTask {
-	timeout: NodeJS.Timeout;
-	promise?: {
-		// eslint-disable-next-line no-unused-vars
-		resolve: (value: any) => void;
-		// eslint-disable-next-line no-unused-vars
-		reject: (reason?: any) => void;
-	};
-	method?: string;
-	resolveRaw?: boolean;
-}
-
-// eslint-disable-next-line no-unused-vars
-type TaskMatcher = (task: TaskPayload) => boolean;
-
-export class WorkerDispatcherManager {
-	private _websocketManager: WebSocketManager;
-	private _methodManager: MethodManager;
-	private _workers: WorkerConnection[] = [];
-	private _taskQueue: TaskPayload[] = [];
-	private _clientRequests: { [taskId: string]: string } = {};
-	private _pendingTasks: Map<string, PendingTask> = new Map();
-	private _debugNoWorkerTaskIds: Set<string> = new Set();
-	private _debugCodexTaskIds: Set<string> = new Set();
-	private _taskTimings: Map<string, {
-		queuedAt: number;
-		dispatchedAt?: number;
-		method?: string;
-		messageId?: number;
-		userContext?: { id_user?: string; user?: string; id_ws?: string };
-		params?: any[];
-	}> = new Map();
-	private _dispatchRetryTimer: NodeJS.Timeout | null = null;
-	private _dispatchRetryDelayMs = 25;
-	private _nonCodexReserve = 0;
-	private _publicationWorkerIndexes: Set<string> | null = null;
-	private _publicationWorkerInstances: Set<string> | null = null;
-	private _codexWorkerIndexes: Set<string> | null = null;
-	private _codexWorkerInstances: Set<string> | null = null;
-
-	private MAX_CONCURRENCY = 10;
-	private WORKER_TASK_LOG_THRESHOLD_MS = 200;
-	private WORKER_PUBLICATION_SLOW_MS = 20000;
-	private _workerTaskDebug = false;
-	private _aiWorkerDebug = false;
-
-	constructor() {}
-
-	static create(websocketManager: WebSocketManager, methodManager: MethodManager) {
-		const workerDispatcherManager = new WorkerDispatcherManager();
-		workerDispatcherManager.initialize(websocketManager, methodManager);
-		return workerDispatcherManager;
-	}
-
-	public initialize(websocketManager: WebSocketManager, methodManager: MethodManager) {
-		this._websocketManager = websocketManager;
-		this._methodManager = methodManager;
-		this._workerTaskDebug = this.parseDebugFlag(process.env.WORKER_TASK_DEBUG ?? process.env.WORKER_DISPATCH_DEBUG);
-		this._aiWorkerDebug = this.parseDebugFlag(process.env.AI_ASSISTANT_WORKER_DEBUG);
-		const slowPublicationMs = parseInt(process.env.WORKER_PUBLICATION_SLOW_MS ?? '', 10);
-		if (!Number.isNaN(slowPublicationMs) && slowPublicationMs > 0) {
-			this.WORKER_PUBLICATION_SLOW_MS = slowPublicationMs;
-		}
-		this._nonCodexReserve = this.parsePositiveInt(
-			process.env.WORKER_NON_CODEX_RESERVE ?? process.env.WORKER_RESERVED_SLOTS,
-			0
-		);
-		this._publicationWorkerIndexes = this.parseWorkerSelector(
-			process.env.PUBLICATION_WORKER_INDEX
-			|| process.env.SUBSCRIPTION_WORKER_INDEX
-			|| process.env.WORKER_PUBLICATION_INDEX
-		);
-		this._publicationWorkerInstances = this.parseWorkerSelector(
-			process.env.PUBLICATION_WORKER_INSTANCE
-			|| process.env.SUBSCRIPTION_WORKER_INSTANCE
-			|| process.env.WORKER_PUBLICATION_INSTANCE
-		);
-		this._codexWorkerIndexes = this.parseWorkerSelector(
-			process.env.AI_ASSISTANT_CODEX_WORKER_INDEX
-			|| process.env.CODEX_WORKER_INDEX
-			|| process.env.WORKER_CODEX_INDEX
-		);
-		this._codexWorkerInstances = this.parseWorkerSelector(
-			process.env.AI_ASSISTANT_CODEX_WORKER_INSTANCE
-			|| process.env.CODEX_WORKER_INSTANCE
-			|| process.env.WORKER_CODEX_INSTANCE
-		);
-	}
-
-	private parseDebugFlag(value: any): boolean {
-		if (value === true) {
-			return true;
-		}
-
-		if (value === false || value === null || value === undefined) {
-			return false;
-		}
-
-		if (typeof value === 'number') {
-			return value === 1;
-		}
-
-		if (typeof value === 'string') {
-			const normalized = value.trim().toLowerCase();
-			return ['1', 'true', 'yes', 'y', 'on'].includes(normalized);
-		}
-
-		return false;
-	}
-
-	private parsePositiveInt(value: any, fallback: number): number {
-		const parsed = parseInt(value ?? '', 10);
-		if (Number.isNaN(parsed) || parsed < 0) {
-			return fallback;
-		}
-		return parsed;
-	}
-
-	private parseWorkerSelector(value: any): Set<string> | null {
-		if (value === null || value === undefined) {
-			return null;
-		}
-
-		const parts: string[] = [];
-
-		if (Array.isArray(value)) {
-			for (const item of value) {
-				const entry = String(item ?? '').split(',');
-				for (const chunk of entry) {
-					const trimmed = String(chunk ?? '').trim();
-					if (trimmed) {
-						parts.push(trimmed);
-					}
-				}
-			}
-		}
-		else {
-			const entry = String(value ?? '').split(',');
-			for (const chunk of entry) {
-				const trimmed = String(chunk ?? '').trim();
-				if (trimmed) {
-					parts.push(trimmed);
-				}
-			}
-		}
-
-		if (!parts.length) {
-			return null;
-		}
-
-		return new Set(parts);
-	}
-
-	private selectorToArray(value: Set<string> | null): string[] | null {
-		if (!value) {
-			return null;
-		}
-		return Array.from(value.values());
-	}
-
-	private hasWorkerSelector(indexes: Set<string> | null, instances: Set<string> | null): boolean {
-		return !!(indexes && indexes.size) || !!(instances && instances.size);
-	}
-
-	private workerMatchesSelector(worker: WorkerConnection, indexes: Set<string> | null, instances: Set<string> | null): boolean {
-		if (!this.hasWorkerSelector(indexes, instances)) {
-			return false;
-		}
-
-		const normalizedIndex = this.normalizeWorkerIndex(worker.workerIndex);
-		const normalizedInstance = this.normalizeWorkerIndex(worker.workerInstance);
-		const indexMatch = indexes ? (normalizedIndex ? indexes.has(normalizedIndex) : false) : true;
-		const instanceMatch = instances ? (normalizedInstance ? instances.has(normalizedInstance) : false) : true;
-
-		return indexMatch && instanceMatch;
-	}
-
-	private resolveTaskQueue(methodName: string): 'publication' | 'codex' | 'default' {
-		const method = this._methodManager?.getMethod(methodName);
-		const queue = method?.workerQueue;
-		if (queue === 'publication' || queue === 'codex') {
-			return queue;
-		}
-		return 'default';
-	}
-
-	private summarizePublicationValue(value: any): any {
-		if (value === null || value === undefined) {
-			return value;
-		}
-
-		const valueType = typeof value;
-		if (valueType === 'string' || valueType === 'number' || valueType === 'boolean') {
-			return value;
-		}
-
-		if (Array.isArray(value)) {
-			return `[Array(${value.length})]`;
-		}
-
-		if (valueType === 'object') {
-			return `[Object keys=${Object.keys(value).length}]`;
-		}
-
-		return `[${valueType}]`;
-	}
-
-	private getPublicationLogInfo(method?: string, params?: any[]): Record<string, any> | null {
-		if (method !== 'runPublication' || !Array.isArray(params)) {
-			return null;
-		}
-
-		const publication = typeof params[0] === 'string' ? params[0] : null;
-		const subscriptionData = Array.isArray(params[1]) ? params[1] : [];
-		const userId = typeof params[2] === 'string' ? params[2] : null;
-		const subscriptionDataPreview = subscriptionData.slice(0, 3).map((item) => this.summarizePublicationValue(item));
-
-		return {
-			publication,
-			subscriptionDataCount: subscriptionData.length,
-			subscriptionDataPreview,
-			userId
-		};
-	}
-
-	public isSafeShutdown() {
-		return !this._taskQueue.length && !this._pendingTasks.size;
-	}
-
-	public addWorker(ws: WebSocket.WebSocket) {
-		this._workers.push({
-			id: ws['id_worker'],
-			ws: ws,
-			activeTasks: [],
-			workerIndex: ws['workerIndex'],
-			workerInstance: ws['workerInstance']
-		});
-
-		setInterval(() => {
-			if (this._methodManager.getEnableDebug()) {
-				console.log(JSON.stringify(this._workers.map(a => {
-					return {
-						id: a.id,
-						activeTasks: a.activeTasks
-					}
-				}), null, 2));
-			}
-		}, 5000);
-
-		if (this._taskQueue.length) {
-			this.dispatchQueue();
-		}
-	}
-
-	private failActiveWorkerTask(task: WorkerConnection['activeTasks'][number], reason: string) {
-		if (!task || !task.taskId) {
-			return;
-		}
-
-		let pendingTask = this._pendingTasks.get(task.taskId);
-		if (pendingTask) {
-			clearTimeout(pendingTask.timeout);
-			if (pendingTask.promise) {
-				pendingTask.promise.reject(reason);
-			}
-			this._pendingTasks.delete(task.taskId);
-		}
-
-		this._taskTimings.delete(task.taskId);
-
-		if (this._clientRequests[task.taskId]) {
-			delete this._clientRequests[task.taskId];
-		}
-
-		if (!task.userContext?.id_ws || task.messageId === undefined || task.messageId === null) {
-			return;
-		}
-
-		let clientWS = this._websocketManager ? this._websocketManager.getWebSocket(task.userContext.id_ws) : null;
-		if (!clientWS) {
-			return;
-		}
-
-		let response: ServerResponseModel = {
-			messageId: task.messageId,
-			hasError: true,
-			data: reason
-		};
-
-		this._websocketManager.send(clientWS, response);
-	}
-
-	public disconnectWorker(workerId: string, reason = 'Worker disconnected.') {
-		let worker = this._workers.find(w => w.id === workerId);
-		this._workers = this._workers.filter(w => w.id !== workerId);
-
-		if (!worker || !worker.activeTasks?.length) {
-			return;
-		}
-
-		let activeTasks = [...worker.activeTasks];
-		worker.activeTasks = [];
-
-		for (const activeTask of activeTasks) {
-			this.failActiveWorkerTask(activeTask, reason);
-		}
-	}
-
-	public hasWorkers() {
-		return this._workers.length > 0;
-	}
-
-	public getQueueSnapshot() {
-		const activeTasks = this._workers.reduce((total, worker) => total + (worker.activeTasks ? worker.activeTasks.length : 0), 0);
-		return {
-			queueDepth: this._taskQueue.length,
-			pendingTasks: this._pendingTasks.size,
-			workerCount: this._workers.length,
-			activeTasks,
-			workers: this._workers.map(worker => ({
-				id: worker.id,
-				index: worker.workerIndex,
-				instance: worker.workerInstance,
-				activeTasks: worker.activeTasks ? worker.activeTasks.length : 0
-			}))
-		};
-	}
-
-	public hasWorkersForMethod(methodName: string) {
-		if (!this._workers.length) {
-			return false;
-		}
-
-		const method = this._methodManager.getMethod(methodName);
-		const taskQueue = this.resolveTaskQueue(methodName);
-		const targetWorkerIndex = this.normalizeWorkerIndex(method?.targetWorkerIndex);
-		const targetWorkerInstance = this.normalizeWorkerIndex(method?.targetWorkerInstance);
-		const hasExplicitTarget = !!targetWorkerIndex || !!targetWorkerInstance;
-
-		let candidates = this.filterGeneralPurposeWorkers([...this._workers], hasExplicitTarget);
-
-		const hasPublicationSelector = this.hasWorkerSelector(this._publicationWorkerIndexes, this._publicationWorkerInstances);
-		if (hasPublicationSelector && !hasExplicitTarget) {
-			if (taskQueue === 'publication') {
-				candidates = candidates.filter(worker =>
-					this.workerMatchesSelector(worker, this._publicationWorkerIndexes, this._publicationWorkerInstances)
-				);
-			}
-			else {
-				candidates = candidates.filter(worker =>
-					!this.workerMatchesSelector(worker, this._publicationWorkerIndexes, this._publicationWorkerInstances)
-				);
-			}
-		}
-
-		const hasCodexSelector = this.hasWorkerSelector(this._codexWorkerIndexes, this._codexWorkerInstances);
-		if (hasCodexSelector) {
-			if (taskQueue === 'codex') {
-				candidates = candidates.filter(worker =>
-					this.workerMatchesSelector(worker, this._codexWorkerIndexes, this._codexWorkerInstances)
-				);
-			}
-			else if (!hasExplicitTarget) {
-				candidates = candidates.filter(worker =>
-					!this.workerMatchesSelector(worker, this._codexWorkerIndexes, this._codexWorkerInstances)
-				);
-			}
-		}
-
-		if (targetWorkerIndex) {
-			candidates = candidates.filter(worker => this.normalizeWorkerIndex(worker.workerIndex) === targetWorkerIndex);
-		}
-		if (targetWorkerInstance) {
-			candidates = candidates.filter(worker => this.normalizeWorkerIndex(worker.workerInstance) === targetWorkerInstance);
-		}
-
-		return candidates.length > 0;
-	}
-
-	/**
-	 * Add a new task to our in-memory queue and try to dispatch.
-	 */
-	public sendClientTask(messageId: number, method: string, params: any[], userContext?: { id_user?: string; user?: string; id_ws?: string }) {
-		let taskId = 'task-' + objectIdHexString();
-		const queuedAt = Date.now();
-
-		this._taskQueue.push({
-			type: 'task',
-			taskId,
-			messageId,
-			method,
-			params,
-			userContext,
-			queuedAt
-		});
-
-		this._taskTimings.set(taskId, {
-			queuedAt,
-			method,
-			messageId,
-			userContext,
-			params
-		});
-
-		if (userContext && userContext.id_ws) {
-			this._clientRequests[taskId] = userContext.id_ws;
-		}
-
-		if (this.shouldDebug()) {
-			const publicationInfo = this.getPublicationLogInfo(method, params);
-			const logPayload: Record<string, any> = {
-				taskId,
-				method,
-				messageId,
-				priority: method === 'runPublication' ? 'publication' : 'method',
-				queueDepth: this._taskQueue.length,
-				userContext
-			};
-			if (publicationInfo) {
-				Object.assign(logPayload, publicationInfo);
-			}
-			this.logWorkerEvent('workerTaskQueued', logPayload);
-		}
-
-		this.dispatchQueue();
-	}
-
-	/**
-	 * Same as sendInternalTask but returns a Promise so you can `await` it.
-	 */
-	public sendInternalPromise(method: string, params: any[] = []): Promise<any> {
-		if (!Array.isArray(params)) {
-			params = [params];
-		}
-
-		// eslint-disable-next-line no-restricted-syntax
-		return new Promise((resolve, reject) => {
-			let taskId = 'task-' + objectIdHexString();
-			const queuedAt = Date.now();
-
-			this._taskQueue.push({
-				type: 'task',
-				taskId,
-				method,
-				params,
-				messageId: 0,
-				userContext: {
-					user: 'Internal System'
-				},
-				queuedAt
-			});
-
-			this._taskTimings.set(taskId, {
-				queuedAt,
-				method,
-				messageId: 0,
-				userContext: { user: 'Internal System' },
-				params
-			});
-
-			this._pendingTasks.set(taskId, {
-				timeout: null,
-				promise: { resolve, reject }
-			});
-
-			if (this._methodManager.getEnableDebug()) {
-				console.log(new Date(), 'Send Internal Promise', this._taskQueue);
-			}
-
-			if (this.shouldDebug()) {
-				const publicationInfo = this.getPublicationLogInfo(method, params);
-				const logPayload: Record<string, any> = {
-					taskId,
-					method,
-					messageId: 0,
-					priority: method === 'runPublication' ? 'publication' : 'method',
-					queueDepth: this._taskQueue.length,
-					userContext: { user: 'Internal System' }
-				};
-				if (publicationInfo) {
-					Object.assign(logPayload, publicationInfo);
-				}
-				this.logWorkerEvent('workerTaskQueued', logPayload);
-			}
-
-			this.dispatchQueue();
-		});
-	}
-
-	/**
-	 * Send a task internally and resolve with the raw TaskResponse (includes packedResult/meta).
-	 */
-	public sendInternalPromiseRaw(method: string, params: any[] = []): Promise<TaskResponse> {
-		if (!Array.isArray(params)) {
-			params = [params];
-		}
-
-		// eslint-disable-next-line no-restricted-syntax
-		return new Promise((resolve, reject) => {
-			let taskId = 'task-' + objectIdHexString();
-			const queuedAt = Date.now();
-
-			this._taskQueue.push({
-				type: 'task',
-				taskId,
-				method,
-				params,
-				messageId: 0,
-				userContext: {
-					user: 'Internal System'
-				},
-				queuedAt
-			});
-
-			this._taskTimings.set(taskId, {
-				queuedAt,
-				method,
-				messageId: 0,
-				userContext: { user: 'Internal System' },
-				params
-			});
-
-			this._pendingTasks.set(taskId, {
-				timeout: null,
-				promise: { resolve, reject },
-				resolveRaw: true
-			});
-
-			if (this._methodManager.getEnableDebug()) {
-				console.log(new Date(), 'Send Internal Raw Promise', this._taskQueue);
-			}
-
-			if (this.shouldDebug()) {
-				const publicationInfo = this.getPublicationLogInfo(method, params);
-				const logPayload: Record<string, any> = {
-					taskId,
-					method,
-					messageId: 0,
-					priority: method === 'runPublication' ? 'publication' : 'method',
-					queueDepth: this._taskQueue.length,
-					userContext: { user: 'Internal System' }
-				};
-				if (publicationInfo) {
-					Object.assign(logPayload, publicationInfo);
-				}
-				this.logWorkerEvent('workerTaskQueued', logPayload);
-			}
-
-			this.dispatchQueue();
-		});
-	}
-
-	/**
-	 * Send a task internally without returning a promise.
-	 */
-	public sendInternalTask(method: string, params = []) {
-		if (!Array.isArray(params)) {
-			params = [params];
-		}
-
-		let taskId = 'task-' + objectIdHexString();
-		const queuedAt = Date.now();
-
-		this._taskQueue.push({
-			type: 'task',
-			taskId,
-			method,
-			params,
-			messageId: 0,
-			userContext: {
-				user: 'Internal System'
-			},
-			queuedAt
-		});
-
-		this._taskTimings.set(taskId, {
-			queuedAt,
-			method,
-			messageId: 0,
-			userContext: { user: 'Internal System' },
-			params
-		});
-
-		if (this._methodManager.getEnableDebug()) {
-			console.log(new Date(), 'Send Internal Task', this._taskQueue);
-		}
-
-		if (this.shouldDebug()) {
-			const publicationInfo = this.getPublicationLogInfo(method, params);
-			const logPayload: Record<string, any> = {
-				taskId,
-				method,
-				messageId: 0,
-				priority: method === 'runPublication' ? 'publication' : 'method',
-				queueDepth: this._taskQueue.length,
-				userContext: { user: 'Internal System' }
-			};
-			if (publicationInfo) {
-				Object.assign(logPayload, publicationInfo);
-			}
-			this.logWorkerEvent('workerTaskQueued', logPayload);
-		}
-
-		this.dispatchQueue();
-	}
-
-	/**
-	 * The main loop that assigns tasks from _taskQueue to any worker that has capacity.
-	 */
-	private dispatchQueue() {
-		if (!this._taskQueue.length) {
-			return;
-		}
-		if (!this._workers.length) {
-			return;
-		}
-
-		const priorityMatchers: TaskMatcher[] = [
-			(task) => this.resolveTaskQueue(task.method) === 'publication',
-			(task) => this.resolveTaskQueue(task.method) === 'default',
-			(task) => this.resolveTaskQueue(task.method) === 'codex'
-		];
-
-		const tryAssignTask = (matcher: TaskMatcher): boolean => {
-			for (let i = 0; i < this._taskQueue.length; i++) {
-				let task = this._taskQueue[i];
-				if (!matcher(task)) {
-					continue;
-				}
-				let targetWorker = this.findWorkerForTask(task);
-
-				if (targetWorker) {
-					this._taskQueue.splice(i, 1);
-					this.assignTaskToWorker(targetWorker, task);
-					return true;
-				}
-			}
-			return false;
-		};
-
-		while (this._taskQueue.length > 0) {
-			let assigned = false;
-			for (const matcher of priorityMatchers) {
-				if (tryAssignTask(matcher)) {
-					assigned = true;
-					break;
-				}
-			}
-
-			if (!assigned) {
-				this.logNoWorkerAvailability();
-				this.scheduleDispatchRetry();
-
-				if (this._methodManager.getEnableDebug()) {
-					console.log(new Date(), 'No Worker Available', JSON.stringify(this._workers, null, 2));
-				}
-
-				return; // nothing can run right now due to per-worker limits
-			}
-		}
-	}
-
-	private scheduleDispatchRetry(delayMs: number = this._dispatchRetryDelayMs) {
-		if (this._dispatchRetryTimer) {
-			return;
-		}
-
-		this._dispatchRetryTimer = setTimeout(() => {
-			this._dispatchRetryTimer = null;
-			this.dispatchQueue();
-		}, delayMs);
-	}
-
-	/**
-	 * Returns the worker with the fewest activeTasks that is under maxConcurrency and per-method limit.
-	 */
-	private findWorkerForTask(task: TaskPayload): WorkerConnection | null {
-		let method = this._methodManager.getMethod(task.method);
-		const taskQueue = this.resolveTaskQueue(task.method);
-		let methodLimit = method && method.maxConcurrency && method.maxConcurrency > 0 ? method.maxConcurrency : null;
-		let methodLimitPerInstance = method && method.maxConcurrencyPerInstance && method.maxConcurrencyPerInstance > 0
-			? method.maxConcurrencyPerInstance
-			: null;
-		let targetWorkerIndex = this.normalizeWorkerIndex(method?.targetWorkerIndex);
-		let targetWorkerInstance = this.normalizeWorkerIndex(method?.targetWorkerInstance);
-		const hasExplicitTarget = !!targetWorkerIndex || !!targetWorkerInstance;
-		const preferNonZero = !targetWorkerIndex;
-		const debugCodex = this._aiWorkerDebug && taskQueue === 'codex';
-		const debugState: Record<string, any> | null = debugCodex ? {
-			taskId: task.taskId,
-			method: task.method,
-			queueDepth: this._taskQueue.length,
-			targetWorkerIndex: targetWorkerIndex || null,
-			targetWorkerInstance: targetWorkerInstance || null,
-			workerCount: this._workers.length,
-			publicationSelector: {
-				indexes: this.selectorToArray(this._publicationWorkerIndexes),
-				instances: this.selectorToArray(this._publicationWorkerInstances)
-			},
-			codexSelector: {
-				indexes: this.selectorToArray(this._codexWorkerIndexes),
-				instances: this.selectorToArray(this._codexWorkerInstances)
-			},
-			maxConcurrency: this.MAX_CONCURRENCY,
-			nonCodexReserve: this._nonCodexReserve
-		} : null;
-
-		let candidates = this._workers.filter(x => x.activeTasks.length < this.MAX_CONCURRENCY);
-		candidates = this.filterGeneralPurposeWorkers(candidates, hasExplicitTarget);
-		if (debugState) {
-			debugState.candidatesInitial = candidates.length;
-		}
-		const hasPublicationSelector = this.hasWorkerSelector(this._publicationWorkerIndexes, this._publicationWorkerInstances);
-		if (hasPublicationSelector && !hasExplicitTarget) {
-			if (taskQueue === 'publication') {
-				candidates = candidates.filter(worker =>
-					this.workerMatchesSelector(worker, this._publicationWorkerIndexes, this._publicationWorkerInstances)
-				);
-			}
-			else {
-				candidates = candidates.filter(worker =>
-					!this.workerMatchesSelector(worker, this._publicationWorkerIndexes, this._publicationWorkerInstances)
-				);
-			}
-		}
-		if (debugState) {
-			debugState.candidatesAfterPublicationFilter = candidates.length;
-			debugState.hasPublicationSelector = hasPublicationSelector;
-		}
-
-		const hasCodexSelector = this.hasWorkerSelector(this._codexWorkerIndexes, this._codexWorkerInstances);
-		if (hasCodexSelector) {
-			if (taskQueue === 'codex') {
-				candidates = candidates.filter(worker =>
-					this.workerMatchesSelector(worker, this._codexWorkerIndexes, this._codexWorkerInstances)
-				);
-			}
-			else if (!hasExplicitTarget) {
-				candidates = candidates.filter(worker =>
-					!this.workerMatchesSelector(worker, this._codexWorkerIndexes, this._codexWorkerInstances)
-				);
-			}
-		}
-		if (debugState) {
-			debugState.candidatesAfterCodexFilter = candidates.length;
-			debugState.hasCodexSelector = hasCodexSelector;
-		}
-		if (targetWorkerIndex) {
-			candidates = candidates.filter(worker => this.normalizeWorkerIndex(worker.workerIndex) === targetWorkerIndex);
-		}
-		if (targetWorkerInstance) {
-			candidates = candidates.filter(worker => this.normalizeWorkerIndex(worker.workerInstance) === targetWorkerInstance);
-		}
-		if (debugState) {
-			debugState.candidatesAfterTargetFilter = candidates.length;
-		}
-		if (!candidates.length) {
-			this.logCodexSelection(debugState, 'noCandidates');
-			return null;
-		}
-
-		let eligible = candidates.filter((worker) => {
-			if (!methodLimit) {
-				if (!methodLimitPerInstance) {
-					return true;
-				}
-			}
-
-			if (methodLimitPerInstance) {
-				const currentInstance = worker.activeTasks.filter(a => a.method === task.method).length;
-				if (currentInstance >= methodLimitPerInstance) {
-					return false;
-				}
-			}
-
-			if (!methodLimit) {
-				return true;
-			}
-
-			let idx = worker.workerIndex || 'unknown';
-			let current = 0;
-
-			this._workers.forEach((w) => {
-				if ((w.workerIndex || 'unknown') === idx) {
-					current += w.activeTasks.filter(a => a.method === task.method).length;
-				}
-			});
-
-			return current < methodLimit;
-		});
-
-		if (debugState) {
-			debugState.eligibleAfterLimits = eligible.length;
-		}
-		if (!eligible.length) {
-			this.logCodexSelection(debugState, 'limitsExceeded');
-			return null;
-		}
-
-		if (taskQueue === 'codex' && this._nonCodexReserve > 0) {
-			eligible = eligible.filter(worker => (this.MAX_CONCURRENCY - worker.activeTasks.length) > this._nonCodexReserve);
-		}
-
-		if (debugState) {
-			debugState.eligibleAfterReserve = eligible.length;
-		}
-		if (!eligible.length) {
-			this.logCodexSelection(debugState, 'reserveBlocked');
-			return null;
-		}
-
-		eligible.sort((x, y) => {
-			const totalX = this.getWorkerLoad(x);
-			const totalY = this.getWorkerLoad(y);
-			if (totalX !== totalY) {
-				return totalX - totalY;
-			}
-
-			if (preferNonZero) {
-				const xZero = this.isWorkerZero(x) ? 1 : 0;
-				const yZero = this.isWorkerZero(y) ? 1 : 0;
-				if (xZero !== yZero) {
-					return xZero - yZero;
-				}
-			}
-
-			const idxX = this.normalizeWorkerIndex(x.workerIndex) || '';
-			const idxY = this.normalizeWorkerIndex(y.workerIndex) || '';
-			const idxCompare = idxX.localeCompare(idxY, undefined, { numeric: true, sensitivity: 'base' });
-			if (idxCompare !== 0) {
-				return idxCompare;
-			}
-
-			const instX = this.normalizeWorkerIndex(x.workerInstance) || '';
-			const instY = this.normalizeWorkerIndex(y.workerInstance) || '';
-			return instX.localeCompare(instY, undefined, { numeric: true, sensitivity: 'base' });
-		});
-
-			const selected = eligible[0];
-			if (this._aiWorkerDebug && taskQueue === 'codex') {
-				console.log(new Date(), '[AI Worker Debug] AI worker selected', {
-					taskId: task.taskId,
-					method: task.method,
-					workerIndex: selected?.workerIndex || null,
-					workerInstance: selected?.workerInstance || null,
-				activeTasks: selected?.activeTasks ? selected.activeTasks.length : 0,
-				queueDepth: this._taskQueue.length
-			});
-		}
-		return selected;
-	}
-
-	private logCodexSelection(debugState: Record<string, any> | null, reason: string): void {
-		if (!debugState || !this._aiWorkerDebug) {
-			return;
-		}
-		const taskId = String(debugState.taskId || '');
-		if (!taskId || this._debugCodexTaskIds.has(taskId)) {
-			return;
-			}
-			this._debugCodexTaskIds.add(taskId);
-			console.warn(new Date(), '[AI Worker Debug] AI worker selection issue', {
-				reason,
-				...debugState
-			});
-		}
-
-	private normalizeWorkerIndex(value?: string | number | null): string | null {
-		if (value === null || value === undefined) {
-			return null;
-		}
-
-		const normalized = String(value).trim();
-		return normalized.length ? normalized : null;
-	}
-
-	private isGeneralPurposeWorker(worker: WorkerConnection): boolean {
-		const normalizedIndex = this.normalizeWorkerIndex(worker?.workerIndex);
-		const normalizedInstance = this.normalizeWorkerIndex(worker?.workerInstance);
-		if (!normalizedIndex || !normalizedInstance) {
-			return false;
-		}
-
-		return /^[0-9]+$/.test(normalizedIndex) && /^[0-9]+$/.test(normalizedInstance);
-	}
-
-	private filterGeneralPurposeWorkers(candidates: WorkerConnection[], hasExplicitTarget: boolean): WorkerConnection[] {
-		if (hasExplicitTarget) {
-			return candidates;
-		}
-
-		return candidates.filter(worker => this.isGeneralPurposeWorker(worker));
-	}
-
-	private getWorkerLoad(worker: WorkerConnection): number {
-		return worker.activeTasks.reduce((sum, task) => sum + task.weight, 0);
-	}
-
-	private isWorkerZero(worker: WorkerConnection): boolean {
-		return this.normalizeWorkerIndex(worker.workerIndex) === '0';
-	}
-
-	private assignTaskToWorker(worker: WorkerConnection, task: TaskPayload) {
-		let method = this._methodManager.getMethod(task.method);
-
-		if (!method) {
-			console.error('Failed to send task to worker - Could not find method:', task.method);
-			this._taskTimings.delete(task.taskId);
-			return;
-		}
-
-		const dispatchedAt = Date.now();
-		const timing = this._taskTimings.get(task.taskId) || { queuedAt: dispatchedAt };
-		timing.dispatchedAt = dispatchedAt;
-		timing.method = task.method;
-		timing.messageId = task.messageId;
-		timing.userContext = task.userContext;
-		if (!timing.params) {
-			timing.params = task.params;
-		}
-		this._taskTimings.set(task.taskId, timing);
-
-		worker.activeTasks.push({
-			taskId: task.taskId,
-			weight: method && method.workerTaskWeight ? method.workerTaskWeight : 1,
-			method: task.method,
-			startedAt: dispatchedAt,
-			messageId: task.messageId,
-			userContext: task.userContext
-		});
-		this._debugNoWorkerTaskIds.delete(task.taskId);
-
-		let payload: TaskPayload = {
-			type: 'task',
-			taskId: task.taskId,
-			messageId: task.messageId,
-			method: task.method,
-			params: task.params,
-			userContext: task.userContext || {},
-			queuedAt: timing.queuedAt,
-			dispatchedAt
-		};
-
-		if (this.shouldDebug()) {
-			const publicationInfo = this.getPublicationLogInfo(task.method, task.params);
-			const logPayload: Record<string, any> = {
-				taskId: task.taskId,
-				method: task.method,
-				messageId: task.messageId,
-				priority: task.method === 'runPublication' ? 'publication' : 'method',
-				worker: {
-					id: worker.id,
-					index: worker.workerIndex,
-					instance: worker.workerInstance
-				},
-				queueDepth: this._taskQueue.length
-			};
-			if (publicationInfo) {
-				Object.assign(logPayload, publicationInfo);
-			}
-			this.logWorkerEvent('workerTaskAssigned', logPayload);
-		}
-
-		let timeoutHandle = setTimeout(() => {
-			const timeoutAt = Date.now();
-			const queuedAt = timing.queuedAt;
-			const queueWaitMs = queuedAt ? round(dispatchedAt - queuedAt) : null;
-			const dispatchToTimeoutMs = round(timeoutAt - dispatchedAt);
-			const timeoutMessage = 'Task timed out after 2m in WorkerDispatcherManager for method: ' + task.method;
-
-			const publicationInfo = this.getPublicationLogInfo(task.method, task.params);
-			const logPayload: Record<string, any> = {
-				taskId: task.taskId,
-				method: task.method,
-				messageId: task.messageId,
-				queueWaitMs,
-				dispatchToTimeoutMs,
-				userContext: task.userContext,
-				worker: {
-					id: worker.id,
-					index: worker.workerIndex,
-					instance: worker.workerInstance
-				},
-				workers: this._workers.map(a => a.activeTasks)
-			};
-			if (publicationInfo) {
-				Object.assign(logPayload, publicationInfo);
-			}
-			this.logWorkerEvent('workerTaskTimeout', logPayload);
-
-			this.disconnectWorker(worker.id, timeoutMessage);
-			if (worker.ws && worker.ws.readyState !== WebSocket.CLOSED) {
-				try {
-					worker.ws.close();
-				}
-				catch {
-					try {
-						worker.ws.terminate();
-					}
-					catch {
-						// Ignore socket cleanup errors after timeout.
-					}
-				}
-			}
-		}, method.timeoutOverride || (1000 * 60 * 2));
-
-		// If we already stored a promise for this task (from sendInternalPromise), add the timeout now
-		let existing = this._pendingTasks.get(task.taskId);
-		if (!existing) {
-			existing = { timeout: null, method: task.method };
-		}
-		else {
-			existing.method = task.method;
-		}
-
-		existing.timeout = timeoutHandle;
-		this._pendingTasks.set(task.taskId, existing);
-
-		try {
-			this.sendWorkerPayload(worker.ws, payload);
-		}
-		catch (err) {
-			console.error('Failed to send task to worker:', err);
-
-			clearTimeout(timeoutHandle);
-			worker.activeTasks = worker.activeTasks.filter(a => a.taskId !== task.taskId);
-
-			// Put task back in front
-			this._taskQueue.unshift(task);
-			this.dispatchQueue();
-		}
-	}
-
-	private logNoWorkerAvailability() {
-		if (!this.shouldDebug()) {
-			return;
-		}
-
-		const workers = this._workers.map(worker => ({
-			id: worker.id,
-			index: this.normalizeWorkerIndex(worker.workerIndex),
-			instance: this.normalizeWorkerIndex(worker.workerInstance),
-			activeTasks: worker.activeTasks.length
-		}));
-
-		for (const task of this._taskQueue) {
-			const method = this._methodManager.getMethod(task.method);
-			const targetWorkerIndex = this.normalizeWorkerIndex(method?.targetWorkerIndex);
-			const targetWorkerInstance = this.normalizeWorkerIndex(method?.targetWorkerInstance);
-
-			if (!targetWorkerIndex && !targetWorkerInstance) {
-				continue;
-			}
-
-			if (this._debugNoWorkerTaskIds.has(task.taskId)) {
-				continue;
-			}
-
-			const hasMatch = this._workers.some(worker =>
-				(!targetWorkerIndex || this.normalizeWorkerIndex(worker.workerIndex) === targetWorkerIndex) &&
-				(!targetWorkerInstance || this.normalizeWorkerIndex(worker.workerInstance) === targetWorkerInstance)
-			);
-
-			if (!hasMatch) {
-				this._debugNoWorkerTaskIds.add(task.taskId);
-				console.warn(new Date(), '[WorkerDispatcher] No matching worker for task', {
-					taskId: task.taskId,
-					method: task.method,
-					targetWorkerIndex,
-					targetWorkerInstance,
-					workers
-				});
-			}
-		}
-	}
-
-	private shouldDebug(): boolean {
-		if (this._methodManager.getEnableDebug()) {
-			return true;
-		}
-
-		if (this._workerTaskDebug) {
-			return true;
-		}
-
-		return this.parseDebugFlag(process.env.WORKER_TASK_DEBUG ?? process.env.WORKER_DISPATCH_DEBUG);
-	}
-
-	private logWorkerEvent(event: string, details: Record<string, any>) {
-		console.log(this.safeStringify({
-			ts: new Date().toISOString(),
-			event,
-			...details
-		}));
-	}
-
-	private safeStringify(value: any): string {
-		try {
-			const seen = new WeakSet();
-			return JSON.stringify(value, (key, val) => {
-				if (key === '__proto__') {
-					return undefined;
-				}
-				if (typeof val === 'bigint') {
-					return val.toString();
-				}
-				if (val instanceof Error) {
-					return {
-						name: val.name,
-						message: val.message,
-						stack: val.stack
-					};
-				}
-				if (val && typeof val === 'object') {
-					if (seen.has(val)) {
-						return '[Circular]';
-					}
-					seen.add(val);
-				}
-				return val;
-			});
-		}
-		catch (err) {
-			return JSON.stringify({
-				error: 'safeStringifyFailed',
-				message: err?.message || String(err)
-			});
-		}
-	}
-
-	/**
-	 * Handle messages coming back from a worker (like 'taskComplete').
-	 */
-	public handleWorkerMessage(workerId: string, rawMessage: WebSocket.RawData) {
-		let data: TaskResponse;
-
-		try {
-			if (typeof rawMessage === 'string') {
-				data = JSON.parse(rawMessage, dateReviver);
-			}
-			else if (Buffer.isBuffer(rawMessage)) {
-				data = unpack(rawMessage);
-			}
-			else if (Array.isArray(rawMessage)) {
-				const chunks = rawMessage as unknown as ReadonlyArray<Uint8Array>;
-				data = unpack(Buffer.concat(chunks));
-			}
-			else if (rawMessage instanceof ArrayBuffer) {
-				data = unpack(Buffer.from(rawMessage));
-			}
-			else if (ArrayBuffer.isView(rawMessage)) {
-				const view = rawMessage as NodeJS.ArrayBufferView;
-				data = unpack(Buffer.from(view.buffer, view.byteOffset, view.byteLength));
-			}
-			else {
-				console.error('Unsupported worker message type received:', typeof rawMessage);
-				return;
-			}
-		}
-		catch (err) {
-			console.error('Failed to parse worker message:', err);
-			return;
-		}
-
-		if (data.type === 'taskComplete') {
-			let worker = this._workers.find(x => x.id === workerId);
-
-			if (!worker) {
-				console.error('Unknown worker for taskComplete:', workerId);
-				return;
-			}
-
-			let { taskId, messageId, error, result } = data;
-			const timing = this._taskTimings.get(taskId);
-			const methodName = timing?.method;
-			const isAiMethod = !!methodName && methodName.startsWith('ai');
-			let unpackedResult = result;
-			let pendingTask = this._pendingTasks.get(taskId);
-			const resolveRaw = !!pendingTask?.resolveRaw;
-
-			if (!error && !resolveRaw && unpackedResult === null && data['packedResult']) {
-				try {
-					unpackedResult = unpack(data['packedResult']);
-				}
-				catch (err) {
-					console.error('Failed to unpack worker result', err);
-				}
-			}
-			worker.activeTasks = worker.activeTasks.filter(a => a.taskId !== taskId);
-			this._taskTimings.delete(taskId);
-
-			if (this._aiWorkerDebug && isAiMethod) {
-				const packedBytes = data['packedResult'] && data['packedResult'].byteLength
-					? data['packedResult'].byteLength
-					: null;
-				let resultBytes: number | null = null;
-				if (!packedBytes) {
-					try {
-						resultBytes = Buffer.byteLength(JSON.stringify(result));
-					}
-					catch {
-						resultBytes = null;
-					}
-				}
-				console.log(new Date(), '[AI Worker Debug] worker result', {
-					taskId,
-					method: methodName,
-					messageId,
-					hasError: !!error,
-					packedBytes,
-					resultBytes,
-					workerIndex: worker?.workerIndex || null,
-					workerInstance: worker?.workerInstance || null,
-					queueDepth: this._taskQueue.length
-				});
-			}
-
-
-			if (pendingTask) {
-				if (pendingTask.promise) {
-					if (error) {
-						pendingTask.promise.reject(unpackedResult);
-					}
-					else {
-						pendingTask.promise.resolve(resolveRaw ? data : unpackedResult);
-					}
-				}
-
-				clearTimeout(pendingTask.timeout);
-				this._pendingTasks.delete(taskId);
-			}
-
-			// Look up original request if it came from a client
-			let clientReqId = this._clientRequests[taskId];
-
-			if (clientReqId) {
-				let res: ServerResponseModel = {
-					messageId: messageId,
-					hasError: false,
-					data: result
-				};
-
-				if (error) {
-					res.hasError = true;
-					res.data = result;
-				}
-
-				let clientReqWS = this._websocketManager.getWebSocket(clientReqId);
-
-				if (clientReqWS) {
-					if (!error && data['packedResult']) {
-						this._websocketManager.sendPackedBuffer(
-							clientReqWS,
-							messageId,
-							false,
-							data['packedResult'],
-							data['encoding'] || 'msgpack',
-							{ passThrough: true }
-						);
-					}
-					else {
-						this._websocketManager.send(clientReqWS, res);
-					}
-				}
-
-				if (this._clientRequests[taskId]) {
-					delete this._clientRequests[taskId];
-				}
-			}
-
-			// Try to dispatch more from the queue
-			if (this._taskQueue.length) {
-				this.dispatchQueue();
-			}
-		}
-	}
-
-	public sendWorkerPayload(ws: WebSocket.WebSocket, payload: TaskPayload | string) {
-		if (!ws) {
-			return;
-		}
-
-		if (ws.readyState !== WebSocket.OPEN) {
-			if (typeof payload !== 'string') {
-				const task = payload;
-				this.disconnectWorker(ws['id_worker'], 'Worker socket not open for method: ' + task.method);
-				try {
-					ws.close();
-				}
-				catch {}
-			}
-			else {
-				this.disconnectWorker(ws['id_worker'], 'Worker socket not open.');
-				try {
-					ws.close();
-				}
-				catch {}
-			}
-
-			return;
-		}
-
-		if (typeof payload === 'string') {
-			try {
-				ws.send(payload);
-			}
-			catch (err) {
-				this.disconnectWorker(ws['id_worker'], 'Failed to send worker payload: ' + err?.toString());
-				try {
-					ws.close();
-				}
-				catch {}
-				console.error('Failed to send worker payload:', err);
-			}
-
-			return;
-		}
-
-		const task = payload;
-		const payloadBuffer = pack(task);
-
-		try {
-			ws.send(payloadBuffer);
-		}
-		catch (err) {
-			if (this._methodManager.getEnableDebug()) {
-				console.log(new Date(), 'Sending to Server', task);
-			}
-
-			this.disconnectWorker(ws['id_worker'], 'Failed to send worker response for method: ' + task.method);
-			try {
-				ws.close();
-			}
-			catch {}
-			console.error('Failed to send worker response:', err);
-		}
-	}
-}