comfyui-node 1.4.4 → 1.6.0

package/dist/pool/WorkflowPool.js

@@ -9,6 +9,8 @@ import { ClientManager } from "./client/ClientManager.js";
 import { hashWorkflow } from "./utils/hash.js";
 import { cloneDeep } from "./utils/clone.js";
 import { JobProfiler } from "./profiling/JobProfiler.js";
+import { analyzeWorkflowFailure } from "./utils/failure-analysis.js";
+import { WorkflowNotSupportedError } from "../types/error.js";
 const DEFAULT_MAX_ATTEMPTS = 3;
 const DEFAULT_RETRY_DELAY = 1000;
 export class WorkflowPool extends TypedEventTarget {
@@ -17,9 +19,18 @@ export class WorkflowPool extends TypedEventTarget {
     clientManager;
     opts;
     jobStore = new Map();
+    jobFailureAnalysis = new Map();
+    affinities = new Map();
     initPromise;
     processing = false;
+    processQueued = false;
     activeJobs = new Map();
+    queueDebug = process.env.WORKFLOW_POOL_DEBUG === "1";
+    debugLog(...args) {
+        if (this.queueDebug) {
+            console.log(...args);
+        }
+    }
     constructor(clients, opts) {
         super();
         this.strategy = opts?.failoverStrategy ?? new SmartFailoverStrategy();
@@ -28,6 +39,11 @@ export class WorkflowPool extends TypedEventTarget {
             healthCheckIntervalMs: opts?.healthCheckIntervalMs ?? 30000
         });
         this.opts = opts ?? {};
+        if (opts?.workflowAffinities) {
+            for (const affinity of opts.workflowAffinities) {
+                this.affinities.set(affinity.workflowHash, affinity);
+            }
+        }
         this.clientManager.on("client:state", (ev) => {
             this.dispatchEvent(new CustomEvent("client:state", { detail: ev.detail }));
         });
@@ -51,6 +67,15 @@ export class WorkflowPool extends TypedEventTarget {
     async ready() {
         await this.initPromise;
     }
+    setAffinity(affinity) {
+        this.affinities.set(affinity.workflowHash, affinity);
+    }
+    removeAffinity(workflowHash) {
+        return this.affinities.delete(workflowHash);
+    }
+    getAffinities() {
+        return Array.from(this.affinities.values());
+    }
     async enqueue(workflowInput, options) {
         await this.ready();
         const workflowJson = this.normalizeWorkflow(workflowInput);
@@ -72,6 +97,13 @@ export class WorkflowPool extends TypedEventTarget {
                 outputAliases: workflowInput.outputAliases ?? {}
             };
         }
+        const affinity = this.affinities.get(workflowHash);
+        const preferredClientIds = options?.preferredClientIds
+            ? [...options.preferredClientIds]
+            : (affinity?.preferredClientIds ? [...affinity.preferredClientIds] : []);
+        const excludeClientIds = options?.excludeClientIds
+            ? [...options.excludeClientIds]
+            : (affinity?.excludeClientIds ? [...affinity.excludeClientIds] : []);
         const payload = {
             jobId,
             workflow: workflowJson,
@@ -83,14 +115,20 @@ export class WorkflowPool extends TypedEventTarget {
                 maxAttempts: options?.maxAttempts ?? DEFAULT_MAX_ATTEMPTS,
                 retryDelayMs: options?.retryDelayMs ?? DEFAULT_RETRY_DELAY,
                 priority: options?.priority ?? 0,
-                preferredClientIds: options?.preferredClientIds ?? [],
-                excludeClientIds: options?.excludeClientIds ?? [],
+                preferredClientIds: preferredClientIds,
+                excludeClientIds: excludeClientIds,
                 metadata: options?.metadata ?? {},
                 includeOutputs: options?.includeOutputs ?? []
             }
         };
         const record = {
             ...payload,
+            options: {
+                ...payload.options,
+                preferredClientIds: payload.options.preferredClientIds ? [...payload.options.preferredClientIds] : [],
+                excludeClientIds: payload.options.excludeClientIds ? [...payload.options.excludeClientIds] : [],
+                includeOutputs: payload.options.includeOutputs ? [...payload.options.includeOutputs] : []
+            },
             attachments: options?.attachments,
             status: "queued"
         };
@@ -113,6 +151,7 @@ export class WorkflowPool extends TypedEventTarget {
             if (removed) {
                 record.status = "cancelled";
                 record.completedAt = Date.now();
+                this.clearJobFailures(jobId);
                 this.dispatchEvent(new CustomEvent("job:cancelled", { detail: { job: record } }));
                 return true;
             }
@@ -122,6 +161,7 @@ export class WorkflowPool extends TypedEventTarget {
             await active.cancel();
             record.status = "cancelled";
             record.completedAt = Date.now();
+            this.clearJobFailures(jobId);
             this.dispatchEvent(new CustomEvent("job:cancelled", { detail: { job: record } }));
             return true;
         }
@@ -159,7 +199,7 @@ export class WorkflowPool extends TypedEventTarget {
         }
     }
     static fallbackId() {
-        return (globalThis.crypto && "randomUUID" in globalThis.crypto)
+        return globalThis.crypto && "randomUUID" in globalThis.crypto
             ? globalThis.crypto.randomUUID()
             : `job_${Math.random().toString(36).slice(2, 10)}`;
     }
@@ -185,44 +225,215 @@ export class WorkflowPool extends TypedEventTarget {
         }
         return autoSeeds;
     }
+    rememberJobFailure(job, clientId, analysis) {
+        let map = this.jobFailureAnalysis.get(job.jobId);
+        if (!map) {
+            map = new Map();
+            this.jobFailureAnalysis.set(job.jobId, map);
+        }
+        map.set(clientId, analysis);
+    }
+    clearJobFailures(jobId) {
+        this.jobFailureAnalysis.delete(jobId);
+    }
+    collectFailureReasons(jobId) {
+        const map = this.jobFailureAnalysis.get(jobId);
+        if (!map) {
+            return {};
+        }
+        const reasons = {};
+        for (const [clientId, analysis] of map.entries()) {
+            reasons[clientId] = analysis.reason;
+        }
+        return reasons;
+    }
+    addPermanentExclusion(job, clientId) {
+        if (!job.options.excludeClientIds) {
+            job.options.excludeClientIds = [];
+        }
+        if (!job.options.excludeClientIds.includes(clientId)) {
+            job.options.excludeClientIds.push(clientId);
+        }
+    }
+    hasRetryPath(job) {
+        const map = this.jobFailureAnalysis.get(job.jobId);
+        const exclude = new Set(job.options.excludeClientIds ?? []);
+        const preferred = job.options.preferredClientIds?.length ? new Set(job.options.preferredClientIds) : null;
+        for (const client of this.clientManager.list()) {
+            if (preferred && !preferred.has(client.id)) {
+                continue;
+            }
+            if (exclude.has(client.id)) {
+                continue;
+            }
+            const analysis = map?.get(client.id);
+            if (analysis?.blockClient === "permanent") {
+                continue;
+            }
+            return true;
+        }
+        return false;
+    }
+    createWorkflowNotSupportedError(job, cause) {
+        const reasons = this.collectFailureReasons(job.jobId);
+        const message = `Workflow ${job.workflowHash} is not supported by any connected clients`;
+        return new WorkflowNotSupportedError(message, {
+            workflowHash: job.workflowHash,
+            reasons,
+            cause
+        });
+    }
     async processQueue() {
+        this.debugLog("[processQueue] Called");
         if (this.processing) {
+            this.debugLog("[processQueue] Already processing, returning early");
+            this.processQueued = true;
             return;
         }
         this.processing = true;
         try {
+            // Continue processing until no more jobs can be assigned
+            let iteration = 0;
             while (true) {
-                const reservation = await this.queue.reserve();
-                if (!reservation) {
-                    break;
+                iteration++;
+                this.debugLog(`[processQueue] Iteration ${iteration}`);
+                const idleClients = this.clientManager.list().filter(c => this.clientManager.isClientStable(c));
+                this.debugLog(`[processQueue] Idle clients: [${idleClients.map(c => c.id).join(", ")}] (${idleClients.length})`);
+                if (!idleClients.length) {
+                    this.debugLog("[processQueue] No idle clients, breaking");
+                    break; // No idle clients available
                 }
-                const job = this.jobStore.get(reservation.payload.jobId);
-                if (!job) {
-                    await this.queue.commit(reservation.reservationId);
-                    continue;
+                const waitingJobs = await this.queue.peek(100); // Peek at top 100 jobs
+                this.debugLog(`[processQueue] Waiting jobs in queue: ${waitingJobs.length}`);
+                if (!waitingJobs.length) {
+                    this.debugLog("[processQueue] No waiting jobs, breaking");
+                    break; // No jobs in queue
                 }
-                const lease = this.clientManager.claim(job);
-                if (!lease) {
-                    await this.queue.retry(reservation.reservationId, { delayMs: job.options.retryDelayMs });
-                    this.scheduleProcess(job.options.retryDelayMs);
-                    break;
+                const leasedClientIds = new Set();
+                const reservedJobIds = new Set();
+                const jobMatchInfos = [];
+                for (const jobPayload of waitingJobs) {
+                    const job = this.jobStore.get(jobPayload.jobId);
+                    if (!job) {
+                        this.debugLog(`[processQueue] Job ${jobPayload.jobId} not in jobStore, skipping`);
+                        continue;
+                    }
+                    const compatibleClients = idleClients
+                        .filter(client => {
+                        const canRun = this.clientManager.canClientRunJob(client, job);
+                        if (!canRun) {
+                            this.debugLog(`[processQueue] Job ${job.jobId.substring(0, 8)}... NOT compatible with ${client.id}. Checking why...`);
+                            this.debugLog(`[processQueue]   - preferredClientIds: ${JSON.stringify(job.options.preferredClientIds)}`);
+                            this.debugLog(`[processQueue]   - excludeClientIds: ${JSON.stringify(job.options.excludeClientIds)}`);
+                            this.debugLog(`[processQueue]   - client.id: ${client.id}`);
+                        }
+                        return canRun;
+                    })
+                        .map(client => client.id);
+                    this.debugLog(`[processQueue] Job ${job.jobId.substring(0, 8)}... compatible with: [${compatibleClients.join(", ")}] (selectivity=${compatibleClients.length})`);
+                    if (compatibleClients.length > 0) {
+                        jobMatchInfos.push({
+                            jobPayload,
+                            job,
+                            compatibleClients,
+                            selectivity: compatibleClients.length
+                        });
+                    }
                 }
-                this.runJob({ reservation, job, clientId: lease.clientId, release: lease.release }).catch((error) => {
-                    console.error("[WorkflowPool] Unhandled job error", error);
+                this.debugLog(`[processQueue] Found ${jobMatchInfos.length} compatible job matches`);
+                if (jobMatchInfos.length === 0) {
+                    this.debugLog("[processQueue] No compatible jobs for idle clients, breaking");
+                    break; // No compatible jobs for idle clients
+                }
+                // Sort jobs by priority first, then selectivity, to maximize throughput
+                // 1. Higher priority jobs execute first (explicit user priority)
+                // 2. More selective jobs (fewer compatible clients) assigned first within same priority
+                // 3. Earlier queue position as final tiebreaker
+                jobMatchInfos.sort((a, b) => {
+                    // Primary: priority (higher priority = higher precedence)
+                    const aPriority = a.job.options.priority ?? 0;
+                    const bPriority = b.job.options.priority ?? 0;
+                    if (aPriority !== bPriority) {
+                        return bPriority - aPriority; // Higher priority first
+                    }
+                    // Secondary: selectivity (fewer compatible clients = higher precedence)
+                    if (a.selectivity !== b.selectivity) {
+                        return a.selectivity - b.selectivity;
+                    }
+                    // Tertiary: maintain queue order (earlier jobs first)
+                    const aIndex = waitingJobs.indexOf(a.jobPayload);
+                    const bIndex = waitingJobs.indexOf(b.jobPayload);
+                    return aIndex - bIndex;
                 });
+                // Assign jobs to clients using the selectivity-based ordering
+                let assignedAnyJob = false;
+                for (const matchInfo of jobMatchInfos) {
+                    if (reservedJobIds.has(matchInfo.job.jobId))
+                        continue;
+                    // Find first available compatible client
+                    const availableClient = matchInfo.compatibleClients.find(clientId => !leasedClientIds.has(clientId));
+                    if (!availableClient) {
+                        this.debugLog(`[processQueue] No available client for job ${matchInfo.job.jobId.substring(0, 8)}...`);
+                        continue; // No available clients for this job
+                    }
+                    this.debugLog(`[processQueue] Reserving job ${matchInfo.job.jobId.substring(0, 8)}... for client ${availableClient}`);
+                    const reservation = await this.queue.reserveById(matchInfo.job.jobId);
+                    if (reservation) {
+                        // Mark as leased/reserved for this cycle
+                        leasedClientIds.add(availableClient);
+                        reservedJobIds.add(matchInfo.job.jobId);
+                        assignedAnyJob = true;
+                        // Get the lease (which marks the client as busy)
+                        const lease = this.clientManager.claim(matchInfo.job, availableClient);
+                        if (lease) {
+                            this.debugLog(`[processQueue] Starting job ${matchInfo.job.jobId.substring(0, 8)}... on client ${availableClient}`);
+                            this.runJob({ reservation, job: matchInfo.job, clientId: lease.clientId, release: lease.release }).catch((error) => {
+                                console.error("[WorkflowPool] Unhandled job error", error);
+                            });
+                        }
+                        else {
+                            // This should not happen since we checked canClientRunJob, but handle defensively
+                            console.error(`[processQueue.processQueue] CRITICAL: Failed to claim client ${availableClient} for job ${matchInfo.job.jobId} after successful check.`);
+                            await this.queue.retry(reservation.reservationId, { delayMs: matchInfo.job.options.retryDelayMs });
+                        }
+                    }
+                    else {
+                        this.debugLog(`[processQueue] Failed to reserve job ${matchInfo.job.jobId.substring(0, 8)}...`);
+                    }
+                }
+                this.debugLog(`[processQueue] Assigned any job in this iteration: ${assignedAnyJob}`);
+                // If we didn't assign any jobs this iteration, no point continuing
+                if (!assignedAnyJob) {
+                    this.debugLog("[processQueue] No jobs assigned, breaking");
+                    break;
+                }
             }
         }
         finally {
+            this.debugLog("[processQueue] Exiting, setting processing = false");
             this.processing = false;
+            if (this.processQueued) {
+                this.debugLog("[processQueue] Pending rerun detected, draining queue again");
+                this.processQueued = false;
+                void this.processQueue();
+            }
         }
     }
     async runJob(ctx) {
         const { reservation, job, clientId, release } = ctx;
+        let released = false;
+        const safeRelease = (opts) => {
+            if (released) {
+                return;
+            }
+            released = true;
+            release(opts);
+        };
         const managed = this.clientManager.getClient(clientId);
         const client = managed?.client;
         if (!client) {
             await this.queue.retry(reservation.reservationId, { delayMs: job.options.retryDelayMs });
-            release({ success: false });
+            safeRelease({ success: false });
             return;
         }
         job.status = "running";
@@ -257,9 +468,9 @@ export class WorkflowPool extends TypedEventTarget {
         // Use stored metadata if available (from Workflow instance), otherwise extract from recreated instance
         const outputNodeIds = reservation.payload.workflowMeta?.outputNodeIds ??
             wfInstance.outputNodeIds ??
-            job.options.includeOutputs ?? [];
-        const outputAliases = reservation.payload.workflowMeta?.outputAliases ??
-            wfInstance.outputAliases ?? {};
+            job.options.includeOutputs ??
+            [];
+        const outputAliases = reservation.payload.workflowMeta?.outputAliases ?? wfInstance.outputAliases ?? {};
         let promptBuilder = new PromptBuilder(wfInstance.json, wfInstance.inputPaths ?? [], outputNodeIds);
         for (const nodeId of outputNodeIds) {
             const alias = outputAliases[nodeId] ?? nodeId;
@@ -267,9 +478,7 @@ export class WorkflowPool extends TypedEventTarget {
         }
         const wrapper = new CallWrapper(client, promptBuilder);
         // Setup profiling if enabled
-        const profiler = this.opts.enableProfiling
-            ? new JobProfiler(job.enqueuedAt, workflowPayload)
-            : undefined;
+        const profiler = this.opts.enableProfiling ? new JobProfiler(job.enqueuedAt, workflowPayload) : undefined;
         // Setup node execution timeout tracking
         const nodeExecutionTimeout = this.opts.nodeExecutionTimeoutMs ?? 300000; // 5 minutes default
         let nodeTimeoutId;
@@ -285,9 +494,10 @@ export class WorkflowPool extends TypedEventTarget {
                 currentExecutingNode = nodeName || null;
                 nodeTimeoutId = setTimeout(() => {
                     const elapsed = Date.now() - (lastNodeStartTime || 0);
-                    const nodeInfo = currentExecutingNode ? ` (node: ${currentExecutingNode})` : '';
-                    rejectCompletion?.(new Error(`Node execution timeout: took longer than ${nodeExecutionTimeout}ms${nodeInfo}. ` +
-                        `Actual time: ${elapsed}ms. Server may be stuck or node is too slow for configured timeout.`));
+                    const nodeInfo = currentExecutingNode ? ` (node: ${currentExecutingNode})` : "";
+                    completionError = new Error(`Node execution timeout: took longer than ${nodeExecutionTimeout}ms${nodeInfo}. ` +
+                        `Actual time: ${elapsed}ms. Server may be stuck or node is too slow for configured timeout.`);
+                    resolveCompletion?.();
                 }, nodeExecutionTimeout);
             }
         };
@@ -326,20 +536,20 @@ export class WorkflowPool extends TypedEventTarget {
             const onExecutionError = (event) => {
                 const detail = event.detail || {};
                 if (detail.node !== undefined) {
-                    profiler.onNodeError(String(detail.node), detail.exception_message || 'Execution error');
+                    profiler.onNodeError(String(detail.node), detail.exception_message || "Execution error");
                 }
             };
             // Attach listeners to client
-            client.addEventListener('execution_start', onExecutionStart);
-            client.addEventListener('execution_cached', onExecutionCached);
-            client.addEventListener('executing', onExecuting);
-            client.addEventListener('execution_error', onExecutionError);
+            client.addEventListener("execution_start", onExecutionStart);
+            client.addEventListener("execution_cached", onExecutionCached);
+            client.addEventListener("executing", onExecuting);
+            client.addEventListener("execution_error", onExecutionError);
             // Cleanup function to remove listeners
             const cleanupProfiler = () => {
-                client.removeEventListener('execution_start', onExecutionStart);
-                client.removeEventListener('execution_cached', onExecutionCached);
-                client.removeEventListener('executing', onExecuting);
-                client.removeEventListener('execution_error', onExecutionError);
+                client.removeEventListener("execution_start", onExecutionStart);
+                client.removeEventListener("execution_cached", onExecutionCached);
+                client.removeEventListener("executing", onExecuting);
+                client.removeEventListener("execution_error", onExecutionError);
             };
             // Ensure cleanup happens when job finishes
             wrapper.onFinished(() => cleanupProfiler());
@@ -365,19 +575,19 @@ export class WorkflowPool extends TypedEventTarget {
         };
         const onExecutionStarted = (event) => {
             // Execution started - reset timeout for first node
-            resetNodeTimeout('execution_start');
+            resetNodeTimeout("execution_start");
         };
         if (nodeExecutionTimeout > 0) {
-            client.addEventListener('execution_start', onExecutionStarted);
-            client.addEventListener('executing', onNodeExecuting);
-            client.addEventListener('progress', onNodeProgress);
+            client.addEventListener("execution_start", onExecutionStarted);
+            client.addEventListener("executing", onNodeExecuting);
+            client.addEventListener("progress", onNodeProgress);
         }
         const cleanupNodeTimeout = () => {
             clearNodeTimeout();
             if (nodeExecutionTimeout > 0) {
-                client.removeEventListener('execution_start', onExecutionStarted);
-                client.removeEventListener('executing', onNodeExecuting);
-                client.removeEventListener('progress', onNodeProgress);
+                client.removeEventListener("execution_start", onExecutionStarted);
+                client.removeEventListener("executing", onNodeExecuting);
+                client.removeEventListener("progress", onNodeProgress);
             }
         };
         let pendingSettled = false;
@@ -398,10 +608,11 @@ export class WorkflowPool extends TypedEventTarget {
             };
         });
         let resolveCompletion;
-        let rejectCompletion;
-        const completionPromise = new Promise((resolve, reject) => {
+        let completionError;
+        // completionPromise is used to track when the wrapper completes (success or failure)
+        // It's resolved in onFinished and onFailed handlers
+        const completionPromise = new Promise((resolve) => {
             resolveCompletion = resolve;
-            rejectCompletion = reject;
         });
         let jobStartedDispatched = false;
         wrapper.onProgress((progress, promptId) => {
@@ -493,16 +704,20 @@ export class WorkflowPool extends TypedEventTarget {
             }
             job.result = resultPayload;
             job.completedAt = Date.now();
+            this.clearJobFailures(job.jobId);
             // Cleanup timeouts
             cleanupNodeTimeout();
             // Attach profiling stats if profiling was enabled
             if (profiler) {
                 job.profileStats = profiler.getStats();
             }
+            completionError = undefined;
             this.dispatchEvent(new CustomEvent("job:completed", { detail: { job } }));
+            safeRelease({ success: true });
             resolveCompletion?.();
         });
         wrapper.onFailed((error, promptId) => {
+            this.debugLog("[debug] wrapper.onFailed", job.jobId, error.name);
             if (!job.promptId && promptId) {
                 job.promptId = promptId;
             }
@@ -510,9 +725,13 @@ export class WorkflowPool extends TypedEventTarget {
             // Cleanup timeouts
             cleanupNodeTimeout();
             rejectPending?.(error);
-            rejectCompletion?.(error);
+            completionError = error;
+            this.debugLog("[debug] resolveCompletion available", Boolean(resolveCompletion));
+            safeRelease({ success: false });
+            resolveCompletion?.();
         });
         try {
+            // Start the workflow execution
             const exec = wrapper.run();
             // Add timeout for execution start to prevent jobs getting stuck
             const executionStartTimeout = this.opts.executionStartTimeoutMs ?? 5000;
@@ -528,18 +747,21 @@ export class WorkflowPool extends TypedEventTarget {
                     })
                 ]);
                 await pendingWithTimeout;
-                clearTimeout(pendingTimeoutId);
             }
             else {
                 await pendingPromise;
             }
+            if (executionStartTimeout > 0) {
+                clearTimeout(pendingTimeoutId);
+            }
             this.activeJobs.set(job.jobId, {
                 reservation,
                 job,
                 clientId,
-                release,
+                release: (opts) => safeRelease(opts),
                 cancel: async () => {
                     try {
+                        wrapper.cancel("workflow pool cancel");
                         if (job.promptId) {
                             await client.ext.queue.interrupt(job.promptId);
                         }
@@ -547,36 +769,41 @@ export class WorkflowPool extends TypedEventTarget {
                     finally {
                         this.activeJobs.delete(job.jobId);
                         await this.queue.discard(reservation.reservationId, new Error("cancelled"));
-                        release({ success: false });
+                        safeRelease({ success: false });
                     }
                 }
             });
             const result = await exec;
+            // Wait for the wrapper to complete (onFinished or onFailed callback)
+            await completionPromise;
             if (result === false) {
-                // Execution failed - try to get the error from completionPromise rejection
-                try {
-                    await completionPromise;
-                }
-                catch (err) {
-                    throw err;
-                }
-                throw job.lastError ?? new Error("Execution failed");
+                const errorToThrow = (completionError instanceof Error ? completionError : undefined) ??
+                    (job.lastError instanceof Error ? job.lastError : undefined) ??
+                    new Error("Execution failed");
+                throw errorToThrow;
             }
-            await completionPromise;
             await this.queue.commit(reservation.reservationId);
-            release({ success: true });
+            safeRelease({ success: true });
         }
         catch (error) {
+            // Immediately release the client on any failure
+            safeRelease({ success: false });
             const latestStatus = this.jobStore.get(job.jobId)?.status;
             if (latestStatus === "cancelled") {
-                release({ success: false });
                 return;
             }
             job.lastError = error;
             job.status = "failed";
-            this.clientManager.recordFailure(clientId, job, error);
             const remainingAttempts = job.options.maxAttempts - job.attempts;
-            const willRetry = remainingAttempts > 0;
+            const failureAnalysis = analyzeWorkflowFailure(error);
+            this.rememberJobFailure(job, clientId, failureAnalysis);
+            if (failureAnalysis.blockClient === "permanent") {
+                this.addPermanentExclusion(job, clientId);
+                reservation.payload.options.excludeClientIds = [...(job.options.excludeClientIds ?? [])];
+            }
+            this.clientManager.recordFailure(clientId, job, error);
+            const hasRetryPath = this.hasRetryPath(job);
+            const willRetry = failureAnalysis.retryable && remainingAttempts > 0 && hasRetryPath;
             this.dispatchEvent(new CustomEvent("job:failed", {
                 detail: { job, willRetry }
             }));
@@ -589,19 +816,24 @@ export class WorkflowPool extends TypedEventTarget {
                 job.startedAt = undefined;
                 job.completedAt = undefined;
                 job.result = undefined;
+                reservation.payload.options.excludeClientIds = [...(job.options.excludeClientIds ?? [])];
                 await this.queue.retry(reservation.reservationId, { delayMs: delay });
                 this.dispatchEvent(new CustomEvent("job:queued", { detail: { job } }));
                 this.scheduleProcess(delay);
-                release({ success: false });
             }
             else {
                 job.completedAt = Date.now();
-                await this.queue.discard(reservation.reservationId, error);
-                release({ success: false });
+                const finalError = !hasRetryPath && failureAnalysis.type === "client_incompatible" && this.jobFailureAnalysis.has(job.jobId)
+                    ? this.createWorkflowNotSupportedError(job, error)
+                    : error;
+                job.lastError = finalError;
+                await this.queue.discard(reservation.reservationId, finalError);
+                this.clearJobFailures(job.jobId);
             }
         }
         finally {
             this.activeJobs.delete(job.jobId);
+            this.debugLog(`[runJob.finally] Job ${job.jobId.substring(0, 8)}... completed, calling processQueue()`);
             void this.processQueue();
         }
     }