dsh-multimodal-node 0.0.0-stage → 0.3.1

package/lib/gateway.js

@@ -0,0 +1,974 @@
+/**
+ * The gateway listener: a standalone HTTP server that speaks ComfyUI's own
+ * protocol on one port (8187 by default) so every existing ComfyUI client —
+ * the official frontend, an API script, another ComfyUI node — can keep posting
+ * JSON to a single address and never learn that two GPUs are behind it.
+ *
+ * What it fakes, and why that is safe:
+ *
+ *   /prompt      accepted here, queued, and forwarded later by the scheduler.
+ *                The id returned is a gateway id; the same id addresses
+ *                /history/<id> and /view forever.
+ *   /queue       our pool merged with whatever the instances' own queues hold.
+ *   /jobs        our pool merged with every instance's own job list, so the
+ *                dsh-comfyui task centre sees one list instead of the primary's
+ *                slice. Cancel routes are matched to the instance that holds
+ *                the job; /jobs/{id} resolves across the pool first.
+ *   /history     our finished runs merged with the instances' own history, so
+ *                a run started directly on 8188/8189 still shows up.
+ *   /system_stats, /object_info, /models, /embeddings, /features
+ *                answered from the primary instance (they describe the
+ *                installation, not the GPU), with the device list merged.
+ *   /ws          a real WebSocket pushing the same event names the frontend
+ *                listens for, sourced from the pool instead of one server.
+ *   /view        streamed from whichever instance holds the file.
+ *
+ * Anything not listed is relayed to the primary instance, so an endpoint this
+ * gateway does not know still behaves like ComfyUI.
+ */
+import { createServer } from 'node:http';
+import { errorMessage, pipeGuarded, readJson, sendJson, closeServer } from './http.js';
+import { WsServer } from './ws.js';
+
+const COMFY_ID = /^[0-9a-f-]{8,64}$/i;
+
+/** Accept `bytes=…` ranges on the merged media route (players seek with them). */
+function normalizePath(url) {
+    const path = url.split('?')[0];
+    return path.startsWith('/api/') ? path.slice(4) : path;
+}
+
+export class GatewayServer {
+    /**
+     * @param {{settings:()=>any, instances: import('./instances.js').InstanceManager,
+     *          pool: import('./pool.js').TaskPool, log:(level:string,message:string)=>void}} deps
+     */
+    constructor(deps) {
+        this.deps = deps;
+        this.server = null;
+        this.ws = new WsServer();
+        this.objectInfo = new Map(); // instanceId -> { ts, value }
+        this.primaryCache = { at: 0, id: null };
+        this.lastError = null;
+        this.port = null;
+        this.host = null;
+    }
+
+    get listening() {
+        return this.server !== null && this.server.listening;
+    }
+
+    /** Primary instance: the first enabled, healthy one; the graph source for object_info. */
+    async primary() {
+        const now = Date.now();
+        if (now - this.primaryCache.at < 3000 && this.primaryCache.id !== null) {
+            const known = this.deps.instances.get(this.primaryCache.id);
+            if (known !== undefined && known.health.ok) return known;
+        }
+        const candidates = this.deps.instances.enabled();
+        const healthy = candidates.find((runtime) => runtime.health.ok);
+        const chosen = healthy ?? candidates[0];
+        this.primaryCache = { at: now, id: chosen?.id ?? null };
+        return chosen;
+    }
+
+    async start() {
+        const settings = this.deps.settings().listen ?? {};
+        if (settings.enabled === false) return { ok: false, skipped: 'disabled' };
+        const host = settings.host ?? '0.0.0.0';
+        const port = Number(settings.port ?? 8287);
+        await this.stop();
+        const server = createServer((request, response) => {
+            this.handle(request, response).catch((error) => {
+                if (!response.headersSent) sendJson(response, 500, { error: errorMessage(error) });
+                else response.end();
+            });
+        });
+        server.on('upgrade', (request, socket, head) => {
+            const path = normalizePath(request.url ?? '/');
+            if (path !== '/ws') {
+                socket.destroy();
+                return;
+            }
+            this.ws.accept(request, socket);
+            // Greet with the current status so a fresh frontend paints immediately.
+            void this.pushStatus();
+        });
+        await new Promise((resolve, reject) => {
+            const onError = (error) => {
+                server.off('listening', onListening);
+                reject(error);
+            };
+            const onListening = () => {
+                server.off('error', onError);
+                resolve();
+            };
+            server.once('error', onError);
+            server.once('listening', onListening);
+            server.listen(port, host);
+        });
+        server.on('error', (error) => {
+            this.lastError = errorMessage(error);
+            this.deps.log('error', `gateway listen error: ${this.lastError}`);
+        });
+        this.server = server;
+        this.port = port;
+        this.host = host;
+        this.lastError = null;
+        this.deps.log('info', `gateway listening on ${host}:${port}`);
+        return { ok: true, host, port };
+    }
+
+    async stop() {
+        this.ws.dispose();
+        const server = this.server;
+        this.server = null;
+        if (server === null) return { ok: true };
+        // closeServer 会强制断开 keep-alive 连接，端口才真的释放（见 http.js）。
+        await closeServer(server);
+        return { ok: true };
+    }
+
+    async handle(request, response) {
+        const url = new URL(request.url ?? '/', 'http://gateway.invalid');
+        const path = normalizePath(request.url ?? '/');
+        const method = request.method ?? 'GET';
+
+        // CORS: callers are usually on the LAN (other PCs, other ComfyUI nodes).
+        response.setHeader('access-control-allow-origin', '*');
+        response.setHeader('access-control-allow-headers', 'content-type,authorization');
+        response.setHeader('access-control-allow-methods', 'GET,POST,HEAD,OPTIONS');
+        if (method === 'OPTIONS') {
+            response.writeHead(204);
+            response.end();
+            return;
+        }
+        /**
+         * 用户节点接口先走：/node/ 前缀独占，处理完就直接返回。
+         *
+         * 放在 CORS 之后、任何 ComfyUI 协议分支之前——否则一个 /node/ping 会被当成
+         * 「未知路由」转发到主实例，用户节点收到的就是一个 ComfyUI 的 404。
+         */
+        if (typeof this.deps.nodeRoutes === 'function' && this.deps.nodeRoutes(request, response, url, path, method) === true) {
+            return undefined;
+        }
+
+        /**
+         * API 接入端口的凭据闸门。
+         *
+         * 一个地方判完，把结果挂在请求上，后面的分支（尤其是 /prompt）按 tier 兜底。
+         * 放在这里而不是每个分支里各判一次：漏判一条路由就是一次未授权提交，而
+         * "每个分支都记得判" 这种约定，第二个人的第一个改动就会破。
+         *
+         * 未开启校验时 verdict.checked 为 false —— 放行但标注，审计日志里能分开。
+         */
+        if (typeof this.deps.apiGuard === 'function') {
+            const verdict = this.deps.apiGuard(request, url);
+            if (verdict.ok !== true) {
+                return sendJson(response, 401, { ok: false, error: verdict.error ?? '缺少 apikey', authRequired: true });
+            }
+            request.dshAuth = verdict;
+        }
+
+        if (path === '/gateway/status') return sendJson(response, 200, this.statusPayload());
+        if (path === '/gateway/instances' && method === 'POST') {
+            const body = await readJson(request).catch(() => ({}));
+            const action = String(body.action ?? '');
+            const id = String(body.id ?? '');
+            try {
+                if (action === 'start') return sendJson(response, 200, await this.deps.instances.start(id));
+                if (action === 'stop') return sendJson(response, 200, await this.deps.instances.stop(id));
+                if (action === 'restart') return sendJson(response, 200, await this.deps.instances.restart(id));
+            } catch (error) {
+                return sendJson(response, 200, { ok: false, error: errorMessage(error) });
+            }
+        }
+
+        if (path === '/prompt' && method === 'POST') {
+            let body;
+            try {
+                body = await readJson(request);
+            } catch (error) {
+                return sendJson(response, 400, { error: errorMessage(error) });
+            }
+            /**
+             * 带 key 的调用按 key 的分级兜底一次。
+             *
+             * /prompt 本身不审查（程序发的就是要的，这是 03 号插件锁了源码级测试的边界），
+             * 但**发出去一把 SFW 的 key** 必须真的限住内容，否则那把 key 等于没发。
+             * 没开校验时 checked=false，这条判断整体跳过。
+             */
+            const auth = request.dshAuth;
+            if (auth !== undefined && auth.checked === true && auth.tier !== 'nsfw') {
+                const { classifyWorkflow, visibleTo } = await import('./tiers.js');
+                const classification = classifyWorkflow({ api: body?.prompt ?? {}, name: body?.extra_data?.workflowName ?? '' });
+                if (!visibleTo(auth.tier, classification.tier)) {
+                    this.deps.log?.('warn', `apikey ${auth.name} 提交了成人工作流，按 tier=sfw 拒绝`);
+                    return sendJson(response, 403, {
+                        ok: false,
+                        error: '这个 apikey 的分级是 SFW，不能提交成人内容的工作流',
+                        tier: auth.tier,
+                        reason: classification.reason,
+                    });
+                }
+            }
+            try {
+                const reply = this.deps.pool.enqueue(body);
+                return sendJson(response, 200, reply);
+            } catch (error) {
+                return sendJson(response, 400, { error: errorMessage(error), node_errors: {} });
+            }
+        }
+
+        if (path === '/queue' && method === 'GET') {
+            const queue = await this.deps.pool.queueView();
+            return sendJson(response, 200, queue);
+        }
+        if (path === '/queue' && method === 'POST') {
+            const body = await readJson(request).catch(() => ({}));
+            if (body.clear === true) {
+                for (const task of this.deps.pool.ordered()) this.deps.pool.cancel(task.id);
+                for (const runtime of this.deps.instances.enabled()) {
+                    try {
+                        await this.deps.pool.backend(runtime).clearQueue();
+                    } catch {
+                        // unreachable instance: its queue is not ours to clear
+                    }
+                }
+                return sendJson(response, 200, {});
+            }
+            if (Array.isArray(body.delete)) {
+                for (const id of body.delete) this.deps.pool.cancel(String(id));
+                return sendJson(response, 200, {});
+            }
+            return sendJson(response, 200, {});
+        }
+
+        if (path === '/interrupt' && method === 'POST') {
+            const body = await readJson(request).catch(() => ({}));
+            if (typeof body.prompt_id === 'string' && body.prompt_id !== '') {
+                const known = this.deps.pool.tasks.has(body.prompt_id);
+                const result = this.deps.pool.cancel(body.prompt_id);
+                // A prompt id we do not track is somebody else's run — it was started
+                // directly on an instance — so the only way to stop it is to ask the
+                // instances. For an id we do track, `cancel` already interrupted the
+                // right upstream prompt; broadcasting it again would stop unrelated
+                // work on another GPU.
+                if (!known) {
+                    await Promise.all(
+                        this.deps.instances.enabled().map(async (runtime) => {
+                            try {
+                                await this.deps.pool.backend(runtime).interrupt(body.prompt_id);
+                            } catch {
+                                // instance asleep: nothing to interrupt there
+                            }
+                        }),
+                    );
+                }
+                return sendJson(response, 200, result);
+            }
+            return sendJson(response, 200, {});
+        }
+
+        if (path === '/free' && method === 'POST') {
+            const body = await readJson(request).catch(() => ({}));
+            await Promise.all(
+                this.deps.instances.enabled().map(async (runtime) => {
+                    try {
+                        await this.deps.pool.backend(runtime).free(body);
+                    } catch {
+                        // ignore a sleeping instance
+                    }
+                }),
+            );
+            return sendJson(response, 200, {});
+        }
+
+        if (path === '/history' && method === 'GET') {
+            const max = Math.min(1000, Math.max(1, Number(url.searchParams.get('max_items') ?? '64') || 64));
+            const history = await this.mergedHistory(max);
+            return sendJson(response, 200, history);
+        }
+        if (path.startsWith('/history/') && method === 'GET') {
+            const id = decodeURIComponent(path.slice('/history/'.length));
+            const entry = await this.mergedHistoryOne(id);
+            return sendJson(response, 200, entry === undefined ? {} : { [id]: entry });
+        }
+
+        /**
+         * The job API — what the dsh-comfyui task centre actually polls.
+         *
+         * Two things had to be handled here rather than by the generic relay:
+         *
+         *  1. The list is per-instance on a real server: 8188 answers with its own history
+         *     and knows nothing about 8189. Relaying the list to the primary made half the
+         *     farm invisible ("生成完的记录只出现一部分"), so every enabled instance is
+         *     queried and the rows are merged with the gateway's own finished tasks.
+         *  2. `status=all` is the gateway's dialect, not ComfyUI's — a real server rejects
+         *     it with 400 ("Invalid status value(s): all"), and the caller shows that as a
+         *     red error box. It is translated to "no status filter" here.
+         */
+        if (path === '/jobs' && method === 'GET') {
+            return sendJson(response, 200, await this.mergedJobs(url));
+        }
+        if (path.startsWith('/jobs/') && method === 'GET') {
+            const id = decodeURIComponent(path.slice('/jobs/'.length));
+            return sendJson(response, 200, await this.mergedJobOne(id));
+        }
+        if ((path === '/jobs/cancel' || /^\/jobs\/[^/]+\/cancel$/.test(path)) && method === 'POST') {
+            const raw = await readRaw(request);
+            return this.proxyJobMutation(path, raw, request, response);
+        }
+
+        if (path === '/view') {
+            return this.proxyView(url, request, response);
+        }
+
+        if (path === '/system_stats') {
+            return sendJson(response, 200, await this.mergedStats());
+        }
+
+        if (path === '/object_info' || path.startsWith('/object_info/')) {
+            const primary = await this.primary();
+            if (primary === undefined) return sendJson(response, 503, { error: '没有可用的 ComfyUI 实例' });
+            try {
+                const value = await this.cachedObjectInfo(primary, path.slice('/object_info'.length));
+                return sendJson(response, 200, value);
+            } catch (error) {
+                return sendJson(response, 502, { error: errorMessage(error) });
+            }
+        }
+
+        if (path === '/models' || path.startsWith('/models/')) {
+            const primary = await this.primary();
+            if (primary === undefined) return sendJson(response, 200, []);
+            try {
+                const value = await this.deps.pool.backend(primary).json(path);
+                return sendJson(response, 200, value);
+            } catch (error) {
+                return sendJson(response, 502, { error: errorMessage(error) });
+            }
+        }
+
+        /**
+         * Everything else: relay to the primary instance.
+         *
+         * Two separate faults used to live here, both surfacing as an unusable message the
+         * calling DSH then showed as 502 or "no such endpoint":
+         *
+         *  1. The `/api` prefix was stripped unconditionally. That is right for the classic
+         *     routes (ComfyUI mirrors every one of them under /api, and an `/api`-only
+         *     reverse proxy is the reason the bare form is worth trying), but WRONG for the
+         *     job API: `/api/jobs` exists only with its prefix, so rewriting it to `/jobs`
+         *     asked a healthy 0.31.1 server for a route it has never had and got 404 back.
+         *     `upstreamCandidates()` therefore offers the caller's own spelling first and
+         *     the bare form only as a 404 fallback.
+         *  2. `fetch` reports DNS failure, refusal, TLS refusal and timeouts as the same
+         *     `fetch failed`, and the Host header was being replaced with the upstream's.
+         *     The caller's authorization is forwarded, their Host is not, a loopback name is
+         *     pinned to 127.0.0.1, and `describeFetchError` names the cause.
+         */
+        const primary = await this.primary();
+        if (primary === undefined) return sendJson(response, 503, { error: '没有可用的 ComfyUI 实例' });
+        const candidates = upstreamCandidates(primary.instance.baseUrl, request.url ?? '/');
+        const body = method === 'GET' || method === 'HEAD' ? undefined : await readRaw(request);
+        const headers = {};
+        if (request.headers.authorization) headers.authorization = request.headers.authorization;
+        if (request.headers['content-type']) headers['content-type'] = request.headers['content-type'];
+        let lastTarget = candidates[0];
+        try {
+            let upstream;
+            for (const target of candidates) {
+                lastTarget = target;
+                upstream = await fetch(target, {
+                    method,
+                    headers,
+                    body,
+                    // A relay for an unknown route cannot know how long the work takes. Jobs
+                    // listing with a big limit against a busy ComfyUI easily exceeds 30s, and
+                    // a timeout surfaces as the same opaque `fetch failed`; 120s is generous
+                    // for anything a caller would poll synchronously.
+                    signal: AbortSignal.timeout(120000),
+                });
+                // Only a 404 justifies the other spelling: every other status is the real
+                // answer to the route the caller asked for and must pass through untouched.
+                if (upstream.status !== 404 || target === candidates[candidates.length - 1]) break;
+                await upstream.body?.cancel().catch(() => undefined);
+                this.deps.log('info', `relay ${target} -> 404, retrying the other spelling`);
+            }
+            const buffer = Buffer.from(await upstream.arrayBuffer());
+            response.writeHead(upstream.status, {
+                'content-type': upstream.headers.get('content-type') ?? 'application/octet-stream',
+                'content-length': String(buffer.length),
+            });
+            response.end(buffer);
+        } catch (error) {
+            const reason = describeFetchError(error);
+            this.deps.log('warn', `relay ${lastTarget} failed: ${reason}`);
+            sendJson(response, 502, {
+                error: `上游 ${primary.id} (${lastTarget}) 请求失败：${reason}`,
+                upstream: primary.id,
+                target: lastTarget,
+                detail: errorMessage(error),
+            });
+        }
+    }
+
+    async cachedObjectInfo(runtime, suffix) {
+        const now = Date.now();
+        const cached = this.objectInfo.get(runtime.id);
+        let value;
+        if (cached !== undefined && now - cached.ts < 60000) {
+            value = cached.value;
+        } else {
+            value = await this.deps.pool.backend(runtime).objectInfo();
+            this.objectInfo.set(runtime.id, { ts: now, value });
+        }
+        if (suffix === '' || suffix === '/') return value;
+        const key = decodeURIComponent(suffix.slice(1));
+        return key in value ? { [key]: value[key] } : {};
+    }
+
+    /**
+     * One merged /history: our finished runs plus each instance's own history.
+     * Ordering is by execution start, newest first, because the frontend sorts
+     * by nothing and simply paints the object in insertion order.
+     */
+    async mergedHistory(max) {
+        const merged = new Map();
+        const limit = Math.min(max, this.deps.settings().historyItems ?? 120);
+        for (const task of this.deps.pool.tasks.values()) {
+            if (task.state !== 'done' && task.state !== 'failed') continue;
+            if (task.historyEntry !== undefined && task.historyEntry !== null) {
+                merged.set(task.id, decorate(task.historyEntry, task));
+            }
+        }
+        await Promise.all(
+            this.deps.instances.enabled().map(async (runtime) => {
+                try {
+                    const history = await this.deps.pool.backend(runtime).history(limit);
+                    for (const [id, entry] of Object.entries(history ?? {})) {
+                        if (merged.has(id)) continue;
+                        // A run this gateway submitted appears under its upstream id;
+                        // its gateway id is what callers poll, so do not double-list it.
+                        if ([...this.deps.pool.tasks.values()].some((task) => task.upstreamId === id)) continue;
+                        merged.set(id, decorate(entry, undefined, runtime.id));
+                    }
+                } catch {
+                    // an offline instance contributes nothing
+                }
+            }),
+        );
+        const entries = [...merged.entries()]
+            .sort((left, right) => startTime(right[1]) - startTime(left[1]))
+            .slice(0, limit);
+        return Object.fromEntries(entries);
+    }
+
+    /**
+     * GET one JSON route from one instance, trying the caller's spelling and then the bare
+     * form on a 404. Needed because the job API lives under `/api` while the classic routes
+     * are mirrored — `Backend` only speaks the bare form.
+     */
+    async instanceJson(runtime, rawPath, timeoutMs = 30000) {
+        let lastError = null;
+        for (const target of upstreamCandidates(runtime.instance.baseUrl, rawPath)) {
+            try {
+                const headers = {};
+                if (typeof runtime.instance.apiKey === 'string' && runtime.instance.apiKey !== '') {
+                    headers.authorization = `Bearer ${runtime.instance.apiKey}`;
+                }
+                const upstream = await fetch(target, { headers, signal: AbortSignal.timeout(timeoutMs) });
+                const text = await upstream.text();
+                if (upstream.status === 404) {
+                    lastError = new Error(`${target} -> HTTP 404`);
+                    continue;
+                }
+                if (!upstream.ok) throw new Error(`${target} -> HTTP ${upstream.status} ${text.slice(0, 160)}`);
+                return text === '' ? undefined : JSON.parse(text);
+            } catch (error) {
+                lastError = error;
+            }
+        }
+        throw lastError ?? new Error(`${runtime.id} ${rawPath}: no route answered`);
+    }
+
+    async mergedHistoryOne(id) {
+        const task = this.deps.pool.tasks.get(id);
+        if (task !== undefined && task.historyEntry !== undefined && task.historyEntry !== null) {
+            return decorate(task.historyEntry, task);
+        }
+        if (task !== undefined && task.upstreamId !== null) {
+            const runtime = task.backendId === null ? undefined : this.deps.instances.get(task.backendId);
+            if (runtime !== undefined) {
+                try {
+                    const history = await this.deps.pool.backend(runtime).historyOf(task.upstreamId);
+                    const entry = history?.[task.upstreamId];
+                    if (entry !== undefined) return decorate(entry, task);
+                } catch {
+                    // fall through to a direct search
+                }
+            }
+        }
+        for (const runtime of this.deps.instances.enabled()) {
+            try {
+                const history = await this.deps.pool.backend(runtime).historyOf(id);
+                if (history?.[id] !== undefined) return decorate(history[id], undefined, runtime.id);
+            } catch {
+                // try the next instance
+            }
+        }
+        return undefined;
+    }
+
+    /**
+     * One merged job list: our own tasks plus every instance's own rows.
+     *
+     * The caller is a ComfyUI frontend (or the dsh-comfyui panel), so the shape must stay
+     * ComfyUI's: `{jobs: [...], pagination: {offset, limit, total, has_more}}`. Paging is
+     * applied AFTER the merge — paging per instance would return `limit` rows from each and
+     * then truncate, which silently drops the newest rows of the second GPU.
+     */
+    async mergedJobs(url) {
+        const query = url.searchParams;
+        const statusParam = (query.get('status') ?? '').split(',').map((s) => s.trim()).filter((s) => s !== '');
+        // `all` is the gateway's dialect: a real ComfyUI rejects it with 400.
+        const statuses = statusParam.includes('all') || statusParam.includes('') ? [] : statusParam;
+        const limitRaw = Number(query.get('limit') ?? '');
+        const limit = Number.isFinite(limitRaw) && limitRaw > 0 ? Math.floor(limitRaw) : null;
+        const offsetRaw = Number(query.get('offset') ?? '');
+        const offset = Number.isFinite(offsetRaw) && offsetRaw > 0 ? Math.floor(offsetRaw) : 0;
+
+        const seen = new Set();
+        const jobs = [];
+
+        // Our own tasks first: they carry the gateway id callers already know.
+        for (const task of this.deps.pool.tasks.values()) {
+            if (task.state === 'queued' || task.state === 'running') {
+                const status = task.state === 'running' ? 'in_progress' : 'pending';
+                if (statuses.length > 0 && !statuses.includes(status)) continue;
+                jobs.push({
+                    id: task.id,
+                    status,
+                    priority: task.priority,
+                    create_time: task.enqueuedAt,
+                    outputs_count: 0,
+                    workflow_id: null,
+                });
+                seen.add(task.id);
+                if (task.upstreamId !== null) seen.add(task.upstreamId);
+                continue;
+            }
+            const status = task.state === 'done' ? 'completed' : task.state === 'cancelled' ? 'cancelled' : 'failed';
+            if (statuses.length > 0 && !statuses.includes(status)) continue;
+            const entry = task.historyEntry ?? null;
+            jobs.push({
+                id: task.id,
+                status,
+                priority: task.priority,
+                create_time: task.enqueuedAt,
+                execution_start_time: task.startedAt,
+                execution_end_time: task.finishedAt,
+                execution_error: status === 'failed' ? { exception_message: task.error ?? 'failed' } : null,
+                outputs_count: countOutputs(task.outputs),
+                workflow_id: null,
+            });
+            void entry;
+            seen.add(task.id);
+            if (task.upstreamId !== null) seen.add(task.upstreamId);
+        }
+
+        // Then whatever each instance reports, minus anything already represented above.
+        await Promise.all(
+            this.deps.instances.enabled().map(async (runtime) => {
+                try {
+                    const params = new URLSearchParams();
+                    if (statuses.length > 0) params.set('status', statuses.join(','));
+                    params.set('sort_by', 'created_at');
+                    params.set('sort_order', 'desc');
+                    // No limit here on purpose: the merge pages once, at the end.
+                    const reply = await this.instanceJson(runtime, `/api/jobs?${params.toString()}`, 120000);
+                    for (const job of Array.isArray(reply?.jobs) ? reply.jobs : []) {
+                        const id = typeof job?.id === 'string' ? job.id : '';
+                        if (id === '' || seen.has(id)) continue;
+                        seen.add(id);
+                        jobs.push({ ...job, backendId: runtime.id });
+                    }
+                } catch {
+                    // an offline instance contributes nothing, same as mergedHistory
+                }
+            }),
+        );
+
+        jobs.sort((left, right) => Number(right.create_time ?? 0) - Number(left.create_time ?? 0));
+        const total = jobs.length;
+        const page = jobs.slice(offset, limit === null ? undefined : offset + limit);
+        return { jobs: page, pagination: { offset, limit, total, has_more: offset + page.length < total } };
+    }
+
+    /** One job by id across the pool and every instance (first hit wins). */
+    async mergedJobOne(id) {
+        const task = this.deps.pool.tasks.get(id);
+        if (task !== undefined) {
+            return {
+                id: task.id,
+                status: task.state === 'queued' ? 'pending' : task.state === 'running' ? 'in_progress' : task.state === 'done' ? 'completed' : task.state,
+                priority: task.priority,
+                create_time: task.enqueuedAt,
+                execution_start_time: task.startedAt,
+                execution_end_time: task.finishedAt,
+                execution_error: task.error === null ? null : { exception_message: task.error },
+                outputs_count: countOutputs(task.outputs),
+                workflow_id: null,
+            };
+        }
+        for (const runtime of this.deps.instances.enabled()) {
+            try {
+                return await this.instanceJson(runtime, `/api/jobs/${encodeURIComponent(id)}`);
+            } catch {
+                // 404 here is "not this instance"; keep looking
+            }
+        }
+        return { error: 'Job not found' };
+    }
+
+    /**
+     * Cancel passes through to whoever holds the job.
+     *
+     * A gateway task is cancelled in the pool (which also stops the right upstream prompt);
+     * anything else is an id that lives on exactly one instance, so the call is broadcast
+     * and the first non-error answer wins. Broadcasting a *cancel* is safe in a way
+     * broadcasting an interrupt is not: the instances ignore ids they do not know.
+     */
+    async proxyJobMutation(path, raw, request, response) {
+        let body = {};
+        try {
+            body = raw === undefined || raw.length === 0 ? {} : JSON.parse(raw.toString('utf8'));
+        } catch {
+            return sendJson(response, 400, { error: 'invalid JSON body' });
+        }
+        const single = /^\/jobs\/([^/]+)\/cancel$/.exec(path);
+        if (single !== null) {
+            const id = decodeURIComponent(single[1]);
+            if (this.deps.pool.tasks.has(id)) {
+                const result = this.deps.pool.cancel(id);
+                return sendJson(response, 200, { cancelled: result.ok === true, ...(result.ok === true ? {} : { error: result.error }) });
+            }
+        } else if (Array.isArray(body.job_ids)) {
+            const known = body.job_ids.filter((id) => typeof id === 'string' && this.deps.pool.tasks.has(id));
+            for (const id of known) this.deps.pool.cancel(id);
+        }
+        const headers = {};
+        if (request.headers.authorization) headers.authorization = request.headers.authorization;
+        if (request.headers['content-type']) headers['content-type'] = request.headers['content-type'];
+        let lastError = null;
+        for (const runtime of this.deps.instances.enabled()) {
+            for (const target of upstreamCandidates(runtime.instance.baseUrl, `${path}`)) {
+                try {
+                    const upstream = await fetch(target, {
+                        method: 'POST',
+                        headers,
+                        body: raw,
+                        signal: AbortSignal.timeout(30000),
+                    });
+                    const buffer = Buffer.from(await upstream.arrayBuffer());
+                    if (upstream.status === 404) {
+                        lastError = `${runtime.id} ${target} -> 404`;
+                        continue;
+                    }
+                    response.writeHead(upstream.status, {
+                        'content-type': upstream.headers.get('content-type') ?? 'application/json',
+                        'content-length': String(buffer.length),
+                    });
+                    response.end(buffer);
+                    return undefined;
+                } catch (error) {
+                    lastError = describeFetchError(error);
+                }
+            }
+        }
+        return sendJson(response, 200, { cancelled: false, error: lastError ?? 'no instance accepted the cancel' });
+    }
+
+    /** /system_stats from the primary instance, with every bound GPU listed. */
+    async mergedStats() {
+        const primary = await this.primary();
+        if (primary === undefined) {
+            return { system: { comfyui_version: 'gateway', os: process.platform, python_version: null }, devices: [] };
+        }
+        let base;
+        try {
+            base = await this.deps.pool.backend(primary).systemStats();
+        } catch (error) {
+            return { system: { comfyui_version: 'gateway', os: process.platform }, devices: [], error: errorMessage(error) };
+        }
+        const devices = [];
+        for (const runtime of this.deps.instances.enabled()) {
+            for (const device of runtime.health.devices ?? []) {
+                devices.push({ ...device, index: devices.length });
+            }
+        }
+        return {
+            ...base,
+            system: {
+                ...base.system,
+                gateway_port: this.port,
+                gateway_instances: this.deps.instances.enabled().map((runtime) => ({
+                    id: runtime.id,
+                    baseUrl: runtime.instance.baseUrl,
+                    healthy: runtime.health.ok,
+                })),
+            },
+            devices: devices.length > 0 ? devices : base.devices ?? [],
+        };
+    }
+
+    /** Find which instance holds this file, then stream it through. */
+    async proxyView(url, request, response) {
+        const filename = url.searchParams.get('filename') ?? url.searchParams.get('file') ?? '';
+        const subfolder = url.searchParams.get('subfolder') ?? '';
+        const type = url.searchParams.get('type') ?? 'output';
+        if (filename === '') return sendJson(response, 400, { error: 'filename is required' });
+
+        const params = new URLSearchParams({ filename, subfolder, type });
+        const promptId = url.searchParams.get('prompt_id');
+        const ordered = [];
+        if (promptId !== null && promptId !== '') {
+            let task = this.deps.pool.tasks.get(promptId);
+            if (task === undefined) {
+                task = [...this.deps.pool.tasks.values()].find((candidate) => candidate.upstreamId === promptId);
+            }
+            const runtime = task?.backendId === null || task?.backendId === undefined ? undefined : this.deps.instances.get(task.backendId);
+            if (runtime !== undefined) ordered.push(runtime);
+        }
+        for (const runtime of this.deps.instances.enabled()) {
+            if (!ordered.includes(runtime)) ordered.push(runtime);
+        }
+        const method = request.method === 'HEAD' ? 'HEAD' : 'GET';
+        let lastError = 'no instance holds that file';
+        for (const runtime of ordered) {
+            try {
+                const stream = await this.deps.pool.backend(runtime).openView(params.toString(), method, request.headers.range);
+                const head = {
+                    'content-type': stream.headers.get('content-type') ?? 'application/octet-stream',
+                    'accept-ranges': 'bytes',
+                    'cache-control': 'private, max-age=3600',
+                    'access-control-allow-origin': '*',
+                };
+                for (const name of ['content-length', 'content-range']) {
+                    const value = stream.headers.get(name);
+                    if (value !== null) head[name] = value;
+                }
+                response.writeHead(stream.status, head);
+                if (method === 'HEAD' || stream.body === null) {
+                    response.end();
+                    return undefined;
+                }
+                // The body carries a 60 s AbortSignal; a bare `.pipe()` here would let
+                // a client that walked away turn that timeout into an uncaught exception
+                // and take the whole Host down. See pipeGuarded().
+                pipeGuarded(stream.body, response);
+                return undefined;
+            } catch (error) {
+                lastError = errorMessage(error);
+            }
+        }
+        return sendJson(response, 404, { error: lastError });
+    }
+
+    statusPayload() {
+        const snapshot = this.deps.pool.snapshot();
+        return {
+            ok: true,
+            listening: this.listening,
+            host: this.host,
+            port: this.port,
+            lastError: this.lastError,
+            wsClients: this.ws.size,
+            ...snapshot,
+            instances: this.deps.instances.list().map((runtime) => ({
+                id: runtime.id,
+                label: runtime.instance.label,
+                baseUrl: runtime.instance.baseUrl,
+                enabled: runtime.instance.enabled,
+                healthy: runtime.health.ok,
+                version: runtime.health.version,
+                latencyMs: runtime.health.latencyMs,
+                devices: runtime.health.devices,
+                lastError: runtime.lastError ?? runtime.health.error,
+                speed: Number((runtime.speed ?? 1).toFixed(2)),
+                speedSamples: runtime.speedSamples,
+                maxConcurrent: runtime.instance.maxConcurrent,
+                weight: runtime.instance.weight,
+                control: runtime.instance.control.type,
+                /**
+                 * The full control block, so the panel can edit batPath/cwd.
+                 *
+                 * `control` above is the *type string* — the panel must never spread it
+                 * into an object (that is what made the 托管方式 dropdown snap back to
+                 * 不托管: `{...'spawn'}` yields `{}` with no type). Read this one instead.
+                 */
+                controlDetail: runtime.instance.control,
+                pid: runtime.pid,
+                starting: runtime.starting,
+                stopping: runtime.stopping,
+                autoStart: runtime.instance.autoStart,
+            })),
+        };
+    }
+
+    /** Push the events the official frontend listens for. */
+    async pushStatus() {
+        if (this.ws.size === 0) return;
+        this.ws.broadcast('status', {
+            status: { exec_info: { queue_remaining: this.deps.pool.ordered().length } },
+            sid: 'gateway',
+        });
+        for (const task of this.deps.pool.tasks.values()) {
+            if (task.state === 'running' && task.upstreamId !== null) {
+                this.ws.broadcast('executing', { node: null, prompt_id: task.id });
+                if (task.progress?.position === 'running') {
+                    this.ws.broadcast('execution_start', { prompt_id: task.id });
+                }
+            }
+            if (task.state === 'done' && task.finishedAt !== null && Date.now() - task.finishedAt < 4000 && !task.announced) {
+                task.announced = true;
+                this.ws.broadcast('executing', { node: null, prompt_id: task.id });
+                this.ws.broadcast('execution_success', { prompt_id: task.id, timestamp: task.finishedAt });
+                this.ws.broadcast('executed', {
+                    node: null,
+                    display_node: null,
+                    output: task.outputs ?? {},
+                    prompt_id: task.id,
+                });
+            }
+            if (task.state === 'failed' && task.finishedAt !== null && Date.now() - task.finishedAt < 4000 && !task.announcedFailure) {
+                task.announcedFailure = true;
+                this.ws.broadcast('execution_error', {
+                    prompt_id: task.id,
+                    node_id: null,
+                    node_type: null,
+                    executed: [],
+                    exception_message: task.error ?? 'failed',
+                    exception_type: 'gateway',
+                    traceback: '',
+                    current_inputs: {},
+                    current_outputs: {},
+                });
+            }
+        }
+    }
+}
+
+function decorate(entry, task, backendId) {
+    if (entry === null || typeof entry !== 'object') return entry;
+    const gateway = task === undefined ? undefined : { id: task.id, backendId: task.backendId, cost: task.cost };
+    return {
+        ...entry,
+        meta: {
+            ...(entry.meta ?? {}),
+            ...(gateway === undefined ? {} : { gateway }),
+            ...(backendId === undefined ? {} : { backendId }),
+        },
+    };
+}
+
+function startTime(entry) {
+    const messages = entry?.status?.messages ?? [];
+    for (const message of messages) {
+        if (Array.isArray(message) && message[0] === 'execution_start') return Number(message[1]?.timestamp ?? 0);
+    }
+    return 0;
+}
+
+async function readRaw(request) {
+    const chunks = [];
+    for await (const chunk of request) chunks.push(chunk);
+    return chunks.length === 0 ? undefined : Buffer.concat(chunks);
+}
+
+/**
+ * Ordered upstream URLs for a relayed request.
+ *
+ * The gateway accepts both spellings (`/api/queue` and `/queue`), and the old relay
+ * assumed ComfyUI did the same, so it stripped `/api` on the way out. That assumption is
+ * wrong for the job API: `server.py` declares `/api/jobs`, `/api/jobs/{job_id}`,
+ * `/api/jobs/{job_id}/cancel` and `/api/jobs/cancel` *literally with the prefix*, and the
+ * "mirror every route under /api" pass only ever ADDS a prefix — nothing registers a bare
+ * `/jobs`. A real 0.31.1 server answers `GET /api/jobs` with 200 and `GET /jobs` with 404.
+ * Stripping the prefix therefore converted a working endpoint into "HTTP 404" for every
+ * caller, which is exactly how the dsh-comfyui task centre ended up showing nothing but
+ * `ComfyUI /api/jobs?... failed: HTTP 404`.
+ *
+ * So the prefix is no longer stripped blind: the request goes out as the caller spelled it
+ * first, and only a 404 falls back to the bare form. That fallback is what keeps an
+ * `/api`-only reverse proxy working (such a proxy forwards just `/api/*` and rejects the
+ * bare path). The query string is preserved throughout — `/jobs?status=…` is meaningless
+ * without it.
+ */
+export function upstreamCandidates(baseUrl, rawUrl) {
+    // Loopback written as a name can resolve to ::1 while ComfyUI listens on 127.0.0.1.
+    // Pinning the literal address removes a whole class of "fetch failed" that says
+    // nothing about the actual fault.
+    const base = String(baseUrl ?? '')
+        .replace(/\/+$/, '')
+        .replace(/\/\/(localhost|\[::1\])(?=[:/]|$)/i, '//127.0.0.1');
+    const raw = String(rawUrl ?? '/');
+    const queryAt = raw.indexOf('?');
+    const rawPath = queryAt < 0 ? raw : raw.slice(0, queryAt);
+    const query = queryAt < 0 ? '' : raw.slice(queryAt);
+    const path = rawPath === '' ? '/' : rawPath;
+
+    const urls = [`${base}${path}${query}`];
+    let bare = path;
+    if (path.startsWith('/api/')) bare = path.slice(4);
+    else if (path === '/api') bare = '/';
+    if (bare !== path) urls.push(`${base}${bare}${query}`);
+    return urls;
+}
+
+/** The preferred upstream URL for a relayed request (first candidate). */
+export function upstreamUrl(baseUrl, rawUrl) {
+    return upstreamCandidates(baseUrl, rawUrl)[0];
+}
+
+/**
+ * Turn an undici/Node fetch failure into something an operator can act on.
+ *
+ * `fetch failed` alone is the string that started this: it is what Node prints for DNS
+ * failure, connection refused, TLS refusal and a dozen other causes, all wrapped in a
+ * `cause`. Unwrapping the cause is the difference between "it is down" and "it timed out".
+ */
+export function describeFetchError(error) {
+    if (error === null || error === undefined) return '未知错误';
+    const name = String(error.name ?? '');
+    if (name === 'TimeoutError' || name === 'AbortError') return '超时（上游没有在限定时间内回应）';
+    const cause = error.cause ?? {};
+    const code = String(cause.code ?? '');
+    /**
+     * `bad port` is not a network failure at all: Node's fetch refuses a list of reserved
+     * ports (1, 9, 25, 110, …) before opening a socket. Reporting it as a connection problem
+     * sends the operator to look at the wrong machine, so it gets its own sentence.
+     */
+    if (/bad port/i.test(String(cause.message ?? ''))) {
+        return '端口被 Node 拒绝（1/9/25 这类保留端口不允许 fetch，请换成真实的服务端口）';
+    }
+    const map = {
+        ECONNREFUSED: '连接被拒绝（该端口没有服务在监听）',
+        ENOTFOUND: '域名解析失败',
+        EAI_AGAIN: '域名解析暂时失败',
+        ECONNRESET: '连接被重置',
+        EHOSTUNREACH: '主机不可达',
+        ENETUNREACH: '网络不可达',
+        ETIMEDOUT: '连接超时',
+        UND_ERR_CONNECT_TIMEOUT: '连接超时',
+        CERT_HAS_EXPIRED: '证书已过期',
+        DEPTH_ZERO_SELF_SIGNED_CERT: '自签名证书被拒绝',
+    };
+    if (code !== '' && map[code] !== undefined) return `${map[code]}（${code}）`;
+    const message = String(cause.message ?? error.message ?? '');
+    if (code !== '') return `${message || '请求失败'}（${code}）`;
+    return message || '请求失败';
+}
+
+export { COMFY_ID };