dsh-multimodal-node 0.0.0-stage → 0.2.2

package/lib/instances.js

@@ -0,0 +1,554 @@
+/**
+ * Instance registry + local process control.
+ *
+ * An "instance" is one ComfyUI server bound to the gateway: one GPU, one port.
+ * `control.type` decides what start/stop mean:
+ *
+ *   none   — the instance is somebody else's process; only drain/bind apply.
+ *   spawn  — this gateway owns the process: start = spawn(command), stop = kill
+ *            the whole tree. Use it for the GPUs on this desktop.
+ *   ssh    — same, but the command runs on another machine over ssh.
+ */
+import { spawn, spawnSync } from 'node:child_process';
+import { existsSync } from 'node:fs';
+import { dirname } from 'node:path';
+import { probe } from './comfy.js';
+
+const ID_PATTERN = /^[a-z0-9_-]{1,32}$/;
+
+export function normalizeBaseUrl(raw) {
+    let text = String(raw ?? '').trim();
+    if (text === '') return '';
+    if (!/^https?:\/\//i.test(text)) text = `http://${text}`;
+    try {
+        const url = new URL(text);
+        return `${url.protocol}//${url.host}`;
+    } catch {
+        return '';
+    }
+}
+
+export function normalizeInstance(input, index = 0) {
+    const baseUrl = normalizeBaseUrl(input.baseUrl ?? input.url);
+    const derived = baseUrl === '' ? `p${index}` : `p${new URL(baseUrl).port || '80'}`;
+    const id = typeof input.id === 'string' && ID_PATTERN.test(input.id) ? input.id : derived;
+    const control = input.control ?? {};
+    /**
+     * Two types only, and the decision is "does a bat file exist?" — not which of
+     * several launch mechanisms to learn.
+     *
+     *   script — the gateway runs `<batPath>`; stopping kills the whole tree.
+     *   none   — somebody else owns the process; only dispatch applies.
+     *
+     * `type: 'ssh'` and free-form `command` are retired. Migration is deliberately
+     * conservative, because the alternative is silently launching something the
+     * operator can no longer see or edit in the panel:
+     *
+     *   script + batPath        → kept as-is
+     *   spawn  + *.bat|*.cmd    → becomes script(batPath = that path)
+     *   spawn  + anything else  → 'none', and the old command is dropped: a bare
+     *                             command line is not a file and must not be run as
+     *                             one ("python main.py" is not a path)
+     *   ssh                     → 'none': a remote host cannot be started by a local
+     *                             .bat, and pretending otherwise would make the Start
+     *                             button lie
+     */
+    const isBat = (value) => /\.(bat|cmd)$/i.test(value.trim());
+    const declared = typeof control.batPath === 'string' ? control.batPath.trim() : '';
+    const legacy = typeof control.command === 'string' ? control.command.trim() : '';
+    const wantsScript = control.type === 'script' || control.type === 'spawn';
+    const batPath = isBat(declared) ? declared : wantsScript && isBat(legacy) ? legacy : '';
+    const type = wantsScript && batPath !== '' ? 'script' : 'none';
+    return {
+        id,
+        label: typeof input.label === 'string' && input.label !== '' ? input.label : id,
+        baseUrl,
+        apiKey: typeof input.apiKey === 'string' ? input.apiKey : '',
+        weight: clampInt(input.weight, 1, 10, 1),
+        maxConcurrent: clampInt(input.maxConcurrent, 0, 32, 1),
+        enabled: input.enabled !== false,
+        autoStart: input.autoStart === true,
+        control: {
+            type,
+            batPath: batPath,
+            cwd: typeof control.cwd === 'string' ? control.cwd : '',
+            stopTimeoutMs: clampInt(control.stopTimeoutMs, 500, 60000, 15000),
+            readyTimeoutMs: clampInt(control.readyTimeoutMs, 5000, 900000, 300000),
+            /**
+             * Kill this process when DSH exits. Default true: a gateway you can start but
+             * not stop leaks VRAM, and the next launch then fights the orphan for the port.
+             * Only an explicit `false` keeps it alive — that is for a box where ComfyUI is
+             * meant to outlive the console.
+             */
+            stopOnExit: control.stopOnExit !== false,
+        },
+    };
+}
+
+function clampInt(value, min, max, fallback) {
+    const number = Number(value);
+    if (!Number.isFinite(number)) return fallback;
+    return Math.min(max, Math.max(min, Math.floor(number)));
+}
+
+/**
+ * Which process is listening on this instance's port?
+ *
+ * Needed because a pid only exists for a process the *current* session spawned. After DSH
+ * restarts, ComfyUI is still running and holding VRAM, but the gateway has no handle — and
+ * a 停止 button that cannot find the process is a button that reports failure at a live
+ * process.
+ *
+ * Windows: `netstat -ano` gives the owning pid per listening socket. Elsewhere: `lsof` if
+ * it exists. Returns null when nothing is found, which the caller treats as "already down"
+ * only after confirming the port really does not answer.
+ */
+function pidListeningOn(baseUrl) {
+    let port;
+    try {
+        port = new URL(baseUrl).port;
+    } catch {
+        return Promise.resolve(null);
+    }
+    if (port === '') return Promise.resolve(null);
+
+    if (process.platform === 'win32') {
+        return new Promise((resolve) => {
+            const run = spawn('netstat', ['-ano', '-p', 'tcp'], { windowsHide: true, stdio: ['ignore', 'pipe', 'ignore'] });
+            let out = '';
+            run.stdout.on('data', (chunk) => { out += String(chunk); });
+            run.on('error', () => resolve(null));
+            run.on('exit', () => {
+                // Lines look like: "  TCP    0.0.0.0:8188    0.0.0.0:0    LISTENING    1234"
+                const suffix = `:${port}`;
+                for (const line of out.split(/\r?\n/)) {
+                    if (!/LISTENING/i.test(line)) continue;
+                    const parts = line.trim().split(/\s+/);
+                    if (parts.length < 5) continue;
+                    const local = parts[1] ?? '';
+                    // Compare the port exactly: `:818` must not match `:8188`.
+                    if (!local.endsWith(suffix)) continue;
+                    const pid = Number(parts[parts.length - 1]);
+                    if (Number.isFinite(pid) && pid > 0) {
+                        resolve(pid);
+                        return;
+                    }
+                }
+                resolve(null);
+            });
+        });
+    }
+
+    return new Promise((resolve) => {
+        const run = spawn('lsof', ['-nP', `-iTCP:${port}`, '-sTCP:LISTEN', '-t'], { stdio: ['ignore', 'pipe', 'ignore'] });
+        let out = '';
+        run.stdout.on('data', (chunk) => { out += String(chunk); });
+        run.on('error', () => resolve(null));
+        run.on('exit', () => {
+            const pid = Number(String(out).trim().split(/\s+/)[0]);
+            resolve(Number.isFinite(pid) && pid > 0 ? pid : null);
+        });
+    });
+}
+
+/** `taskkill /T /F` on Windows and wait for it; harmless elsewhere. */
+function taskkillTree(pid, wait) {
+    const run = spawn('taskkill', ['/PID', String(pid), '/T', '/F'], { windowsHide: true, stdio: 'ignore' });
+    if (wait !== true) return Promise.resolve();
+    return new Promise((resolve) => {
+        let settled = false;
+        const done = () => {
+            if (settled) return;
+            settled = true;
+            resolve();
+        };
+        run.on('exit', done);
+        run.on('error', done);
+        setTimeout(done, 5000);
+    });
+}
+
+/** Runtime state of one instance: health, load, and (if owned) its process. */
+export class InstanceRuntime {
+    constructor(instance) {
+        this.instance = instance;
+        this.health = { ok: false, checkedAt: 0, version: null, devices: [], latencyMs: 0, error: 'not probed' };
+        this.child = null;
+        this.pid = null;
+        this.starting = false;
+        this.stopping = false;
+        this.lastError = null;
+        this.startedAt = null;
+        this.speed = 1; // seconds per cost unit, EWMA; 1 = unknown/baseline
+        this.speedSamples = 0;
+    }
+
+    get id() {
+        return this.instance.id;
+    }
+
+    /**
+     * Bind the process to this plugin's lifetime so a restart cannot orphan it.
+     *
+     * On Windows `child.kill()` only reaches the immediate child — a ComfyUI
+     * launched through a wrapper (.bat, a launcher stub, a shell) leaves its real
+     * process alive and the VRAM stays taken. `taskkill /T` walks the tree, so it
+     * must run **before** the direct kill: killing the wrapper first reparents
+     * the grandchild and `taskkill` then finds nothing to walk.
+     */
+    async killNow() {
+        const pid = this.pid;
+        const child = this.child;
+        this.pid = null;
+        this.child = null;
+        if (process.platform === 'win32' && typeof pid === 'number') {
+            await taskkillTree(pid, true);
+        }
+        if (child !== null) {
+            try {
+                child.kill('SIGKILL');
+            } catch {
+                // already dead
+            }
+        }
+    }
+
+    /**
+     * Synchronous kill for shutdown, because an async kill is not a kill at all.
+     *
+     * `spawn('taskkill', …)` returns immediately and the child only does its work on the
+     * next event-loop turn. During host shutdown the process is already exiting: the
+     * taskkill we just spawned is a child of *this* process and dies with it, so the
+     * ComfyUI tree was never actually killed and VRAM stayed held until someone noticed.
+     *
+     * `spawnSync` blocks until taskkill has finished, which is exactly what a shutdown
+     * path needs. The 8s ceiling matters: a wedged process must not stop DSH from exiting.
+     */
+    killDetached() {
+        const pid = this.pid;
+        const child = this.child;
+        this.pid = null;
+        this.child = null;
+        if (process.platform === 'win32' && typeof pid === 'number') {
+            try {
+                spawnSync('taskkill', ['/PID', String(pid), '/T', '/F'], {
+                    windowsHide: true,
+                    stdio: 'ignore',
+                    timeout: 8000,
+                });
+            } catch {
+                // taskkill unavailable: the direct kill below is the best effort
+            }
+        }
+        if (child !== null) {
+            try {
+                child.kill('SIGKILL');
+            } catch {
+                // already dead
+            }
+        }
+    }
+}
+
+export class InstanceManager {
+    /**
+     * @param {{getInstances:()=>any[], onChange:()=>void, sse?:object}} options
+     */
+    constructor(options) {
+        this.options = options;
+        /** @type {Map<string, InstanceRuntime>} */
+        this.runtimes = new Map();
+        this.log = [];
+        this.sync();
+    }
+
+    /** Rebuild runtimes after a settings change, keeping live processes alive. */
+    sync() {
+        const wanted = new Map();
+        for (const raw of this.options.getInstances()) {
+            const instance = normalizeInstance(raw, wanted.size);
+            if (instance.baseUrl === '') continue;
+            wanted.set(instance.id, instance);
+        }
+        for (const [id, runtime] of this.runtimes) {
+            if (!wanted.has(id)) {
+                runtime.killDetached();
+                this.runtimes.delete(id);
+            }
+        }
+        for (const [id, instance] of wanted) {
+            const existing = this.runtimes.get(id);
+            if (existing === undefined) {
+                this.runtimes.set(id, new InstanceRuntime(instance));
+            } else {
+                existing.instance = instance;
+            }
+        }
+    }
+
+    list() {
+        return [...this.runtimes.values()];
+    }
+
+    get(id) {
+        return this.runtimes.get(id);
+    }
+
+    require(id) {
+        const runtime = this.runtimes.get(id);
+        if (runtime === undefined) throw new Error(`unknown instance ${id}`);
+        return runtime;
+    }
+
+    enabled() {
+        return this.list().filter((runtime) => runtime.instance.enabled && runtime.instance.baseUrl !== '');
+    }
+
+    record(level, message) {
+        const entry = { ts: new Date().toISOString(), level, message };
+        this.log.unshift(entry);
+        if (this.log.length > 300) this.log.length = 300;
+    }
+
+    /** Probe every instance in parallel; health feeds the scheduler. */
+    async probeAll(force = false) {
+        const now = Date.now();
+        const jobs = this.list().map(async (runtime) => {
+            if (!force && now - runtime.health.checkedAt < 4000) return;
+            const result = await probe({
+                id: runtime.id,
+                baseUrl: runtime.instance.baseUrl,
+                apiKey: runtime.instance.apiKey,
+            });
+            runtime.health = {
+                ok: result.ok,
+                checkedAt: Date.now(),
+                version: result.version,
+                devices: result.devices,
+                latencyMs: result.latencyMs,
+                error: result.error,
+                argv: result.argv ?? [],
+            };
+            if (result.ok) {
+                runtime.lastError = null;
+                if (runtime.startedAt === null) runtime.startedAt = Date.now();
+            } else if (!runtime.starting) {
+                runtime.lastError = result.error;
+            }
+        });
+        await Promise.all(jobs);
+    }
+
+    async start(id) {
+        const runtime = this.require(id);
+        if (runtime.instance.control.type === 'none') {
+            throw new Error(`${id} 什么都没配（control.type=none）：在面板里给它选一个 bat 文件才能启停`);
+        }
+        if (runtime.starting) return { ok: true, alreadyStarting: true };
+        runtime.starting = true;
+        runtime.lastError = null;
+        this.record('info', `starting ${id}`);
+        try {
+            this.spawnScript(runtime);
+            const ready = await this.waitReady(runtime);
+            runtime.starting = false;
+            if (!ready.ok) {
+                runtime.lastError = ready.error;
+                this.record('error', `${id} start failed: ${ready.error}`);
+            } else {
+                this.record('info', `${id} ready (${ready.latencyMs} ms)`);
+            }
+            return ready;
+        } catch (error) {
+            runtime.starting = false;
+            runtime.lastError = error instanceof Error ? error.message : String(error);
+            this.record('error', `${id} start error: ${runtime.lastError}`);
+            return { ok: false, error: runtime.lastError };
+        } finally {
+            this.options.onChange();
+        }
+    }
+
+    /**
+     * Run the configured `.bat` through cmd.exe.
+     *
+     * Why cmd.exe and not `shell: true`: Node's `shell: true` picks `%ComSpec%`,
+     * and a path with spaces or non-ASCII characters gets mangled on the way in.
+     * Handing cmd.exe the literal `/d /s /c ""C:\某 目录\启动.bat""` form is the one
+     * quoting that survives both. `/d` also skips AutoRun, so a stray registry
+     * AutoRun entry cannot prepend commands to our launch.
+     *
+     * The bat is executed, not parsed: whatever it does (cd, activate a venv, start
+     * python with the right port) is the operator's business, which is the whole
+     * point of naming a file instead of typing a command.
+     */
+    spawnScript(runtime) {
+        const { batPath, cwd } = runtime.instance.control;
+        const file = batPath.trim();
+        if (file === '') throw new Error('没有指定 bat 文件');
+        if (!existsSync(file)) throw new Error(`bat 文件不存在: ${file}`);
+        if (!/\.(bat|cmd)$/i.test(file)) throw new Error(`不是 .bat / .cmd 文件: ${file}`);
+        if (cwd !== '' && !existsSync(cwd)) throw new Error(`工作目录不存在: ${cwd}`);
+        // cmd /s /c needs one extra layer of quotes around the whole command when the
+        // path may contain spaces: `cmd /d /s /c ""C:\some dir\start.bat""`.
+        //
+        // `windowsVerbatimArguments` is not optional here. Without it Node re-escapes
+        // the argument list for CreateProcess and cmd receives literal \" — which it
+        // then tries to execute as a program name ("'\"C:\...cmd\"' 不是内部或外部命令").
+        const child = spawn(process.env.ComSpec ?? 'cmd.exe', ['/d', '/s', '/c', `""${file}""`], {
+            // A bat that does not cd for itself still finds its own folder first.
+            cwd: cwd === '' ? dirname(file) : cwd,
+            shell: false,
+            windowsHide: true,
+            detached: false,
+            windowsVerbatimArguments: true,
+            stdio: ['ignore', 'pipe', 'pipe'],
+            env: { ...process.env, PYTHONUNBUFFERED: '1' },
+        });
+        runtime.child = child;
+        runtime.pid = child.pid ?? null;
+        const tail = [];
+        const keep = (chunk) => {
+            tail.push(String(chunk));
+            if (tail.length > 40) tail.shift();
+            runtime.consoleTail = tail.join('');
+        };
+        child.stdout?.on('data', keep);
+        child.stderr?.on('data', keep);
+        child.on('exit', (code, signal) => {
+            this.record('warn', `${runtime.id} exited code=${code} signal=${signal}`);
+            runtime.child = null;
+            runtime.pid = null;
+            runtime.health = { ...runtime.health, ok: false, checkedAt: Date.now(), error: 'process exited' };
+            this.options.onChange();
+        });
+    }
+
+    /** Poll /system_stats until the freshly spawned server answers. */
+    async waitReady(runtime) {
+        const deadline = Date.now() + runtime.instance.control.readyTimeoutMs;
+        let last = 'timeout';
+        while (Date.now() < deadline) {
+            const result = await probe({
+                id: runtime.id,
+                baseUrl: runtime.instance.baseUrl,
+                apiKey: runtime.instance.apiKey,
+            }, 3000);
+            if (result.ok) {
+                runtime.health = {
+                    ok: true,
+                    checkedAt: Date.now(),
+                    version: result.version,
+                    devices: result.devices,
+                    latencyMs: result.latencyMs,
+                    error: null,
+                    argv: result.argv ?? [],
+                };
+                return { ok: true, latencyMs: result.latencyMs, version: result.version };
+            }
+            last = result.error ?? 'no answer';
+            if (runtime.child === null && runtime.pid === null && runtime.instance.control.type === 'script') {
+                const hint = (runtime.consoleTail ?? '').split('\n').slice(-4).join(' | ');
+                return { ok: false, error: `进程已退出: ${hint || last}` };
+            }
+            await new Promise((resolve) => setTimeout(resolve, 1500));
+        }
+        return { ok: false, error: `等待就绪超时: ${last}` };
+    }
+
+    async stop(id) {
+        const runtime = this.require(id);
+        if (runtime.instance.control.type === 'none') {
+            throw new Error(`${id} 不受本网关托管，只能 drain（停止派发），不能停进程`);
+        }
+        runtime.stopping = true;
+        this.record('info', `stopping ${id}`);
+        try {
+            /**
+             * No pid? Find the process before deciding there is nothing to stop.
+             *
+             * `this.pid` only exists for a process THIS session spawned. After a DSH
+             * restart the ComfyUI tree is still running (it outlived the console) but the
+             * gateway has no handle on it, so the old code fell straight through to the
+             * poll loop, waited the full 15 s and reported "端口仍在响应" — while the
+             * process it was asked to stop sat there untouched. That is the "stop doesn't
+             * work" report: a button that waits 15 seconds and fails.
+             *
+             * Recovering the pid means asking the OS which process owns the port, then
+             * killing that tree. It is the same authority the user expects the button to
+             * have; guessing nothing, verifying by port ownership.
+             */
+            let pid = runtime.pid;
+            if (typeof pid !== 'number') {
+                pid = await pidListeningOn(runtime.instance.baseUrl);
+                if (typeof pid === 'number') {
+                    this.record('info', `${id}: recovered pid ${pid} from the port`);
+                }
+            }
+            if (typeof pid !== 'number') {
+                // Nothing is listening on that port at all: the instance is already down,
+                // which is what "stop" was asking for. Reporting failure here was wrong.
+                const gone = await probe({ id, baseUrl: runtime.instance.baseUrl, apiKey: runtime.instance.apiKey }, 1500);
+                if (!gone.ok) {
+                    runtime.health = { ...runtime.health, ok: false, checkedAt: Date.now(), error: 'stopped' };
+                    this.record('info', `${id}: nothing listening, treated as stopped`);
+                    return { ok: true, stopped: true, note: '端口本来就没有服务' };
+                }
+                return { ok: false, error: '找不到该端口的进程，也无法确认它已停止' };
+            }
+
+            await runtime.killNow();
+            if (runtime.pid === null && typeof pid === 'number') {
+                // killNow only acts on its own handle; make sure a recovered pid dies too.
+                await taskkillTree(pid, true);
+            }
+            // A wrapper script may keep the port open for a moment; poll the port.
+            const deadline = Date.now() + runtime.instance.control.stopTimeoutMs;
+            while (Date.now() < deadline) {
+                const result = await probe({ id, baseUrl: runtime.instance.baseUrl, apiKey: runtime.instance.apiKey }, 1500);
+                if (!result.ok) {
+                    runtime.health = { ...runtime.health, ok: false, checkedAt: Date.now(), error: 'stopped' };
+                    this.record('info', `${id} stopped`);
+                    return { ok: true, stopped: true };
+                }
+                await new Promise((resolve) => setTimeout(resolve, 700));
+            }
+            return { ok: false, error: '进程未在超时内退出（端口仍在响应）' };
+        } finally {
+            runtime.stopping = false;
+            this.options.onChange();
+        }
+    }
+
+    async restart(id) {
+        const stopped = await this.stop(id);
+        if (!stopped.ok) return stopped;
+        return this.start(id);
+    }
+
+    /**
+     * Shutdown: take every owned process down with us.
+     *
+     * `stopOnExit` is the operator's choice, per instance. It defaults to true because the
+     * alternative — a gateway you can start but not stop — leaks VRAM: the process keeps
+     * running with nothing driving it, and the next launch fights it for the port.
+     *
+     * The summary is returned so the caller can log what actually happened. "We tried" is
+     * not the same as "it died", and on a shutdown path that distinction is the only thing
+     * worth recording.
+     */
+    dispose() {
+        const stopped = [];
+        const kept = [];
+        for (const runtime of this.runtimes.values()) {
+            if (runtime.instance.control.type === 'none') continue;
+            if (runtime.instance.control.stopOnExit === false) {
+                kept.push(runtime.id);
+                continue;
+            }
+            runtime.killDetached();
+            stopped.push(runtime.id);
+        }
+        return { stopped, kept };
+    }
+}