@minionry/minion 0.7.36 → 0.7.39

package/dist/server/engines/hermes/HermesEngine.js

@@ -1,28 +1,17 @@
 // Copyright (c) 2025-present Minionry, Inc.
 // SPDX-License-Identifier: LicenseRef-Minionry-Software
-import { spawn as nodeSpawn } from 'node:child_process';
 import { randomUUID } from 'node:crypto';
 import { mkdirSync } from 'node:fs';
 import { hlog } from '../../cli/headless/headless-logger.js';
 import { BROWSER_SYSTEM_PROMPT } from '../../cli/headless/mcp-config.js';
-import { killProcessGroup } from '../../cli/headless/runner.js';
-import { isDirectory } from '../../services/pathUtils.js';
-import { findAvailablePort } from '../../utils/port-manager.js';
 import { EngineEventStream } from '../engine-event-stream.js';
 import { HERMES_READ_ONLY_REFUSAL, READ_ONLY_UNSUPPORTED, sessionIsReadOnly } from '../read-only.js';
 import { AppRunUnsupportedError, EngineStartError, } from '../types.js';
 import { HERMES_CONNECT_MESSAGE, } from './hermes-auth.js';
 import { hermesAppRunBaseEnv } from './hermes-gateway-home.js';
+import { HermesGatewayRegistry, userHermesHome, } from './hermes-gateway-registry.js';
 import { HermesEventMapper } from './hermes-invoker.js';
 import { hermesAvailability, resolveHermesCommand } from './hermes-provision.js';
-function asRecord(value) {
-    return typeof value === 'object' && value !== null
-        ? value
-        : undefined;
-}
-function asString(value) {
-    return typeof value === 'string' ? value : undefined;
-}
 const HERMES_THINKING_MODEL_PATTERN = /(^|\/)(hermes-4|deephermes)/i;
 export function resolveHermesReasoningEffort(model, effortLevel) {
     if (!model || !HERMES_THINKING_MODEL_PATTERN.test(model))
@@ -31,36 +20,13 @@ export function resolveHermesReasoningEffort(model, effortLevel) {
         return undefined;
     return effortLevel === 'ultracode' ? 'ultra' : effortLevel;
 }
-function delay(ms) {
-    return new Promise((resolve) => {
-        setTimeout(resolve, ms).unref?.();
-    });
-}
-const GATEWAY_BASE_PORT = 8642;
-const claimedGatewayPorts = new Set();
-async function claimGatewayPort() {
-    let from = GATEWAY_BASE_PORT;
-    for (;;) {
-        const port = await findAvailablePort(from);
-        if (!claimedGatewayPorts.has(port)) {
-            claimedGatewayPorts.add(port);
-            return port;
-        }
-        from = port + 1;
-    }
-}
-const STARTUP_OUTPUT_LINES = 5;
-const STARTUP_OUTPUT_LINE_CHARS = 300;
+export const HERMES_APP_ENDPOINT_GONE_MESSAGE = "This app's model endpoint is no longer registered with your minion, so the message was not sent. The app has to register its endpoint again before this conversation can continue.";
 export class HermesEngine extends EngineEventStream {
     engineId = 'hermes';
     resolveCommand;
-    spawn;
     fetchImpl;
-    resolvePort;
-    generateApiKey;
+    gateways;
     generateConversationId;
-    healthTimeoutMs;
-    healthPollIntervalMs;
     auth;
     provisionGatewayHome;
     checkAppRunAvailability;
@@ -72,12 +38,9 @@ export class HermesEngine extends EngineEventStream {
     sessionOptions = null;
     conversationId = 'pending';
     previousResponseId;
-    apiKey = '';
-    baseUrl = '';
+    command = '';
     gateway = null;
-    gatewayStartError = null;
-    claimedPort = null;
-    startupOutput = [];
+    acquiring = null;
     currentAbort = null;
     currentRunSettled = null;
     cancelling = false;
@@ -91,14 +54,19 @@ export class HermesEngine extends EngineEventStream {
     constructor(options = {}) {
         super();
         this.resolveCommand = options.resolveCommand ?? (() => resolveHermesCommand());
-        this.spawn = options.spawn ?? nodeSpawn;
         this.fetchImpl =
             options.fetch ?? ((url, init) => globalThis.fetch(url, init));
-        this.resolvePort = options.resolvePort ?? (() => this.claimPort());
-        this.generateApiKey = options.generateApiKey ?? (() => randomUUID());
+        this.gateways =
+            options.gateways ??
+                new HermesGatewayRegistry({
+                    spawn: options.spawn,
+                    fetch: this.fetchImpl,
+                    resolvePort: options.resolvePort,
+                    generateApiKey: options.generateApiKey,
+                    healthTimeoutMs: options.healthTimeoutMs,
+                    healthPollIntervalMs: options.healthPollIntervalMs,
+                });
         this.generateConversationId = options.generateConversationId ?? (() => randomUUID());
-        this.healthTimeoutMs = options.healthTimeoutMs ?? 30_000;
-        this.healthPollIntervalMs = options.healthPollIntervalMs ?? 250;
         this.auth = options.auth;
         this.provisionGatewayHome = options.provisionGatewayHome;
         this.checkAppRunAvailability = options.checkAppRunAvailability ?? (() => hermesAvailability());
@@ -116,11 +84,9 @@ export class HermesEngine extends EngineEventStream {
         this.sessionOptions = options;
         this.conversationId = options.resumeSessionId ?? this.generateConversationId();
         this.previousResponseId = undefined;
-        this.apiKey = this.generateApiKey();
         this.mapper = new HermesEventMapper({ sessionId: this.conversationId });
-        let command;
         try {
-            command = await this.resolveCommand();
+            this.command = await this.resolveCommand();
         }
         catch (err) {
             throw new EngineStartError('HERMES_NOT_INSTALLED', err instanceof Error ? err.message : String(err));
@@ -130,42 +96,63 @@ export class HermesEngine extends EngineEventStream {
             return;
         if (this.auth && !appRun) {
             this.providerEnv = this.auth.providerEnv();
-            this.connection = await this.auth.connectionStatus(command);
+            this.connection = await this.auth.connectionStatus(this.command);
             if (this.disposed)
                 return;
         }
         this.provisionSessionHome(options);
         this.nativeResume = !options.resumeSessionId || !this.gatewayHome || this.gatewayHome.persistent;
         this.resolveBrowserMcpActive(options);
-        const port = await this.resolvePort();
-        this.baseUrl = `http://127.0.0.1:${port}`;
-        const cwd = appRun?.cwd ?? options.workingDir;
-        const child = this.spawnGateway(command, cwd, port, options);
-        this.gateway = child;
-        this.wireGateway(child);
-        await this.waitForHealthy(cwd);
-    }
-    spawnGateway(command, cwd, port, options) {
+        const acquiring = this.acquireGateway(options);
+        this.acquiring = acquiring;
         try {
-            return this.spawn(command, ['gateway', 'run'], {
-                cwd,
-                env: {
-                    ...this.inheritedSpawnEnv(options),
-                    ...this.providerEnv,
-                    ...(this.gatewayHome?.env ?? {}),
-                    API_SERVER_ENABLED: 'true',
-                    API_SERVER_KEY: this.apiKey,
-                    API_SERVER_PORT: String(port),
-                    API_SERVER_HOST: '127.0.0.1',
-                },
-                detached: true,
-                stdio: ['ignore', 'pipe', 'pipe'],
-            });
+            await acquiring;
         }
-        catch (err) {
-            throw this.gatewayStartFailure(err instanceof Error ? err : new Error(String(err)), cwd);
+        finally {
+            this.acquiring = null;
         }
     }
+    async acquireGateway(options) {
+        const lease = await this.gateways.acquire(this.gatewayLaunch(options), (failure) => this.onGatewayLost(failure));
+        if (this.disposed) {
+            await lease.release();
+            throw new Error('HermesEngine: engine disposed during startup');
+        }
+        if (!lease.endpoint)
+            throw new Error('HermesEngine: gateway failed to start — it exited as this session attached');
+        this.gateway = lease;
+        this.healthy = true;
+    }
+    gatewayLaunch(options) {
+        const homeEnv = this.gatewayHome?.env ?? {};
+        const env = {
+            ...this.inheritedSpawnEnv(options),
+            ...this.providerEnv,
+            ...homeEnv,
+        };
+        return {
+            home: this.gatewayHome?.home ?? userHermesHome(env),
+            owned: this.gatewayHome !== null,
+            command: this.command,
+            cwd: options.appRun?.cwd ?? options.workingDir,
+            env,
+            secrets: [...Object.values(this.providerEnv), ...Object.values(homeEnv)],
+            quietOnceServing: !!options.appRun,
+        };
+    }
+    onGatewayLost(failure) {
+        this.gateway = null;
+        if (!this.healthy || this.disposed)
+            return;
+        this.emit({
+            kind: 'engine.error',
+            sessionId: this.sessionIdForEvent(),
+            timestamp: Date.now(),
+            code: failure.code,
+            message: failure.message,
+            fatal: true,
+        });
+    }
     async sendPrompt(prompt, _attachments) {
         if (!this.sessionOptions) {
             throw new Error('HermesEngine: sendPrompt called before startSession');
@@ -224,12 +211,14 @@ export class HermesEngine extends EngineEventStream {
             return;
         this.disposed = true;
         this.currentAbort?.abort();
-        this.killGateway();
-        this.releasePort();
+        const released = this.gateway?.release();
+        this.gateway = null;
         if (this.currentRunSettled) {
             await this.currentRunSettled;
         }
         this.closeIterator();
+        await released;
+        await this.acquiring?.catch(() => { });
         this.gatewayHome?.cleanup();
         this.gatewayHome = null;
     }
@@ -291,107 +280,9 @@ export class HermesEngine extends EngineEventStream {
             .filter(Boolean)
             .join('\n\n');
     }
-    wireGateway(child) {
-        this.drainGatewayOutput(child.stdout, 'stdout');
-        this.drainGatewayOutput(child.stderr, 'stderr');
-        child.on('error', (err) => {
-            this.gatewayStartError = err;
-            if (this.gateway === child && this.sessionOptions && !this.disposed) {
-                this.gateway = null;
-            }
-            if (this.healthy && !this.disposed) {
-                this.emit({
-                    kind: 'engine.error',
-                    sessionId: this.sessionIdForEvent(),
-                    timestamp: Date.now(),
-                    code: 'HERMES_GATEWAY_ERROR',
-                    message: err.message,
-                    fatal: true,
-                });
-            }
-        });
-        child.on('exit', (code, signal) => {
-            if (this.gateway === child)
-                this.gateway = null;
-            if (this.disposed)
-                return;
-            this.gatewayStartError ??= new Error(`hermes gateway exited (${signal ? `signal ${signal}` : `code ${code}`})`);
-            if (this.healthy) {
-                this.emit({
-                    kind: 'engine.error',
-                    sessionId: this.sessionIdForEvent(),
-                    timestamp: Date.now(),
-                    code: 'HERMES_GATEWAY_EXIT',
-                    message: this.gatewayStartError.message,
-                    fatal: true,
-                });
-            }
-        });
-    }
-    drainGatewayOutput(stream, label) {
-        if (!stream)
-            return;
-        stream.on('data', (chunk) => {
-            if (this.disposed)
-                return;
-            if (this.sessionOptions?.appRun && this.healthy)
-                return;
-            const text = chunk.toString('utf8').trim();
-            if (!text)
-                return;
-            hlog(`[hermes gateway ${label}] ${text.slice(0, 2000)}`);
-            if (!this.healthy)
-                this.keepStartupOutput(text);
-        });
-        stream.on('error', () => { });
-    }
-    keepStartupOutput(text) {
-        for (const line of text.split('\n')) {
-            const trimmed = line.trim();
-            if (trimmed)
-                this.startupOutput.push(trimmed.slice(0, STARTUP_OUTPUT_LINE_CHARS));
-        }
-        this.startupOutput.splice(0, Math.max(0, this.startupOutput.length - STARTUP_OUTPUT_LINES));
-    }
-    gatewayStartFailure(err, cwd) {
-        const errno = err.code;
-        if ((errno === 'ENOENT' || errno === 'ENOTDIR') && !isDirectory(cwd)) {
-            return new EngineStartError('WORKING_DIRECTORY_NOT_FOUND', `Could not start Hermes: its working directory ${cwd} is not a directory — it may have been moved, renamed, or deleted.`);
-        }
-        return new Error(`HermesEngine: gateway failed to start — ${err.message}${this.startupOutputSuffix()}`);
-    }
-    async waitForHealthy(cwd) {
-        const deadline = Date.now() + this.healthTimeoutMs;
-        while (Date.now() < deadline) {
-            if (this.disposed) {
-                throw new Error('HermesEngine: engine disposed during startup');
-            }
-            if (this.gatewayStartError) {
-                const err = this.gatewayStartError;
-                this.killGateway();
-                throw this.gatewayStartFailure(err, cwd);
-            }
-            try {
-                const res = await this.fetchImpl(`${this.baseUrl}/health`, {
-                    method: 'GET',
-                    headers: { Authorization: `Bearer ${this.apiKey}` },
-                });
-                if (res.ok) {
-                    const body = asRecord(await res.json());
-                    if (asString(body?.status) === 'ok') {
-                        this.healthy = true;
-                        return;
-                    }
-                }
-            }
-            catch {
-            }
-            await delay(this.healthPollIntervalMs);
-        }
-        this.killGateway();
-        throw new Error(`HermesEngine: gateway did not become healthy within ${this.healthTimeoutMs}ms${this.startupOutputSuffix()}`);
-    }
     async runTurn(prompt, signal) {
+        if (!(await this.refreshAppRunGateway(signal)))
+            return;
         const resumeId = this.previousResponseId;
         const degrade = await this.attemptTurn(prompt, signal, resumeId, resumeId !== undefined);
         if (degrade && !this.cancelling && !signal.aborted) {
@@ -400,6 +291,43 @@ export class HermesEngine extends EngineEventStream {
             await this.attemptTurn(prompt, signal, undefined, false);
         }
     }
+    async refreshAppRunGateway(signal) {
+        const options = this.sessionOptions;
+        const lease = this.gateway;
+        if (!options?.appRun || !options.model || !this.provisionGatewayHome || !lease)
+            return true;
+        let home;
+        try {
+            home = this.provisionGatewayHome(options.model, undefined, options.appRun);
+        }
+        catch (err) {
+            return this.settleUnreadyTurn('HERMES_GATEWAY_ERROR', errMessage(err), signal);
+        }
+        if (!home)
+            return this.settleUnreadyTurn('HERMES_APP_ENDPOINT_GONE', HERMES_APP_ENDPOINT_GONE_MESSAGE, signal);
+        this.gatewayHome = home;
+        try {
+            await lease.reconcile(this.gatewayLaunch(options));
+        }
+        catch (err) {
+            return this.settleUnreadyTurn('HERMES_GATEWAY_ERROR', errMessage(err), signal);
+        }
+        if (this.cancelling || signal.aborted) {
+            this.emitAbortedIdle();
+            return false;
+        }
+        return true;
+    }
+    settleUnreadyTurn(code, message, signal) {
+        if (this.cancelling || signal.aborted) {
+            this.emitAbortedIdle();
+        }
+        else {
+            this.emitTurnError(code, message);
+            this.emitTurnIdle('error');
+        }
+        return false;
+    }
     async attemptTurn(prompt, signal, previousResponseId, canDegrade) {
         const outcome = await this.postTurn(prompt, signal, previousResponseId);
         if (outcome.kind === 'aborted') {
@@ -453,13 +381,17 @@ export class HermesEngine extends EngineEventStream {
         const reasoningEffort = resolveHermesReasoningEffort(model, this.sessionOptions?.effortLevel);
         if (reasoningEffort)
             body.model_options = { reasoning: { effort: reasoningEffort } };
+        const endpoint = this.gateway?.endpoint;
+        if (!endpoint) {
+            return { kind: 'error', code: 'HERMES_GATEWAY_EXIT', detail: 'The Hermes gateway for this session is not running.' };
+        }
         let response;
         try {
-            response = await this.fetchImpl(`${this.baseUrl}/v1/responses`, {
+            response = await this.fetchImpl(`${endpoint.baseUrl}/v1/responses`, {
                 method: 'POST',
                 headers: {
                     'Content-Type': 'application/json',
-                    Authorization: `Bearer ${this.apiKey}`,
+                    Authorization: `Bearer ${endpoint.apiKey}`,
                     'X-Hermes-Session-Key': this.conversationId,
                 },
                 body: JSON.stringify(body),
@@ -583,45 +515,6 @@ export class HermesEngine extends EngineEventStream {
             raw,
         });
     }
-    async claimPort() {
-        const port = await claimGatewayPort();
-        if (this.disposed) {
-            claimedGatewayPorts.delete(port);
-        }
-        else {
-            this.claimedPort = port;
-        }
-        return port;
-    }
-    releasePort() {
-        if (this.claimedPort === null)
-            return;
-        claimedGatewayPorts.delete(this.claimedPort);
-        this.claimedPort = null;
-    }
-    startupOutputSuffix() {
-        if (this.startupOutput.length === 0)
-            return '';
-        const secrets = [
-            this.apiKey,
-            ...Object.values(this.providerEnv),
-            ...Object.values(this.gatewayHome?.env ?? {}),
-        ].filter((value) => value.length >= 8);
-        const lines = this.startupOutput.map((line) => secrets.reduce((masked, secret) => masked.split(secret).join('[redacted]'), line));
-        return `\nLast gateway output:\n${lines.join('\n')}`;
-    }
-    killGateway() {
-        const child = this.gateway;
-        if (!child || child.pid === undefined)
-            return;
-        const pid = child.pid;
-        killProcessGroup(pid, 'SIGTERM');
-        setTimeout(() => {
-            if (!child.killed) {
-                killProcessGroup(pid, 'SIGKILL');
-            }
-        }, 5000).unref?.();
-    }
 }
 function errMessage(err) {
     return err instanceof Error ? err.message : String(err);