hoi4-agent-tools 3.10.0 → 4.0.1

package/dist/hoi4_agent_tools/core/job-worker-host.js

@@ -1,5 +1,7 @@
 import { spawn } from 'node:child_process';
+import { closeSync, fstatSync, openSync, readSync, readdirSync, rmSync, statSync } from 'node:fs';
 import { mkdir } from 'node:fs/promises';
+import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { setTimeout as delay } from 'node:timers/promises';
 import { JobService } from './job-service.js';
@@ -8,8 +10,16 @@ import { RequestScheduler } from './request-scheduler.js';
 import { SharedRequestCapacity } from './shared-request-capacity.js';
 import { containedGeneratedPath } from './workspace.js';
 import { ServiceError } from './result.js';
+import { isDomainToolName } from './domain-tools.js';
+import { classifyWorkerExit, defaultWorkerHeapMiB, } from './worker-exit.js';
 const maximumCheckpointRecoveries = 2;
 const maximumPreDispatchRecoveries = 2;
+/** Used only when the exit status of the worker is unknown to this process. */
+const WORKER_EXIT_MESSAGE = 'The worker stopped without publishing a terminal job outcome; its exit status is not available to this server, so retry the call';
+/** Bytes of a worker's standard error read back when it exits unexpectedly. */
+const STDERR_TAIL_BYTES = 16_384;
+/** Standard-error logs of workers that exited unexpectedly kept for diagnosis. */
+const RETAINED_STDERR_LOGS = 10;
 // Native and IPC codes are fixed vocabulary. Error messages can contain paths or source
 // text, so only an allowlisted code and the fixed coordination stage are reported.
 const nativeFailureCodes = new Set([
@@ -58,13 +68,68 @@ export function workerFailure(error, stage) {
         };
 }
 /** Stop a supervised worker and wait, bounded, until the operating system reports its exit. */
-async function stopChild(child) {
+async function stopChild(child, stopped) {
     if (child.exitCode !== null || child.signalCode !== null)
         return;
+    stopped?.add(child);
     const exited = new Promise((resolve) => child.once('exit', () => resolve()));
     child.kill('SIGKILL');
     await Promise.race([exited, delay(10_000)]);
 }
+/** Read the last bytes of a worker's standard-error log; empty when it cannot be read. */
+function readTail(file) {
+    try {
+        const fd = openSync(file, 'r');
+        try {
+            const size = fstatSync(fd).size;
+            const length = Math.min(size, STDERR_TAIL_BYTES);
+            const buffer = Buffer.alloc(length);
+            readSync(fd, buffer, 0, length, size - length);
+            return buffer.toString('utf8');
+        }
+        finally {
+            closeSync(fd);
+        }
+    }
+    catch {
+        return '';
+    }
+}
+/** Keep only the newest few worker logs; the rest are older diagnostics. */
+function pruneWorkerLogs(logRoot) {
+    try {
+        const logs = readdirSync(logRoot)
+            .filter((name) => name.startsWith('worker-') && name.endsWith('.log'))
+            .map((name) => ({ name, modified: statSync(path.join(logRoot, name)).mtimeMs }))
+            .sort((left, right) => right.modified - left.modified);
+        // A recent log may belong to a worker that is still running, so only older ones go.
+        const cutoff = Date.now() - 10 * 60_000;
+        for (const { name, modified } of logs.slice(RETAINED_STDERR_LOGS))
+            if (modified < cutoff)
+                rmSync(path.join(logRoot, name), { force: true });
+    }
+    catch {
+        // Diagnostics are best effort and must never change a job outcome.
+    }
+}
+/**
+ * Remove the log of a worker that ended normally or because the host stopped it. Keep the
+ * newest few logs of workers that ended unexpectedly, because their stack is the evidence.
+ */
+function retainOrRemoveLog(file, evidence, logRoot) {
+    try {
+        const unexpected = !evidence.stoppedByHost &&
+            (evidence.signal !== null || (evidence.code !== null && evidence.code !== 0));
+        if (!unexpected) {
+            rmSync(file, { force: true });
+            return;
+        }
+        pruneWorkerLogs(logRoot);
+    }
+    catch {
+        // Diagnostics are best effort and must never change a job outcome.
+    }
+}
 /**
  * Bounded, fixed-entry child execution. Client disconnection never kills a worker; an
  * overdue or unresponsive cancelled read-only worker is stopped by its supervising host.
@@ -74,12 +139,14 @@ export class JobWorkerHost {
     jobs;
     capacity;
     scheduler;
+    logRoot;
     owner = {};
-    constructor(engine, jobs, capacity, scheduler) {
+    constructor(engine, jobs, capacity, scheduler, logRoot) {
         this.engine = engine;
         this.jobs = jobs;
         this.capacity = capacity;
         this.scheduler = scheduler;
+        this.logRoot = logRoot;
     }
     static async create(engine, jobs) {
         const state = engine.resolver.serverState();
@@ -87,43 +154,17 @@ export class JobWorkerHost {
             throw new ServiceError('JOB_STORAGE_UNAVAILABLE', 'Workers require persistent server state');
         const root = await containedGeneratedPath(state.root, 'job-workers');
         await mkdir(root, { recursive: true, mode: 0o700 });
+        const logRoot = path.join(root, 'logs');
+        await mkdir(logRoot, { recursive: true, mode: 0o700 });
+        pruneWorkerLogs(logRoot);
         const config = engine.resolver.config();
-        return new JobWorkerHost(engine, jobs ?? (await JobService.create(engine.resolver)), new SharedRequestCapacity(root, config.maxSharedTools), new RequestScheduler(config.maxConcurrentTools));
+        return new JobWorkerHost(engine, jobs ?? (await JobService.create(engine.resolver)), new SharedRequestCapacity(root, config.maxSharedTools), new RequestScheduler(config.maxConcurrentTools), logRoot);
     }
     async run(workspaceId, id, principal) {
         const initial = await this.jobs.get(workspaceId, id, principal);
         if (['completed', 'cancelled', 'failed'].includes(initial.status))
             return initial;
-        if (![
-            'hoi4.impact_inspect',
-            'hoi4.decision_inspect',
-            'hoi4.mechanic_test',
-            'hoi4.package_check',
-            'hoi4.scenario_test',
-            'hoi4.event_inspect',
-            'hoi4.event_render',
-            'hoi4.event_compare',
-            'hoi4.tech_inspect',
-            'hoi4.tech_render',
-            'hoi4.tech_compare',
-            'hoi4.probability_inspect',
-            'hoi4.probability_evaluate',
-            'hoi4.probability_sweep',
-            'hoi4.probability_simulate',
-            'hoi4.probability_sequence',
-            'hoi4.probability_compare',
-            'hoi4.probability_render',
-            'hoi4.map_inspect',
-            'hoi4.map_render',
-            'hoi4.map_rewrite',
-            'hoi4.gui_inspect',
-            'hoi4.gui_render',
-            'hoi4.gui_rewrite',
-            'hoi4.focus_inspect',
-            'hoi4.focus_render',
-            'hoi4.focus_raster',
-            'hoi4.focus_rewrite',
-        ].includes(initial.request.toolName))
+        if (!isDomainToolName(initial.request.toolName))
             throw new ServiceError('JOB_OPERATION_UNAVAILABLE', 'This domain has not yet been registered for worker execution');
         const signal = new AbortController().signal;
         const recovered = new Set();
@@ -139,7 +180,7 @@ export class JobWorkerHost {
                     return current;
                 if (current.owner !== undefined && jobOwnerLiveness(current.owner) === 'alive')
                     return await this.followOwner(workspaceId, id, principal);
-                throw new ServiceError('JOB_WORKER_EXIT', 'The worker stopped without publishing a terminal job outcome');
+                throw new ServiceError('JOB_WORKER_EXIT', WORKER_EXIT_MESSAGE);
             }
             catch (error) {
                 const current = await this.jobs.get(workspaceId, id, principal);
@@ -178,15 +219,19 @@ export class JobWorkerHost {
             if (liveness !== 'alive')
                 throw new ServiceError(liveness === 'unknown' ? 'JOB_OWNER_UNRESOLVED' : 'JOB_WORKER_EXIT', liveness === 'unknown'
                     ? 'The replacement execution owner cannot be verified on this host'
-                    : 'The replacement worker stopped without publishing a terminal job outcome');
+                    : `The replacement worker stopped without publishing a terminal job outcome; its exit status belongs to the server that started it, so retry the call`);
             await delay(100);
         }
     }
     async launch(workspaceId, id, lease, progress, principal) {
-        // Resolve the same configured workspace and grants again immediately before dispatch.
-        await this.jobs.get(workspaceId, id, principal);
-        const sourceMode = import.meta.url.endsWith('.ts');
-        const entry = fileURLToPath(new URL(sourceMode ? './job-worker.ts' : './job-worker.js', import.meta.url));
+        // Resolve the same configured workspace and grants again immediately before dispatch. A
+        // job that finished, or that a live worker already runs, needs no new worker: another
+        // launcher may have run it while this one waited for capacity.
+        const current = await this.jobs.get(workspaceId, id, principal);
+        if (['completed', 'failed', 'cancelled'].includes(current.status))
+            return;
+        if (current.owner !== undefined && jobOwnerLiveness(current.owner) === 'alive')
+            return;
         const configuration = this.engine.resolver.config();
         const registeredWorkspaceIds = new Set(configuration.workspaces.map((workspace) => workspace.id));
         const resolvedWorkspaces = this.engine.resolver.list();
@@ -199,96 +244,54 @@ export class JobWorkerHost {
                 workspaceIds: [...new Set([...grant.workspaceIds, ...discoveredWorkspaceIds])],
             }
             : grant;
+        const message = {
+            configuration: {
+                ...configuration,
+                modRoots: [],
+                workspaces: resolvedWorkspaces.map(({ registration }) => registration),
+                http: {
+                    ...configuration.http,
+                    tokens: configuration.http.tokens.map(preserveDiscoveredGrants),
+                    principals: configuration.http.principals.map(preserveDiscoveredGrants),
+                },
+            },
+            workspaceId,
+            jobId: id,
+            ...(principal === undefined ? {} : { principal }),
+        };
+        const warm = configuration.jobWorkerIdleSeconds > 0;
         progress.stage = 'startup';
-        const child = spawn(process.execPath, [
-            ...(configuration.jobWorkerMaxHeapMiB === undefined
-                ? []
-                : [`--max-old-space-size=${configuration.jobWorkerMaxHeapMiB}`]),
-            ...(sourceMode ? ['--import', import.meta.resolve('tsx')] : []),
-            entry,
-        ], {
-            detached: true,
-            cwd: process.cwd(),
-            stdio: ['ignore', 'ignore', 'ignore', 'ipc'],
-            windowsHide: true,
-        });
-        let phase = 'waiting_for_ready';
-        let acknowledgmentTimer;
-        const initialized = new Promise((resolve, reject) => {
-            child.once('error', reject);
-            child.once('exit', (code, signal) => reject(new ServiceError('JOB_WORKER_STARTUP_EXIT', `The worker exited before completing its startup handshake (phase ${phase}, exit ${code ?? 'none'}, signal ${signal ?? 'none'})`)));
-            child.once('message', (value) => {
-                void (async () => {
-                    if (typeof value !== 'object' ||
-                        value === null ||
-                        !('type' in value) ||
-                        value.type !== 'ready' ||
-                        child.pid === undefined)
-                        throw new ServiceError('JOB_WORKER_PROTOCOL', 'The worker did not provide its readiness handshake');
-                    progress.stage = 'handoff';
-                    await lease.handoffToProcess(child.pid);
-                    progress.stage = 'dispatch';
-                    await new Promise((resolve, reject) => {
-                        phase = 'waiting_for_acceptance';
-                        acknowledgmentTimer = setTimeout(() => reject(new ServiceError('JOB_WORKER_ACCEPT_TIMEOUT', 'The worker did not acknowledge receipt within its startup window')), 30_000);
-                        child.once('message', (accepted) => {
-                            if (typeof accepted !== 'object' ||
-                                accepted === null ||
-                                !('type' in accepted) ||
-                                accepted.type !== 'accepted') {
-                                reject(new ServiceError('JOB_WORKER_PROTOCOL', 'The worker did not acknowledge the dispatched job'));
-                                return;
-                            }
-                            phase = 'accepted';
-                            // Only close IPC after the child confirms receipt. The persistent job,
-                            // rather than the launcher's lifetime, owns execution after this point.
-                            if (child.connected)
-                                child.disconnect();
-                            resolve();
-                        });
-                        child.send({
-                            configuration: {
-                                ...configuration,
-                                modRoots: [],
-                                workspaces: resolvedWorkspaces.map(({ registration }) => registration),
-                                http: {
-                                    ...configuration.http,
-                                    tokens: configuration.http.tokens.map(preserveDiscoveredGrants),
-                                    principals: configuration.http.principals.map(preserveDiscoveredGrants),
-                                },
-                            },
-                            workspaceId,
-                            jobId: id,
-                            ...(principal === undefined ? {} : { principal }),
-                        }, (error) => {
-                            if (error !== null)
-                                reject(error);
-                        });
-                    });
-                    resolve();
-                })().catch((error) => {
-                    if (child.connected)
-                        child.disconnect();
-                    reject(error instanceof Error
-                        ? error
-                        : new ServiceError('JOB_WORKER_PROTOCOL', 'Worker initialization failed'));
-                });
-            });
-        });
+        const reused = warm ? this.takeIdleWorker() : undefined;
+        const child = reused ?? (await this.startWorker(configuration.jobWorkerMaxHeapMiB));
         try {
-            await initialized;
+            progress.stage = 'handoff';
+            await lease.handoffToProcess(child.pid);
         }
-        finally {
-            clearTimeout(acknowledgmentTimer);
+        catch (error) {
+            // A reused worker that just exited cannot take the slot; no job reached it.
+            if (reused !== undefined && error instanceof ServiceError)
+                throw new ServiceError('REQUEST_LEASE_HANDOFF_LOST', 'The warm worker stopped before it received the job');
+            if (reused === undefined)
+                await stopChild(child, this.stoppedByHost);
+            throw error;
         }
+        progress.stage = 'dispatch';
+        await this.dispatch(child, message, reused !== undefined);
+        // Without warm reuse the persistent job, not the launcher's lifetime, owns execution once
+        // the child confirmed receipt, so close the channel.
+        if (!warm && child.connected)
+            child.disconnect();
         progress.stage = 'supervision';
         const { jobDeadlineSeconds, jobCancelGraceSeconds } = this.engine.resolver.config();
         const supervisedSince = Date.now();
         let cancellationSeenAt;
         for (;;) {
             const record = await this.jobs.get(workspaceId, id, principal);
-            if (['completed', 'failed', 'cancelled'].includes(record.status))
+            if (['completed', 'failed', 'cancelled'].includes(record.status)) {
+                if (warm)
+                    await this.keepWarm(child, lease);
                 return;
+            }
             if (record.owner !== undefined && record.owner.pid !== child.pid) {
                 if (jobOwnerLiveness(record.owner) === 'alive')
                     return;
@@ -303,7 +306,7 @@ export class JobWorkerHost {
                 const unresponsive = cancellationSeenAt !== undefined &&
                     Date.now() - cancellationSeenAt > jobCancelGraceSeconds * 1000;
                 if (overdue || unresponsive) {
-                    await stopChild(child);
+                    await stopChild(child, this.stoppedByHost);
                     await this.jobs.failInterrupted(workspaceId, id, {
                         code: 'JOB_DEADLINE_EXCEEDED',
                         message: `The job ran longer than ${jobDeadlineSeconds} seconds and was stopped; narrow its selector or limits, or raise jobDeadlineSeconds`,
@@ -311,16 +314,200 @@ export class JobWorkerHost {
                     return;
                 }
             }
-            if (child.pid !== undefined) {
-                try {
-                    process.kill(child.pid, 0);
-                }
-                catch (error) {
-                    if (error.code === 'ESRCH')
-                        throw new ServiceError('JOB_WORKER_EXIT', 'The worker stopped without publishing a terminal job outcome');
-                }
+            if (child.exitCode !== null || child.signalCode !== null) {
+                const evidence = await this.exits.get(child);
+                throw new ServiceError('JOB_WORKER_EXIT', evidence === undefined ? WORKER_EXIT_MESSAGE : classifyWorkerExit(evidence).message);
             }
             await delay(100);
         }
     }
+    /** Workers this host stopped itself, so their exit is not reported as a fault. */
+    stoppedByHost = new WeakSet();
+    /** Settles with the exit status and last standard error of each worker this host started. */
+    exits = new WeakMap();
+    /** Idle warm workers of this host, most recently used last. */
+    idleWorkers = [];
+    /**
+     * Workers that reported idle since their last dispatch. The report usually arrives before the
+     * supervision loop notices the finished job, so it is recorded as it comes.
+     */
+    reportedIdle = new WeakSet();
+    watched = new WeakSet();
+    workerSequence = 0;
+    watchIdle(child) {
+        if (this.watched.has(child))
+            return;
+        this.watched.add(child);
+        child.on('message', (value) => {
+            if (typeof value === 'object' && value !== null && 'type' in value && value.type === 'idle')
+                this.reportedIdle.add(child);
+        });
+    }
+    /** Let every idle warm worker exit; a busy one finishes its job first. */
+    releaseIdleWorkers() {
+        for (const child of this.idleWorkers.splice(0))
+            if (child.connected)
+                child.disconnect();
+    }
+    takeIdleWorker() {
+        for (;;) {
+            const child = this.idleWorkers.pop();
+            if (child === undefined)
+                return undefined;
+            if (child.connected &&
+                child.exitCode === null &&
+                child.signalCode === null &&
+                child.pid !== undefined) {
+                child.ref();
+                child.channel?.ref();
+                return child;
+            }
+        }
+    }
+    /** Start a fixed-entry worker and wait for its readiness handshake. */
+    async startWorker(maxHeapMiB) {
+        const sourceMode = import.meta.url.endsWith('.ts');
+        const entry = fileURLToPath(new URL(sourceMode ? './job-worker.ts' : './job-worker.js', import.meta.url));
+        const heapLimitMiB = maxHeapMiB ?? defaultWorkerHeapMiB();
+        const stderrLog = path.join(this.logRoot, `worker-${process.pid}-${++this.workerSequence}.log`);
+        const stderrFd = openSync(stderrLog, 'w', 0o600);
+        let child;
+        try {
+            child = spawn(process.execPath, [
+                `--max-old-space-size=${heapLimitMiB}`,
+                ...(sourceMode ? ['--import', import.meta.resolve('tsx')] : []),
+                entry,
+            ], {
+                detached: true,
+                cwd: process.cwd(),
+                stdio: ['ignore', 'ignore', stderrFd, 'ipc'],
+                windowsHide: true,
+            });
+        }
+        finally {
+            closeSync(stderrFd);
+        }
+        this.exits.set(child, new Promise((resolve) => {
+            child.once('exit', (code, signal) => {
+                const evidence = {
+                    code,
+                    signal,
+                    stderrTail: readTail(stderrLog),
+                    stoppedByHost: this.stoppedByHost.has(child),
+                    heapLimitMiB,
+                };
+                retainOrRemoveLog(stderrLog, evidence, this.logRoot);
+                resolve(evidence);
+            });
+        }));
+        try {
+            await new Promise((resolve, reject) => {
+                const onExit = (code, signal) => reject(new ServiceError('JOB_WORKER_STARTUP_EXIT', `The worker exited before completing its startup handshake (phase waiting_for_ready, exit ${code ?? 'none'}, signal ${signal ?? 'none'})`));
+                child.once('error', reject);
+                child.once('exit', onExit);
+                child.once('message', (value) => {
+                    child.off('exit', onExit);
+                    child.off('error', reject);
+                    if (typeof value !== 'object' ||
+                        value === null ||
+                        !('type' in value) ||
+                        value.type !== 'ready' ||
+                        child.pid === undefined)
+                        reject(new ServiceError('JOB_WORKER_PROTOCOL', 'The worker did not provide its readiness handshake'));
+                    else
+                        resolve();
+                });
+            });
+        }
+        catch (error) {
+            if (child.connected)
+                child.disconnect();
+            throw error;
+        }
+        return child;
+    }
+    /** Send the job and wait until the worker acknowledges receipt. */
+    async dispatch(child, message, reused) {
+        let acknowledgmentTimer;
+        this.watchIdle(child);
+        this.reportedIdle.delete(child);
+        try {
+            await new Promise((resolve, reject) => {
+                const onExit = (code, signal) => reject(reused
+                    ? new ServiceError('REQUEST_LEASE_HANDOFF_LOST', 'The warm worker stopped before it received the job')
+                    : new ServiceError('JOB_WORKER_STARTUP_EXIT', `The worker exited before completing its startup handshake (phase waiting_for_acceptance, exit ${code ?? 'none'}, signal ${signal ?? 'none'})`));
+                child.once('exit', onExit);
+                acknowledgmentTimer = setTimeout(() => reject(new ServiceError('JOB_WORKER_ACCEPT_TIMEOUT', 'The worker did not acknowledge receipt within its startup window')), 30_000);
+                const onMessage = (accepted) => {
+                    if (typeof accepted === 'object' &&
+                        accepted !== null &&
+                        'type' in accepted &&
+                        accepted.type === 'idle')
+                        return;
+                    child.off('message', onMessage);
+                    child.off('exit', onExit);
+                    if (typeof accepted !== 'object' ||
+                        accepted === null ||
+                        !('type' in accepted) ||
+                        accepted.type !== 'accepted')
+                        reject(new ServiceError('JOB_WORKER_PROTOCOL', 'The worker did not acknowledge the dispatched job'));
+                    else
+                        resolve();
+                };
+                child.on('message', onMessage);
+                child.send(message, (error) => {
+                    if (error !== null)
+                        reject(error);
+                });
+            });
+        }
+        catch (error) {
+            if (child.connected)
+                child.disconnect();
+            throw error;
+        }
+        finally {
+            clearTimeout(acknowledgmentTimer);
+        }
+    }
+    /**
+     * After a job, wait briefly for the worker to report itself idle, release its capacity slot
+     * and keep it for the next job. A worker that does not report idle is left to exit.
+     */
+    async keepWarm(child, lease) {
+        if (!child.connected || child.exitCode !== null || child.signalCode !== null)
+            return;
+        const idle = await new Promise((resolve) => {
+            if (this.reportedIdle.has(child)) {
+                resolve(true);
+                return;
+            }
+            const timer = setTimeout(() => finish(false), 10_000);
+            const onMessage = (value) => {
+                if (typeof value === 'object' && value !== null && 'type' in value && value.type === 'idle')
+                    finish(true);
+            };
+            const onExit = () => finish(false);
+            function finish(result) {
+                clearTimeout(timer);
+                child.off('message', onMessage);
+                child.off('exit', onExit);
+                resolve(result);
+            }
+            child.on('message', onMessage);
+            child.once('exit', onExit);
+        });
+        // The channel can close while waiting; read it afresh rather than from the earlier check.
+        const connected = () => child.connected;
+        if (!idle || !connected()) {
+            if (connected())
+                child.disconnect();
+            return;
+        }
+        await lease.reclaimFromProcess();
+        // An idle worker must not keep this server's event loop alive.
+        child.unref();
+        child.channel?.unref();
+        this.idleWorkers.push(child);
+    }
 }