llm-cli-gateway 2.14.0-rc.1 → 2.14.1

package/dist/async-job-manager.js

@@ -1,4 +1,6 @@
 import { randomUUID } from "crypto";
+import os from "os";
+import { hrtime } from "process";
 import { envWithExtendedPath, getExtendedPath, killProcessGroup, providerCommandName, spawnCliProcess, unregisterProcessGroup, } from "./executor.js";
 import { noopLogger, logWarn } from "./logger.js";
 import { ProcessMonitor } from "./process-monitor.js";
@@ -8,6 +10,16 @@ import { codexFrResponse } from "./codex-json-parser.js";
 import { extractProviderOutputMetadata } from "./provider-output-metadata.js";
 import { getRequestContext, resolveOwnerPrincipal, principalCanAccess } from "./request-context.js";
 import { runApiRequest, ApiHttpError, } from "./api-provider.js";
+import { DEFAULT_INSTANCE_HEARTBEAT_MS, DEFAULT_INSTANCE_LEASE_TTL_MS, DEFAULT_HTTP_JOB_GRACE_MS, DEFAULT_ORPHAN_SWEEP_INTERVAL_MS, DEFAULT_INSTANCE_GC_MS, } from "./config.js";
+const DEFAULT_LEASE_RUNTIME_CONFIG = {
+    instanceHeartbeatMs: DEFAULT_INSTANCE_HEARTBEAT_MS,
+    instanceLeaseTtlMs: DEFAULT_INSTANCE_LEASE_TTL_MS,
+    httpJobGraceMs: DEFAULT_HTTP_JOB_GRACE_MS,
+    orphanSweepIntervalMs: DEFAULT_ORPHAN_SWEEP_INTERVAL_MS,
+    instanceGcMs: DEFAULT_INSTANCE_GC_MS,
+    role: "gateway",
+};
+const MAX_CONSECUTIVE_HEARTBEAT_FAILURES = 3;
 export function extractApiHttpStatus(error) {
     for (const candidate of [error, error?.cause]) {
         if (candidate instanceof ApiHttpError && typeof candidate.status === "number") {
@@ -238,13 +250,26 @@ export class AsyncJobManager {
     limiter;
     completedJobMemoryTtlMs;
     maxJobOutputBytes;
-    constructor(logger = noopLogger, onJobComplete, store = null, flightRecorder = new NoopFlightRecorder(), limits = DEFAULT_MANAGER_JOB_LIMITS) {
+    instanceId = randomUUID();
+    hostname = os.hostname();
+    instancePid = process.pid;
+    lease;
+    heartbeatTimer = null;
+    sweepTimer = null;
+    durableAdmission;
+    consecutiveHeartbeatFailures = 0;
+    skipSweepThisCycle = false;
+    nextHeartbeatExpectedAt = 0;
+    disposed = false;
+    pendingWrites = new Set();
+    constructor(logger = noopLogger, onJobComplete, store = null, flightRecorder = new NoopFlightRecorder(), limits = DEFAULT_MANAGER_JOB_LIMITS, _deprecatedOwnsOrphanRecovery = true, leaseConfig = DEFAULT_LEASE_RUNTIME_CONFIG) {
         this.logger = logger;
         this.onJobComplete = onJobComplete;
         this.processMonitor = new ProcessMonitor(logger);
         this.store = store;
         this.flightRecorder = flightRecorder;
         this.limits = limits;
+        this.lease = leaseConfig;
         this.completedJobMemoryTtlMs = limits.completedJobMemoryTtlMs;
         this.maxJobOutputBytes = limits.maxJobOutputBytes;
         this.limiter = new JobLimiter({
@@ -253,23 +278,24 @@ export class AsyncJobManager {
             maxQueuedJobs: limits.maxQueuedJobs,
             queueTimeoutMs: limits.queueTimeoutMs,
         }, logger);
+        this.durableAdmission = store !== null;
         if (this.store) {
             try {
-                const { count, orphaned } = this.store.markOrphanedOnStartup();
-                if (count > 0) {
-                    this.logger.info(`Marked ${count} in-flight job(s) as orphaned after gateway restart`);
-                }
-                for (const orphan of orphaned) {
-                    try {
-                        this.flightRecorder.logComplete(orphan.correlationId, this.buildOrphanFlightResult(orphan));
-                    }
-                    catch (err) {
-                        this.logger.error(`Async-path FR logComplete for orphaned job ${orphan.id} failed`, err);
-                    }
-                }
+                this.store.registerInstance({
+                    instanceId: this.instanceId,
+                    role: this.lease.role ?? "gateway",
+                    hostname: this.hostname,
+                    pid: this.instancePid,
+                });
             }
             catch (err) {
-                this.logger.error("markOrphanedOnStartup failed", err);
+                this.durableAdmission = false;
+                this.logger.error("#139 registerInstance failed; durable async admission is disabled for this instance", err);
+            }
+            if (this.durableAdmission) {
+                this.runOrphanSweep();
+                this.startHeartbeat();
+                this.startReaper();
             }
         }
         this.evictionTimer = setInterval(() => this.evictCompletedJobs(), EVICTION_INTERVAL_MS);
@@ -281,6 +307,212 @@ export class AsyncJobManager {
             this.stallTimer.unref();
         }
     }
+    canAdmitDurableJobs() {
+        return this.store !== null && this.durableAdmission;
+    }
+    assertDurableAdmission(provider) {
+        if (this.store && !this.durableAdmission) {
+            throw new Error(`Durable async admission is disabled for ${provider}: this gateway instance could not register or lost its heartbeat lease. Retry after the gateway recovers.`);
+        }
+    }
+    trackPendingWrite(p) {
+        this.pendingWrites.add(p);
+        void p.finally(() => this.pendingWrites.delete(p));
+    }
+    async dispose(opts = {}) {
+        const timeoutMs = opts.timeoutMs ?? 5000;
+        if (this.disposed)
+            return;
+        this.disposed = true;
+        this.durableAdmission = false;
+        for (const t of [this.heartbeatTimer, this.sweepTimer, this.evictionTimer, this.stallTimer]) {
+            if (t)
+                clearInterval(t);
+        }
+        this.heartbeatTimer = null;
+        this.sweepTimer = null;
+        this.evictionTimer = null;
+        this.stallTimer = null;
+        if (!this.store)
+            return;
+        const active = [...this.jobs.values()].filter(job => isAsyncJobInProgress(job.status));
+        for (const job of active) {
+            try {
+                if (job.transport === "http") {
+                    job.abort?.abort();
+                }
+                else if (job.process) {
+                    killProcessGroup(job.process, "SIGTERM");
+                }
+            }
+            catch (err) {
+                this.logger.error(`#139 dispose: failed to signal job ${job.id}`, err);
+            }
+        }
+        const deadline = Date.now() + timeoutMs;
+        while (Date.now() < deadline) {
+            const stillActive = [...this.jobs.values()].some(job => isAsyncJobInProgress(job.status));
+            if (!stillActive && this.pendingWrites.size === 0)
+                break;
+            await Promise.race([
+                Promise.allSettled([...this.pendingWrites]),
+                new Promise(resolve => setTimeout(resolve, 50)),
+            ]);
+        }
+        const stillActive = [...this.jobs.values()].some(job => isAsyncJobInProgress(job.status));
+        if (stillActive) {
+            logWarn(this.logger, "#139 dispose timed out with active owned jobs; skipping deregister and letting the lease expire");
+            return;
+        }
+        try {
+            this.store.deregisterInstance(this.instanceId);
+        }
+        catch (err) {
+            this.logger.error("#139 dispose: deregisterInstance failed", err);
+        }
+    }
+    getInstanceId() {
+        return this.instanceId;
+    }
+    startHeartbeat() {
+        const intervalMs = this.lease.instanceHeartbeatMs;
+        this.nextHeartbeatExpectedAt = Number(hrtime.bigint() / 1000000n) + intervalMs;
+        this.heartbeatTimer = setInterval(() => this.onHeartbeatTick(intervalMs), intervalMs);
+        if (this.heartbeatTimer.unref)
+            this.heartbeatTimer.unref();
+    }
+    onHeartbeatTick(intervalMs) {
+        if (this.disposed || !this.store)
+            return;
+        const nowMonoMs = Number(hrtime.bigint() / 1000000n);
+        const driftMs = nowMonoMs - this.nextHeartbeatExpectedAt;
+        this.nextHeartbeatExpectedAt = nowMonoMs + intervalMs;
+        if (driftMs > intervalMs) {
+            this.skipSweepThisCycle = true;
+            logWarn(this.logger, `#139 heartbeat scheduling drift ${Math.round(driftMs)}ms exceeded interval ${intervalMs}ms; skipping the next orphan sweep`);
+        }
+        try {
+            this.store.heartbeat(this.instanceId);
+            this.consecutiveHeartbeatFailures = 0;
+        }
+        catch (err) {
+            this.consecutiveHeartbeatFailures++;
+            this.logger.error(`#139 heartbeat failed (${this.consecutiveHeartbeatFailures}/${MAX_CONSECUTIVE_HEARTBEAT_FAILURES})`, err);
+            if (this.consecutiveHeartbeatFailures >= MAX_CONSECUTIVE_HEARTBEAT_FAILURES) {
+                if (this.durableAdmission) {
+                    this.durableAdmission = false;
+                    this.logger.error("#139 sustained heartbeat failure; disabling durable admission and orphan sweeping on this instance");
+                }
+            }
+        }
+    }
+    runOrphanSweepNow() {
+        this.runOrphanSweep();
+    }
+    startReaper() {
+        this.sweepTimer = setInterval(() => {
+            this.runOrphanSweep();
+            try {
+                this.store?.gcInstances(this.lease.instanceGcMs);
+            }
+            catch (err) {
+                this.logger.error("#139 gateway_instances GC failed", err);
+            }
+        }, this.lease.orphanSweepIntervalMs);
+        if (this.sweepTimer.unref)
+            this.sweepTimer.unref();
+    }
+    runOrphanSweep() {
+        if (!this.store || this.disposed || !this.durableAdmission)
+            return;
+        if (this.skipSweepThisCycle) {
+            this.skipSweepThisCycle = false;
+            return;
+        }
+        const leaseTtl = this.lease.instanceLeaseTtlMs;
+        const httpGrace = this.lease.httpJobGraceMs;
+        let liveConfirmedIds = [];
+        try {
+            const candidates = this.store.selectStaleProcessCandidates(leaseTtl, httpGrace);
+            liveConfirmedIds = this.confirmLiveProcessCandidates(candidates);
+        }
+        catch (err) {
+            this.logger.error("#139 selecting stale process candidates failed", err);
+        }
+        let orphaned;
+        try {
+            orphaned = this.store.recoverStaleJobs(leaseTtl, httpGrace, liveConfirmedIds);
+        }
+        catch (err) {
+            this.logger.error("#139 recoverStaleJobs failed", err);
+            return;
+        }
+        if (orphaned.length > 0) {
+            this.logger.info(`#139 orphaned ${orphaned.length} stale job(s) whose owning instance is gone`);
+        }
+        for (const orphan of orphaned) {
+            try {
+                this.flightRecorder.logComplete(orphan.correlationId, this.buildOrphanFlightResult(orphan));
+            }
+            catch (err) {
+                this.logger.error(`#139 FR logComplete for orphaned job ${orphan.id} failed`, err);
+            }
+        }
+    }
+    confirmLiveProcessCandidates(candidates) {
+        const live = [];
+        for (const c of candidates) {
+            if (c.transport !== "process" || c.pid == null)
+                continue;
+            if (c.hostname !== null && c.hostname !== this.hostname)
+                continue;
+            if (c.hostname === null)
+                continue;
+            try {
+                process.kill(c.pid, 0);
+                live.push(c.id);
+            }
+            catch (err) {
+                const code = err.code;
+                if (code === "EPERM")
+                    live.push(c.id);
+            }
+        }
+        return live;
+    }
+    recordStartOrFailClosed(job, acq, input) {
+        if (!this.store)
+            return;
+        try {
+            this.store.recordStart(input);
+        }
+        catch (err) {
+            if (acq.state === "granted")
+                acq.permit.release();
+            else
+                acq.cancel();
+            this.jobs.delete(job.id);
+            this.logger.error(`#139 durable recordStart failed for job ${job.id}; failing the request (fail-closed)`, err);
+            throw new Error(`Durable job admission failed for ${job.cli}: the job store rejected recordStart`);
+        }
+    }
+    markRunningDurable(job, pid, failClosed = false) {
+        if (!this.store)
+            return;
+        let transitioned = false;
+        try {
+            transitioned = this.store.markRunning(job.id, { pid });
+        }
+        catch (err) {
+            if (failClosed)
+                throw err;
+            this.logger.error(`#139 markRunning (best-effort) failed for job ${job.id}`, err);
+            return;
+        }
+        if (!transitioned && failClosed) {
+            throw new Error(`#139 markRunning matched no queued row for job ${job.id} (already recovered or terminal); refusing to run a child against a stale durable row`);
+        }
+    }
     buildOrphanFlightResult(orphan) {
         const durationMs = Math.max(0, Date.now() - new Date(orphan.startedAt).getTime());
         const hasCapturedStdout = orphan.stdout.length > 0;
@@ -500,6 +732,7 @@ export class AsyncJobManager {
             if (reused)
                 return reused;
         }
+        this.assertDurableAdmission(provider.name);
         const id = randomUUID();
         const startedAt = new Date().toISOString();
         const abort = new AbortController();
@@ -555,16 +788,18 @@ export class AsyncJobManager {
                 return;
             }
             job.status = "running";
-            runApiRequest(provider, apiRequest, this.logger, { signal: abort.signal })
+            this.markRunningDurable(job, null);
+            const settle = runApiRequest(provider, apiRequest, this.logger, { signal: abort.signal })
                 .then(result => this.finalizeHttpJob(job, result, null))
                 .catch(error => this.finalizeHttpJob(job, null, error));
+            this.trackPendingWrite(settle);
         };
         const acq = this.limiter.acquire(provider.name, permit => launch(permit), () => this.failQueuedJob(job, "queue wait timed out before a run slot was free"));
         if (acq.state === "rejected") {
             this.jobs.delete(id);
             throw new JobSaturationError(provider.name, `running limit (${this.limits.maxRunningJobs} global / ${this.limits.maxRunningJobsPerProvider} per provider) reached and the queue is full (max ${this.limits.maxQueuedJobs}); retry shortly`);
         }
-        this.safeStoreCall("recordStart", () => this.store.recordStart({
+        this.recordStartOrFailClosed(job, acq, {
             id,
             correlationId,
             requestKey,
@@ -573,9 +808,10 @@ export class AsyncJobManager {
             startedAt,
             pid: null,
             ownerPrincipal,
+            ownerInstance: this.instanceId,
             transport: "http",
             payloadJson,
-        }));
+        });
         if (writeFlightStart && flightRecorderEntry) {
             try {
                 this.flightRecorder.logStart({
@@ -829,7 +1065,7 @@ export class AsyncJobManager {
             abort: null,
             httpStatus: row.httpStatus,
             payloadJson: row.payloadJson,
-            exited: row.status !== "running",
+            exited: row.status !== "running" && row.status !== "queued",
             metricsRecorded: true,
             outputFormat: row.outputFormat ?? undefined,
             ownerPrincipal: row.ownerPrincipal,
@@ -867,6 +1103,7 @@ export class AsyncJobManager {
             if (reused)
                 return reused;
         }
+        this.assertDurableAdmission(cli);
         const id = randomUUID();
         const startedAt = new Date().toISOString();
         const ownerPrincipal = resolveOwnerPrincipal(getRequestContext());
@@ -934,7 +1171,7 @@ export class AsyncJobManager {
             this.jobs.delete(id);
             throw new JobSaturationError(cli, `running limit (${this.limits.maxRunningJobs} global / ${this.limits.maxRunningJobsPerProvider} per provider) reached and the queue is full (max ${this.limits.maxQueuedJobs}); retry shortly`);
         }
-        this.safeStoreCall("recordStart", () => this.store.recordStart({
+        this.recordStartOrFailClosed(job, acq, {
             id,
             correlationId,
             requestKey,
@@ -944,7 +1181,8 @@ export class AsyncJobManager {
             startedAt,
             pid: null,
             ownerPrincipal,
-        }));
+            ownerInstance: this.instanceId,
+        });
         if (writeFlightStart && flightRecorderEntry) {
             try {
                 this.flightRecorder.logStart({
@@ -986,6 +1224,24 @@ export class AsyncJobManager {
             env: { ...baseEnv, ...(extraEnv ?? {}) },
         });
         job.process = child;
+        try {
+            this.markRunningDurable(job, child.pid ?? null, true);
+        }
+        catch (err) {
+            try {
+                killProcessGroup(child, "SIGKILL");
+            }
+            catch {
+            }
+            if (child.pid) {
+                try {
+                    unregisterProcessGroup(child.pid);
+                }
+                catch {
+                }
+            }
+            throw err;
+        }
         if (stdin !== undefined && child.stdin) {
             try {
                 child.stdin.write(stdin);

package/dist/config.d.ts

@@ -20,6 +20,11 @@ export declare const PERSISTENCE_BACKENDS: readonly ["sqlite", "postgres", "memo
 export type PersistenceBackend = (typeof PERSISTENCE_BACKENDS)[number];
 export declare const DEFAULT_JOB_RETENTION_DAYS = 30;
 export declare const DEFAULT_DEDUP_WINDOW_MS: number;
+export declare const DEFAULT_INSTANCE_HEARTBEAT_MS = 15000;
+export declare const DEFAULT_INSTANCE_LEASE_TTL_MS = 90000;
+export declare const DEFAULT_HTTP_JOB_GRACE_MS = 300000;
+export declare const DEFAULT_ORPHAN_SWEEP_INTERVAL_MS = 30000;
+export declare const DEFAULT_INSTANCE_GC_MS = 3600000;
 export interface PersistenceConfig {
     backend: PersistenceBackend;
     path: string | null;
@@ -27,6 +32,12 @@ export interface PersistenceConfig {
     retentionDays: number;
     dedupWindowMs: number;
     acknowledgeEphemeral: boolean;
+    ownsOrphanRecovery: boolean;
+    instanceHeartbeatMs: number;
+    instanceLeaseTtlMs: number;
+    httpJobGraceMs: number;
+    orphanSweepIntervalMs: number;
+    instanceGcMs: number;
     asyncJobsEnabled: boolean;
     sources: PersistenceConfigSources;
 }

package/dist/config.js

@@ -45,6 +45,11 @@ export function loadConfig() {
 export const PERSISTENCE_BACKENDS = ["sqlite", "postgres", "memory", "none"];
 export const DEFAULT_JOB_RETENTION_DAYS = 30;
 export const DEFAULT_DEDUP_WINDOW_MS = 60 * 60 * 1000;
+export const DEFAULT_INSTANCE_HEARTBEAT_MS = 15000;
+export const DEFAULT_INSTANCE_LEASE_TTL_MS = 90000;
+export const DEFAULT_HTTP_JOB_GRACE_MS = 300000;
+export const DEFAULT_ORPHAN_SWEEP_INTERVAL_MS = 30000;
+export const DEFAULT_INSTANCE_GC_MS = 3600000;
 const PersistenceSchema = z
     .object({
     backend: z.enum(PERSISTENCE_BACKENDS).default("sqlite"),
@@ -53,8 +58,30 @@ const PersistenceSchema = z
     retentionDays: z.number().positive().default(DEFAULT_JOB_RETENTION_DAYS),
     dedupWindowMs: z.number().int().nonnegative().default(DEFAULT_DEDUP_WINDOW_MS),
     acknowledgeEphemeral: z.boolean().default(false),
+    ownsOrphanRecovery: z.boolean().default(false),
+    instanceHeartbeatMs: z.number().int().positive().default(DEFAULT_INSTANCE_HEARTBEAT_MS),
+    instanceLeaseTtlMs: z.number().int().positive().default(DEFAULT_INSTANCE_LEASE_TTL_MS),
+    httpJobGraceMs: z.number().int().positive().default(DEFAULT_HTTP_JOB_GRACE_MS),
+    orphanSweepIntervalMs: z.number().int().positive().default(DEFAULT_ORPHAN_SWEEP_INTERVAL_MS),
+    instanceGcMs: z.number().int().positive().default(DEFAULT_INSTANCE_GC_MS),
 })
-    .strict();
+    .strict()
+    .superRefine((cfg, ctx) => {
+    if (cfg.instanceLeaseTtlMs < 2 * cfg.instanceHeartbeatMs) {
+        ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            path: ["instanceLeaseTtlMs"],
+            message: `instanceLeaseTtlMs (${cfg.instanceLeaseTtlMs}) must be >= 2 * instanceHeartbeatMs (${2 * cfg.instanceHeartbeatMs})`,
+        });
+    }
+    if (cfg.httpJobGraceMs < cfg.instanceLeaseTtlMs) {
+        ctx.addIssue({
+            code: z.ZodIssueCode.custom,
+            path: ["httpJobGraceMs"],
+            message: `httpJobGraceMs (${cfg.httpJobGraceMs}) must be >= instanceLeaseTtlMs (${cfg.instanceLeaseTtlMs})`,
+        });
+    }
+});
 const DEFAULT_SQLITE_PATH = path.join(os.homedir(), ".llm-cli-gateway", "logs.db");
 function defaultPersistenceConfigPath() {
     return (process.env.LLM_GATEWAY_CONFIG ?? path.join(os.homedir(), ".llm-cli-gateway", "config.toml"));
@@ -153,6 +180,9 @@ export function loadPersistenceConfig(logger = noopLogger) {
         envOverrides: [],
     };
     const merged = applyEnvOverrides(raw ?? {}, logger, sources);
+    if (merged.ownsOrphanRecovery !== undefined) {
+        logWarn(logger, "[persistence].ownsOrphanRecovery is deprecated and no longer used; the durable per-job lease recovery (#139) runs safely from every instance. Remove it from config.toml.");
+    }
     let parsed;
     try {
         parsed = PersistenceSchema.parse(merged);
@@ -182,6 +212,12 @@ export function loadPersistenceConfig(logger = noopLogger) {
         retentionDays: parsed.retentionDays,
         dedupWindowMs: parsed.dedupWindowMs,
         acknowledgeEphemeral: parsed.acknowledgeEphemeral,
+        ownsOrphanRecovery: parsed.ownsOrphanRecovery,
+        instanceHeartbeatMs: parsed.instanceHeartbeatMs,
+        instanceLeaseTtlMs: parsed.instanceLeaseTtlMs,
+        httpJobGraceMs: parsed.httpJobGraceMs,
+        orphanSweepIntervalMs: parsed.orphanSweepIntervalMs,
+        instanceGcMs: parsed.instanceGcMs,
         asyncJobsEnabled,
         sources,
     };

package/dist/index.js

@@ -291,9 +291,17 @@ function getJobStore(runtimeLogger = logger) {
     return jobStore;
 }
 function newAsyncJobManager(metrics, runtimeLogger, store = getJobStore(runtimeLogger), fr = getFlightRecorder(runtimeLogger)) {
+    const pc = getPersistenceConfig(runtimeLogger);
     return new AsyncJobManager(runtimeLogger, (cli, durationMs, success) => {
         metrics.recordRequest(cli, durationMs, success);
-    }, store, fr, getLimitsConfig(runtimeLogger).jobs);
+    }, store, fr, getLimitsConfig(runtimeLogger).jobs, true, {
+        instanceHeartbeatMs: pc.instanceHeartbeatMs,
+        instanceLeaseTtlMs: pc.instanceLeaseTtlMs,
+        httpJobGraceMs: pc.httpJobGraceMs,
+        orphanSweepIntervalMs: pc.orphanSweepIntervalMs,
+        instanceGcMs: pc.instanceGcMs,
+        role: process.argv.includes("--transport=http") ? "http" : "stdio",
+    });
 }
 function getAsyncJobManager(runtimeLogger = logger) {
     asyncJobManager ??= newAsyncJobManager(performanceMetrics, runtimeLogger);
@@ -573,7 +581,7 @@ async function awaitJobOrDefer(cli, args, corrId, idleTimeoutMs, outputFormat, f
     }
     const deferralAvailable = runtime.persistence.backend !== "none" &&
         runtime.persistence.asyncJobsEnabled &&
-        runtime.asyncJobManager.hasStore();
+        runtime.asyncJobManager.canAdmitDurableJobs();
     if (SYNC_DEADLINE_MS === 0 || !deferralAvailable) {
         const command = providerCommandName(cli);
         let slot;
@@ -662,7 +670,7 @@ async function awaitApiJobOrDefer(provider, apiRequest, corrId, runtime = resolv
     };
     const deferralAvailable = runtime.persistence.backend !== "none" &&
         runtime.persistence.asyncJobsEnabled &&
-        runtime.asyncJobManager.hasStore();
+        runtime.asyncJobManager.canAdmitDurableJobs();
     if (SYNC_DEADLINE_MS === 0 || !deferralAvailable || forceInline) {
         let slot;
         try {
@@ -8282,6 +8290,15 @@ function registerHealthResource(server) {
 async function shutdown(signal) {
     logger.info(`Received ${signal}, shutting down gracefully...`);
     try {
+        if (asyncJobManager) {
+            try {
+                await asyncJobManager.dispose({ timeoutMs: 5000 });
+                logger.info("Async job manager disposed");
+            }
+            catch (err) {
+                logger.error("Async job manager dispose failed", err);
+            }
+        }
         await killAllProcessGroups();
         logger.info("All process groups terminated");
         if (activeHttpGateway) {