llm-cli-gateway 2.14.0-rc.1 → 2.14.0

package/dist/index.js

@@ -291,9 +291,17 @@ function getJobStore(runtimeLogger = logger) {
     return jobStore;
 }
 function newAsyncJobManager(metrics, runtimeLogger, store = getJobStore(runtimeLogger), fr = getFlightRecorder(runtimeLogger)) {
+    const pc = getPersistenceConfig(runtimeLogger);
     return new AsyncJobManager(runtimeLogger, (cli, durationMs, success) => {
         metrics.recordRequest(cli, durationMs, success);
-    }, store, fr, getLimitsConfig(runtimeLogger).jobs);
+    }, store, fr, getLimitsConfig(runtimeLogger).jobs, true, {
+        instanceHeartbeatMs: pc.instanceHeartbeatMs,
+        instanceLeaseTtlMs: pc.instanceLeaseTtlMs,
+        httpJobGraceMs: pc.httpJobGraceMs,
+        orphanSweepIntervalMs: pc.orphanSweepIntervalMs,
+        instanceGcMs: pc.instanceGcMs,
+        role: process.argv.includes("--transport=http") ? "http" : "stdio",
+    });
 }
 function getAsyncJobManager(runtimeLogger = logger) {
     asyncJobManager ??= newAsyncJobManager(performanceMetrics, runtimeLogger);
@@ -573,7 +581,7 @@ async function awaitJobOrDefer(cli, args, corrId, idleTimeoutMs, outputFormat, f
     }
     const deferralAvailable = runtime.persistence.backend !== "none" &&
         runtime.persistence.asyncJobsEnabled &&
-        runtime.asyncJobManager.hasStore();
+        runtime.asyncJobManager.canAdmitDurableJobs();
     if (SYNC_DEADLINE_MS === 0 || !deferralAvailable) {
         const command = providerCommandName(cli);
         let slot;
@@ -662,7 +670,7 @@ async function awaitApiJobOrDefer(provider, apiRequest, corrId, runtime = resolv
     };
     const deferralAvailable = runtime.persistence.backend !== "none" &&
         runtime.persistence.asyncJobsEnabled &&
-        runtime.asyncJobManager.hasStore();
+        runtime.asyncJobManager.canAdmitDurableJobs();
     if (SYNC_DEADLINE_MS === 0 || !deferralAvailable || forceInline) {
         let slot;
         try {
@@ -8282,6 +8290,15 @@ function registerHealthResource(server) {
 async function shutdown(signal) {
     logger.info(`Received ${signal}, shutting down gracefully...`);
     try {
+        if (asyncJobManager) {
+            try {
+                await asyncJobManager.dispose({ timeoutMs: 5000 });
+                logger.info("Async job manager disposed");
+            }
+            catch (err) {
+                logger.error("Async job manager dispose failed", err);
+            }
+        }
         await killAllProcessGroups();
         logger.info("All process groups terminated");
         if (activeHttpGateway) {

package/dist/job-store.d.ts

@@ -1,6 +1,7 @@
 import type { Logger } from "./logger.js";
 import type { PersistenceConfig } from "./config.js";
-export type JobStoreStatus = "running" | "completed" | "failed" | "canceled" | "orphaned";
+export type JobStoreStatus = "queued" | "running" | "completed" | "failed" | "canceled" | "orphaned";
+export type JobStoreActiveStatus = Extract<JobStoreStatus, "queued" | "running">;
 export type JobTransport = "process" | "http";
 export interface JobRecord {
     id: string;
@@ -23,11 +24,26 @@ export interface JobRecord {
     transport: JobTransport;
     httpStatus: number | null;
     payloadJson: string | null;
+    ownerInstance: string | null;
+    leaseDeadline: number | null;
 }
 export declare function resolveJobStoreDbPath(): string | null;
 export declare function resolveJobRetentionMs(): number;
 export declare function resolveDedupWindowMs(): number;
 export declare function computeRequestKey(cli: string, args: string[], extra?: string): string;
+export interface GatewayInstanceMeta {
+    instanceId: string;
+    role: string | null;
+    hostname: string | null;
+    pid: number | null;
+}
+export interface SweepCandidate {
+    id: string;
+    pid: number | null;
+    transport: JobTransport;
+    ownerInstance: string | null;
+    hostname: string | null;
+}
 export interface JobStore {
     recordStart(input: {
         id: string;
@@ -39,13 +55,23 @@ export interface JobStore {
         startedAt: string;
         pid: number | null;
         ownerPrincipal?: string | null;
+        ownerInstance?: string | null;
         transport?: JobTransport;
         payloadJson?: string | null;
     }): void;
+    markRunning(id: string, opts: {
+        pid: number | null;
+    }): boolean;
+    registerInstance(meta: GatewayInstanceMeta): void;
+    heartbeat(instanceId: string): void;
+    deregisterInstance(instanceId: string): void;
+    selectStaleProcessCandidates(leaseTtlMs: number, httpJobGraceMs: number): SweepCandidate[];
+    recoverStaleJobs(leaseTtlMs: number, httpJobGraceMs: number, liveConfirmedIds?: string[]): OrphanedJobSnapshot[];
+    gcInstances(instanceGcMs: number): number;
     recordOutput(id: string, stdout: string, stderr: string, outputTruncated: boolean): void;
     recordComplete(input: {
         id: string;
-        status: Exclude<JobStoreStatus, "running">;
+        status: Exclude<JobStoreStatus, "running" | "queued">;
         exitCode: number | null;
         stdout: string;
         stderr: string;
@@ -117,6 +143,7 @@ export declare class SqliteJobStore implements JobStore, ValidationRunStore {
     private db;
     private retentionMs;
     private dedupWindowMs;
+    private leaseTtlMs;
     private insertStmt;
     private updateOutputStmt;
     private updateCompleteStmt;
@@ -125,9 +152,19 @@ export declare class SqliteJobStore implements JobStore, ValidationRunStore {
     private selectRunningOrphansStmt;
     private markOrphanedStmt;
     private deleteExpiredStmt;
+    private markRunningStmt;
+    private registerInstanceStmt;
+    private heartbeatInstanceStmt;
+    private heartbeatJobsStmt;
+    private deregisterInstanceStmt;
+    private selectStaleCandidatesStmt;
+    private orphanExpiredStmt;
+    private advanceLeaseStmt;
+    private gcInstancesStmt;
     constructor(dbPath: string, logger?: Logger, options?: {
         retentionMs?: number;
         dedupWindowMs?: number;
+        leaseTtlMs?: number;
     });
     recordStart(input: {
         id: string;
@@ -139,13 +176,23 @@ export declare class SqliteJobStore implements JobStore, ValidationRunStore {
         startedAt: string;
         pid: number | null;
         ownerPrincipal?: string | null;
+        ownerInstance?: string | null;
         transport?: JobTransport;
         payloadJson?: string | null;
     }): void;
+    markRunning(id: string, opts: {
+        pid: number | null;
+    }): boolean;
+    registerInstance(meta: GatewayInstanceMeta): void;
+    heartbeat(instanceId: string): void;
+    deregisterInstance(instanceId: string): void;
+    selectStaleProcessCandidates(_leaseTtlMs: number, _httpJobGraceMs: number): SweepCandidate[];
+    recoverStaleJobs(leaseTtlMs: number, httpJobGraceMs: number, liveConfirmedIds?: string[]): OrphanedJobSnapshot[];
+    gcInstances(instanceGcMs: number): number;
     recordOutput(id: string, stdout: string, stderr: string, outputTruncated: boolean): void;
     recordComplete(input: {
         id: string;
-        status: Exclude<JobStoreStatus, "running">;
+        status: Exclude<JobStoreStatus, "running" | "queued">;
         exitCode: number | null;
         stdout: string;
         stderr: string;
@@ -176,9 +223,11 @@ export declare class MemoryJobStore implements JobStore {
     private rows;
     private retentionMs;
     private dedupWindowMs;
+    private leaseTtlMs;
     constructor(options?: {
         retentionMs?: number;
         dedupWindowMs?: number;
+        leaseTtlMs?: number;
     });
     recordStart(input: {
         id: string;
@@ -190,13 +239,23 @@ export declare class MemoryJobStore implements JobStore {
         startedAt: string;
         pid: number | null;
         ownerPrincipal?: string | null;
+        ownerInstance?: string | null;
         transport?: JobTransport;
         payloadJson?: string | null;
     }): void;
+    markRunning(id: string, opts: {
+        pid: number | null;
+    }): boolean;
+    registerInstance(_meta: GatewayInstanceMeta): void;
+    heartbeat(instanceId: string): void;
+    deregisterInstance(_instanceId: string): void;
+    selectStaleProcessCandidates(_leaseTtlMs: number, _httpJobGraceMs: number): SweepCandidate[];
+    recoverStaleJobs(_leaseTtlMs: number, _httpJobGraceMs: number, _liveConfirmedIds?: string[]): OrphanedJobSnapshot[];
+    gcInstances(_instanceGcMs: number): number;
     recordOutput(id: string, stdout: string, stderr: string, outputTruncated: boolean): void;
     recordComplete(input: {
         id: string;
-        status: Exclude<JobStoreStatus, "running">;
+        status: Exclude<JobStoreStatus, "running" | "queued">;
         exitCode: number | null;
         stdout: string;
         stderr: string;
@@ -223,6 +282,7 @@ export declare class PostgresJobStore implements JobStore, ValidationRunStore {
     constructor(dsn: string, logger?: Logger, options?: {
         retentionMs?: number;
         dedupWindowMs?: number;
+        leaseTtlMs?: number;
     });
     private syncCall;
     recordStart(input: {
@@ -235,13 +295,23 @@ export declare class PostgresJobStore implements JobStore, ValidationRunStore {
         startedAt: string;
         pid: number | null;
         ownerPrincipal?: string | null;
+        ownerInstance?: string | null;
         transport?: JobTransport;
         payloadJson?: string | null;
     }): void;
+    markRunning(id: string, opts: {
+        pid: number | null;
+    }): boolean;
+    registerInstance(meta: GatewayInstanceMeta): void;
+    heartbeat(instanceId: string): void;
+    deregisterInstance(instanceId: string): void;
+    selectStaleProcessCandidates(leaseTtlMs: number, httpJobGraceMs: number): SweepCandidate[];
+    recoverStaleJobs(leaseTtlMs: number, httpJobGraceMs: number, liveConfirmedIds?: string[]): OrphanedJobSnapshot[];
+    gcInstances(instanceGcMs: number): number;
     recordOutput(id: string, stdout: string, stderr: string, outputTruncated: boolean): void;
     recordComplete(input: {
         id: string;
-        status: Exclude<JobStoreStatus, "running">;
+        status: Exclude<JobStoreStatus, "running" | "queued">;
         exitCode: number | null;
         stdout: string;
         stderr: string;