llm-cli-gateway 2.14.0-rc.1 → 2.14.1

package/dist/job-store.d.ts

@@ -1,6 +1,7 @@
 import type { Logger } from "./logger.js";
 import type { PersistenceConfig } from "./config.js";
-export type JobStoreStatus = "running" | "completed" | "failed" | "canceled" | "orphaned";
+export type JobStoreStatus = "queued" | "running" | "completed" | "failed" | "canceled" | "orphaned";
+export type JobStoreActiveStatus = Extract<JobStoreStatus, "queued" | "running">;
 export type JobTransport = "process" | "http";
 export interface JobRecord {
     id: string;
@@ -23,11 +24,26 @@ export interface JobRecord {
     transport: JobTransport;
     httpStatus: number | null;
     payloadJson: string | null;
+    ownerInstance: string | null;
+    leaseDeadline: number | null;
 }
 export declare function resolveJobStoreDbPath(): string | null;
 export declare function resolveJobRetentionMs(): number;
 export declare function resolveDedupWindowMs(): number;
 export declare function computeRequestKey(cli: string, args: string[], extra?: string): string;
+export interface GatewayInstanceMeta {
+    instanceId: string;
+    role: string | null;
+    hostname: string | null;
+    pid: number | null;
+}
+export interface SweepCandidate {
+    id: string;
+    pid: number | null;
+    transport: JobTransport;
+    ownerInstance: string | null;
+    hostname: string | null;
+}
 export interface JobStore {
     recordStart(input: {
         id: string;
@@ -39,13 +55,23 @@ export interface JobStore {
         startedAt: string;
         pid: number | null;
         ownerPrincipal?: string | null;
+        ownerInstance?: string | null;
         transport?: JobTransport;
         payloadJson?: string | null;
     }): void;
+    markRunning(id: string, opts: {
+        pid: number | null;
+    }): boolean;
+    registerInstance(meta: GatewayInstanceMeta): void;
+    heartbeat(instanceId: string): void;
+    deregisterInstance(instanceId: string): void;
+    selectStaleProcessCandidates(leaseTtlMs: number, httpJobGraceMs: number): SweepCandidate[];
+    recoverStaleJobs(leaseTtlMs: number, httpJobGraceMs: number, liveConfirmedIds?: string[]): OrphanedJobSnapshot[];
+    gcInstances(instanceGcMs: number): number;
     recordOutput(id: string, stdout: string, stderr: string, outputTruncated: boolean): void;
     recordComplete(input: {
         id: string;
-        status: Exclude<JobStoreStatus, "running">;
+        status: Exclude<JobStoreStatus, "running" | "queued">;
         exitCode: number | null;
         stdout: string;
         stderr: string;
@@ -117,6 +143,7 @@ export declare class SqliteJobStore implements JobStore, ValidationRunStore {
     private db;
     private retentionMs;
     private dedupWindowMs;
+    private leaseTtlMs;
     private insertStmt;
     private updateOutputStmt;
     private updateCompleteStmt;
@@ -125,9 +152,19 @@ export declare class SqliteJobStore implements JobStore, ValidationRunStore {
     private selectRunningOrphansStmt;
     private markOrphanedStmt;
     private deleteExpiredStmt;
+    private markRunningStmt;
+    private registerInstanceStmt;
+    private heartbeatInstanceStmt;
+    private heartbeatJobsStmt;
+    private deregisterInstanceStmt;
+    private selectStaleCandidatesStmt;
+    private orphanExpiredStmt;
+    private advanceLeaseStmt;
+    private gcInstancesStmt;
     constructor(dbPath: string, logger?: Logger, options?: {
         retentionMs?: number;
         dedupWindowMs?: number;
+        leaseTtlMs?: number;
     });
     recordStart(input: {
         id: string;
@@ -139,13 +176,23 @@ export declare class SqliteJobStore implements JobStore, ValidationRunStore {
         startedAt: string;
         pid: number | null;
         ownerPrincipal?: string | null;
+        ownerInstance?: string | null;
         transport?: JobTransport;
         payloadJson?: string | null;
     }): void;
+    markRunning(id: string, opts: {
+        pid: number | null;
+    }): boolean;
+    registerInstance(meta: GatewayInstanceMeta): void;
+    heartbeat(instanceId: string): void;
+    deregisterInstance(instanceId: string): void;
+    selectStaleProcessCandidates(_leaseTtlMs: number, _httpJobGraceMs: number): SweepCandidate[];
+    recoverStaleJobs(leaseTtlMs: number, httpJobGraceMs: number, liveConfirmedIds?: string[]): OrphanedJobSnapshot[];
+    gcInstances(instanceGcMs: number): number;
     recordOutput(id: string, stdout: string, stderr: string, outputTruncated: boolean): void;
     recordComplete(input: {
         id: string;
-        status: Exclude<JobStoreStatus, "running">;
+        status: Exclude<JobStoreStatus, "running" | "queued">;
         exitCode: number | null;
         stdout: string;
         stderr: string;
@@ -176,9 +223,11 @@ export declare class MemoryJobStore implements JobStore {
     private rows;
     private retentionMs;
     private dedupWindowMs;
+    private leaseTtlMs;
     constructor(options?: {
         retentionMs?: number;
         dedupWindowMs?: number;
+        leaseTtlMs?: number;
     });
     recordStart(input: {
         id: string;
@@ -190,13 +239,23 @@ export declare class MemoryJobStore implements JobStore {
         startedAt: string;
         pid: number | null;
         ownerPrincipal?: string | null;
+        ownerInstance?: string | null;
         transport?: JobTransport;
         payloadJson?: string | null;
     }): void;
+    markRunning(id: string, opts: {
+        pid: number | null;
+    }): boolean;
+    registerInstance(_meta: GatewayInstanceMeta): void;
+    heartbeat(instanceId: string): void;
+    deregisterInstance(_instanceId: string): void;
+    selectStaleProcessCandidates(_leaseTtlMs: number, _httpJobGraceMs: number): SweepCandidate[];
+    recoverStaleJobs(_leaseTtlMs: number, _httpJobGraceMs: number, _liveConfirmedIds?: string[]): OrphanedJobSnapshot[];
+    gcInstances(_instanceGcMs: number): number;
     recordOutput(id: string, stdout: string, stderr: string, outputTruncated: boolean): void;
     recordComplete(input: {
         id: string;
-        status: Exclude<JobStoreStatus, "running">;
+        status: Exclude<JobStoreStatus, "running" | "queued">;
         exitCode: number | null;
         stdout: string;
         stderr: string;
@@ -223,6 +282,7 @@ export declare class PostgresJobStore implements JobStore, ValidationRunStore {
     constructor(dsn: string, logger?: Logger, options?: {
         retentionMs?: number;
         dedupWindowMs?: number;
+        leaseTtlMs?: number;
     });
     private syncCall;
     recordStart(input: {
@@ -235,13 +295,23 @@ export declare class PostgresJobStore implements JobStore, ValidationRunStore {
         startedAt: string;
         pid: number | null;
         ownerPrincipal?: string | null;
+        ownerInstance?: string | null;
         transport?: JobTransport;
         payloadJson?: string | null;
     }): void;
+    markRunning(id: string, opts: {
+        pid: number | null;
+    }): boolean;
+    registerInstance(meta: GatewayInstanceMeta): void;
+    heartbeat(instanceId: string): void;
+    deregisterInstance(instanceId: string): void;
+    selectStaleProcessCandidates(leaseTtlMs: number, httpJobGraceMs: number): SweepCandidate[];
+    recoverStaleJobs(leaseTtlMs: number, httpJobGraceMs: number, liveConfirmedIds?: string[]): OrphanedJobSnapshot[];
+    gcInstances(instanceGcMs: number): number;
     recordOutput(id: string, stdout: string, stderr: string, outputTruncated: boolean): void;
     recordComplete(input: {
         id: string;
-        status: Exclude<JobStoreStatus, "running">;
+        status: Exclude<JobStoreStatus, "running" | "queued">;
         exitCode: number | null;
         stdout: string;
         stderr: string;

package/dist/job-store.js

@@ -6,6 +6,7 @@ import { Worker } from "worker_threads";
 import { fileURLToPath } from "url";
 import { openDatabase } from "./sqlite-driver.js";
 import { noopLogger } from "./logger.js";
+import { DEFAULT_INSTANCE_LEASE_TTL_MS, DEFAULT_HTTP_JOB_GRACE_MS } from "./config.js";
 export function resolveJobStoreDbPath() {
     const configured = process.env.LLM_GATEWAY_JOBS_DB ?? process.env.LLM_GATEWAY_LOGS_DB;
     if (configured !== undefined) {
@@ -19,6 +20,7 @@ export function resolveJobStoreDbPath() {
 }
 const DEFAULT_RETENTION_DAYS = 30;
 const FAR_FUTURE_ISO = "9999-12-31T23:59:59.999Z";
+const SQLITE_NOW_MS = "CAST(ROUND((julianday('now') - 2440587.5) * 86400000.0) AS INTEGER)";
 export function resolveJobRetentionMs() {
     const raw = process.env.LLM_GATEWAY_JOB_RETENTION_DAYS;
     const days = raw ? Number(raw) : DEFAULT_RETENTION_DAYS;
@@ -63,6 +65,8 @@ function rowToRecord(row) {
         transport: row.transport ?? "process",
         httpStatus: row.http_status ?? null,
         payloadJson: row.payload_json ?? null,
+        ownerInstance: row.owner_instance ?? null,
+        leaseDeadline: row.lease_deadline == null ? null : Number(row.lease_deadline),
     };
 }
 function ensureJobsOwnerColumn(db) {
@@ -85,6 +89,16 @@ function ensureJobsTransportColumns(db) {
         db.exec("ALTER TABLE jobs ADD COLUMN payload_json TEXT");
     }
 }
+function ensureJobsLeaseColumns(db) {
+    const cols = db.prepare("PRAGMA table_info(jobs)").all();
+    const names = new Set(cols.map(col => col?.name));
+    if (!names.has("owner_instance")) {
+        db.exec("ALTER TABLE jobs ADD COLUMN owner_instance TEXT");
+    }
+    if (!names.has("lease_deadline")) {
+        db.exec("ALTER TABLE jobs ADD COLUMN lease_deadline INTEGER");
+    }
+}
 export function isValidationRunStore(store) {
     return (typeof store === "object" &&
         store !== null &&
@@ -97,6 +111,7 @@ export class SqliteJobStore {
     db;
     retentionMs;
     dedupWindowMs;
+    leaseTtlMs;
     insertStmt;
     updateOutputStmt;
     updateCompleteStmt;
@@ -105,11 +120,21 @@ export class SqliteJobStore {
     selectRunningOrphansStmt;
     markOrphanedStmt;
     deleteExpiredStmt;
+    markRunningStmt;
+    registerInstanceStmt;
+    heartbeatInstanceStmt;
+    heartbeatJobsStmt;
+    deregisterInstanceStmt;
+    selectStaleCandidatesStmt;
+    orphanExpiredStmt;
+    advanceLeaseStmt;
+    gcInstancesStmt;
     constructor(dbPath, logger = noopLogger, options = {}) {
         this.logger = logger;
         this.db = openDatabase(dbPath);
         this.db.exec("PRAGMA journal_mode = WAL");
         this.db.exec("PRAGMA synchronous = NORMAL");
+        this.db.exec("PRAGMA busy_timeout = 5000");
         this.db.exec(`
       CREATE TABLE IF NOT EXISTS jobs (
         id TEXT PRIMARY KEY,
@@ -131,12 +156,25 @@ export class SqliteJobStore {
         owner_principal TEXT,
         transport TEXT NOT NULL DEFAULT 'process',
         http_status INTEGER,
-        payload_json TEXT
+        payload_json TEXT,
+        owner_instance TEXT,
+        lease_deadline INTEGER
       );
       CREATE INDEX IF NOT EXISTS idx_jobs_request_key ON jobs(request_key);
       CREATE INDEX IF NOT EXISTS idx_jobs_status ON jobs(status);
       CREATE INDEX IF NOT EXISTS idx_jobs_expires_at ON jobs(expires_at);
       CREATE INDEX IF NOT EXISTS idx_jobs_request_key_finished ON jobs(request_key, finished_at);
+
+      CREATE TABLE IF NOT EXISTS gateway_instances (
+        instance_id TEXT PRIMARY KEY,
+        role TEXT,
+        hostname TEXT,
+        pid INTEGER,
+        started_at INTEGER NOT NULL,
+        last_heartbeat INTEGER NOT NULL
+      );
+      CREATE INDEX IF NOT EXISTS idx_gateway_instances_heartbeat
+        ON gateway_instances(last_heartbeat);
     `);
         this.db.exec(`
       CREATE TABLE IF NOT EXISTS validation_runs (
@@ -176,6 +214,8 @@ export class SqliteJobStore {
     `);
         ensureJobsOwnerColumn(this.db);
         ensureJobsTransportColumns(this.db);
+        ensureJobsLeaseColumns(this.db);
+        this.db.exec("CREATE INDEX IF NOT EXISTS idx_jobs_owner_status ON jobs(owner_instance, status)");
         if (process.platform !== "win32") {
             try {
                 chmodSync(dbPath, 0o600);
@@ -185,15 +225,17 @@ export class SqliteJobStore {
         }
         this.retentionMs = options.retentionMs ?? resolveJobRetentionMs();
         this.dedupWindowMs = options.dedupWindowMs ?? resolveDedupWindowMs();
+        this.leaseTtlMs = options.leaseTtlMs ?? DEFAULT_INSTANCE_LEASE_TTL_MS;
         this.insertStmt = this.db.prepare(`
       INSERT INTO jobs (id, correlation_id, request_key, cli, args_json, output_format,
                         status, exit_code, stdout, stderr, output_truncated, error,
                         started_at, finished_at, pid, expires_at, owner_principal,
-                        transport, http_status, payload_json)
+                        transport, http_status, payload_json, owner_instance, lease_deadline)
       VALUES (@id, @correlation_id, @request_key, @cli, @args_json, @output_format,
-              @status, @exit_code, @stdout, @stderr, @output_truncated, @error,
+              'queued', @exit_code, @stdout, @stderr, @output_truncated, @error,
               @started_at, @finished_at, @pid, @expires_at, @owner_principal,
-              @transport, @http_status, @payload_json)
+              @transport, @http_status, @payload_json, @owner_instance,
+              ${SQLITE_NOW_MS} + @lease_ttl_ms)
     `);
         this.updateOutputStmt = this.db.prepare(`
       UPDATE jobs SET stdout = @stdout, stderr = @stderr, output_truncated = @output_truncated
@@ -203,15 +245,18 @@ export class SqliteJobStore {
       UPDATE jobs SET status = @status, exit_code = @exit_code, stdout = @stdout, stderr = @stderr,
                       output_truncated = @output_truncated, error = @error,
                       finished_at = @finished_at, expires_at = @expires_at,
-                      http_status = @http_status
-      WHERE id = @id
+                      http_status = @http_status, lease_deadline = NULL
+      WHERE id = @id AND status IN ('queued', 'running', 'orphaned')
     `);
         this.getByIdStmt = this.db.prepare(`SELECT * FROM jobs WHERE id = ?`);
         this.findByRequestKeyStmt = this.db.prepare(`
       SELECT * FROM jobs
       WHERE request_key = ?
         AND started_at >= ?
-        AND status IN ('running', 'completed')
+        AND (
+          status IN ('running', 'completed')
+          OR (status = 'queued' AND lease_deadline IS NOT NULL AND lease_deadline >= ${SQLITE_NOW_MS})
+        )
       ORDER BY started_at DESC
       LIMIT 1
     `);
@@ -228,6 +273,56 @@ export class SqliteJobStore {
       WHERE status = 'running'
     `);
         this.deleteExpiredStmt = this.db.prepare(`DELETE FROM jobs WHERE expires_at < ?`);
+        this.markRunningStmt = this.db.prepare(`
+      UPDATE jobs
+      SET status = 'running', pid = @pid, lease_deadline = ${SQLITE_NOW_MS} + @lease_ttl_ms
+      WHERE id = @id AND status = 'queued'
+    `);
+        this.registerInstanceStmt = this.db.prepare(`
+      INSERT INTO gateway_instances (instance_id, role, hostname, pid, started_at, last_heartbeat)
+      VALUES (@instance_id, @role, @hostname, @pid, ${SQLITE_NOW_MS}, ${SQLITE_NOW_MS})
+      ON CONFLICT(instance_id) DO UPDATE SET
+        role = excluded.role, hostname = excluded.hostname, pid = excluded.pid,
+        last_heartbeat = excluded.last_heartbeat
+    `);
+        this.heartbeatInstanceStmt = this.db.prepare(`
+      UPDATE gateway_instances SET last_heartbeat = ${SQLITE_NOW_MS} WHERE instance_id = @instance_id
+    `);
+        this.heartbeatJobsStmt = this.db.prepare(`
+      UPDATE jobs SET lease_deadline = ${SQLITE_NOW_MS} + @lease_ttl_ms
+      WHERE owner_instance = @instance_id AND status IN ('queued', 'running')
+    `);
+        this.deregisterInstanceStmt = this.db.prepare(`DELETE FROM gateway_instances WHERE instance_id = @instance_id`);
+        this.selectStaleCandidatesStmt = this.db.prepare(`
+      SELECT j.id AS id, j.pid AS pid, j.transport AS transport,
+             j.owner_instance AS owner_instance, gi.hostname AS hostname
+      FROM jobs j
+      LEFT JOIN gateway_instances gi ON gi.instance_id = j.owner_instance
+      WHERE j.status IN ('queued', 'running')
+        AND j.transport = 'process'
+        AND j.pid IS NOT NULL
+        AND (j.lease_deadline IS NULL OR j.lease_deadline < ${SQLITE_NOW_MS})
+    `);
+        this.orphanExpiredStmt = this.db.prepare(`
+      UPDATE jobs
+      SET status = 'orphaned',
+          error = COALESCE(error, 'owning gateway instance is no longer alive'),
+          finished_at = COALESCE(finished_at, @now_iso),
+          expires_at = @expires_iso,
+          lease_deadline = NULL
+      WHERE status IN ('queued', 'running')
+        AND (lease_deadline IS NULL OR lease_deadline < ${SQLITE_NOW_MS})
+        AND (transport <> 'http'
+             OR started_at < strftime('%Y-%m-%dT%H:%M:%fZ', 'now', @http_grace_modifier))
+        AND id NOT IN (SELECT value FROM json_each(@exclude_json))
+      RETURNING id, correlation_id, started_at, stdout, stderr, exit_code, transport, http_status
+    `);
+        this.advanceLeaseStmt = this.db.prepare(`
+      UPDATE jobs SET lease_deadline = ${SQLITE_NOW_MS} + @lease_ttl_ms, pid = NULL
+      WHERE status IN ('queued', 'running')
+        AND id IN (SELECT value FROM json_each(@ids_json))
+    `);
+        this.gcInstancesStmt = this.db.prepare(`DELETE FROM gateway_instances WHERE last_heartbeat < ${SQLITE_NOW_MS} - @gc_ms`);
     }
     recordStart(input) {
         this.insertStmt.run({
@@ -237,7 +332,6 @@ export class SqliteJobStore {
             cli: input.cli,
             args_json: JSON.stringify(input.args),
             output_format: input.outputFormat ?? null,
-            status: "running",
             exit_code: null,
             stdout: "",
             stderr: "",
@@ -251,7 +345,74 @@ export class SqliteJobStore {
             transport: input.transport ?? "process",
             http_status: null,
             payload_json: input.payloadJson ?? null,
+            owner_instance: input.ownerInstance ?? null,
+            lease_ttl_ms: this.leaseTtlMs,
+        });
+    }
+    markRunning(id, opts) {
+        const result = this.markRunningStmt.run({
+            id,
+            pid: opts.pid,
+            lease_ttl_ms: this.leaseTtlMs,
+        });
+        return Number(result.changes) > 0;
+    }
+    registerInstance(meta) {
+        this.registerInstanceStmt.run({
+            instance_id: meta.instanceId,
+            role: meta.role ?? null,
+            hostname: meta.hostname ?? null,
+            pid: meta.pid ?? null,
+        });
+    }
+    heartbeat(instanceId) {
+        this.heartbeatInstanceStmt.run({ instance_id: instanceId });
+        this.heartbeatJobsStmt.run({ instance_id: instanceId, lease_ttl_ms: this.leaseTtlMs });
+    }
+    deregisterInstance(instanceId) {
+        this.deregisterInstanceStmt.run({ instance_id: instanceId });
+    }
+    selectStaleProcessCandidates(_leaseTtlMs, _httpJobGraceMs) {
+        const rows = this.selectStaleCandidatesStmt.all();
+        return rows.map(r => ({
+            id: r.id,
+            pid: r.pid,
+            transport: r.transport ?? "process",
+            ownerInstance: r.owner_instance ?? null,
+            hostname: r.hostname ?? null,
+        }));
+    }
+    recoverStaleJobs(leaseTtlMs, httpJobGraceMs, liveConfirmedIds = []) {
+        const excludeJson = JSON.stringify(liveConfirmedIds);
+        const httpGraceModifier = `-${httpJobGraceMs / 1000} seconds`;
+        const run = this.db.withTransaction(() => {
+            if (liveConfirmedIds.length > 0) {
+                this.advanceLeaseStmt.run({ ids_json: excludeJson, lease_ttl_ms: leaseTtlMs });
+            }
+            const nowIso = new Date().toISOString();
+            const expiresAt = new Date(Date.now() + this.retentionMs).toISOString();
+            const rows = this.orphanExpiredStmt.all({
+                now_iso: nowIso,
+                expires_iso: expiresAt,
+                http_grace_modifier: httpGraceModifier,
+                exclude_json: excludeJson,
+            });
+            return rows.map(r => ({
+                id: r.id,
+                correlationId: r.correlation_id,
+                startedAt: r.started_at,
+                stdout: r.stdout ?? "",
+                stderr: r.stderr ?? "",
+                exitCode: r.exit_code,
+                transport: r.transport ?? "process",
+                httpStatus: r.http_status ?? null,
+            }));
         });
+        return run();
+    }
+    gcInstances(instanceGcMs) {
+        const result = this.gcInstancesStmt.run({ gc_ms: instanceGcMs });
+        return Number(result.changes);
     }
     recordOutput(id, stdout, stderr, outputTruncated) {
         this.updateOutputStmt.run({
@@ -286,21 +447,8 @@ export class SqliteJobStore {
         return row ? rowToRecord(row) : null;
     }
     markOrphanedOnStartup() {
-        const now = new Date().toISOString();
-        const expiresAt = new Date(Date.now() + this.retentionMs).toISOString();
-        const rows = this.selectRunningOrphansStmt.all();
-        const orphaned = rows.map(row => ({
-            id: row.id,
-            correlationId: row.correlation_id,
-            startedAt: row.started_at,
-            stdout: row.stdout ?? "",
-            stderr: row.stderr ?? "",
-            exitCode: row.exit_code,
-            transport: row.transport ?? "process",
-            httpStatus: row.http_status ?? null,
-        }));
-        const result = this.markOrphanedStmt.run(now, expiresAt);
-        return { count: Number(result.changes), orphaned };
+        const orphaned = this.recoverStaleJobs(this.leaseTtlMs, DEFAULT_HTTP_JOB_GRACE_MS);
+        return { count: orphaned.length, orphaned };
     }
     evictExpired() {
         const now = new Date().toISOString();
@@ -462,9 +610,11 @@ export class MemoryJobStore {
     rows = new Map();
     retentionMs;
     dedupWindowMs;
+    leaseTtlMs;
     constructor(options = {}) {
         this.retentionMs = options.retentionMs ?? resolveJobRetentionMs();
         this.dedupWindowMs = options.dedupWindowMs ?? resolveDedupWindowMs();
+        this.leaseTtlMs = options.leaseTtlMs ?? DEFAULT_INSTANCE_LEASE_TTL_MS;
     }
     recordStart(input) {
         this.rows.set(input.id, {
@@ -474,7 +624,7 @@ export class MemoryJobStore {
             cli: input.cli,
             argsJson: JSON.stringify(input.args),
             outputFormat: input.outputFormat ?? null,
-            status: "running",
+            status: "queued",
             exitCode: null,
             stdout: "",
             stderr: "",
@@ -488,8 +638,39 @@ export class MemoryJobStore {
             transport: input.transport ?? "process",
             httpStatus: null,
             payloadJson: input.payloadJson ?? null,
+            ownerInstance: input.ownerInstance ?? null,
+            leaseDeadline: Date.now() + this.leaseTtlMs,
         });
     }
+    markRunning(id, opts) {
+        const row = this.rows.get(id);
+        if (!row || row.status !== "queued")
+            return false;
+        row.status = "running";
+        row.pid = opts.pid;
+        row.leaseDeadline = Date.now() + this.leaseTtlMs;
+        return true;
+    }
+    registerInstance(_meta) { }
+    heartbeat(instanceId) {
+        const deadline = Date.now() + this.leaseTtlMs;
+        for (const row of this.rows.values()) {
+            if (row.ownerInstance === instanceId &&
+                (row.status === "queued" || row.status === "running")) {
+                row.leaseDeadline = deadline;
+            }
+        }
+    }
+    deregisterInstance(_instanceId) { }
+    selectStaleProcessCandidates(_leaseTtlMs, _httpJobGraceMs) {
+        return [];
+    }
+    recoverStaleJobs(_leaseTtlMs, _httpJobGraceMs, _liveConfirmedIds) {
+        return [];
+    }
+    gcInstances(_instanceGcMs) {
+        return 0;
+    }
     recordOutput(id, stdout, stderr, outputTruncated) {
         const row = this.rows.get(id);
         if (!row)
@@ -502,6 +683,8 @@ export class MemoryJobStore {
         const row = this.rows.get(input.id);
         if (!row)
             return;
+        if (row.status !== "queued" && row.status !== "running" && row.status !== "orphaned")
+            return;
         row.status = input.status;
         row.exitCode = input.exitCode;
         row.stdout = input.stdout;
@@ -510,6 +693,7 @@ export class MemoryJobStore {
         row.error = input.error;
         row.finishedAt = input.finishedAt;
         row.expiresAt = new Date(Date.parse(input.finishedAt) + this.retentionMs).toISOString();
+        row.leaseDeadline = null;
         if (input.httpStatus !== undefined)
             row.httpStatus = input.httpStatus;
     }
@@ -519,11 +703,15 @@ export class MemoryJobStore {
     }
     findByRequestKey(requestKey) {
         const cutoffMs = Date.now() - this.dedupWindowMs;
+        const nowMs = Date.now();
         let best = null;
         for (const row of this.rows.values()) {
             if (row.requestKey !== requestKey)
                 continue;
-            if (row.status !== "running" && row.status !== "completed")
+            const reusable = row.status === "running" ||
+                row.status === "completed" ||
+                (row.status === "queued" && row.leaseDeadline !== null && row.leaseDeadline >= nowMs);
+            if (!reusable)
                 continue;
             if (Date.parse(row.startedAt) < cutoffMs)
                 continue;
@@ -569,6 +757,7 @@ export class PostgresJobStore {
                 dsn,
                 retentionMs: options.retentionMs ?? resolveJobRetentionMs(),
                 dedupWindowMs: options.dedupWindowMs ?? resolveDedupWindowMs(),
+                leaseTtlMs: options.leaseTtlMs ?? DEFAULT_INSTANCE_LEASE_TTL_MS,
                 farFutureIso: FAR_FUTURE_ISO,
                 connectionTimeoutMillis: 5000,
             },
@@ -616,6 +805,37 @@ export class PostgresJobStore {
     recordStart(input) {
         this.syncCall("recordStart", input);
     }
+    markRunning(id, opts) {
+        return this.syncCall("markRunning", id, opts);
+    }
+    registerInstance(meta) {
+        this.syncCall("registerInstance", meta);
+    }
+    heartbeat(instanceId) {
+        this.syncCall("heartbeat", instanceId);
+    }
+    deregisterInstance(instanceId) {
+        this.syncCall("deregisterInstance", instanceId);
+    }
+    selectStaleProcessCandidates(leaseTtlMs, httpJobGraceMs) {
+        return this.syncCall("selectStaleProcessCandidates", leaseTtlMs, httpJobGraceMs);
+    }
+    recoverStaleJobs(leaseTtlMs, httpJobGraceMs, liveConfirmedIds = []) {
+        const result = this.syncCall("recoverStaleJobs", leaseTtlMs, httpJobGraceMs, liveConfirmedIds);
+        return result.orphaned.map(row => ({
+            id: row.id,
+            correlationId: row.correlation_id,
+            startedAt: row.started_at,
+            stdout: row.stdout ?? "",
+            stderr: row.stderr ?? "",
+            exitCode: row.exit_code,
+            transport: row.transport ?? "process",
+            httpStatus: row.http_status ?? null,
+        }));
+    }
+    gcInstances(instanceGcMs) {
+        return this.syncCall("gcInstances", instanceGcMs);
+    }
     recordOutput(id, stdout, stderr, outputTruncated) {
         this.syncCall("recordOutput", id, stdout, stderr, outputTruncated);
     }
@@ -631,20 +851,8 @@ export class PostgresJobStore {
         return row ? rowToRecord(row) : null;
     }
     markOrphanedOnStartup() {
-        const result = this.syncCall("markOrphanedOnStartup");
-        return {
-            count: result.count,
-            orphaned: result.orphaned.map(row => ({
-                id: row.id,
-                correlationId: row.correlation_id,
-                startedAt: row.started_at,
-                stdout: row.stdout ?? "",
-                stderr: row.stderr ?? "",
-                exitCode: row.exit_code,
-                transport: row.transport ?? "process",
-                httpStatus: row.http_status ?? null,
-            })),
-        };
+        const orphaned = this.recoverStaleJobs(DEFAULT_INSTANCE_LEASE_TTL_MS, DEFAULT_HTTP_JOB_GRACE_MS);
+        return { count: orphaned.length, orphaned };
     }
     evictExpired() {
         return this.syncCall("evictExpired");
@@ -703,6 +911,7 @@ export function createJobStore(config, logger = noopLogger) {
     const opts = {
         retentionMs: config.retentionDays * 24 * 60 * 60 * 1000,
         dedupWindowMs: config.dedupWindowMs,
+        leaseTtlMs: config.instanceLeaseTtlMs,
     };
     switch (config.backend) {
         case "none":