llm-cli-gateway 2.14.0-rc.1 → 2.14.0

package/dist/job-store.js

@@ -6,6 +6,7 @@ import { Worker } from "worker_threads";
 import { fileURLToPath } from "url";
 import { openDatabase } from "./sqlite-driver.js";
 import { noopLogger } from "./logger.js";
+import { DEFAULT_INSTANCE_LEASE_TTL_MS, DEFAULT_HTTP_JOB_GRACE_MS } from "./config.js";
 export function resolveJobStoreDbPath() {
     const configured = process.env.LLM_GATEWAY_JOBS_DB ?? process.env.LLM_GATEWAY_LOGS_DB;
     if (configured !== undefined) {
@@ -19,6 +20,7 @@ export function resolveJobStoreDbPath() {
 }
 const DEFAULT_RETENTION_DAYS = 30;
 const FAR_FUTURE_ISO = "9999-12-31T23:59:59.999Z";
+const SQLITE_NOW_MS = "CAST(ROUND((julianday('now') - 2440587.5) * 86400000.0) AS INTEGER)";
 export function resolveJobRetentionMs() {
     const raw = process.env.LLM_GATEWAY_JOB_RETENTION_DAYS;
     const days = raw ? Number(raw) : DEFAULT_RETENTION_DAYS;
@@ -63,6 +65,8 @@ function rowToRecord(row) {
         transport: row.transport ?? "process",
         httpStatus: row.http_status ?? null,
         payloadJson: row.payload_json ?? null,
+        ownerInstance: row.owner_instance ?? null,
+        leaseDeadline: row.lease_deadline == null ? null : Number(row.lease_deadline),
     };
 }
 function ensureJobsOwnerColumn(db) {
@@ -85,6 +89,16 @@ function ensureJobsTransportColumns(db) {
         db.exec("ALTER TABLE jobs ADD COLUMN payload_json TEXT");
     }
 }
+function ensureJobsLeaseColumns(db) {
+    const cols = db.prepare("PRAGMA table_info(jobs)").all();
+    const names = new Set(cols.map(col => col?.name));
+    if (!names.has("owner_instance")) {
+        db.exec("ALTER TABLE jobs ADD COLUMN owner_instance TEXT");
+    }
+    if (!names.has("lease_deadline")) {
+        db.exec("ALTER TABLE jobs ADD COLUMN lease_deadline INTEGER");
+    }
+}
 export function isValidationRunStore(store) {
     return (typeof store === "object" &&
         store !== null &&
@@ -97,6 +111,7 @@ export class SqliteJobStore {
     db;
     retentionMs;
     dedupWindowMs;
+    leaseTtlMs;
     insertStmt;
     updateOutputStmt;
     updateCompleteStmt;
@@ -105,11 +120,21 @@ export class SqliteJobStore {
     selectRunningOrphansStmt;
     markOrphanedStmt;
     deleteExpiredStmt;
+    markRunningStmt;
+    registerInstanceStmt;
+    heartbeatInstanceStmt;
+    heartbeatJobsStmt;
+    deregisterInstanceStmt;
+    selectStaleCandidatesStmt;
+    orphanExpiredStmt;
+    advanceLeaseStmt;
+    gcInstancesStmt;
     constructor(dbPath, logger = noopLogger, options = {}) {
         this.logger = logger;
         this.db = openDatabase(dbPath);
         this.db.exec("PRAGMA journal_mode = WAL");
         this.db.exec("PRAGMA synchronous = NORMAL");
+        this.db.exec("PRAGMA busy_timeout = 5000");
         this.db.exec(`
       CREATE TABLE IF NOT EXISTS jobs (
         id TEXT PRIMARY KEY,
@@ -131,12 +156,25 @@ export class SqliteJobStore {
         owner_principal TEXT,
         transport TEXT NOT NULL DEFAULT 'process',
         http_status INTEGER,
-        payload_json TEXT
+        payload_json TEXT,
+        owner_instance TEXT,
+        lease_deadline INTEGER
       );
       CREATE INDEX IF NOT EXISTS idx_jobs_request_key ON jobs(request_key);
       CREATE INDEX IF NOT EXISTS idx_jobs_status ON jobs(status);
       CREATE INDEX IF NOT EXISTS idx_jobs_expires_at ON jobs(expires_at);
       CREATE INDEX IF NOT EXISTS idx_jobs_request_key_finished ON jobs(request_key, finished_at);
+
+      CREATE TABLE IF NOT EXISTS gateway_instances (
+        instance_id TEXT PRIMARY KEY,
+        role TEXT,
+        hostname TEXT,
+        pid INTEGER,
+        started_at INTEGER NOT NULL,
+        last_heartbeat INTEGER NOT NULL
+      );
+      CREATE INDEX IF NOT EXISTS idx_gateway_instances_heartbeat
+        ON gateway_instances(last_heartbeat);
     `);
         this.db.exec(`
       CREATE TABLE IF NOT EXISTS validation_runs (
@@ -176,6 +214,8 @@ export class SqliteJobStore {
     `);
         ensureJobsOwnerColumn(this.db);
         ensureJobsTransportColumns(this.db);
+        ensureJobsLeaseColumns(this.db);
+        this.db.exec("CREATE INDEX IF NOT EXISTS idx_jobs_owner_status ON jobs(owner_instance, status)");
         if (process.platform !== "win32") {
             try {
                 chmodSync(dbPath, 0o600);
@@ -185,15 +225,17 @@ export class SqliteJobStore {
         }
         this.retentionMs = options.retentionMs ?? resolveJobRetentionMs();
         this.dedupWindowMs = options.dedupWindowMs ?? resolveDedupWindowMs();
+        this.leaseTtlMs = options.leaseTtlMs ?? DEFAULT_INSTANCE_LEASE_TTL_MS;
         this.insertStmt = this.db.prepare(`
       INSERT INTO jobs (id, correlation_id, request_key, cli, args_json, output_format,
                         status, exit_code, stdout, stderr, output_truncated, error,
                         started_at, finished_at, pid, expires_at, owner_principal,
-                        transport, http_status, payload_json)
+                        transport, http_status, payload_json, owner_instance, lease_deadline)
       VALUES (@id, @correlation_id, @request_key, @cli, @args_json, @output_format,
-              @status, @exit_code, @stdout, @stderr, @output_truncated, @error,
+              'queued', @exit_code, @stdout, @stderr, @output_truncated, @error,
               @started_at, @finished_at, @pid, @expires_at, @owner_principal,
-              @transport, @http_status, @payload_json)
+              @transport, @http_status, @payload_json, @owner_instance,
+              ${SQLITE_NOW_MS} + @lease_ttl_ms)
     `);
         this.updateOutputStmt = this.db.prepare(`
       UPDATE jobs SET stdout = @stdout, stderr = @stderr, output_truncated = @output_truncated
@@ -203,15 +245,18 @@ export class SqliteJobStore {
       UPDATE jobs SET status = @status, exit_code = @exit_code, stdout = @stdout, stderr = @stderr,
                       output_truncated = @output_truncated, error = @error,
                       finished_at = @finished_at, expires_at = @expires_at,
-                      http_status = @http_status
-      WHERE id = @id
+                      http_status = @http_status, lease_deadline = NULL
+      WHERE id = @id AND status IN ('queued', 'running', 'orphaned')
     `);
         this.getByIdStmt = this.db.prepare(`SELECT * FROM jobs WHERE id = ?`);
         this.findByRequestKeyStmt = this.db.prepare(`
       SELECT * FROM jobs
       WHERE request_key = ?
         AND started_at >= ?
-        AND status IN ('running', 'completed')
+        AND (
+          status IN ('running', 'completed')
+          OR (status = 'queued' AND lease_deadline IS NOT NULL AND lease_deadline >= ${SQLITE_NOW_MS})
+        )
       ORDER BY started_at DESC
       LIMIT 1
     `);
@@ -228,6 +273,56 @@ export class SqliteJobStore {
       WHERE status = 'running'
     `);
         this.deleteExpiredStmt = this.db.prepare(`DELETE FROM jobs WHERE expires_at < ?`);
+        this.markRunningStmt = this.db.prepare(`
+      UPDATE jobs
+      SET status = 'running', pid = @pid, lease_deadline = ${SQLITE_NOW_MS} + @lease_ttl_ms
+      WHERE id = @id AND status = 'queued'
+    `);
+        this.registerInstanceStmt = this.db.prepare(`
+      INSERT INTO gateway_instances (instance_id, role, hostname, pid, started_at, last_heartbeat)
+      VALUES (@instance_id, @role, @hostname, @pid, ${SQLITE_NOW_MS}, ${SQLITE_NOW_MS})
+      ON CONFLICT(instance_id) DO UPDATE SET
+        role = excluded.role, hostname = excluded.hostname, pid = excluded.pid,
+        last_heartbeat = excluded.last_heartbeat
+    `);
+        this.heartbeatInstanceStmt = this.db.prepare(`
+      UPDATE gateway_instances SET last_heartbeat = ${SQLITE_NOW_MS} WHERE instance_id = @instance_id
+    `);
+        this.heartbeatJobsStmt = this.db.prepare(`
+      UPDATE jobs SET lease_deadline = ${SQLITE_NOW_MS} + @lease_ttl_ms
+      WHERE owner_instance = @instance_id AND status IN ('queued', 'running')
+    `);
+        this.deregisterInstanceStmt = this.db.prepare(`DELETE FROM gateway_instances WHERE instance_id = @instance_id`);
+        this.selectStaleCandidatesStmt = this.db.prepare(`
+      SELECT j.id AS id, j.pid AS pid, j.transport AS transport,
+             j.owner_instance AS owner_instance, gi.hostname AS hostname
+      FROM jobs j
+      LEFT JOIN gateway_instances gi ON gi.instance_id = j.owner_instance
+      WHERE j.status IN ('queued', 'running')
+        AND j.transport = 'process'
+        AND j.pid IS NOT NULL
+        AND (j.lease_deadline IS NULL OR j.lease_deadline < ${SQLITE_NOW_MS})
+    `);
+        this.orphanExpiredStmt = this.db.prepare(`
+      UPDATE jobs
+      SET status = 'orphaned',
+          error = COALESCE(error, 'owning gateway instance is no longer alive'),
+          finished_at = COALESCE(finished_at, @now_iso),
+          expires_at = @expires_iso,
+          lease_deadline = NULL
+      WHERE status IN ('queued', 'running')
+        AND (lease_deadline IS NULL OR lease_deadline < ${SQLITE_NOW_MS})
+        AND (transport <> 'http'
+             OR started_at < strftime('%Y-%m-%dT%H:%M:%fZ', 'now', @http_grace_modifier))
+        AND id NOT IN (SELECT value FROM json_each(@exclude_json))
+      RETURNING id, correlation_id, started_at, stdout, stderr, exit_code, transport, http_status
+    `);
+        this.advanceLeaseStmt = this.db.prepare(`
+      UPDATE jobs SET lease_deadline = ${SQLITE_NOW_MS} + @lease_ttl_ms, pid = NULL
+      WHERE status IN ('queued', 'running')
+        AND id IN (SELECT value FROM json_each(@ids_json))
+    `);
+        this.gcInstancesStmt = this.db.prepare(`DELETE FROM gateway_instances WHERE last_heartbeat < ${SQLITE_NOW_MS} - @gc_ms`);
     }
     recordStart(input) {
         this.insertStmt.run({
@@ -237,7 +332,6 @@ export class SqliteJobStore {
             cli: input.cli,
             args_json: JSON.stringify(input.args),
             output_format: input.outputFormat ?? null,
-            status: "running",
             exit_code: null,
             stdout: "",
             stderr: "",
@@ -251,7 +345,74 @@ export class SqliteJobStore {
             transport: input.transport ?? "process",
             http_status: null,
             payload_json: input.payloadJson ?? null,
+            owner_instance: input.ownerInstance ?? null,
+            lease_ttl_ms: this.leaseTtlMs,
+        });
+    }
+    markRunning(id, opts) {
+        const result = this.markRunningStmt.run({
+            id,
+            pid: opts.pid,
+            lease_ttl_ms: this.leaseTtlMs,
+        });
+        return Number(result.changes) > 0;
+    }
+    registerInstance(meta) {
+        this.registerInstanceStmt.run({
+            instance_id: meta.instanceId,
+            role: meta.role ?? null,
+            hostname: meta.hostname ?? null,
+            pid: meta.pid ?? null,
+        });
+    }
+    heartbeat(instanceId) {
+        this.heartbeatInstanceStmt.run({ instance_id: instanceId });
+        this.heartbeatJobsStmt.run({ instance_id: instanceId, lease_ttl_ms: this.leaseTtlMs });
+    }
+    deregisterInstance(instanceId) {
+        this.deregisterInstanceStmt.run({ instance_id: instanceId });
+    }
+    selectStaleProcessCandidates(_leaseTtlMs, _httpJobGraceMs) {
+        const rows = this.selectStaleCandidatesStmt.all();
+        return rows.map(r => ({
+            id: r.id,
+            pid: r.pid,
+            transport: r.transport ?? "process",
+            ownerInstance: r.owner_instance ?? null,
+            hostname: r.hostname ?? null,
+        }));
+    }
+    recoverStaleJobs(leaseTtlMs, httpJobGraceMs, liveConfirmedIds = []) {
+        const excludeJson = JSON.stringify(liveConfirmedIds);
+        const httpGraceModifier = `-${httpJobGraceMs / 1000} seconds`;
+        const run = this.db.withTransaction(() => {
+            if (liveConfirmedIds.length > 0) {
+                this.advanceLeaseStmt.run({ ids_json: excludeJson, lease_ttl_ms: leaseTtlMs });
+            }
+            const nowIso = new Date().toISOString();
+            const expiresAt = new Date(Date.now() + this.retentionMs).toISOString();
+            const rows = this.orphanExpiredStmt.all({
+                now_iso: nowIso,
+                expires_iso: expiresAt,
+                http_grace_modifier: httpGraceModifier,
+                exclude_json: excludeJson,
+            });
+            return rows.map(r => ({
+                id: r.id,
+                correlationId: r.correlation_id,
+                startedAt: r.started_at,
+                stdout: r.stdout ?? "",
+                stderr: r.stderr ?? "",
+                exitCode: r.exit_code,
+                transport: r.transport ?? "process",
+                httpStatus: r.http_status ?? null,
+            }));
         });
+        return run();
+    }
+    gcInstances(instanceGcMs) {
+        const result = this.gcInstancesStmt.run({ gc_ms: instanceGcMs });
+        return Number(result.changes);
     }
     recordOutput(id, stdout, stderr, outputTruncated) {
         this.updateOutputStmt.run({
@@ -286,21 +447,8 @@ export class SqliteJobStore {
         return row ? rowToRecord(row) : null;
     }
     markOrphanedOnStartup() {
-        const now = new Date().toISOString();
-        const expiresAt = new Date(Date.now() + this.retentionMs).toISOString();
-        const rows = this.selectRunningOrphansStmt.all();
-        const orphaned = rows.map(row => ({
-            id: row.id,
-            correlationId: row.correlation_id,
-            startedAt: row.started_at,
-            stdout: row.stdout ?? "",
-            stderr: row.stderr ?? "",
-            exitCode: row.exit_code,
-            transport: row.transport ?? "process",
-            httpStatus: row.http_status ?? null,
-        }));
-        const result = this.markOrphanedStmt.run(now, expiresAt);
-        return { count: Number(result.changes), orphaned };
+        const orphaned = this.recoverStaleJobs(this.leaseTtlMs, DEFAULT_HTTP_JOB_GRACE_MS);
+        return { count: orphaned.length, orphaned };
     }
     evictExpired() {
         const now = new Date().toISOString();
@@ -462,9 +610,11 @@ export class MemoryJobStore {
     rows = new Map();
     retentionMs;
     dedupWindowMs;
+    leaseTtlMs;
     constructor(options = {}) {
         this.retentionMs = options.retentionMs ?? resolveJobRetentionMs();
         this.dedupWindowMs = options.dedupWindowMs ?? resolveDedupWindowMs();
+        this.leaseTtlMs = options.leaseTtlMs ?? DEFAULT_INSTANCE_LEASE_TTL_MS;
     }
     recordStart(input) {
         this.rows.set(input.id, {
@@ -474,7 +624,7 @@ export class MemoryJobStore {
             cli: input.cli,
             argsJson: JSON.stringify(input.args),
             outputFormat: input.outputFormat ?? null,
-            status: "running",
+            status: "queued",
             exitCode: null,
             stdout: "",
             stderr: "",
@@ -488,8 +638,39 @@ export class MemoryJobStore {
             transport: input.transport ?? "process",
             httpStatus: null,
             payloadJson: input.payloadJson ?? null,
+            ownerInstance: input.ownerInstance ?? null,
+            leaseDeadline: Date.now() + this.leaseTtlMs,
         });
     }
+    markRunning(id, opts) {
+        const row = this.rows.get(id);
+        if (!row || row.status !== "queued")
+            return false;
+        row.status = "running";
+        row.pid = opts.pid;
+        row.leaseDeadline = Date.now() + this.leaseTtlMs;
+        return true;
+    }
+    registerInstance(_meta) { }
+    heartbeat(instanceId) {
+        const deadline = Date.now() + this.leaseTtlMs;
+        for (const row of this.rows.values()) {
+            if (row.ownerInstance === instanceId &&
+                (row.status === "queued" || row.status === "running")) {
+                row.leaseDeadline = deadline;
+            }
+        }
+    }
+    deregisterInstance(_instanceId) { }
+    selectStaleProcessCandidates(_leaseTtlMs, _httpJobGraceMs) {
+        return [];
+    }
+    recoverStaleJobs(_leaseTtlMs, _httpJobGraceMs, _liveConfirmedIds) {
+        return [];
+    }
+    gcInstances(_instanceGcMs) {
+        return 0;
+    }
     recordOutput(id, stdout, stderr, outputTruncated) {
         const row = this.rows.get(id);
         if (!row)
@@ -502,6 +683,8 @@ export class MemoryJobStore {
         const row = this.rows.get(input.id);
         if (!row)
             return;
+        if (row.status !== "queued" && row.status !== "running" && row.status !== "orphaned")
+            return;
         row.status = input.status;
         row.exitCode = input.exitCode;
         row.stdout = input.stdout;
@@ -510,6 +693,7 @@ export class MemoryJobStore {
         row.error = input.error;
         row.finishedAt = input.finishedAt;
         row.expiresAt = new Date(Date.parse(input.finishedAt) + this.retentionMs).toISOString();
+        row.leaseDeadline = null;
         if (input.httpStatus !== undefined)
             row.httpStatus = input.httpStatus;
     }
@@ -519,11 +703,15 @@ export class MemoryJobStore {
     }
     findByRequestKey(requestKey) {
         const cutoffMs = Date.now() - this.dedupWindowMs;
+        const nowMs = Date.now();
         let best = null;
         for (const row of this.rows.values()) {
             if (row.requestKey !== requestKey)
                 continue;
-            if (row.status !== "running" && row.status !== "completed")
+            const reusable = row.status === "running" ||
+                row.status === "completed" ||
+                (row.status === "queued" && row.leaseDeadline !== null && row.leaseDeadline >= nowMs);
+            if (!reusable)
                 continue;
             if (Date.parse(row.startedAt) < cutoffMs)
                 continue;
@@ -569,6 +757,7 @@ export class PostgresJobStore {
                 dsn,
                 retentionMs: options.retentionMs ?? resolveJobRetentionMs(),
                 dedupWindowMs: options.dedupWindowMs ?? resolveDedupWindowMs(),
+                leaseTtlMs: options.leaseTtlMs ?? DEFAULT_INSTANCE_LEASE_TTL_MS,
                 farFutureIso: FAR_FUTURE_ISO,
                 connectionTimeoutMillis: 5000,
             },
@@ -616,6 +805,37 @@ export class PostgresJobStore {
     recordStart(input) {
         this.syncCall("recordStart", input);
     }
+    markRunning(id, opts) {
+        return this.syncCall("markRunning", id, opts);
+    }
+    registerInstance(meta) {
+        this.syncCall("registerInstance", meta);
+    }
+    heartbeat(instanceId) {
+        this.syncCall("heartbeat", instanceId);
+    }
+    deregisterInstance(instanceId) {
+        this.syncCall("deregisterInstance", instanceId);
+    }
+    selectStaleProcessCandidates(leaseTtlMs, httpJobGraceMs) {
+        return this.syncCall("selectStaleProcessCandidates", leaseTtlMs, httpJobGraceMs);
+    }
+    recoverStaleJobs(leaseTtlMs, httpJobGraceMs, liveConfirmedIds = []) {
+        const result = this.syncCall("recoverStaleJobs", leaseTtlMs, httpJobGraceMs, liveConfirmedIds);
+        return result.orphaned.map(row => ({
+            id: row.id,
+            correlationId: row.correlation_id,
+            startedAt: row.started_at,
+            stdout: row.stdout ?? "",
+            stderr: row.stderr ?? "",
+            exitCode: row.exit_code,
+            transport: row.transport ?? "process",
+            httpStatus: row.http_status ?? null,
+        }));
+    }
+    gcInstances(instanceGcMs) {
+        return this.syncCall("gcInstances", instanceGcMs);
+    }
     recordOutput(id, stdout, stderr, outputTruncated) {
         this.syncCall("recordOutput", id, stdout, stderr, outputTruncated);
     }
@@ -631,20 +851,8 @@ export class PostgresJobStore {
         return row ? rowToRecord(row) : null;
     }
     markOrphanedOnStartup() {
-        const result = this.syncCall("markOrphanedOnStartup");
-        return {
-            count: result.count,
-            orphaned: result.orphaned.map(row => ({
-                id: row.id,
-                correlationId: row.correlation_id,
-                startedAt: row.started_at,
-                stdout: row.stdout ?? "",
-                stderr: row.stderr ?? "",
-                exitCode: row.exit_code,
-                transport: row.transport ?? "process",
-                httpStatus: row.http_status ?? null,
-            })),
-        };
+        const orphaned = this.recoverStaleJobs(DEFAULT_INSTANCE_LEASE_TTL_MS, DEFAULT_HTTP_JOB_GRACE_MS);
+        return { count: orphaned.length, orphaned };
     }
     evictExpired() {
         return this.syncCall("evictExpired");
@@ -703,6 +911,7 @@ export function createJobStore(config, logger = noopLogger) {
     const opts = {
         retentionMs: config.retentionDays * 24 * 60 * 60 * 1000,
         dedupWindowMs: config.dedupWindowMs,
+        leaseTtlMs: config.instanceLeaseTtlMs,
     };
     switch (config.backend) {
         case "none":

package/dist/postgres-job-store-worker.js

@@ -1,6 +1,7 @@
 import { writeFileSync } from "node:fs";
 import { parentPort, workerData } from "node:worker_threads";
 let pool = null;
+const PG_NOW_MS = "(EXTRACT(EPOCH FROM now()) * 1000)::bigint";
 function signal(shared, status) {
     const view = new Int32Array(shared);
     Atomics.store(view, 0, status);
@@ -70,13 +71,26 @@ async function init() {
       owner_principal TEXT,
       transport TEXT NOT NULL DEFAULT 'process',
       http_status INTEGER,
-      payload_json TEXT
+      payload_json TEXT,
+      owner_instance TEXT,
+      lease_deadline BIGINT
     );
     CREATE INDEX IF NOT EXISTS idx_jobs_request_key ON jobs(request_key);
     CREATE INDEX IF NOT EXISTS idx_jobs_status ON jobs(status);
     CREATE INDEX IF NOT EXISTS idx_jobs_expires_at ON jobs(expires_at);
     CREATE INDEX IF NOT EXISTS idx_jobs_request_key_finished ON jobs(request_key, finished_at);
 
+    CREATE TABLE IF NOT EXISTS gateway_instances (
+      instance_id TEXT PRIMARY KEY,
+      role TEXT,
+      hostname TEXT,
+      pid INTEGER,
+      started_at BIGINT NOT NULL,
+      last_heartbeat BIGINT NOT NULL
+    );
+    CREATE INDEX IF NOT EXISTS idx_gateway_instances_heartbeat
+      ON gateway_instances(last_heartbeat);
+
     CREATE TABLE IF NOT EXISTS validation_runs (
       validation_id TEXT PRIMARY KEY,
       owner_principal TEXT NOT NULL,
@@ -116,6 +130,9 @@ async function init() {
     await getPool().query("ALTER TABLE jobs ADD COLUMN IF NOT EXISTS transport TEXT NOT NULL DEFAULT 'process'");
     await getPool().query("ALTER TABLE jobs ADD COLUMN IF NOT EXISTS http_status INTEGER");
     await getPool().query("ALTER TABLE jobs ADD COLUMN IF NOT EXISTS payload_json TEXT");
+    await getPool().query("ALTER TABLE jobs ADD COLUMN IF NOT EXISTS owner_instance TEXT");
+    await getPool().query("ALTER TABLE jobs ADD COLUMN IF NOT EXISTS lease_deadline BIGINT");
+    await getPool().query("CREATE INDEX IF NOT EXISTS idx_jobs_owner_status ON jobs(owner_instance, status)");
 }
 async function op(method, args) {
     switch (method) {
@@ -127,9 +144,9 @@ async function op(method, args) {
             await getPool().query(`INSERT INTO jobs (id, correlation_id, request_key, cli, args_json, output_format,
                            status, exit_code, stdout, stderr, output_truncated, error,
                            started_at, finished_at, pid, expires_at, owner_principal,
-                           transport, http_status, payload_json)
-         VALUES ($1, $2, $3, $4, $5, $6, 'running', NULL, '', '', FALSE, NULL,
-                 $7, NULL, $8, $9, $10, $11, NULL, $12)`, [
+                           transport, http_status, payload_json, owner_instance, lease_deadline)
+         VALUES ($1, $2, $3, $4, $5, $6, 'queued', NULL, '', '', FALSE, NULL,
+                 $7, NULL, $8, $9, $10, $11, NULL, $12, $13, ${PG_NOW_MS} + $14)`, [
                 input.id,
                 input.correlationId,
                 input.requestKey,
@@ -142,9 +159,106 @@ async function op(method, args) {
                 input.ownerPrincipal ?? null,
                 input.transport ?? "process",
                 input.payloadJson ?? null,
+                input.ownerInstance ?? null,
+                workerData.leaseTtlMs,
             ]);
             return null;
         }
+        case "markRunning": {
+            const [id, opts] = args;
+            const result = await getPool().query(`UPDATE jobs
+         SET status = 'running', pid = $2, lease_deadline = ${PG_NOW_MS} + $3::bigint
+         WHERE id = $1 AND status = 'queued'`, [id, opts?.pid ?? null, workerData.leaseTtlMs]);
+            return (result.rowCount ?? 0) > 0;
+        }
+        case "registerInstance": {
+            const meta = args[0];
+            await getPool().query(`INSERT INTO gateway_instances (instance_id, role, hostname, pid, started_at, last_heartbeat)
+         VALUES ($1, $2, $3, $4, ${PG_NOW_MS}, ${PG_NOW_MS})
+         ON CONFLICT (instance_id) DO UPDATE SET
+           role = EXCLUDED.role, hostname = EXCLUDED.hostname, pid = EXCLUDED.pid,
+           last_heartbeat = EXCLUDED.last_heartbeat`, [meta.instanceId, meta.role ?? null, meta.hostname ?? null, meta.pid ?? null]);
+            return null;
+        }
+        case "heartbeat": {
+            const instanceId = args[0];
+            await withClient(async (client) => {
+                await client.query("BEGIN");
+                try {
+                    await client.query(`UPDATE gateway_instances SET last_heartbeat = ${PG_NOW_MS} WHERE instance_id = $1`, [instanceId]);
+                    await client.query(`UPDATE jobs SET lease_deadline = ${PG_NOW_MS} + $2
+             WHERE owner_instance = $1 AND status IN ('queued', 'running')`, [instanceId, workerData.leaseTtlMs]);
+                    await client.query("COMMIT");
+                }
+                catch (error) {
+                    await client.query("ROLLBACK");
+                    throw error;
+                }
+            });
+            return null;
+        }
+        case "deregisterInstance": {
+            await getPool().query("DELETE FROM gateway_instances WHERE instance_id = $1", [args[0]]);
+            return null;
+        }
+        case "selectStaleProcessCandidates": {
+            const result = await getPool().query(`SELECT j.id AS id, j.pid AS pid, j.transport AS transport,
+                j.owner_instance AS owner_instance, gi.hostname AS hostname
+         FROM jobs j
+         LEFT JOIN gateway_instances gi ON gi.instance_id = j.owner_instance
+         WHERE j.status IN ('queued', 'running')
+           AND j.transport = 'process'
+           AND j.pid IS NOT NULL
+           AND (j.lease_deadline IS NULL OR j.lease_deadline < ${PG_NOW_MS})`);
+            return result.rows.map(r => ({
+                id: r.id,
+                pid: r.pid,
+                transport: r.transport ?? "process",
+                ownerInstance: r.owner_instance ?? null,
+                hostname: r.hostname ?? null,
+            }));
+        }
+        case "recoverStaleJobs": {
+            const [leaseTtlMs, httpJobGraceMs, liveConfirmedIds] = args;
+            const excludeIds = liveConfirmedIds ?? [];
+            const httpGraceCutoffMs = "(" + PG_NOW_MS + " - $1::bigint)";
+            return await withClient(async (client) => {
+                await client.query("BEGIN");
+                try {
+                    if (excludeIds.length > 0) {
+                        await client.query(`UPDATE jobs SET lease_deadline = ${PG_NOW_MS} + $1::bigint, pid = NULL
+               WHERE status IN ('queued', 'running') AND id = ANY($2::text[])`, [leaseTtlMs, excludeIds]);
+                    }
+                    const orphan = await client.query(`UPDATE jobs
+             SET status = 'orphaned',
+                 error = COALESCE(error, 'owning gateway instance is no longer alive'),
+                 finished_at = COALESCE(finished_at, $2),
+                 expires_at = $3,
+                 lease_deadline = NULL
+             WHERE status IN ('queued', 'running')
+               AND (lease_deadline IS NULL OR lease_deadline < ${PG_NOW_MS})
+               AND (transport <> 'http'
+                    OR (EXTRACT(EPOCH FROM started_at::timestamptz) * 1000)::bigint < ${httpGraceCutoffMs})
+               AND NOT (id = ANY($4::text[]))
+             RETURNING id, correlation_id, started_at, stdout, stderr, exit_code, transport, http_status`, [
+                        httpJobGraceMs,
+                        new Date().toISOString(),
+                        new Date(Date.now() + workerData.retentionMs).toISOString(),
+                        excludeIds,
+                    ]);
+                    await client.query("COMMIT");
+                    return { orphaned: orphan.rows };
+                }
+                catch (error) {
+                    await client.query("ROLLBACK");
+                    throw error;
+                }
+            });
+        }
+        case "gcInstances": {
+            const result = await getPool().query(`DELETE FROM gateway_instances WHERE last_heartbeat < ${PG_NOW_MS} - $1::bigint`, [args[0]]);
+            return result.rowCount ?? 0;
+        }
         case "recordOutput":
             await getPool().query("UPDATE jobs SET stdout = $2, stderr = $3, output_truncated = $4 WHERE id = $1", args);
             return null;
@@ -154,8 +268,8 @@ async function op(method, args) {
             await getPool().query(`UPDATE jobs
          SET status = $2, exit_code = $3, stdout = $4, stderr = $5,
              output_truncated = $6, error = $7, finished_at = $8,
-             expires_at = $9, http_status = $10
-         WHERE id = $1`, [
+             expires_at = $9, http_status = $10, lease_deadline = NULL
+         WHERE id = $1 AND status IN ('queued', 'running', 'orphaned')`, [
                 input.id,
                 input.status,
                 input.exitCode,
@@ -178,7 +292,10 @@ async function op(method, args) {
             const result = await getPool().query(`SELECT * FROM jobs
          WHERE request_key = $1
            AND started_at >= $2
-           AND status IN ('running', 'completed')
+           AND (
+             status IN ('running', 'completed')
+             OR (status = 'queued' AND lease_deadline IS NOT NULL AND lease_deadline >= ${PG_NOW_MS})
+           )
          ORDER BY started_at DESC
          LIMIT 1`, [args[0], cutoff]);
             return result.rows[0] ?? null;

package/dist/sqlite-driver.d.ts

@@ -9,7 +9,7 @@ export interface GatewayStatement {
 export interface GatewayDatabase {
     exec(sql: string): void;
     prepare(sql: string): GatewayStatement;
-    withTransaction<A extends unknown[]>(fn: (...args: A) => void): (...args: A) => void;
+    withTransaction<A extends unknown[], R = void>(fn: (...args: A) => R): (...args: A) => R;
     close(): void;
 }
 export declare function openDatabase(dbPath: string): GatewayDatabase;

package/dist/sqlite-driver.js

@@ -115,8 +115,9 @@ class GatewayDatabaseImpl {
             this.db.exec("BEGIN");
             this.inTransaction = true;
             try {
-                fn(...args);
+                const result = fn(...args);
                 this.db.exec("COMMIT");
+                return result;
             }
             catch (error) {
                 try {