@pasko70/pibo 3.0.1 → 3.1.0

package/dist/reliability/store.js

@@ -5,6 +5,8 @@ import { ensurePrivatePiboHomeForPath, piboHomePath } from "../core/pibo-home.js
 import { protectPrivateFileSync } from "../core/private-path.js";
 import { sqliteTableColumns } from "../data/sqlite-schema.js";
 import { DatabaseSync } from "node:sqlite";
+const MAX_ATTEMPTS_ERROR = "Job exhausted retry attempts.";
+const MAX_ATTEMPTS_REASON = "max_attempts";
 function now() {
     return new Date().toISOString();
 }
@@ -36,6 +38,11 @@ function ensurePiboRunColumn(db, name, definition) {
     if (!columns.has(name))
         db.exec(`ALTER TABLE pibo_runs ADD COLUMN ${name} ${definition}`);
 }
+function ensurePiboJobColumn(db, name, definition) {
+    const columns = sqliteTableColumns(db, "pibo_jobs");
+    if (!columns.has(name))
+        db.exec(`ALTER TABLE pibo_jobs ADD COLUMN ${name} ${definition}`);
+}
 export class PiboReliabilityStore {
     db;
     appendEventStatement;
@@ -74,6 +81,13 @@ export class PiboReliabilityStore {
 			CREATE INDEX IF NOT EXISTS idx_pibo_event_stream_topic_stream
 				ON pibo_event_stream(topic, stream_id);
 
+			CREATE TABLE IF NOT EXISTS pibo_delivery_receipts (
+				delivery_id TEXT NOT NULL,
+				projection TEXT NOT NULL,
+				delivered_at TEXT NOT NULL,
+				PRIMARY KEY(delivery_id, projection)
+			);
+
 			CREATE TABLE IF NOT EXISTS pibo_event_consumers (
 				consumer TEXT NOT NULL,
 				topic TEXT NOT NULL,
@@ -91,6 +105,7 @@ export class PiboReliabilityStore {
 				priority INTEGER NOT NULL DEFAULT 0,
 				worker_id TEXT,
 				claim_expires_at TEXT,
+				claim_token INTEGER NOT NULL DEFAULT 0,
 				attempts INTEGER NOT NULL DEFAULT 0,
 				max_attempts INTEGER NOT NULL DEFAULT 1,
 				idempotency_key TEXT,
@@ -157,6 +172,7 @@ export class PiboReliabilityStore {
 			CREATE INDEX IF NOT EXISTS idx_pibo_runs_status
 				ON pibo_runs(status);
 		`);
+        ensurePiboJobColumn(this.db, "claim_token", "INTEGER NOT NULL DEFAULT 0");
         ensurePiboRunColumn(this.db, "timeout_ms", "INTEGER");
         ensurePiboRunColumn(this.db, "timeout_at", "TEXT");
         ensurePiboRunColumn(this.db, "timeout_phase", "TEXT");
@@ -189,6 +205,21 @@ export class PiboReliabilityStore {
             throw error;
         }
     }
+    hasDeliveryReceipt(deliveryId, projection) {
+        return this.db
+            .prepare("SELECT 1 AS found FROM pibo_delivery_receipts WHERE delivery_id = ? AND projection = ?")
+            .get(deliveryId, projection) !== undefined;
+    }
+    recordDeliveryReceipt(deliveryId, projection, deliveredAt = now()) {
+        this.db.prepare(`
+			INSERT OR IGNORE INTO pibo_delivery_receipts (delivery_id, projection, delivered_at)
+			VALUES (?, ?, ?)
+		`).run(deliveryId, projection, deliveredAt);
+        const row = this.db
+            .prepare("SELECT delivery_id, projection, delivered_at FROM pibo_delivery_receipts WHERE delivery_id = ? AND projection = ?")
+            .get(deliveryId, projection);
+        return { deliveryId: row.delivery_id, projection: row.projection, deliveredAt: row.delivered_at };
+    }
     list(input = {}) {
         const clauses = [];
         const values = [];
@@ -323,56 +354,123 @@ export class PiboReliabilityStore {
     claimBatch(workerId, limit, options = {}) {
         const timestamp = now();
         const claimExpiresAt = new Date(Date.now() + (options.visibilityTimeoutMs ?? 30000)).toISOString();
-        const clauses = [
-            "((state = 'pending' AND run_at <= ?) OR (state = 'running' AND claim_expires_at IS NOT NULL AND claim_expires_at <= ?))",
-            "(expires_at IS NULL OR expires_at > ?)",
-        ];
-        const values = [timestamp, timestamp, timestamp];
-        if (options.queue) {
-            clauses.push("queue = ?");
-            values.push(options.queue);
-        }
-        this.moveExpiredJobs(timestamp);
-        const rows = this.db
-            .prepare(`
-				UPDATE pibo_jobs
-				SET state = 'running',
-					worker_id = ?,
-					claim_expires_at = ?,
-					attempts = attempts + 1,
-					updated_at = ?
-				WHERE job_id IN (
-					SELECT job_id FROM pibo_jobs
-					WHERE ${clauses.join(" AND ")}
-					ORDER BY priority DESC, run_at ASC, created_at ASC
-					LIMIT ?
-				)
-				RETURNING *
-			`)
-            .all(workerId, claimExpiresAt, timestamp, ...values, clampLimit(limit));
-        return rows.map(jobFromRow);
+        return this.inImmediateTransaction(() => {
+            const clauses = [
+                "((state = 'pending' AND run_at <= ?) OR (state = 'running' AND claim_expires_at IS NOT NULL AND claim_expires_at <= ?))",
+                "(expires_at IS NULL OR expires_at > ?)",
+                "attempts < max_attempts",
+            ];
+            const values = [timestamp, timestamp, timestamp];
+            if (options.queue) {
+                clauses.push("queue = ?");
+                values.push(options.queue);
+            }
+            this.moveExpiredJobs(timestamp);
+            this.moveExhaustedRecoverableJobs(timestamp, { queue: options.queue });
+            const rows = this.db
+                .prepare(`
+					UPDATE pibo_jobs
+					SET state = 'running',
+						worker_id = ?,
+						claim_expires_at = ?,
+						claim_token = claim_token + 1,
+						attempts = attempts + 1,
+						updated_at = ?
+					WHERE job_id IN (
+						SELECT job_id FROM pibo_jobs
+						WHERE ${clauses.join(" AND ")}
+						ORDER BY priority DESC, run_at ASC, created_at ASC
+						LIMIT ?
+					)
+					RETURNING *
+				`)
+                .all(workerId, claimExpiresAt, timestamp, ...values, clampLimit(limit));
+            return rows.map(jobFromRow);
+        });
     }
     claimJob(jobId, workerId, visibilityTimeoutMs = 30000) {
         const timestamp = now();
         const claimExpiresAt = new Date(Date.now() + visibilityTimeoutMs).toISOString();
-        const row = this.db
-            .prepare(`
+        return this.inImmediateTransaction(() => {
+            this.moveExpiredJobs(timestamp);
+            this.moveExhaustedRecoverableJobs(timestamp, { jobId });
+            const row = this.db
+                .prepare(`
+					UPDATE pibo_jobs
+					SET state = 'running',
+						worker_id = ?,
+						claim_expires_at = ?,
+						claim_token = claim_token + 1,
+						attempts = attempts + 1,
+						updated_at = ?
+					WHERE job_id = ?
+						AND state = 'pending'
+						AND run_at <= ?
+						AND attempts < max_attempts
+						AND (expires_at IS NULL OR expires_at > ?)
+					RETURNING *
+				`)
+                .get(workerId, claimExpiresAt, timestamp, jobId, timestamp, timestamp);
+            return row ? jobFromRow(row) : undefined;
+        });
+    }
+    claimRecoverableJob(jobId, workerId, visibilityTimeoutMs = 30000, forcePending = false) {
+        const timestamp = now();
+        const claimExpiresAt = new Date(Date.now() + visibilityTimeoutMs).toISOString();
+        return this.inImmediateTransaction(() => {
+            this.moveExpiredJobs(timestamp);
+            this.moveExhaustedRecoverableJobs(timestamp, { jobId });
+            const row = this.db.prepare(`
 				UPDATE pibo_jobs
 				SET state = 'running',
 					worker_id = ?,
 					claim_expires_at = ?,
+					claim_token = claim_token + 1,
 					attempts = attempts + 1,
 					updated_at = ?
 				WHERE job_id = ?
-					AND state = 'pending'
-					AND run_at <= ?
+					AND (
+						(state = 'pending' AND (? = 1 OR run_at <= ?))
+						OR (state = 'running' AND claim_expires_at IS NOT NULL AND claim_expires_at <= ?)
+					)
+					AND attempts < max_attempts
 					AND (expires_at IS NULL OR expires_at > ?)
 				RETURNING *
-			`)
-            .get(workerId, claimExpiresAt, timestamp, jobId, timestamp, timestamp);
-        return row ? jobFromRow(row) : undefined;
+			`).get(workerId, claimExpiresAt, timestamp, jobId, forcePending ? 1 : 0, timestamp, timestamp, timestamp);
+            return row ? jobFromRow(row) : undefined;
+        });
+    }
+    updateJobPayload(jobId, workerId, payload, claimToken) {
+        const result = this.db.prepare(`
+			UPDATE pibo_jobs
+			SET payload_json = ?, updated_at = ?
+			WHERE job_id = ? AND state = 'running' AND worker_id = ?
+				AND (? IS NULL OR claim_token = ?)
+				AND claim_expires_at IS NOT NULL AND claim_expires_at > ?
+		`).run(json(payload), now(), jobId, workerId, claimToken ?? null, claimToken ?? null, now());
+        return Number(result.changes ?? 0) > 0;
     }
-    ack(jobId, workerId) {
+    releaseJob(jobId, workerId, delayMs = 0, claimToken) {
+        const timestamp = now();
+        return this.inImmediateTransaction(() => {
+            const row = this.getLiveWorkerJob(jobId, workerId, timestamp, claimToken);
+            if (!row)
+                return false;
+            if (row.attempts >= row.max_attempts) {
+                this.moveJobToDead(row, MAX_ATTEMPTS_ERROR, MAX_ATTEMPTS_REASON, timestamp);
+                return true;
+            }
+            const runAt = new Date(Date.parse(timestamp) + Math.max(0, delayMs)).toISOString();
+            const result = this.db.prepare(`
+				UPDATE pibo_jobs
+				SET state = 'pending', worker_id = NULL, claim_expires_at = NULL, run_at = ?, updated_at = ?
+				WHERE job_id = ? AND state = 'running' AND worker_id = ?
+					AND (? IS NULL OR claim_token = ?)
+			`).run(runAt, timestamp, jobId, workerId, claimToken ?? null, claimToken ?? null);
+            return Number(result.changes ?? 0) > 0;
+        });
+    }
+    ack(jobId, workerId, claimToken) {
         const timestamp = now();
         const result = this.db
             .prepare(`
@@ -380,47 +478,53 @@ export class PiboReliabilityStore {
 				WHERE job_id = ?
 					AND state = 'running'
 					AND worker_id = ?
+					AND (? IS NULL OR claim_token = ?)
 					AND claim_expires_at IS NOT NULL
 					AND claim_expires_at > ?
 			`)
-            .run(jobId, workerId, timestamp);
+            .run(jobId, workerId, claimToken ?? null, claimToken ?? null, timestamp);
         return Number(result.changes ?? 0) > 0;
     }
     retry(jobId, workerId, input = {}) {
         const timestamp = now();
-        const row = this.getLiveWorkerJob(jobId, workerId, timestamp);
-        if (!row)
-            return false;
-        const job = jobFromRow(row);
-        if (job.attempts >= job.maxAttempts) {
-            this.moveJobToDead(row, input.error ?? "Job exhausted retry attempts.", "max_attempts", timestamp);
-            return true;
-        }
-        const delayMs = input.delayMs ?? retryDelayMs(job.attempts, input);
-        const runAt = new Date(asDate(input.now).getTime() + delayMs).toISOString();
-        this.db
-            .prepare(`
-				UPDATE pibo_jobs
-				SET state = 'pending',
-					worker_id = NULL,
-					claim_expires_at = NULL,
-					run_at = ?,
-					updated_at = ?,
-					last_error = ?
-				WHERE job_id = ?
-			`)
-            .run(runAt, timestamp, input.error ?? null, jobId);
-        return true;
+        return this.inImmediateTransaction(() => {
+            const row = this.getLiveWorkerJob(jobId, workerId, timestamp, input.claimToken);
+            if (!row)
+                return false;
+            const job = jobFromRow(row);
+            if (job.attempts >= job.maxAttempts) {
+                this.moveJobToDead(row, input.error ?? MAX_ATTEMPTS_ERROR, MAX_ATTEMPTS_REASON, timestamp);
+                return true;
+            }
+            const delayMs = input.delayMs ?? retryDelayMs(job.attempts, input);
+            const runAt = new Date(asDate(input.now).getTime() + delayMs).toISOString();
+            const result = this.db
+                .prepare(`
+					UPDATE pibo_jobs
+					SET state = 'pending',
+						worker_id = NULL,
+						claim_expires_at = NULL,
+						run_at = ?,
+						updated_at = ?,
+						last_error = ?
+					WHERE job_id = ? AND state = 'running' AND worker_id = ?
+						AND (? IS NULL OR claim_token = ?)
+				`)
+                .run(runAt, timestamp, input.error ?? null, jobId, workerId, input.claimToken ?? null, input.claimToken ?? null);
+            return Number(result.changes ?? 0) > 0;
+        });
     }
-    fail(jobId, workerId, error) {
+    fail(jobId, workerId, error, claimToken) {
         const timestamp = now();
-        const row = this.getLiveWorkerJob(jobId, workerId, timestamp);
-        if (!row)
-            return false;
-        this.moveJobToDead(row, error, "failed", timestamp);
-        return true;
+        return this.inImmediateTransaction(() => {
+            const row = this.getLiveWorkerJob(jobId, workerId, timestamp, claimToken);
+            if (!row)
+                return false;
+            this.moveJobToDead(row, error, "failed", timestamp);
+            return true;
+        });
     }
-    heartbeat(jobId, workerId, extendMs) {
+    heartbeat(jobId, workerId, extendMs, claimToken) {
         const timestamp = now();
         const claimExpiresAt = new Date(Date.now() + Math.max(1, extendMs)).toISOString();
         const result = this.db
@@ -430,10 +534,11 @@ export class PiboReliabilityStore {
 				WHERE job_id = ?
 					AND state = 'running'
 					AND worker_id = ?
+					AND (? IS NULL OR claim_token = ?)
 					AND claim_expires_at IS NOT NULL
 					AND claim_expires_at > ?
 			`)
-            .run(claimExpiresAt, timestamp, jobId, workerId, timestamp);
+            .run(claimExpiresAt, timestamp, jobId, workerId, claimToken ?? null, claimToken ?? null, timestamp);
         return Number(result.changes ?? 0) > 0;
     }
     listJobs(input = {}) {
@@ -447,11 +552,56 @@ export class PiboReliabilityStore {
             clauses.push("state = ?");
             values.push(input.state);
         }
+        if (input.excludeJobIds?.length) {
+            clauses.push(`job_id NOT IN (${input.excludeJobIds.map(() => "?").join(", ")})`);
+            values.push(...input.excludeJobIds);
+        }
         const where = clauses.length ? `WHERE ${clauses.join(" AND ")}` : "";
         const rows = this.db
             .prepare(`SELECT * FROM pibo_jobs ${where} ORDER BY priority DESC, run_at ASC, created_at ASC LIMIT ?`)
             .all(...values, clampLimit(input.limit, 100));
-        return rows.map(jobFromRow);
+        const jobs = [];
+        for (const row of rows) {
+            try {
+                jobs.push(jobFromRow(row));
+            }
+            catch {
+                this.quarantineJobRow(row, "payload_malformed");
+            }
+        }
+        return jobs;
+    }
+    listRecoverableJobs(input = {}) {
+        const timestamp = now();
+        const clauses = [
+            "((state = 'pending' AND (? = 1 OR run_at <= ?)) OR (state = 'running' AND claim_expires_at IS NOT NULL AND claim_expires_at <= ?))",
+            "(expires_at IS NULL OR expires_at > ?)",
+        ];
+        const values = [input.includeDeferredPending ? 1 : 0, timestamp, timestamp, timestamp];
+        if (input.queue) {
+            clauses.push("queue = ?");
+            values.push(input.queue);
+        }
+        if (input.excludeJobIds?.length) {
+            clauses.push(`job_id NOT IN (${input.excludeJobIds.map(() => "?").join(", ")})`);
+            values.push(...input.excludeJobIds);
+        }
+        const rows = this.db.prepare(`
+			SELECT * FROM pibo_jobs
+			WHERE ${clauses.join(" AND ")}
+			ORDER BY priority DESC, run_at ASC, created_at ASC
+			LIMIT ?
+		`).all(...values, clampLimit(input.limit, 100));
+        const jobs = [];
+        for (const row of rows) {
+            try {
+                jobs.push(jobFromRow(row));
+            }
+            catch {
+                this.quarantineJobRow(row, "payload_malformed");
+            }
+        }
+        return jobs;
     }
     listDead(input = {}) {
         const clauses = [];
@@ -466,6 +616,51 @@ export class PiboReliabilityStore {
             .all(...values, clampLimit(input.limit, 100));
         return rows.map(deadJobFromRow);
     }
+    hasLiveJob(jobId) {
+        return this.db.prepare("SELECT 1 AS found FROM pibo_jobs WHERE job_id = ?").get(jobId) !== undefined;
+    }
+    hasJobs(queue) {
+        return queue
+            ? this.db.prepare("SELECT 1 AS found FROM pibo_jobs WHERE queue = ? LIMIT 1").get(queue) !== undefined
+            : this.db.prepare("SELECT 1 AS found FROM pibo_jobs LIMIT 1").get() !== undefined;
+    }
+    hasRecoverableJobs(queue, includeDeferredPending = false) {
+        const timestamp = now();
+        return this.inImmediateTransaction(() => {
+            this.moveExpiredJobs(timestamp);
+            this.moveExhaustedRecoverableJobs(timestamp, { queue });
+            return this.db.prepare(`
+				SELECT 1 AS found FROM pibo_jobs
+				WHERE queue = ?
+					AND (
+						(state = 'pending' AND (? = 1 OR run_at <= ?))
+						OR (state = 'running' AND claim_expires_at IS NOT NULL AND claim_expires_at <= ?)
+					)
+					AND attempts < max_attempts
+					AND (expires_at IS NULL OR expires_at > ?)
+				LIMIT 1
+			`).get(queue, includeDeferredPending ? 1 : 0, timestamp, timestamp, timestamp) !== undefined;
+        });
+    }
+    quarantineJob(jobId, reason, correlation = {}) {
+        const row = this.db.prepare("SELECT * FROM pibo_jobs WHERE job_id = ?").get(jobId);
+        if (!row)
+            return false;
+        this.quarantineJobRow(row, reason, correlation);
+        return true;
+    }
+    deadLetter(input) {
+        const timestamp = now();
+        const jobId = input.jobId ?? `job_${randomUUID()}`;
+        this.db.prepare(`
+			INSERT OR REPLACE INTO pibo_dead_jobs (
+				job_id, queue, payload_json, attempts, max_attempts, idempotency_key, created_at,
+				updated_at, expires_at, last_error, dead_at, dead_reason
+			) VALUES (?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?, ?)
+		`).run(jobId, input.queue, json(input.payload ?? null), Math.max(0, input.attempts ?? 0), Math.max(1, input.maxAttempts ?? 1), input.idempotencyKey ?? null, timestamp, timestamp, input.error ?? input.reason, timestamp, input.reason);
+        const row = this.db.prepare("SELECT * FROM pibo_dead_jobs WHERE job_id = ?").get(jobId);
+        return deadJobFromRow(row);
+    }
     requeueDead(jobId, input = {}) {
         const dead = this.db.prepare("SELECT * FROM pibo_dead_jobs WHERE job_id = ?").get(jobId);
         if (!dead)
@@ -583,46 +778,62 @@ export class PiboReliabilityStore {
         return Number(result.changes ?? 0);
     }
     recoverInterruptedRuns(workerId = `run-registry:${process.pid}`) {
-        const rows = this.db.prepare("SELECT * FROM pibo_runs WHERE status = 'running'").all();
-        const recovered = [];
-        const timestamp = now();
-        for (const row of rows) {
-            if (row.job_id && this.hasUnexpiredJobClaim(row.job_id, timestamp, workerId))
-                continue;
-            const run = runFromRow(row);
-            if (run.timeoutAt && run.timeoutAt <= timestamp) {
-                const error = `Run deadline ${run.timeoutAt} elapsed before the interrupted runtime recovered.`;
-                if (run.jobId)
-                    this.moveLiveJobToDead(run.jobId, error, "timeout", timestamp);
-                recovered.push(this.updateRun(run.runId, {
-                    status: "timed_out",
-                    error,
-                    timeoutPhase: "lifetime",
-                    summary: `${run.toolName} run started successfully, then reached its configured timeout.`,
-                    completedAt: timestamp,
-                }) ?? run);
-                continue;
-            }
-            if (run.retryable && run.maxAttempts > 1) {
-                if (run.jobId)
-                    this.releaseJobForRetry(run.jobId, timestamp);
-                recovered.push(this.updateRun(run.runId, {
-                    status: "queued",
-                    summary: `${run.toolName} run is queued for retry after interruption.`,
-                }) ?? run);
-            }
-            else {
-                if (run.jobId)
-                    this.moveLiveJobToDead(run.jobId, "Run was interrupted before completion and the tool is not retryable.", "interrupted", timestamp);
+        return this.inImmediateTransaction(() => {
+            const rows = this.db.prepare("SELECT * FROM pibo_runs WHERE status = 'running'").all();
+            const recovered = [];
+            const timestamp = now();
+            for (const row of rows) {
+                if (row.job_id && this.hasUnexpiredJobClaim(row.job_id, timestamp, workerId))
+                    continue;
+                const run = runFromRow(row);
+                if (run.timeoutAt && run.timeoutAt <= timestamp) {
+                    const error = `Run deadline ${run.timeoutAt} elapsed before the interrupted runtime recovered.`;
+                    if (run.jobId)
+                        this.moveLiveJobToDead(run.jobId, error, "timeout", timestamp);
+                    recovered.push(this.updateRun(run.runId, {
+                        status: "timed_out",
+                        error,
+                        timeoutPhase: "lifetime",
+                        summary: `${run.toolName} run started successfully, then reached its configured timeout.`,
+                        completedAt: timestamp,
+                    }) ?? run);
+                    continue;
+                }
+                const jobRow = run.jobId
+                    ? this.db.prepare("SELECT * FROM pibo_jobs WHERE job_id = ?").get(run.jobId)
+                    : undefined;
+                if (run.retryable && jobRow && jobRow.attempts < jobRow.max_attempts) {
+                    this.releaseJobForRetry(jobRow.job_id, timestamp);
+                    recovered.push(this.updateRun(run.runId, {
+                        status: "queued",
+                        summary: `${run.toolName} run is queued for retry after interruption.`,
+                    }) ?? run);
+                    continue;
+                }
+                if (run.retryable && jobRow && jobRow.attempts >= jobRow.max_attempts) {
+                    this.moveJobToDead(jobRow, MAX_ATTEMPTS_ERROR, MAX_ATTEMPTS_REASON, timestamp);
+                    recovered.push(this.updateRun(run.runId, {
+                        status: "failed",
+                        error: MAX_ATTEMPTS_ERROR,
+                        summary: `${run.toolName} run failed after exhausting retry attempts.`,
+                        completedAt: timestamp,
+                    }) ?? run);
+                    continue;
+                }
+                const error = run.retryable
+                    ? "Run was interrupted before completion and its retry job is no longer available."
+                    : "Run was interrupted before completion and the tool is not retryable.";
+                if (jobRow)
+                    this.moveJobToDead(jobRow, error, "interrupted", timestamp);
                 recovered.push(this.updateRun(run.runId, {
                     status: "failed",
-                    error: "Run was interrupted before completion and the tool is not retryable.",
+                    error,
                     summary: `${run.toolName} run failed.`,
                     completedAt: timestamp,
                 }) ?? run);
             }
-        }
-        return recovered;
+            return recovered;
+        });
     }
     close() {
         this.db.close();
@@ -659,23 +870,42 @@ export class PiboReliabilityStore {
             .get(queue, idempotencyKey);
         return row ? jobFromRow(row) : undefined;
     }
-    getLiveWorkerJob(jobId, workerId, timestamp) {
+    getLiveWorkerJob(jobId, workerId, timestamp, claimToken) {
         return this.db
             .prepare(`
 				SELECT * FROM pibo_jobs
 				WHERE job_id = ?
 					AND state = 'running'
 					AND worker_id = ?
+					AND (? IS NULL OR claim_token = ?)
 					AND claim_expires_at IS NOT NULL
 					AND claim_expires_at > ?
 			`)
-            .get(jobId, workerId, timestamp);
+            .get(jobId, workerId, claimToken ?? null, claimToken ?? null, timestamp);
     }
     moveExpiredJobs(timestamp) {
         const expired = this.db.prepare("SELECT * FROM pibo_jobs WHERE expires_at IS NOT NULL AND expires_at <= ?").all(timestamp);
         for (const row of expired)
             this.moveJobToDead(row, row.last_error ?? "Job expired.", "expired", timestamp);
     }
+    moveExhaustedRecoverableJobs(timestamp, options = {}) {
+        const clauses = [
+            "attempts >= max_attempts",
+            "(state = 'pending' OR (state = 'running' AND claim_expires_at IS NOT NULL AND claim_expires_at <= ?))",
+        ];
+        const values = [timestamp];
+        if (options.jobId) {
+            clauses.push("job_id = ?");
+            values.push(options.jobId);
+        }
+        if (options.queue) {
+            clauses.push("queue = ?");
+            values.push(options.queue);
+        }
+        const exhausted = this.db.prepare(`SELECT * FROM pibo_jobs WHERE ${clauses.join(" AND ")}`).all(...values);
+        for (const row of exhausted)
+            this.moveJobToDead(row, MAX_ATTEMPTS_ERROR, MAX_ATTEMPTS_REASON, timestamp);
+    }
     hasUnexpiredJobClaim(jobId, timestamp, workerId) {
         const row = this.db
             .prepare(`
@@ -718,6 +948,39 @@ export class PiboReliabilityStore {
             .run(row.job_id, row.queue, row.payload_json, row.attempts, row.max_attempts, row.idempotency_key, row.created_at, timestamp, row.expires_at, error, timestamp, reason);
         this.db.prepare("DELETE FROM pibo_jobs WHERE job_id = ?").run(row.job_id);
     }
+    quarantineJobRow(row, reason, correlation = {}) {
+        this.inImmediateTransaction(() => {
+            const current = this.db.prepare("SELECT * FROM pibo_jobs WHERE job_id = ?").get(row.job_id);
+            if (!current)
+                return;
+            const payload = {
+                version: 1,
+                key: correlation.key ?? current.job_id,
+                ...(correlation.piboSessionId ? { piboSessionId: correlation.piboSessionId } : {}),
+                ...(correlation.eventId ? { eventId: correlation.eventId } : {}),
+                state: { phase: "quarantined", reasonCode: reason },
+            };
+            const sanitized = {
+                ...current,
+                payload_json: json(payload),
+                idempotency_key: correlation.key ?? null,
+                last_error: reason,
+            };
+            this.moveJobToDead(sanitized, reason, reason, now());
+        });
+    }
+    inImmediateTransaction(operation) {
+        this.db.exec("BEGIN IMMEDIATE");
+        try {
+            const result = operation();
+            this.db.exec("COMMIT");
+            return result;
+        }
+        catch (error) {
+            this.db.exec("ROLLBACK");
+            throw error;
+        }
+    }
     requireRun(runId) {
         const run = this.getRun(runId);
         if (!run)
@@ -750,6 +1013,7 @@ function jobFromRow(row) {
         priority: row.priority,
         workerId: row.worker_id ?? undefined,
         claimExpiresAt: row.claim_expires_at ?? undefined,
+        claimToken: row.claim_token,
         attempts: row.attempts,
         maxAttempts: row.max_attempts,
         idempotencyKey: row.idempotency_key ?? undefined,
@@ -763,7 +1027,7 @@ function deadJobFromRow(row) {
     return {
         jobId: row.job_id,
         queue: row.queue,
-        payload: parseJson(row.payload_json),
+        payload: safeDeadPayload(row.payload_json, row.dead_reason, row.job_id),
         attempts: row.attempts,
         maxAttempts: row.max_attempts,
         idempotencyKey: row.idempotency_key ?? undefined,
@@ -775,6 +1039,14 @@ function deadJobFromRow(row) {
         deadReason: row.dead_reason,
     };
 }
+function safeDeadPayload(value, reason, jobId) {
+    try {
+        return parseJson(value);
+    }
+    catch {
+        return { version: 1, key: jobId, state: { phase: "quarantined", reasonCode: reason } };
+    }
+}
 function runFromRow(row) {
     const output = {
         runId: row.run_id,