@pasko70/pibo 1.9.13 → 1.10.1

package/dist/reliability/store.js

@@ -30,6 +30,11 @@ function migratePiboRunControllerColumn(db) {
     db.exec("DROP INDEX IF EXISTS idx_pibo_runs_" + ["o", "wner_updated"].join(""));
     db.exec(`ALTER TABLE pibo_runs RENAME COLUMN ${legacyColumn} TO controller_pibo_session_id`);
 }
+function ensurePiboRunColumn(db, name, definition) {
+    const columns = sqliteTableColumns(db, "pibo_runs");
+    if (!columns.has(name))
+        db.exec(`ALTER TABLE pibo_runs ADD COLUMN ${name} ${definition}`);
+}
 export class PiboReliabilityStore {
     db;
     appendEventStatement;
@@ -46,105 +51,113 @@ export class PiboReliabilityStore {
         if (resolvedPath !== ":memory:")
             this.db.exec("PRAGMA journal_mode = WAL");
         migratePiboRunControllerColumn(this.db);
-        this.db.exec(`
-			CREATE TABLE IF NOT EXISTS pibo_event_stream (
-				stream_id INTEGER PRIMARY KEY,
-				topic TEXT NOT NULL,
-				key TEXT,
-				event_id TEXT NOT NULL,
-				idempotency_key TEXT,
-				created_at TEXT NOT NULL,
-				retention_class TEXT NOT NULL,
-				payload_json TEXT NOT NULL
-			);
-			CREATE UNIQUE INDEX IF NOT EXISTS idx_pibo_event_stream_event
-				ON pibo_event_stream(topic, event_id);
-			CREATE UNIQUE INDEX IF NOT EXISTS idx_pibo_event_stream_idempotency
-				ON pibo_event_stream(topic, idempotency_key)
-				WHERE idempotency_key IS NOT NULL;
-			CREATE INDEX IF NOT EXISTS idx_pibo_event_stream_topic_stream
-				ON pibo_event_stream(topic, stream_id);
-
-			CREATE TABLE IF NOT EXISTS pibo_event_consumers (
-				consumer TEXT NOT NULL,
-				topic TEXT NOT NULL,
-				last_stream_id INTEGER NOT NULL,
-				updated_at TEXT NOT NULL,
-				PRIMARY KEY(consumer, topic)
-			);
-
-			CREATE TABLE IF NOT EXISTS pibo_jobs (
-				job_id TEXT PRIMARY KEY,
-				queue TEXT NOT NULL,
-				state TEXT NOT NULL,
-				payload_json TEXT NOT NULL,
-				run_at TEXT NOT NULL,
-				priority INTEGER NOT NULL DEFAULT 0,
-				worker_id TEXT,
-				claim_expires_at TEXT,
-				attempts INTEGER NOT NULL DEFAULT 0,
-				max_attempts INTEGER NOT NULL DEFAULT 1,
-				idempotency_key TEXT,
-				created_at TEXT NOT NULL,
-				updated_at TEXT NOT NULL,
-				expires_at TEXT,
-				last_error TEXT
-			);
-			CREATE UNIQUE INDEX IF NOT EXISTS idx_pibo_jobs_idempotency
-				ON pibo_jobs(queue, idempotency_key)
-				WHERE idempotency_key IS NOT NULL;
-			CREATE INDEX IF NOT EXISTS idx_pibo_jobs_live_claim
-				ON pibo_jobs(queue, state, run_at, priority DESC, created_at)
-				WHERE state = 'pending';
-			CREATE INDEX IF NOT EXISTS idx_pibo_jobs_expired_claim
-				ON pibo_jobs(queue, state, claim_expires_at, priority DESC, created_at)
-				WHERE state = 'running';
-
-			CREATE TABLE IF NOT EXISTS pibo_dead_jobs (
-				job_id TEXT PRIMARY KEY,
-				queue TEXT NOT NULL,
-				payload_json TEXT NOT NULL,
-				attempts INTEGER NOT NULL,
-				max_attempts INTEGER NOT NULL,
-				idempotency_key TEXT,
-				created_at TEXT NOT NULL,
-				updated_at TEXT NOT NULL,
-				expires_at TEXT,
-				last_error TEXT,
-				dead_at TEXT NOT NULL,
-				dead_reason TEXT NOT NULL
-			);
-			CREATE INDEX IF NOT EXISTS idx_pibo_dead_jobs_queue_dead_at
-				ON pibo_dead_jobs(queue, dead_at);
-
-			CREATE TABLE IF NOT EXISTS pibo_runs (
-				run_id TEXT PRIMARY KEY,
-				kind TEXT NOT NULL,
-				controller_pibo_session_id TEXT NOT NULL,
-				status TEXT NOT NULL,
-				completion_policy TEXT NOT NULL,
-				consumed INTEGER NOT NULL DEFAULT 0,
-				tool_name TEXT NOT NULL,
-				summary TEXT,
-				result_json TEXT,
-				error TEXT,
-				notified_status TEXT,
-				acknowledged_status TEXT,
-				created_at TEXT NOT NULL,
-				updated_at TEXT NOT NULL,
-				completed_at TEXT,
-				job_id TEXT,
-				retryable INTEGER NOT NULL DEFAULT 0,
-				max_attempts INTEGER NOT NULL DEFAULT 1
-			);
-			CREATE INDEX IF NOT EXISTS idx_pibo_runs_controller_updated
-				ON pibo_runs(controller_pibo_session_id, updated_at);
-			CREATE INDEX IF NOT EXISTS idx_pibo_runs_status
-				ON pibo_runs(status);
+        this.db.exec(`
+			CREATE TABLE IF NOT EXISTS pibo_event_stream (
+				stream_id INTEGER PRIMARY KEY,
+				topic TEXT NOT NULL,
+				key TEXT,
+				event_id TEXT NOT NULL,
+				idempotency_key TEXT,
+				created_at TEXT NOT NULL,
+				retention_class TEXT NOT NULL,
+				payload_json TEXT NOT NULL
+			);
+			CREATE UNIQUE INDEX IF NOT EXISTS idx_pibo_event_stream_event
+				ON pibo_event_stream(topic, event_id);
+			CREATE UNIQUE INDEX IF NOT EXISTS idx_pibo_event_stream_idempotency
+				ON pibo_event_stream(topic, idempotency_key)
+				WHERE idempotency_key IS NOT NULL;
+			CREATE INDEX IF NOT EXISTS idx_pibo_event_stream_topic_stream
+				ON pibo_event_stream(topic, stream_id);
+
+			CREATE TABLE IF NOT EXISTS pibo_event_consumers (
+				consumer TEXT NOT NULL,
+				topic TEXT NOT NULL,
+				last_stream_id INTEGER NOT NULL,
+				updated_at TEXT NOT NULL,
+				PRIMARY KEY(consumer, topic)
+			);
+
+			CREATE TABLE IF NOT EXISTS pibo_jobs (
+				job_id TEXT PRIMARY KEY,
+				queue TEXT NOT NULL,
+				state TEXT NOT NULL,
+				payload_json TEXT NOT NULL,
+				run_at TEXT NOT NULL,
+				priority INTEGER NOT NULL DEFAULT 0,
+				worker_id TEXT,
+				claim_expires_at TEXT,
+				attempts INTEGER NOT NULL DEFAULT 0,
+				max_attempts INTEGER NOT NULL DEFAULT 1,
+				idempotency_key TEXT,
+				created_at TEXT NOT NULL,
+				updated_at TEXT NOT NULL,
+				expires_at TEXT,
+				last_error TEXT
+			);
+			CREATE UNIQUE INDEX IF NOT EXISTS idx_pibo_jobs_idempotency
+				ON pibo_jobs(queue, idempotency_key)
+				WHERE idempotency_key IS NOT NULL;
+			CREATE INDEX IF NOT EXISTS idx_pibo_jobs_live_claim
+				ON pibo_jobs(queue, state, run_at, priority DESC, created_at)
+				WHERE state = 'pending';
+			CREATE INDEX IF NOT EXISTS idx_pibo_jobs_expired_claim
+				ON pibo_jobs(queue, state, claim_expires_at, priority DESC, created_at)
+				WHERE state = 'running';
+
+			CREATE TABLE IF NOT EXISTS pibo_dead_jobs (
+				job_id TEXT PRIMARY KEY,
+				queue TEXT NOT NULL,
+				payload_json TEXT NOT NULL,
+				attempts INTEGER NOT NULL,
+				max_attempts INTEGER NOT NULL,
+				idempotency_key TEXT,
+				created_at TEXT NOT NULL,
+				updated_at TEXT NOT NULL,
+				expires_at TEXT,
+				last_error TEXT,
+				dead_at TEXT NOT NULL,
+				dead_reason TEXT NOT NULL
+			);
+			CREATE INDEX IF NOT EXISTS idx_pibo_dead_jobs_queue_dead_at
+				ON pibo_dead_jobs(queue, dead_at);
+
+			CREATE TABLE IF NOT EXISTS pibo_runs (
+				run_id TEXT PRIMARY KEY,
+				kind TEXT NOT NULL,
+				controller_pibo_session_id TEXT NOT NULL,
+				status TEXT NOT NULL,
+				completion_policy TEXT NOT NULL,
+				consumed INTEGER NOT NULL DEFAULT 0,
+				tool_name TEXT NOT NULL,
+				summary TEXT,
+				result_json TEXT,
+				error TEXT,
+				notified_status TEXT,
+				acknowledged_status TEXT,
+				created_at TEXT NOT NULL,
+				updated_at TEXT NOT NULL,
+				completed_at TEXT,
+				job_id TEXT,
+				retryable INTEGER NOT NULL DEFAULT 0,
+				max_attempts INTEGER NOT NULL DEFAULT 1,
+				timeout_ms INTEGER,
+				timeout_at TEXT,
+				timeout_phase TEXT,
+				service_warning TEXT
+			);
+			CREATE INDEX IF NOT EXISTS idx_pibo_runs_controller_updated
+				ON pibo_runs(controller_pibo_session_id, updated_at);
+			CREATE INDEX IF NOT EXISTS idx_pibo_runs_status
+				ON pibo_runs(status);
 		`);
-        this.appendEventStatement = this.db.prepare(`
-			INSERT INTO pibo_event_stream (topic, key, event_id, idempotency_key, created_at, retention_class, payload_json)
-			VALUES (?, ?, ?, ?, ?, ?, ?)
+        ensurePiboRunColumn(this.db, "timeout_ms", "INTEGER");
+        ensurePiboRunColumn(this.db, "timeout_at", "TEXT");
+        ensurePiboRunColumn(this.db, "timeout_phase", "TEXT");
+        ensurePiboRunColumn(this.db, "service_warning", "TEXT");
+        this.appendEventStatement = this.db.prepare(`
+			INSERT INTO pibo_event_stream (topic, key, event_id, idempotency_key, created_at, retention_class, payload_json)
+			VALUES (?, ?, ?, ?, ?, ?, ?)
 		`);
         this.getEventByStreamIdStatement = this.db.prepare("SELECT * FROM pibo_event_stream WHERE stream_id = ?");
         this.getEventByTopicEventIdStatement = this.db.prepare("SELECT * FROM pibo_event_stream WHERE topic = ? AND event_id = ?");
@@ -202,12 +215,12 @@ export class PiboReliabilityStore {
         }
         const where = clauses.length ? `WHERE ${clauses.join(" AND ")}` : "";
         const rows = this.db
-            .prepare(`
-				SELECT topic, key, retention_class, COUNT(*) AS count
-				FROM pibo_event_stream
-				${where}
-				GROUP BY topic, key, retention_class
-				ORDER BY topic, key, retention_class
+            .prepare(`
+				SELECT topic, key, retention_class, COUNT(*) AS count
+				FROM pibo_event_stream
+				${where}
+				GROUP BY topic, key, retention_class
+				ORDER BY topic, key, retention_class
 			`)
             .all(...values);
         return rows.map((row) => ({
@@ -223,12 +236,12 @@ export class PiboReliabilityStore {
     }
     saveConsumerOffset(topic, consumer, streamId) {
         this.db
-            .prepare(`
-				INSERT INTO pibo_event_consumers (consumer, topic, last_stream_id, updated_at)
-				VALUES (?, ?, ?, ?)
-				ON CONFLICT(consumer, topic) DO UPDATE SET
-					last_stream_id = MAX(pibo_event_consumers.last_stream_id, excluded.last_stream_id),
-					updated_at = excluded.updated_at
+            .prepare(`
+				INSERT INTO pibo_event_consumers (consumer, topic, last_stream_id, updated_at)
+				VALUES (?, ?, ?, ?)
+				ON CONFLICT(consumer, topic) DO UPDATE SET
+					last_stream_id = MAX(pibo_event_consumers.last_stream_id, excluded.last_stream_id),
+					updated_at = excluded.updated_at
 			`)
             .run(consumer, topic, streamId, now());
     }
@@ -260,23 +273,23 @@ export class PiboReliabilityStore {
             values.push(input.beforeStreamId);
         }
         if (!input.destructive) {
-            clauses.push(`
-				stream_id <= COALESCE(
-					(SELECT MIN(last_stream_id) FROM pibo_event_consumers WHERE pibo_event_consumers.topic = pibo_event_stream.topic),
-					stream_id
-				)
+            clauses.push(`
+				stream_id <= COALESCE(
+					(SELECT MIN(last_stream_id) FROM pibo_event_consumers WHERE pibo_event_consumers.topic = pibo_event_stream.topic),
+					stream_id
+				)
 			`);
         }
         const where = clauses.length ? `WHERE ${clauses.join(" AND ")}` : "";
         const result = this.db
-            .prepare(`
-				DELETE FROM pibo_event_stream
-				WHERE stream_id IN (
-					SELECT stream_id FROM pibo_event_stream
-					${where}
-					ORDER BY stream_id ASC
-					LIMIT ${clampLimit(input.limit, 500)}
-				)
+            .prepare(`
+				DELETE FROM pibo_event_stream
+				WHERE stream_id IN (
+					SELECT stream_id FROM pibo_event_stream
+					${where}
+					ORDER BY stream_id ASC
+					LIMIT ${clampLimit(input.limit, 500)}
+				)
 			`)
             .run(...values);
         return Number(result.changes ?? 0);
@@ -285,11 +298,11 @@ export class PiboReliabilityStore {
         const timestamp = now();
         const jobId = input.jobId ?? `job_${randomUUID()}`;
         const result = this.db
-            .prepare(`
-				INSERT OR IGNORE INTO pibo_jobs (
-					job_id, queue, state, payload_json, run_at, priority, worker_id, claim_expires_at,
-					attempts, max_attempts, idempotency_key, created_at, updated_at, expires_at, last_error
-				) VALUES (?, ?, 'pending', ?, ?, ?, NULL, NULL, 0, ?, ?, ?, ?, ?, NULL)
+            .prepare(`
+				INSERT OR IGNORE INTO pibo_jobs (
+					job_id, queue, state, payload_json, run_at, priority, worker_id, claim_expires_at,
+					attempts, max_attempts, idempotency_key, created_at, updated_at, expires_at, last_error
+				) VALUES (?, ?, 'pending', ?, ?, ?, NULL, NULL, 0, ?, ?, ?, ?, ?, NULL)
 			`)
             .run(jobId, input.queue, json(input.payload), input.runAt ?? timestamp, input.priority ?? 0, Math.max(1, input.maxAttempts ?? 1), input.idempotencyKey ?? null, timestamp, timestamp, input.expiresAt ?? null);
         if (Number(result.changes ?? 0) === 0 && input.idempotencyKey) {
@@ -313,20 +326,20 @@ export class PiboReliabilityStore {
         }
         this.moveExpiredJobs(timestamp);
         const rows = this.db
-            .prepare(`
-				UPDATE pibo_jobs
-				SET state = 'running',
-					worker_id = ?,
-					claim_expires_at = ?,
-					attempts = attempts + 1,
-					updated_at = ?
-				WHERE job_id IN (
-					SELECT job_id FROM pibo_jobs
-					WHERE ${clauses.join(" AND ")}
-					ORDER BY priority DESC, run_at ASC, created_at ASC
-					LIMIT ?
-				)
-				RETURNING *
+            .prepare(`
+				UPDATE pibo_jobs
+				SET state = 'running',
+					worker_id = ?,
+					claim_expires_at = ?,
+					attempts = attempts + 1,
+					updated_at = ?
+				WHERE job_id IN (
+					SELECT job_id FROM pibo_jobs
+					WHERE ${clauses.join(" AND ")}
+					ORDER BY priority DESC, run_at ASC, created_at ASC
+					LIMIT ?
+				)
+				RETURNING *
 			`)
             .all(workerId, claimExpiresAt, timestamp, ...values, clampLimit(limit));
         return rows.map(jobFromRow);
@@ -335,18 +348,18 @@ export class PiboReliabilityStore {
         const timestamp = now();
         const claimExpiresAt = new Date(Date.now() + visibilityTimeoutMs).toISOString();
         const row = this.db
-            .prepare(`
-				UPDATE pibo_jobs
-				SET state = 'running',
-					worker_id = ?,
-					claim_expires_at = ?,
-					attempts = attempts + 1,
-					updated_at = ?
-				WHERE job_id = ?
-					AND state = 'pending'
-					AND run_at <= ?
-					AND (expires_at IS NULL OR expires_at > ?)
-				RETURNING *
+            .prepare(`
+				UPDATE pibo_jobs
+				SET state = 'running',
+					worker_id = ?,
+					claim_expires_at = ?,
+					attempts = attempts + 1,
+					updated_at = ?
+				WHERE job_id = ?
+					AND state = 'pending'
+					AND run_at <= ?
+					AND (expires_at IS NULL OR expires_at > ?)
+				RETURNING *
 			`)
             .get(workerId, claimExpiresAt, timestamp, jobId, timestamp, timestamp);
         return row ? jobFromRow(row) : undefined;
@@ -354,13 +367,13 @@ export class PiboReliabilityStore {
     ack(jobId, workerId) {
         const timestamp = now();
         const result = this.db
-            .prepare(`
-				DELETE FROM pibo_jobs
-				WHERE job_id = ?
-					AND state = 'running'
-					AND worker_id = ?
-					AND claim_expires_at IS NOT NULL
-					AND claim_expires_at > ?
+            .prepare(`
+				DELETE FROM pibo_jobs
+				WHERE job_id = ?
+					AND state = 'running'
+					AND worker_id = ?
+					AND claim_expires_at IS NOT NULL
+					AND claim_expires_at > ?
 			`)
             .run(jobId, workerId, timestamp);
         return Number(result.changes ?? 0) > 0;
@@ -378,15 +391,15 @@ export class PiboReliabilityStore {
         const delayMs = input.delayMs ?? retryDelayMs(job.attempts, input);
         const runAt = new Date(asDate(input.now).getTime() + delayMs).toISOString();
         this.db
-            .prepare(`
-				UPDATE pibo_jobs
-				SET state = 'pending',
-					worker_id = NULL,
-					claim_expires_at = NULL,
-					run_at = ?,
-					updated_at = ?,
-					last_error = ?
-				WHERE job_id = ?
+            .prepare(`
+				UPDATE pibo_jobs
+				SET state = 'pending',
+					worker_id = NULL,
+					claim_expires_at = NULL,
+					run_at = ?,
+					updated_at = ?,
+					last_error = ?
+				WHERE job_id = ?
 			`)
             .run(runAt, timestamp, input.error ?? null, jobId);
         return true;
@@ -403,14 +416,14 @@ export class PiboReliabilityStore {
         const timestamp = now();
         const claimExpiresAt = new Date(Date.now() + Math.max(1, extendMs)).toISOString();
         const result = this.db
-            .prepare(`
-				UPDATE pibo_jobs
-				SET claim_expires_at = ?, updated_at = ?
-				WHERE job_id = ?
-					AND state = 'running'
-					AND worker_id = ?
-					AND claim_expires_at IS NOT NULL
-					AND claim_expires_at > ?
+            .prepare(`
+				UPDATE pibo_jobs
+				SET claim_expires_at = ?, updated_at = ?
+				WHERE job_id = ?
+					AND state = 'running'
+					AND worker_id = ?
+					AND claim_expires_at IS NOT NULL
+					AND claim_expires_at > ?
 			`)
             .run(claimExpiresAt, timestamp, jobId, workerId, timestamp);
         return Number(result.changes ?? 0) > 0;
@@ -466,6 +479,7 @@ export class PiboReliabilityStore {
         const timestamp = now();
         const runId = input.runId ?? `run_${randomUUID()}`;
         const maxAttempts = Math.max(1, input.maxAttempts ?? 1);
+        const timeoutAt = input.timeoutMs === undefined ? undefined : new Date(Date.parse(timestamp) + input.timeoutMs).toISOString();
         const job = this.enqueue({
             queue: "runs",
             payload: {
@@ -473,18 +487,19 @@ export class PiboReliabilityStore {
                 controllerPiboSessionId: input.controllerPiboSessionId,
                 toolName: input.toolName,
                 params: input.params,
+                timeoutMs: input.timeoutMs,
             },
             maxAttempts,
         });
         this.db
-            .prepare(`
-				INSERT INTO pibo_runs (
-					run_id, kind, controller_pibo_session_id, status, completion_policy, consumed, tool_name,
-					summary, result_json, error, notified_status, acknowledged_status, created_at, updated_at,
-					completed_at, job_id, retryable, max_attempts
-				) VALUES (?, 'tool', ?, 'running', ?, 0, ?, ?, NULL, NULL, NULL, NULL, ?, ?, NULL, ?, ?, ?)
+            .prepare(`
+				INSERT INTO pibo_runs (
+					run_id, kind, controller_pibo_session_id, status, completion_policy, consumed, tool_name,
+					summary, result_json, error, notified_status, acknowledged_status, created_at, updated_at,
+					completed_at, job_id, retryable, max_attempts, timeout_ms, timeout_at, timeout_phase, service_warning
+				) VALUES (?, 'tool', ?, 'running', ?, 0, ?, ?, NULL, NULL, NULL, NULL, ?, ?, NULL, ?, ?, ?, ?, ?, NULL, ?)
 			`)
-            .run(runId, input.controllerPiboSessionId, input.completionPolicy, input.toolName, `${input.toolName} run is running.`, timestamp, timestamp, job.jobId, input.retryable ? 1 : 0, maxAttempts);
+            .run(runId, input.controllerPiboSessionId, input.completionPolicy, input.toolName, `${input.toolName} run is running.`, timestamp, timestamp, job.jobId, input.retryable ? 1 : 0, maxAttempts, input.timeoutMs ?? null, timeoutAt ?? null, input.serviceWarning ?? null);
         this.claimJob(job.jobId, `run-registry:${process.pid}`, 24 * 60 * 60 * 1000);
         return this.requireRun(runId);
     }
@@ -494,24 +509,28 @@ export class PiboReliabilityStore {
             return undefined;
         const next = { ...existing, ...patch, updatedAt: patch.updatedAt ?? now() };
         this.db
-            .prepare(`
-				UPDATE pibo_runs SET
-					status = ?,
-					completion_policy = ?,
-					consumed = ?,
-					summary = ?,
-					result_json = ?,
-					error = ?,
-					notified_status = ?,
-					acknowledged_status = ?,
-					updated_at = ?,
-					completed_at = ?,
-					job_id = ?,
-					retryable = ?,
-					max_attempts = ?
-				WHERE run_id = ?
+            .prepare(`
+				UPDATE pibo_runs SET
+					status = ?,
+					completion_policy = ?,
+					consumed = ?,
+					summary = ?,
+					result_json = ?,
+					error = ?,
+					notified_status = ?,
+					acknowledged_status = ?,
+					updated_at = ?,
+					completed_at = ?,
+					job_id = ?,
+					retryable = ?,
+					max_attempts = ?,
+					timeout_ms = ?,
+					timeout_at = ?,
+					timeout_phase = ?,
+					service_warning = ?
+				WHERE run_id = ?
 			`)
-            .run(next.status, next.completionPolicy, next.consumed ? 1 : 0, next.summary ?? null, next.result ? JSON.stringify(next.result) : null, next.error ?? null, next.notifiedStatus ?? null, next.acknowledgedStatus ?? null, next.updatedAt, next.completedAt ?? null, next.jobId ?? null, next.retryable ? 1 : 0, next.maxAttempts, runId);
+            .run(next.status, next.completionPolicy, next.consumed ? 1 : 0, next.summary ?? null, next.result ? JSON.stringify(next.result) : null, next.error ?? null, next.notifiedStatus ?? null, next.acknowledgedStatus ?? null, next.updatedAt, next.completedAt ?? null, next.jobId ?? null, next.retryable ? 1 : 0, next.maxAttempts, next.timeoutMs ?? null, next.timeoutAt ?? null, next.timeoutPhase ?? null, next.serviceWarning ?? null, runId);
         return this.requireRun(runId);
     }
     getRun(runId) {
@@ -535,7 +554,7 @@ export class PiboReliabilityStore {
     }
     pruneRuns(input) {
         const rows = this.db
-            .prepare("SELECT * FROM pibo_runs WHERE status IN ('completed', 'failed', 'cancelled') AND completed_at IS NOT NULL")
+            .prepare("SELECT * FROM pibo_runs WHERE status IN ('completed', 'failed', 'timed_out', 'cancelled') AND completed_at IS NOT NULL")
             .all();
         const ids = rows
             .map(runFromRow)
@@ -619,13 +638,13 @@ export class PiboReliabilityStore {
     }
     getLiveWorkerJob(jobId, workerId, timestamp) {
         return this.db
-            .prepare(`
-				SELECT * FROM pibo_jobs
-				WHERE job_id = ?
-					AND state = 'running'
-					AND worker_id = ?
-					AND claim_expires_at IS NOT NULL
-					AND claim_expires_at > ?
+            .prepare(`
+				SELECT * FROM pibo_jobs
+				WHERE job_id = ?
+					AND state = 'running'
+					AND worker_id = ?
+					AND claim_expires_at IS NOT NULL
+					AND claim_expires_at > ?
 			`)
             .get(jobId, workerId, timestamp);
     }
@@ -636,26 +655,26 @@ export class PiboReliabilityStore {
     }
     hasUnexpiredJobClaim(jobId, timestamp) {
         const row = this.db
-            .prepare(`
-				SELECT job_id FROM pibo_jobs
-				WHERE job_id = ?
-					AND state = 'running'
-					AND claim_expires_at IS NOT NULL
-					AND claim_expires_at > ?
+            .prepare(`
+				SELECT job_id FROM pibo_jobs
+				WHERE job_id = ?
+					AND state = 'running'
+					AND claim_expires_at IS NOT NULL
+					AND claim_expires_at > ?
 			`)
             .get(jobId, timestamp);
         return row !== undefined;
     }
     releaseJobForRetry(jobId, timestamp) {
         this.db
-            .prepare(`
-				UPDATE pibo_jobs
-				SET state = 'pending',
-					worker_id = NULL,
-					claim_expires_at = NULL,
-					run_at = ?,
-					updated_at = ?
-				WHERE job_id = ?
+            .prepare(`
+				UPDATE pibo_jobs
+				SET state = 'pending',
+					worker_id = NULL,
+					claim_expires_at = NULL,
+					run_at = ?,
+					updated_at = ?
+				WHERE job_id = ?
 			`)
             .run(timestamp, timestamp, jobId);
     }
@@ -666,11 +685,11 @@ export class PiboReliabilityStore {
     }
     moveJobToDead(row, error, reason, timestamp) {
         this.db
-            .prepare(`
-				INSERT OR REPLACE INTO pibo_dead_jobs (
-					job_id, queue, payload_json, attempts, max_attempts, idempotency_key, created_at,
-					updated_at, expires_at, last_error, dead_at, dead_reason
-				) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            .prepare(`
+				INSERT OR REPLACE INTO pibo_dead_jobs (
+					job_id, queue, payload_json, attempts, max_attempts, idempotency_key, created_at,
+					updated_at, expires_at, last_error, dead_at, dead_reason
+				) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
 			`)
             .run(row.job_id, row.queue, row.payload_json, row.attempts, row.max_attempts, row.idempotency_key, row.created_at, timestamp, row.expires_at, error, timestamp, reason);
         this.db.prepare("DELETE FROM pibo_jobs WHERE job_id = ?").run(row.job_id);
@@ -760,6 +779,14 @@ function runFromRow(row) {
         output.completedAt = row.completed_at;
     if (row.job_id)
         output.jobId = row.job_id;
+    if (row.timeout_ms !== null)
+        output.timeoutMs = row.timeout_ms;
+    if (row.timeout_at)
+        output.timeoutAt = row.timeout_at;
+    if (row.timeout_phase === "startup" || row.timeout_phase === "lifetime")
+        output.timeoutPhase = row.timeout_phase;
+    if (row.service_warning)
+        output.serviceWarning = row.service_warning;
     return output;
 }
 function retryDelayMs(attempts, input) {