@fungi.computer/watchdog 0.1.0 → 0.1.1

package/dist/internal/sqlite-store.js

@@ -1,6 +1,8 @@
 import * as Cause from "effect/Cause";
 import * as Effect from "effect/Effect";
 import * as Schema from "effect/Schema";
+import { reserveRetry, retryDue } from "./retry.js";
+import { disappearanceRecovery } from "./recovery.js";
 import { WatchdogError } from "./watchdog-error.js";
 import { laneSchema } from "./lane.js";
 const Identifier = Schema.String.check(Schema.isMinLength(1), Schema.isMaxLength(256));
@@ -31,8 +33,12 @@ const JobRowSchema = Schema.Struct({
     lane: laneSchema,
     priority: Priority,
     payload_json: Schema.String,
+    not_before: Count,
+    attempt_count: Count,
     recovery_max: Schema.NullOr(Count),
-    state: Schema.Literals(["queued", "running", "settled"]),
+    recovery_window_ms: Schema.NullOr(Count),
+    recovery_window_started_at: Schema.NullOr(Count),
+    state: Schema.Literals(["queued", "running", "completing", "settled"]),
     outcome: Schema.NullOr(Schema.Literals([
         "completed",
         "failed",
@@ -59,9 +65,37 @@ function laneQuery(partition) {
             values: [partition.lane],
         };
 }
-const columns = `sequence, job_id, queue_name, lane, priority, payload_json,
-                 recovery_max, state, outcome, claim_token, recovery_count,
+const columns = `sequence, job_id, queue_name, lane, priority, payload_json, not_before, attempt_count,
+                 recovery_max, recovery_window_ms, recovery_window_started_at, state, outcome, claim_token, recovery_count,
                  cancellation_id, failure_reason`;
+/** Queue order is decided before readiness; a sleeping head still owns its queue. */
+export const queuedHeadPredicate = `(cancellation_id IS NOT NULL OR NOT EXISTS (
+  SELECT 1 FROM watchdog_jobs predecessor
+  WHERE predecessor.queue_name = watchdog_jobs.queue_name
+    AND predecessor.state IN ('queued', 'running', 'completing')
+    AND (predecessor.state IN ('running', 'completing') OR
+      (predecessor.state = 'queued' AND
+        (predecessor.priority > watchdog_jobs.priority OR
+          (predecessor.priority = watchdog_jobs.priority AND
+            predecessor.sequence < watchdog_jobs.sequence))))))`;
+const WakeRow = Schema.Struct({
+    pending: Schema.Literals([0, 1]),
+    next_wake_at: Schema.NullOr(Count),
+});
+export const parseWakeTime = Schema.decodeUnknownSync(Count);
+export function parseWakeState(raw) {
+    const row = Schema.decodeUnknownSync(WakeRow)(raw);
+    return { pending: row.pending === 1, nextWakeAt: row.next_wake_at };
+}
+function readWakeState(owner, now) {
+    parseWakeTime(now);
+    return parseWakeState(owner.sql
+        .exec(`SELECT
+    COALESCE(MAX(CASE WHEN state IN ('running','completing') OR cancellation_id IS NOT NULL OR not_before <= ? THEN 1 ELSE 0 END),0) AS pending,
+    MIN(CASE WHEN state='queued' AND cancellation_id IS NULL AND not_before > ? THEN not_before END) AS next_wake_at
+    FROM watchdog_jobs WHERE state IN ('running','completing') OR (state='queued' AND ${queuedHeadPredicate})`, now, now)
+        .toArray()[0]);
+}
 function unavailable(cause) {
     return new WatchdogError("store_unavailable", cause);
 }
@@ -87,22 +121,21 @@ export function parseJobRow(raw) {
     const payload = decodePayload(row.payload_json);
     if (payload._tag === "Failure")
         throw unavailable(Cause.squash(payload.cause));
-    const common = row.recovery_max === null
-        ? {
-            jobId: row.job_id,
-            queue: row.queue_name,
-            lane: row.lane,
-            priority: row.priority,
-            payload: payload.value,
-        }
-        : {
-            jobId: row.job_id,
-            queue: row.queue_name,
-            lane: row.lane,
-            priority: row.priority,
-            payload: payload.value,
-            recovery: { maxRecoveries: row.recovery_max },
-        };
+    let common = {
+        jobId: row.job_id,
+        queue: row.queue_name,
+        lane: row.lane,
+        priority: row.priority,
+        payload: payload.value,
+    };
+    if (row.not_before !== 0)
+        common = { ...common, notBefore: row.not_before };
+    if (row.recovery_max !== null) {
+        const recovery = row.recovery_window_ms === null
+            ? { maxRecoveries: row.recovery_max }
+            : { maxRecoveries: row.recovery_max, windowMs: row.recovery_window_ms };
+        common = { ...common, recovery };
+    }
     if (row.state === "queued")
         return { row, job: { ...common, state: "queued" } };
     if (row.state === "running") {
@@ -114,7 +147,7 @@ export function parseJobRow(raw) {
         throw unavailable();
     const settled = {
         ...common,
-        state: "settled",
+        state: row.state,
         outcome: row.outcome,
     };
     return {
@@ -133,13 +166,22 @@ function readJob(owner, jobId) {
         .toArray()[0];
     return raw === undefined ? undefined : parseJobRow(raw);
 }
-function readSettled(owner, limit, afterSequence, partition) {
-    const lane = laneQuery(partition);
+function readSettled(owner, limit, afterSequence, partition, selectedLane) {
+    const partitionQuery = laneQuery(partition);
+    const lane = selectedLane === undefined
+        ? partitionQuery
+        : {
+            predicate: `${partitionQuery.predicate} AND lane = ?`,
+            values: [
+                ...partitionQuery.values,
+                Schema.decodeUnknownSync(laneSchema)(selectedLane),
+            ],
+        };
     const rows = owner.sql
         .exec(`SELECT ${columns} FROM watchdog_jobs NOT INDEXED WHERE state = 'settled' AND ${lane.predicate} AND sequence > ?
        AND sequence < COALESCE(
         (SELECT MIN(sequence) FROM watchdog_jobs INDEXED BY idx_watchdog_ready
-         WHERE state IN ('queued', 'running') AND ${lane.predicate}),
+         WHERE state IN ('queued', 'running', 'completing') AND ${lane.predicate}),
         9223372036854775807)
        ORDER BY sequence ASC LIMIT ?`, ...lane.values, afterSequence ?? 0, ...lane.values, limit)
         .toArray();
@@ -154,8 +196,8 @@ function readQueueHead(owner, queue) {
     const raw = owner.sql
         .exec(`SELECT ${columns}
          FROM watchdog_jobs
-        WHERE queue_name = ? AND state IN ('running', 'queued')
-        ORDER BY CASE state WHEN 'running' THEN 0 ELSE 1 END,
+        WHERE queue_name = ? AND state IN ('running', 'completing', 'queued')
+        ORDER BY CASE state WHEN 'queued' THEN 1 ELSE 0 END,
                  priority DESC, sequence ASC
         LIMIT 1`, queue)
         .toArray()[0];
@@ -165,7 +207,7 @@ function readQueueHead(owner, queue) {
     if (parsed.job.state === "settled")
         throw unavailable();
     return {
-        state: parsed.job.state,
+        state: parsed.job.state === "completing" ? "running" : parsed.job.state,
         jobId: parsed.job.jobId,
         cancellationRequested: parsed.row.cancellation_id !== null,
     };
@@ -202,8 +244,12 @@ function prepare(owner) {
         lane TEXT NOT NULL,
         priority INTEGER NOT NULL,
         payload_json TEXT NOT NULL,
+        not_before INTEGER NOT NULL DEFAULT 0 CHECK (not_before >= 0),
+        attempt_count INTEGER NOT NULL DEFAULT 0 CHECK (attempt_count >= 0),
         recovery_max INTEGER,
-        state TEXT NOT NULL CHECK (state IN ('queued', 'running', 'settled')),
+        recovery_window_ms INTEGER CHECK (recovery_window_ms IS NULL OR recovery_window_ms > 0),
+        recovery_window_started_at INTEGER,
+        state TEXT NOT NULL CHECK (state IN ('queued', 'running', 'completing', 'settled')),
         outcome TEXT CHECK (
           outcome IS NULL OR outcome IN (
             'completed', 'failed', 'cancelled', 'outcome_unknown', 'interrupted'
@@ -214,17 +260,29 @@ function prepare(owner) {
         cancellation_id TEXT UNIQUE,
         failure_reason TEXT,
         CHECK (recovery_max IS NULL OR recovery_max >= 0),
-        CHECK (
+        CONSTRAINT watchdog_execution_state CHECK (
           (state = 'queued' AND outcome IS NULL AND claim_token IS NULL) OR
           (state = 'running' AND outcome IS NULL AND claim_token IS NOT NULL) OR
+          (state = 'completing' AND outcome IS NOT NULL AND claim_token IS NOT NULL) OR
           (state = 'settled' AND outcome IS NOT NULL)
         )
       )
     `);
+        // Refuse the old state grammar before an executor can perform effects.
+        if (owner.sql
+            .exec("SELECT 1 AS current FROM sqlite_schema WHERE name = 'watchdog_jobs' AND instr(sql, 'watchdog_execution_state') > 0")
+            .toArray().length !== 1)
+            throw new WatchdogError("store_invalid", new Error("Obsolete Watchdog execution schema"));
+        try {
+            owner.sql.exec(`SELECT ${columns} FROM watchdog_jobs LIMIT 0`);
+        }
+        catch (error) {
+            throw new WatchdogError("store_invalid", error);
+        }
         owner.sql.exec(`CREATE INDEX IF NOT EXISTS idx_watchdog_lane_failure
          ON watchdog_jobs(lane, state DESC, sequence ASC)`);
         owner.sql.exec(`CREATE INDEX IF NOT EXISTS idx_watchdog_ready
-         ON watchdog_jobs(state, priority DESC, sequence ASC)`);
+         ON watchdog_jobs(state, not_before, priority DESC, sequence ASC)`);
         owner.sql.exec(`CREATE INDEX IF NOT EXISTS idx_watchdog_queue_head
          ON watchdog_jobs(queue_name, state, priority DESC, sequence ASC)`);
     });
@@ -239,14 +297,15 @@ function enqueue(owner, project, job) {
             prior.job.lane !== job.lane ||
             prior.job.priority !== job.priority ||
             prior.row.payload_json !== payloadJson ||
-            priorRecovery !== nextRecovery)
+            priorRecovery !== nextRecovery ||
+            prior.job.recovery?.windowMs !== job.recovery?.windowMs)
             throw unavailable();
         return false;
     }
     owner.sql.exec(`INSERT INTO watchdog_jobs
-       (job_id, queue_name, lane, priority, payload_json, recovery_max,
+       (job_id, queue_name, lane, priority, payload_json, recovery_max, recovery_window_ms, not_before,
         state, outcome, claim_token, recovery_count, cancellation_id)
-     VALUES (?, ?, ?, ?, ?, ?, 'queued', NULL, NULL, 0, NULL)`, job.jobId, job.queue, job.lane, job.priority, payloadJson, job.recovery?.maxRecoveries ?? null);
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'queued', NULL, NULL, 0, NULL)`, job.jobId, job.queue, job.lane, job.priority, payloadJson, job.recovery?.maxRecoveries ?? null, job.recovery?.windowMs ?? null, job.notBefore ?? 0);
     projectQueue(owner, project, job.queue);
     return true;
 }
@@ -254,6 +313,8 @@ function recordCancellationIntent(owner, project, intent) {
     const prior = readJob(owner, intent.jobId);
     if (prior === undefined)
         return { status: "missing" };
+    if (prior.job.state === "completing" || prior.job.state === "settled")
+        return { status: "decided", outcome: prior.job.outcome };
     if (prior.row.cancellation_id !== null)
         return prior.row.cancellation_id === intent.cancellationId
             ? { status: "replayed", intent }
@@ -273,17 +334,10 @@ function selectAndClaim(owner, project, claimToken, partition) {
             .exec(`SELECT ${columns}
            FROM watchdog_jobs
           WHERE state = 'queued' AND ${lane.predicate}
-            AND (cancellation_id IS NOT NULL OR NOT EXISTS (
-              SELECT 1 FROM watchdog_jobs predecessor INDEXED BY idx_watchdog_queue_head
-              WHERE predecessor.queue_name = watchdog_jobs.queue_name
-                AND predecessor.state IN ('queued', 'running')
-                AND (predecessor.state = 'running' OR
-                  (predecessor.state = 'queued' AND
-                    (predecessor.priority > watchdog_jobs.priority OR
-                      (predecessor.priority = watchdog_jobs.priority AND
-                        predecessor.sequence < watchdog_jobs.sequence))))))
+            AND (cancellation_id IS NOT NULL OR not_before <= ?)
+            AND ${queuedHeadPredicate}
           ORDER BY priority DESC, sequence ASC
-          LIMIT 1`, ...lane.values)
+          LIMIT 1`, ...lane.values, Date.now())
             .toArray()[0];
         if (raw === undefined)
             return { status: "no_ready" };
@@ -310,13 +364,13 @@ function selectAndClaim(owner, project, claimToken, partition) {
         return { status: "claimed", job: claimed, claimToken };
     });
 }
-function settleExactClaim(owner, project, input) {
+function recordExecutionDecision(owner, input) {
     return owner.transactionSync(() => {
         const prior = readJob(owner, input.jobId);
         if (prior === undefined)
             return { status: "missing" };
         const outcome = prior.row.cancellation_id === null ? input.decision.kind : "cancelled";
-        if (prior.job.state === "settled")
+        if (prior.job.state === "completing" || prior.job.state === "settled")
             return prior.row.claim_token === input.claimToken &&
                 prior.job.outcome === outcome
                 ? { status: "replayed", job: prior.job }
@@ -324,11 +378,28 @@ function settleExactClaim(owner, project, input) {
         if (prior.job.state !== "running" ||
             prior.row.claim_token !== input.claimToken)
             return { status: "conflict" };
-        owner.sql.exec(`UPDATE watchdog_jobs
-          SET state = 'settled', outcome = ?, failure_reason = ?
-        WHERE job_id = ? AND state = 'running' AND claim_token = ?`, outcome, outcome === "failed" && input.decision.kind === "failed"
+        owner.sql.exec(`UPDATE watchdog_jobs SET state = 'completing', outcome = ?, failure_reason = ?
+       WHERE job_id = ? AND state = 'running' AND claim_token = ?`, outcome, outcome === "failed" && input.decision.kind === "failed"
             ? (input.decision.reason ?? null)
             : null, input.jobId, input.claimToken);
+        const decided = readJob(owner, input.jobId)?.job;
+        if (decided?.state !== "completing")
+            return { status: "conflict" };
+        return { status: "recorded", job: decided };
+    });
+}
+function finishCompletingClaim(owner, project, input) {
+    return owner.transactionSync(() => {
+        const prior = readJob(owner, input.jobId);
+        if (prior === undefined)
+            return { status: "missing" };
+        if (prior.row.claim_token !== input.claimToken)
+            return { status: "conflict" };
+        if (prior.job.state === "settled")
+            return { status: "replayed", job: prior.job };
+        if (prior.job.state !== "completing")
+            return { status: "conflict" };
+        owner.sql.exec("UPDATE watchdog_jobs SET state = 'settled' WHERE job_id = ? AND state = 'completing' AND claim_token = ?", input.jobId, input.claimToken);
         const settled = readJob(owner, input.jobId)?.job;
         if (settled?.state !== "settled")
             return { status: "conflict" };
@@ -340,6 +411,19 @@ function settleExactClaim(owner, project, input) {
 export function openSqliteState(owner, project, partition) {
     prepare(owner);
     const transactional = Object.freeze({
+        wakeState: (now) => readWakeState(owner, now),
+        readJob: (jobId) => {
+            const parsed = readJob(owner, jobId);
+            return parsed === undefined
+                ? undefined
+                : { state: parsed.row.state, notBefore: parsed.row.not_before };
+        },
+        expediteQueuedJob: (jobId, now) => {
+            parseWakeTime(now);
+            return (owner.sql
+                .exec("UPDATE watchdog_jobs SET not_before=? WHERE job_id=? AND state='queued' AND not_before>? RETURNING job_id", now, jobId, now)
+                .toArray().length === 1);
+        },
         readQueueHead: (queue) => readQueueHead(owner, queue),
         readLatestSettledJob: (queue) => readLatestSettledJob(owner, queue),
         enqueueAcceptedJob: (job) => {
@@ -350,9 +434,7 @@ export function openSqliteState(owner, project, partition) {
     const createStore = (partition) => {
         const lane = laneQuery(partition);
         return Object.freeze({
-            hasPendingWork: storeEffect(() => owner.sql
-                .exec("SELECT 1 FROM watchdog_jobs WHERE state IN ('queued', 'running') LIMIT 1")
-                .toArray().length !== 0),
+            wakeState: (now) => storeEffect(() => readWakeState(owner, now)),
             ingest: (job) => storeEffect(() => owner.transactionSync(() => ({
                 status: enqueue(owner, project, job) ? "queued" : "duplicate",
             }))),
@@ -377,12 +459,51 @@ export function openSqliteState(owner, project, partition) {
                 };
             }),
             readQueueHead: (queue) => storeEffect(() => readQueueHead(owner, queue)),
-            readSettled: (limit, afterSequence) => storeEffect(() => readSettled(owner, limit, afterSequence, partition)),
+            readSettled: (limit, afterSequence, selectedLane) => storeEffect(() => readSettled(owner, limit, afterSequence, partition, selectedLane)),
             selectAndClaim: (claimToken) => storeEffect(() => selectAndClaim(owner, project, claimToken, partition)),
-            settleExactClaim: (input) => storeEffect(() => settleExactClaim(owner, project, input)),
+            reserveAttempt: (input) => storeEffect(() => owner.transactionSync(() => {
+                const prior = readJob(owner, input.jobId);
+                if (prior?.job.state !== "running" ||
+                    prior.row.claim_token !== input.claimToken ||
+                    prior.row.cancellation_id !== null)
+                    throw new WatchdogError("claim_unavailable");
+                const reserved = reserveRetry(prior.row.attempt_count, Date.now(), input.policy);
+                owner.sql.exec("UPDATE watchdog_jobs SET attempt_count=?, not_before=? WHERE job_id=? AND state='running' AND claim_token=?", reserved.attempt.count, reserved.notBefore, input.jobId, input.claimToken);
+                return reserved.attempt;
+            })),
+            retryExactClaim: (input) => storeEffect(() => owner.transactionSync(() => {
+                const prior = readJob(owner, input.jobId);
+                if (prior?.job.state !== "running" ||
+                    prior.row.claim_token !== input.claimToken)
+                    return { status: "conflict" };
+                if (prior.row.cancellation_id !== null)
+                    return { status: "cancelled" };
+                const due = input.policy === undefined
+                    ? prior.row.not_before
+                    : retryDue(prior.row.attempt_count, Date.now(), input.policy);
+                owner.sql.exec("UPDATE watchdog_jobs SET state='queued', claim_token=NULL, not_before=? WHERE job_id=? AND state='running' AND claim_token=?", due, input.jobId, input.claimToken);
+                projectQueue(owner, project, prior.job.queue);
+                return { status: "requeued" };
+            })),
+            recordExecutionDecision: (input) => storeEffect(() => recordExecutionDecision(owner, input)),
+            nextCompletingClaim: () => storeEffect(() => {
+                const raw = owner.sql
+                    .exec(`SELECT ${columns} FROM watchdog_jobs WHERE state = 'completing' AND ${lane.predicate} ORDER BY sequence LIMIT 1`, ...lane.values)
+                    .toArray()[0];
+                if (raw === undefined)
+                    return undefined;
+                const parsed = parseJobRow(raw);
+                if (parsed.job.state !== "completing" ||
+                    parsed.row.claim_token === null)
+                    throw unavailable();
+                return { job: parsed.job, claimToken: parsed.row.claim_token };
+            }),
+            finishCompletingClaim: (input) => storeEffect(() => finishCompletingClaim(owner, project, input)),
             recordCancellation: (intent) => storeEffect(() => owner.transactionSync(() => {
                 const result = recordCancellationIntent(owner, project, intent);
-                return { status: result.status };
+                return result.status === "decided"
+                    ? result
+                    : { status: result.status };
             })),
             failLane: (raw) => storeEffect(() => {
                 const input = parseLaneFailure(raw);
@@ -390,14 +511,14 @@ export function openSqliteState(owner, project, partition) {
                     // Under the declared state grammar, DESC puts running before queued.
                     const rows = owner.sql
                         .exec(`SELECT ${columns} FROM watchdog_jobs
-             WHERE lane = ? AND state IN ('queued', 'running')
+             WHERE lane = ? AND state IN ('queued', 'running', 'completing')
              ORDER BY state DESC, sequence ASC LIMIT ?`, input.lane, laneFailureBatchSize)
                         .toArray();
                     const jobs = [];
                     for (const row of rows) {
                         const prior = parseJobRow(row);
                         owner.sql.exec(`UPDATE watchdog_jobs SET state = 'settled', outcome = 'failed', failure_reason = ?, claim_token = NULL
-               WHERE job_id = ? AND state IN ('queued', 'running')`, input.reason, prior.job.jobId);
+               WHERE job_id = ? AND state IN ('queued', 'running', 'completing')`, input.reason, prior.job.jobId);
                         const settled = readJob(owner, prior.job.jobId)?.job;
                         if (settled?.state !== "settled" || settled.outcome !== "failed")
                             throw unavailable();
@@ -405,7 +526,7 @@ export function openSqliteState(owner, project, partition) {
                         jobs.push(settled);
                     }
                     const more = owner.sql
-                        .exec("SELECT 1 FROM watchdog_jobs WHERE lane = ? AND state IN ('queued', 'running') LIMIT 1", input.lane)
+                        .exec("SELECT 1 FROM watchdog_jobs WHERE lane = ? AND state IN ('queued', 'running', 'completing') LIMIT 1", input.lane)
                         .toArray().length !== 0;
                     return { jobs, more };
                 });
@@ -417,6 +538,19 @@ export function openSqliteState(owner, project, partition) {
                     projectQueue(owner, project, queue);
                 });
             }),
+            handoffExactClaim: (input) => storeEffect(() => owner.transactionSync(() => {
+                const prior = readJob(owner, input.jobId);
+                if (prior === undefined)
+                    return { status: "missing" };
+                if (prior.job.state !== "running" ||
+                    prior.row.claim_token !== input.claimToken)
+                    return { status: "conflict" };
+                if (prior.row.cancellation_id !== null)
+                    return { status: "cancelled" };
+                owner.sql.exec("UPDATE watchdog_jobs SET state = 'queued', claim_token = NULL WHERE job_id = ? AND state = 'running' AND claim_token = ?", input.jobId, input.claimToken);
+                projectQueue(owner, project, prior.job.queue);
+                return { status: "requeued" };
+            })),
             nextRunningClaim: () => storeEffect(() => {
                 const raw = owner.sql
                     .exec(`SELECT ${columns}
@@ -444,12 +578,19 @@ export function openSqliteState(owner, project, partition) {
                     prior.row.claim_token !== input.claimToken ||
                     prior.row.recovery_count !== input.recoveryCount)
                     return { status: "conflict" };
-                if (input.disposition === "requeue" &&
+                const recovery = input.requeue
+                    ? {
+                        kind: "requeue",
+                        recoveryCount: prior.row.recovery_count,
+                        windowStartedAt: prior.row.recovery_window_started_at,
+                    }
+                    : disappearanceRecovery(prior.job.recovery, prior.row.recovery_count, prior.row.recovery_window_started_at, Date.now());
+                if (recovery.kind === "requeue" &&
                     prior.row.cancellation_id === null) {
                     owner.sql.exec(`UPDATE watchdog_jobs
                   SET state = 'queued', claim_token = NULL,
-                      recovery_count = recovery_count + 1
-                WHERE job_id = ? AND state = 'running' AND claim_token = ?`, input.job.jobId, input.claimToken);
+                      recovery_count = ?, recovery_window_started_at = ?
+                WHERE job_id = ? AND state = 'running' AND claim_token = ?`, recovery.recoveryCount, recovery.windowStartedAt, input.job.jobId, input.claimToken);
                     projectQueue(owner, project, input.job.queue);
                     return { status: "requeued" };
                 }

package/dist/internal/watchdog-error.d.ts

@@ -1,9 +1,9 @@
 /** Typed failures owned by the generic Watchdog engine. */
 export declare class WatchdogError extends Error {
     readonly _tag: "WatchdogError";
-    readonly cause: "store_unavailable" | "store_invalid" | "claim_unavailable" | "execution_failed" | "settlement_rejected" | "wake_unavailable" | "liveness_unavailable";
+    readonly cause: "store_unavailable" | "store_invalid" | "claim_unavailable" | "drain_failed" | "execution_failed" | "settlement_rejected" | "wake_unavailable" | "liveness_unavailable";
     readonly underlying?: unknown;
-    constructor(failureKind: "store_unavailable" | "store_invalid" | "claim_unavailable" | "execution_failed" | "settlement_rejected" | "wake_unavailable" | "liveness_unavailable", 
+    constructor(failureKind: "store_unavailable" | "store_invalid" | "claim_unavailable" | "drain_failed" | "execution_failed" | "settlement_rejected" | "wake_unavailable" | "liveness_unavailable", 
     /**
      * The original thrown value, kept for server-side diagnosis only. `cause`
      * is already the closed category above, so this is a separate field.

package/dist/postgres.js

@@ -5,6 +5,7 @@ export const watchdogPostgresSchema = schemaStatements(true);
 /** Open the shared Effect engine on a migrated PostgreSQL database. */
 export function createPostgresWatchdog(options) {
     const runtime = createAsyncRuntime({
+        dialect: "postgres",
         clock: options.now === undefined
             ? "EXTRACT(EPOCH FROM clock_timestamp())*1000"
             : undefined,

package/docs/api/d1.md

@@ -65,6 +65,7 @@ Parameterized native purge; caller commits it with its own erasure facts.
 ```ts
 type AsyncWatchdogOptions = Readonly<{
   execute: WatchdogExecutor;
+  retryPolicy?: WatchdogOptions["retryPolicy"];
   wake: Readonly<{
     recompute: () => Promise<void>;
   }>;
@@ -107,9 +108,9 @@ writes.
 
 #### Parameters
 
-| Parameter | Type                                                                                                                                                                                                                         |
-| --------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `options` | `Readonly`\<\{ `execute`: [`WatchdogExecutor`](index.md#watchdogexecutor); `wake`: `Readonly`\<\{ `recompute`: () => `Promise`\<`void`\>; \}\>; `now?`: () => `number`; \}\> & `Readonly`\<\{ `database`: `D1Database`; \}\> |
+| Parameter | Type                                                                                                                                                                                                                                                                                                                                                                                                 |
+| --------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `options` | `Readonly`\<\{ `execute`: [`WatchdogExecutor`](index.md#watchdogexecutor); `retryPolicy?`: (`job`) => `Promise`\< \| `Readonly`\<\{ `baseMs`: `number`; `maxMs`: `number`; `limit`: `Readonly`\<...\> \| `Readonly`\<...\>; \}\> \| `undefined`\>; `wake`: `Readonly`\<\{ `recompute`: () => `Promise`\<`void`\>; \}\>; `now?`: () => `number`; \}\> & `Readonly`\<\{ `database`: `D1Database`; \}\> |
 
 #### Returns