@fungi.computer/watchdog 0.1.0 → 0.1.1

package/dist/index.d.ts

@@ -1,9 +1,9 @@
 import * as core from "./effect.js";
 export { WatchdogError } from "./internal/watchdog-error.js";
-export { WatchdogCancellationIntentSchema, WatchdogCancellationResultSchema, WatchdogExecutionDecisionSchema, WatchdogIngestResultSchema, WatchdogQueuedJobSchema, WatchdogQueueHeadSchema, WatchdogReadResultSchema, WatchdogRunningJobSchema, WatchdogSettledJobSchema, WatchdogStoredJobSchema, } from "./effect.js";
-export type { EffectWatchdogOptions, EffectWatchdogExecutor, WatchdogCancellationDecision, WatchdogCancellationIntent, WatchdogCancellationResult, WatchdogExecutionDecision, WatchdogIngestResult, WatchdogLivenessUnknown, WatchdogLaneFailure, WatchdogLaneFailureResult, WatchdogQueuedJob, WatchdogQueueHead, WatchdogQueueProjector, WatchdogQueueTransition, WatchdogReadResult, WatchdogRunningJob, WatchdogRuntimeEffect, ConcurrentWatchdogRuntimeEffect, WatchdogSettledJob, WatchdogSqliteCursor, WatchdogSqliteOwner, WatchdogSqliteValue, WatchdogSqliteRow, WatchdogStoredJob, WatchdogTickResult, WatchdogTransactionalProjection, } from "./effect.js";
+export { WatchdogCompletingJobSchema, WatchdogCancellationIntentSchema, WatchdogCancellationResultSchema, WatchdogExecutionDecisionSchema, WatchdogIngestResultSchema, WatchdogQueuedJobSchema, WatchdogQueueHeadSchema, WatchdogReadResultSchema, WatchdogRunningJobSchema, WatchdogSettledJobSchema, WatchdogStoredJobSchema, } from "./effect.js";
+export type { EffectWatchdogOptions, EffectWatchdogExecutor, WatchdogCompletingJob, WatchdogCancellationDecision, WatchdogCancellationIntent, WatchdogCancellationResult, WatchdogExecutionDecision, WatchdogIngestResult, WatchdogLivenessUnknown, WatchdogLaneFailure, WatchdogLaneFailureResult, WatchdogQueuedJob, WatchdogQueueHead, WatchdogQueueProjector, WatchdogQueueTransition, WatchdogReadResult, WatchdogRunningJob, WatchdogRuntimeEffect, ConcurrentWatchdogRuntimeEffect, WatchdogSettledJob, WatchdogSqliteCursor, WatchdogSqliteOwner, WatchdogSqliteValue, WatchdogSqliteRow, WatchdogStoredJob, WatchdogTickResult, WatchdogTransactionalProjection, WatchdogWakeState, WatchdogRetryPolicy, WatchdogAttempt, } from "./effect.js";
 /** Promise host executor for one generic running job. */
-export type WatchdogExecutor = (job: core.WatchdogRunningJob, signal: AbortSignal) => Promise<core.WatchdogExecutionDecision>;
+export type WatchdogExecutor = (job: core.WatchdogRunningJob, signal: AbortSignal, attempt: core.WatchdogAttempt | undefined) => Promise<core.WatchdogExecutionDecision>;
 /** Options for constructing the Promise facade. */
 export type WatchdogOptions = Readonly<{
     /** Optional opaque lane with its own single-flight execution slot. */
@@ -14,6 +14,8 @@ export type WatchdogOptions = Readonly<{
     isAlive: (job: core.WatchdogRunningJob) => Promise<boolean>;
     /** Executes one generic claimed job. */
     execute: WatchdogExecutor;
+    /** Resolve the code-owned delivery policy before reserving a receiver attempt. */
+    retryPolicy?: (job: core.WatchdogRunningJob) => Promise<core.WatchdogRetryPolicy | undefined>;
     /** Recomputes the owner wake after durable state changes. */
     wake: Readonly<{
         /** Arms the next wake from current owner state. */
@@ -24,8 +26,8 @@ export type WatchdogOptions = Readonly<{
 }>;
 /** Promise facade over the canonical Effect runtime. */
 export type WatchdogRuntime = Readonly<{
-    /** Whether any queued or running job still needs owner reconciliation. */
-    hasPendingWork: () => Promise<boolean>;
+    /** Read readiness and the next deadline from the same durable work rows. */
+    wakeState: (now: number) => Promise<core.WatchdogWakeState>;
     /** Persists one queued job. */
     ingest: (job: core.WatchdogQueuedJob) => Promise<core.WatchdogIngestResult>;
     /** Reads one durable job. */

package/dist/index.js

@@ -6,7 +6,7 @@ import * as core from "./effect.js";
 import { adaptPromiseWatchdogHost } from "./internal/promise-watchdog-host.js";
 import { WatchdogError } from "./internal/watchdog-error.js";
 export { WatchdogError } from "./internal/watchdog-error.js";
-export { WatchdogCancellationIntentSchema, WatchdogCancellationResultSchema, WatchdogExecutionDecisionSchema, WatchdogIngestResultSchema, WatchdogQueuedJobSchema, WatchdogQueueHeadSchema, WatchdogReadResultSchema, WatchdogRunningJobSchema, WatchdogSettledJobSchema, WatchdogStoredJobSchema, } from "./effect.js";
+export { WatchdogCompletingJobSchema, WatchdogCancellationIntentSchema, WatchdogCancellationResultSchema, WatchdogExecutionDecisionSchema, WatchdogIngestResultSchema, WatchdogQueuedJobSchema, WatchdogQueueHeadSchema, WatchdogReadResultSchema, WatchdogRunningJobSchema, WatchdogSettledJobSchema, WatchdogStoredJobSchema, } from "./effect.js";
 const runPromise = (effect) => Effect.runPromiseExit(effect).then((exit) => {
     if (Exit.isSuccess(exit))
         return exit.value;
@@ -25,7 +25,7 @@ function make(options) {
 }
 function promiseRuntime(runtime) {
     return Object.freeze({
-        hasPendingWork: () => runPromise(runtime.hasPendingWork),
+        wakeState: (now) => runPromise(runtime.wakeState(now)),
         ingest: (job) => runPromise(runtime.ingest(job)),
         read: (jobId) => runPromise(runtime.read(jobId)),
         readQueueHead: (queue) => runPromise(runtime.readQueueHead(queue)),

package/dist/internal/async-runtime.d.ts

@@ -1,9 +1,11 @@
-import type { WatchdogExecutor, WatchdogRuntime } from "../index.js";
+import type { WatchdogExecutor, WatchdogRuntime, WatchdogOptions } from "../index.js";
 import type { SqlDatabase } from "./async-sql.js";
 /** Host execution and wake scheduling for a shared-database Watchdog. */
 export type AsyncWatchdogOptions = Readonly<{
     /** Execute one claimed job; honour abort when its lease is lost. */
     execute: WatchdogExecutor;
+    /** Resolve the delivery owner policy before receiver attempt admission. */
+    retryPolicy?: WatchdogOptions["retryPolicy"];
     /** Recompute a durable wake after a transition commits. */
     wake: Readonly<{
         /** Arm the next durable owner wake. */

package/dist/internal/async-runtime.js

@@ -19,15 +19,24 @@ export function createAsyncRuntime(database, options) {
     const runtime = bindWatchdogEngine({
         isAlive: (job) => state.isAlive(job.jobId),
         executor: {
-            execute: (job) => renew(job.jobId).pipe(Effect.andThen(Effect.raceFirst(Effect.tryPromise({
-                try: (signal) => options.execute(job, signal),
+            retryPolicy: (job) => {
+                const policy = options.retryPolicy;
+                return policy === undefined
+                    ? Effect.succeed(undefined)
+                    : Effect.tryPromise({
+                        try: () => policy(job),
+                        catch: (error) => new WatchdogError("execution_failed", error),
+                    });
+            },
+            execute: (job, reserved) => renew(job.jobId).pipe(Effect.andThen(Effect.raceFirst(Effect.tryPromise({
+                try: (signal) => options.execute(job, signal, reserved),
                 catch: () => new WatchdogError("execution_failed"),
             }), Effect.forever(Effect.sleep(heartbeatMs).pipe(Effect.andThen(renew(job.jobId))))))),
         },
         wake: { recompute: () => attempt(options.wake.recompute) },
     }, state.store);
     return {
-        hasPendingWork: () => run(runtime.hasPendingWork),
+        wakeState: (now) => run(runtime.wakeState(now)),
         ingest: (job) => run(runtime.ingest(job)),
         read: (id) => run(runtime.read(id)),
         readQueueHead: (queue) => run(runtime.readQueueHead(queue)),

package/dist/internal/async-sql.d.ts

@@ -6,10 +6,11 @@ export type Statement = Readonly<{
     values: readonly SqlValue[];
 }>;
 export type SqlDatabase = Readonly<{
+    dialect: "sqlite" | "postgres";
     clock?: string | undefined;
     query: (statement: Statement) => Promise<readonly SqlRow[]>;
 }>;
-export declare const columns = "sequence, job_id, queue_name, lane, priority, payload_json,\n  recovery_max, state, outcome, claim_token, recovery_count, cancellation_id, failure_reason";
+export declare const columns = "sequence, job_id, queue_name, lane, priority, payload_json, not_before, attempt_count,\n  recovery_max, recovery_window_ms, recovery_window_started_at, state, outcome, claim_token, recovery_count, cancellation_id, failure_reason";
 export declare function schemaStatements(postgres: boolean): readonly string[];
 export declare function enqueueSelect(source: Statement): Statement;
 export declare function enqueueStatement(input: WatchdogQueuedJob): Statement;

package/dist/internal/async-sql.js

@@ -1,8 +1,8 @@
 import * as Schema from "effect/Schema";
 import { WatchdogQueuedJobSchema } from "../effect.js";
 import { laneMaxLength } from "./lane.js";
-export const columns = `sequence, job_id, queue_name, lane, priority, payload_json,
-  recovery_max, state, outcome, claim_token, recovery_count, cancellation_id, failure_reason`;
+export const columns = `sequence, job_id, queue_name, lane, priority, payload_json, not_before, attempt_count,
+  recovery_max, recovery_window_ms, recovery_window_started_at, state, outcome, claim_token, recovery_count, cancellation_id, failure_reason`;
 export function schemaStatements(postgres) {
     return [
         `CREATE TABLE IF NOT EXISTS watchdog_jobs (
@@ -12,20 +12,25 @@ export function schemaStatements(postgres) {
       lane TEXT NOT NULL CHECK(length(lane) BETWEEN 1 AND ${String(laneMaxLength)}),
       priority ${postgres ? "DOUBLE PRECISION" : "INTEGER"} NOT NULL,
       payload_json TEXT NOT NULL,
+      not_before ${postgres ? "DOUBLE PRECISION" : "INTEGER"} NOT NULL DEFAULT 0 CHECK(not_before>=0),
+      attempt_count ${postgres ? "DOUBLE PRECISION" : "INTEGER"} NOT NULL DEFAULT 0 CHECK(attempt_count>=0),
       recovery_max ${postgres ? "DOUBLE PRECISION" : "INTEGER"} CHECK(recovery_max IS NULL OR recovery_max >= 0),
-      state TEXT NOT NULL CHECK(state IN ('queued','running','settled')),
+      recovery_window_ms ${postgres ? "DOUBLE PRECISION" : "INTEGER"} CHECK(recovery_window_ms IS NULL OR recovery_window_ms > 0),
+      recovery_window_started_at ${postgres ? "DOUBLE PRECISION" : "INTEGER"},
+      state TEXT NOT NULL CHECK(state IN ('queued','running','completing','settled')),
       outcome TEXT CHECK(outcome IN ('completed','failed','cancelled','outcome_unknown','interrupted')),
       claim_token TEXT,
       recovery_count ${postgres ? "DOUBLE PRECISION" : "INTEGER"} NOT NULL DEFAULT 0,
       cancellation_id TEXT UNIQUE,
       failure_reason TEXT,
       lease_until ${postgres ? "DOUBLE PRECISION" : "INTEGER"},
-      CHECK ((state='queued' AND outcome IS NULL AND claim_token IS NULL) OR
+      CONSTRAINT watchdog_execution_state CHECK ((state='queued' AND outcome IS NULL AND claim_token IS NULL) OR
         (state='running' AND outcome IS NULL AND claim_token IS NOT NULL AND lease_until IS NOT NULL) OR
+        (state='completing' AND outcome IS NOT NULL AND claim_token IS NOT NULL) OR
         (state='settled' AND outcome IS NOT NULL))
     )`,
         "CREATE INDEX IF NOT EXISTS idx_watchdog_lane_failure ON watchdog_jobs(lane,state DESC,sequence ASC)",
-        "CREATE INDEX IF NOT EXISTS idx_watchdog_ready ON watchdog_jobs(state,priority DESC,sequence)",
+        "CREATE INDEX IF NOT EXISTS idx_watchdog_ready ON watchdog_jobs(state,not_before,priority DESC,sequence)",
         "CREATE INDEX IF NOT EXISTS idx_watchdog_queue_head ON watchdog_jobs(queue_name,state,priority DESC,sequence)",
         "CREATE INDEX IF NOT EXISTS idx_watchdog_lease ON watchdog_jobs(state,lease_until,sequence)",
     ];
@@ -36,14 +41,16 @@ export function schemaStatements(postgres) {
 export function enqueueSelect(source) {
     return {
         text: `INSERT INTO watchdog_jobs
-      (job_id,queue_name,lane,priority,payload_json,recovery_max,state)
-      SELECT job_id,queue_name,lane,priority,payload_json,recovery_max,'queued'
+      (job_id,queue_name,lane,priority,payload_json,recovery_max,recovery_window_ms,not_before,state)
+      SELECT job_id,queue_name,lane,priority,payload_json,recovery_max,recovery_window_ms,not_before,'queued'
       FROM (${source.text}) AS admission WHERE true
       ON CONFLICT(job_id) DO UPDATE SET job_id = NULL WHERE NOT (
         watchdog_jobs.queue_name=excluded.queue_name AND watchdog_jobs.lane=excluded.lane AND
         watchdog_jobs.priority=excluded.priority AND watchdog_jobs.payload_json=excluded.payload_json AND
         (watchdog_jobs.recovery_max=excluded.recovery_max OR
-          (watchdog_jobs.recovery_max IS NULL AND excluded.recovery_max IS NULL))
+          (watchdog_jobs.recovery_max IS NULL AND excluded.recovery_max IS NULL)) AND
+        (watchdog_jobs.recovery_window_ms=excluded.recovery_window_ms OR
+          (watchdog_jobs.recovery_window_ms IS NULL AND excluded.recovery_window_ms IS NULL))
         )
       RETURNING ${columns}`,
         values: source.values,
@@ -53,7 +60,7 @@ export function enqueueStatement(input) {
     const job = Schema.decodeUnknownSync(WatchdogQueuedJobSchema)(input);
     return enqueueSelect({
         text: `SELECT CAST($1 AS TEXT) AS job_id,CAST($2 AS TEXT) AS queue_name,CAST($3 AS TEXT) AS lane,CAST($4 AS DOUBLE PRECISION) AS priority,
-      CAST($5 AS TEXT) AS payload_json,CAST($6 AS DOUBLE PRECISION) AS recovery_max`,
+      CAST($5 AS TEXT) AS payload_json,CAST($6 AS DOUBLE PRECISION) AS recovery_max,CAST($7 AS DOUBLE PRECISION) AS recovery_window_ms,CAST($8 AS DOUBLE PRECISION) AS not_before`,
         values: [
             job.jobId,
             job.queue,
@@ -61,6 +68,8 @@ export function enqueueStatement(input) {
             job.priority,
             Schema.encodeSync(Schema.fromJsonString(Schema.Unknown))(job.payload),
             job.recovery?.maxRecoveries ?? null,
+            job.recovery?.windowMs ?? null,
+            job.notBefore ?? 0,
         ],
     });
 }

package/dist/internal/async-store.d.ts

@@ -5,7 +5,7 @@ export declare const heartbeatMs = 10000;
 export declare const attempt: <A>(run: () => Promise<A>) => Effect.Effect<A, WatchdogError>;
 export declare function openAsyncStore(database: SqlDatabase, now: () => number): {
     store: Readonly<{
-        hasPendingWork: Effect.Effect<boolean, WatchdogError>;
+        wakeState: (now: number) => Effect.Effect<import("../effect.js").WatchdogWakeState, WatchdogError>;
         ingest: (job: import("../effect.js").WatchdogQueuedJob) => Effect.Effect<import("../effect.js").WatchdogIngestResult, WatchdogError>;
         read: (jobId: string) => Effect.Effect<(Extract<import("../effect.js").WatchdogReadResult, {
             status: "found";
@@ -16,24 +16,57 @@ export declare function openAsyncStore(database: SqlDatabase, now: () => number)
             status: "missing";
         }>, WatchdogError>;
         readQueueHead: (queue: string) => Effect.Effect<import("../effect.js").WatchdogQueueHead, WatchdogError>;
-        readSettled: (limit: number, afterSequence?: number) => Effect.Effect<readonly import("../effect.js").WatchdogSettledRecord[], WatchdogError>;
+        readSettled: (limit: number, afterSequence?: number, lane?: string) => Effect.Effect<readonly import("../effect.js").WatchdogSettledRecord[], WatchdogError>;
         selectAndClaim: (claimToken: string) => Effect.Effect<import("./sqlite-store.js").WatchdogSelection, WatchdogError>;
-        settleExactClaim: (input: Readonly<{
+        reserveAttempt: (input: {
             jobId: string;
             claimToken: string;
-            decision: import("../effect.js").WatchdogExecutionDecision;
+            policy: import("../effect.js").WatchdogRetryPolicy;
+        }) => Effect.Effect<import("../effect.js").WatchdogAttempt, WatchdogError>;
+        retryExactClaim: (input: {
+            jobId: string;
+            claimToken: string;
+            policy?: import("../effect.js").WatchdogRetryPolicy;
+        }) => Effect.Effect<{
+            status: "requeued" | "cancelled" | "conflict";
+        }, WatchdogError>;
+        recordExecutionDecision: (input: Readonly<{
+            jobId: string;
+            claimToken: string;
+            decision: Exclude<import("../effect.js").WatchdogExecutionDecision, {
+                kind: "retry";
+            }>;
         }>) => Effect.Effect<Readonly<{
-            status: "settled" | "replayed";
-            job: import("../effect.js").WatchdogSettledJob;
+            status: "recorded" | "replayed";
+            job: import("../effect.js").WatchdogCompletingJob | import("../effect.js").WatchdogSettledJob;
         }> | Readonly<{
             status: "conflict" | "missing";
         }>, WatchdogError>;
+        nextCompletingClaim: () => Effect.Effect<{
+            job: import("../effect.js").WatchdogCompletingJob;
+            claimToken: string;
+        } | undefined, WatchdogError>;
+        finishCompletingClaim: (input: {
+            jobId: string;
+            claimToken: string;
+        }) => Effect.Effect<{
+            status: "settled" | "replayed";
+            job: import("../effect.js").WatchdogSettledJob;
+        } | {
+            status: "conflict" | "missing";
+        }, WatchdogError>;
         recordCancellation: (input: import("../effect.js").WatchdogCancellationIntent) => Effect.Effect<import("../effect.js").WatchdogCancellationResult, WatchdogError>;
         failLane: (input: import("../effect.js").WatchdogLaneFailure) => Effect.Effect<Readonly<{
             jobs: readonly import("../effect.js").WatchdogSettledJob[];
             more: boolean;
         }>, WatchdogError>;
         purgeQueued: (queue: string) => Effect.Effect<void, WatchdogError>;
+        handoffExactClaim: (input: {
+            jobId: string;
+            claimToken: string;
+        }) => Effect.Effect<{
+            status: "requeued" | "cancelled" | "conflict" | "missing";
+        }, WatchdogError>;
         nextRunningClaim: () => Effect.Effect<Readonly<{
             job: import("../effect.js").WatchdogRunningJob;
             claimToken: string;
@@ -43,7 +76,7 @@ export declare function openAsyncStore(database: SqlDatabase, now: () => number)
             job: import("../effect.js").WatchdogRunningJob;
             claimToken: string;
             recoveryCount: number;
-            disposition: "requeue" | "interrupted";
+            requeue?: boolean;
         }>) => Effect.Effect<Readonly<{
             status: "requeued";
         }> | Readonly<{

package/dist/internal/async-store.js

@@ -1,6 +1,9 @@
+import { laneSchema } from "./lane.js";
+import { reserveRetry, retryDue } from "./retry.js";
+import { disappearanceRecovery } from "./recovery.js";
 import * as Effect from "effect/Effect";
 import * as Schema from "effect/Schema";
-import { parseJobRow, parseLaneFailure, laneFailureBatchSize, } from "./sqlite-store.js";
+import { parseJobRow, queuedHeadPredicate, parseWakeState, parseWakeTime, parseLaneFailure, laneFailureBatchSize, } from "./sqlite-store.js";
 import { WatchdogError } from "./watchdog-error.js";
 import { columns, enqueueStatement, } from "./async-sql.js";
 const leaseMs = 120_000;
@@ -18,7 +21,30 @@ export function openAsyncStore(database, now) {
     const at = (parameter) => database.clock === undefined
         ? `CAST($${String(parameter)} AS DOUBLE PRECISION)`
         : `(CASE WHEN $${String(parameter)}>${database.clock} THEN $${String(parameter)} ELSE ${database.clock} END)`;
-    const query = (text, values = []) => database.query({ text, values });
+    let schemaChecked = false;
+    const query = async (text, values = []) => {
+        if (!schemaChecked) {
+            const current = await database.query({
+                text: database.dialect === "postgres"
+                    ? "SELECT 1 AS current FROM pg_constraint WHERE conrelid = 'watchdog_jobs'::regclass AND conname = 'watchdog_execution_state'"
+                    : "SELECT 1 AS current FROM sqlite_schema WHERE name = 'watchdog_jobs' AND instr(sql, 'watchdog_execution_state') > 0",
+                values: [],
+            });
+            if (current.length !== 1)
+                throw new WatchdogError("store_invalid", new Error("Obsolete Watchdog execution schema"));
+            try {
+                await database.query({
+                    text: `SELECT ${columns} FROM watchdog_jobs LIMIT 0`,
+                    values: [],
+                });
+            }
+            catch (error) {
+                throw new WatchdogError("store_invalid", error);
+            }
+            schemaChecked = true;
+        }
+        return database.query({ text, values });
+    };
     const row = async (id) => {
         const raw = (await query(`SELECT ${columns} FROM watchdog_jobs WHERE job_id=$1`, [id]))[0];
         return raw === undefined ? undefined : parseJobRow(raw);
@@ -26,12 +52,25 @@ export function openAsyncStore(database, now) {
     // Private to this runtime: claims never escape through a public job.
     const claims = new Map();
     const store = {
-        hasPendingWork: attempt(async () => (await query("SELECT job_id FROM watchdog_jobs WHERE state IN ('queued','running') LIMIT 1")).length > 0),
-        ingest: (job) => attempt(async () => ({
-            status: (await database.query(enqueueStatement(job))).length > 0
-                ? "queued"
-                : "duplicate",
-        })),
+        wakeState: (instant) => attempt(async () => {
+            parseWakeTime(instant);
+            return parseWakeState((await query(`SELECT
+        COALESCE(MAX(CASE WHEN state='completing' OR cancellation_id IS NOT NULL OR
+          (state='running' AND lease_until <= ${at(1)}) OR
+          (state='queued' AND not_before <= ${at(1)}) THEN 1 ELSE 0 END),0) AS pending,
+        MIN(CASE WHEN state='running' AND lease_until > ${at(1)} THEN lease_until
+          WHEN state='queued' AND cancellation_id IS NULL AND not_before > ${at(1)} THEN not_before END) AS next_wake_at
+        FROM watchdog_jobs WHERE state IN ('running','completing') OR
+          (state='queued' AND ${queuedHeadPredicate})`, [instant]))[0]);
+        }),
+        ingest: (job) => attempt(async () => {
+            const statement = enqueueStatement(job);
+            return {
+                status: (await query(statement.text, statement.values)).length > 0
+                    ? "queued"
+                    : "duplicate",
+            };
+        }),
         read: (id) => attempt(async () => {
             const observedClaim = claims.get(id);
             const found = await row(id);
@@ -60,7 +99,7 @@ export function openAsyncStore(database, now) {
         }),
         readQueueHead: (queue) => attempt(async () => {
             const raw = (await query(`SELECT ${columns} FROM watchdog_jobs WHERE queue_name=$1
-        AND state IN ('queued','running') ORDER BY CASE state WHEN 'running' THEN 0 ELSE 1 END,
+        AND state IN ('queued','running','completing') ORDER BY CASE state WHEN 'queued' THEN 1 ELSE 0 END,
         priority DESC,sequence LIMIT 1`, [queue]))[0];
             if (raw === undefined)
                 return { state: "idle" };
@@ -68,15 +107,21 @@ export function openAsyncStore(database, now) {
             if (found.job.state === "settled")
                 throw new WatchdogError("store_unavailable");
             return {
-                state: found.job.state,
+                state: found.job.state === "completing" ? "running" : found.job.state,
                 jobId: found.job.jobId,
                 cancellationRequested: found.row.cancellation_id !== null,
             };
         }),
-        readSettled: (limit, afterSequence = 0) => attempt(async () => {
+        readSettled: (limit, afterSequence = 0, lane) => attempt(async () => {
             Schema.decodeUnknownSync(Schema.Int.check(Schema.isGreaterThanOrEqualTo(1), Schema.isLessThanOrEqualTo(256)))(limit);
             const rows = await query(`SELECT ${columns} FROM watchdog_jobs WHERE state='settled'
-        AND sequence>$1 ORDER BY sequence LIMIT $2`, [afterSequence, limit]);
+        AND sequence>$1 ${lane === undefined ? "" : "AND lane=$3"} ORDER BY sequence LIMIT $2`, lane === undefined
+                ? [afterSequence, limit]
+                : [
+                    afterSequence,
+                    limit,
+                    Schema.decodeUnknownSync(laneSchema)(lane),
+                ]);
             return rows.map((raw) => {
                 const parsed = parseJobRow(raw);
                 if (parsed.job.state !== "settled")
@@ -91,6 +136,8 @@ export function openAsyncStore(database, now) {
         claim_token=CASE WHEN cancellation_id IS NULL THEN $1 ELSE NULL END,
         lease_until=CASE WHEN cancellation_id IS NULL THEN ${at(2)}+${String(leaseMs)} ELSE NULL END
         WHERE job_id=(SELECT job_id FROM watchdog_jobs WHERE state='queued'
+          AND (cancellation_id IS NOT NULL OR not_before<=${at(2)})
+          AND ${queuedHeadPredicate}
           ORDER BY priority DESC,sequence LIMIT 1) AND state='queued' RETURNING ${columns}`, [token, now()]))[0];
             if (raw === undefined)
                 return { status: "no_ready" };
@@ -102,9 +149,50 @@ export function openAsyncStore(database, now) {
             claims.set(parsed.job.jobId, token);
             return { status: "claimed", job: parsed.job, claimToken: token };
         }),
-        settleExactClaim: (input) => attempt(async () => {
+        reserveAttempt: (input) => attempt(async () => {
+            const prior = await row(input.jobId);
+            if (prior?.job.state !== "running" ||
+                prior.row.claim_token !== input.claimToken ||
+                prior.row.cancellation_id !== null)
+                throw new WatchdogError("claim_unavailable");
+            const reserved = reserveRetry(prior.row.attempt_count, now(), input.policy);
+            const updated = await query(`UPDATE watchdog_jobs SET attempt_count=$1,not_before=$2 WHERE job_id=$3 AND state='running' AND claim_token=$4 AND attempt_count=$5 AND cancellation_id IS NULL AND lease_until>${at(6)} RETURNING job_id`, [
+                reserved.attempt.count,
+                reserved.notBefore,
+                input.jobId,
+                input.claimToken,
+                prior.row.attempt_count,
+                now(),
+            ]);
+            if (updated.length !== 1)
+                throw new WatchdogError("claim_unavailable");
+            return reserved.attempt;
+        }),
+        retryExactClaim: (input) => attempt(async () => {
+            const prior = await row(input.jobId);
+            if (prior?.job.state !== "running" ||
+                prior.row.claim_token !== input.claimToken)
+                return { status: "conflict" };
+            const due = input.policy === undefined
+                ? prior.row.not_before
+                : retryDue(prior.row.attempt_count, now(), input.policy);
+            const updated = await query(`UPDATE watchdog_jobs SET state='queued',claim_token=NULL,lease_until=NULL,not_before=$1 WHERE job_id=$2 AND state='running' AND claim_token=$3 AND cancellation_id IS NULL AND lease_until>${at(4)} RETURNING job_id`, [due, input.jobId, input.claimToken, now()]);
+            if (updated.length === 1) {
+                claims.delete(input.jobId);
+                return { status: "requeued" };
+            }
+            const current = await row(input.jobId);
+            return {
+                status: current?.job.state === "running" &&
+                    current.row.claim_token === input.claimToken &&
+                    current.row.cancellation_id !== null
+                    ? "cancelled"
+                    : "conflict",
+            };
+        }),
+        recordExecutionDecision: (input) => attempt(async () => {
             try {
-                const raw = (await query(`UPDATE watchdog_jobs SET state='settled',
+                const raw = (await query(`UPDATE watchdog_jobs SET state='completing',
           outcome=CASE WHEN cancellation_id IS NULL THEN $1 ELSE 'cancelled' END,
           failure_reason=CASE WHEN cancellation_id IS NULL THEN $2 ELSE NULL END,lease_until=NULL
           WHERE job_id=$3 AND state='running' AND claim_token=$4 AND lease_until>${at(5)} RETURNING ${columns}`, [
@@ -118,8 +206,8 @@ export function openAsyncStore(database, now) {
                 ]))[0];
                 if (raw !== undefined) {
                     const parsed = parseJobRow(raw);
-                    if (parsed.job.state === "settled")
-                        return { status: "settled", job: parsed.job };
+                    if (parsed.job.state === "completing")
+                        return { status: "recorded", job: parsed.job };
                 }
                 const prior = await row(input.jobId);
                 if (prior === undefined)
@@ -127,7 +215,8 @@ export function openAsyncStore(database, now) {
                 const outcome = prior.row.cancellation_id === null
                     ? input.decision.kind
                     : "cancelled";
-                return prior.job.state === "settled" &&
+                return (prior.job.state === "completing" ||
+                    prior.job.state === "settled") &&
                     prior.row.claim_token === input.claimToken &&
                     prior.job.outcome === outcome
                     ? { status: "replayed", job: prior.job }
@@ -137,13 +226,40 @@ export function openAsyncStore(database, now) {
                 claims.delete(input.jobId);
             }
         }),
+        nextCompletingClaim: () => attempt(async () => {
+            const raw = (await query(`SELECT ${columns} FROM watchdog_jobs WHERE state='completing' ORDER BY sequence LIMIT 1`))[0];
+            if (raw === undefined)
+                return undefined;
+            const found = parseJobRow(raw);
+            if (found.job.state !== "completing" || found.row.claim_token === null)
+                throw new WatchdogError("store_unavailable");
+            return { job: found.job, claimToken: found.row.claim_token };
+        }),
+        finishCompletingClaim: (input) => attempt(async () => {
+            const raw = (await query(`UPDATE watchdog_jobs SET state='settled' WHERE job_id=$1 AND state='completing' AND claim_token=$2 RETURNING ${columns}`, [input.jobId, input.claimToken]))[0];
+            if (raw !== undefined) {
+                const found = parseJobRow(raw);
+                if (found.job.state !== "settled")
+                    throw new WatchdogError("store_unavailable");
+                return { status: "settled", job: found.job };
+            }
+            const prior = await row(input.jobId);
+            if (prior === undefined)
+                return { status: "missing" };
+            return prior.job.state === "settled" &&
+                prior.row.claim_token === input.claimToken
+                ? { status: "replayed", job: prior.job }
+                : { status: "conflict" };
+        }),
         recordCancellation: (intent) => attempt(async () => {
             const updated = await query(`UPDATE watchdog_jobs SET cancellation_id=$1 WHERE job_id=$2
-        AND cancellation_id IS NULL ${intent.onlyIfQueued === true ? "AND state='queued'" : ""}
+        AND state IN ('queued','running') AND cancellation_id IS NULL ${intent.onlyIfQueued === true ? "AND state='queued'" : ""}
         RETURNING job_id`, [intent.cancellationId, intent.jobId]);
             if (updated.length > 0)
                 return { status: "recorded" };
             const prior = await row(intent.jobId);
+            if (prior?.job.state === "completing" || prior?.job.state === "settled")
+                return { status: "decided", outcome: prior.job.outcome };
             return {
                 status: prior === undefined
                     ? "missing"
@@ -156,9 +272,9 @@ export function openAsyncStore(database, now) {
             const input = parseLaneFailure(raw);
             // Under the declared state grammar, DESC puts running before queued.
             const rows = await query(`UPDATE watchdog_jobs SET state='settled', outcome='failed', failure_reason=$1, lease_until=NULL, claim_token=NULL
-         WHERE job_id IN (SELECT job_id FROM watchdog_jobs WHERE lane=$2 AND state IN ('queued','running')
+         WHERE job_id IN (SELECT job_id FROM watchdog_jobs WHERE lane=$2 AND state IN ('queued','running','completing')
            ORDER BY state DESC, sequence ASC LIMIT $3)
-           AND state IN ('queued','running') RETURNING ${columns}`, [input.reason, input.lane, laneFailureBatchSize]);
+           AND state IN ('queued','running','completing') RETURNING ${columns}`, [input.reason, input.lane, laneFailureBatchSize]);
             const jobs = rows.map((row) => {
                 const parsed = parseJobRow(row);
                 if (parsed.job.state !== "settled")
@@ -166,12 +282,28 @@ export function openAsyncStore(database, now) {
                 claims.delete(parsed.job.jobId);
                 return parsed.job;
             });
-            const more = (await query("SELECT 1 AS pending FROM watchdog_jobs WHERE lane=$1 AND state IN ('queued','running') LIMIT 1", [input.lane])).length !== 0;
+            const more = (await query("SELECT 1 AS pending FROM watchdog_jobs WHERE lane=$1 AND state IN ('queued','running','completing') LIMIT 1", [input.lane])).length !== 0;
             return { jobs, more };
         }),
         purgeQueued: (queue) => attempt(async () => {
             await query("DELETE FROM watchdog_jobs WHERE queue_name=$1 AND state='queued'", [queue]);
         }),
+        handoffExactClaim: (input) => attempt(async () => {
+            const changed = await query(`UPDATE watchdog_jobs SET state='queued',claim_token=NULL,lease_until=NULL
+        WHERE job_id=$1 AND state='running' AND claim_token=$2 AND lease_until>${at(3)} AND cancellation_id IS NULL RETURNING job_id`, [input.jobId, input.claimToken, now()]);
+            if (changed.length > 0) {
+                claims.delete(input.jobId);
+                return { status: "requeued" };
+            }
+            const prior = await row(input.jobId);
+            if (prior === undefined)
+                return { status: "missing" };
+            return prior.job.state === "running" &&
+                prior.row.claim_token === input.claimToken &&
+                prior.row.cancellation_id !== null
+                ? { status: "cancelled" }
+                : { status: "conflict" };
+        }),
         nextRunningClaim: () => attempt(async () => {
             const raw = (await query(`SELECT ${columns} FROM watchdog_jobs WHERE state='running'
         ORDER BY lease_until,sequence LIMIT 1`))[0];
@@ -187,7 +319,21 @@ export function openAsyncStore(database, now) {
             };
         }),
         reconcileDisappearedClaim: (input) => attempt(async () => {
-            const requeue = input.disposition === "requeue";
+            const prior = await row(input.job.jobId);
+            if (prior === undefined)
+                return { status: "missing" };
+            if (prior.job.state !== "running" ||
+                prior.row.claim_token !== input.claimToken ||
+                prior.row.recovery_count !== input.recoveryCount)
+                return { status: "conflict" };
+            const recovery = input.requeue
+                ? {
+                    kind: "requeue",
+                    recoveryCount: prior.row.recovery_count,
+                    windowStartedAt: prior.row.recovery_window_started_at,
+                }
+                : disappearanceRecovery(prior.job.recovery, prior.row.recovery_count, prior.row.recovery_window_started_at, now());
+            const requeue = recovery.kind === "requeue";
             const reason = input.job.recovery === undefined
                 ? "Execution disappeared without a recovery policy"
                 : `Restart recovery budget exhausted (${String(input.job.recovery.maxRecoveries)} recoveries)`;
@@ -195,15 +341,22 @@ export function openAsyncStore(database, now) {
         state=CASE WHEN $1=1 AND cancellation_id IS NULL THEN 'queued' ELSE 'settled' END,
         outcome=CASE WHEN cancellation_id IS NOT NULL THEN 'cancelled' WHEN $1=1 THEN NULL ELSE 'interrupted' END,
         claim_token=CASE WHEN $1=1 AND cancellation_id IS NULL THEN NULL ELSE claim_token END,
-        recovery_count=recovery_count+CASE WHEN $1=1 AND cancellation_id IS NULL THEN 1 ELSE 0 END,
+        recovery_count=CASE WHEN $1=1 AND cancellation_id IS NULL THEN $6 ELSE recovery_count END,
+        recovery_window_started_at=CASE WHEN $1=1 AND cancellation_id IS NULL THEN $7 ELSE recovery_window_started_at END,
         failure_reason=CASE WHEN $1=0 AND cancellation_id IS NULL THEN $2 ELSE NULL END,lease_until=NULL
-        WHERE job_id=$3 AND state='running' AND claim_token=$4 AND recovery_count=$5 AND lease_until<=${at(6)}
+        WHERE job_id=$3 AND state='running' AND claim_token=$4 AND recovery_count=$5 AND lease_until<=${at(8)}
         RETURNING ${columns}`, [
                 requeue ? 1 : 0,
                 reason,
                 input.job.jobId,
                 input.claimToken,
                 input.recoveryCount,
+                recovery.kind === "requeue"
+                    ? recovery.recoveryCount
+                    : input.recoveryCount,
+                recovery.kind === "requeue"
+                    ? recovery.windowStartedAt
+                    : prior.row.recovery_window_started_at,
                 now(),
             ]))[0];
             if (raw === undefined)

package/dist/internal/promise-watchdog-host.js

@@ -10,8 +10,17 @@ export const adaptPromiseWatchdogHost = (options) => {
             catch: (error) => new WatchdogError("liveness_unavailable", error),
         }),
         executor: {
-            execute: (job) => Effect.tryPromise({
-                try: (signal) => options.execute(job, signal),
+            retryPolicy: (job) => {
+                const policy = options.retryPolicy;
+                return policy === undefined
+                    ? Effect.succeed(undefined)
+                    : Effect.tryPromise({
+                        try: () => policy(job),
+                        catch: (error) => new WatchdogError("execution_failed", error),
+                    });
+            },
+            execute: (job, attempt) => Effect.tryPromise({
+                try: (signal) => options.execute(job, signal, attempt),
                 catch: () => new WatchdogError("execution_failed"),
             }),
         },

package/dist/internal/recovery.d.ts

@@ -0,0 +1,9 @@
+import type { WatchdogRunningJob } from "../effect.js";
+/** Decide one unexpected disappearance from its durable count and window. */
+export declare function disappearanceRecovery(policy: WatchdogRunningJob["recovery"], recoveryCount: number, windowStartedAt: number | null, now: number): Readonly<{
+    kind: "interrupted";
+}> | Readonly<{
+    kind: "requeue";
+    recoveryCount: number;
+    windowStartedAt: number | null;
+}>;