@fungi.computer/watchdog 0.1.0 → 0.1.1

package/dist/internal/recovery.js

@@ -0,0 +1,20 @@
+/** Decide one unexpected disappearance from its durable count and window. */
+export function disappearanceRecovery(policy, recoveryCount, windowStartedAt, now) {
+    if (policy === undefined)
+        return { kind: "interrupted" };
+    const expired = policy.windowMs !== undefined &&
+        windowStartedAt !== null &&
+        now - windowStartedAt >= policy.windowMs;
+    const count = expired ? 0 : recoveryCount;
+    if (count >= policy.maxRecoveries)
+        return { kind: "interrupted" };
+    return {
+        kind: "requeue",
+        recoveryCount: count + 1,
+        windowStartedAt: policy.windowMs === undefined
+            ? null
+            : expired || windowStartedAt === null
+                ? now
+                : windowStartedAt,
+    };
+}

package/dist/internal/retry.d.ts

@@ -0,0 +1,19 @@
+import type { WatchdogRetryPolicy } from "../effect.js";
+export declare const parseRetryPolicy: (input: unknown, options?: import("effect/SchemaAST").ParseOptions) => {
+    readonly baseMs: number;
+    readonly maxMs: number;
+    readonly limit: {
+        readonly maxAttempts: number;
+    } | {
+        readonly deadlineAt: number;
+    };
+};
+export declare function retryDue(count: number, now: number, policy: WatchdogRetryPolicy): number;
+export declare function reserveRetry(count: number, now: number, policy: WatchdogRetryPolicy): {
+    attempt: {
+        count: number;
+        status: "exhausted" | "admitted";
+        final: boolean;
+    };
+    notBefore: number;
+};

package/dist/internal/retry.js

@@ -0,0 +1,32 @@
+import { backoffDelay } from "@fungi.computer/almanac";
+import * as Schema from "effect/Schema";
+const Count = Schema.Int.check(Schema.isGreaterThanOrEqualTo(0), Schema.isLessThanOrEqualTo(Number.MAX_SAFE_INTEGER));
+const Policy = Schema.Struct({
+    baseMs: Count,
+    maxMs: Count,
+    limit: Schema.Union([
+        Schema.Struct({ maxAttempts: Count }),
+        Schema.Struct({ deadlineAt: Count }),
+    ]),
+});
+export const parseRetryPolicy = Schema.decodeUnknownSync(Policy);
+export function retryDue(count, now, policy) {
+    return Math.min(now + backoffDelay(count, policy), "deadlineAt" in policy.limit
+        ? policy.limit.deadlineAt
+        : Number.MAX_SAFE_INTEGER);
+}
+export function reserveRetry(count, now, policy) {
+    const exhausted = "maxAttempts" in policy.limit
+        ? count >= policy.limit.maxAttempts
+        : now >= policy.limit.deadlineAt;
+    const charged = exhausted ? count : count + 1;
+    return {
+        attempt: {
+            count: charged,
+            status: exhausted ? "exhausted" : "admitted",
+            final: exhausted ||
+                ("maxAttempts" in policy.limit && charged >= policy.limit.maxAttempts),
+        },
+        notBefore: retryDue(charged, now, policy),
+    };
+}

package/dist/internal/runtime.js

@@ -1,3 +1,4 @@
+import { parseRetryPolicy } from "./retry.js";
 import * as Cause from "effect/Cause";
 import * as Deferred from "effect/Deferred";
 import * as Effect from "effect/Effect";
@@ -18,6 +19,30 @@ function failureReasonOf(cause) {
 export const bindWatchdogEngine = (options, store) => {
     let tickState = "idle";
     let activeExecution;
+    let lifecycle = "open";
+    let currentTick;
+    let closing;
+    const close = Effect.suspend(() => {
+        if (closing !== undefined)
+            return Deferred.await(closing);
+        const done = Deferred.makeUnsafe();
+        closing = done;
+        lifecycle = "stopping";
+        const pendingTick = currentTick;
+        activeExecution?.fiber.interruptUnsafe();
+        return (pendingTick === undefined
+            ? Effect.succeed(undefined)
+            : Deferred.await(pendingTick)).pipe(Effect.onExit((exit) => {
+            lifecycle = "stopped";
+            return Deferred.done(done, Exit.map(exit, () => undefined));
+        }), Effect.uninterruptible);
+    });
+    const policyFor = (job) => (options.executor.retryPolicy?.(job) ?? Effect.succeed(undefined)).pipe(Effect.flatMap((policy) => policy === undefined
+        ? Effect.succeed(undefined)
+        : Effect.try({
+            try: () => parseRetryPolicy(policy),
+            catch: (error) => new WatchdogError("store_invalid", error),
+        })));
     const ingest = (job) => store.ingest(job);
     const read = (jobId) => store.read(jobId).pipe(Effect.map((result) => result.status === "missing"
         ? result
@@ -27,7 +52,7 @@ export const bindWatchdogEngine = (options, store) => {
             cancellation: result.cancellation,
         }));
     const readQueueHead = (queue) => store.readQueueHead(queue);
-    const readSettled = (limit, afterSequence) => store.readSettled(limit, afterSequence);
+    const readSettled = (limit, afterSequence, lane) => store.readSettled(limit, afterSequence, lane);
     const cancel = (input) => store.recordCancellation(input).pipe(Effect.tap((result) => {
         if (result.status !== "recorded" && result.status !== "replayed")
             return Effect.void;
@@ -70,13 +95,11 @@ export const bindWatchdogEngine = (options, store) => {
                 };
             if (probe.success)
                 return { status: "none" };
-            const disposition = claim.job.recovery !== undefined &&
-                claim.recoveryCount < claim.job.recovery.maxRecoveries
-                ? "requeue"
-                : "interrupted";
+            const policy = yield* policyFor(claim.job);
             const result = yield* store.reconcileDisappearedClaim({
                 ...claim,
-                disposition,
+                requeue: policy !== undefined ||
+                    options.executor.replaySafe?.(claim.job) === true,
             });
             switch (result.status) {
                 case "requeued":
@@ -100,93 +123,172 @@ export const bindWatchdogEngine = (options, store) => {
             }
         });
     };
+    const finishCompletion = (claim) => store.finishCompletingClaim(claim).pipe(Effect.flatMap((result) => result.status === "settled" || result.status === "replayed"
+        ? options.wake.recompute().pipe(Effect.as({
+            status: "settled",
+            job: result.job,
+        }))
+        : Effect.fail(new WatchdogError("settlement_rejected"))));
     const tick = () => Effect.suspend(() => {
+        if (lifecycle !== "open")
+            return Effect.succeed({ status: "stopped" });
         if (tickState !== "idle")
             return Effect.succeed({ status: "busy" });
         tickState = "running";
-        return inspectDisappearedClaim().pipe(Effect.flatMap((inspection) => {
-            if (inspection.status === "recovered")
-                return Effect.succeed({
-                    status: "recovered",
-                    job: inspection.job,
-                });
-            // Other work still proceeds; only an otherwise idle tick has
-            // nothing to say but the unanswered probe.
-            return claimNext().pipe(Effect.map((selection) => selection.status === "no_ready" &&
-                inspection.status === "liveness_unknown"
-                ? inspection
-                : selection));
-        }), Effect.flatMap((selection) => {
-            if (selection.status === "no_ready")
-                return Effect.succeed({ status: "no_ready" });
-            if (selection.status === "liveness_unknown")
-                return Effect.succeed(selection);
-            if (selection.status === "claim_unavailable")
-                return Effect.fail(new WatchdogError("claim_unavailable"));
-            if (selection.status === "cancelled")
-                return options.wake.recompute().pipe(Effect.as({
-                    status: "settled",
-                    job: selection.job,
-                }));
-            if (selection.status === "recovered")
-                return Effect.succeed({
-                    status: "settled",
-                    job: selection.job,
-                });
-            return Effect.gen(function* () {
-                const start = yield* Deferred.make();
-                const fiber = yield* Effect.forkChild(Deferred.await(start).pipe(Effect.andThen(options.executor.execute(selection.job))));
-                const owned = {
-                    jobId: selection.job.jobId,
-                    fiber,
-                };
-                activeExecution = owned;
-                const current = yield* store.read(selection.job.jobId);
-                if (current.status !== "found" ||
-                    current.job.state !== "running" ||
-                    current.cancellation.status === "present") {
-                    if (current.status === "found" &&
-                        current.job.state === "settled" &&
-                        current.job.outcome === "failed")
-                        owned.failed = current.job;
-                    yield* Fiber.interrupt(fiber);
-                }
-                else
-                    yield* Deferred.succeed(start, undefined);
-                const execution = yield* Fiber.await(fiber).pipe(Effect.ensuring(Effect.sync(() => {
-                    if (activeExecution?.fiber === fiber)
-                        activeExecution = undefined;
-                })));
-                if (owned.failed !== undefined)
-                    return yield* options.wake
-                        .recompute()
-                        .pipe(Effect.as({ status: "settled", job: owned.failed }));
-                const decision = Exit.isSuccess(execution)
-                    ? execution.value
-                    : { kind: "failed", reason: failureReasonOf(execution.cause) };
-                const settled = yield* store.settleExactClaim({
-                    jobId: selection.job.jobId,
-                    claimToken: selection.claimToken,
-                    decision,
+        const done = Deferred.makeUnsafe();
+        currentTick = done;
+        return Effect.gen(function* () {
+            // An execution decision survives projection failure. Retry only its
+            // transaction, once per tick; it is never a disappeared execution.
+            const completing = yield* store.nextCompletingClaim();
+            if (completing !== undefined)
+                return yield* finishCompletion({
+                    jobId: completing.job.jobId,
+                    claimToken: completing.claimToken,
                 });
-                switch (settled.status) {
-                    case "settled":
-                    case "replayed":
+            return yield* inspectDisappearedClaim().pipe(Effect.flatMap((inspection) => {
+                if (inspection.status === "recovered")
+                    return Effect.succeed({
+                        status: "recovered",
+                        job: inspection.job,
+                    });
+                // Other work still proceeds; only an otherwise idle tick has
+                // nothing to say but the unanswered probe.
+                return claimNext().pipe(Effect.map((selection) => selection.status === "no_ready" &&
+                    inspection.status === "liveness_unknown"
+                    ? inspection
+                    : selection));
+            }), Effect.flatMap((selection) => {
+                if (selection.status === "no_ready")
+                    return Effect.succeed({ status: "no_ready" });
+                if (selection.status === "liveness_unknown")
+                    return Effect.succeed(selection);
+                if (selection.status === "claim_unavailable")
+                    return Effect.fail(new WatchdogError("claim_unavailable"));
+                if (selection.status === "cancelled")
+                    return options.wake.recompute().pipe(Effect.as({
+                        status: "settled",
+                        job: selection.job,
+                    }));
+                if (selection.status === "recovered")
+                    return Effect.succeed({
+                        status: "settled",
+                        job: selection.job,
+                    });
+                return Effect.gen(function* () {
+                    const policy = yield* policyFor(selection.job);
+                    const replaySafe = options.executor.replaySafe?.(selection.job) === true;
+                    const start = yield* Deferred.make();
+                    const dispatch = { started: false };
+                    const fiber = yield* Effect.forkChild(Deferred.await(start).pipe(Effect.andThen(Effect.sync(() => {
+                        dispatch.started = true;
+                    })), Effect.andThen(policy === undefined
+                        ? Effect.succeed(undefined)
+                        : store.reserveAttempt({
+                            jobId: selection.job.jobId,
+                            claimToken: selection.claimToken,
+                            policy,
+                        })), Effect.flatMap((attempt) => options.executor.execute(selection.job, attempt))));
+                    const owned = {
+                        jobId: selection.job.jobId,
+                        fiber,
+                    };
+                    activeExecution = owned;
+                    const current = yield* store.read(selection.job.jobId);
+                    if (lifecycle !== "open" ||
+                        current.status !== "found" ||
+                        current.job.state !== "running" ||
+                        current.cancellation.status === "present") {
+                        if (current.status === "found" &&
+                            current.job.state === "settled" &&
+                            current.job.outcome === "failed")
+                            owned.failed = current.job;
+                        yield* Fiber.interrupt(fiber);
+                    }
+                    else
+                        yield* Deferred.succeed(start, undefined);
+                    const execution = yield* Fiber.await(fiber).pipe(Effect.ensuring(Effect.sync(() => {
+                        if (activeExecution?.fiber === fiber)
+                            activeExecution = undefined;
+                    })));
+                    if (owned.failed !== undefined)
                         return yield* options.wake.recompute().pipe(Effect.as({
                             status: "settled",
-                            job: settled.job,
+                            job: owned.failed,
                         }));
-                    case "conflict":
-                    case "missing":
-                        return yield* Effect.fail(new WatchdogError("settlement_rejected"));
-                }
-            });
-        }), Effect.ensuring(Effect.sync(() => {
+                    if (lifecycle !== "open" && Exit.isFailure(execution)) {
+                        const drained = Cause.hasInterruptsOnly(execution.cause) &&
+                            (!dispatch.started ||
+                                (yield* options.executor.isDrained?.(selection.job) ??
+                                    Effect.succeed(false)));
+                        if (!drained)
+                            return yield* Effect.fail(new WatchdogError("drain_failed"));
+                        const handoff = yield* store.handoffExactClaim({
+                            jobId: selection.job.jobId,
+                            claimToken: selection.claimToken,
+                        });
+                        if (handoff.status === "requeued")
+                            return yield* options.wake
+                                .recompute()
+                                .pipe(Effect.as({ status: "stopped" }));
+                        if (handoff.status !== "cancelled")
+                            return yield* Effect.fail(new WatchdogError("settlement_rejected"));
+                    }
+                    const decision = Exit.isSuccess(execution)
+                        ? execution.value
+                        : { kind: "failed", reason: failureReasonOf(execution.cause) };
+                    if ((policy !== undefined &&
+                        (decision.kind === "retry" || Exit.isFailure(execution))) ||
+                        (replaySafe && Exit.isFailure(execution))) {
+                        const retryInput = {
+                            jobId: selection.job.jobId,
+                            claimToken: selection.claimToken,
+                        };
+                        if (policy !== undefined)
+                            Object.assign(retryInput, { policy });
+                        const retried = yield* store.retryExactClaim(retryInput);
+                        if (retried.status === "requeued" &&
+                            replaySafe &&
+                            Exit.isFailure(execution))
+                            return yield* Effect.failCause(execution.cause);
+                        if (retried.status === "requeued")
+                            return yield* options.wake.recompute().pipe(Effect.as({
+                                status: "requeued",
+                                jobId: selection.job.jobId,
+                            }));
+                        if (retried.status === "conflict")
+                            return yield* Effect.fail(new WatchdogError("settlement_rejected"));
+                        // A recorded cancellation wins over retry and follows normal terminal settlement.
+                    }
+                    const settled = yield* store.recordExecutionDecision({
+                        jobId: selection.job.jobId,
+                        claimToken: selection.claimToken,
+                        decision: decision.kind === "retry"
+                            ? { kind: "failed", reason: "retry_policy_missing" }
+                            : decision,
+                    });
+                    switch (settled.status) {
+                        case "recorded":
+                        case "replayed":
+                            return yield* finishCompletion({
+                                jobId: selection.job.jobId,
+                                claimToken: selection.claimToken,
+                            });
+                        case "conflict":
+                        case "missing":
+                            return yield* Effect.fail(new WatchdogError("settlement_rejected"));
+                    }
+                });
+            }));
+        }).pipe(Effect.onExit((exit) => {
             tickState = "idle";
-        })));
+            currentTick = undefined;
+            return Deferred.done(done, Exit.map(exit, () => undefined));
+        }));
     });
     return Object.freeze({
-        hasPendingWork: store.hasPendingWork,
+        close,
+        wakeState: store.wakeState,
         ingest,
         read,
         readQueueHead,

package/dist/internal/sqlite-store.d.ts

@@ -1,6 +1,6 @@
 import * as Effect from "effect/Effect";
 import * as Schema from "effect/Schema";
-import type { WatchdogLaneFailure, WatchdogCancellationIntent, WatchdogCancellationResult, WatchdogExecutionDecision, WatchdogIngestResult, WatchdogQueuedJob, WatchdogQueueHead, WatchdogQueueProjector, WatchdogReadResult, WatchdogRunningJob, WatchdogSettledJob, WatchdogSettledRecord, WatchdogSqliteOwner, WatchdogSqliteRow, WatchdogStoredJob, WatchdogTransactionalProjection } from "../effect.js";
+import type { WatchdogCompletingJob, WatchdogLaneFailure, WatchdogCancellationIntent, WatchdogCancellationResult, WatchdogExecutionDecision, WatchdogIngestResult, WatchdogQueuedJob, WatchdogQueueHead, WatchdogQueueProjector, WatchdogReadResult, WatchdogRunningJob, WatchdogSettledJob, WatchdogSettledRecord, WatchdogSqliteOwner, WatchdogSqliteRow, WatchdogStoredJob, WatchdogTransactionalProjection, WatchdogWakeState, WatchdogRetryPolicy, WatchdogAttempt } from "../effect.js";
 import { WatchdogError } from "./watchdog-error.js";
 export type { WatchdogSqliteRow } from "../effect.js";
 export declare const laneFailureBatchSize = 256;
@@ -13,8 +13,12 @@ declare const JobRowSchema: Schema.Struct<{
     readonly lane: Schema.String;
     readonly priority: Schema.Int;
     readonly payload_json: Schema.String;
+    readonly not_before: Schema.Int;
+    readonly attempt_count: Schema.Int;
     readonly recovery_max: Schema.NullOr<Schema.Int>;
-    readonly state: Schema.Literals<readonly ["queued", "running", "settled"]>;
+    readonly recovery_window_ms: Schema.NullOr<Schema.Int>;
+    readonly recovery_window_started_at: Schema.NullOr<Schema.Int>;
+    readonly state: Schema.Literals<readonly ["queued", "running", "completing", "settled"]>;
     readonly outcome: Schema.NullOr<Schema.Literals<readonly ["completed", "failed", "cancelled", "outcome_unknown", "interrupted"]>>;
     readonly claim_token: Schema.NullOr<Schema.String>;
     readonly recovery_count: Schema.Int;
@@ -48,7 +52,7 @@ export type LanePartition = Readonly<{
 }>;
 /** Package-private storage operations consumed by Watchdog's Effect engine. */
 export type WatchdogStore = Readonly<{
-    hasPendingWork: Effect.Effect<boolean, WatchdogError>;
+    wakeState: (now: number) => Effect.Effect<WatchdogWakeState, WatchdogError>;
     ingest: (job: WatchdogQueuedJob) => Effect.Effect<WatchdogIngestResult, WatchdogError>;
     read: (jobId: string) => Effect.Effect<(Extract<WatchdogReadResult, {
         status: "found";
@@ -59,24 +63,57 @@ export type WatchdogStore = Readonly<{
         status: "missing";
     }>, WatchdogError>;
     readQueueHead: (queue: string) => Effect.Effect<WatchdogQueueHead, WatchdogError>;
-    readSettled: (limit: number, afterSequence?: number) => Effect.Effect<readonly WatchdogSettledRecord[], WatchdogError>;
+    readSettled: (limit: number, afterSequence?: number, lane?: string) => Effect.Effect<readonly WatchdogSettledRecord[], WatchdogError>;
     selectAndClaim: (claimToken: string) => Effect.Effect<WatchdogSelection, WatchdogError>;
-    settleExactClaim: (input: Readonly<{
+    reserveAttempt: (input: {
         jobId: string;
         claimToken: string;
-        decision: WatchdogExecutionDecision;
+        policy: WatchdogRetryPolicy;
+    }) => Effect.Effect<WatchdogAttempt, WatchdogError>;
+    retryExactClaim: (input: {
+        jobId: string;
+        claimToken: string;
+        policy?: WatchdogRetryPolicy;
+    }) => Effect.Effect<{
+        status: "requeued" | "cancelled" | "conflict";
+    }, WatchdogError>;
+    recordExecutionDecision: (input: Readonly<{
+        jobId: string;
+        claimToken: string;
+        decision: Exclude<WatchdogExecutionDecision, {
+            kind: "retry";
+        }>;
     }>) => Effect.Effect<Readonly<{
-        status: "settled" | "replayed";
-        job: WatchdogSettledJob;
+        status: "recorded" | "replayed";
+        job: WatchdogCompletingJob | WatchdogSettledJob;
     }> | Readonly<{
         status: "conflict" | "missing";
     }>, WatchdogError>;
+    nextCompletingClaim: () => Effect.Effect<{
+        job: WatchdogCompletingJob;
+        claimToken: string;
+    } | undefined, WatchdogError>;
+    finishCompletingClaim: (input: {
+        jobId: string;
+        claimToken: string;
+    }) => Effect.Effect<{
+        status: "settled" | "replayed";
+        job: WatchdogSettledJob;
+    } | {
+        status: "conflict" | "missing";
+    }, WatchdogError>;
     recordCancellation: (input: WatchdogCancellationIntent) => Effect.Effect<WatchdogCancellationResult, WatchdogError>;
     failLane: (input: WatchdogLaneFailure) => Effect.Effect<Readonly<{
         jobs: readonly WatchdogSettledJob[];
         more: boolean;
     }>, WatchdogError>;
     purgeQueued: (queue: string) => Effect.Effect<void, WatchdogError>;
+    handoffExactClaim: (input: {
+        jobId: string;
+        claimToken: string;
+    }) => Effect.Effect<{
+        status: "requeued" | "cancelled" | "conflict" | "missing";
+    }, WatchdogError>;
     nextRunningClaim: () => Effect.Effect<Readonly<{
         job: WatchdogRunningJob;
         claimToken: string;
@@ -86,7 +123,7 @@ export type WatchdogStore = Readonly<{
         job: WatchdogRunningJob;
         claimToken: string;
         recoveryCount: number;
-        disposition: "requeue" | "interrupted";
+        requeue?: boolean;
     }>) => Effect.Effect<Readonly<{
         status: "requeued";
     }> | Readonly<{
@@ -102,6 +139,10 @@ export type WatchdogSqliteState = Readonly<{
     partition: (partition: LanePartition) => WatchdogStore;
     transactional: WatchdogTransactionalProjection;
 }>;
+/** Queue order is decided before readiness; a sleeping head still owns its queue. */
+export declare const queuedHeadPredicate = "(cancellation_id IS NOT NULL OR NOT EXISTS (\n  SELECT 1 FROM watchdog_jobs predecessor\n  WHERE predecessor.queue_name = watchdog_jobs.queue_name\n    AND predecessor.state IN ('queued', 'running', 'completing')\n    AND (predecessor.state IN ('running', 'completing') OR\n      (predecessor.state = 'queued' AND\n        (predecessor.priority > watchdog_jobs.priority OR\n          (predecessor.priority = watchdog_jobs.priority AND\n            predecessor.sequence < watchdog_jobs.sequence))))))";
+export declare const parseWakeTime: (input: unknown, options?: import("effect/SchemaAST").ParseOptions) => number;
+export declare function parseWakeState(raw: WatchdogSqliteRow | undefined): WatchdogWakeState;
 export declare function parseJobRow(raw: WatchdogSqliteRow): ParsedWatchdogSqliteJob;
 /** Construct Watchdog's private owner-local SQLite state. */
 export declare function openSqliteState(owner: WatchdogSqliteOwner, project?: WatchdogQueueProjector, partition?: LanePartition): WatchdogSqliteState;