@earendil-works/pi-durable 1.3.0 → 1.4.0

package/dist/harness/scheduler.js

@@ -1,13 +1,25 @@
 import { copyJson } from "@earendil-works/chord";
-import { awaitWithContext, withAbortSignal } from "@earendil-works/chord/context";
+import { awaitWithContext, withAbortSignal, withoutAbortSignal, withTelemetry } from "@earendil-works/chord/context";
+import { createTypedMetrics, createTypedSpanStarter, NOOP_TELEMETRY_CONTEXT, NOOP_TELEMETRY_METRICS, } from "@earendil-works/pi-telemetry";
 import { errorMessage } from "../errors.js";
-import { agentHooks } from "./agent.js";
+import { agentHooks, DEFAULT_RESTART_LIMIT } from "./agent.js";
 import { readContextFrom } from "./context.js";
-import { closedError, scanAll, Waiters } from "./util.js";
+import { DURABLE_TELEMETRY_SCHEMA } from "./telemetry.js";
+import { closedError, optionalExtension, scanAll, Waiters } from "./util.js";
 const SCAN_PAGE_SIZE = 256;
-/** Longest delay `setTimeout` supports; longer sleeps wait in several steps. */
-const MAX_TIMER_DELAY = 2_147_483_647;
-const LIVE_STATUSES = ["pending", "running", "waiting", "completing"];
+const LIVE_STATUSES = ["pending", "running", "waiting", "sleeping", "completing"];
+/** The part of a record `blockedReason()` reads. */
+export function blockedSubject(record) {
+    return {
+        id: record.id,
+        kind: record.kind,
+        version: record.version,
+        abortRequested: record.abortRequested,
+        ...(record.abortReason === undefined ? {} : { abortReason: record.abortReason }),
+        ...(record.restarts === undefined ? {} : { restarts: record.restarts }),
+        status: record.state.status,
+    };
+}
 function conversationNode(id) {
     return `c${id}`;
 }
@@ -21,8 +33,9 @@ function parentNode(node) {
 /**
  * Durable task scheduler of one Harness.
  *
- * `#live` mirrors every committed non-terminal task record: pending, running, waiting, and completing. The synchronous
- * commit listener updates it on the Session line, so code running on the line reads exactly the committed state from it.
+ * `#live` mirrors every committed non-terminal task record: pending, running, waiting, sleeping, and completing. The
+ * synchronous commit listener updates it on the Session line, so code running on the line reads exactly the committed
+ * state from it.
  *
  * Tasks and conversations form one ownership tree (spec §5.5): a task's parent is its owner task, or its conversation;
  * a conversation's parent is its owner task, if any. Walks up that tree decide what reaches one task: a cancelling
@@ -40,6 +53,10 @@ function parentNode(node) {
  * step that decides its end, so a runtime commit it queued either lands before that decision or is rejected.
  */
 export class TaskScheduler {
+    #telemetry;
+    #activeMetric;
+    #durationMetric;
+    #resumed = new Set();
     #session;
     #storage;
     #registry;
@@ -48,6 +65,9 @@ export class TaskScheduler {
     #settings;
     #env;
     #now;
+    #onWake;
+    #timers;
+    #writeAtClose;
     #report;
     #settleOutcome;
     #withdrawInputs;
@@ -60,7 +80,7 @@ export class TaskScheduler {
      * first idle check after `settings.contextRetentionMs` of idleness, and at close.
      */
     #contexts = new Map();
-    /** Timer for the earliest expiry of an idle context, only where timers can be unreferenced (see `#scheduleExpiry`). */
+    /** Background timer for the earliest expiry of an idle context, unless the host declined it (see `#scheduleExpiry`). */
     #expiry;
     #invocations = new Map();
     #taskWaiters = new Waiters();
@@ -103,7 +123,21 @@ export class TaskScheduler {
      * them with reason `restart`, so nothing of them or below them runs a phase again.
      */
     #abandoned = new Set();
+    /** Sleeping live tasks, due or not: the candidates of `#wakeDue()`. */
+    #sleeping = new Set();
+    /** Timer for the earliest sleep, without `onWake` while scheduling runs. */
+    #timer;
+    /** Last value given to `onWake`, so an unchanged one is not repeated. */
+    #lastWake;
+    #wakeScheduled = false;
+    #blockedListeners = new Set();
+    /** A registry snapshot that threw for the task graph was reported; it is read per record, so only once. */
+    #snapshotReported = false;
+    /** Tasks a pass left out as parked; each pass checks them again, as `settings.restartLimit` may have risen. */
+    #parkedTasks = new Set();
     #reconcileScheduled = false;
+    /** Reconcile passes underway with work (they may overlap): their commits finalize held outcomes and mark cascades. */
+    #reconcilingWork = 0;
     #sweepScheduled = false;
     #cascadePending = false;
     #unsubscribeRegistry = () => { };
@@ -112,6 +146,10 @@ export class TaskScheduler {
     #dirty = false;
     #draining = false;
     constructor(options) {
+        this.#telemetry = options.telemetry ?? NOOP_TELEMETRY_CONTEXT;
+        const metrics = createTypedMetrics(this.#telemetry.metrics ?? NOOP_TELEMETRY_METRICS, [DURABLE_TELEMETRY_SCHEMA]);
+        this.#activeMetric = metrics.upDownCounter("pi.durable.tasks.active");
+        this.#durationMetric = metrics.histogram("pi.durable.phase.duration");
         this.#session = options.session;
         this.#storage = options.storage;
         this.#registry = options.registry;
@@ -120,17 +158,28 @@ export class TaskScheduler {
         this.#settings = options.settings;
         this.#env = options.env;
         this.#now = options.now;
+        this.#onWake = options.onWake;
+        this.#timers = options.timers;
+        this.#writeAtClose = options.writeAtClose;
         this.#report = options.report;
         this.#settleOutcome = options.settleOutcome;
         this.#withdrawInputs = options.withdrawInputs;
         this.#conversation = options.conversation;
         this.#context = options.context;
     }
-    /** Load live tasks and change surviving `running` tasks back to `pending`. Dispatches nothing. */
+    /**
+     * Load live tasks and change surviving `running` tasks back to `pending`, counting the restart. Dispatches nothing. A
+     * task that reaches the restart limit this way is parked and reported.
+     */
     async open(context) {
         this.#session.observeCommits((publication) => this.#observe(publication));
         this.#session.subscribeClose(() => this.#seal());
-        this.#unsubscribeRegistry = this.#registry.subscribe(() => this.#kick());
+        this.#unsubscribeRegistry = this.#registry.subscribe(() => {
+            this.#notifyBlocked();
+            this.#kick();
+        });
+        const limit = this.#restartLimit();
+        const parked = [];
         await this.#session.commitWith(async (tx) => {
             // Every table read before the first write.
             const scans = [];
@@ -140,10 +189,15 @@ export class TaskScheduler {
             for (const records of scans) {
                 for (const record of records) {
                     this.#track(undefined, record);
+                    this.#resumed.add(record.id);
                     if (record.abandonOnRestart === true)
                         this.#abandoned.add(record.id);
                     if (record.state.status === "running") {
-                        tx.setTask(withState(record, { status: "pending", checkpoint: record.state.checkpoint }));
+                        const restarts = (record.restarts ?? 0) + 1;
+                        const pending = withState(record, { status: "pending", checkpoint: record.state.checkpoint });
+                        tx.setTask({ ...pending, restarts });
+                        if (restarts >= limit && !record.abortRequested)
+                            parked.push(record);
                     }
                     if (record.state.status === "waiting" && record.state.policy === "failFast") {
                         this.#failFastChecks.add(record.id);
@@ -151,23 +205,143 @@ export class TaskScheduler {
                 }
             }
         }, context);
+        for (const record of parked) {
+            const restarts = (record.restarts ?? 0) + 1;
+            const error = new Error(`Task ${record.id} (${record.kind}) was cut off ${restarts} times without progress; it is parked until retryTask()`);
+            this.#report(error, { type: "task_parked", ...this.#aboutTask(recordTask(record)) });
+        }
         // Derive abort marks a crash left unapplied below cancelled owners, and finalize held outcomes.
         this.#cascadePending = true;
         this.#scheduleReconcile();
+        this.#scheduleWake();
     }
-    /** Enable scheduling. Idempotent; the kick does nothing once closing. */
+    /**
+     * Enable scheduling, and make sleeping tasks that are due runnable: a host wakeup, such as an alarm, calls this.
+     * Idempotent; the kick does nothing once closing.
+     */
     resume() {
         this.#enabled = true;
+        this.#wakeDue();
         this.#kick();
     }
     /**
-     * Signal every invocation and wait for them; writes nothing. Close calls this after every close listener has run, so
-     * waits, watches, and streams a task holds end with the Session's reason, not as cancelled by its signal.
+     * Run a task parked by the restart limit again: clears its restart count and enables scheduling. Does nothing for a
+     * task that is not parked; rejects for an unknown one.
+     */
+    async retry(id, context) {
+        await this.#session.commitWith(async (tx) => {
+            const current = await tx.task(id);
+            if (current === undefined)
+                throw new Error(`Task ${id} does not exist`);
+            const status = current.state.status;
+            if (status === "terminal" || status === "completing" || !this.#parked(current))
+                return;
+            tx.setTask(progressed(current));
+        }, context);
+        this.resume();
+    }
+    /**
+     * Why a live task cannot run here, if it cannot: its definition (missing, too old, failed migration) or the restart
+     * limit. A running task, an abort-marked one, and one holding its outcome are never blocked.
+     */
+    blockedReason(record) {
+        const status = record.status;
+        if (status === "running" || status === "completing" || status === "terminal")
+            return undefined;
+        // An abort request runs its handler or orphans the task; a `restart` mark waits for the definition, blocked.
+        if (record.abortRequested && record.abortReason === undefined)
+            return undefined;
+        if (this.#parked(record))
+            return "restart_limit";
+        let snapshot;
+        try {
+            snapshot = this.#registry.snapshot();
+        }
+        catch (error) {
+            if (!this.#snapshotReported)
+                this.#report(error, REGISTRY);
+            this.#snapshotReported = true;
+            return undefined;
+        }
+        const fit = this.#fit(record, snapshot.task(record.kind));
+        if ("reason" in fit)
+            return fit.reason;
+        return fit.migrates && fit.task.definition.migrate === undefined ? "migration_failed" : undefined;
+    }
+    /** Call `listener` when blocked reasons may have changed without a commit: a registry change or a failed migration. */
+    subscribeBlocked(listener) {
+        this.#blockedListeners.add(listener);
+        return () => this.#blockedListeners.delete(listener);
+    }
+    /**
+     * When the Session next needs a process: `active` while some task can run now or is running, `at` for the earliest
+     * sleep not yet due. Synchronous, from memory; a task whose chain is not loaded yet counts as it would alone.
+     */
+    wake() {
+        const now = this.#now();
+        // A reconcile pass with work to do is work too: it finalizes held outcomes and marks cascades, at open as
+        // recovery. A process that dies before it lands must be woken again. A pass with nothing to do is not.
+        let active = this.#invocations.size > 0 || this.#reconcilingWork > 0 || this.#reconcileWork();
+        let at;
+        let snapshot;
+        let snapshotFailed = false;
+        for (const record of this.#live.values()) {
+            const state = record.state;
+            if (state.status === "completing")
+                continue;
+            if (state.status === "waiting" && !record.abortRequested && state.on.some((id) => this.#live.has(id)))
+                continue;
+            if (state.status === "sleeping" && !record.abortRequested && state.until > now) {
+                at = at === undefined ? state.until : Math.min(at, state.until);
+                continue;
+            }
+            if (active)
+                continue;
+            if (this.#parked(record))
+                continue;
+            // An abort invocation waits for the live work below it, which counts itself.
+            if (record.abortRequested && this.#waitingOn(record, true).length > 0)
+                continue;
+            // A task no definition can take waits for a registry change, except an abort-marked one, which is orphaned.
+            if (!record.abortRequested || record.abortReason !== undefined) {
+                if (snapshot === undefined && !snapshotFailed) {
+                    try {
+                        snapshot = this.#registry.snapshot();
+                    }
+                    catch (error) {
+                        snapshotFailed = true;
+                        this.#report(error, REGISTRY);
+                    }
+                }
+                if (snapshot !== undefined && "reason" in this.#fit(record, snapshot.task(record.kind)))
+                    continue;
+            }
+            active = true;
+        }
+        return at === undefined ? { active } : { active, at };
+    }
+    /**
+     * Signal every invocation and wait for them, then write the tasks they ran back as `pending` (`writeAtClose`), as a
+     * close is not a crash. Close calls this after every close listener has run, so waits, watches, and streams a task
+     * holds end with the Session's reason, not as cancelled by its signal.
      */
     async join() {
-        for (const invocation of this.#invocations.values())
+        const stopped = [...this.#invocations.values()];
+        for (const invocation of stopped)
             invocation.controller.abort();
-        await Promise.allSettled([...this.#invocations.values()].map((invocation) => invocation.done));
+        await Promise.allSettled(stopped.map((invocation) => invocation.done));
+        if (stopped.length === 0)
+            return;
+        // A close is not a crash: a task it stopped is `pending` again, so the next open counts no restart for it. Read
+        // on the line after commits admitted before the seal, such as an abort mark, so the write keeps them. A failure
+        // leaves them `running`, counted as if cut off; the Storage guard fails the Session and reports it.
+        await this.#writeAtClose(() => stopped.flatMap((invocation) => {
+            const record = this.#live.get(invocation.taskId);
+            if (record?.state.status !== "running")
+                return [];
+            const value = withState(record, { status: "pending", checkpoint: record.state.checkpoint });
+            return [{ type: "task", value }];
+        })).catch(() => { });
     }
     /**
      * Commit the abort mark, or settle a task that no registered definition can take as `orphaned` when nothing it owns
@@ -362,7 +536,7 @@ export class TaskScheduler {
             return this.#settings().contextRetentionMs;
         }
         catch (error) {
-            this.#report(error);
+            this.#report(error, SETTINGS);
             return 0;
         }
     }
@@ -391,9 +565,9 @@ export class TaskScheduler {
         this.#scheduleExpiry();
     }
     /**
-     * Run `#settleIdle()` when the earliest idle context expires. The timer is unreferenced, so it never keeps the process
-     * alive. Where timers cannot be unreferenced, as in Cloudflare Workers, none is kept: a pending timer could keep a
-     * Durable Object from being evicted, and eviction frees the contexts. There, task changes alone check expiry.
+     * Run `#settleIdle()` when the earliest idle context expires, on a background timer, which never keeps the process
+     * alive. Where the host declines it, as the default does in Cloudflare Workers, none is kept: a pending timer could
+     * keep a Durable Object from being evicted, and eviction frees the contexts. There, task changes alone check expiry.
      */
     #scheduleExpiry() {
         const retention = this.#contextRetentionMs();
@@ -405,22 +579,16 @@ export class TaskScheduler {
         }
         if (this.#expiry !== undefined && this.#expiry.at === at)
             return;
-        if (this.#expiry !== undefined)
-            clearTimeout(this.#expiry.timer);
+        this.#expiry?.disarm();
         this.#expiry = undefined;
         if (at === undefined || this.#closing)
             return;
-        const timer = setTimeout(() => {
+        const disarm = this.#timers.background(() => {
             this.#expiry = undefined;
             this.#settleIdle();
-        }, Math.min(Math.max(0, at - this.#now()), MAX_TIMER_DELAY));
-        const unref = timer.unref;
-        if (typeof unref !== "function") {
-            clearTimeout(timer);
-            return;
-        }
-        unref.call(timer);
-        this.#expiry = { at, timer };
+        }, at - this.#now());
+        if (disarm !== undefined)
+            this.#expiry = { at, disarm };
     }
     // ─── Ownership ───────────────────────────────────────────────────────────
     #scheduleReconcile() {
@@ -437,6 +605,9 @@ export class TaskScheduler {
      */
     async #reconcile() {
         this.#reconcileScheduled = false;
+        const work = this.#reconcileWork();
+        if (work)
+            this.#reconcilingWork++;
         const cascade = this.#cascadePending;
         this.#cascadePending = false;
         const checks = [...this.#failFastChecks];
@@ -493,8 +664,15 @@ export class TaskScheduler {
             // running the pass again, so it fails the Session, which reports it.
             this.#failSession(error);
         }
+        if (work)
+            this.#reconcilingWork--;
+        this.#scheduleWake();
         this.#settleIdle();
     }
+    /** Whether a reconcile pass would have work: a cascade to derive, held outcomes to check, or `failFast` waits. */
+    #reconcileWork() {
+        return this.#cascadePending || this.#finalizeChecks.size > 0 || this.#failFastChecks.size > 0;
+    }
     /** Whether any of `ids` holds or ended with an outcome other than `completed`. */
     async #anyFailed(ids) {
         for (const id of ids) {
@@ -672,12 +850,24 @@ export class TaskScheduler {
      */
     #track(previous, record) {
         const id = record.id;
+        const invocation = this.#invocations.get(id);
+        if (invocation !== undefined)
+            invocation.record = record;
+        if (record.state.status === "terminal") {
+            if (previous !== undefined && !this.#closing)
+                this.#activeMetric.add(-1, { task: record.kind });
+            this.#resumed.delete(id);
+        }
+        else if (previous === undefined && !this.#closing) {
+            this.#activeMetric.add(1, { task: record.kind });
+        }
         if (previous?.state.status === "waiting")
             for (const on of previous.state.on)
                 deleteFrom(this.#waiters, on, id);
         if (record.state.status === "terminal") {
             const node = nodeOf(record);
             this.#live.delete(id);
+            this.#sleeping.delete(id);
             this.#intent.delete(id);
             this.#runnable.delete(id);
             this.#finalizeChecks.delete(id);
@@ -711,6 +901,10 @@ export class TaskScheduler {
         if (record.state.status === "waiting")
             for (const on of record.state.on)
                 addTo(this.#waiters, on, id);
+        if (record.state.status === "sleeping")
+            this.#sleeping.add(id);
+        else
+            this.#sleeping.delete(id);
     }
     /** The nearest live task above ended ordinary work may be free now: one holding its outcome, or an aborting one. */
     #release(node, overlay) {
@@ -962,28 +1156,152 @@ export class TaskScheduler {
         return undefined;
     }
     /** Fail the Session, which reports it, for a throw in one of the scheduler's own commits; ignored once closing. */
-    #failSession(error) {
+    #failSession(error, context) {
         if (!this.#closing)
-            this.#session.fail(error);
+            this.#session.fail(error, context);
     }
     /** Close listener: runs synchronously once admission is sealed, before `join()`, or when the Session fails. */
     #seal() {
+        for (const record of this.#live.values())
+            this.#activeMetric.add(-1, { task: record.kind });
+        this.#resumed.clear();
         this.#closing = true;
         try {
             this.#unsubscribeRegistry();
         }
         catch (error) {
-            this.#report(error);
+            this.#report(error, REGISTRY);
         }
         const error = closedError(this.#session);
         this.#taskWaiters.rejectAll(error);
         this.#idleWaiters.rejectAll(error);
         this.#contexts.clear();
-        if (this.#expiry !== undefined)
-            clearTimeout(this.#expiry.timer);
+        this.#expiry?.disarm();
         this.#expiry = undefined;
+        this.#timer?.disarm();
+        this.#timer = undefined;
+    }
+    /** Recompute the wake in a microtask, off the Session line: arm the built-in timer, or tell `onWake` of a change. */
+    #scheduleWake() {
+        if (this.#wakeScheduled || this.#closing)
+            return;
+        this.#wakeScheduled = true;
+        queueMicrotask(() => {
+            this.#wakeScheduled = false;
+            if (this.#closing)
+                return;
+            const wake = this.wake();
+            if (this.#onWake === undefined) {
+                this.#armTimer(this.#enabled ? wake.at : undefined);
+                return;
+            }
+            const last = this.#lastWake;
+            if (last !== undefined && last.active === wake.active && last.at === wake.at)
+                return;
+            try {
+                this.#onWake(wake);
+                // Only a delivered wake counts: after a throw, the next change or kick offers it again.
+                this.#lastWake = wake;
+            }
+            catch (error) {
+                this.#report(error, ON_WAKE);
+            }
+        });
+    }
+    /**
+     * Keep the timer for the earliest sleep. It keeps the process alive, unlike the context expiry timer: a process with a
+     * sleeping task has work left, such as a retry backoff. Rechecks the clock when it fires, so a wake before `until`
+     * only arms it again.
+     */
+    #armTimer(at) {
+        if (this.#timer !== undefined && this.#timer.at === at)
+            return;
+        this.#timer?.disarm();
+        this.#timer = undefined;
+        if (at === undefined || this.#closing)
+            return;
+        const disarm = this.#timers.foreground(() => {
+            this.#timer = undefined;
+            this.#wakeDue();
+            this.#kick();
+        }, at - this.#now());
+        this.#timer = { at, disarm };
+    }
+    /** Make every sleeping task whose time has come a reservation candidate. */
+    #wakeDue() {
+        for (const id of this.#sleeping) {
+            const record = this.#live.get(id);
+            if (record !== undefined && !this.#notDue(record))
+                this.#runnable.add(id);
+        }
+    }
+    /** A task cut off `restartLimit` times in a row; an abort mark still runs its abort handler. */
+    #parked(record) {
+        return !record.abortRequested && record.restarts !== undefined && record.restarts >= this.#restartLimit();
+    }
+    /**
+     * The report info of a task's failure, with the extension that installed its definition: the code that failed is
+     * that extension's. Built-in tasks have none.
+     */
+    /** What a report about `task`'s own code says of it: the task, and the extension that installed its definition. */
+    #aboutTask(task) {
+        let extension;
+        if ("extension" in task)
+            extension = task.extension;
+        else {
+            try {
+                const snapshot = this.#registry.snapshot();
+                extension = installerOf(snapshot, snapshot.task(task.kind));
+            }
+            catch {
+                // A registry that throws is reported where it is read for scheduling; a report goes out without the name.
+            }
+        }
+        return { task: reportTask(task), ...optionalExtension(extension) };
+    }
+    /** Make parked tasks that a raised `settings.restartLimit` no longer parks candidates again, and tell the graph. */
+    #unparkRaised() {
+        let unparked = false;
+        for (const id of this.#parkedTasks) {
+            const record = this.#live.get(id);
+            if (record !== undefined && this.#parked(record))
+                continue;
+            this.#parkedTasks.delete(id);
+            if (record === undefined)
+                continue;
+            this.#runnable.add(id);
+            unparked = true;
+        }
+        if (unparked)
+            queueMicrotask(() => this.#notifyBlocked());
+    }
+    /** `settings.restartLimit`, or the default when the host's settings throw. */
+    #restartLimit() {
+        try {
+            return this.#settings().restartLimit;
+        }
+        catch (error) {
+            this.#report(error, SETTINGS);
+            return DEFAULT_RESTART_LIMIT;
+        }
+    }
+    #notifyBlocked() {
+        for (const listener of [...this.#blockedListeners]) {
+            try {
+                listener();
+            }
+            catch (error) {
+                this.#report(error, { type: "listener_error" });
+            }
+        }
+    }
+    /** A sleep whose `until` the clock has not reached; an abort mark ends a sleep. */
+    #notDue(record) {
+        const state = record.state;
+        return state.status === "sleeping" && !record.abortRequested && state.until > this.#now();
     }
     #kick() {
+        this.#scheduleWake();
         this.#dirty = true;
         if (this.#draining || !this.#enabled || this.#closing)
             return;
@@ -1016,6 +1334,7 @@ export class TaskScheduler {
      */
     async #reserve() {
         const reservations = [];
+        const faulted = [];
         let abandoned = [];
         try {
             await this.#session.commitWith(async (tx) => {
@@ -1031,6 +1350,10 @@ export class TaskScheduler {
                     return;
                 }
                 await this.#loadScopes(false);
+                // Every pass picks up sleeps whose time has come: a pass started by any change may be the first to run after
+                // a deadline, and the wake it schedules no longer keeps a timer for a due sleep.
+                this.#wakeDue();
+                this.#unparkRaised();
                 // Taken once per pass, and only when some task is a candidate.
                 let snapshot;
                 for (const id of [...this.#runnable]) {
@@ -1044,6 +1367,20 @@ export class TaskScheduler {
                         this.#runnable.delete(id);
                         continue;
                     }
+                    // It returns at the first pass after its time comes, which the timer, `resume()`, or any change starts.
+                    if (this.#notDue(record)) {
+                        this.#runnable.delete(id);
+                        continue;
+                    }
+                    // It returns when `retry()` clears its restart count, or at a pass after `settings.restartLimit` rose.
+                    if (this.#parked(record)) {
+                        this.#runnable.delete(id);
+                        // Newly parked without a commit, as after a lowered limit: the graph must hear it.
+                        if (!this.#parkedTasks.has(id))
+                            queueMicrotask(() => this.#notifyBlocked());
+                        this.#parkedTasks.add(id);
+                        continue;
+                    }
                     const runnable = record;
                     const mode = record.abortRequested ? "abort" : "run";
                     // Work below an owner with cancellation intent waits for its cascade mark instead of running a phase; schedule
@@ -1053,6 +1390,16 @@ export class TaskScheduler {
                         this.#scheduleReconcile();
                         continue;
                     }
+                    // An abort handler cut off `restartLimit` times in a row would run forever; nothing is left to park it for.
+                    if (mode === "abort" && (record.restarts ?? 0) >= this.#restartLimit()) {
+                        const message = `Abort handler of task ${id} (${record.kind}) was cut off ${record.restarts} times`;
+                        await this.#terminate(tx, record, {
+                            status: "faulted",
+                            error: { message, detail: { reason: "restart_limit" } },
+                        });
+                        faulted.push({ record, message });
+                        continue;
+                    }
                     snapshot ??= this.#registry.snapshot();
                     const resolution = this.#resolve(runnable, snapshot);
                     if (resolution.kind === "blocked") {
@@ -1071,7 +1418,7 @@ export class TaskScheduler {
                     // Registered on the line, so marks and later reservations see it and close joins it; `#end` makes it a
                     // candidate again.
                     this.#runnable.delete(id);
-                    const invocation = this.#createInvocation(record, mode);
+                    const invocation = this.#createInvocation(runnable, mode, installerOf(snapshot, resolution.task));
                     reservations.push({ invocation, task: resolution.task, snapshot });
                 }
             }, this.#context);
@@ -1084,6 +1431,13 @@ export class TaskScheduler {
             }
             throw error;
         }
+        for (const { record, message } of faulted) {
+            this.#report(new Error(message), {
+                type: "task_faulted",
+                ...this.#aboutTask(recordTask(record)),
+                reason: "restart_limit",
+            });
+        }
         // Marked, or already marked or gone: done with them. Reserve again, now seeing the marks.
         for (const id of abandoned)
             this.#abandoned.delete(id);
@@ -1123,7 +1477,11 @@ export class TaskScheduler {
         }
         catch (error) {
             this.#failedMigrations.set(record.id, { task: fit.task, error });
-            this.#report(error);
+            // Blocked now without a commit: the graph and the host's wake must hear it.
+            queueMicrotask(() => this.#notifyBlocked());
+            this.#scheduleWake();
+            const extension = installerOf(snapshot, fit.task);
+            this.#report(error, { type: "migration_failed", ...this.#aboutTask({ ...recordTask(record), extension }) });
             return { kind: "blocked", reason: "migration_failed" };
         }
     }
@@ -1176,6 +1534,10 @@ export class TaskScheduler {
         const on = this.#waitingOn(record);
         if (on.length > 0)
             return { kind: "waiting", on };
+        if (this.#notDue(record))
+            return { kind: "sleeping", until: record.state.until };
+        if (this.#parked(record))
+            return { kind: "blocked", reason: "restart_limit" };
         const fit = this.#fit(record, snapshot.task(record.kind));
         if ("reason" in fit)
             return { kind: "blocked", ...fit };
@@ -1184,16 +1546,21 @@ export class TaskScheduler {
         }
         return { kind: "ready", migrates: fit.migrates };
     }
-    #createInvocation(record, mode) {
+    #createInvocation(record, mode, extension) {
         const controller = new AbortController();
         const { promise: done, resolve: finish } = Promise.withResolvers();
         const invocation = {
             taskId: record.id,
+            kind: record.kind,
+            extension,
             conversationId: record.conversationId,
             mode,
             controller,
+            links: record.startedAt === undefined ? record.links : undefined,
             context: withAbortSignal(controller.signal, this.#context),
+            record: record,
             watches: new Set(),
+            timers: new Set(),
             ended: false,
             done,
             finish: () => finish(),
@@ -1211,10 +1578,13 @@ export class TaskScheduler {
                     await this.#runAbort(reservation);
             }
             catch (error) {
-                this.#report(error);
+                this.#report(error, { type: "task_error", ...this.#aboutTask(invocation) }, invocation.context);
             }
             finally {
                 this.#end(invocation);
+                invocation.record = undefined;
+                invocation.links = undefined;
+                invocation.fault = undefined;
                 invocation.finish();
                 this.#kick();
             }
@@ -1226,29 +1596,87 @@ export class TaskScheduler {
         const state = { task: reservation.task, snapshot: reservation.snapshot, reported: undefined };
         const phase = { snapshot: () => state.snapshot, task: () => state.task, agent: undefined };
         const runtime = this.#runtime(invocation, phase);
-        let previous;
-        for (;;) {
-            const current = await this.#step(invocation, (tx, current) => this.#decide(tx, current, previous, state));
-            // Close may seal between the decision and dispatch.
-            if (current === undefined || this.#closing)
-                return;
-            const checkpoint = current.state.checkpoint;
-            // Each phase handler resolves its agent afresh, at first use.
-            phase.agent = undefined;
+        let current = await this.#step(invocation, (tx, next) => this.#decide(tx, next, undefined, state, invocation.extension, invocation.context));
+        while (current !== undefined && !this.#closing) {
+            const task = current;
+            current = await this.#phase(invocation, task, async () => {
+                const checkpoint = task.state.checkpoint;
+                // Each phase handler resolves its agent afresh, at first use.
+                phase.agent = undefined;
+                let previous;
+                try {
+                    await erased(state.task).phases[checkpoint.phase](task, runtime, invocation.context);
+                    previous = { checkpoint };
+                }
+                catch (error) {
+                    previous = { checkpoint, failure: { error } };
+                }
+                return this.#step(invocation, (tx, next) => this.#decide(tx, next, previous, state, invocation.extension, invocation.context));
+            });
+        }
+    }
+    /** A phase owns its decision commit, not the next handler, a durable wait, or recovery. */
+    #phase(invocation, task, run) {
+        const parent = task.trace === undefined
+            ? this.#telemetry
+            : (this.#telemetry.propagation?.extract(task.trace) ?? this.#telemetry);
+        const checkpoint = task.state.checkpoint;
+        const attempt = checkpoint.attempt;
+        const resumed = this.#resumed.delete(task.id);
+        return createTypedSpanStarter(parent, [DURABLE_TELEMETRY_SCHEMA])("pi.durable.phase", {
+            "task.id": task.id,
+            "task.name": task.kind,
+            phase: checkpoint.phase,
+            kind: invocation.mode,
+            resumed,
+            ...(attempt === undefined ? {} : { attempt }),
+        }, async (span) => {
+            if (span.propagation !== undefined && invocation.links !== undefined) {
+                for (const carrier of invocation.links)
+                    span.addLink({ context: span.propagation.extract(carrier) });
+            }
+            const previous = invocation.context;
+            invocation.context = withTelemetry(span, previous);
+            const started = performance.now();
             try {
-                await erased(state.task).phases[checkpoint.phase](current, runtime, invocation.context);
-                previous = { checkpoint };
+                return await run();
             }
-            catch (error) {
-                previous = { checkpoint, failure: { error } };
+            finally {
+                invocation.context = previous;
+                const state = invocation.record.state;
+                const terminal = invocation.fault ??
+                    (state.status === "terminal" || state.status === "completing" ? state.outcome : undefined);
+                const outcome = terminal !== undefined
+                    ? terminal.status === "completed"
+                        ? "done"
+                        : terminal.status === "aborted"
+                            ? "aborted"
+                            : "faulted"
+                    : state.status === "waiting" || state.status === "sleeping"
+                        ? state.status
+                        : invocation.ended || this.#closing
+                            ? "aborted"
+                            : `next=${state.checkpoint.phase}`;
+                span.setAttributes({ outcome });
+                if (terminal?.status === "faulted" || terminal?.status === "failed" || terminal?.status === "orphaned") {
+                    const message = terminal.status === "orphaned" ? terminal.reason : terminal.error.message;
+                    span.setStatus({
+                        status: "error",
+                        error: { name: `task_${terminal.status}`, message: message.slice(0, 500) },
+                    });
+                }
+                else if (outcome === "aborted") {
+                    span.setStatus({ status: "ok" });
+                }
+                this.#durationMetric.record(performance.now() - started, { task: task.kind, phase: checkpoint.phase, outcome }, span);
             }
-        }
+        });
     }
     /**
-     * Precedence rules for a run invocation, on the line. Rules 1 (terminal, `completing`, or `waiting`) and 2 (closing)
-     * are applied by `#step`. Returns whether the invocation continues with the next phase.
+     * Precedence rules for a run invocation, on the line. Rules 1 (terminal, `completing`, `waiting`, or `sleeping`) and 2
+     * (closing) are applied by `#step`. Returns whether the invocation continues with the next phase.
      */
-    #decide(tx, current, previous, state) {
+    #decide(tx, current, previous, state, extension, context) {
         // 3. abort mark: end; a fresh abort invocation starts once the task's ordinary owned work is gone.
         if (current.abortRequested)
             return false;
@@ -1278,8 +1706,11 @@ export class TaskScheduler {
             }
             if (state.reported === undefined || state.reported.task !== next) {
                 state.reported = { task: next };
-                const cause = next === undefined ? "missing_task" : "incompatible_task";
-                this.#report(new Error(`Task ${current.id} keeps running under its old ${current.kind} definition`, { cause }));
+                this.#report(new Error(`Task ${current.id} keeps running under its old ${current.kind} definition`), {
+                    type: "task_outdated",
+                    ...this.#aboutTask({ ...recordTask(current), extension }),
+                    reason: next === undefined ? "missing_task" : "incompatible_task",
+                }, context);
             }
         }
         return true;
@@ -1290,27 +1721,34 @@ export class TaskScheduler {
         const current = this.#live.get(invocation.taskId);
         if (current === undefined || this.#closing)
             return;
-        let failure;
-        try {
-            const phase = { snapshot: () => reservation.snapshot, task: () => reservation.task, agent: undefined };
-            const runtime = this.#runtime(invocation, phase);
-            await erased(reservation.task).abort(current, runtime, invocation.context);
-        }
-        catch (error) {
-            failure = { error };
-        }
-        const message = `Abort handler of task ${invocation.taskId} returned without a terminal outcome`;
-        await this.#step(invocation, () => ({ fault: failure?.error ?? new Error(message) }));
+        await this.#phase(invocation, current, async () => {
+            let failure;
+            try {
+                const phase = {
+                    snapshot: () => reservation.snapshot,
+                    task: () => reservation.task,
+                    agent: undefined,
+                };
+                const runtime = this.#runtime(invocation, phase);
+                await erased(reservation.task).abort(current, runtime, invocation.context);
+            }
+            catch (error) {
+                failure = { error };
+            }
+            const message = `Abort handler of task ${invocation.taskId} returned without a terminal outcome`;
+            await this.#step(invocation, () => ({ fault: failure?.error ?? new Error(message) }));
+        });
     }
     /**
-     * One synchronous decision on the Session line. A task that is no longer running (rule 1: terminal, `completing`, or
-     * `waiting`) or a closing Harness (rule 2) ends the invocation without a write; otherwise `decide` may stage a write
-     * and returns whether the invocation continues. Ending happens inside the callback, before a fault's Harness
-     * cleanup. A rejected step, such as admission after close, also ends the invocation.
+     * One synchronous decision on the Session line. A task that is no longer running (rule 1: terminal, `completing`,
+     * `waiting`, or `sleeping`) or a closing Harness (rule 2) ends the invocation without a write; otherwise `decide` may
+     * stage a write and returns whether the invocation continues. Ending happens inside the callback, before a fault's
+     * Harness cleanup. A rejected step, such as admission after close, also ends the invocation.
      */
     async #step(invocation, decide) {
+        let fault;
         try {
-            return await this.#session.commitWith(async (tx) => {
+            const continued = await this.#session.commitWith(async (tx) => {
                 const found = this.#live.get(invocation.taskId);
                 const current = found?.state.status === "running" ? found : undefined;
                 const decision = current !== undefined && !this.#closing ? decide(tx, current) : false;
@@ -1318,17 +1756,23 @@ export class TaskScheduler {
                     return current;
                 this.#end(invocation);
                 if (decision !== false) {
+                    fault = { error: decision.fault };
                     const message = errorMessage(decision.fault);
-                    await this.#terminate(tx, current, { status: "faulted", error: { message } });
+                    invocation.fault = { status: "faulted", error: { message } };
+                    await this.#terminate(tx, current, invocation.fault);
                 }
                 return undefined;
-            }, this.#context);
+            }, withoutAbortSignal(invocation.context));
+            // Reported once the fault is durable: a host may count it, for example toward a breaker.
+            if (fault !== undefined)
+                this.#report(fault.error, { type: "task_faulted", ...this.#aboutTask(invocation), reason: "threw" }, invocation.context);
+            return continued;
         }
         catch (error) {
             // The step writes the scheduler's own decision, a fault or a terminal record; a throw there would otherwise
             // leave the task running, to be reserved and run again.
             this.#end(invocation);
-            this.#failSession(error);
+            this.#failSession(error, invocation.context);
             return undefined;
         }
     }
@@ -1353,17 +1797,22 @@ export class TaskScheduler {
     async #commitState(tx, invocation, current, next) {
         if (next.status === "waiting")
             await this.#validateWait(tx, invocation, current, next.on, next.policy);
+        if (next.status === "sleeping")
+            validateSleep(invocation, current.id, next.until);
         if (next.status === "terminal") {
             const overlay = overlayOf(tx);
             await this.#loadScopes(false);
             for (const record of overlay.tasks.values())
                 await this.#loadChain(parentOf(record), overlay);
             if (this.#hasOwnedLive(current.id, overlay)) {
-                tx.setTask(withState(current, { status: "completing", outcome: next.outcome }));
+                tx.setTask(withState(progressed(current), { status: "completing", outcome: next.outcome }));
                 return;
             }
         }
-        tx.setTask(withState(current, next));
+        // A state the task committed that differs from its current one is progress: its restart count starts over. The
+        // same checkpoint committed again, for example as intent before an effect, is not.
+        const same = jsonEqual(current.state, next);
+        tx.setTask(withState(same ? current : progressed(current), next));
     }
     /**
      * A wait names existing tasks, and `failFast` only tasks the waiter owns. It may not close a cycle in the wait-for
@@ -1464,6 +1913,9 @@ export class TaskScheduler {
         }
         for (const watch of invocation.watches)
             void watch.stop();
+        for (const disarm of invocation.timers)
+            disarm();
+        invocation.timers.clear();
         // Pending waits bound to the invocation, such as a tool's waitForTask(), reject with it.
         invocation.controller.abort(endedError(invocation));
     }
@@ -1501,7 +1953,7 @@ export class TaskScheduler {
         };
         const hooks = {
             each: async (name, invoke) => {
-                for (const handlers of agentHooks(await agent(), phase.task().definition.name)) {
+                for (const { extension, handlers } of agentHooks(await agent(), phase.task().definition.name)) {
                     const handler = handlers[name];
                     if (typeof handler !== "function")
                         continue;
@@ -1511,17 +1963,20 @@ export class TaskScheduler {
                     catch (error) {
                         if (invocation.controller.signal.aborted)
                             throw error;
-                        this.#report(error);
+                        this.#report(error, { type: "extension_error", extension, task: reportTask(invocation) }, invocation.context);
                     }
                 }
             },
         };
         const settings = this.#settings;
+        const models = this.#models;
         return {
             taskId: invocation.taskId,
             conversationId: invocation.conversationId,
             signal: invocation.controller.signal,
-            models: this.#models,
+            get models() {
+                return models();
+            },
             agent: (context) => invocation.ended ? Promise.reject(endedError(invocation)) : awaitWithContext(agent(), context),
             get settings() {
                 return settings();
@@ -1531,11 +1986,22 @@ export class TaskScheduler {
             get registry() {
                 return phase.snapshot();
             },
-            commit: (change, context) => this.#gated(invocation, async (tx, current) => {
-                const next = await change(tx, current);
-                if (next !== undefined)
-                    await this.#commitState(tx, invocation, current, next);
-            }, context),
+            commit: async (change, context) => {
+                let checkpoint;
+                await this.#gated(invocation, async (tx, current) => {
+                    const next = await change(tx, current);
+                    if (next !== undefined)
+                        await this.#commitState(tx, invocation, current, next);
+                    if (next?.status === "running" && current.state.checkpoint.phase !== next.checkpoint.phase) {
+                        checkpoint = next.checkpoint.phase;
+                    }
+                }, context);
+                if (checkpoint !== undefined) {
+                    invocation.context.telemetry.addEvent("pi.durable.checkpoint", {
+                        phase: checkpoint,
+                    });
+                }
+            },
             memo: ((name, ...rest) => {
                 if (rest.length === 1) {
                     return this.#read(invocation, async () => memoOf(this.#live.get(invocation.taskId), name));
@@ -1549,7 +2015,7 @@ export class TaskScheduler {
                     return candidate;
                 }, rest[1]);
             }),
-            sleep: (until, context) => this.#sleep(invocation, until, context),
+            sleepUntil: (until, checkpoint, context) => this.#gated(invocation, (tx, current) => this.#commitState(tx, invocation, current, { status: "sleeping", checkpoint, until }), context),
             watchDoc: ((...args) => this.#watchDoc(invocation, args)),
             snapshot: ((...args) => this.#read(invocation, () => sessionMethod(this.#session, "snapshot")(...args))),
             snapshotAsOf: ((...args) => this.#read(invocation, () => sessionMethod(this.#session, "snapshotAsOf")(...args))),
@@ -1634,10 +2100,31 @@ export class TaskScheduler {
                     throw endedError(invocation);
                 return this.#now();
             },
-            report: (error) => {
+            setTimer: (callback, delayMs) => {
+                // An ended invocation's timers never fire.
+                if (invocation.ended)
+                    return () => { };
+                const disarm = this.#timers.foreground(() => {
+                    invocation.timers.delete(disarm);
+                    if (!invocation.ended)
+                        callback();
+                }, delayMs);
+                invocation.timers.add(disarm);
+                return () => {
+                    invocation.timers.delete(disarm);
+                    disarm();
+                };
+            },
+            report: (error, from) => {
                 if (invocation.ended)
                     throw endedError(invocation);
-                this.#report(error);
+                // The task's own failure names the task's extension; any other names whose code failed, if anyone's.
+                const task = reportTask(invocation);
+                this.#report(error, from === undefined
+                    ? { type: "task_error", ...this.#aboutTask(invocation) }
+                    : from.type === "extension"
+                        ? { type: "extension_error", task, ...optionalExtension(from.extension) }
+                        : { type: "host_error", callback: from.callback, task }, invocation.context);
             },
         };
     }
@@ -1668,22 +2155,6 @@ export class TaskScheduler {
             return change(tx, current);
         }, context, { conversationId: invocation.conversationId, taskId: invocation.taskId });
     }
-    /** Wait until the Harness clock reaches `until`, rechecking it after every timer. */
-    async #sleep(invocation, until, context) {
-        if (invocation.ended)
-            throw endedError(invocation);
-        const signals = [invocation.controller.signal];
-        if (context.abortSignal !== undefined)
-            signals.push(context.abortSignal);
-        const signal = AbortSignal.any(signals);
-        for (;;) {
-            signal.throwIfAborted();
-            const remaining = until - this.#now();
-            if (remaining <= 0)
-                return;
-            await delay(Math.min(remaining, MAX_TIMER_DELAY), signal);
-        }
-    }
     async #watchDoc(invocation, args) {
         if (invocation.ended)
             throw endedError(invocation);
@@ -1699,9 +2170,29 @@ export class TaskScheduler {
         return watch;
     }
 }
-/** The record with an abort mark: `restart` for an abandonment; without a reason, a request, which replaces `restart`. */
+const REGISTRY = { type: "host_error", callback: "registry" };
+const SETTINGS = { type: "host_error", callback: "settings" };
+const ON_WAKE = { type: "host_error", callback: "onWake" };
+/** The extension that installed `task` in `snapshot`, the last one that lists it; none for a built-in task. */
+function installerOf(snapshot, task) {
+    if (task === undefined)
+        return undefined;
+    return snapshot.installed().findLast((installed) => installed.tasks?.includes(task) === true)?.name;
+}
+/** The report info of a failure of a task. */
+function reportTask(task) {
+    return { id: task.taskId, kind: task.kind, conversationId: task.conversationId };
+}
+/** A stored task's identity, as an invocation names it. */
+function recordTask(record) {
+    return { taskId: record.id, kind: record.kind, conversationId: record.conversationId };
+}
+/**
+ * The record with an abort mark: `restart` for an abandonment; without a reason, a request, which replaces `restart`.
+ * The restart count starts over: it now counts cut-offs of the abort handler.
+ */
 function withAbortMark(record, reason) {
-    const { abortReason: _reason, ...rest } = record;
+    const { abortReason: _reason, restarts: _restarts, ...rest } = record;
     return { ...rest, abortRequested: true, ...(reason === undefined ? {} : { abortReason: reason }) };
 }
 /** A live owner's durable cancellation intent: its abort mark, or a held outcome other than `completed`. */
@@ -1761,11 +2252,18 @@ function erased(task) {
 function endedError(invocation) {
     return new Error(`Task ${invocation.taskId} invocation has ended`);
 }
-/** Replace a live record's state; memos disappear once an outcome is decided. */
+/** The record without its restart count, as after a state the task committed. */
+function progressed(record) {
+    if (record.restarts === undefined)
+        return record;
+    const { restarts: _restarts, ...rest } = record;
+    return rest;
+}
+/** Replace a live record's state; memos and the restart count disappear once an outcome is decided. */
 function withState(record, state) {
     if (state.status !== "terminal" && state.status !== "completing")
         return { ...record, state };
-    const { memos: _memos, ...rest } = record;
+    const { memos: _memos, restarts: _restarts, ...rest } = record;
     return { ...rest, state };
 }
 /** Whether a definition can take the task at reservation: same version, or newer with a migration. */
@@ -1773,18 +2271,13 @@ function canReserve(task, record) {
     const definition = task.definition;
     return (definition.version === record.version || (definition.version > record.version && definition.migrate !== undefined));
 }
-function delay(ms, signal) {
-    return new Promise((resolve, reject) => {
-        const onAbort = () => {
-            clearTimeout(timer);
-            reject(signal.reason);
-        };
-        const timer = setTimeout(() => {
-            signal.removeEventListener("abort", onAbort);
-            resolve();
-        }, ms);
-        signal.addEventListener("abort", onAbort, { once: true });
-    });
+/** A sleep's `until` is a time on the Harness clock; an abort handler cannot sleep. */
+function validateSleep(invocation, task, until) {
+    if (invocation.mode === "abort")
+        throw new Error(`Abort handler of task ${task} cannot sleep`);
+    if (typeof until !== "number" || !Number.isFinite(until) || until < 0) {
+        throw new Error(`Task ${task} cannot sleep until ${until}: not a time in milliseconds`);
+    }
 }
 /** Structural equality of two JSON values; object key order is ignored. */
 function jsonEqual(left, right) {