@vibesdotdev/workers 0.0.0-stage → 1.0.0-rc.3

package/dist/runtime/worker-runner.js

@@ -0,0 +1,970 @@
+/** Bridges a runtime worker implementation to a QueueAdapter (dispatch by metadata.type). */
+import { jobWriteGuard, JobExecutionOperations } from "../core/queue-adapter.js";
+import { shouldRetryJob, parseJobInput, readJobTimeoutMs, runWithJobTimeout } from "./job-contract.js";
+import { retryDelayMs } from "../core/enqueue-metadata.js";
+import { QUEUED_INVOCATION_METADATA_KEY, QueuedInvocationSchema } from "../core/queued-invocation.js";
+import { recordJobRun } from "./ledger/record-job-run.js";
+import { getLogger } from '@vibesdotdev/logging';
+import { CaughtFailureSchema, CapabilityError, getRuntimeScopedValues, isCapabilityError, valueFreeFailure } from '@vibesdotdev/kernel';
+import * as z from 'zod/v4';
+import { settle, settleSync } from "../core/settle.js";
+import { jobRefusalOf, JOB_REFUSAL_METADATA_KEY, JOB_FAILURE_FACTS_METADATA_KEY } from "../core/job-refusal.js";
+import { createJobEvents, createSubtasks, getJobMetadata, getJobType, getMaxAttempts, readMetadataString, resolveJobStallThresholdMs } from "./job-context.js";
+import { resolveUsageAttributionHardware } from "./usage-hardware.js";
+import { servedJobTypes } from "./served-jobs.js";
+import { SUPERVISED_METADATA_KEY } from "../core/supervised-workloads.js";
+import { runAsExecutingJob } from "../core/keyed-dispatch.js";
+import { recordDeadLetter } from "../dead-letter/dead-letters.js";
+const JOB_HEARTBEAT_INTERVAL_MS = 30_000;
+const SERVED_JOBS_REFRESH_MS = 30_000;
+const JOB_CANCELLATION_POLL_INTERVAL_MS = 2_000;
+const JOB_DRAIN_TIMEOUT_MS = 30_000;
+const HEARTBEAT_FAILURE_WARN_THRESHOLD = 3;
+/**
+ * A job-scoped tick on the runtime's lifecycle timers (RC07). Each run arms
+ * the next one-shot `lifecycle.timeout` only after it settles, so runs never
+ * overlap, and nothing is armed once the runtime's lifecycle signal aborts.
+ * A one-shot chain rather than `lifecycle.interval`: a job running in a
+ * finite host (a CLI drain) still owes its lease a heartbeat until it ends,
+ * and the tick lives exactly as long as that job. A run that rejects is
+ * logged and the chain continues; the returned disposer stops it.
+ */
+function startJobTick(runtime, logger, name, everyMs, run) {
+    let stopped = false;
+    let disarm = null;
+    const arm = () => {
+        if (stopped || runtime.lifecycle.signal.aborted)
+            return;
+        disarm = runtime.lifecycle.timeout(() => {
+            disarm = null;
+            void settle(run).then((ticked) => {
+                if (!ticked.ok)
+                    logger.error('Job tick failed', { tick: name, error: valueFreeFailure(ticked.caught) });
+                arm();
+            });
+        }, everyMs);
+    };
+    arm();
+    return () => {
+        stopped = true;
+        disarm?.();
+        disarm = null;
+    };
+}
+function isLifecycleEnd(failure) {
+    return (isCapabilityError(failure) &&
+        (failure.code === 'TIMEOUT' || failure.code === 'CANCELLED' || failure.reason === 'runtime-disposed'));
+}
+/** The pause ended because the loop is stopping (stop() or runtime disposal), not because it failed. */
+function isLoopShutdown(failure) {
+    return isCapabilityError(failure) && (failure.code === 'CANCELLED' || failure.reason === 'runtime-disposed');
+}
+/** The handler ids a worker declares; a list that is not ids marks the worker invalid. */
+const DeclaredHandlerIdsSchema = z.array(z.string());
+/** A worker implementation that carries its `getHandlerIds` method. */
+function declaresHandlerIds(value) {
+    return typeof value === 'object' && value !== null && 'getHandlerIds' in value && typeof value.getHandlerIds === 'function';
+}
+/**
+ * A queued job's payload as its implementation receives it. Producers enqueue
+ * a `workers/job`'s JSON input (every dispatch path takes the job's own input),
+ * and the queue returns it unchanged.
+ */
+function queuedJobData(job) {
+    // SAFETY: the queue stores and returns the enqueued payload unchanged, and
+    // every producer of a workers/job enqueues that job's JSON object input.
+    return job.data;
+}
+/**
+ * Walk a failure's cause chain into loggable detail (drizzle wraps the real
+ * D1 HTTP failure in a generic "Failed query" — without the chain the log
+ * cannot say WHY a dequeue failed).
+ */
+function describeFailure(failure) {
+    if (!(failure instanceof Error))
+        return { error: failure.description };
+    const causes = [];
+    let cursor = failure.cause;
+    for (let depth = 0; depth < 3 && cursor; depth += 1) {
+        const cause = CaughtFailureSchema.parse(cursor);
+        causes.push(cause instanceof Error ? cause.message : cause.description);
+        cursor = cause instanceof Error ? cause.cause : undefined;
+    }
+    return causes.length > 0 ? { error: failure.message, cause: causes.join(' ← ') } : { error: failure.message };
+}
+/**
+ * `work`'s outcome, or the signal's reason once it aborts first. Its selected
+ * owner still joins abandoned work: child jobs own their work, and Runtime
+ * retains a timed handler after its wait has ended.
+ */
+function untilAborted(work, signal) {
+    if (!signal)
+        return work;
+    if (signal.aborted)
+        return Promise.reject(signal.reason);
+    return new Promise((resolve, reject) => {
+        const abort = () => reject(signal.reason);
+        signal.addEventListener('abort', abort, { once: true });
+        void settle(() => work).then((outcome) => {
+            signal.removeEventListener('abort', abort);
+            if (outcome.ok)
+                resolve(outcome.value);
+            else
+                reject(outcome.thrown);
+        });
+    });
+}
+export class WorkerRunner {
+    runtime;
+    worker;
+    queue;
+    adapter;
+    options;
+    logger;
+    runningTasks = new Set();
+    activeJobs = new Set();
+    /** Active jobs waiting on children or producer settlement do not hold a handler slot. */
+    parkedJobs = 0;
+    /** Parked jobs whose wait is over, queued to take back a slot (handed over the moment one frees, before any claim). */
+    resumeWaiters = [];
+    running = false;
+    /** Aborted by stop() so an idle worker loop wakes now, not a poll interval later. */
+    loopIdle = null;
+    unsubscribeJobAvailable = null;
+    dequeueInProgress = false;
+    maxConcurrency = 1;
+    workerHandlerIdsInvalid = false;
+    consecutiveDequeueFailures = 0;
+    dequeueBackoffUntil = 0;
+    consecutiveHeartbeatFailures = 0;
+    activeCancellationControllers = new Set();
+    servedJobs;
+    constructor(runtime, worker, queue, adapter, options) {
+        this.runtime = runtime;
+        this.worker = worker;
+        this.queue = queue;
+        this.adapter = adapter;
+        this.options = options;
+        this.logger =
+            options?.logger ??
+                getLogger('worker-runtime', runtime)
+                    .child('runner')
+                    .withContext({
+                    workerId: worker.id,
+                    queue: worker.descriptor?.queue
+                });
+    }
+    /**
+     * The job types this worker claims: every job the registry puts on the
+     * worker's queue, plus any handler ids the worker declares. Refreshed so
+     * jobs registered after start are served too. An empty set claims nothing.
+     */
+    resolveJobTypes() {
+        const now = Date.now();
+        if (!this.servedJobs || now - this.servedJobs.at >= SERVED_JOBS_REFRESH_MS) {
+            const declared = DeclaredHandlerIdsSchema.safeParse(declaresHandlerIds(this.worker) ? this.worker.getHandlerIds() : []);
+            if (!declared.success) {
+                this.workerHandlerIdsInvalid = true;
+                this.logger.error('Skipping worker with invalid handler id list', {
+                    workerId: this.worker.id,
+                    issues: declared.error.message
+                });
+                return null;
+            }
+            const descriptor = this.worker.descriptor ?? this.runtime.assets('workers/worker').descriptor(this.worker.id);
+            const derived = descriptor
+                ? servedJobTypes(this.runtime, String(this.runtime.scope?.hardware), descriptor)
+                : [];
+            this.servedJobs = { at: now, ids: [...new Set([...declared.data, ...derived])] };
+        }
+        return this.servedJobs.ids.length > 0 ? this.servedJobs.ids : null;
+    }
+    async start() {
+        if (this.running)
+            return;
+        this.running = true;
+        const requestedConcurrency = this.worker.descriptor?.concurrency ?? 1;
+        this.maxConcurrency = Math.max(1, requestedConcurrency);
+        this.loopIdle = new AbortController();
+        for (let i = 0; i < this.maxConcurrency; i++) {
+            this.runningTasks.add(this.workerLoop());
+        }
+        // Polling is the worker loops' lifecycle-owned delay (workerLoop);
+        // no separate poll timer runs beside them.
+        if (this.queue.onJobAvailable) {
+            this.unsubscribeJobAvailable = this.queue.onJobAvailable(() => void this.tryDequeue());
+        }
+        this.logger.info('Worker runner started', {
+            workerId: this.worker.id,
+            concurrency: this.maxConcurrency
+        });
+    }
+    async stop() {
+        if (!this.running)
+            return;
+        this.running = false;
+        this.loopIdle?.abort();
+        this.loopIdle = null;
+        // Stopping: parked jobs resume at once so they can unwind.
+        for (const resume of this.resumeWaiters.splice(0)) {
+            this.parkedJobs -= 1;
+            resume();
+        }
+        this.unsubscribeJobAvailable?.();
+        this.unsubscribeJobAvailable = null;
+        // Each task's own failure is logged where it ran (runJobInvocation's catch); stop() only joins them.
+        await Promise.allSettled(this.runningTasks.values());
+        // Drain with a deadline: a hung job must not make the daemon unkillable
+        // except by SIGKILL. Past the deadline, abort every in-flight job's
+        // cancellation signal and wait once more for the handlers to unwind.
+        // Aborting the logical wait does not retire a producer. Its queue claim
+        // stays held until the owned continuation joins it and its issued writes.
+        // The drain is a bounded lifecycle wait: it ends at the deadline, when
+        // runtime disposal begins, or (inside a plugin's onDispose) when the
+        // finalization budget runs out, whichever comes first.
+        const drainTimeoutMs = this.options?.drainTimeoutMs ?? JOB_DRAIN_TIMEOUT_MS;
+        const drain = await settle(() => this.runtime.lifecycle.wait(() => this.waitForActiveJobs(), {
+            timeoutMs: drainTimeoutMs,
+            reason: 'worker-runner-drain'
+        }));
+        if (!drain.ok && !isLifecycleEnd(drain.caught))
+            throw drain.thrown;
+        if (!drain.ok) {
+            this.logger.warn('Worker runner drain timed out; aborting in-flight jobs', {
+                workerId: this.worker.id,
+                drainTimeoutMs,
+                activeJobs: this.activeJobs.size
+            });
+            for (const controller of this.activeCancellationControllers) {
+                controller.abort(new Error(`Worker runner drain timed out after ${drainTimeoutMs}ms`));
+            }
+            this.activeCancellationControllers.clear();
+            await this.waitForActiveJobs();
+        }
+        this.runningTasks.clear();
+        this.logger.info('Worker runner stopped', { workerId: this.worker.id });
+    }
+    async workerLoop() {
+        // An EMPTY queue sleeps a full poll interval — the old 50ms idle spin
+        // meant ~20 SELECTs/sec/slot against D1 over HTTP for a quiet queue
+        // (2 pods × 2 slots ≈ 80 qps of no-op polls), which is both cost and
+        // the likely source of the intermittent transient failures. The short
+        // yield remains only AFTER a processed job, to drain a hot queue fast.
+        //
+        // Poll pacing (RC07): owner Workers. A queue that emits job-available
+        // (`onJobAvailable`: IPC, memory) is also driven by that event from
+        // start(); the pause below exists for adapters that have no such event
+        // (the database adapter), and is a lifecycle-owned delay.
+        const idleSleepMs = this.options?.pollIntervalMs ?? 1000;
+        const idle = this.loopIdle?.signal;
+        while (this.running) {
+            const job = await this.tryDequeue();
+            // stop() or runtime disposal cancels this pause: shutdown never
+            // waits out a poll interval.
+            const paused = await settle(() => this.runtime.lifecycle.delay(job ? 50 : idleSleepMs, idle));
+            if (!paused.ok && isLoopShutdown(paused.caught))
+                break;
+            if (!paused.ok)
+                throw paused.thrown;
+        }
+    }
+    /** A parked job takes its slot back, waiting (first come, first served) while every slot is held. */
+    async unpark(parking) {
+        if (!this.running || (this.resumeWaiters.length === 0 && this.activeJobs.size - this.parkedJobs < this.maxConcurrency)) {
+            this.parkedJobs -= 1;
+            return;
+        }
+        await new Promise((resume) => {
+            parking.resume = resume;
+            this.resumeWaiters.push(resume);
+        });
+        delete parking.resume;
+    }
+    /** The handler ended (returned, threw or timed out): a job still parked stops counting, and leaves the resume queue. */
+    endParking(parking) {
+        if (parking.ended)
+            return;
+        parking.ended = true;
+        const queued = parking.resume ? this.resumeWaiters.indexOf(parking.resume) : -1;
+        if (queued >= 0)
+            this.resumeWaiters.splice(queued, 1);
+        if (parking.depth > 0 || queued >= 0) {
+            this.parkedJobs -= 1;
+            this.resumeParked();
+        }
+        parking.resume?.();
+    }
+    /** Hand freed slots to parked jobs waiting to resume (each stops counting as parked as it resumes). */
+    resumeParked() {
+        while (this.resumeWaiters.length > 0 && this.activeJobs.size - this.parkedJobs < this.maxConcurrency) {
+            this.parkedJobs -= 1;
+            this.resumeWaiters.shift()();
+        }
+    }
+    async tryDequeue() {
+        if (!this.running || this.workerHandlerIdsInvalid)
+            return null;
+        if (this.dequeueInProgress)
+            return null;
+        if (this.activeJobs.size - this.parkedJobs >= this.maxConcurrency)
+            return null;
+        if (Date.now() < this.dequeueBackoffUntil)
+            return null;
+        this.dequeueInProgress = true;
+        // A successful claim releases `dequeueInProgress` the moment it settles,
+        // so a job-available signal arriving right after is served at once.
+        const dequeued = await settle(() => this.claimNextJob());
+        if (dequeued.ok)
+            return dequeued.value;
+        // Transient adapter failures (D1 HTTP blips) back off exponentially
+        // instead of re-erroring every poll: 1s, 2s, 4s … capped at 30s.
+        // The first failure and every 5th thereafter log at error (with the
+        // cause chain); the rest at warn — one blip is noise, a streak is an
+        // outage and stays loud.
+        this.consecutiveDequeueFailures += 1;
+        const backoffMs = Math.min(30_000, 1000 * 2 ** (this.consecutiveDequeueFailures - 1));
+        this.dequeueBackoffUntil = Date.now() + backoffMs;
+        const detail = {
+            ...describeFailure(dequeued.caught),
+            consecutiveFailures: this.consecutiveDequeueFailures,
+            backoffMs
+        };
+        if (this.consecutiveDequeueFailures === 1 || this.consecutiveDequeueFailures % 5 === 0) {
+            this.logger.error('Failed to dequeue job', detail);
+        }
+        else {
+            this.logger.warn('Failed to dequeue job (backing off)', detail);
+        }
+        this.dequeueInProgress = false;
+        return null;
+    }
+    /** Claim the next job this worker serves and start running it; `null` when there is none (or nothing to serve). */
+    claimNextJob() {
+        const jobTypes = this.resolveJobTypes();
+        if (!jobTypes)
+            return Promise.resolve(this.releaseDequeue(null));
+        const { promise, resolve, reject } = Promise.withResolvers();
+        try {
+            const owned = this.runtime.lifecycle.own(async () => {
+                const heartbeatIntervalMs = this.options?.heartbeatIntervalMs ?? JOB_HEARTBEAT_INTERVAL_MS;
+                const job = await this.queue.dequeue({
+                    jobTypes,
+                    executionOwner: this.worker.id,
+                    claimMetadata: {
+                        workerId: this.worker.id,
+                        workerHeartbeatAt: new Date().toISOString(),
+                        workerHeartbeatState: 'claimed',
+                        workerHeartbeatIntervalMs: heartbeatIntervalMs,
+                        // The job runs in this process. Whatever starts a PM workload for it appends that
+                        // workload before it starts; a restore retires the job only on this explicit record.
+                        [SUPERVISED_METADATA_KEY]: []
+                    }
+                });
+                this.consecutiveDequeueFailures = 0;
+                if (!job) {
+                    resolve(this.releaseDequeue(null));
+                    return;
+                }
+                const processing = settle(() => this.runJobInvocation(job)).then((ran) => {
+                    if (!ran.ok)
+                        this.logger.error('Unhandled job processing error', { jobId: job.id, error: valueFreeFailure(ran.caught) });
+                }).finally(() => {
+                    this.activeJobs.delete(processing);
+                    this.resumeParked();
+                });
+                this.activeJobs.add(processing);
+                resolve(this.releaseDequeue(job));
+                await processing;
+            });
+            void owned.catch(reject);
+        }
+        catch (error) {
+            CaughtFailureSchema.parse(error);
+            reject(error);
+        }
+        return promise;
+    }
+    /** End this claim: the next job-available signal may dequeue at once. (A failed claim is released by tryDequeue after its backoff is set.) */
+    releaseDequeue(job) {
+        this.dequeueInProgress = false;
+        return job;
+    }
+    async waitForActiveJobs() {
+        // Each active job carries its own logging `.catch` (see processing above); this only joins them.
+        while (this.activeJobs.size > 0)
+            await Promise.allSettled(this.activeJobs.values());
+    }
+    /**
+     * Run one job as its own invocation, acting as the envelope actor the
+     * producer stamped from its host (`metadata.actorUserId`, never caller
+     * input), with its organization. A job without one runs with no actor.
+     */
+    async runJobInvocation(job) {
+        let started = false;
+        try {
+            const metadata = getJobMetadata(job);
+            const userId = readMetadataString(metadata, 'actorUserId');
+            const orgId = readMetadataString(metadata, 'organizationId');
+            const actor = userId ? { userId } : undefined;
+            if (actor && orgId)
+                actor.orgId = orgId;
+            const invocation = Object.hasOwn(metadata, QUEUED_INVOCATION_METADATA_KEY)
+                ? QueuedInvocationSchema.parse(metadata[QUEUED_INVOCATION_METADATA_KEY])
+                : { actor };
+            // A queue can wake this loop from a producer's request; the job carries
+            // only its queued invocation, never that producer's request values.
+            const isolate = await this.requestValuesIsolation();
+            await isolate(() => this.runtime.runOutsideInvocation(() => this.runtime.runInvocation(() => {
+                started = true;
+                return this.processJob(job, metadata);
+            }, { ...invocation, actorReplaces: true })));
+        }
+        catch (error) {
+            if (!started && job.executionClaim.state === 'held') {
+                const identity = job.executionClaim.identity;
+                const recorded = await settle(() => this.queue.updateJob(job.id, {
+                    status: 'failed', error: 'Job invocation could not be admitted'
+                }, jobWriteGuard(job)));
+                if (!recorded.ok)
+                    this.runtime.diagnostics.error('job-invocation-record-failed', {
+                        scope: 'workers:runner', plugin: 'workers', error: recorded.caught
+                    });
+                const released = await settle(() => this.queue.releaseExecution(job.id, identity));
+                if (!released.ok)
+                    this.runtime.diagnostics.error('job-invocation-release-failed', {
+                        scope: 'workers:runner', plugin: 'workers', error: released.caught
+                    });
+            }
+            throw error;
+        }
+    }
+    /** The host's empty request-values scope; a host without request values has none to leak. */
+    async requestValuesIsolation() {
+        const selected = await settle(() => getRuntimeScopedValues(this.runtime));
+        if (selected.ok)
+            return callback => selected.value.isolate(callback);
+        const failure = selected.caught;
+        if (isCapabilityError(failure) && failure.code === 'UNAVAILABLE' && failure.reason === 'scoped-values-unavailable')
+            return callback => callback();
+        throw selected.thrown;
+    }
+    async processJob(job, metadata) {
+        const claim = job.executionClaim;
+        if (claim.state !== 'held')
+            throw new CapabilityError('The queue did not admit an execution claim.', 'INTERNAL', {
+                reason: 'job-execution-claim-missing'
+            });
+        const operations = new JobExecutionOperations(this.queue, job);
+        let claimReleased = false;
+        const releaseClaim = async () => {
+            if (claimReleased)
+                return;
+            await this.queue.releaseExecution(job.id, claim.identity);
+            claimReleased = true;
+        };
+        const jobType = getJobType(job);
+        if (!jobType) {
+            try {
+                await operations.update({ status: 'failed', error: 'Job missing metadata.type' });
+            }
+            finally {
+                const retired = await settle(() => operations.retire());
+                if (!retired.ok)
+                    this.runtime.diagnostics.error('job-issued-operation-failed', {
+                        scope: 'workers:runner', plugin: 'workers', error: retired.caught
+                    });
+                await releaseClaim();
+            }
+            return;
+        }
+        const spawnedQueues = new Map();
+        const attempt = claim.identity.attempt;
+        const maxAttempts = getMaxAttempts(job, metadata);
+        const heartbeatIntervalMs = this.options?.heartbeatIntervalMs ?? JOB_HEARTBEAT_INTERVAL_MS;
+        let stopHeartbeat = null;
+        const stopHeartbeatNow = () => {
+            stopHeartbeat?.();
+            stopHeartbeat = null;
+        };
+        let heartbeatInFlight = null;
+        let logicalEnded = false;
+        let observeAttemptStall;
+        let resetAttemptStallWindow;
+        let stopCancellationPoll;
+        let unsubscribeCancellation;
+        let cancellationCheckInFlight = null;
+        // The provenance seen while detecting the cancellation. The ledger record
+        // uses it rather than re-reading the queue, so a cancelled run is never
+        // recorded unattributed just because a later read failed.
+        let observedCancellation;
+        const cancellationController = new AbortController();
+        this.activeCancellationControllers.add(cancellationController);
+        // This job is lifecycle-owned work (claimNextJob), and disposal joins owned
+        // work before any plugin's onDispose can stop the runner: the handler's
+        // signal also ends when disposal begins, or an untimed handler holds
+        // disposal for the whole finalization budget. Disposal is not cancellation:
+        // nothing is recorded or retried after it begins; the job is left processing
+        // for the next owner to recover, as on a drain timeout.
+        const handlerSignal = AbortSignal.any([cancellationController.signal, this.runtime.lifecycle.signal]);
+        const runStartedAt = new Date();
+        let runRecorded = false;
+        const recordRun = async (status, failure) => {
+            if (runRecorded)
+                return;
+            const run = {
+                jobId: job.id,
+                jobType,
+                mode: 'background',
+                status,
+                attempt,
+                durationMs: Math.round(Date.now() - runStartedAt.getTime()),
+                hardware: String(this.runtime.scope?.hardware ?? 'unknown'),
+                startedAt: runStartedAt.toISOString()
+            };
+            if (failure !== undefined)
+                run.failure = failure;
+            if (status === 'cancelled') {
+                // Prefer the provenance already observed. Only when nothing observed
+                // it is the queue read again, and a failed read is surfaced rather than
+                // silently recording a current cancellation as an unattributed row.
+                if (observedCancellation)
+                    run.cancellation = observedCancellation;
+                else {
+                    const cancelledJob = await settle(() => this.queue.getJob(job.id));
+                    if (cancelledJob.ok) {
+                        if (cancelledJob.value?.cancellation)
+                            run.cancellation = cancelledJob.value.cancellation;
+                    }
+                    else {
+                        this.logger.warn('Cancelled job provenance could not be read; no ledger row was recorded.', {
+                            jobId: job.id,
+                            jobType,
+                            ...describeFailure(cancelledJob.caught)
+                        });
+                    }
+                }
+            }
+            if (status === 'cancelled' && !run.cancellation) {
+                throw new CapabilityError('Cancellation provenance is unavailable; the ledger row was not recorded.', 'UNAVAILABLE', {
+                    reason: 'job-cancellation-provenance-unavailable', details: { jobId: job.id }
+                });
+            }
+            runRecorded = true;
+            await recordJobRun(this.runtime, run);
+        };
+        const abortIfCancelled = async () => {
+            if (cancellationController.signal.aborted)
+                return true;
+            if (cancellationCheckInFlight)
+                return cancellationCheckInFlight;
+            cancellationCheckInFlight = (async () => {
+                const read = await settle(() => this.queue.getJob(job.id));
+                if (!read.ok) {
+                    // An unreadable status is not a cancellation: log it and let the
+                    // next check (or the queue's cancelled event) decide.
+                    this.logger.warn('Job cancellation check could not read the job', {
+                        jobId: job.id,
+                        jobType,
+                        ...describeFailure(read.caught)
+                    });
+                    return false;
+                }
+                if (read.value?.generation !== job.generation || read.value.status !== 'cancelled')
+                    return false;
+                if (read.value.cancellation)
+                    observedCancellation = read.value.cancellation;
+                if (!cancellationController.signal.aborted) {
+                    cancellationController.abort(new Error(`Job cancelled: ${job.id}`));
+                }
+                return true;
+            })();
+            try {
+                return await cancellationCheckInFlight;
+            }
+            finally {
+                cancellationCheckInFlight = null;
+            }
+        };
+        const writeHeartbeat = async (state) => {
+            if (heartbeatInFlight)
+                return heartbeatInFlight;
+            const nowIso = new Date().toISOString();
+            heartbeatInFlight = (async () => {
+                const written = await settle(async () => {
+                    await operations.renew();
+                    if (logicalEnded || await abortIfCancelled())
+                        return;
+                    await operations.update({
+                        metadata: {
+                            workerId: this.worker.id,
+                            jobType,
+                            attempt,
+                            workerHeartbeatAt: nowIso,
+                            workerHeartbeatState: state,
+                            workerHeartbeatIntervalMs: heartbeatIntervalMs
+                        }
+                    });
+                    if (this.queue.broadcastEvent) {
+                        await operations.emit('heartbeat', {
+                            workerId: this.worker.id,
+                            jobType,
+                            attempt,
+                            state,
+                            heartbeatAt: nowIso,
+                            heartbeatIntervalMs
+                        });
+                    }
+                    if (state === 'running' && !logicalEnded && observeAttemptStall) {
+                        await observeAttemptStall();
+                    }
+                    this.consecutiveHeartbeatFailures = 0;
+                });
+                if (written.ok)
+                    return;
+                // A missed heartbeat is suspicion, never retirement authority.
+                // Keep the claim held and surface sustained native renewal failures.
+                this.consecutiveHeartbeatFailures += 1;
+                const detail = {
+                    jobId: job.id,
+                    jobType,
+                    error: valueFreeFailure(written.caught),
+                    consecutiveHeartbeatFailures: this.consecutiveHeartbeatFailures
+                };
+                if (this.consecutiveHeartbeatFailures >= HEARTBEAT_FAILURE_WARN_THRESHOLD) {
+                    this.logger.warn('Job execution renewal updates failing repeatedly', detail);
+                }
+                else {
+                    this.logger.debug('Job heartbeat update failed', detail);
+                }
+            })();
+            try {
+                await heartbeatInFlight;
+            }
+            finally {
+                heartbeatInFlight = null;
+            }
+        };
+        let producer;
+        try {
+            const attempted = await settle(async () => {
+                if (this.queue.subscribeToEvents) {
+                    unsubscribeCancellation = this.queue.subscribeToEvents(job.id, (event) => {
+                        if (event.stamp?.generation === job.generation && event.event === 'cancelled' && !cancellationController.signal.aborted) {
+                            cancellationController.abort(new Error(`Job cancelled: ${job.id}`));
+                        }
+                    });
+                }
+                // Cancellation poll (RC07 named poll). Owner: the job's queue adapter.
+                // `subscribeToEvents` delivers `cancelled` in-process and through a
+                // composed job-event bus, but no adapter guarantees a cross-process
+                // cancellation event (the database queue without a bus, IPC across
+                // hosts), so the job re-reads its own status on a job-scoped tick.
+                stopCancellationPoll = startJobTick(this.runtime, this.logger, 'cancellation-poll', this.options?.cancellationPollIntervalMs ?? JOB_CANCELLATION_POLL_INTERVAL_MS, async () => {
+                    await abortIfCancelled();
+                });
+                await operations.update({ status: 'processing' });
+                await writeHeartbeat('started');
+                stopHeartbeat = startJobTick(this.runtime, this.logger, 'heartbeat', heartbeatIntervalMs, () => writeHeartbeat('running'));
+                const impl = await this.runtime
+                    .query('workers/job')
+                    .withId(jobType)
+                    .forHardware(this.runtime.scope.hardware)
+                    .enabled()
+                    .resolve();
+                // Materialize the descriptor's input-schema `.default()` values BEFORE
+                // the impl sees the data. Producers/enqueue paths only forward the fields
+                // a caller supplied, so omitted-but-defaulted fields (e.g. pdf-ocr's
+                // maxPages/scaleTo/dpi) would otherwise reach handle() as `undefined` and
+                // silently break defaulted behavior. Data that does not satisfy the job's
+                // own schema is a terminal failure: the handler never runs on it, and a
+                // retry would fail the same way. Runtime-level so EVERY workers/job gets
+                // the same contract, not a per-impl bolt-on.
+                const timeoutMs = readJobTimeoutMs(impl.descriptor);
+                const contextData = parseJobInput(impl.descriptor, jobType, queuedJobData(job));
+                // Ending the attempt releases its slot; Runtime still owns an unfinished timed handler.
+                const parking = { depth: 0, ended: false };
+                // A park honors the signal of the handler it is handed to: a deadline, cancellation or
+                // disposal ends the wait, so the handler can unwind and Runtime can join it.
+                const parkFor = (signal) => async (work) => {
+                    if (parking.ended)
+                        return operations.own(work);
+                    // The job gives up its one slot while it only waits — once, however many children it awaits at once.
+                    if (parking.depth === 0) {
+                        this.parkedJobs += 1;
+                        this.resumeParked();
+                        resetAttemptStallWindow?.();
+                    }
+                    parking.depth += 1;
+                    try {
+                        return await untilAborted(operations.own(work), signal);
+                    }
+                    finally {
+                        parking.depth -= 1;
+                        // Resuming takes a slot back: at once when one is free, else the next one that frees.
+                        if (parking.depth === 0 && !parking.ended) {
+                            await this.unpark(parking);
+                            resetAttemptStallWindow?.();
+                        }
+                    }
+                };
+                const stallThresholdMs = await resolveJobStallThresholdMs(this.runtime, this.options?.stallThresholdMs);
+                const jobEvents = createJobEvents(operations, {
+                    jobId: job.id,
+                    workerId: this.worker.id,
+                    queueName: this.worker.descriptor?.queue ?? jobType,
+                    stallThresholdMs,
+                    now: this.options?.now,
+                    isParked: () => logicalEnded ||
+                        parking.ended ||
+                        parking.depth > 0 ||
+                        parking.resume !== undefined,
+                    eventSink: this.options?.eventSink,
+                    initialMetadata: metadata
+                });
+                observeAttemptStall = () => jobEvents.observeStall();
+                resetAttemptStallWindow = () => jobEvents.resetStallWindow();
+                const context = {
+                    jobId: job.id,
+                    data: contextData,
+                    attempt,
+                    maxAttempts,
+                    metadata,
+                    events: jobEvents,
+                    subtasks: createSubtasks({
+                        runtime: this.runtime,
+                        adapter: this.adapter,
+                        parentQueue: this.queue,
+                        parentJob: job,
+                        operations,
+                        spawnedQueues,
+                        hardware: this.runtime.scope.hardware,
+                        park: parkFor(undefined)
+                    }),
+                    signal: handlerSignal,
+                    park: parkFor(handlerSignal)
+                };
+                if (impl.beforeHandle)
+                    await impl.beforeHandle(context);
+                if (await abortIfCancelled())
+                    return;
+                // Usage/billing follows the declared hosting owner, not the execution ABI.
+                const hardware = resolveUsageAttributionHardware(this.runtime.scope);
+                const billingUserId = readMetadataString(metadata, 'billingUserId') ?? null;
+                const organizationId = readMetadataString(metadata, 'organizationId') ?? null;
+                const surface = `worker.${jobType}`;
+                const handleStartedAt = performance.now();
+                let result;
+                let handleFailed = true;
+                let metrics;
+                try {
+                    result = await runWithJobTimeout(this.runtime.lifecycle, jobType, timeoutMs, handlerSignal, (signal) => {
+                        // Children this execution dispatches are keyed by it, so the job re-entered after a restart finds them (keyed-dispatch.ts).
+                        const park = parkFor(signal);
+                        const emit = context.events?.emit.bind(context.events);
+                        const notify = this.queue.notifyJobChanged?.bind(this.queue);
+                        const changed = notify ? () => notify(job.id) : undefined;
+                        const receipt = this.queue.queueId === undefined ? undefined : {
+                            adapterId: this.adapter.descriptor.id, queueId: this.queue.queueId, generation: job.generation
+                        };
+                        producer = runAsExecutingJob(this.runtime, { jobId: job.id, attempt, park, emit, changed, receipt }, () => impl.handle({ ...context, signal, park }));
+                        return producer;
+                    });
+                    handleFailed = false;
+                }
+                finally {
+                    this.endParking(parking);
+                    const aggregator = this.options?.aggregator;
+                    if (aggregator) {
+                        const seconds = (performance.now() - handleStartedAt) / 1000;
+                        metrics = settleSync(() => aggregator.record({
+                            billingUserId,
+                            organizationId,
+                            surface,
+                            hardware,
+                            seconds,
+                            jobId: job.id
+                        }));
+                        if (handleFailed && !metrics.ok) {
+                            // The handler refusal stays primary; report the independent
+                            // metrics failure through Runtime's non-throwing diagnostics.
+                            this.runtime.diagnostics.error('job-attempt-metrics-failed', {
+                                scope: 'workers:runner',
+                                plugin: jobType,
+                                error: metrics.caught
+                            });
+                        }
+                    }
+                }
+                if (metrics && !metrics.ok)
+                    throw metrics.thrown;
+                if (await abortIfCancelled())
+                    return;
+                if (impl.afterHandle)
+                    await impl.afterHandle(context, result);
+                // A handler that returned once disposal ended its signal may hold a
+                // partial result: leave the job processing for the next owner.
+                if (await abortIfCancelled() || this.runtime.lifecycle.signal.aborted)
+                    return;
+                await writeHeartbeat('completed');
+                if (await abortIfCancelled())
+                    return;
+                logicalEnded = true;
+                const stamp = await operations.update({ status: 'completed', result });
+                await recordRun('completed');
+                await operations.emit('completed', { result }, Object.freeze({
+                    ...operations.guard, expectedStatus: stamp.status
+                }));
+            });
+            if (attempted.ok)
+                return;
+            const error = attempted.caught;
+            // The failure text the queue and the dead letter keep (`String(thrown)` for a non-Error throw).
+            const message = error instanceof Error ? error.message : error.description;
+            this.logger.error('Job failed', {
+                jobId: job.id,
+                jobType,
+                attempt,
+                maxAttempts,
+                error: valueFreeFailure(error)
+            });
+            if (await abortIfCancelled())
+                return;
+            const hooked = await settle(() => this.runtime
+                .query('workers/job')
+                .withId(jobType)
+                .forHardware(this.runtime.scope.hardware)
+                .enabled()
+                .resolve()
+                .then(async (impl) => {
+                if (!impl.onError)
+                    return;
+                await impl.onError({
+                    jobId: job.id,
+                    data: queuedJobData(job),
+                    attempt,
+                    maxAttempts,
+                    metadata
+                }, error instanceof Error ? error : new Error(message));
+            }));
+            if (!hooked.ok) {
+                this.logger.warn('Job onError hook failed', {
+                    jobId: job.id,
+                    jobType,
+                    error: valueFreeFailure(hooked.caught)
+                });
+            }
+            if (await abortIfCancelled() || cancellationController.signal.aborted || this.runtime.lifecycle.signal.aborted)
+                return;
+            if (shouldRetryJob(error, attempt, maxAttempts)) {
+                // Retry admission follows this producer and the work it issued, not
+                // the shape of a timeout or a later failure. Keep its lease without
+                // holding a slot; take one back (waiting while every slot is held) to finish.
+                const pendingProducer = producer;
+                const retryParking = { depth: 0, ended: false };
+                this.parkedJobs += 1;
+                this.resumeParked();
+                try {
+                    // Runtime reports late producer failures; this join only controls re-admission.
+                    if (pendingProducer)
+                        await settle(() => pendingProducer);
+                    await operations.joinWork();
+                }
+                finally {
+                    await this.unpark(retryParking);
+                }
+                if (await abortIfCancelled() || cancellationController.signal.aborted || this.runtime.lifecycle.signal.aborted)
+                    return;
+                stopHeartbeatNow();
+                if (heartbeatInFlight)
+                    await heartbeatInFlight;
+                const retired = await settle(() => operations.retire());
+                if (!retired.ok)
+                    this.runtime.diagnostics.error('job-issued-operation-failed', {
+                        scope: 'workers:runner', plugin: jobType, error: retired.caught
+                    });
+                await releaseClaim();
+                // The job's backoff policy delays the retry as adapter-native
+                // availability (`delay`), not an in-process wait.
+                const delay = retryDelayMs(metadata, attempt);
+                const retried = await settle(() => this.queue.retryJob(job.id, { expectedGeneration: job.generation, delay }));
+                if (!retried.ok) {
+                    // Native failure remains visible; no expired heartbeat may clear ownership.
+                    this.logger.error('Failed to schedule job retry', {
+                        jobId: job.id,
+                        jobType,
+                        attempt,
+                        ...describeFailure(retried.caught)
+                    });
+                }
+                return;
+            }
+            await writeHeartbeat('failed');
+            if (await abortIfCancelled())
+                return;
+            await recordDeadLetter(this.runtime, {
+                id: job.id,
+                jobType,
+                queue: this.worker.descriptor?.queue ?? String(metadata.queueName ?? 'unknown'),
+                attempts: attempt,
+                lastError: message,
+                failedAt: new Date().toISOString(),
+                payload: job.data ?? null
+            });
+            const recorded = await settle(async () => {
+                // Keep the typed refusal alongside the Kernel's bounded failure facts.
+                const refusal = jobRefusalOf(error);
+                const failureMetadata = {
+                    [JOB_FAILURE_FACTS_METADATA_KEY]: { ...valueFreeFailure(error) },
+                    [JOB_REFUSAL_METADATA_KEY]: refusal ?? null
+                };
+                const failed = {
+                    status: 'failed', error: message,
+                    metadata: failureMetadata
+                };
+                logicalEnded = true;
+                const stamp = await operations.update(failed);
+                await recordRun('failed', error);
+                await operations.emit('failed', { error: message }, Object.freeze({
+                    ...operations.guard, expectedStatus: stamp.status
+                }));
+            });
+            if (!recorded.ok) {
+                this.logger.error('Failed to record job failure', {
+                    jobId: job.id,
+                    jobType,
+                    ...describeFailure(recorded.caught)
+                });
+            }
+        }
+        finally {
+            // Free logical capacity while the exact producer and its operations retire.
+            this.parkedJobs += 1;
+            this.resumeParked();
+            try {
+                if (producer)
+                    await settle(() => producer);
+                await operations.joinWork();
+                stopHeartbeatNow();
+                if (heartbeatInFlight)
+                    await heartbeatInFlight;
+                stopCancellationPoll?.();
+                unsubscribeCancellation?.();
+                if (cancellationCheckInFlight)
+                    await settle(() => cancellationCheckInFlight);
+                const retired = await settle(() => operations.retire());
+                if (!retired.ok)
+                    this.runtime.diagnostics.error('job-issued-operation-failed', {
+                        scope: 'workers:runner', plugin: jobType, error: retired.caught
+                    });
+                await releaseClaim();
+                if (cancellationController.signal.aborted)
+                    await recordRun('cancelled');
+            }
+            finally {
+                this.parkedJobs -= 1;
+                this.activeCancellationControllers.delete(cancellationController);
+            }
+        }
+    }
+}
+//# sourceMappingURL=worker-runner.js.map