@vibesdotdev/workers 0.0.0-stage → 1.0.0-rc.2

package/dist/adapters/ipc/store/store.js

@@ -0,0 +1,982 @@
+import { tryGetVibesClient } from '@vibesdotdev/kernel/client';
+import * as z from 'zod/v4';
+import { getRuntimeProcessService } from '@vibesdotdev/kernel/process';
+import { PmTaskAdmissionSchema, pmTaskProcessIdentityOf } from '@vibesdotdev/pm/schemas/process';
+/**
+ * Consumer queue store — the job table behind the IPC adapter.
+ *
+ * The worker runtime process owns one store. Claims and reads come from
+ * memory; every change is written through to the `workers.queue-job` record,
+ * and `open()` rebuilds the queue from that record. A job accepted before a
+ * restart therefore still runs, and a finished job's result stays readable.
+ */
+import { getLogger } from '@vibesdotdev/logging';
+import { CapabilityError, isCapabilityError, valueFreeFailure } from '@vibesdotdev/kernel';
+import { JsonObjectSchema } from '@vibesdotdev/kernel/schemas/json';
+import { recordAtomic } from '@vibesdotdev/storage/record';
+import { assertJobMutationGuard, assertJobWriteGuard, assertJobReadmission, assertExecutionClaim, keepsCallerAdmission, createJobExecutionClaim, jobExecutionRecord, jobMutationStamp, omitJobFields, pageJobList, restoreExecutionClaim, snapshotJob } from "../../../core/queue-adapter.js";
+import { COALESCE_METADATA_KEY, joinsCoalescedJob } from "../../../core/coalesced-enqueue.js";
+import { retryDelayMs, stampRetryPolicy, withRetryAvailability } from "../../../core/enqueue-metadata.js";
+import { decideKeyedClaim, jobGeneration, nextGeneration } from "../../../core/keyed-claim.js";
+import { priorityInsertIndex } from "../../../core/priority-order.js";
+import { settle } from "../../../core/settle.js";
+import { SUPERVISED_METADATA_KEY, SupervisedWorkloadsSchema } from "../../../core/supervised-workloads.js";
+import { QueueJobMetadataRecordSchema, queueJobMetadataId } from "./queue-job-metadata.record.descriptor.js";
+const LEGACY_DEFAULT_QUEUE_NAME = 'ipc-queue';
+const FINISHED_STATUSES = new Set(['completed', 'failed', 'cancelled']);
+/** PM's statuses for a process that has ended; every other status (or none) is not proof of exit. */
+const PM_ENDED_STATUSES = new Set(['stopped', 'crashed', 'completed', 'pass', 'fail']);
+/** Missed heartbeats identify a suspect execution, never prove its retirement. */
+const LEASE_MISSED_HEARTBEATS = 3;
+const DEFAULT_HEARTBEAT_INTERVAL_MS = 30_000;
+const MetadataKeysSchema = z.array(z.string()).refine(keys => new Set(keys).size === keys.length);
+function isNumber(value) {
+    return typeof value === 'number';
+}
+function isString(value) {
+    return typeof value === 'string';
+}
+function getJobPriority(job) {
+    const priority = job?.metadata?.priority;
+    return isNumber(priority) ? priority : undefined;
+}
+function metadataQueueName(metadata) {
+    const queueName = metadata?.queueName;
+    return isString(queueName) && queueName.length > 0 ? queueName : undefined;
+}
+function getJobQueueName(job) {
+    return metadataQueueName(job?.metadata);
+}
+function resolveJobQueueName(job) {
+    return getJobQueueName(job) ?? LEGACY_DEFAULT_QUEUE_NAME;
+}
+function matchesDequeueOptions(job, options) {
+    if (!job)
+        return false;
+    if (options?.queueName && getJobQueueName(job) !== options.queueName) {
+        return false;
+    }
+    if (options?.jobTypes && options.jobTypes.length > 0) {
+        const declaredType = job.metadata?.type;
+        const jobType = isString(declaredType) ? declaredType : undefined;
+        return !!jobType && options.jobTypes.includes(jobType);
+    }
+    return true;
+}
+function normalizeDelayMs(delay) {
+    return delay !== undefined && Number.isFinite(delay) && delay > 0
+        ? Math.ceil(delay)
+        : 0;
+}
+function jsonMetadata(metadata) {
+    const parsed = JsonObjectSchema.safeParse(metadata ?? {});
+    if (!parsed.success) {
+        throw new CapabilityError('IPC queue metadata must contain JSON values.', 'INVALID_INPUT', {
+            reason: 'worker-metadata-invalid'
+        });
+    }
+    return parsed.data;
+}
+function toRecord(job) {
+    const execution = jobExecutionRecord(job);
+    return {
+        ...execution,
+        id: job.id,
+        queue: resolveJobQueueName(job),
+        status: job.status,
+        cancellation: job.cancellation,
+        data: job.data ?? null,
+        result: job.result ?? null,
+        error: job.error ?? null,
+        progress: job.progress ?? null,
+        attemptsMade: job.attemptsMade,
+        generation: job.generation,
+        metadata: {},
+        metadataFormat: 'entries-v1',
+        metadataKeys: Object.keys({ ...job.metadata, queueName: resolveJobQueueName(job) }),
+        createdAt: job.createdAt.toISOString(),
+        startedAt: job.startedAt?.toISOString() ?? null,
+        finishedAt: job.completedAt?.toISOString() ?? null,
+        retiredAt: job.retiredAt?.toISOString() ?? null
+    };
+}
+function fromRecord(row, metadata) {
+    const job = {
+        id: row.id,
+        data: row.data,
+        status: row.status,
+        cancellation: row.cancellation,
+        attemptsMade: row.attemptsMade,
+        createdAt: new Date(row.createdAt),
+        generation: row.generation ?? 0,
+        revision: row.revision ?? 0,
+        executionClaim: restoreExecutionClaim(row),
+        metadata: { ...metadata, queueName: row.queue }
+    };
+    if (row.result !== null)
+        job.result = row.result;
+    if (row.error !== null)
+        job.error = row.error;
+    if (row.progress !== null)
+        job.progress = row.progress;
+    if (row.startedAt)
+        job.startedAt = new Date(row.startedAt);
+    if (row.finishedAt)
+        job.completedAt = new Date(row.finishedAt);
+    if (row.retiredAt)
+        job.retiredAt = new Date(row.retiredAt);
+    return job;
+}
+function byClaimOrder(a, b) {
+    const priority = (getJobPriority(b) ?? 0) - (getJobPriority(a) ?? 0);
+    return priority !== 0 ? priority : a.createdAt.getTime() - b.createdAt.getTime();
+}
+function sameRecoveryFence(left, right) {
+    return left.id === right.id && left.queue === right.queue && left.status === right.status &&
+        left.generation === right.generation && left.revision === right.revision &&
+        left.executionState === right.executionState && left.claimToken === right.claimToken;
+}
+function samePmAdmission(left, right) {
+    return left.admissionId === right.admissionId && left.processId === right.processId &&
+        left.scope.workspacePath === right.scope.workspacePath && left.scope.toolingId === right.scope.toolingId;
+}
+/** The fence of a stored row: the fields a later write must still find there. */
+function rowFence(row) {
+    return {
+        id: row.id, queue: row.queue, status: row.status, generation: row.generation,
+        revision: row.revision, executionState: row.executionState, claimToken: row.claimToken
+    };
+}
+/** How many times one job write is tried when SQLite refuses it as busy, and its backoff between tries. */
+const STORE_WRITE_ATTEMPTS = 5;
+const STORE_WRITE_BACKOFF = { backoff: { type: 'exponential', delay: 250 } };
+function recordFence(job) {
+    return {
+        id: job.id, queue: resolveJobQueueName(job), status: job.status,
+        generation: job.generation, revision: job.revision, executionState: job.executionClaim.state,
+        claimToken: job.executionClaim.state === 'held' ? job.executionClaim.identity.token : null
+    };
+}
+export class IpcJobStore {
+    runtime;
+    jobs = new Map();
+    queue = [];
+    jobAvailableCallbacks = new Set();
+    delayedTimers = new Map();
+    writes = new Map();
+    /** Jobs whose latest change was refused by storage (a full disk, say): stored again once a write lands. */
+    unstored = new Set();
+    /** The fence of each job's row as last durably stored: the next write is admitted against it. */
+    storedFences = new Map();
+    record;
+    metadataRecord;
+    normalizedJobs = new Set();
+    storedMetadataKeys = new Map();
+    metadataVersions = new Map();
+    storedMetadataVersions = new Map();
+    metadataSequence = 0;
+    executionOwner;
+    /** This process's incarnation of the queue: every claim it takes is stamped with it and its pid. */
+    incarnation = crypto.randomUUID();
+    incarnationPid;
+    /** The serve process each held claim (by token) was taken in; `null` when it was never recorded. */
+    claimStamps = new Map();
+    /** `runtime` owns the delayed-job timers; they end with it. */
+    constructor(runtime) {
+        this.runtime = runtime;
+    }
+    /**
+     * Restore the stored admission and claim exactly. A restart or a silent
+     * heartbeat is not proof that its producer or supervised workload stopped.
+     * The one exception is proved, not inferred: a held claim taken in another
+     * serve process whose pid is no longer running, and whose job explicitly
+     * recorded its supervised workloads (`[]` for none) with PM reporting each
+     * ended, is interrupted (`worker-incarnation-ended`).
+     * A row whose stored claim is malformed opens unresolved with its typed
+     * fault and stays as stored: that job is fenced, the others are served.
+     */
+    async open(record, metadataRecord) {
+        if (this.record)
+            return;
+        recordAtomic(record);
+        await metadataRecord.initialize();
+        // Rows stored before claims carry no generation, revision or execution
+        // state. One guarded pass per field migrates them all: a per-row filter
+        // rescans the whole table (record filters read the stored JSON), which
+        // made opening quadratic in the stored jobs.
+        await record.updateMany({ generation: null }, { generation: 0 });
+        await record.updateMany({ revision: null }, { revision: 0 });
+        await record.updateMany({ executionState: null }, { executionState: 'unknown' });
+        const { items } = await record.list();
+        const restoredMetadata = new Map();
+        for (const row of items) {
+            if (this.jobs.has(row.id))
+                throw new CapabilityError('A stored admission conflicts with a job accepted before storage opened.', 'CONFLICT', {
+                    reason: 'job-storage-open-conflict', retryable: false
+                });
+            restoredMetadata.set(row.id, await this.restoreMetadata(row, metadataRecord));
+        }
+        this.record = record;
+        this.metadataRecord = metadataRecord;
+        this.incarnationPid = (await this.processService())?.currentPid?.();
+        const stored = new Set();
+        const pending = [];
+        const now = Date.now();
+        let unresolved = 0;
+        const invalid = [];
+        for (const row of items) {
+            if (row.generation == null || row.revision == null || row.executionState == null) {
+                throw new CapabilityError('The stored job changed during claim migration.', 'CONFLICT', {
+                    reason: 'job-execution-migration-conflict', retryable: false
+                });
+            }
+            stored.add(row.id);
+            this.storedFences.set(row.id, rowFence(row));
+            if (this.jobs.has(row.id))
+                throw new CapabilityError('A stored admission conflicts with a job accepted before storage opened.', 'CONFLICT', {
+                    reason: 'job-storage-open-conflict', retryable: false
+                });
+            const job = fromRecord(row, restoredMetadata.get(row.id) ?? {});
+            if (row.metadataFormat === 'entries-v1') {
+                this.normalizedJobs.add(row.id);
+                this.storedMetadataKeys.set(row.id, new Set(row.metadataKeys ?? []));
+            }
+            this.jobs.set(job.id, job);
+            if (row.claimToken)
+                this.claimStamps.set(row.claimToken, row.claimIncarnation && row.claimPid ? { incarnation: row.claimIncarnation, pid: row.claimPid } : null);
+            if (job.executionClaim.state !== 'none') {
+                unresolved++;
+                if (job.executionClaim.state === 'unknown' && job.executionClaim.invalidRecord) {
+                    invalid.push({ jobId: job.id, queue: row.queue, status: job.status, fields: job.executionClaim.invalidRecord.fields });
+                }
+                continue;
+            }
+            if (job.status === 'pending')
+                pending.push(job);
+            if (job.status === 'delayed') {
+                const until = Date.parse(String(job.metadata?.delayedUntil ?? ''));
+                this.scheduleDelayedJob(job.id, Number.isFinite(until) ? Math.max(0, until - now) : 0);
+            }
+        }
+        const interrupted = await this.interruptEndedIncarnations();
+        unresolved -= interrupted.length;
+        for (const job of this.jobs.values()) {
+            if (!stored.has(job.id))
+                this.persist(job);
+        }
+        for (const job of pending.sort(byClaimOrder))
+            this.queue.push(job.id);
+        await this.flush();
+        const logger = getLogger('worker-adapter.ipc', this.runtime).child('store');
+        if (items.length > 0) {
+            logger.info('Consumer queue restored from storage', {
+                jobs: items.length,
+                pending: pending.length,
+                unresolved,
+                invalid: invalid.length
+            });
+        }
+        if (interrupted.length > 0) {
+            logger.warn('Jobs held by an ended worker process were interrupted; their owners may retry them', {
+                reason: 'worker-incarnation-ended',
+                jobs: interrupted
+            });
+        }
+        if (invalid.length > 0) {
+            logger.error('Stored job ownership is invalid; those jobs stay unresolved until their execution is confirmed retired', {
+                reason: 'job-execution-record-invalid',
+                jobs: invalid
+            });
+        }
+        if (pending.length > 0)
+            this.notifyJobAvailable();
+    }
+    /** Recovery refuses takeover until the selected owner supplies real retirement. */
+    checkExpiredExecutions(now = Date.now()) {
+        let suspect = 0;
+        for (const job of this.jobs.values()) {
+            const claim = job.executionClaim;
+            if (claim.state === 'none')
+                continue;
+            const heartbeat = Date.parse(claim.state === 'held' ? claim.heartbeatAt : String(job.metadata?.workerHeartbeatAt ?? ''));
+            const lastSeen = Number.isFinite(heartbeat) ? heartbeat : job.startedAt?.getTime();
+            if (lastSeen === undefined)
+                continue;
+            const interval = job.metadata?.workerHeartbeatIntervalMs;
+            const leaseMs = (isNumber(interval) && interval > 0 ? interval : DEFAULT_HEARTBEAT_INTERVAL_MS) * LEASE_MISSED_HEARTBEATS;
+            if (now - lastSeen > leaseMs)
+                suspect++;
+        }
+        if (suspect > 0) {
+            throw new CapabilityError('Expired job heartbeats do not confirm execution retirement.', 'UNAVAILABLE', {
+                reason: 'job-execution-exit-unconfirmed', retryable: false, details: { suspect }
+            });
+        }
+    }
+    /** Resolve once every change made so far is stored. */
+    async flush() {
+        await Promise.all(this.writes.values());
+    }
+    async flushJob(jobId) {
+        await this.writes.get(jobId);
+    }
+    onJobAvailable(handler) {
+        this.jobAvailableCallbacks.add(handler);
+        return () => {
+            this.jobAvailableCallbacks.delete(handler);
+        };
+    }
+    enqueueJob(jobId, data, options) {
+        // A caller-supplied id is a dedupe key: an existing job it still names stays as it is (keepsCallerAdmission).
+        const current = this.jobs.get(jobId);
+        if (current && keepsCallerAdmission(current, options?.resume))
+            return jobId;
+        this.writeJob(jobId, data, options);
+        return jobId;
+    }
+    /**
+     * Claim a caller key (`QueueAdapter.claimKeyed`): the read, the decision and
+     * the write happen in this one synchronous section of the owning process.
+     */
+    claimKeyed(jobId, data, options) {
+        // SAFETY: The heterogeneous store preserves each job's data unchanged; the claim's caller selects TJob for this key's job type.
+        const current = this.jobs.get(jobId) ?? null;
+        // One store holds every queue's jobs: a key another queue holds is neither
+        // replaced nor reported as this queue's job.
+        const requestedQueue = metadataQueueName(options.metadata) ?? LEGACY_DEFAULT_QUEUE_NAME;
+        if (current && resolveJobQueueName(current) !== requestedQueue) {
+            throw new CapabilityError('This job id is held by another queue.', 'CONFLICT', {
+                reason: 'job-key-held-by-another-queue',
+                retryable: false,
+                details: { jobId, requestedQueue, holdingQueue: resolveJobQueueName(current) }
+            });
+        }
+        const decision = decideKeyedClaim(current, options.restartGeneration);
+        if (current && !decision.write) {
+            return { outcome: decision.outcome, job: snapshotJob(current), generation: jobGeneration(current) };
+        }
+        const { restartGeneration: _restartGeneration, ...enqueueOptions } = options;
+        const job = this.writeJob(jobId, data, enqueueOptions);
+        return { outcome: decision.outcome, job, generation: jobGeneration(job) };
+    }
+    /**
+     * Coalesced enqueue (`QueueAdapter.enqueueCoalesced`): join this queue's
+     * delayed or pending job carrying the key, else write `jobId` carrying it —
+     * decided in this one synchronous section. Only jobs that have not started
+     * are scanned, so the cost follows the queue's backlog, not its history.
+     */
+    enqueueCoalesced(jobId, data, options) {
+        const requestedQueue = metadataQueueName(options.metadata) ?? LEGACY_DEFAULT_QUEUE_NAME;
+        for (const candidateId of [...this.queue, ...this.delayedTimers.keys()]) {
+            const candidate = this.jobs.get(candidateId);
+            if (candidate && resolveJobQueueName(candidate) === requestedQueue && joinsCoalescedJob(candidate, options.coalesceKey)) {
+                return { jobId: candidate.id, coalesced: true };
+            }
+        }
+        const { coalesceKey, ...enqueueOptions } = options;
+        this.writeJob(jobId, data, { ...enqueueOptions, metadata: { ...enqueueOptions.metadata, [COALESCE_METADATA_KEY]: coalesceKey } });
+        return { jobId, coalesced: false };
+    }
+    /** Write a new admission of `jobId`, one generation past the job it replaces. */
+    writeJob(jobId, data, options) {
+        const current = this.jobs.get(jobId);
+        if (current)
+            assertJobReadmission(current, current.generation);
+        const generation = nextGeneration(current);
+        this.clearDelayedTimer(jobId);
+        this.removeQueuedJobOccurrences(jobId);
+        const metadata = jsonMetadata(options?.metadata);
+        stampRetryPolicy(metadata, options);
+        const delayMs = normalizeDelayMs(options?.delay);
+        if (delayMs > 0) {
+            metadata.delayMs = delayMs;
+            metadata.delayedUntil = new Date(Date.now() + delayMs).toISOString();
+        }
+        const job = {
+            id: jobId,
+            data,
+            status: delayMs > 0 ? 'delayed' : 'pending',
+            attemptsMade: 0,
+            createdAt: new Date(),
+            metadata,
+            generation,
+            revision: (current?.revision ?? -1) + 1,
+            executionClaim: { state: 'none' }
+        };
+        this.jobs.set(jobId, job);
+        this.markMetadata(jobId, [...Object.keys(current?.metadata ?? {}), ...Object.keys(metadata)]);
+        if (delayMs > 0) {
+            this.scheduleDelayedJob(jobId, delayMs, options?.priority);
+            this.persist(job);
+            return snapshotJob(job);
+        }
+        this.enqueuePendingJob(jobId, options?.priority);
+        this.persist(job);
+        this.notifyJobAvailable();
+        return snapshotJob(job);
+    }
+    dequeueJob(options) {
+        let jobId;
+        const index = this.queue.findIndex((id) => {
+            const job = this.jobs.get(id);
+            return job?.status === 'pending' && job.executionClaim.state === 'none' && matchesDequeueOptions(job, options);
+        });
+        if (index === -1)
+            return null;
+        const claimMetadata = options?.claimMetadata === undefined ? undefined : jsonMetadata(options.claimMetadata);
+        jobId = this.queue.splice(index, 1)[0];
+        if (!jobId)
+            return null;
+        const job = this.jobs.get(jobId);
+        if (!job) {
+            getLogger('worker-adapter.ipc', this.runtime).child('store').error('Queued job is missing from the job table', { jobId });
+            return null;
+        }
+        const identity = createJobExecutionClaim(job, options?.executionOwner ?? (this.executionOwner ??= crypto.randomUUID()));
+        job.executionClaim = { state: 'held', identity, heartbeatAt: new Date().toISOString() };
+        this.claimStamps.set(identity.token, this.incarnationPid ? { incarnation: this.incarnation, pid: this.incarnationPid } : null);
+        job.revision++;
+        job.retiredAt = undefined;
+        job.status = 'processing';
+        job.startedAt = new Date();
+        if (claimMetadata) {
+            job.metadata = { ...job.metadata, ...claimMetadata };
+            this.markMetadata(jobId, Object.keys(claimMetadata));
+        }
+        this.persist(job);
+        // SAFETY: The heterogeneous store preserves enqueueJob's data unchanged; the queue caller selects TJob for its job type at this erased retrieval boundary.
+        return snapshotJob(job);
+    }
+    getJob(jobId) {
+        const job = this.jobs.get(jobId);
+        // SAFETY: Stored data is unchanged from enqueueJob; getJob's caller supplies its job payload type because the heterogeneous map erases it.
+        return job ? snapshotJob(job) : null;
+    }
+    updateJob(jobId, update, guard) {
+        const job = this.jobs.get(jobId);
+        if (!job)
+            throw new CapabilityError('Job not found.', 'NOT_FOUND', { reason: 'job-not-found' });
+        assertJobMutationGuard(job, guard);
+        const metadata = update.metadata === undefined ? undefined : jsonMetadata(update.metadata);
+        if (update.status !== undefined) {
+            if (update.status !== 'delayed')
+                this.clearDelayedTimer(jobId);
+            job.status = update.status;
+            if (FINISHED_STATUSES.has(update.status))
+                job.completedAt = new Date();
+            if (FINISHED_STATUSES.has(update.status) && job.executionClaim.state === 'none')
+                job.retiredAt = new Date();
+            if (!FINISHED_STATUSES.has(update.status)) {
+                job.completedAt = undefined;
+                job.retiredAt = undefined;
+            }
+        }
+        if (update.progress !== undefined)
+            job.progress = update.progress;
+        if (update.result !== undefined)
+            job.result = update.result;
+        if (update.error !== undefined)
+            job.error = update.error;
+        if (metadata !== undefined) {
+            job.metadata = { ...job.metadata, ...metadata };
+            this.markMetadata(jobId, Object.keys(metadata));
+        }
+        job.revision++;
+        this.persist(job);
+        return jobMutationStamp(job);
+    }
+    renewExecution(jobId, claim) {
+        const job = this.jobs.get(jobId);
+        if (!job)
+            throw new CapabilityError('Job not found.', 'NOT_FOUND', { reason: 'job-not-found' });
+        const identity = assertExecutionClaim(job, claim);
+        job.executionClaim = { state: 'held', identity, heartbeatAt: new Date().toISOString() };
+        this.persist(job);
+    }
+    releaseExecution(jobId, claim) {
+        const job = this.jobs.get(jobId);
+        if (!job)
+            throw new CapabilityError('Job not found.', 'NOT_FOUND', { reason: 'job-not-found' });
+        assertExecutionClaim(job, claim);
+        job.executionClaim = { state: 'none' };
+        if (FINISHED_STATUSES.has(job.status))
+            job.retiredAt = new Date();
+        this.persist(job);
+    }
+    admitEvent(jobId, guard) {
+        const job = this.jobs.get(jobId);
+        if (!job)
+            throw new CapabilityError('Job not found.', 'NOT_FOUND', { reason: 'job-not-found' });
+        assertJobWriteGuard(job, guard);
+        job.revision++;
+        this.persist(job);
+        return jobMutationStamp(job);
+    }
+    cancelJob(jobId, cancellation, guard) {
+        const job = this.jobs.get(jobId);
+        if (!job)
+            return null;
+        if (guard)
+            assertJobMutationGuard(job, guard);
+        if (job.executionClaim.state === 'unknown')
+            throw new CapabilityError('Job execution retirement is unconfirmed.', 'UNAVAILABLE', {
+                reason: 'job-execution-exit-unconfirmed', retryable: false
+            });
+        this.clearDelayedTimer(jobId);
+        this.removeQueuedJobOccurrences(jobId);
+        job.status = 'cancelled';
+        job.cancellation = cancellation;
+        job.completedAt = new Date();
+        job.revision++;
+        if (job.executionClaim.state === 'none')
+            job.retiredAt = new Date();
+        this.persist(job);
+        return jobMutationStamp(job);
+    }
+    retryJob(jobId, options) {
+        const job = this.jobs.get(jobId);
+        if (!job)
+            throw new Error(`Job ${jobId} not found`);
+        assertJobReadmission(job, options.expectedGeneration, options.expectedRevision, options.expectedStatus);
+        this.clearDelayedTimer(jobId);
+        this.removeQueuedJobOccurrences(jobId);
+        job.generation = nextGeneration(job);
+        job.attemptsMade++;
+        job.startedAt = undefined;
+        job.completedAt = undefined;
+        job.cancellation = undefined;
+        job.retiredAt = undefined;
+        job.revision++;
+        job.error = undefined;
+        job.result = undefined;
+        const delayMs = normalizeDelayMs(options.delay);
+        job.metadata = withRetryAvailability(job.metadata, delayMs);
+        this.markMetadata(jobId, ['delayedUntil']);
+        if (delayMs > 0) {
+            job.status = 'delayed';
+            this.scheduleDelayedJob(jobId, delayMs);
+            this.persist(job);
+            return jobId;
+        }
+        job.status = 'pending';
+        this.enqueuePendingJob(jobId, getJobPriority(job));
+        this.persist(job);
+        this.notifyJobAvailable();
+        return jobId;
+    }
+    getStatus(jobId) {
+        const job = this.jobs.get(jobId);
+        if (!job)
+            throw new Error(`Job ${jobId} not found`);
+        return job.status;
+    }
+    listJobs(state, options) {
+        // SAFETY: listJobs retains the queue API's caller-selected payload type; filtering and sorting do not alter the data stored by enqueueJob.
+        let allJobs = Array.from(this.jobs.values());
+        const queueName = options?.queueName;
+        if (queueName)
+            allJobs = allJobs.filter((job) => resolveJobQueueName(job) === queueName);
+        return pageJobList(allJobs, state, options).map((job) => omitJobFields(snapshotJob(job), options?.omit));
+    }
+    /** Count one queue's jobs by status in place; no job is snapshotted or serialized. */
+    getMetrics(queueName) {
+        const metrics = { waiting: 0, active: 0, completed: 0, failed: 0 };
+        for (const job of this.jobs.values()) {
+            if (resolveJobQueueName(job) !== queueName)
+                continue;
+            if (job.status === 'pending')
+                metrics.waiting++;
+            else if (job.status === 'processing')
+                metrics.active++;
+            else if (job.status === 'completed')
+                metrics.completed++;
+            else if (job.status === 'failed')
+                metrics.failed++;
+        }
+        return metrics;
+    }
+    /**
+     * Remove one queue's finished jobs older than the given ages. Cancelled jobs
+     * age with completed ones. Unfinished jobs are never removed.
+     */
+    async cleanup(queueName, options = {}) {
+        const now = Date.now();
+        const expired = [];
+        let completedRemoved = 0;
+        let failedRemoved = 0;
+        for (const job of this.jobs.values()) {
+            if (resolveJobQueueName(job) !== queueName || job.executionClaim.state !== 'none' || !job.retiredAt)
+                continue;
+            const age = now - job.retiredAt.getTime();
+            if (job.status === 'failed') {
+                if (options.failedOlderThan === undefined || age < options.failedOlderThan)
+                    continue;
+                failedRemoved++;
+            }
+            else if (job.status === 'completed' || job.status === 'cancelled') {
+                if (options.completedOlderThan === undefined || age < options.completedOlderThan)
+                    continue;
+                completedRemoved++;
+            }
+            else {
+                continue;
+            }
+            expired.push({ fence: recordFence(job), retiredAt: job.retiredAt.toISOString() });
+        }
+        if (options.dryRun || expired.length === 0)
+            return { completedRemoved, failedRemoved };
+        await this.flush();
+        completedRemoved = 0;
+        failedRemoved = 0;
+        for (const expiredJob of expired) {
+            const { fence, retiredAt } = expiredJob;
+            await this.enqueueWrite(fence.id, async () => {
+                const current = this.jobs.get(fence.id);
+                if (!current || current.executionClaim.state !== 'none' ||
+                    current.generation !== fence.generation || current.revision !== fence.revision ||
+                    current.retiredAt?.toISOString() !== retiredAt || current.status !== fence.status)
+                    return;
+                if (this.record) {
+                    const removed = await settle(() => recordAtomic(this.record).commit([
+                        { record: this.record, type: 'delete', id: current.id,
+                            where: { ...fence, retiredAt, claimOwner: null, claimAttempt: null, claimHeartbeatAt: null } },
+                        ...[...(this.storedMetadataKeys.get(current.id) ?? [])].map(key => ({
+                            record: this.metadataRecord, type: 'delete', id: queueJobMetadataId(current.id, key)
+                        }))
+                    ]));
+                    if (!removed.ok) {
+                        if (this.isFenceMiss(removed.caught))
+                            return;
+                        throw removed.caught;
+                    }
+                }
+                // The durable row is gone even if readmission changed memory while commit prepared.
+                // A later queued write must create its row and every current metadata key.
+                this.storedFences.delete(fence.id);
+                this.normalizedJobs.delete(fence.id);
+                this.storedMetadataKeys.delete(fence.id);
+                this.storedMetadataVersions.delete(fence.id);
+                if (this.jobs.get(fence.id) !== current || current.executionClaim.state !== 'none' ||
+                    current.generation !== fence.generation || current.revision !== fence.revision ||
+                    current.retiredAt?.toISOString() !== retiredAt || current.status !== fence.status)
+                    return;
+                this.jobs.delete(current.id);
+                this.metadataVersions.delete(current.id);
+                if (current.status === 'failed')
+                    failedRemoved++;
+                else
+                    completedRemoved++;
+            });
+        }
+        return { completedRemoved, failedRemoved };
+    }
+    notifyJobAvailable() {
+        for (const callback of this.jobAvailableCallbacks) {
+            queueMicrotask(callback);
+        }
+    }
+    /** Admit every durable mutation for one job to its existing writer chain. */
+    enqueueWrite(jobId, run) {
+        const priorWrite = this.writes.get(jobId) ?? Promise.resolve();
+        const write = settle(() => priorWrite).then(run);
+        this.writes.set(jobId, write);
+        void settle(() => write).then(() => {
+            if (this.writes.get(jobId) === write)
+                this.writes.delete(jobId);
+        });
+        return write;
+    }
+    /**
+     * Write the job's current state behind any earlier write for the same job. Each write stores the
+     * current row and every dirty metadata key against the row last durably stored, so a refused earlier write never blocks a
+     * later one: the later write carries every change the refused one held. A retryable refusal
+     * (`CONFLICT sqlite-busy`) is retried with backoff on the same write; a non-retryable one
+     * (`job-write-stale`: another writer owns the row) is logged and not retried.
+     */
+    persist(job) {
+        const record = this.record;
+        if (!record)
+            return;
+        const write = this.enqueueWrite(job.id, async () => {
+            await this.storeCurrent(record, job.id);
+            this.unstored.delete(job.id);
+            // Storage admits writes again: a job whose last change was refused, and that may never
+            // change again, is stored now, so memory and the stored queue converge.
+            for (const id of this.unstored) {
+                this.unstored.delete(id);
+                const pending = this.jobs.get(id);
+                if (pending)
+                    this.persist(pending);
+            }
+        });
+        void settle(() => write).then((stored) => {
+            if (!stored.ok) {
+                // A stale write belongs to another writer's row and stays refused; any other refusal is retried later.
+                if (!(isCapabilityError(stored.caught) && stored.caught.reason === 'job-write-stale'))
+                    this.unstored.add(job.id);
+                getLogger('worker-adapter.ipc', this.runtime).child('store').error('Queue job change was not stored', {
+                    jobId: job.id, status: job.status, error: valueFreeFailure(stored.caught)
+                });
+            }
+        });
+    }
+    /** Store the job's row as it is now, retrying a retryable refusal up to `STORE_WRITE_ATTEMPTS` times. */
+    async storeCurrent(record, jobId) {
+        for (let attempt = 1;; attempt++) {
+            const job = this.jobs.get(jobId);
+            if (!job)
+                return;
+            const row = this.storedRecord(job);
+            const metadata = { ...job.metadata, queueName: resolveJobQueueName(job) };
+            const versions = new Map(this.metadataVersions.get(jobId));
+            const stored = await settle(() => this.storeRow(record, row, metadata, versions));
+            if (stored.ok)
+                return;
+            const caught = stored.caught;
+            if (attempt >= STORE_WRITE_ATTEMPTS || (isCapabilityError(caught) && !caught.retryable))
+                throw caught;
+            await this.runtime.lifecycle.delay(retryDelayMs(STORE_WRITE_BACKOFF, attempt));
+        }
+    }
+    async storeRow(record, row, metadata, versions) {
+        const fence = this.storedFences.get(row.id);
+        const normalized = this.normalizedJobs.has(row.id);
+        const previousKeys = this.storedMetadataKeys.get(row.id) ?? new Set();
+        const committedVersions = this.storedMetadataVersions.get(row.id);
+        const mutations = [fence
+                ? { record, type: 'replace', id: row.id, data: row, where: fence }
+                : { record, type: 'create', id: row.id, data: row }];
+        for (const [key, value] of Object.entries(metadata)) {
+            if (normalized && (versions.get(key) ?? 0) <= (committedVersions?.get(key) ?? 0))
+                continue;
+            const id = queueJobMetadataId(row.id, key);
+            mutations.push({ record: this.metadataRecord, type: normalized && previousKeys.has(key) ? 'replace' : 'create',
+                id, data: { id, jobId: row.id, key, value } });
+        }
+        for (const key of previousKeys) {
+            if (!Object.hasOwn(metadata, key))
+                mutations.push({ record: this.metadataRecord, type: 'delete', id: queueJobMetadataId(row.id, key) });
+        }
+        const stored = await settle(() => recordAtomic(record).commit(mutations));
+        if (!stored.ok) {
+            if (!this.isFenceMiss(stored.caught))
+                throw stored.caught;
+            throw new CapabilityError('The intended durable job change was not admitted.', 'CONFLICT', {
+                reason: 'job-write-stale', retryable: false
+            });
+        }
+        this.storedFences.set(row.id, rowFence(row));
+        this.normalizedJobs.add(row.id);
+        this.storedMetadataKeys.set(row.id, new Set(Object.keys(metadata)));
+        this.storedMetadataVersions.set(row.id, versions);
+    }
+    markMetadata(jobId, keys) {
+        const versions = this.metadataVersions.get(jobId) ?? new Map();
+        const sequence = ++this.metadataSequence;
+        for (const key of keys)
+            versions.set(key, sequence);
+        this.metadataVersions.set(jobId, versions);
+    }
+    isFenceMiss(error) {
+        return isCapabilityError(error) && error.reason === 'record-atomic-precondition-failed' && error.details?.index === 0;
+    }
+    async restoreMetadata(row, record) {
+        if (row.metadataFormat == null)
+            return jsonMetadata(row.metadata);
+        const keys = MetadataKeysSchema.safeParse(row.metadataKeys);
+        if (row.metadataFormat !== 'entries-v1' || !keys.success || Object.keys(row.metadata ?? {}).length > 0) {
+            throw new CapabilityError('Stored queue metadata has an invalid format.', 'INVALID_INPUT', { reason: 'worker-metadata-record-invalid' });
+        }
+        const { items } = await record.list({ where: { jobId: row.id } });
+        const expected = new Set(keys.data);
+        const entries = [];
+        for (const input of items) {
+            const entry = QueueJobMetadataRecordSchema.safeParse(input);
+            if (!entry.success || entry.data.jobId !== row.id || !expected.delete(entry.data.key) ||
+                entry.data.id !== queueJobMetadataId(row.id, entry.data.key)) {
+                throw new CapabilityError('A stored queue metadata entry is invalid.', 'INVALID_INPUT', { reason: 'worker-metadata-record-invalid' });
+            }
+            entries.push([entry.data.key, entry.data.value]);
+        }
+        if (expected.size > 0)
+            throw new CapabilityError('A stored queue metadata entry is missing.', 'NOT_FOUND', { reason: 'worker-metadata-record-missing' });
+        return Object.fromEntries(entries);
+    }
+    /** This host's process service; `undefined` when it composes none (nothing is stamped or proved then). */
+    async processService() {
+        const resolved = await settle(() => getRuntimeProcessService(this.runtime));
+        if (resolved.ok)
+            return resolved.value;
+        if (isCapabilityError(resolved.caught) && resolved.caught.reason === 'process-unavailable')
+            return undefined;
+        throw resolved.caught;
+    }
+    /** The job's row, with the serve process its held claim was taken in. */
+    storedRecord(job) {
+        const claim = job.executionClaim;
+        const stamp = claim.state === 'held' ? this.claimStamps.get(claim.identity.token) : undefined;
+        return { ...toRecord(job), claimIncarnation: stamp?.incarnation ?? null, claimPid: stamp?.pid ?? null };
+    }
+    ownsRecoveryFence(job, fence) {
+        return this.jobs.get(job.id) === job && sameRecoveryFence(recordFence(job), fence);
+    }
+    hasDurableRecoveryFence(job, fence) {
+        const stored = this.storedFences.get(job.id);
+        return this.ownsRecoveryFence(job, fence) && stored !== undefined && sameRecoveryFence(stored, fence);
+    }
+    async interruptEndedIncarnations() {
+        const processes = await this.processService();
+        const interrupted = [];
+        if (!processes)
+            return interrupted;
+        for (const job of this.jobs.values()) {
+            const claim = job.executionClaim;
+            if (claim.state !== 'held')
+                continue;
+            const stamp = this.claimStamps.get(claim.identity.token);
+            if (!stamp || stamp.incarnation === this.incarnation)
+                continue;
+            const captured = recordFence(job);
+            const running = await processes.isRunning(stamp.pid);
+            if (running || !this.ownsRecoveryFence(job, captured))
+                continue;
+            const proven = await this.supervisedWorkloadsEnded(job, captured);
+            if (!proven || !this.ownsRecoveryFence(job, proven))
+                continue;
+            const now = new Date();
+            if (!FINISHED_STATUSES.has(job.status)) {
+                job.status = 'failed';
+                job.error = 'The worker process that held this job ended before it finished (worker-incarnation-ended).';
+                job.completedAt = now;
+            }
+            job.metadata = { ...job.metadata, interruption: { reason: 'worker-incarnation-ended', incarnation: stamp.incarnation, pid: stamp.pid } };
+            this.markMetadata(job.id, ['interruption']);
+            job.executionClaim = { state: 'none' };
+            this.claimStamps.delete(claim.identity.token);
+            job.retiredAt = now;
+            job.revision++;
+            this.persist(job);
+            interrupted.push(job.id);
+        }
+        return interrupted;
+    }
+    /** Return the exact proved queue revision; every awaited owner call is fenced. */
+    async supervisedWorkloadsEnded(job, initial) {
+        const value = job.metadata?.[SUPERVISED_METADATA_KEY];
+        if (value === undefined)
+            return undefined;
+        // This owner-held snapshot traverses only the small supervision key, never graph metadata.
+        const snapshot = z.array(JsonObjectSchema).safeParse(value);
+        if (!snapshot.success)
+            return undefined;
+        const recorded = SupervisedWorkloadsSchema.safeParse(snapshot.data);
+        if (!recorded.success || !this.ownsRecoveryFence(job, initial))
+            return undefined;
+        if (recorded.data.length === 0)
+            return initial;
+        if (!this.runtime.hasContext('pm/client'))
+            return undefined;
+        const selected = await settle(() => this.runtime.context('pm/client'));
+        if (!selected.ok || !this.ownsRecoveryFence(job, initial))
+            return undefined;
+        const pm = selected.value;
+        let fence = initial;
+        let entries = snapshot.data;
+        for (const [index, workload] of recorded.data.entries()) {
+            if (workload.admissionId === undefined) {
+                const status = await settle(() => pm.getStatus(workload.processId));
+                if (!this.ownsRecoveryFence(job, fence) || !status.ok || !status.value ||
+                    !PM_ENDED_STATUSES.has(status.value.status))
+                    return undefined;
+                continue;
+            }
+            // Retained old PM clients must fail closed before any new capability call.
+            const lookup = z.custom(value => z.function().safeParse(value).success).safeParse(pm.supervisedAdmission);
+            const acknowledge = z.custom(value => z.function().safeParse(value).success).safeParse(pm.acknowledgeSupervised);
+            if (!lookup.success || !acknowledge.success)
+                return undefined;
+            let receipt = workload.settlement;
+            if (!receipt) {
+                const observed = await settle(() => lookup.data.call(pm, pmTaskProcessIdentityOf(workload)));
+                if (!this.ownsRecoveryFence(job, fence) || !observed.ok)
+                    return undefined;
+                const admission = PmTaskAdmissionSchema.safeParse(observed.value);
+                if (!admission.success || !samePmAdmission(workload, admission.data) ||
+                    admission.data.startedAt === undefined || !admission.data.settlement ||
+                    (workload.startedAt !== undefined && workload.startedAt !== admission.data.startedAt))
+                    return undefined;
+                const settled = admission.data.settlement;
+                receipt = settled;
+                // Exact index comes from an immutable, duplicate-checked identity snapshot.
+                // Preserve unknown JSON fields and all other supervised admissions.
+                entries = entries.map((entry, entryIndex) => entryIndex === index
+                    ? { ...entry, startedAt: settled.startedAt, settlement: settled }
+                    : entry);
+                job.metadata = { ...job.metadata, [SUPERVISED_METADATA_KEY]: entries };
+                this.markMetadata(job.id, [SUPERVISED_METADATA_KEY]);
+                job.revision++;
+                fence = recordFence(job);
+                this.persist(job);
+                const stored = await settle(() => this.flushJob(job.id));
+                if (!stored.ok || !this.hasDurableRecoveryFence(job, fence))
+                    return undefined;
+            }
+            // A receipt seen only in an unstored in-memory mutation is never ACK authority.
+            if (!this.hasDurableRecoveryFence(job, fence))
+                return undefined;
+            const acknowledged = await settle(() => acknowledge.data.call(pm, receipt));
+            if (!this.ownsRecoveryFence(job, fence) || !acknowledged.ok || !z.boolean().safeParse(acknowledged.value).success)
+                return undefined;
+            // false is idempotent absence only; the independently durable exact receipt is proof.
+        }
+        return this.ownsRecoveryFence(job, fence) ? fence : undefined;
+    }
+    clearDelayedTimer(jobId) {
+        const timer = this.delayedTimers.get(jobId);
+        if (!timer)
+            return;
+        timer();
+        this.delayedTimers.delete(jobId);
+    }
+    removeQueuedJobOccurrences(jobId) {
+        for (let i = this.queue.length - 1; i >= 0; i--) {
+            if (this.queue[i] === jobId)
+                this.queue.splice(i, 1);
+        }
+    }
+    enqueuePendingJob(jobId, priority) {
+        this.removeQueuedJobOccurrences(jobId);
+        this.queue.splice(priorityInsertIndex(this.queue, (id) => getJobPriority(this.jobs.get(id)), priority), 0, jobId);
+        if (priority !== undefined) {
+            const job = this.jobs.get(jobId);
+            if (job)
+                job.metadata = { ...job.metadata, priority };
+            if (job)
+                this.markMetadata(jobId, ['priority']);
+        }
+    }
+    scheduleDelayedJob(jobId, delayMs, priority) {
+        this.clearDelayedTimer(jobId);
+        const timer = this.runtime.lifecycle.timeout(() => {
+            this.delayedTimers.delete(jobId);
+            const job = this.jobs.get(jobId);
+            if (!job || job.status !== 'delayed' || job.executionClaim.state !== 'none')
+                return;
+            job.status = 'pending';
+            job.revision++;
+            this.enqueuePendingJob(jobId, priority ?? getJobPriority(job));
+            this.persist(job);
+            this.notifyJobAvailable();
+        }, delayMs);
+        this.delayedTimers.set(jobId, timer);
+    }
+}
+export function ipcJobStoreFor(runtime) {
+    const owner = tryGetVibesClient('workers.consumer', runtime);
+    if (!owner)
+        throw new CapabilityError('Compose the Workers consumer plugin before accessing its queue store.', 'UNAVAILABLE', {
+            reason: 'workers-consumer-unavailable', retryable: false
+        });
+    return owner.store;
+}
+//# sourceMappingURL=store.js.map