onadit 0.0.1 → 0.1.0-beta.45

package/dist/src/jobs/queue.js

@@ -0,0 +1,759 @@
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { EXCLUSIVE_JOBS } from "./types.js";
+/**
+ * Terminal statuses: the job is done and its `payload` (full
+ * conversation / export text) is dead weight — the durable record lives
+ * in raw files + the vector index, not the queue ledger. Only
+ * queued/running/interrupted jobs still need their payload.
+ */
+const TERMINAL_STATUSES = new Set([
+    "completed",
+    "failed",
+    "cancelled",
+]);
+/**
+ * Terminal statuses whose `payload` is safe to drop. A `failed` job is
+ * still retryable — `create()` with the same idempotency_key turns it into
+ * a retry whose chain root (the failed job) must still carry the canonical
+ * source/raw payload (save.ts resolveChainRoot). The failure path persists
+ * (and thus compacts) the job the instant it fails, before any retry exists,
+ * so stripping `failed` payloads here would silently break changed-source
+ * retries. Failed jobs are few (cap still bounds them); keep their payload.
+ */
+const STRIPPABLE_STATUSES = new Set([
+    "completed",
+    "cancelled",
+]);
+/**
+ * Cap on retained terminal-job records. Metadata is tiny (~200 B/job),
+ * but unbounded retention eventually blows `JSON.stringify`'s max string
+ * length (V8 ~512 MB) — the failure that turned every save into a 500.
+ * We keep the most recent N; idempotency dedup and the friend-status
+ * projection only need recent terminal jobs, and older sessions are
+ * re-guarded by capture-state.json + content-hash dedup.
+ */
+const MAX_TERMINAL_JOBS = 2000;
+const PAYLOAD_OFFLOAD_THRESHOLD_BYTES = 64 * 1024;
+const MAX_LEDGER_BYTES = 64 * 1024 * 1024;
+const PAYLOAD_REF_MARKER = "__rift_queue_payload_ref";
+const PERSIST_RETRY_DELAY_MS = 100;
+const START_RETRY_INITIAL_DELAY_MS = 100;
+const START_RETRY_MAX_DELAY_MS = 5000;
+/**
+ * In-process job queue with:
+ * - Persistence to `queue.json` (atomic write: temp + rename)
+ * - State machine: queued -> running -> completed | failed, plus cancelled, interrupted
+ * - Idempotency checked before mutual exclusion (PRD 4.1)
+ * - Mutual exclusion for {compact, reindex, reconcile}
+ * - Crash recovery: running -> interrupted -> re-queued on startup
+ */
+export class JobQueue {
+    queuePath;
+    jobs = new Map();
+    payloadRefs = new Map();
+    handlers = new Map();
+    processing = false;
+    held = false;
+    persistRetryTimer;
+    startRetryTimer;
+    startRetryDelayMs = START_RETRY_INITIAL_DELAY_MS;
+    constructor(queuePath) {
+        this.queuePath = queuePath;
+    }
+    /**
+     * Hold processing — registerHandler and create won't auto-start queued
+     * work until releaseHold() is called. Used at startup to prevent
+     * recovered jobs from executing before the integrity check completes.
+     */
+    hold() {
+        this.held = true;
+    }
+    /** Release hold and kick processing for any waiting jobs. */
+    releaseHold() {
+        this.held = false;
+        this.processNext();
+    }
+    /** Register a handler for a job type. Kicks processing for any queued work. */
+    registerHandler(type, handler) {
+        this.handlers.set(type, handler);
+        this.processNext();
+    }
+    /** Check if a handler is registered for a job type. */
+    hasHandler(type) {
+        return this.handlers.has(type);
+    }
+    /**
+     * Load persisted queue from disk. Mark any `running` jobs as
+     * `interrupted` and re-queue them — timers/progress don't survive restart.
+     *
+     * Only jobs that were `running` at crash time get re-queued. Jobs that
+     * were already `interrupted` (from a previous recovery) are left as-is
+     * so repeated loads don't multiply work.
+     */
+    async load() {
+        try {
+            const raw = fs.readFileSync(this.queuePath, "utf-8");
+            const entries = JSON.parse(raw);
+            this.jobs.clear();
+            this.payloadRefs.clear();
+            // Collect jobs that need recovery (were running at crash time).
+            const toRequeue = [];
+            for (const entry of entries) {
+                const job = this.hydrateJob(entry);
+                if (job.status === "running") {
+                    job.status = "interrupted";
+                    job.updated_at = new Date().toISOString();
+                    toRequeue.push(job);
+                }
+                this.jobs.set(job.id, job);
+            }
+            // Re-queue only the newly-interrupted jobs.
+            for (const job of toRequeue) {
+                const requeuedId = crypto.randomUUID();
+                const requeued = {
+                    id: requeuedId,
+                    type: job.type,
+                    status: "queued",
+                    created_at: new Date().toISOString(),
+                    updated_at: new Date().toISOString(),
+                    ...(job.idempotency_key !== undefined ? { idempotency_key: job.idempotency_key } : {}),
+                    retry_of: job.id,
+                    ...(job.payload !== undefined ? { payload: job.payload } : {}),
+                };
+                job.retried_by = requeuedId;
+                this.jobs.set(requeuedId, requeued);
+            }
+            // Heal a ledger that bloated before this fix shipped: strip terminal
+            // payloads / cap retention on load, even when nothing needs re-queuing.
+            const compacted = this.compactLedger();
+            if (toRequeue.length > 0 || compacted) {
+                await this.persist();
+            }
+        }
+        catch (err) {
+            if (err.code === "ENOENT") {
+                // No queue file yet — fresh start.
+                return;
+            }
+            throw err;
+        }
+    }
+    /**
+     * Create a job. Returns `{ job, duplicate }`.
+     *
+     * Idempotency is checked first (PRD 4.1):
+     * - If a job with the same `idempotency_key` exists in
+     *   `queued | running | completed` -> return it with `duplicate: true`.
+     * - If it exists in `failed` -> create a new retry job with `retry_of`.
+     *
+     * Mutual exclusion is checked second:
+     * - If the job type is in the exclusive set and another exclusive job
+     *   is `queued | running`, return null (caller should 409).
+     */
+    async create(type, opts) {
+        const key = opts?.idempotency_key;
+        // --- Idempotency (checked first per PRD 4.1) ---
+        if (key) {
+            const existing = this.findByIdempotencyKey(key);
+            if (existing) {
+                if (existing.status === "queued" ||
+                    existing.status === "running" ||
+                    existing.status === "completed") {
+                    return { job: existing, duplicate: true };
+                }
+                if (existing.status === "failed") {
+                    // Create retry
+                    return this.createRetryJob(existing, type, opts);
+                }
+                // cancelled / interrupted with same key — fall through to create new
+            }
+        }
+        // --- Mutual exclusion ---
+        if (EXCLUSIVE_JOBS.has(type) && this.hasActiveExclusiveJob()) {
+            return { conflict: true };
+        }
+        const job = {
+            id: crypto.randomUUID(),
+            type,
+            status: "queued",
+            created_at: new Date().toISOString(),
+            updated_at: new Date().toISOString(),
+            ...(key !== undefined ? { idempotency_key: key } : {}),
+            ...(opts?.payload !== undefined ? { payload: opts.payload } : {}),
+        };
+        const snapshot = this.snapshotState();
+        this.jobs.set(job.id, job);
+        try {
+            await this.persist();
+        }
+        catch (err) {
+            this.restoreSnapshot(snapshot);
+            this.deletePayloadFilesForJob(job.id);
+            throw err;
+        }
+        this.processNext();
+        return { job, duplicate: false };
+    }
+    /** Get a job by ID, or undefined. */
+    get(id) {
+        return this.jobs.get(id);
+    }
+    /**
+     * Return all jobs whose `idempotency_key` starts with `prefix`. Used
+     * by the friend-status projection to surface inbox freshness — jobs
+     * created from the inbox watcher all use keys of the form
+     * `inbox:<hex>`, so a prefix scan answers "what's the inbox lane
+     * doing" without leaking queue internals to the route layer.
+     *
+     * Read-only iteration of the in-memory map; safe to call on a hot
+     * queue. Returns the snapshot in insertion order.
+     */
+    listByIdempotencyPrefix(prefix) {
+        const out = [];
+        for (const job of this.jobs.values()) {
+            if (job.idempotency_key?.startsWith(prefix))
+                out.push(job);
+        }
+        return out;
+    }
+    /**
+     * Check whether an idempotency key matches an existing active job.
+     * Returns the job if found in queued/running/completed state, undefined otherwise.
+     * Does NOT create or mutate anything — safe for pre-flight checks.
+     */
+    findDuplicate(idempotencyKey) {
+        const existing = this.findByIdempotencyKey(idempotencyKey);
+        if (existing &&
+            (existing.status === "queued" ||
+                existing.status === "running" ||
+                existing.status === "completed")) {
+            return existing;
+        }
+        return undefined;
+    }
+    /**
+     * Cancel a job.
+     * - `queued` or `interrupted` -> `cancelled`, returns the job.
+     * - `running` -> returns `{ conflict: true }`.
+     * - Others -> returns undefined (not found or not cancellable).
+     */
+    async cancel(id) {
+        const job = this.jobs.get(id);
+        if (!job)
+            return undefined;
+        if (job.status === "queued" || job.status === "interrupted") {
+            const snapshot = this.snapshotState();
+            job.status = "cancelled";
+            job.updated_at = new Date().toISOString();
+            try {
+                await this.persist();
+            }
+            catch (err) {
+                this.restoreSnapshot(snapshot);
+                throw err;
+            }
+            return job;
+        }
+        if (job.status === "running") {
+            return { conflict: true };
+        }
+        return undefined;
+    }
+    /**
+     * Keep the persisted ledger bounded so `JSON.stringify` can never exceed
+     * V8's max string length (the failure mode that turned every save into a
+     * 500). Two cheap, safe reductions on terminal jobs only:
+     *   1. Drop `payload` — only active jobs need it.
+     *   2. Cap how many terminal jobs we retain (oldest dropped first).
+     * Non-terminal jobs (queued/running/interrupted) are never touched.
+     * Returns true if anything changed.
+     */
+    compactLedger() {
+        let changed = false;
+        // Protect every job reachable by walking `retry_of` from a non-terminal
+        // job. The save/ingest handlers call resolveChainRoot() to recover the
+        // canonical row id + source from the chain root. If an ancestor is
+        // dropped (payload stripped OR row deleted) while a retry is still
+        // queued/running, resolveChainRoot stops short (save.ts:55), the handler
+        // falls back to the retry's own id/source, and a duplicate row is
+        // created. So these ancestors are exempt from BOTH reductions below.
+        const protectedIds = new Set();
+        for (const job of this.jobs.values()) {
+            if (TERMINAL_STATUSES.has(job.status))
+                continue;
+            let cursor = job;
+            let depth = 0;
+            while (cursor?.retry_of && depth < 50) {
+                const parent = this.jobs.get(cursor.retry_of);
+                if (!parent)
+                    break;
+                protectedIds.add(parent.id);
+                cursor = parent;
+                depth++;
+            }
+        }
+        const terminal = [];
+        for (const job of this.jobs.values()) {
+            if (TERMINAL_STATUSES.has(job.status)) {
+                // Keep payload on retry-chain ancestors: the save handler walks
+                // `retry_of` to the root job and reads its payload to resolve the
+                // canonical source/raw artifact (src/jobs/handlers/save.ts). A job
+                // with `retried_by` set is such an ancestor, as is any ancestor of
+                // a still-active retry (protectedIds). Standalone/leaf terminal jobs
+                // no longer need their payload.
+                if (job.payload !== undefined &&
+                    STRIPPABLE_STATUSES.has(job.status) &&
+                    job.retried_by === undefined &&
+                    !protectedIds.has(job.id)) {
+                    delete job.payload;
+                    changed = true;
+                }
+                terminal.push(job);
+            }
+        }
+        if (terminal.length > MAX_TERMINAL_JOBS) {
+            // Drop the oldest terminal jobs beyond the cap (oldest by updated_at),
+            // but never an ancestor of a still-active retry chain. If protected
+            // jobs prevent reaching the cap, the ledger stays slightly above it —
+            // correctness over a hard bound (the overflow is bounded by in-flight
+            // work, which is small).
+            terminal.sort((a, b) => a.updated_at.localeCompare(b.updated_at));
+            const overflow = terminal.length - MAX_TERMINAL_JOBS;
+            let dropped = 0;
+            for (const job of terminal) {
+                if (dropped >= overflow)
+                    break;
+                if (protectedIds.has(job.id))
+                    continue;
+                this.jobs.delete(job.id);
+                dropped++;
+                changed = true;
+            }
+        }
+        return changed;
+    }
+    /** Atomic write: temp file + rename. Ledger is compacted first so the
+     * serialized form stays well under V8's max string length. */
+    async persist() {
+        this.compactLedger();
+        this.prunePayloadRefCache();
+        const activePayloadRefs = new Set();
+        const entries = [...this.jobs.values()].map((job) => this.serializeJobForLedger(job, activePayloadRefs));
+        let data;
+        try {
+            data = JSON.stringify(entries, null, 2);
+            if (Buffer.byteLength(data, "utf-8") > MAX_LEDGER_BYTES) {
+                throw this.createLedgerTooLargeError(`Job ledger exceeds ${MAX_LEDGER_BYTES} bytes after payload offload.`);
+            }
+        }
+        catch (err) {
+            // Defense in depth: even after compaction, a pathological volume of
+            // active payloads could exceed V8's max string length. Surface a
+            // typed, actionable error instead of a bare 500 — and never silently
+            // corrupt active jobs by dropping their payloads.
+            if (err instanceof RangeError) {
+                throw this.createLedgerTooLargeError("Job ledger too large to serialize (exceeds V8 max string length).");
+            }
+            throw err;
+        }
+        const tmp = this.queuePath + `.tmp.${process.pid}`;
+        fs.writeFileSync(tmp, data, "utf-8");
+        fs.renameSync(tmp, this.queuePath);
+        this.cleanupPayloadFiles(activePayloadRefs);
+        this.clearPersistRetry();
+    }
+    createLedgerTooLargeError(message) {
+        const e = new Error(`${message} Payloads above ${PAYLOAD_OFFLOAD_THRESHOLD_BYTES} bytes are ` +
+            "offloaded automatically, so this means there are too many queued jobs " +
+            "or a malformed payload reference. Retry after current jobs drain, or " +
+            "see reports/2026-05-25-capture-500-queue-bloat.md for manual recovery.");
+        e.code = "queue_ledger_too_large";
+        return e;
+    }
+    snapshotState() {
+        return {
+            jobs: new Map([...this.jobs.entries()].map(([id, job]) => [id, { ...job }])),
+            payloadRefs: new Map([...this.payloadRefs.entries()].map(([id, entry]) => [
+                id,
+                { ref: { ...entry.ref }, value: entry.value },
+            ])),
+        };
+    }
+    restoreSnapshot(snapshot) {
+        this.jobs = snapshot.jobs;
+        this.payloadRefs = snapshot.payloadRefs;
+    }
+    payloadDir() {
+        return path.join(path.dirname(this.queuePath), "payloads");
+    }
+    payloadPath(jobId, sha256) {
+        return path.join(this.payloadDir(), `${jobId}.${sha256}.json`);
+    }
+    payloadRefPath(jobId, sha256) {
+        return `payloads/${jobId}.${sha256}.json`;
+    }
+    payloadAbsPath(ref) {
+        return path.isAbsolute(ref.path)
+            ? ref.path
+            : path.join(path.dirname(this.queuePath), ref.path);
+    }
+    isPayloadRef(payload) {
+        return (typeof payload === "object" &&
+            payload !== null &&
+            payload[PAYLOAD_REF_MARKER] === true &&
+            typeof payload["path"] === "string" &&
+            typeof payload["byte_length"] === "number" &&
+            typeof payload["sha256"] === "string");
+    }
+    hydrateJob(entry) {
+        const job = { ...entry };
+        if (this.isPayloadRef(entry.payload)) {
+            const payload = this.readPayloadRef(entry.payload);
+            job.payload = payload;
+            this.payloadRefs.set(job.id, {
+                ref: { ...entry.payload },
+                value: payload,
+            });
+        }
+        return job;
+    }
+    readPayloadRef(ref) {
+        const refPath = this.payloadAbsPath(ref);
+        const raw = fs.readFileSync(refPath, "utf-8");
+        const bytes = Buffer.byteLength(raw, "utf-8");
+        const sha256 = crypto.createHash("sha256").update(raw).digest("hex");
+        if (bytes !== ref.byte_length || sha256 !== ref.sha256) {
+            throw new Error(`Job payload ref failed integrity check: ${ref.path}`);
+        }
+        return JSON.parse(raw);
+    }
+    serializeJobForLedger(job, activePayloadRefs) {
+        const persisted = { ...job };
+        if (job.payload === undefined) {
+            delete persisted.payload;
+            this.payloadRefs.delete(job.id);
+            return persisted;
+        }
+        const cached = this.payloadRefs.get(job.id);
+        if (cached &&
+            cached.value === job.payload &&
+            fs.existsSync(this.payloadAbsPath(cached.ref))) {
+            activePayloadRefs.add(this.payloadAbsPath(cached.ref));
+            persisted.payload = cached.ref;
+            return persisted;
+        }
+        const payloadJson = JSON.stringify(job.payload);
+        if (payloadJson === undefined) {
+            throw new Error(`Job ${job.id} payload is not JSON-serializable`);
+        }
+        const byteLength = Buffer.byteLength(payloadJson, "utf-8");
+        if (byteLength <= PAYLOAD_OFFLOAD_THRESHOLD_BYTES) {
+            this.payloadRefs.delete(job.id);
+            persisted.payload = job.payload;
+            return persisted;
+        }
+        const sha256 = crypto.createHash("sha256").update(payloadJson).digest("hex");
+        const refPath = this.payloadRefPath(job.id, sha256);
+        const ref = {
+            [PAYLOAD_REF_MARKER]: true,
+            path: refPath,
+            byte_length: byteLength,
+            sha256,
+        };
+        this.writePayloadFile(this.payloadPath(job.id, sha256), payloadJson);
+        this.payloadRefs.set(job.id, { ref, value: job.payload });
+        activePayloadRefs.add(this.payloadAbsPath(ref));
+        persisted.payload = ref;
+        return persisted;
+    }
+    writePayloadFile(filePath, data) {
+        fs.mkdirSync(path.dirname(filePath), { recursive: true });
+        const tmp = `${filePath}.tmp.${process.pid}`;
+        fs.writeFileSync(tmp, data, "utf-8");
+        fs.renameSync(tmp, filePath);
+    }
+    prunePayloadRefCache() {
+        for (const jobId of this.payloadRefs.keys()) {
+            if (!this.jobs.has(jobId)) {
+                this.payloadRefs.delete(jobId);
+            }
+        }
+    }
+    deletePayloadFilesForJob(jobId) {
+        const dir = this.payloadDir();
+        let files;
+        try {
+            if (!fs.existsSync(dir))
+                return;
+            files = fs.readdirSync(dir);
+        }
+        catch (err) {
+            process.stderr.write(`queue: warning: failed to scan payload files for ${jobId}: ${err instanceof Error ? err.message : String(err)}\n`);
+            return;
+        }
+        for (const file of files) {
+            if (file === `${jobId}.json` || file.startsWith(`${jobId}.`)) {
+                this.deletePayloadPath(path.join(dir, file), jobId);
+            }
+        }
+    }
+    cleanupPayloadFiles(activePayloadRefs) {
+        const dir = this.payloadDir();
+        let files;
+        try {
+            if (!fs.existsSync(dir))
+                return;
+            files = fs.readdirSync(dir);
+        }
+        catch (err) {
+            process.stderr.write(`queue: warning: failed to scan payload directory for cleanup: ${err instanceof Error ? err.message : String(err)}\n`);
+            return;
+        }
+        for (const file of files) {
+            if (!file.endsWith(".json"))
+                continue;
+            const filePath = path.join(dir, file);
+            if (!activePayloadRefs.has(filePath)) {
+                this.deletePayloadPath(filePath, file);
+            }
+        }
+    }
+    deletePayloadPath(filePath, label) {
+        try {
+            fs.unlinkSync(filePath);
+        }
+        catch (err) {
+            if (err.code !== "ENOENT") {
+                process.stderr.write(`queue: warning: failed to remove payload file ${label}: ${err instanceof Error ? err.message : String(err)}\n`);
+            }
+        }
+    }
+    /**
+     * Find the active job for a given idempotency key.
+     * Skip superseded jobs (those with `retried_by` set) so we always
+     * find the leaf of the retry chain, not the old parent.
+     */
+    findByIdempotencyKey(key) {
+        for (const job of this.jobs.values()) {
+            if (job.idempotency_key === key && job.retried_by === undefined)
+                return job;
+        }
+        return undefined;
+    }
+    /**
+     * Create a job that bypasses the mutual exclusion check and is
+     * inserted at the front of the queue (runs before other queued work).
+     *
+     * Used exclusively for startup corruption repair: the mandatory
+     * rebuild must run before any other queued work touches storage,
+     * regardless of what else is in the queue. Other exclusive jobs
+     * naturally wait (one-at-a-time processing) and run after the
+     * rebuild against healthy storage.
+     */
+    async createUrgent(type, opts) {
+        const job = {
+            id: crypto.randomUUID(),
+            type,
+            status: "queued",
+            created_at: new Date().toISOString(),
+            updated_at: new Date().toISOString(),
+            ...(opts?.payload !== undefined ? { payload: opts.payload } : {}),
+        };
+        // Rebuild the Map with this job first so findNextQueued() picks it up
+        // before any previously queued work.
+        const snapshot = this.snapshotState();
+        const entries = [...this.jobs.entries()];
+        this.jobs.clear();
+        this.jobs.set(job.id, job);
+        for (const [id, j] of entries) {
+            this.jobs.set(id, j);
+        }
+        try {
+            await this.persist();
+        }
+        catch (err) {
+            this.restoreSnapshot(snapshot);
+            this.deletePayloadFilesForJob(job.id);
+            throw err;
+        }
+        this.processNext();
+        return { job, duplicate: false };
+    }
+    /** Whether any exclusive job is queued or running. */
+    hasActiveExclusiveJob() {
+        for (const job of this.jobs.values()) {
+            if (EXCLUSIVE_JOBS.has(job.type) &&
+                (job.status === "queued" || job.status === "running")) {
+                return true;
+            }
+        }
+        return false;
+    }
+    /**
+     * Return the exclusive job that is currently running, or null.
+     * Only returns jobs with status "running" — queued exclusive jobs
+     * have not entered their coordinated maintenance phase.
+     */
+    getRunningExclusiveJob() {
+        for (const job of this.jobs.values()) {
+            if (EXCLUSIVE_JOBS.has(job.type) && job.status === "running") {
+                return job;
+            }
+        }
+        return null;
+    }
+    async createRetryJob(failed, type, opts) {
+        // Mutual exclusion still applies to retries of exclusive jobs.
+        if (EXCLUSIVE_JOBS.has(type) && this.hasActiveExclusiveJob()) {
+            return { conflict: true };
+        }
+        const retryId = crypto.randomUUID();
+        const retry = {
+            id: retryId,
+            type,
+            status: "queued",
+            created_at: new Date().toISOString(),
+            updated_at: new Date().toISOString(),
+            ...(opts?.idempotency_key !== undefined ? { idempotency_key: opts.idempotency_key } : {}),
+            retry_of: failed.id,
+            ...(opts?.payload !== undefined ? { payload: opts.payload } : {}),
+        };
+        const snapshot = this.snapshotState();
+        failed.retried_by = retryId;
+        failed.updated_at = new Date().toISOString();
+        this.jobs.set(retryId, retry);
+        try {
+            await this.persist();
+        }
+        catch (err) {
+            this.restoreSnapshot(snapshot);
+            this.deletePayloadFilesForJob(retryId);
+            throw err;
+        }
+        this.processNext();
+        return { job: retry, duplicate: false };
+    }
+    /** Process the next queued job if no job is currently running. */
+    processNext() {
+        if (this.held)
+            return;
+        if (this.processing)
+            return;
+        const next = this.findNextQueued();
+        if (!next)
+            return;
+        const handler = this.handlers.get(next.type);
+        if (!handler)
+            return; // No handler registered — leave queued.
+        this.processing = true;
+        void (async () => {
+            try {
+                await this.applyPersistedMutation(() => {
+                    next.status = "running";
+                    next.updated_at = new Date().toISOString();
+                });
+            }
+            catch (err) {
+                this.processing = false;
+                process.stderr.write(`queue: failed to persist start of job ${next.id}: ${err instanceof Error ? err.message : String(err)}\n`);
+                this.scheduleStartRetry();
+                return;
+            }
+            this.clearStartRetry();
+            try {
+                await handler(next);
+                await this.applyTerminalMutation(`completion of job ${next.id}`, () => {
+                    next.status = "completed";
+                    next.updated_at = new Date().toISOString();
+                });
+            }
+            catch (err) {
+                await this.applyTerminalMutation(`failure of job ${next.id}`, () => {
+                    next.status = "failed";
+                    next.error = err instanceof Error ? err.message : String(err);
+                    next.updated_at = new Date().toISOString();
+                });
+            }
+            finally {
+                this.processing = false;
+                this.processNext();
+            }
+        })();
+    }
+    async applyPersistedMutation(mutator) {
+        const snapshot = this.snapshotState();
+        mutator();
+        try {
+            await this.persist();
+        }
+        catch (err) {
+            this.restoreSnapshot(snapshot);
+            throw err;
+        }
+    }
+    async applyTerminalMutation(label, mutator) {
+        mutator();
+        try {
+            await this.persist();
+        }
+        catch (err) {
+            this.markPersistDirty(label, err);
+        }
+    }
+    markPersistDirty(label, err) {
+        process.stderr.write(`queue: failed to persist ${label}: ${err instanceof Error ? err.message : String(err)}; will retry\n`);
+        this.schedulePersistRetry();
+    }
+    schedulePersistRetry() {
+        if (this.persistRetryTimer)
+            return;
+        this.persistRetryTimer = setTimeout(() => {
+            this.persistRetryTimer = undefined;
+            void this.retryDirtyPersist();
+        }, PERSIST_RETRY_DELAY_MS);
+        this.persistRetryTimer.unref?.();
+    }
+    scheduleStartRetry() {
+        if (this.startRetryTimer)
+            return;
+        const delayMs = this.startRetryDelayMs;
+        this.startRetryDelayMs = Math.min(this.startRetryDelayMs * 2, START_RETRY_MAX_DELAY_MS);
+        this.startRetryTimer = setTimeout(() => {
+            this.startRetryTimer = undefined;
+            this.processNext();
+        }, delayMs);
+        this.startRetryTimer.unref?.();
+    }
+    clearStartRetry() {
+        if (this.startRetryTimer) {
+            clearTimeout(this.startRetryTimer);
+            this.startRetryTimer = undefined;
+        }
+        this.startRetryDelayMs = START_RETRY_INITIAL_DELAY_MS;
+    }
+    clearPersistRetry() {
+        if (!this.persistRetryTimer)
+            return;
+        clearTimeout(this.persistRetryTimer);
+        this.persistRetryTimer = undefined;
+    }
+    async retryDirtyPersist() {
+        try {
+            await this.persist();
+        }
+        catch (err) {
+            process.stderr.write(`queue: failed to retry dirty ledger persist: ${err instanceof Error ? err.message : String(err)}\n`);
+            this.schedulePersistRetry();
+        }
+    }
+    findNextQueued() {
+        for (const job of this.jobs.values()) {
+            if (job.status === "queued")
+                return job;
+        }
+        return undefined;
+    }
+}
+//# sourceMappingURL=queue.js.map