@voybio/ace-swarm 0.1.0

package/dist/job-scheduler.js

@@ -0,0 +1,1217 @@
+import { randomUUID } from "node:crypto";
+import { readHandoffRegistry } from "./handoff-registry.js";
+import { safeRead, safeWrite, withFileLock, wsPath } from "./helpers.js";
+import { isReadError } from "./shared.js";
+import { appendStatusEventSafe, readStatusEvents } from "./status-events.js";
+import { appendRunLedgerEntrySafe } from "./run-ledger.js";
+import { readTodoState } from "./todo-state.js";
+// ── Buffered transition-event infrastructure ────────────────────────
+// Events are collected during the scheduler lock and flushed after
+// the lock is released.  Emission is failure-tolerant: a broken
+// STATUS_EVENTS file never blocks a state transition.
+const SCHEDULER_SOURCE_MODULE = "capability-scheduler";
+function newEventBuffer() {
+    return { status: [], ledger: [] };
+}
+async function flushTransitionEvents(buf) {
+    for (const ev of buf.status) {
+        try {
+            await appendStatusEventSafe({
+                source_module: SCHEDULER_SOURCE_MODULE,
+                event_type: ev.event_type,
+                status: ev.status,
+                summary: ev.summary,
+                payload: ev.payload,
+            });
+        }
+        catch {
+            // Transition events must never break scheduler operations
+        }
+    }
+    for (const le of buf.ledger) {
+        try {
+            await appendRunLedgerEntrySafe(le);
+        }
+        catch {
+            // Ledger entries must never break scheduler operations
+        }
+    }
+}
+export const JOB_QUEUE_REL = "agent-state/job-queue.json";
+export const JOB_LOCK_TABLE_REL = "agent-state/job-locks.json";
+export const SCHEDULER_LEASE_REL = "agent-state/scheduler-lease.json";
+export const SCHEDULER_LOCK_REL = "agent-state/job-scheduler.lock";
+export const PRIORITY_BANDS = ["P0", "P1", "P2", "P3"];
+export const JOB_STATUS = [
+    "pending",
+    "accepted",
+    "blocked",
+    "ready",
+    "running",
+    "done",
+    "failed",
+    "failed_terminal",
+    "canceled",
+    "unknown_recovery",
+];
+export const JOB_LOCK_STATUS = [
+    "active",
+    "completed",
+    "released",
+    "failed",
+];
+const PRIORITY_WEIGHT = {
+    P0: 0,
+    P1: 1,
+    P2: 2,
+    P3: 3,
+};
+const TERMINAL_STATUSES = new Set(["done", "failed_terminal", "canceled"]);
+const BLOCKED_REASONS = new Set(["blocked", "failed"]);
+function nowIso(now = new Date()) {
+    return now.toISOString();
+}
+function parseDate(input) {
+    if (!input)
+        return undefined;
+    const ms = Date.parse(input);
+    if (!Number.isFinite(ms))
+        return undefined;
+    return new Date(ms);
+}
+function toPriority(input) {
+    if (!input)
+        return "P2";
+    return PRIORITY_BANDS.includes(input)
+        ? input
+        : "P2";
+}
+function matchesExpected(actual, expected) {
+    const candidates = expected
+        .split("|")
+        .map((row) => row.trim())
+        .filter((row) => row.length > 0);
+    if (candidates.length === 0)
+        return actual === expected;
+    return candidates.includes(actual);
+}
+function defaultQueueFile() {
+    return {
+        version: 1,
+        updated_at: nowIso(),
+        jobs: [],
+    };
+}
+function defaultJobLockFile() {
+    return {
+        version: 1,
+        updated_at: nowIso(),
+        locks: [],
+    };
+}
+function defaultLease(owner = "scheduler") {
+    const now = new Date();
+    const expires = new Date(now.getTime() + 30_000);
+    return {
+        version: 1,
+        lease_id: randomUUID(),
+        owner,
+        acquired_at: nowIso(now),
+        heartbeat_at: nowIso(now),
+        expires_at: nowIso(expires),
+    };
+}
+function parseQueueFile(raw) {
+    try {
+        const parsed = JSON.parse(raw);
+        if (!parsed || typeof parsed !== "object" || Array.isArray(parsed))
+            return undefined;
+        const candidate = parsed;
+        if (candidate.version !== 1 || !Array.isArray(candidate.jobs))
+            return undefined;
+        const jobs = [];
+        for (const row of candidate.jobs) {
+            if (!row || typeof row !== "object")
+                continue;
+            const job = row;
+            if (typeof job.job_id !== "string" || typeof job.kind !== "string")
+                continue;
+            const status = JOB_STATUS.includes(job.status)
+                ? job.status
+                : "pending";
+            const priority = toPriority(job.priority);
+            const deps = Array.isArray(job.dependencies)
+                ? job.dependencies
+                    .filter((dep) => {
+                    if (!dep || typeof dep !== "object")
+                        return false;
+                    const typed = dep;
+                    return ((typed.kind === "gate" ||
+                        typed.kind === "handoff" ||
+                        typed.kind === "todo" ||
+                        typed.kind === "time") &&
+                        typeof typed.ref === "string");
+                })
+                    .map((dep) => ({ kind: dep.kind, ref: dep.ref, expected: dep.expected }))
+                : [];
+            jobs.push({
+                job_id: job.job_id,
+                kind: job.kind,
+                payload: job.payload && typeof job.payload === "object" ? job.payload : {},
+                status,
+                priority,
+                dependencies: deps,
+                resource_requirements: Array.isArray(job.resource_requirements)
+                    ? job.resource_requirements.filter((value) => typeof value === "string" && value.trim().length > 0)
+                    : ["agent-state/job-scheduler.default"],
+                attempt: typeof job.attempt === "number" ? job.attempt : 0,
+                max_attempts: typeof job.max_attempts === "number" ? job.max_attempts : 3,
+                backoff_ms: typeof job.backoff_ms === "number" ? job.backoff_ms : 5_000,
+                next_attempt_at: typeof job.next_attempt_at === "string" ? job.next_attempt_at : undefined,
+                created_at: typeof job.created_at === "string" ? job.created_at : nowIso(),
+                accepted_at: typeof job.accepted_at === "string" ? job.accepted_at : undefined,
+                started_at: typeof job.started_at === "string" ? job.started_at : undefined,
+                completed_at: typeof job.completed_at === "string" ? job.completed_at : undefined,
+                evidence_ref: typeof job.evidence_ref === "string" ? job.evidence_ref : undefined,
+                error: typeof job.error === "string" ? job.error : undefined,
+                reason_code: typeof job.reason_code === "string" ? job.reason_code : undefined,
+            });
+        }
+        return {
+            version: 1,
+            updated_at: typeof candidate.updated_at === "string" ? candidate.updated_at : nowIso(),
+            jobs,
+        };
+    }
+    catch {
+        return undefined;
+    }
+}
+function parseJobLockFile(raw) {
+    try {
+        const parsed = JSON.parse(raw);
+        if (!parsed || typeof parsed !== "object" || Array.isArray(parsed))
+            return undefined;
+        const candidate = parsed;
+        if (candidate.version !== 1 || !Array.isArray(candidate.locks))
+            return undefined;
+        const locks = [];
+        for (const row of candidate.locks) {
+            if (!row || typeof row !== "object")
+                continue;
+            const record = row;
+            if (typeof record.lock_id !== "string" ||
+                typeof record.job_id !== "string" ||
+                typeof record.resource !== "string") {
+                continue;
+            }
+            const status = JOB_LOCK_STATUS.includes(record.status)
+                ? record.status
+                : "active";
+            locks.push({
+                lock_id: record.lock_id,
+                job_id: record.job_id,
+                phase: typeof record.phase === "string" ? record.phase : "dispatch",
+                resource: record.resource,
+                status,
+                priority: toPriority(record.priority),
+                lease_id: typeof record.lease_id === "string" ? record.lease_id : undefined,
+                reason_code: typeof record.reason_code === "string" ? record.reason_code : undefined,
+                acquired_at: typeof record.acquired_at === "string" ? record.acquired_at : undefined,
+                heartbeat_at: typeof record.heartbeat_at === "string" ? record.heartbeat_at : undefined,
+                released_at: typeof record.released_at === "string" ? record.released_at : undefined,
+                created_at: typeof record.created_at === "string" ? record.created_at : nowIso(),
+            });
+        }
+        return {
+            version: 1,
+            updated_at: typeof candidate.updated_at === "string" ? candidate.updated_at : nowIso(),
+            locks,
+        };
+    }
+    catch {
+        return undefined;
+    }
+}
+function parseLease(raw) {
+    try {
+        const parsed = JSON.parse(raw);
+        if (!parsed || typeof parsed !== "object" || Array.isArray(parsed))
+            return undefined;
+        const candidate = parsed;
+        if (candidate.version !== 1 ||
+            typeof candidate.lease_id !== "string" ||
+            typeof candidate.owner !== "string" ||
+            typeof candidate.acquired_at !== "string" ||
+            typeof candidate.heartbeat_at !== "string" ||
+            typeof candidate.expires_at !== "string") {
+            return undefined;
+        }
+        return {
+            version: 1,
+            lease_id: candidate.lease_id,
+            owner: candidate.owner,
+            acquired_at: candidate.acquired_at,
+            heartbeat_at: candidate.heartbeat_at,
+            expires_at: candidate.expires_at,
+        };
+    }
+    catch {
+        return undefined;
+    }
+}
+export function getJobQueuePath() {
+    return wsPath(JOB_QUEUE_REL);
+}
+export function getJobLockTablePath() {
+    return wsPath(JOB_LOCK_TABLE_REL);
+}
+export function getSchedulerLeasePath() {
+    return wsPath(SCHEDULER_LEASE_REL);
+}
+export function readJobQueue() {
+    const raw = safeRead(JOB_QUEUE_REL);
+    if (isReadError(raw))
+        return defaultQueueFile();
+    return parseQueueFile(raw) ?? defaultQueueFile();
+}
+export function readJobLockTable() {
+    const raw = safeRead(JOB_LOCK_TABLE_REL);
+    if (isReadError(raw))
+        return defaultJobLockFile();
+    return parseJobLockFile(raw) ?? defaultJobLockFile();
+}
+export function readSchedulerLease() {
+    const raw = safeRead(SCHEDULER_LEASE_REL);
+    if (isReadError(raw))
+        return undefined;
+    return parseLease(raw);
+}
+function writeQueue(queue) {
+    queue.updated_at = nowIso();
+    return safeWrite(JOB_QUEUE_REL, JSON.stringify(queue, null, 2));
+}
+function writeJobLocks(table) {
+    table.updated_at = nowIso();
+    return safeWrite(JOB_LOCK_TABLE_REL, JSON.stringify(table, null, 2));
+}
+function writeLease(lease) {
+    return safeWrite(SCHEDULER_LEASE_REL, JSON.stringify(lease, null, 2));
+}
+function compareJobs(a, b) {
+    const priorityCmp = PRIORITY_WEIGHT[a.priority] - PRIORITY_WEIGHT[b.priority];
+    if (priorityCmp !== 0)
+        return priorityCmp;
+    const createdCmp = Date.parse(a.created_at) - Date.parse(b.created_at);
+    if (createdCmp !== 0)
+        return createdCmp;
+    return a.job_id.localeCompare(b.job_id);
+}
+function parseGateResults(payload) {
+    const rows = Array.isArray(payload?.gate_results)
+        ? payload.gate_results
+        : [];
+    return rows
+        .filter((row) => {
+        if (!row || typeof row !== "object" || Array.isArray(row))
+            return false;
+        const candidate = row;
+        return typeof candidate.id === "string" && typeof candidate.status === "string";
+    })
+        .map((row) => ({
+        id: row.id,
+        status: row.status,
+        detail: typeof row.detail === "string" ? row.detail : undefined,
+    }));
+}
+function resolveGateDependencyStatus(dependency, statusEvents) {
+    const gateEvents = statusEvents
+        .filter((event) => event.event_type === "GATES_EXECUTED")
+        .reverse();
+    for (const event of gateEvents) {
+        const gateResults = parseGateResults(event.payload);
+        if (dependency.ref) {
+            const match = gateResults.find((row) => row.id === dependency.ref);
+            if (match) {
+                return { status: match.status, detail: match.detail };
+            }
+        }
+        const gatesRun = Array.isArray(event.payload?.gates_run)
+            ? event.payload.gates_run.filter((value) => typeof value === "string")
+            : [];
+        if (!dependency.ref) {
+            return { status: event.status };
+        }
+        if (gatesRun.includes(dependency.ref)) {
+            return { status: event.status };
+        }
+    }
+    return undefined;
+}
+function evaluateDependencyBlocks(job, now, context) {
+    const blocks = [];
+    const { todoState, handoffState, statusEvents } = context;
+    if (job.next_attempt_at) {
+        const nextAttempt = parseDate(job.next_attempt_at);
+        if (nextAttempt && nextAttempt.getTime() > now.getTime()) {
+            blocks.push({
+                dependency: { kind: "time", ref: job.next_attempt_at, expected: "retry" },
+                reason_code: "retry_backoff",
+                detail: `Retry blocked until ${job.next_attempt_at}`,
+            });
+        }
+    }
+    for (const dependency of job.dependencies) {
+        if (dependency.kind === "time") {
+            const marker = dependency.expected ?? dependency.ref;
+            const readyAt = parseDate(marker);
+            if (!readyAt) {
+                blocks.push({
+                    dependency,
+                    reason_code: "invalid_time",
+                    detail: `Invalid time dependency: ${marker}`,
+                });
+                continue;
+            }
+            if (readyAt.getTime() > now.getTime()) {
+                blocks.push({
+                    dependency,
+                    reason_code: "time_not_reached",
+                    detail: `Waiting until ${readyAt.toISOString()}`,
+                });
+            }
+            continue;
+        }
+        if (dependency.kind === "todo") {
+            const expected = (dependency.expected ?? "done");
+            const row = todoState.nodes[dependency.ref];
+            if (!row) {
+                blocks.push({
+                    dependency,
+                    reason_code: "todo_missing",
+                    detail: `TODO dependency not found: ${dependency.ref}`,
+                });
+                continue;
+            }
+            if (!matchesExpected(row.status, expected)) {
+                blocks.push({
+                    dependency,
+                    reason_code: "todo_unmet",
+                    detail: `TODO ${dependency.ref} is ${row.status}, expected ${expected}`,
+                });
+            }
+            continue;
+        }
+        if (dependency.kind === "handoff") {
+            const expected = dependency.expected ?? "completed";
+            const row = handoffState.handoffs[dependency.ref];
+            if (!row) {
+                blocks.push({
+                    dependency,
+                    reason_code: "handoff_missing",
+                    detail: `Handoff dependency not found: ${dependency.ref}`,
+                });
+                continue;
+            }
+            if (!matchesExpected(row.status ?? "open", expected)) {
+                blocks.push({
+                    dependency,
+                    reason_code: "handoff_unmet",
+                    detail: `Handoff ${dependency.ref} is ${row.status ?? "open"}, expected ${expected}`,
+                });
+            }
+            continue;
+        }
+        if (dependency.kind === "gate") {
+            const expected = dependency.expected ?? "pass";
+            const relevant = resolveGateDependencyStatus(dependency, statusEvents);
+            if (!relevant) {
+                blocks.push({
+                    dependency,
+                    reason_code: "gate_unknown",
+                    detail: `No gate evidence found for ${dependency.ref || "<any>"}`,
+                });
+                continue;
+            }
+            if (!matchesExpected(relevant.status, expected)) {
+                blocks.push({
+                    dependency,
+                    reason_code: "gate_unmet",
+                    detail: `Gate ${dependency.ref || "<any>"} latest status=${relevant.status}, expected=${expected}${relevant.detail ? ` (${relevant.detail})` : ""}`,
+                });
+            }
+        }
+    }
+    return blocks;
+}
+function normalizeResources(input) {
+    const rows = (input ?? []).map((row) => row.trim()).filter((row) => row.length > 0);
+    if (rows.length === 0)
+        return ["agent-state/job-scheduler.default"];
+    return Array.from(new Set(rows));
+}
+function computeBackoffMs(job) {
+    const base = Math.max(1000, Math.floor(job.backoff_ms || 1000));
+    const exponent = Math.max(0, job.attempt - 1);
+    const factor = Math.min(32, 2 ** exponent);
+    return base * factor;
+}
+function isJobLockActive(record) {
+    return record.status === "active";
+}
+function buildOccupancy(locks) {
+    const occupied = new Set();
+    for (const lock of locks) {
+        if (!isJobLockActive(lock))
+            continue;
+        occupied.add(lock.resource);
+    }
+    return occupied;
+}
+function deriveOwner(input) {
+    const trimmed = input?.trim();
+    if (trimmed && trimmed.length > 0)
+        return trimmed;
+    return "capability-ops";
+}
+function defaultLeaseTtlSeconds() {
+    return 1800;
+}
+function hasActiveLockForJob(locks, jobId) {
+    return locks.some((lock) => lock.job_id === jobId && isJobLockActive(lock));
+}
+function acquireLease(owner, ttlSeconds, now) {
+    const existing = readSchedulerLease();
+    const expiry = new Date(now.getTime() + ttlSeconds * 1000);
+    if (!existing) {
+        const lease = {
+            version: 1,
+            lease_id: randomUUID(),
+            owner,
+            acquired_at: nowIso(now),
+            heartbeat_at: nowIso(now),
+            expires_at: nowIso(expiry),
+        };
+        writeLease(lease);
+        return { acquired: true, lease, recovery_mode: false };
+    }
+    const existingExpiry = parseDate(existing.expires_at);
+    const expired = !existingExpiry || existingExpiry.getTime() <= now.getTime();
+    if (expired || existing.owner === owner) {
+        const lease = {
+            version: 1,
+            lease_id: expired ? randomUUID() : existing.lease_id,
+            owner,
+            acquired_at: expired ? nowIso(now) : existing.acquired_at,
+            heartbeat_at: nowIso(now),
+            expires_at: nowIso(expiry),
+        };
+        writeLease(lease);
+        return { acquired: true, lease, recovery_mode: expired };
+    }
+    return { acquired: false, lease: existing, recovery_mode: false };
+}
+function hasActiveLeaseForOwner(lease, owner, now) {
+    if (!lease)
+        return false;
+    if (lease.owner !== owner)
+        return false;
+    const expires = parseDate(lease.expires_at);
+    if (!expires)
+        return false;
+    return expires.getTime() > now.getTime();
+}
+function ensureLeaseForOwner(owner, now, ttlSeconds = defaultLeaseTtlSeconds()) {
+    const leaseResult = acquireLease(owner, ttlSeconds, now);
+    if (!leaseResult.acquired) {
+        return leaseResult;
+    }
+    const refreshedLease = {
+        ...leaseResult.lease,
+        owner,
+        heartbeat_at: nowIso(now),
+        expires_at: nowIso(new Date(now.getTime() + ttlSeconds * 1000)),
+    };
+    writeLease(refreshedLease);
+    return {
+        ...leaseResult,
+        lease: refreshedLease,
+    };
+}
+function releaseLocksForRecoveredJob(table, jobId, reasonCode, now) {
+    for (const lock of table.locks) {
+        if (lock.job_id !== jobId)
+            continue;
+        if (lock.status !== "active")
+            continue;
+        lock.status = "released";
+        lock.reason_code = reasonCode;
+        lock.released_at = nowIso(now);
+        lock.heartbeat_at = nowIso(now);
+    }
+}
+function heartbeatRunningLocks(table, leaseId, now) {
+    for (const lock of table.locks) {
+        if (lock.status !== "active")
+            continue;
+        lock.lease_id = leaseId;
+        lock.heartbeat_at = nowIso(now);
+    }
+}
+function startJobNow(job, table, lease, now, buf, sourceTool) {
+    const acquiredAt = nowIso(now);
+    const resources = normalizeResources(job.resource_requirements);
+    for (const lock of table.locks) {
+        if (lock.job_id !== job.job_id)
+            continue;
+        if (lock.status === "active") {
+            lock.heartbeat_at = acquiredAt;
+            lock.lease_id = lease.lease_id;
+        }
+    }
+    const existingResources = new Set(table.locks
+        .filter((row) => row.job_id === job.job_id && row.status === "active")
+        .map((row) => row.resource));
+    for (const resource of resources) {
+        if (existingResources.has(resource))
+            continue;
+        table.locks.push({
+            lock_id: `LOCK-${randomUUID().slice(0, 12)}`,
+            job_id: job.job_id,
+            phase: "dispatch",
+            resource,
+            status: "active",
+            priority: job.priority,
+            lease_id: lease.lease_id,
+            acquired_at: acquiredAt,
+            heartbeat_at: acquiredAt,
+            created_at: acquiredAt,
+        });
+    }
+    job.status = "running";
+    job.started_at = acquiredAt;
+    job.error = undefined;
+    job.reason_code = undefined;
+    buf.status.push({
+        event_type: "SCHEDULER_JOB_STARTED",
+        status: "in_progress",
+        summary: `Started job ${job.job_id}`,
+        payload: {
+            job_id: job.job_id,
+            acquired_at: acquiredAt,
+            resources,
+            source_tool: sourceTool,
+        },
+    });
+    buf.ledger.push({
+        tool: sourceTool,
+        category: "major_update",
+        message: `Started scheduler job ${job.job_id}`,
+        artifacts: ["agent-state/job-queue.json", "agent-state/job-locks.json"],
+        metadata: {
+            job_id: job.job_id,
+            acquired_at: acquiredAt,
+            resources,
+        },
+    });
+    return { acquired_at: acquiredAt, resources };
+}
+function reconcileRecoveryState(queue, table, now, recoveryMode, buf) {
+    let recovered = 0;
+    for (const job of queue.jobs) {
+        if (job.status === "unknown_recovery") {
+            releaseLocksForRecoveredJob(table, job.job_id, "recovery_hold", now);
+            continue;
+        }
+        if (job.status !== "running")
+            continue;
+        const runningLocks = table.locks.filter((lock) => lock.job_id === job.job_id && lock.status === "active");
+        const missingLockState = runningLocks.length === 0;
+        if (!missingLockState && !recoveryMode) {
+            continue;
+        }
+        const previousStatus = job.status;
+        const reasonCode = missingLockState ? "recovery_missing_lock" : "lease_recovered";
+        job.status = "unknown_recovery";
+        job.reason_code = reasonCode;
+        job.error = missingLockState
+            ? "Running job lost its resource lock state and requires manual recovery."
+            : `Running job lease was recovered by another owner at ${nowIso(now)}`;
+        job.next_attempt_at = undefined;
+        releaseLocksForRecoveredJob(table, job.job_id, "crash_recovery", now);
+        recovered += 1;
+        buf.status.push({
+            event_type: "SCHEDULER_RECOVERY",
+            status: "blocked",
+            summary: `Job ${job.job_id} moved to unknown_recovery (${reasonCode})`,
+            payload: {
+                job_id: job.job_id,
+                from_status: previousStatus,
+                reason_code: reasonCode,
+                error: job.error,
+                recovery_mode: recoveryMode,
+            },
+        });
+    }
+    return recovered;
+}
+export async function enqueueJob(input) {
+    const buf = newEventBuffer();
+    const result = await withFileLock(SCHEDULER_LOCK_REL, () => {
+        if (!input.kind || input.kind.trim().length === 0) {
+            const queue = readJobQueue();
+            return {
+                ok: false,
+                path: getJobQueuePath(),
+                queue,
+                nack: {
+                    reason_code: "invalid_kind",
+                    reason: "`kind` is required.",
+                },
+            };
+        }
+        const queue = readJobQueue();
+        const now = new Date();
+        const created = nowIso(now);
+        const jobId = input.job_id && input.job_id.trim().length > 0
+            ? input.job_id.trim()
+            : `JOB-${created.replace(/[-:.TZ]/g, "").slice(0, 14)}-${randomUUID().slice(0, 8)}`;
+        if (queue.jobs.some((job) => job.job_id === jobId)) {
+            return {
+                ok: false,
+                path: getJobQueuePath(),
+                queue,
+                nack: {
+                    reason_code: "duplicate_job_id",
+                    reason: `job_id already exists: ${jobId}`,
+                },
+            };
+        }
+        const dependencies = [
+            ...(Array.isArray(input.dependencies) ? input.dependencies : []),
+        ];
+        if (input.run_after && input.run_after.trim().length > 0) {
+            dependencies.push({
+                kind: "time",
+                ref: input.run_after,
+            });
+        }
+        const job = {
+            job_id: jobId,
+            kind: input.kind.trim(),
+            payload: input.payload ?? {},
+            status: "accepted",
+            priority: toPriority(input.priority),
+            dependencies,
+            resource_requirements: normalizeResources(input.resource_requirements),
+            attempt: 0,
+            max_attempts: typeof input.max_attempts === "number" && Number.isFinite(input.max_attempts)
+                ? Math.max(1, Math.floor(input.max_attempts))
+                : 3,
+            backoff_ms: typeof input.backoff_ms === "number" && Number.isFinite(input.backoff_ms)
+                ? Math.max(1000, Math.floor(input.backoff_ms))
+                : 5000,
+            created_at: created,
+            accepted_at: created,
+        };
+        queue.jobs.push(job);
+        const path = writeQueue(queue);
+        buf.status.push({
+            event_type: "SCHEDULER_JOB_ENQUEUED",
+            status: "started",
+            summary: `Enqueued ${job.job_id} (${job.kind}, ${job.priority})`,
+            payload: {
+                job_id: job.job_id,
+                kind: job.kind,
+                priority: job.priority,
+                dependency_count: job.dependencies.length,
+                resource_requirements: job.resource_requirements,
+            },
+        });
+        buf.ledger.push({
+            tool: "enqueue_job",
+            category: "major_update",
+            message: `Enqueued scheduler job ${job.job_id} (${job.kind})`,
+            artifacts: ["agent-state/job-queue.json"],
+            metadata: {
+                job_id: job.job_id,
+                priority: job.priority,
+                dependencies: job.dependencies.length,
+            },
+        });
+        return {
+            ok: true,
+            path,
+            queue,
+            ack: {
+                job_id: job.job_id,
+                status: job.status,
+            },
+        };
+    });
+    await flushTransitionEvents(buf);
+    return result;
+}
+export async function acknowledgeJob(input) {
+    const buf = newEventBuffer();
+    const result = await withFileLock(SCHEDULER_LOCK_REL, () => {
+        const queue = readJobQueue();
+        const job = queue.jobs.find((row) => row.job_id === input.job_id);
+        if (!job) {
+            return {
+                ok: false,
+                path: getJobQueuePath(),
+                queue,
+                error: `Unknown job_id: ${input.job_id}`,
+            };
+        }
+        const from = job.status;
+        if (input.action === "accept") {
+            if (job.status === "pending" || job.status === "blocked") {
+                job.status = "accepted";
+                job.reason_code = undefined;
+                job.error = undefined;
+                job.accepted_at = nowIso();
+            }
+        }
+        else if (input.action === "reject" || input.action === "cancel") {
+            job.status = "canceled";
+            job.reason_code = input.action;
+            job.error = input.reason;
+            job.completed_at = nowIso();
+        }
+        else if (input.action === "resume") {
+            if (BLOCKED_REASONS.has(job.status) ||
+                job.status === "canceled" ||
+                job.status === "unknown_recovery") {
+                const wasUnknownRecovery = job.status === "unknown_recovery";
+                job.status = "accepted";
+                job.reason_code = undefined;
+                job.error = undefined;
+                job.next_attempt_at = undefined;
+                if (wasUnknownRecovery) {
+                    job.started_at = undefined;
+                }
+            }
+        }
+        const path = writeQueue(queue);
+        buf.status.push({
+            event_type: "SCHEDULER_JOB_ACK",
+            status: job.status === "canceled" ? "done" : "pass",
+            summary: `Job ${input.job_id} ${input.action} (${from} -> ${job.status})`,
+            payload: {
+                job_id: input.job_id,
+                action: input.action,
+                from_status: from,
+                to_status: job.status,
+                reason: input.reason ?? "",
+            },
+        });
+        buf.ledger.push({
+            tool: "ack_job",
+            category: "info",
+            message: `Scheduler job ${input.job_id}: ${from} -> ${job.status} via ${input.action}`,
+            artifacts: ["agent-state/job-queue.json"],
+            metadata: {
+                job_id: input.job_id,
+                action: input.action,
+                from: from,
+                to: job.status,
+            },
+        });
+        return {
+            ok: true,
+            path,
+            queue,
+            from,
+            to: job.status,
+        };
+    });
+    await flushTransitionEvents(buf);
+    return result;
+}
+export async function dispatchJobs(input = {}) {
+    const buf = newEventBuffer();
+    const result = await withFileLock(SCHEDULER_LOCK_REL, () => {
+        const owner = deriveOwner(input.owner);
+        const now = parseDate(input.now_iso) ?? new Date();
+        const table = readJobLockTable();
+        const leaseTtl = typeof input.lease_ttl_seconds === "number" && input.lease_ttl_seconds > 0
+            ? Math.floor(input.lease_ttl_seconds)
+            : defaultLeaseTtlSeconds();
+        const leaseResult = acquireLease(owner, leaseTtl, now);
+        if (!leaseResult.acquired) {
+            const queue = readJobQueue();
+            return {
+                ok: false,
+                owner,
+                lease_acquired: false,
+                lease: leaseResult.lease,
+                queue_path: getJobQueuePath(),
+                lock_path: getJobLockTablePath(),
+                queue,
+                locks: table,
+                summary: {
+                    total_jobs: queue.jobs.length,
+                    blocked_jobs: queue.jobs.filter((job) => job.status === "blocked").length,
+                    ready_jobs: queue.jobs.filter((job) => job.status === "ready").length,
+                    running_jobs: queue.jobs.filter((job) => job.status === "running").length,
+                    started_jobs: 0,
+                    recovered_jobs: 0,
+                },
+            };
+        }
+        const queue = readJobQueue();
+        const dependencyContext = {
+            todoState: readTodoState(),
+            handoffState: readHandoffRegistry(),
+            statusEvents: readStatusEvents(500),
+        };
+        const recoveredJobs = reconcileRecoveryState(queue, table, now, leaseResult.recovery_mode, buf);
+        heartbeatRunningLocks(table, leaseResult.lease.lease_id, now);
+        for (const job of queue.jobs) {
+            if (TERMINAL_STATUSES.has(job.status))
+                continue;
+            if (job.status === "running" || job.status === "unknown_recovery")
+                continue;
+            if (job.status === "pending") {
+                job.status = "accepted";
+                job.accepted_at = job.accepted_at ?? nowIso(now);
+            }
+            const blocks = evaluateDependencyBlocks(job, now, dependencyContext);
+            if (blocks.length > 0) {
+                job.status = "blocked";
+                job.reason_code = blocks[0]?.reason_code;
+                job.error = blocks[0]?.detail;
+            }
+            else {
+                job.status = "ready";
+                job.reason_code = undefined;
+                job.error = undefined;
+            }
+        }
+        const occupancy = buildOccupancy(table.locks);
+        const readyJobs = queue.jobs
+            .filter((job) => job.status === "ready")
+            .sort(compareJobs);
+        let startedJobs = 0;
+        for (const job of readyJobs) {
+            const resources = normalizeResources(job.resource_requirements);
+            const free = resources.every((resource) => !occupancy.has(resource));
+            if (!free) {
+                continue;
+            }
+            for (const resource of resources) {
+                occupancy.add(resource);
+            }
+            startJobNow(job, table, leaseResult.lease, now, buf, "dispatch_jobs");
+            startedJobs += 1;
+        }
+        const refreshedLease = {
+            ...leaseResult.lease,
+            owner,
+            heartbeat_at: nowIso(now),
+            expires_at: nowIso(new Date(now.getTime() + leaseTtl * 1000)),
+        };
+        writeLease(refreshedLease);
+        const finalQueuePath = writeQueue(queue);
+        const finalLockPath = writeJobLocks(table);
+        const summary = {
+            total_jobs: queue.jobs.length,
+            blocked_jobs: queue.jobs.filter((job) => job.status === "blocked").length,
+            ready_jobs: queue.jobs.filter((job) => job.status === "ready").length,
+            running_jobs: queue.jobs.filter((job) => job.status === "running").length,
+            started_jobs: startedJobs,
+            recovered_jobs: recoveredJobs,
+        };
+        buf.status.push({
+            event_type: "SCHEDULER_TICK",
+            status: "pass",
+            summary: `Dispatch started=${summary.started_jobs} ready=${summary.ready_jobs} running=${summary.running_jobs} recovered=${recoveredJobs}`,
+            payload: {
+                owner,
+                lease_acquired: true,
+                lease_id: refreshedLease.lease_id,
+                recovery_mode: leaseResult.recovery_mode,
+                ...summary,
+            },
+        });
+        buf.ledger.push({
+            tool: "dispatch_jobs",
+            category: summary.started_jobs > 0 || summary.recovered_jobs > 0 ? "major_update" : "info",
+            message: `Scheduler dispatch complete (started=${summary.started_jobs}, running=${summary.running_jobs})`,
+            artifacts: ["agent-state/job-queue.json", "agent-state/job-locks.json"],
+            metadata: {
+                owner,
+                lease_id: refreshedLease.lease_id,
+                lease_acquired: true,
+                recovery_mode: leaseResult.recovery_mode,
+                summary,
+            },
+        });
+        return {
+            ok: true,
+            owner,
+            lease_acquired: true,
+            lease: refreshedLease,
+            queue_path: finalQueuePath,
+            lock_path: finalLockPath,
+            queue,
+            locks: table,
+            summary,
+        };
+    });
+    await flushTransitionEvents(buf);
+    return result;
+}
+export async function dispatchJobNow(input) {
+    const buf = newEventBuffer();
+    const result = await withFileLock(SCHEDULER_LOCK_REL, () => {
+        const owner = deriveOwner(input.owner);
+        const now = parseDate(input.now_iso) ?? new Date();
+        const leaseResult = ensureLeaseForOwner(owner, now);
+        if (!leaseResult.acquired) {
+            return {
+                ok: false,
+                queue_path: getJobQueuePath(),
+                lock_path: getJobLockTablePath(),
+                queue: readJobQueue(),
+                locks: readJobLockTable(),
+                error: `Owner ${owner} does not hold an active scheduler lease.`,
+            };
+        }
+        const queue = readJobQueue();
+        const table = readJobLockTable();
+        const job = queue.jobs.find((row) => row.job_id === input.job_id);
+        if (!job) {
+            return {
+                ok: false,
+                queue_path: getJobQueuePath(),
+                lock_path: getJobLockTablePath(),
+                queue,
+                locks: table,
+                error: `Unknown job_id: ${input.job_id}`,
+            };
+        }
+        if (job.status === "unknown_recovery") {
+            return {
+                ok: false,
+                queue_path: getJobQueuePath(),
+                lock_path: getJobLockTablePath(),
+                queue,
+                locks: table,
+                error: `Job ${input.job_id} is parked in unknown_recovery; resume it before dispatch.`,
+            };
+        }
+        if (TERMINAL_STATUSES.has(job.status)) {
+            return {
+                ok: false,
+                queue_path: getJobQueuePath(),
+                lock_path: getJobLockTablePath(),
+                queue,
+                locks: table,
+                error: `Job ${input.job_id} is ${job.status} and cannot be started.`,
+            };
+        }
+        if (job.status === "running") {
+            const runningResources = table.locks
+                .filter((row) => row.job_id === input.job_id && row.status === "active")
+                .map((row) => row.resource);
+            const acquiredAt = table.locks.find((row) => row.job_id === input.job_id && row.status === "active")
+                ?.acquired_at ??
+                job.started_at ??
+                nowIso(now);
+            heartbeatRunningLocks(table, leaseResult.lease.lease_id, now);
+            const queuePath = writeQueue(queue);
+            const lockPath = writeJobLocks(table);
+            return {
+                ok: true,
+                queue_path: queuePath,
+                lock_path: lockPath,
+                queue,
+                locks: table,
+                started_lock: {
+                    acquired_at: acquiredAt,
+                    resources: runningResources,
+                },
+            };
+        }
+        const dependencyContext = {
+            todoState: readTodoState(),
+            handoffState: readHandoffRegistry(),
+            statusEvents: readStatusEvents(500),
+        };
+        const blocks = evaluateDependencyBlocks(job, now, dependencyContext);
+        if (blocks.length > 0) {
+            return {
+                ok: false,
+                queue_path: getJobQueuePath(),
+                lock_path: getJobLockTablePath(),
+                queue,
+                locks: table,
+                error: blocks[0]?.detail ?? `Job ${input.job_id} is blocked.`,
+            };
+        }
+        const occupancy = buildOccupancy(table.locks);
+        const resources = normalizeResources(job.resource_requirements);
+        const busy = resources.find((resource) => table.locks.some((row) => row.job_id !== input.job_id && row.resource === resource && row.status === "active"));
+        if (busy) {
+            return {
+                ok: false,
+                queue_path: getJobQueuePath(),
+                lock_path: getJobLockTablePath(),
+                queue,
+                locks: table,
+                error: `Resource ${busy} is already held by another running job.`,
+            };
+        }
+        for (const resource of resources) {
+            occupancy.add(resource);
+        }
+        const startedLock = startJobNow(job, table, leaseResult.lease, now, buf, "dispatch_job_now");
+        const queuePath = writeQueue(queue);
+        const lockPath = writeJobLocks(table);
+        return {
+            ok: true,
+            queue_path: queuePath,
+            lock_path: lockPath,
+            queue,
+            locks: table,
+            started_lock: startedLock,
+        };
+    });
+    await flushTransitionEvents(buf);
+    return result;
+}
+export async function completeJob(input) {
+    const buf = newEventBuffer();
+    const result = await withFileLock(SCHEDULER_LOCK_REL, () => {
+        const owner = deriveOwner(input.owner);
+        const now = parseDate(input.now_iso) ?? new Date();
+        const leaseResult = ensureLeaseForOwner(owner, now);
+        if (!leaseResult.acquired) {
+            return {
+                ok: false,
+                queue_path: getJobQueuePath(),
+                lock_path: getJobLockTablePath(),
+                queue: readJobQueue(),
+                locks: readJobLockTable(),
+                error: `Owner ${owner} does not hold an active scheduler lease.`,
+            };
+        }
+        const queue = readJobQueue();
+        const table = readJobLockTable();
+        const job = queue.jobs.find((row) => row.job_id === input.job_id);
+        if (!job) {
+            return {
+                ok: false,
+                queue_path: getJobQueuePath(),
+                lock_path: getJobLockTablePath(),
+                queue,
+                locks: table,
+                error: `Unknown job_id: ${input.job_id}`,
+            };
+        }
+        if (job.status !== "running") {
+            return {
+                ok: false,
+                queue_path: getJobQueuePath(),
+                lock_path: getJobLockTablePath(),
+                queue,
+                locks: table,
+                error: `Job ${input.job_id} is ${job.status}; only running jobs can be completed.`,
+            };
+        }
+        const runningLocks = table.locks.filter((lock) => lock.job_id === input.job_id && lock.status === "active");
+        if (runningLocks.length === 0) {
+            return {
+                ok: false,
+                queue_path: getJobQueuePath(),
+                lock_path: getJobLockTablePath(),
+                queue,
+                locks: table,
+                error: `Job ${input.job_id} has no active resource locks.`,
+            };
+        }
+        const success = input.success !== false;
+        for (const lock of table.locks) {
+            if (lock.job_id !== input.job_id)
+                continue;
+            if (lock.status === "active") {
+                lock.status = success ? "completed" : "failed";
+                lock.reason_code = success ? undefined : "execution_failed";
+                lock.released_at = nowIso(now);
+                lock.heartbeat_at = nowIso(now);
+            }
+        }
+        if (success) {
+            job.status = "done";
+            job.completed_at = nowIso(now);
+            job.evidence_ref = input.evidence_ref ?? job.evidence_ref;
+            job.error = undefined;
+            job.reason_code = undefined;
+            job.next_attempt_at = undefined;
+        }
+        else {
+            job.attempt += 1;
+            job.error = input.error ?? "Job execution failed";
+            job.evidence_ref = input.evidence_ref ?? job.evidence_ref;
+            if (job.attempt >= job.max_attempts) {
+                job.status = "failed_terminal";
+                job.completed_at = nowIso(now);
+                job.reason_code = "retry_exhausted";
+                job.next_attempt_at = undefined;
+            }
+            else {
+                const retryAt = new Date(now.getTime() + computeBackoffMs(job));
+                job.status = "blocked";
+                job.reason_code = "retry_scheduled";
+                job.next_attempt_at = nowIso(retryAt);
+                job.started_at = undefined;
+            }
+        }
+        const queuePath = writeQueue(queue);
+        const lockPath = writeJobLocks(table);
+        const finalStatus = job.status;
+        const isSuccess = finalStatus === "done";
+        buf.status.push({
+            event_type: "SCHEDULER_JOB_COMPLETED",
+            status: isSuccess ? "done" : finalStatus === "failed_terminal" ? "fail" : "blocked",
+            summary: isSuccess
+                ? `Completed job ${input.job_id}`
+                : `Job ${input.job_id} ended as ${finalStatus}`,
+            payload: {
+                job_id: input.job_id,
+                final_status: finalStatus,
+                retry_scheduled_for: job.next_attempt_at ?? null,
+                evidence_ref: input.evidence_ref ?? "",
+                error: input.error ?? "",
+            },
+        });
+        buf.ledger.push({
+            tool: "complete_job",
+            category: isSuccess ? "major_update" : "regression",
+            message: isSuccess
+                ? `Completed scheduler job ${input.job_id}`
+                : `Scheduler job ${input.job_id} ended with ${finalStatus}`,
+            artifacts: ["agent-state/job-queue.json", "agent-state/job-locks.json"],
+            metadata: {
+                job_id: input.job_id,
+                final_status: finalStatus,
+                retry_scheduled_for: job.next_attempt_at,
+            },
+        });
+        return {
+            ok: true,
+            queue_path: queuePath,
+            lock_path: lockPath,
+            queue,
+            locks: table,
+            status: job.status,
+            retry_scheduled_for: job.next_attempt_at,
+        };
+    });
+    await flushTransitionEvents(buf);
+    return result;
+}
+//# sourceMappingURL=job-scheduler.js.map