@voybio/ace-swarm 2.4.8 → 2.4.10

package/dist/job-scheduler.js

@@ -1,6 +1,6 @@
 import { randomUUID } from "node:crypto";
 import { readHandoffRegistry } from "./handoff-registry.js";
-import { resolveWorkspaceRoot, safeRead, safeWrite, withFileLock, wsPath } from "./helpers.js";
+import { resolveWorkspaceRoot, withWorkspaceRoot, safeRead, safeWrite, withFileLock, wsPath } from "./helpers.js";
 import { isReadError } from "./shared.js";
 import { appendStatusEventSafe, readStatusEvents } from "./status-events.js";
 import { appendRunLedgerEntrySafe } from "./run-ledger.js";
@@ -10,6 +10,7 @@ import { ProjectionManager } from "./store/materializers/projection-manager.js";
 import { SchedulerRepository } from "./store/repositories/scheduler-repository.js";
 import { getWorkspaceStorePath, readStoreJsonSync, storeExistsSync, } from "./store/store-snapshot.js";
 import { withStoreWriteCoordinator } from "./store/write-coordinator.js";
+import { probeProcess, readProcessIdentity } from "./process-identity.js";
 // ── Buffered transition-event infrastructure ────────────────────────
 // Events are collected during the scheduler lock and flushed after
 // the lock is released.  Emission is failure-tolerant: a broken
@@ -204,6 +205,8 @@ function parseQueueFile(raw) {
                 evidence_ref: typeof job.evidence_ref === "string" ? job.evidence_ref : undefined,
                 error: typeof job.error === "string" ? job.error : undefined,
                 reason_code: typeof job.reason_code === "string" ? job.reason_code : undefined,
+                execution: job.execution && typeof job.execution === "object" ? { ...job.execution } : undefined,
+                execution_children: Array.isArray(job.execution_children) ? job.execution_children.map(identity => ({ ...identity })) : undefined,
             });
         }
         return {
@@ -289,16 +292,28 @@ function parseLease(raw) {
         return undefined;
     }
 }
-export function getJobQueuePath() {
+export function getJobQueuePath(workspaceRoot = resolveWorkspaceRoot()) {
+    return withWorkspaceRoot(workspaceRoot, () => getJobQueuePathInWorkspace());
+}
+function getJobQueuePathInWorkspace() {
     return wsPath(JOB_QUEUE_REL);
 }
-export function getJobLockTablePath() {
+export function getJobLockTablePath(workspaceRoot = resolveWorkspaceRoot()) {
+    return withWorkspaceRoot(workspaceRoot, () => getJobLockTablePathInWorkspace());
+}
+function getJobLockTablePathInWorkspace() {
     return wsPath(JOB_LOCK_TABLE_REL);
 }
-export function getSchedulerLeasePath() {
+export function getSchedulerLeasePath(workspaceRoot = resolveWorkspaceRoot()) {
+    return withWorkspaceRoot(workspaceRoot, () => getSchedulerLeasePathInWorkspace());
+}
+function getSchedulerLeasePathInWorkspace() {
     return wsPath(SCHEDULER_LEASE_REL);
 }
-export function readJobQueue() {
+export function readJobQueue(workspaceRoot = resolveWorkspaceRoot()) {
+    return withWorkspaceRoot(workspaceRoot, () => readJobQueueInWorkspace());
+}
+function readJobQueueInWorkspace() {
     const root = resolveWorkspaceRoot();
     if (storeExistsSync(root)) {
         const stored = readStoreJsonSync(root, "state/scheduler/queue");
@@ -311,7 +326,10 @@ export function readJobQueue() {
         return defaultQueueFile();
     return parseQueueFile(raw) ?? defaultQueueFile();
 }
-export function readJobLockTable() {
+export function readJobLockTable(workspaceRoot = resolveWorkspaceRoot()) {
+    return withWorkspaceRoot(workspaceRoot, () => readJobLockTableInWorkspace());
+}
+function readJobLockTableInWorkspace() {
     const root = resolveWorkspaceRoot();
     if (storeExistsSync(root)) {
         const stored = readStoreJsonSync(root, "state/scheduler/locks");
@@ -324,11 +342,14 @@ export function readJobLockTable() {
         return defaultJobLockFile();
     return parseJobLockFile(raw) ?? defaultJobLockFile();
 }
-export function readSchedulerLease() {
+export function readSchedulerLease(workspaceRoot = resolveWorkspaceRoot()) {
+    return withWorkspaceRoot(workspaceRoot, () => readSchedulerLeaseInWorkspace());
+}
+function readSchedulerLeaseInWorkspace() {
     const root = resolveWorkspaceRoot();
     if (storeExistsSync(root)) {
         const stored = readStoreJsonSync(root, "state/scheduler/lease");
-        if (stored)
+        if (stored !== undefined)
             return stored ?? undefined;
     }
     const raw = safeRead(SCHEDULER_LEASE_REL);
@@ -336,28 +357,54 @@ export function readSchedulerLease() {
         return undefined;
     return parseLease(raw);
 }
+/** Serialize admission as well as persistence across processes. */
+async function withSchedulerTransaction(fn) {
+    const storePath = getWorkspaceStorePath(resolveWorkspaceRoot());
+    return withFileLock(SCHEDULER_LOCK_REL, () => withStoreWriteCoordinator(storePath, fn, { operation_label: "scheduler_transaction" }));
+}
+function probeJobExecution(job) {
+    const identities = [...(job.execution ? [job.execution] : []), ...(job.execution_children ?? [])];
+    const states = identities.map(probeProcess);
+    if (states.includes("alive"))
+        return "alive";
+    return states.includes("unknown") ? "unknown" : "dead";
+}
+function hasExecutionOwnership(queue, locks) {
+    return queue.jobs.some((job) => job.status === "running" ||
+        (job.status === "unknown_recovery" && (job.reason_code === "handoff_execution_unverified" ||
+            probeJobExecution(job) !== "dead"))) ||
+        locks.locks.some((lock) => isJobLockActive(lock));
+}
+function leaseForAcquisition() {
+    // Called inside the transaction: legacy idle leases can be reclaimed immediately.
+    return hasExecutionOwnership(readJobQueue(), readJobLockTable()) ? readSchedulerLease() : undefined;
+}
+// Only called from withSchedulerTransaction. Do not reacquire the store coordinator.
 async function persistSchedulerState(input) {
     const root = resolveWorkspaceRoot();
+    const lease = hasExecutionOwnership(input.queue, input.locks) ? input.lease : undefined;
     if (!storeExistsSync(root)) {
+        // Legacy workspaces share the same cross-process transaction lock. Do not
+        // create a partial store that would hide their handoffs, events, and todos.
         safeWrite(JOB_QUEUE_REL, JSON.stringify({ ...input.queue, updated_at: nowIso() }, null, 2));
         safeWrite(JOB_LOCK_TABLE_REL, JSON.stringify({ ...input.locks, updated_at: nowIso() }, null, 2));
-        safeWrite(SCHEDULER_LEASE_REL, JSON.stringify(input.lease ?? null, null, 2));
+        safeWrite(SCHEDULER_LEASE_REL, JSON.stringify(lease ?? null, null, 2));
         return;
     }
-    const storePath = getWorkspaceStorePath(root);
-    await withStoreWriteCoordinator(storePath, async () => {
-        const store = await openStore(storePath);
-        try {
-            const repo = new SchedulerRepository(store);
-            await repo.replaceState(input);
-            await store.commit();
-            const projections = new ProjectionManager(store, root);
-            await projections.projectAfterCommit(["scheduler"]);
-        }
-        finally {
-            await store.close();
-        }
-    }, { operation_label: "persistSchedulerState" });
+    const store = await openStore(getWorkspaceStorePath(root));
+    try {
+        const repo = new SchedulerRepository(store);
+        await repo.replaceState({
+            ...input,
+            lease,
+        });
+        await store.commit();
+        const projections = new ProjectionManager(store, root);
+        await projections.projectAfterCommit(["scheduler"]);
+    }
+    finally {
+        await store.close();
+    }
 }
 function compareJobs(a, b) {
     const priorityCmp = PRIORITY_WEIGHT[a.priority] - PRIORITY_WEIGHT[b.priority];
@@ -583,6 +630,11 @@ function acquireLease(existing, owner, ttlSeconds, now) {
     }
     const existingExpiry = parseDate(existing.expires_at);
     const expired = !existingExpiry || existingExpiry.getTime() <= now.getTime();
+    // A missed heartbeat is not evidence that an executing process has exited.
+    // Do not steal its owner lease while its identity is live or uncertain.
+    if (expired && existing.owner !== owner && readJobQueue().jobs.some((job) => (job.status === "running" || job.status === "unknown_recovery") &&
+        probeJobExecution(job) !== "dead"))
+        return { acquired: false, lease: existing, recovery_mode: false };
     if (expired || existing.owner === owner) {
         const lease = {
             version: 1,
@@ -607,7 +659,7 @@ function hasActiveLeaseForOwner(lease, owner, now) {
     return expires.getTime() > now.getTime();
 }
 function ensureLeaseForOwner(owner, now, ttlSeconds = defaultLeaseTtlSeconds()) {
-    const leaseResult = acquireLease(readSchedulerLease(), owner, ttlSeconds, now);
+    const leaseResult = acquireLease(leaseForAcquisition(), owner, ttlSeconds, now);
     if (!leaseResult.acquired) {
         return leaseResult;
     }
@@ -700,9 +752,18 @@ function startJobNow(job, table, lease, now, buf, sourceTool) {
     });
     return { acquired_at: acquiredAt, resources };
 }
-function reconcileRecoveryState(queue, table, now, recoveryMode, buf) {
+function reconcileRecoveryState(queue, table, now, recoveryMode, buf, selected) {
     let recovered = 0;
     for (const job of queue.jobs) {
+        if (selected && !selected.has(job.job_id))
+            continue;
+        if (job.kind === "handoff.lifecycle")
+            continue;
+        if ((job.status === "running" || job.status === "unknown_recovery") && probeJobExecution(job) !== "dead") {
+            // Preserve ownership even during lease recovery or missing lock metadata.
+            // A retained uncertain execution must never be dispatched a second time.
+            continue;
+        }
         if (job.status === "unknown_recovery") {
             releaseLocksForRecoveredJob(table, job.job_id, "recovery_hold", now);
             continue;
@@ -749,11 +810,15 @@ function reconcileRecoveryState(queue, table, now, recoveryMode, buf) {
  * Missing-lock jobs are intentionally left to `reconcileRecoveryState`, which
  * runs first; this pass only inspects jobs that still hold active locks.
  */
-function reapStaleRunningJobs(queue, table, now, staleMs, buf) {
+function reapStaleRunningJobs(queue, table, now, staleMs, buf, selected) {
     if (staleMs <= 0)
         return [];
     const reaped = [];
     for (const job of queue.jobs) {
+        if (selected && !selected.has(job.job_id))
+            continue;
+        if (job.kind === "handoff.lifecycle")
+            continue;
         if (job.status !== "running")
             continue;
         const activeLocks = table.locks.filter((lock) => lock.job_id === job.job_id && lock.status === "active");
@@ -765,6 +830,14 @@ function reapStaleRunningJobs(queue, table, now, staleMs, buf) {
         const ageMs = now.getTime() - lastBeat;
         if (ageMs <= staleMs)
             continue;
+        if (job.execution || job.execution_children?.length) {
+            const processState = probeJobExecution(job);
+            if (processState !== "dead") {
+                job.reason_code = `heartbeat_stale_process_${processState}`;
+                job.error = `Worker heartbeat is stale, but execution process is ${processState}; execution and resource locks are retained pending recovery.`;
+                continue;
+            }
+        }
         const previousStatus = job.status;
         job.status = "unknown_recovery";
         job.reason_code = "heartbeat_stale";
@@ -799,9 +872,12 @@ function reapStaleRunningJobs(queue, table, now, staleMs, buf) {
     }
     return reaped;
 }
-export async function enqueueJob(input) {
+export async function enqueueJob(input, workspaceRoot = resolveWorkspaceRoot()) {
+    return withWorkspaceRoot(workspaceRoot, () => enqueueJobInWorkspace(input));
+}
+async function enqueueJobInWorkspace(input) {
     const buf = newEventBuffer();
-    const result = await withFileLock(SCHEDULER_LOCK_REL, async () => {
+    const result = await withSchedulerTransaction(async () => {
         if (!input.kind || input.kind.trim().length === 0) {
             const queue = readJobQueue();
             return {
@@ -900,9 +976,79 @@ export async function enqueueJob(input) {
     await flushTransitionEvents(buf);
     return result;
 }
-export async function acknowledgeJob(input) {
+/** Mirror handoff lifecycle without admitting or executing a job. */
+export async function syncHandoffLifecycleJob(input, workspaceRoot = resolveWorkspaceRoot()) {
+    return withWorkspaceRoot(workspaceRoot, () => syncHandoffLifecycleJobInWorkspace(input));
+}
+async function syncHandoffLifecycleJobInWorkspace(input) {
+    return withSchedulerTransaction(async () => {
+        const queue = readJobQueue();
+        const locks = readJobLockTable();
+        const now = parseDate(input.now_iso) ?? new Date();
+        let job = queue.jobs.find((row) => row.job_id === input.job_id);
+        if (job && (job.kind !== "handoff.lifecycle" || job.payload.handoff_id !== input.payload?.handoff_id)) {
+            return {
+                ok: false,
+                enqueued: false,
+                path: getJobQueuePath(),
+                queue,
+                nack: { reason_code: "duplicate_job_id", reason: "Job belongs to another lifecycle or execution." },
+            };
+        }
+        const enqueued = !job;
+        if (!job) {
+            job = {
+                job_id: input.job_id,
+                kind: "handoff.lifecycle",
+                payload: input.payload ?? {},
+                status: "blocked",
+                priority: toPriority(input.priority),
+                dependencies: input.dependencies ?? [],
+                resource_requirements: normalizeResources(input.resource_requirements),
+                attempt: 0,
+                max_attempts: 3,
+                backoff_ms: 5000,
+                created_at: nowIso(now),
+                accepted_at: nowIso(now),
+            };
+            queue.jobs.push(job);
+        }
+        job.payload = { ...job.payload, ...input.payload };
+        if (job.payload.status === "completed") {
+            // The canonical handoff completion is the receipt resolving legacy bookkeeping.
+            job.status = "done";
+            job.completed_at = job.completed_at ?? nowIso(now);
+            job.reason_code = undefined;
+            job.error = undefined;
+            releaseLocksForRecoveredJob(locks, job.job_id, "handoff_completed", now);
+        }
+        else if (job.status === "running" || job.status === "unknown_recovery" || hasActiveLockForJob(locks.locks, job.job_id)) {
+            // Do not infer worker death from a legacy synthetic job's kind.
+            job.status = "unknown_recovery";
+            job.reason_code = "handoff_execution_unverified";
+            job.error = "Legacy handoff execution requires a completion receipt; ownership and locks are preserved.";
+        }
+        else if (!TERMINAL_STATUSES.has(job.status)) {
+            job.status = "blocked";
+            job.reason_code = "handoff_metadata_only";
+            job.error = "Lifecycle metadata is synchronized by the handoff registry, not dispatched.";
+        }
+        await persistSchedulerState({ queue, locks, lease: readSchedulerLease() });
+        return {
+            ok: true,
+            enqueued,
+            path: getJobQueuePath(),
+            queue,
+            ack: { job_id: job.job_id, status: job.status },
+        };
+    });
+}
+export async function acknowledgeJob(input, workspaceRoot = resolveWorkspaceRoot()) {
+    return withWorkspaceRoot(workspaceRoot, () => acknowledgeJobInWorkspace(input));
+}
+async function acknowledgeJobInWorkspace(input) {
     const buf = newEventBuffer();
-    const result = await withFileLock(SCHEDULER_LOCK_REL, async () => {
+    const result = await withSchedulerTransaction(async () => {
         const queue = readJobQueue();
         const job = queue.jobs.find((row) => row.job_id === input.job_id);
         if (!job) {
@@ -982,22 +1128,29 @@ export async function acknowledgeJob(input) {
     await flushTransitionEvents(buf);
     return result;
 }
-export async function dispatchJobs(input = {}) {
+export async function dispatchJobs(input = {}, workspaceRoot = resolveWorkspaceRoot()) {
+    return withWorkspaceRoot(workspaceRoot, () => dispatchJobsInWorkspace(input));
+}
+async function dispatchJobsInWorkspace(input = {}) {
     const buf = newEventBuffer();
-    const result = await withFileLock(SCHEDULER_LOCK_REL, async () => {
+    const selected = input.job_ids === undefined ? undefined : new Set(input.job_ids);
+    const result = await withSchedulerTransaction(async () => {
         const owner = deriveOwner(input.owner);
         const now = parseDate(input.now_iso) ?? new Date();
         const table = readJobLockTable();
         const leaseTtl = typeof input.lease_ttl_seconds === "number" && input.lease_ttl_seconds > 0
             ? Math.floor(input.lease_ttl_seconds)
             : defaultLeaseTtlSeconds();
-        const leaseResult = acquireLease(readSchedulerLease(), owner, leaseTtl, now);
+        const leaseResult = acquireLease(leaseForAcquisition(), owner, leaseTtl, now);
         if (!leaseResult.acquired) {
             const queue = readJobQueue();
             return {
                 ok: false,
+                reason_code: "scheduler_lease_held",
+                error: `Scheduler lease is held by ${leaseResult.lease.owner} until ${leaseResult.lease.expires_at}; dispatch requested by ${owner}.`,
                 owner,
                 lease_acquired: false,
+                lease_retained: true,
                 lease: leaseResult.lease,
                 queue_path: getJobQueuePath(),
                 lock_path: getJobLockTablePath(),
@@ -1006,7 +1159,7 @@ export async function dispatchJobs(input = {}) {
                 summary: {
                     total_jobs: queue.jobs.length,
                     blocked_jobs: queue.jobs.filter((job) => job.status === "blocked").length,
-                    ready_jobs: queue.jobs.filter((job) => job.status === "ready").length,
+                    ready_jobs: queue.jobs.filter((job) => job.status === "ready" && job.kind !== "handoff.lifecycle").length,
                     running_jobs: queue.jobs.filter((job) => job.status === "running").length,
                     started_jobs: 0,
                     recovered_jobs: 0,
@@ -1020,15 +1173,19 @@ export async function dispatchJobs(input = {}) {
             handoffState: readHandoffRegistry(),
             statusEvents: readStatusEvents(500),
         };
-        const recoveredJobs = reconcileRecoveryState(queue, table, now, leaseResult.recovery_mode, buf);
+        const recoveredJobs = reconcileRecoveryState(queue, table, now, leaseResult.recovery_mode, buf, selected);
         // Heartbeat watchdog: reap running jobs whose worker heartbeat is stale.
         // NOTE: dispatch deliberately does NOT re-stamp running-job heartbeats here.
         // Only a live worker (heartbeatJob) or a fresh start proves liveness; if the
         // dispatcher renewed heartbeats it would mask worker death (root cause of D3).
         const staleMs = resolveStaleMs(input.heartbeat_stale_seconds);
-        const reapedJobIds = reapStaleRunningJobs(queue, table, now, staleMs, buf);
+        const reapedJobIds = reapStaleRunningJobs(queue, table, now, staleMs, buf, selected);
         const reapedJobs = reapedJobIds.length;
         for (const job of queue.jobs) {
+            if (selected && !selected.has(job.job_id))
+                continue;
+            if (job.kind === "handoff.lifecycle")
+                continue;
             if (TERMINAL_STATUSES.has(job.status))
                 continue;
             if (job.status === "running" || job.status === "unknown_recovery")
@@ -1051,7 +1208,7 @@ export async function dispatchJobs(input = {}) {
         }
         const occupancy = buildOccupancy(table.locks);
         const readyJobs = queue.jobs
-            .filter((job) => job.status === "ready")
+            .filter((job) => job.status === "ready" && job.kind !== "handoff.lifecycle" && (!selected || selected.has(job.job_id)))
             .sort(compareJobs);
         let startedJobs = 0;
         for (const job of readyJobs) {
@@ -1080,7 +1237,7 @@ export async function dispatchJobs(input = {}) {
         const summary = {
             total_jobs: queue.jobs.length,
             blocked_jobs: queue.jobs.filter((job) => job.status === "blocked").length,
-            ready_jobs: queue.jobs.filter((job) => job.status === "ready").length,
+            ready_jobs: queue.jobs.filter((job) => job.status === "ready" && job.kind !== "handoff.lifecycle").length,
             running_jobs: queue.jobs.filter((job) => job.status === "running").length,
             started_jobs: startedJobs,
             recovered_jobs: recoveredJobs,
@@ -1093,6 +1250,7 @@ export async function dispatchJobs(input = {}) {
             payload: {
                 owner,
                 lease_acquired: true,
+                lease_retained: hasExecutionOwnership(queue, table),
                 lease_id: refreshedLease.lease_id,
                 recovery_mode: leaseResult.recovery_mode,
                 ...summary,
@@ -1107,6 +1265,7 @@ export async function dispatchJobs(input = {}) {
                 owner,
                 lease_id: refreshedLease.lease_id,
                 lease_acquired: true,
+                lease_retained: hasExecutionOwnership(queue, table),
                 recovery_mode: leaseResult.recovery_mode,
                 summary,
             },
@@ -1115,6 +1274,7 @@ export async function dispatchJobs(input = {}) {
             ok: true,
             owner,
             lease_acquired: true,
+            lease_retained: hasExecutionOwnership(queue, table),
             lease: refreshedLease,
             queue_path: getJobQueuePath(),
             lock_path: getJobLockTablePath(),
@@ -1126,9 +1286,12 @@ export async function dispatchJobs(input = {}) {
     await flushTransitionEvents(buf);
     return result;
 }
-export async function dispatchJobNow(input) {
+export async function dispatchJobNow(input, workspaceRoot = resolveWorkspaceRoot()) {
+    return withWorkspaceRoot(workspaceRoot, () => dispatchJobNowInWorkspace(input));
+}
+async function dispatchJobNowInWorkspace(input) {
     const buf = newEventBuffer();
-    const result = await withFileLock(SCHEDULER_LOCK_REL, async () => {
+    const result = await withSchedulerTransaction(async () => {
         const owner = deriveOwner(input.owner);
         const now = parseDate(input.now_iso) ?? new Date();
         const leaseResult = ensureLeaseForOwner(owner, now);
@@ -1155,6 +1318,16 @@ export async function dispatchJobNow(input) {
                 error: `Unknown job_id: ${input.job_id}`,
             };
         }
+        if (job.kind === "handoff.lifecycle") {
+            return {
+                ok: false,
+                queue_path: getJobQueuePath(),
+                lock_path: getJobLockTablePath(),
+                queue,
+                locks: table,
+                error: "Handoff lifecycle jobs are metadata; synchronize the handoff instead of dispatching it.",
+            };
+        }
         if (job.status === "unknown_recovery") {
             return {
                 ok: false,
@@ -1251,9 +1424,72 @@ export async function dispatchJobNow(input) {
     await flushTransitionEvents(buf);
     return result;
 }
-export async function completeJob(input) {
+/** Atomically claim the side-effecting handler, independently of scheduler owner. */
+export async function claimJobExecution(input, workspaceRoot = resolveWorkspaceRoot()) {
+    return withWorkspaceRoot(workspaceRoot, () => withSchedulerTransaction(async () => {
+        const now = parseDate(input.now_iso) ?? new Date();
+        const owner = deriveOwner(input.owner);
+        const lease = ensureLeaseForOwner(owner, now);
+        if (!lease.acquired)
+            return { ok: false, reason_code: "scheduler_lease_held", error: "Execution owner does not hold the scheduler lease." };
+        const queue = readJobQueue();
+        const locks = readJobLockTable();
+        const job = queue.jobs.find(row => row.job_id === input.job_id);
+        if (!job || job.status !== "running")
+            return { ok: false, reason_code: "job_not_running", job };
+        if (job.execution || job.execution_children?.length) {
+            const state = probeJobExecution(job);
+            if (state === "dead") {
+                // The prior process may have performed side effects before it exited.
+                // Preserve uncertainty instead of replaying the handler automatically.
+                job.status = "unknown_recovery";
+                job.reason_code = "execution_process_exited";
+                job.error = "Claimed execution process exited; inspect retained outcomes before retrying.";
+                job.next_attempt_at = undefined;
+                releaseLocksForRecoveredJob(locks, job.job_id, "execution_process_exited", now);
+                await persistSchedulerState({ queue, locks, lease: lease.lease });
+            }
+            return { ok: false, reason_code: state === "alive" ? "execution_claimed" : state === "unknown" ? "execution_uncertain" : "execution_dead", job };
+        }
+        if (!hasActiveLockForJob(locks.locks, job.job_id))
+            return { ok: false, reason_code: "execution_missing_locks", job };
+        const identity = readProcessIdentity(process.pid);
+        if (!identity)
+            return { ok: false, reason_code: "identity_unavailable", error: "Cannot safely identify the worker process; handler was not invoked.", job };
+        const token = randomUUID();
+        job.execution = { ...identity, token };
+        job.execution_children = undefined;
+        await persistSchedulerState({ queue, locks, lease: lease.lease });
+        return { ok: true, token, job };
+    }));
+}
+/** Register a command before waiting on it; only its claimed attempt may do so. */
+export async function recordJobExecutionChild(input, workspaceRoot = resolveWorkspaceRoot()) {
+    return withWorkspaceRoot(workspaceRoot, () => withSchedulerTransaction(async () => {
+        const queue = readJobQueue();
+        const job = queue.jobs.find(row => row.job_id === input.job_id);
+        if (!job || job.status !== "running")
+            return { ok: false, reason_code: "job_not_running" };
+        if (!input.token || job.execution?.token !== input.token)
+            return { ok: false, reason_code: "execution_token_mismatch" };
+        const identity = input.identity;
+        if (!Number.isSafeInteger(identity.pid) || identity.pid <= 1 || !identity.started_at || !identity.hostname) {
+            return { ok: false, reason_code: "execution_identity_invalid" };
+        }
+        job.execution_children ??= [];
+        if (!job.execution_children.some(child => child.pid === identity.pid && child.started_at === identity.started_at && child.hostname === identity.hostname)) {
+            job.execution_children.push({ ...identity });
+            await persistSchedulerState({ queue, locks: readJobLockTable(), lease: readSchedulerLease() });
+        }
+        return { ok: true };
+    }));
+}
+export async function completeJob(input, workspaceRoot = resolveWorkspaceRoot()) {
+    return withWorkspaceRoot(workspaceRoot, () => completeJobInWorkspace(input));
+}
+async function completeJobInWorkspace(input) {
     const buf = newEventBuffer();
-    const result = await withFileLock(SCHEDULER_LOCK_REL, async () => {
+    const result = await withSchedulerTransaction(async () => {
         const owner = deriveOwner(input.owner);
         const now = parseDate(input.now_iso) ?? new Date();
         const leaseResult = ensureLeaseForOwner(owner, now);
@@ -1302,6 +1538,9 @@ export async function completeJob(input) {
             };
         }
         const success = input.success !== false;
+        // The claim belongs to this attempt only. A deliberate retry gets a new one.
+        job.execution = undefined;
+        job.execution_children = undefined;
         for (const lock of table.locks) {
             if (lock.job_id !== input.job_id)
                 continue;
@@ -1393,8 +1632,11 @@ export async function completeJob(input) {
  * job's heartbeat — the dispatcher deliberately does not, so a missing
  * heartbeat reliably signals a dead worker.
  */
-export async function heartbeatJob(input) {
-    return withFileLock(SCHEDULER_LOCK_REL, async () => {
+export async function heartbeatJob(input, workspaceRoot = resolveWorkspaceRoot()) {
+    return withWorkspaceRoot(workspaceRoot, () => heartbeatJobInWorkspace(input));
+}
+async function heartbeatJobInWorkspace(input) {
+    return withSchedulerTransaction(async () => {
         const owner = deriveOwner(input.owner);
         const now = parseDate(input.now_iso) ?? new Date();
         const leaseTtl = typeof input.lease_ttl_seconds === "number" && input.lease_ttl_seconds > 0
@@ -1438,6 +1680,10 @@ export async function heartbeatJob(input) {
             };
         }
         const beatAt = nowIso(now);
+        if (job.reason_code?.startsWith("heartbeat_stale_process_")) {
+            job.reason_code = undefined;
+            job.error = undefined;
+        }
         for (const lock of activeLocks) {
             lock.heartbeat_at = beatAt;
             lock.lease_id = leaseResult.lease.lease_id;
@@ -1462,21 +1708,25 @@ export async function heartbeatJob(input) {
  * watchdog cadence separate from dispatch. Does not require lease ownership —
  * reaping a dead worker's job must not depend on holding the lease.
  */
-export async function reapStaleRunningJobsTick(input = {}) {
+export async function reapStaleRunningJobsTick(input = {}, workspaceRoot = resolveWorkspaceRoot()) {
+    return withWorkspaceRoot(workspaceRoot, () => reapStaleRunningJobsTickInWorkspace(input));
+}
+async function reapStaleRunningJobsTickInWorkspace(input = {}) {
     const buf = newEventBuffer();
-    const result = await withFileLock(SCHEDULER_LOCK_REL, async () => {
+    const result = await withSchedulerTransaction(async () => {
         const now = parseDate(input.now_iso) ?? new Date();
         const staleMs = resolveStaleMs(input.heartbeat_stale_seconds);
         const queue = readJobQueue();
         const table = readJobLockTable();
+        const priorJobs = JSON.stringify(queue.jobs);
         const reapedJobIds = reapStaleRunningJobs(queue, table, now, staleMs, buf);
-        if (reapedJobIds.length > 0) {
+        // Retained live/unknown claims also carry a persisted diagnostic.
+        if (priorJobs !== JSON.stringify(queue.jobs))
             await persistSchedulerState({
                 queue,
                 locks: table,
                 lease: readSchedulerLease(),
             });
-        }
         return {
             ok: true,
             queue_path: getJobQueuePath(),

package/dist/job-worker.d.ts

@@ -35,6 +35,8 @@ export declare function listJobHandlerKinds(): string[];
 export declare function hasJobHandler(kind: string): boolean;
 export interface RunJobWorkerInput {
     owner?: string;
+    /** Execute only these jobs; unrelated jobs are not admitted by this tick. */
+    job_ids?: string[];
     /** Deterministic clock override; threaded through every scheduler call. */
     now_iso?: string;
     lease_ttl_seconds?: number;
@@ -60,7 +62,7 @@ export interface WorkerJobOutcome {
 export interface WorkerSkippedJob {
     job_id: string;
     kind: string;
-    reason: "no_handler" | "max_jobs";
+    reason: "no_handler" | "max_jobs" | "execution_claimed" | "execution_uncertain";
 }
 export interface RunJobWorkerResult {
     owner: string;

package/dist/job-worker.js

@@ -17,7 +17,7 @@
  * across a handler invocation, so a slow or hung handler cannot wedge the
  * scheduler for other owners.
  */
-import { completeJob, dispatchJobs, heartbeatJob, readJobQueue, readSchedulerLease, } from "./job-scheduler.js";
+import { completeJob, claimJobExecution, dispatchJobs, heartbeatJob, readJobQueue, readSchedulerLease, } from "./job-scheduler.js";
 import { resolveWorkspaceRoot } from "./helpers.js";
 const handlers = new Map();
 export function registerJobHandler(kind, handler) {
@@ -66,6 +66,7 @@ export async function runJobWorkerOnce(input = {}) {
         now_iso: input.now_iso,
         lease_ttl_seconds: input.lease_ttl_seconds,
         heartbeat_stale_seconds: input.heartbeat_stale_seconds,
+        job_ids: input.job_ids,
     });
     const executed = [];
     const skipped = [];
@@ -83,7 +84,8 @@ export async function runJobWorkerOnce(input = {}) {
         };
     }
     const queue = readJobQueue();
-    const runningJobs = queue.jobs.filter((job) => job.status === "running");
+    const selected = input.job_ids === undefined ? undefined : new Set(input.job_ids);
+    const runningJobs = queue.jobs.filter((job) => job.status === "running" && (!selected || selected.has(job.job_id)));
     const maxJobs = typeof input.max_jobs === "number" && input.max_jobs > 0
         ? Math.floor(input.max_jobs)
         : Number.POSITIVE_INFINITY;
@@ -98,6 +100,11 @@ export async function runJobWorkerOnce(input = {}) {
             skipped.push({ job_id: job.job_id, kind: job.kind, reason: "max_jobs" });
             continue;
         }
+        const claim = await claimJobExecution({ job_id: job.job_id, owner, now_iso: input.now_iso });
+        if (!claim.ok) {
+            skipped.push({ job_id: job.job_id, kind: job.kind, reason: claim.reason_code === "execution_claimed" ? "execution_claimed" : "execution_uncertain" });
+            continue;
+        }
         executedCount += 1;
         // Prove liveness immediately, then keep beating while the handler runs.
         await heartbeatJob({ job_id: job.job_id, owner, now_iso: input.now_iso });
@@ -113,7 +120,7 @@ export async function runJobWorkerOnce(input = {}) {
         }
         let handlerResult;
         try {
-            handlerResult = await handler({ job, owner });
+            handlerResult = await handler({ job: claim.job ?? job, owner });
         }
         catch (err) {
             handlerResult = { success: false, error: errorMessage(err) };