@deksden-com/dd-flow-cli 0.9.0-beta.123 → 0.9.0-beta.127

package/dist/services/managed-lease-monitor.js

@@ -0,0 +1,128 @@
+import { setTimeout as delay } from "node:timers/promises";
+import { AppError } from "../shared/errors.js";
+import { createRenewalState, maintenanceRetryable, renewalConfirmed, renewalDelay, renewalFailure, renewalRemaining, RENEWAL_POLICY } from "../harness-runtime/lib/lease-renewal.mjs";
+/** Registry renewals happen outside RUN writers. Assertions never renew. */
+export function managedLeaseMonitor(renew) {
+    const episode = createRenewalState();
+    let expiry = 0;
+    let failure = null;
+    let lost = false;
+    let fatal = null;
+    let closing = false;
+    let pending;
+    const shutdown = new AbortController();
+    const uncertain = () => new AppError("controller_lease_unconfirmed", "Managed lease requires confirmation outside the writer", 1, { cause: failure, exhausted: episode.exhausted, closing });
+    const assertNoRenewal = () => {
+        if (closing)
+            throw uncertain();
+        if (fatal)
+            throw fatal;
+        if (lost)
+            throw new AppError("controller_lease_lost", "Managed owner was replaced or retired", 1);
+        if (failure || Date.now() >= expiry)
+            throw uncertain();
+    };
+    const tick = () => {
+        if (closing)
+            throw uncertain();
+        if (fatal)
+            throw fatal;
+        if (lost || failure && !renewalRemaining(episode) || episode.exhausted)
+            return;
+        try {
+            episode.uncertainSince ??= performance.now();
+            const remaining = renewalRemaining(episode);
+            if (!remaining) {
+                failure = uncertain();
+                return;
+            }
+            const receipt = renew(Math.min(RENEWAL_POLICY.attemptMs, remaining));
+            if (!receipt.owned) {
+                lost = true;
+                return;
+            }
+            const value = Date.parse(receipt.lease_expires_at);
+            if (!Number.isFinite(value))
+                throw new AppError("managed_lease_receipt_invalid", "Renewal has no valid persisted expiry", 1);
+            if (value <= Date.now())
+                throw new AppError("controller_lease_unconfirmed", "Renewal receipt has already expired", 1);
+            if ((!failure || renewalRemaining(episode)) && renewalConfirmed(episode)) {
+                expiry = value;
+                failure = null;
+            }
+            else
+                failure = uncertain();
+        }
+        catch (error) {
+            failure = error;
+            if (maintenanceRetryable(error) || error instanceof AppError && error.code === "controller_lease_unconfirmed")
+                renewalFailure(episode);
+            else {
+                fatal = error;
+                throw error;
+            }
+        }
+    };
+    const assertConfirmed = () => { tick(); assertNoRenewal(); };
+    const runConfirmation = async (assertCurrent) => {
+        for (;;) {
+            if (closing)
+                throw uncertain();
+            assertCurrent();
+            tick();
+            try {
+                assertNoRenewal();
+                return;
+            }
+            catch (error) {
+                if (lost || episode.exhausted || !renewalRemaining(episode))
+                    throw error;
+                try {
+                    await delay(Math.min(renewalDelay(episode), renewalRemaining(episode)), undefined, { signal: shutdown.signal });
+                }
+                catch (cause) {
+                    if (closing)
+                        throw uncertain();
+                    throw cause;
+                }
+            }
+        }
+    };
+    const ensureConfirmed = async (assertCurrent = () => { }) => {
+        if (closing)
+            throw uncertain();
+        if (pending)
+            assertCurrent();
+        pending ??= runConfirmation(assertCurrent).finally(() => { pending = undefined; });
+        await pending;
+        if (closing)
+            throw uncertain();
+        assertCurrent();
+        assertNoRenewal();
+    };
+    const background = (assertCurrent, onFailure, onConfirmed = () => { }) => {
+        void ensureConfirmed(assertCurrent).then(() => {
+            if (closing)
+                return;
+            try {
+                onConfirmed();
+            }
+            catch (error) {
+                fatal = failure = error;
+                throw error;
+            }
+        }).catch(error => {
+            if (closing)
+                return;
+            try {
+                onFailure(error);
+            }
+            catch (reporting) {
+                fatal = failure = reporting;
+            }
+        });
+    };
+    return { tick, assertNoRenewal, assertConfirmed, ensureConfirmed, background,
+        close: () => { closing = true; shutdown.abort(); },
+        isConfirmed: () => !closing && !lost && !failure && Date.now() < expiry };
+}

package/dist/services/managed-processes.js

@@ -4,10 +4,17 @@ import path from "node:path";
 import { spawnSync } from "node:child_process";
 import net from "node:net";
 import { getResourceDatabase } from "../storage/database.js";
-import { AppError } from "../shared/errors.js";
+import { AppError, errorRecord } from "../shared/errors.js";
 export function managedProcessBinding(record, home) {
     return { resource_home: home, process_id: record.id, registration_sha256: crypto.createHash("sha256").update(JSON.stringify([home, record.id, record.kind, record.owner_id, record.lease_token, record.pid, record.pid_started_at, record.project_id, record.run_id, record.work_id, record.check_id, record.operation_id, record.metadata_json])).digest("hex") };
 }
+export function assertManagedProcessAdmission(context, record) {
+    const expiry = Date.parse(record.lease_expires_at);
+    if (!Number.isFinite(expiry))
+        throw new AppError("runtime_artifact_invalid", "Managed lease expiry is malformed", 1, { process_id: record.id });
+    if (expiry <= Date.parse(context.now()))
+        throw new AppError("process_ownership_unknown", "Managed owner needs fresh renewal before admission", 1, { process_id: record.id, reason: "renewal_required", lease_expires_at: record.lease_expires_at });
+}
 const defaultLeaseMs = 15 * 60_000;
 export function resourceHome(context) {
     return context.env?.DD_FLOW_RESOURCE_HOME ?? context.ddFlowHome ?? "/tmp/dd-flow-runtime";
@@ -53,8 +60,10 @@ export function prepareManagedProcessRegistration(context, input) {
     const observer = input.kind === "eval-observer";
     const observerBirth = observer && input.pid ? processStartedAt(input.pid) : null;
     if (observer) {
-        if (!Number.isSafeInteger(input.pid) || !input.pid || input.pid < 1 || !observerBirth || input.ownerPid !== undefined && input.ownerPid !== input.pid || input.processGroupId !== undefined && input.processGroupId !== input.pid)
+        if (!Number.isSafeInteger(input.pid) || !input.pid || input.pid < 1 || input.ownerPid !== undefined && input.ownerPid !== input.pid || input.processGroupId !== undefined && input.processGroupId !== input.pid)
             throw new AppError("process_ownership_unknown", "EVAL observer registration requires its live self-owned process identity", 1);
+        if (!observerBirth)
+            throw new AppError("process_ownership_unknown", "EVAL observer physical identity observation is unavailable", 1, { process_id: input.id, reason: "physical_identity_unavailable", phase: "observer_registration", effect: "no_effect", native_dispatch_started: false });
         input = { ...input, ownerPid: input.pid, uniqueActiveOperation: true };
     }
     if (input.uniqueActiveOperation && !input.operationId)
@@ -71,7 +80,7 @@ export function prepareManagedProcessRegistration(context, input) {
     catch {
         throw new AppError("validation", "metadata must be JSON-serializable", 2);
     }
-    const reader = getResourceDatabase(resourceHome(context), "read_existing");
+    const reader = getResourceDatabase(resourceHome(context), "read_existing", context.databaseDeadlineAt);
     try {
         if (input.id && reader.get("SELECT 1 FROM managed_processes WHERE id = ?", [input.id]))
             throw new AppError("validation", "Managed process ID already exists; use its retained registration", 2, { process_id: input.id });
@@ -89,11 +98,14 @@ export function prepareManagedProcessRegistration(context, input) {
 export function registerManagedProcess(context, rawInput) {
     const { input, observer, observerBirth, metadataJson } = prepareManagedProcessRegistration(context, rawInput);
     if (!input.uniqueActiveOperation)
-        return insertManagedProcess(context, input, metadataJson);
-    const db = registry(context);
+        return registry(context).writeTransaction(() => insertManagedProcess(context, input, metadataJson));
     const observerProcessBirth = observer ? stableProcessBirth(input.pid) : null;
+    const observerOwner = observer ? parseMetadata(metadataJson) : null;
+    if (observer && (!observerProcessBirth || !observerOwner?.owner_pid_started_at || !parseBirth(observerOwner.owner_birth)))
+        throw new AppError("process_ownership_unknown", "EVAL observer physical identity observation is unavailable", 1, { process_id: input.id, reason: "physical_identity_unavailable", phase: "observer_registration", effect: "no_effect", native_dispatch_started: false });
     if (observer && !processIsAlive({ pid: input.pid, pid_started_at: observerBirth, metadata_json: JSON.stringify({ process_birth: observerProcessBirth }) }))
         throw new AppError("process_ownership_unknown", "EVAL observer changed before registration", 1);
+    const db = registry(context);
     return db.writeTransaction(() => {
         const existing = db.get("SELECT * FROM managed_processes WHERE operation_id = ? AND state IN ('starting','running','stopping','orphaned')", [input.operationId]);
         if (existing)
@@ -133,29 +145,53 @@ export function prepareManagedProcessConfirmation(context, input) {
     const current = readManagedProcess(context, input.id);
     if (current.lease_token !== input.leaseToken)
         throw new AppError("process_lease_lost", "Managed process lease does not match", 1);
-    if (current.state !== "starting")
-        throw new AppError("process_confirmation_rejected", "Only a starting managed process can be confirmed", 1, { process_id: current.id, state: current.state });
+    if (current.state !== "starting" && !(current.state === "running" && current.pid === input.pid && processIsAlive(current) && (!input.processGroupId || parseMetadata(current.metadata_json).process_group_id === input.processGroupId) && (!input.ownerPid || parseMetadata(current.metadata_json).owner_pid === input.ownerPid)))
+        throw new AppError("process_confirmation_rejected", "Only the retained physical process may be confirmed", 1, { process_id: current.id, state: current.state });
     return current;
 }
 export function confirmManagedProcess(context, input) {
-    const current = prepareManagedProcessConfirmation(context, input);
+    const prepared = prepareManagedProcessConfirmation(context, input);
+    const retainedIdentity = parseMetadata(prepared.metadata_json);
+    const retainedOwner = retainedIdentity.owner_pid;
+    const validRetainedOwner = typeof retainedOwner === "number" && Number.isSafeInteger(retainedOwner) && retainedOwner > 0;
+    const ownerUnknown = Object.hasOwn(retainedIdentity, "owner_pid") && (!validRetainedOwner || typeof retainedIdentity.owner_pid_started_at !== "string" || !retainedIdentity.owner_pid_started_at.trim() || Object.hasOwn(retainedIdentity, "owner_birth") && !parseBirth(retainedIdentity.owner_birth));
+    if (prepared.state === "running" && (!prepared.pid_started_at || Object.hasOwn(retainedIdentity, "process_birth") && !parseBirth(retainedIdentity.process_birth) || ownerUnknown) || input.ownerPid === undefined && Object.hasOwn(retainedIdentity, "owner_pid") && !validRetainedOwner)
+        throw new AppError("process_ownership_unknown", "Retained managed physical identity is unconfirmed", 1, { process_id: input.id, reason: "stored_identity_unconfirmed", phase: "process_confirmation", effect: "no_effect", native_dispatch_started: false });
+    // Initial confirmation must observe the registered owner too, even when a
+    // bootstrap caller omits its optional override. Running bindings never repair.
+    const ownerPid = input.ownerPid ?? (validRetainedOwner ? retainedOwner : undefined);
+    const pidStartedAt = processStartedAt(input.pid), processBirth = stableProcessBirth(input.pid);
+    const ownerStartedAt = ownerPid ? processStartedAt(ownerPid) : null, ownerBirth = ownerPid ? stableProcessBirth(ownerPid) : null;
+    if (!pidStartedAt || !processBirth || ownerPid && (!ownerStartedAt || !ownerBirth))
+        throw new AppError("process_ownership_unknown", "Managed physical identity observation is unavailable", 1, { process_id: input.id, reason: "physical_identity_unavailable", phase: "process_confirmation", effect: "no_effect", native_dispatch_started: false });
     const db = registry(context);
-    const now = context.now();
-    const metadata = parseMetadata(current.metadata_json);
-    if (input.processGroupId)
-        metadata.process_group_id = input.processGroupId;
-    metadata.process_birth = stableProcessBirth(input.pid);
-    if (input.ownerPid) {
-        metadata.owner_pid = input.ownerPid;
-        metadata.owner_pid_started_at = processStartedAt(input.ownerPid);
-        metadata.owner_birth = stableProcessBirth(input.ownerPid);
-    }
-    const result = db.run(`UPDATE managed_processes
+    return db.writeTransaction(() => {
+        const current = requireProcess(db, input.id);
+        if (current.lease_token !== input.leaseToken || !["starting", "running"].includes(current.state))
+            throw new AppError("process_confirmation_rejected", "Managed process changed before confirmation", 1);
+        if (current.state === "running") {
+            if (prepared.state !== "running" || managedProcessBinding(current, resourceHome(context)).registration_sha256 !== managedProcessBinding(prepared, resourceHome(context)).registration_sha256 || current.pid !== input.pid || input.processGroupId && parseMetadata(current.metadata_json).process_group_id !== input.processGroupId || input.ownerPid && parseMetadata(current.metadata_json).owner_pid !== input.ownerPid)
+                throw new AppError("process_confirmation_rejected", "Confirmed process binding changed", 1);
+            return { ...current, registration_sha256: managedProcessBinding(current, resourceHome(context)).registration_sha256 };
+        }
+        const now = context.now();
+        const metadata = parseMetadata(current.metadata_json);
+        if (input.processGroupId)
+            metadata.process_group_id = input.processGroupId;
+        metadata.process_birth = processBirth;
+        if (ownerPid) {
+            metadata.owner_pid = ownerPid;
+            metadata.owner_pid_started_at = ownerStartedAt;
+            metadata.owner_birth = ownerBirth;
+        }
+        const result = db.run(`UPDATE managed_processes
      SET pid = ?, pid_started_at = ?, state = 'running', lease_expires_at = ?, updated_at = ?, metadata_json = ?
-     WHERE id = ? AND lease_token = ? AND state = 'starting'`, [input.pid, processStartedAt(input.pid), leaseExpiry(now, input.leaseMs), now, JSON.stringify(metadata), input.id, input.leaseToken]);
-    if (result.changes !== 1)
-        throw new Error(`Managed process cannot be confirmed: ${input.id}`);
-    return requireProcess(db, input.id);
+     WHERE id = ? AND lease_token = ? AND state = 'starting'`, [input.pid, pidStartedAt, leaseExpiry(now, input.leaseMs), now, JSON.stringify(metadata), input.id, input.leaseToken]);
+        if (result.changes !== 1)
+            throw new Error(`Managed process cannot be confirmed: ${input.id}`);
+        const confirmed = requireProcess(db, input.id);
+        return { ...confirmed, registration_sha256: managedProcessBinding(confirmed, resourceHome(context)).registration_sha256 };
+    });
 }
 export function prepareManagedProcessHeartbeat(context, input) {
     requireProcessText(input.id, "id");
@@ -163,28 +199,43 @@ export function prepareManagedProcessHeartbeat(context, input) {
     prepareManagedProcessLease(context, input.leaseMs);
 }
 export function heartbeatManagedProcess(context, input) {
+    return renewManagedProcess(context, input).owned;
+}
+export function renewManagedProcess(context, input) {
     prepareManagedProcessHeartbeat(context, input);
-    const reader = getResourceDatabase(resourceHome(context), "read_existing");
+    if (context.resourceDatabaseMode === "prepared_existing")
+        registry(context);
+    const reader = getResourceDatabase(resourceHome(context), "read_existing", context.databaseDeadlineAt);
     try {
-        if (!reader.get("SELECT 1 FROM managed_processes WHERE id = ? AND lease_token = ? AND state IN ('starting','running','stopping')", [input.id, input.leaseToken]))
-            return false;
+        if (!reader.get("SELECT 1 FROM managed_processes WHERE id=? AND lease_token=? AND state IN ('starting','running','stopping')", [input.id, input.leaseToken]))
+            return { owned: false, process_id: input.id };
     }
     finally {
         reader.close?.();
     }
     const db = registry(context);
-    return db.run("UPDATE managed_processes SET lease_expires_at = ?, updated_at = ? WHERE id = ? AND lease_token = ? AND state IN ('starting','running','stopping')", [leaseExpiry(context.now(), input.leaseMs), context.now(), input.id, input.leaseToken]).changes === 1;
+    return db.writeTransaction(() => {
+        const now = context.now();
+        if (!db.get("SELECT 1 FROM managed_processes WHERE id=? AND lease_token=? AND state IN ('starting','running','stopping')", [input.id, input.leaseToken]))
+            return { owned: false, process_id: input.id };
+        const changed = db.run("UPDATE managed_processes SET lease_expires_at = ?, updated_at = ? WHERE id = ? AND lease_token = ? AND state IN ('starting','running','stopping')", [leaseExpiry(now, input.leaseMs), now, input.id, input.leaseToken]).changes === 1;
+        if (!changed)
+            return { owned: false, process_id: input.id };
+        const record = requireProcess(db, input.id);
+        return { owned: true, process_id: record.id, lease_expires_at: record.lease_expires_at, registration_sha256: managedProcessBinding(record, resourceHome(context)).registration_sha256 };
+    });
 }
 /** Read-only lease check for a caller already holding a RUN writer. Renewal
  * must happen before entering that writer, never from this assertion. */
 export function assertManagedProcessLease(context, input) {
-    const reader = getResourceDatabase(resourceHome(context), "read_isolated");
+    const reader = getResourceDatabase(resourceHome(context), "read_isolated", context.databaseDeadlineAt);
     try {
         const current = reader.get("SELECT pid, lease_expires_at, state FROM managed_processes WHERE id = ? AND lease_token = ?", [input.id, input.leaseToken]);
-        if (!current || current.pid !== input.pid || !["starting", "running"].includes(current.state)
-            || Date.parse(current.lease_expires_at) <= Date.now()) {
-            throw new AppError("controller_lease_lost", "Managed owner lease changed or expired before productive dispatch", 1, { process_id: input.id });
+        if (!current || current.pid !== input.pid || !["starting", "running"].includes(current.state)) {
+            throw new AppError("controller_lease_lost", "Managed owner lease changed before productive dispatch", 1, { process_id: input.id });
         }
+        if (!Number.isFinite(Date.parse(current.lease_expires_at)) || Date.parse(current.lease_expires_at) <= Date.now())
+            throw new AppError("controller_lease_unconfirmed", "Managed owner requires renewal before productive dispatch", 1, { process_id: input.id, reason: "renewal_required" });
     }
     finally {
         reader.close?.();
@@ -200,21 +251,26 @@ export function prepareManagedProcessFinish(context, input) {
     return readManagedProcess(context, input.id);
 }
 export function finishManagedProcess(context, input) {
-    const existing = prepareManagedProcessFinish(context, input);
-    if (existing.lease_token === input.leaseToken && existing.state === input.state)
-        return true;
-    if (existing.lease_token !== input.leaseToken || !["starting", "running", "stopping", "orphaned"].includes(existing.state))
-        return false;
+    prepareManagedProcessFinish(context, input);
     const db = registry(context);
-    const now = context.now();
-    const updated = db.run("UPDATE managed_processes SET state = ?, termination_reason = ?, finished_at = ?, updated_at = ?, lease_expires_at = ? WHERE id = ? AND lease_token = ? AND state IN ('starting','running','stopping','orphaned')", [input.state, input.reason ?? null, now, now, now, input.id, input.leaseToken]).changes === 1;
-    if (updated)
-        db.run("DELETE FROM managed_resources WHERE process_id = ?", [input.id]);
-    if (updated)
-        return true;
-    // Another client may have completed the same lease after our initial read.
-    const current = requireProcess(db, input.id);
-    return current.lease_token === input.leaseToken && current.state === input.state;
+    return db.writeTransaction(() => {
+        const existing = requireProcess(db, input.id);
+        if (existing.lease_token !== input.leaseToken || existing.state !== input.state && !["starting", "running", "stopping", "orphaned"].includes(existing.state))
+            return false;
+        if (existing.state === input.state) {
+            db.run("DELETE FROM managed_resources WHERE process_id = ?", [input.id]);
+            return true;
+        }
+        const now = context.now();
+        const updated = db.run("UPDATE managed_processes SET state = ?, termination_reason = ?, finished_at = ?, updated_at = ?, lease_expires_at = ? WHERE id = ? AND lease_token = ? AND state IN ('starting','running','stopping','orphaned')", [input.state, input.reason ?? null, now, now, now, input.id, input.leaseToken]).changes === 1;
+        if (updated)
+            db.run("DELETE FROM managed_resources WHERE process_id = ?", [input.id]);
+        if (updated)
+            return true;
+        // Another client may have completed the same lease after our initial read.
+        const current = requireProcess(db, input.id);
+        return current.lease_token === input.leaseToken && current.state === input.state;
+    });
 }
 export function processIsAlive(record) {
     if (!record.pid)
@@ -300,7 +356,7 @@ export function managedLeaseReleased(record) {
         && ["stopped", "failed"].includes(record.state) && Boolean(record.finished_at);
 }
 export function managedProcessStatus(context) {
-    const db = getResourceDatabase(resourceHome(context), "read_existing");
+    const db = getResourceDatabase(resourceHome(context), "read_existing", context.databaseDeadlineAt);
     try {
         return db.all("SELECT * FROM managed_processes ORDER BY updated_at DESC, id");
     }
@@ -315,7 +371,7 @@ export function prepareManagedProcessStopOperation(context, input) {
     return { record, graceMs };
 }
 function readManagedProcess(context, id) {
-    const db = getResourceDatabase(resourceHome(context), "read_existing");
+    const db = getResourceDatabase(resourceHome(context), "read_existing", context.databaseDeadlineAt);
     try {
         return requireProcess(db, id);
     }
@@ -339,14 +395,19 @@ export async function stopManagedProcess(context, input) {
         return record;
     db.run("UPDATE managed_processes SET state = 'stopping', updated_at = ? WHERE id = ? AND lease_token = ? AND state IN ('starting','running','orphaned')", [context.now(), record.id, record.lease_token]);
     if (processTreeIsAlive(record)) {
-        terminateOwnedProcess(record, "SIGTERM");
+        const signalErrors = [];
+        const termError = terminateOwnedProcess(record, "SIGTERM");
+        if (termError)
+            signalErrors.push(errorRecord(termError));
         await delay(graceMs);
-        if (input.force !== false && processTreeIsAlive(record)) {
-            terminateOwnedProcess(record, "SIGKILL");
+        if (!termError && input.force !== false && processTreeIsAlive(record)) {
+            const killError = terminateOwnedProcess(record, "SIGKILL");
+            if (killError)
+                signalErrors.push(errorRecord(killError));
             await delay(250);
         }
         if (processTreeIsAlive(record))
-            throw Object.assign(new Error("Owned process tree could not be safely stopped; resources retained"), { code: "process_tree_not_settled", details: { process_id: record.id } });
+            throw Object.assign(new Error("Owned process tree could not be safely stopped; resources retained"), { code: "process_tree_not_settled", details: { process_id: record.id, signal_errors: signalErrors } });
     }
     if (!finishManagedProcess(context, { id: record.id, leaseToken: record.lease_token, state: "stopped", reason: input.reason ?? "requested_stop" }))
         throw Object.assign(new Error("Managed process ownership changed before settlement"), { code: "process_lease_lost" });
@@ -428,7 +489,7 @@ export async function reservePorts(context, input) {
     }
     return { ports, release: () => releasePortClaims(db, claims) };
 }
-function registry(context) { return getResourceDatabase(resourceHome(context)); }
+function registry(context) { return getResourceDatabase(resourceHome(context), context.resourceDatabaseMode, context.databaseDeadlineAt); }
 function requireProcess(db, id) { const row = db.get("SELECT * FROM managed_processes WHERE id = ?", [id]); if (!row)
     throw new AppError("not_found", `Managed process is missing: ${id}`, 1); return row; }
 export function prepareManagedProcessLease(context, leaseMs) { return leaseExpiry(context.now(), leaseMs); }
@@ -460,7 +521,10 @@ function terminateOwnedProcess(record, signal) {
     try {
         process.kill(process.platform === "win32" || group !== record.pid ? record.pid : -group, signal);
     }
-    catch { /* inspect again below */ }
+    catch (error) {
+        if (error.code !== "ESRCH")
+            return Object.assign(error, { details: { process_id: record.id, pid: record.pid, pgid: group ?? null, signal, phase: "owned_process_stop" } });
+    }
 }
 function parseMetadata(value) { try {
     const parsed = JSON.parse(value);

package/dist/services/merge-server.js

@@ -7,6 +7,7 @@ import fs from "node:fs";
 import path from "node:path";
 import { AppError } from "../shared/errors.js";
 import { runtimeBudgetForRun } from "./runtime-budget.js";
+import { managedLeaseMonitor } from "./managed-lease-monitor.js";
 import { flowCommand } from "./stage-pause.js";
 import { adapterSessionId, prepareHarnessFlowExecutable, runHarnessAdapter } from "./harness-adapter.js";
 import { harnessRuntimeArguments, resolveHarnessCommand } from "./harness-config.js";
@@ -106,36 +107,73 @@ async function dispatch(context, input) {
     const executable = resolveHarnessCommand(context.ddFlowHome, input.profile.harness, "adapter");
     const common = ["--dd-flow-bin", flowExecutable, "--state-dir", stateDir, "--journal", adapterJournal, "--cwd", input.request.target_workspace, "--provider", input.profile.provider, "--model", input.profile.model, "--reasoning", input.profile.reasoning, "--mode", input.profile.mode, "--permission", input.profile.permission, "--project-root", run.project_root, "--dd-flow-home", context.ddFlowHome, ...harnessRuntimeArguments(context.ddFlowHome, input.profile.harness), "--json"];
     const retained = { profile: input.profile, executable, owner: runtimeOwner, session_id: null, state_dir: stateDir, create_operation_id: `${input.request.merge_request_id}:${token}:session.create`, prompt_operation_id: `${input.request.merge_request_id}:${token}:session.prompt` };
-    const retain = () => { context.db.run("UPDATE merge_requests SET adapter_receipt_json = ?, updated_at = ? WHERE merge_request_id = ? AND dispatch_owner = ? AND dispatch_lease_token = ?", [JSON.stringify(retained), context.now(), input.request.merge_request_id, input.serverId, token]); };
+    const retain = () => { const changed = context.db.run("UPDATE merge_requests SET adapter_receipt_json = ?, updated_at = ? WHERE merge_request_id = ? AND dispatch_owner = ? AND dispatch_lease_token = ?", [JSON.stringify(retained), context.now(), input.request.merge_request_id, input.serverId, token]); if (changed.changes !== 1)
+        throw new AppError("capacity_continuation_blocked", "MERGE dispatch owner changed before durable receipt retention", 1); };
     retain();
-    const leaseHeartbeat = setInterval(() => { if (guarded(context, input.request) || (recoveryGuard(context, input.request.project_id, input.request.run_id)?.generation ?? 0) !== generation)
-        return; const now = context.now(); context.db.run("UPDATE merge_requests SET dispatch_lease_expires_at = ?, updated_at = ? WHERE merge_request_id = ? AND status IN ('dispatching','active') AND dispatch_owner = ? AND dispatch_lease_token = ?", [new Date(Date.now() + 120_000).toISOString(), now, input.request.merge_request_id, input.serverId, token]); }, 30_000);
+    const lease = managedLeaseMonitor(attemptMs => {
+        const db = context.db.withDeadline(Date.now() + attemptMs);
+        return db.writeTransaction(() => {
+            assertRunMutationAllowed({ ...context, db }, input.request.project_id, input.request.run_id, generation);
+            const expiry = new Date(Date.now() + 120_000).toISOString();
+            const changed = db.run("UPDATE merge_requests SET dispatch_lease_expires_at = ?, updated_at = ? WHERE merge_request_id = ? AND status IN ('dispatching','active') AND dispatch_owner = ? AND dispatch_lease_token = ?", [expiry, context.now(), input.request.merge_request_id, input.serverId, token]);
+            return changed.changes === 1 ? { owned: true, lease_expires_at: expiry } : { owned: false };
+        });
+    });
+    const ownership = new AbortController();
+    const leaseHeartbeat = setInterval(() => {
+        lease.background(() => assertRunMutationAllowed(context, input.request.project_id, input.request.run_id, generation), error => {
+            ownership.abort(error);
+            try {
+                event(input.journal, { type: "dispatch_lease_unconfirmed", request_id: input.request.merge_request_id, error: errorRecord(error), at: context.now() });
+            }
+            catch { /* No unhandled timer exception; the next boundary sees retained monitor failure. */ }
+        });
+    }, 30_000);
     leaseHeartbeat.unref();
     try {
-        const adapterCall = (args, timeoutMs, progressMessage, id = `${input.request.merge_request_id}:${token}:${args[0]}.${args[1]}`) => { assertRunMutationAllowed(context, input.request.project_id, input.request.run_id, generation); return runHarnessAdapter({ executable, args, env: runtimeContext.env, runtimeOwner, operationId: id, ...(timeoutMs ? { timeoutMs } : {}), ...(input.progress ? { progress: input.progress } : {}), ...(progressMessage ? { progressMessage } : {}), onEvidence: (value) => event(input.journal, { type: "adapter_call", ...value }) }); };
+        const adapterCall = async (args, timeoutMs, progressMessage, id = `${input.request.merge_request_id}:${token}:${args[0]}.${args[1]}`) => {
+            await lease.ensureConfirmed(() => assertRunMutationAllowed(context, input.request.project_id, input.request.run_id, generation));
+            try {
+                return await runHarnessAdapter({ executable, args, harness: runtimeHarness(input.profile), env: runtimeContext.env, runtimeOwner, operationId: id, signal: ownership.signal, ...(timeoutMs !== undefined ? { timeoutMs } : {}), ...(input.progress ? { progress: input.progress } : {}), ...(progressMessage ? { progressMessage } : {}), onEvidence: (value) => event(input.journal, { type: "adapter_call", ...value }), onNativeOutcome: value => {
+                        if (args[0] === "session" && args[1] === "create" && adapterSessionId(value)) {
+                            Object.assign(retained, { session_id: adapterSessionId(value), create: value, settlement_pending: true });
+                            retain();
+                        }
+                    } });
+            }
+            catch (error) {
+                if (ownership.signal.aborted)
+                    throw ownership.signal.reason;
+                throw error;
+            }
+        };
         await adapterCall(["daemon", "start", ...common]);
-        const created = await adapterCall(["session", "create", ...common], runtimeHarness(input.profile) === "zcode-acp" ? 180_000 : undefined);
+        const created = await adapterCall(["session", "create", ...common]);
         const sessionId = adapterSessionId(created);
         if (!sessionId)
             throw new AppError("merge_adapter_invalid", "Harness adapter did not return a Session id", 1, { harness: input.profile.harness, response: created });
-        Object.assign(retained, { session_id: sessionId, create: created });
+        Object.assign(retained, { session_id: sessionId, create: created, settlement_pending: false });
         retain();
         runtimeContext = managedInvocationContext(runtimeContext, { harness: runtimeHarness(input.profile), stateDir, sessionId, projectRoot: run.project_root, runId: input.request.run_id, generation });
         const stageCommand = managedLifecycleCommand(runtimeContext, `${flowCommand(runtimeContext)} stage start ${input.request.run_id} --stage merge --project-root ${JSON.stringify(run.project_root)} --json --progress-jsonl`);
         fs.writeFileSync(promptFile, `Start the assigned MERGE stage now. Your first tool call must be this exact standalone command:\n\n${stageCommand}\n\nTrust and follow the complete stage packet it returns. Continue through merge apply and stage finish. Stop only when the stage reaches a terminal result or explicitly requests user/operator action.\n\n${stageLifecycleInstruction}\n`);
         let prompted;
-        const promptDeadline = Date.now() + 45 * 60_000;
+        retained.progress_policy = "native-inactivity@1";
+        retain();
         try {
-            prompted = await adapterCall(["session", "prompt", "--session-id", sessionId, "--prompt-file", promptFile, ...common], 45 * 60 * 1000, `MERGE ${input.request.merge_request_id} Session ${sessionId} is still running`);
+            prompted = await adapterCall(["session", "prompt", "--session-id", sessionId, "--prompt-file", promptFile, ...common], null, `MERGE ${input.request.merge_request_id} Session ${sessionId} is still running`);
         }
         catch (error) {
             if (runtimeHarness(input.profile) !== "codex-desktop")
                 throw error;
             const capacityIntents = (retained.capacity_intents ?? []);
+            const capacityFailures = (retained.capacity_failures ?? []);
             const resumed = await continueCodexCapacity({ error, sessionId, operationId: String(retained.prompt_operation_id), allowSettledDescendants: true,
-                deadline: promptDeadline,
+                signal: ownership.signal,
                 continuationText: `Continue only MERGE ${input.request.merge_request_id}, Work ${input.request.executor_work_id}, RUN ${input.request.run_id} in this same Session and target ${input.request.target_workspace}. First inspect current durable Stage/Work/MERGE receipts. Never repeat Stage start, accepted answer, completed apply or checks. Follow only the current exact issued successor and stop at the original boundary.`,
                 retainedIntents: () => capacityIntents,
+                retainedFailures: () => capacityFailures,
+                retainFailure: (_id, refusal) => { capacityFailures.push(refusal); retained.capacity_failures = capacityFailures; retain(); },
                 inspectOperation: id => adapterCall(["daemon", "operation", "--state-dir", stateDir, "--operation-id", id, "--json"], 30_000, undefined, `${id}:capacity-observe`),
                 inspectSession: turnId => adapterCall(["session", "inspect", "--session-id", sessionId, ...(turnId ? ["--turn-id", turnId] : []), ...common], 30_000, undefined, `${input.request.merge_request_id}:${token}:capacity-inspect`),
                 assertCurrent: () => {
@@ -163,7 +201,7 @@ async function dispatch(context, input) {
                     }
                     else
                         fs.writeFileSync(file, text, { flag: "wx", mode: 0o600 });
-                    return adapterCall(["session", "prompt", "--session-id", sessionId, "--prompt-file", file, ...common], Math.max(1, promptDeadline - Date.now()), `MERGE ${input.request.merge_request_id} Session ${sessionId} capacity continuation ${ordinal}/2`, id);
+                    return adapterCall(["session", "prompt", "--session-id", sessionId, "--prompt-file", file, ...common], null, `MERGE ${input.request.merge_request_id} Session ${sessionId} capacity continuation ${ordinal}`, id);
                 } });
             if (!resumed)
                 throw error;
@@ -175,7 +213,7 @@ async function dispatch(context, input) {
         const request = context.db.get("SELECT status FROM merge_requests WHERE merge_request_id = ?", [input.request.merge_request_id]);
         event(input.journal, { type: "adapter_settled", request_id: input.request.merge_request_id, session_id: sessionId, request_status: request?.status ?? null, at: context.now() });
         if (request?.status === "dispatching")
-            context.db.run("UPDATE merge_requests SET status = 'recovery_required', last_error_json = ?, updated_at = ? WHERE merge_request_id = ?", [JSON.stringify({ code: "adapter_did_not_start_stage", response: prompted }), context.now(), input.request.merge_request_id]);
+            context.db.run("UPDATE merge_requests SET status = 'recovery_required', last_error_json = ?, updated_at = ? WHERE merge_request_id = ? AND status = 'dispatching' AND dispatch_owner = ? AND dispatch_lease_token = ? AND ? = COALESCE((SELECT MAX(generation) FROM run_recovery_guards WHERE project_id = merge_requests.project_id AND run_id = merge_requests.run_id), 0)", [JSON.stringify({ code: "adapter_did_not_start_stage", response: prompted }), context.now(), input.request.merge_request_id, input.serverId, token, generation]);
     }
     catch (error) {
         if (error instanceof AppError && ["run_recovery_guarded", "run_recovery_generation_stale"].includes(error.code)) {
@@ -183,24 +221,29 @@ async function dispatch(context, input) {
             throw error;
         }
         const request = context.db.get("SELECT status FROM merge_requests WHERE merge_request_id = ?", [input.request.merge_request_id]);
-        if (request?.status === "dispatching")
-            context.db.run("UPDATE merge_requests SET status = 'recovery_required', last_error_json = ?, updated_at = ? WHERE merge_request_id = ?", [JSON.stringify({ code: "adapter_launch_failed", error: errorRecord(error) }), context.now(), input.request.merge_request_id]);
-        else if (request?.status === "active")
-            context.db.run("UPDATE merge_requests SET status = 'recovery_required', last_error_json = ?, updated_at = ? WHERE merge_request_id = ?", [JSON.stringify({ code: "adapter_lost_after_stage_start", error: errorRecord(error) }), context.now(), input.request.merge_request_id]);
+        if (request?.status === "dispatching" || request?.status === "active")
+            context.db.run("UPDATE merge_requests SET status = 'recovery_required', last_error_json = ?, updated_at = ? WHERE merge_request_id = ? AND status = ? AND dispatch_owner = ? AND dispatch_lease_token = ? AND ? = COALESCE((SELECT MAX(generation) FROM run_recovery_guards WHERE project_id = merge_requests.project_id AND run_id = merge_requests.run_id), 0)", [JSON.stringify({ code: request.status === "dispatching" ? "adapter_launch_failed" : "adapter_lost_after_stage_start", error: errorRecord(error) }), context.now(), input.request.merge_request_id, request.status, input.serverId, token, generation]);
         throw error;
     }
     finally {
+        lease.close();
         clearInterval(leaseHeartbeat);
     }
 }
 function nextServerRequests(context, limit) { const selected = new Set(); return context.db.all("SELECT merge_request_id, project_id, run_id, executor_work_id, target_workspace, execution_route, status, created_at FROM merge_requests WHERE status = 'queued' AND execution_route = 'server' ORDER BY created_at, merge_request_id").filter((request) => { if (guarded(context, request) || selected.has(request.project_id) || context.db.get("SELECT 1 FROM merge_requests WHERE project_id = ? AND status NOT IN ('completed','failed','cancelled','superseded') AND (created_at < ? OR (created_at = ? AND merge_request_id < ?)) LIMIT 1", [request.project_id, request.created_at, request.created_at, request.merge_request_id]))
     return false; selected.add(request.project_id); return true; }).slice(0, limit); }
-function reconcileExpiredDispatches(context) { const expired = context.db.all("SELECT merge_request_id, executor_work_id, project_id, run_id FROM merge_requests WHERE status = 'dispatching' AND dispatch_lease_expires_at < ?", [context.now()]); for (const request of expired) {
-    if (guarded(context, request))
-        continue;
-    const work = context.db.get("SELECT status FROM works WHERE work_id = ?", [request.executor_work_id]);
-    context.db.run("UPDATE merge_requests SET status = 'recovery_required', last_error_json = ?, updated_at = ? WHERE merge_request_id = ?", [JSON.stringify({ code: "expired_dispatch_outcome_unknown", work_status: work?.status ?? null }), context.now(), request.merge_request_id]);
-} }
+export function reconcileExpiredDispatches(context) {
+    context.db.writeTransaction(() => {
+        const now = context.now();
+        const expired = context.db.all("SELECT merge_request_id, executor_work_id, project_id, run_id, dispatch_owner, dispatch_lease_token, dispatch_lease_expires_at FROM merge_requests WHERE status = 'dispatching' AND dispatch_lease_expires_at < ?", [now]);
+        for (const request of expired) {
+            if (guarded(context, request))
+                continue;
+            const work = context.db.get("SELECT status FROM works WHERE work_id = ?", [request.executor_work_id]);
+            context.db.run("UPDATE merge_requests SET status = 'recovery_required', last_error_json = ?, updated_at = ? WHERE merge_request_id = ? AND status = 'dispatching' AND dispatch_owner IS ? AND dispatch_lease_token IS ? AND dispatch_lease_expires_at = ? AND dispatch_lease_expires_at < ?", [JSON.stringify({ code: "expired_dispatch_outcome_unknown", work_status: work?.status ?? null }), now, request.merge_request_id, request.dispatch_owner, request.dispatch_lease_token, request.dispatch_lease_expires_at, now]);
+        }
+    });
+}
 function loadProfile(context, id) { return loadAgentProfile(context.ddFlowHome, id); }
 function event(file, value) { fs.appendFileSync(file, `${JSON.stringify(value)}\n`); }
 function guarded(context, request) { const guard = recoveryGuard(context, request.project_id, request.run_id); return Boolean(guard && ["draining", "sealed", "resuming"].includes(guard.status)); }
@@ -210,6 +253,6 @@ function processAlive(pid) { if (!pid)
     process.kill(pid, 0);
     return true;
 }
-catch {
-    return false;
+catch (error) {
+    return error.code !== "ESRCH";
 } }