@deksden-com/dd-flow-cli 0.9.0-beta.15 → 0.9.0-beta.150

package/dist/services/provider-capacity.js

@@ -0,0 +1,455 @@
+import { setTimeout as delay } from "node:timers/promises";
+import crypto from "node:crypto";
+import { CAPACITY_POLICY, MAX_OVERLOAD_RETRIES, capacityBackoff, nativeItemsSettled, normalizeProviderFailure, overloadExhausted, parseCapacityOperation, refusalObservation, terminalProviderOverload, streamRequestKey } from "../harness-runtime/lib/provider-overload-policy.mjs";
+import { AppError } from "../shared/errors.js";
+import { isObservationLoss } from "../harness-runtime/lib/operation-errors.mjs";
+import { parseLifecycleCommand, commandOption } from "./lifecycle-command.js";
+import { nativeTreeReadiness } from "../harness-runtime/lib/native-children.mjs";
+import { ObservationClock } from "../harness-runtime/lib/observation-clock.mjs";
+/** Native output is evidence, not authority: match the committed current scope. */
+export function ownedProviderBoundary(context, input, boundary) {
+    if (typeof boundary.native_hook_event_key !== "string" || typeof boundary.daemon_instance !== "string"
+        || !context.db.get("SELECT 1 FROM hook_events WHERE project_id = ? AND event_key = ? AND daemon_id = ? AND provider_session_id = ? AND status = 'claimed'", [input.projectId, boundary.native_hook_event_key, boundary.daemon_instance, input.sessionId]))
+        return false;
+    const invocation = context.db.get("SELECT scope_json, outcome_json, command FROM lifecycle_invocations WHERE event_key = ? AND status = 'settled'", [boundary.native_hook_event_key]);
+    if (!invocation)
+        return false;
+    const scope = JSON.parse(invocation.scope_json);
+    const outcome = JSON.parse(invocation.outcome_json);
+    if (scope.runId !== input.runId || scope.daemonId !== boundary.daemon_instance || scope.rootSessionId !== input.sessionId || scope.generation !== input.generation
+        || outcome.error || outcome.effect !== undefined && outcome.effect !== "committed" || outcome.result?.ok !== true
+        || outcome.result.effect !== undefined && outcome.result.effect !== "committed")
+        return false;
+    const parsed = parseLifecycleCommand(invocation.command);
+    if (parsed.kind === "none" || parsed.kind === "compound" && !(parsed.supported === true && parsed.invocations?.length === 1) || !["stage_pause", "stage_finish", "work_finish"].includes(parsed.invocation.operation)
+        || boundary.outcome === "completed" && parsed.invocation.operation === "stage_pause" || boundary.outcome === "paused" && parsed.invocation.operation !== "stage_pause"
+        || boundary.lifecycle_operation !== undefined && boundary.lifecycle_operation !== parsed.invocation.operation
+        || parsed.invocation.operation !== "work_finish" && commandOption(parsed.invocation, "stage") !== input.stage)
+        return false;
+    const result = outcome.result;
+    if (result.run_id !== undefined && result.run_id !== input.runId || result.stage !== undefined && result.stage !== input.stage
+        || boundary.outcome === "paused" && (result.pause?.id !== boundary.pause?.id || result.outcome !== "paused")
+        || boundary.work_id !== undefined && result.work_id !== undefined && boundary.work_id !== result.work_id)
+        return false;
+    if (boundary.run_id !== undefined && boundary.run_id !== input.runId || boundary.stage !== undefined && boundary.stage !== input.stage
+        || boundary.work_id !== undefined && input.workId && boundary.work_id !== input.workId)
+        return false;
+    const run = context.db.get("SELECT index_json FROM runs WHERE project_id = ? AND id = ?", [input.projectId, input.runId]);
+    const index = JSON.parse(run?.index_json ?? "null");
+    const stage = index?.stage_runs?.find(item => item.stage === input.stage);
+    if (!stage?.attempt)
+        return false;
+    const workId = input.workId ?? (typeof boundary.work_id === "string" ? boundary.work_id : null);
+    const rows = context.db.all(`SELECT DISTINCT w.work_id, w.status FROM works w JOIN work_sessions ws ON ws.work_id = w.work_id JOIN sessions s ON s.project_id = w.project_id AND s.session_id = ws.session_id JOIN stage_work_memberships m ON m.project_id = w.project_id AND m.run_id = w.run_id AND m.work_id = w.work_id WHERE w.project_id = ? AND w.run_id = ? AND ws.status IN ('running', 'completed') AND s.provider_session_id = ? AND m.stage = ? AND m.attempt = ? AND (? IS NULL OR w.work_id = ?) ${!workId && input.scope === "stage" ? "AND w.parent_work_id IS NULL" : ""}`, [input.projectId, input.runId, input.sessionId, input.stage, stage.attempt, workId, workId]);
+    if (rows.length !== 1 || boundary.work_id !== undefined && boundary.work_id !== rows[0].work_id)
+        return false;
+    if (boundary.outcome === "paused")
+        return rows[0].status === "paused" && stage.pause?.work_id === rows[0].work_id && stage.pause.id === boundary.pause?.id;
+    return boundary.outcome === "completed" && (input.scope === "work" ? rows[0].status === "completed" : stage.status === "done");
+}
+export function providerChildFailure(child) {
+    return child.provider_failure ?? { code: child.native_source ? "provider_capacity_transient" : "turn_interrupted", details: { native_source: child.native_source, provider_session_id: child.session_id, turn_id: child.terminal_turn_id, native_turn_id: child.terminal_turn_id, terminal_status: child.status, observed_at: child.terminal_observed_at ?? null, provider_error: child.provider_error, native_turn_items: child.native_turn_items } };
+}
+export function eligibleProviderChildOverload(child, parentSessionId) {
+    return typeof child.session_id === "string" && child.session_id !== parentSessionId && child.parent_session_id === parentSessionId && child.provenance !== "parent_mismatch" && child.status === "failed" && typeof child.terminal_turn_id === "string" && nativeItemsSettled(child.native_turn_items)
+        && terminalProviderOverload(providerChildFailure(child), child.session_id, child.terminal_turn_id);
+}
+/** A terminal root is not a settled tree. Only the daemon's exact owned tree
+ * with a settled root and positively live descendants permits waiting. */
+export function capacityReadiness(seen, sessionId, operationId, turnId, items, allowDescendants = false, requestProof) {
+    const readiness = nativeTreeReadiness(seen, { sessionId, operationId, turnId, allowDescendants, requestProof });
+    return readiness === "inspection_pending" ? readiness : nativeItemsSettled(items) ? readiness : "unproven";
+}
+/** Permit acquisition must still belong to the original task, not merely an
+ * open RUN. Controls/inspection and ordinary successor prompts are unaffected. */
+export function assertCapacityOwnerAdmission(context, record, operationId, binding) {
+    if (!operationId || !record.project_id || !record.run_id)
+        return;
+    const root = parseCapacityOperation(operationId).root;
+    const childOrdinal = /:child:[a-f0-9]+:followup:(\d+)$/.exec(root)?.[1];
+    if (childOrdinal && (!Number.isSafeInteger(Number(childOrdinal)) || Number(childOrdinal) < 1 || Number(childOrdinal) > MAX_OVERLOAD_RETRIES || String(Number(childOrdinal)) !== childOrdinal))
+        throw new AppError("capacity_continuation_conflict", "Invalid native child continuation ordinal", 1);
+    if (root === operationId && !childOrdinal)
+        return;
+    const reject = () => { throw new AppError("capacity_owner_boundary_changed", "Queued continuation no longer belongs to an unfinished task; reconcile its durable state", 1, { operation_id: operationId, work_id: record.work_id, run_id: record.run_id }); };
+    const metadata = JSON.parse(record.metadata_json);
+    const hasTable = (name) => Boolean(context.db.get("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?", [name]));
+    const parsed = parseCapacityOperation(operationId);
+    if (parsed.ordinal > 0 && !["judge", "probe"].includes(metadata.role ?? "")) {
+        if (!binding?.nativeSessionId || !binding.promptSha256)
+            reject();
+        let intent;
+        const predecessor = parsed.ordinal === 1 ? root : `${root}:capacity:${parsed.ordinal - 1}:session.prompt`;
+        if (hasTable("run_controller_operations")) {
+            const saved = context.db.get("SELECT receipt_json FROM run_controller_operations WHERE operation_id = ? AND controller_id = ? AND generation = ? AND status = 'failed'", [predecessor, record.owner_id, metadata.run_generation ?? 0]);
+            if (saved)
+                intent = JSON.parse(saved.receipt_json).capacity_intent;
+        }
+        if (!intent && record.work_id)
+            for (const table of ["work_launches", "merge_requests"]) {
+                if (!hasTable(table))
+                    continue;
+                const rows = table === "work_launches"
+                    ? context.db.all("SELECT l.adapter_receipt_json FROM work_launches l JOIN works w ON w.work_id = l.work_id WHERE l.work_id = ? AND w.project_id = ? AND w.run_id = ?", [record.work_id, record.project_id, record.run_id])
+                    : context.db.all("SELECT adapter_receipt_json FROM merge_requests WHERE executor_work_id = ? AND project_id = ? AND run_id = ?", [record.work_id, record.project_id, record.run_id]);
+                for (const saved of rows) {
+                    const retained = JSON.parse(saved.adapter_receipt_json ?? "null");
+                    intent = retained?.capacity_intents?.find((item) => item.operation_id === operationId);
+                    if (intent)
+                        break;
+                }
+                if (intent)
+                    break;
+            }
+        if (intent?.policy && intent.policy !== CAPACITY_POLICY)
+            throw new AppError("capacity_policy_mismatch", "Continuation is pinned to another retry policy", 1);
+        if (!intent || parsed.ordinal > MAX_OVERLOAD_RETRIES || intent.policy !== CAPACITY_POLICY || intent.root_operation_id !== root || intent.ordinal !== parsed.ordinal
+            || intent.predecessor !== predecessor || intent.operation_id !== operationId || intent.refusal?.turn_id !== intent.turn_id
+            || intent.refusal?.ordinal !== parsed.ordinal - 1 || intent.refusal?.session_id !== intent.session_id
+            || !Number.isFinite(Date.parse(intent.not_before)) || Date.parse(intent.not_before) > Date.parse(context.now())
+            || intent.deadline !== null && (typeof intent.deadline !== "number" || intent.deadline <= Date.parse(context.now()))
+            || binding?.nativeSessionId !== undefined && intent.session_id !== binding.nativeSessionId
+            || binding?.promptSha256 !== undefined && intent.prompt_sha256 !== binding.promptSha256)
+            reject();
+    }
+    if (context.db.get("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'run_recovery_bindings'")) {
+        if (context.db.get("SELECT 1 FROM run_recovery_bindings b JOIN run_recovery_guards g ON g.recovery_id = b.recovery_id WHERE g.project_id = ? AND g.run_id = ? AND g.status = 'recovered' AND b.status = 'accepted' AND json_extract(b.runtime_dispatch_json, '$.operation_id') = ?", [record.project_id, record.run_id, root]))
+            reject();
+    }
+    if (record.work_id) {
+        const work = context.db.get("SELECT status FROM works WHERE work_id = ? AND project_id = ? AND run_id = ?", [record.work_id, record.project_id, record.run_id]);
+        const merge = context.db.get("SELECT status FROM merge_requests WHERE executor_work_id = ? AND project_id = ? AND run_id = ?", [record.work_id, record.project_id, record.run_id]);
+        if (merge && !["dispatching", "active"].includes(merge.status))
+            reject();
+        // MERGE owns the Stage/request boundary: finishing its Work does not
+        // necessarily commit stage finish yet. External workers own only Work.
+        const mergeOwner = metadata.role === "merge";
+        if (!work || !(merge && mergeOwner ? ["created", "running", "completed"] : ["created", "running"]).includes(work.status))
+            reject();
+    }
+    if (!context.db.get("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'run_controllers'"))
+        return;
+    const controller = context.db.get("SELECT state_json FROM run_controllers WHERE controller_id = ? AND project_id = ? AND run_id = ?", [record.owner_id, record.project_id, record.run_id]);
+    if (!controller)
+        return;
+    const state = JSON.parse(controller.state_json);
+    const run = context.db.get("SELECT index_json FROM runs WHERE project_id = ? AND id = ?", [record.project_id, record.run_id]);
+    const index = JSON.parse(run?.index_json ?? "{}");
+    if (state.stage && (index.current_stage && index.current_stage !== state.stage || index.stage_runs?.some(stage => stage.stage === state.stage && ["done", "skipped"].includes(stage.status))))
+        reject();
+    const child = Object.entries(state.child_capacity ?? {}).find(([, intent]) => intent.operation_id === root)?.[0];
+    if (childOrdinal && !child)
+        reject();
+    if (child && context.db.get("SELECT 1 FROM work_sessions ws JOIN works w ON w.work_id = ws.work_id JOIN sessions s ON s.session_id = ws.session_id WHERE w.project_id = ? AND w.run_id = ? AND s.provider_session_id = ? AND w.status <> 'running'", [record.project_id, record.run_id, child]))
+        reject();
+}
+export function capacityDelay(error, ordinal, now = Date.now()) {
+    return capacityBackoff(error, ordinal, now);
+}
+/** Continue a confirmed terminal Turn, never replay its completed effects. */
+export async function continueProviderCapacity(input) {
+    let { error, operationId } = input;
+    const history = [...(input.retainedIntents?.() ?? [])];
+    if (history.some(intent => intent.policy !== CAPACITY_POLICY) || input.retainedFailures?.().some(refusal => refusal.policy !== CAPACITY_POLICY))
+        throw new AppError("capacity_policy_mismatch", "Retained continuation uses another retry policy", 1);
+    if (history.length > MAX_OVERLOAD_RETRIES)
+        throw new AppError("capacity_continuation_conflict", "Retained episode exceeds its retry budget", 1);
+    const parsed = parseCapacityOperation(input.operationId);
+    const rootId = parsed.root;
+    const text = input.continuationText ?? "Continue the same assigned task in this Session. Check current durable RUN, Work and Stage state first. Never repeat an observed lifecycle command, accepted HITL answer, check or merge apply; use only an exact issued successor. Stop at the assigned boundary.";
+    const promptSha = crypto.createHash("sha256").update(text).digest("hex");
+    let ordinal = parsed.ordinal + 1;
+    const conflict = (message) => { throw new AppError("capacity_continuation_conflict", message, 1, { operation_id: operationId, root_operation_id: rootId, session_id: input.sessionId }); };
+    const blocked = (message) => new AppError("capacity_continuation_blocked", message, 1, { capacity_uncertainty: true, operation_id: operationId, root_operation_id: rootId, provider_session_id: input.sessionId, primary_error: error });
+    const observe = async (action, observedOperationId = operationId) => {
+        try {
+            return await action();
+        }
+        catch (cause) {
+            let primary = cause;
+            for (let depth = 0; depth < 4 && primary?.code === "harness_adapter_failed"; depth++)
+                primary = (primary.cause ?? primary.details?.cause);
+            if (isObservationLoss(primary))
+                throw new AppError("operation_observation_lost", "Capacity native observation was lost; do not cancel or resend", 1, { capacity_uncertainty: true, root_operation_id: rootId, operation_id: observedOperationId, provider_session_id: input.sessionId, predecessor_error: error, cause });
+            throw cause;
+        }
+    };
+    const refusals = [...(input.retainedFailures?.() ?? [])];
+    if (parsed.ordinal > history.length)
+        conflict("Continuation has no complete retained predecessor chain");
+    for (let index = 0; index < history.length; index++) {
+        const intent = history[index];
+        const expected = index + 1;
+        if (intent.ordinal !== expected || intent.operation_id !== `${rootId}:capacity:${expected}:session.prompt`
+            || intent.predecessor !== (expected === 1 ? rootId : `${rootId}:capacity:${expected - 1}:session.prompt`)
+            || intent.prompt_sha256 !== promptSha || intent.policy !== CAPACITY_POLICY || intent.root_operation_id !== rootId || intent.session_id !== input.sessionId
+            || intent.deadline !== (input.deadline ?? null) || !intent.refusal || intent.refusal.ordinal !== expected - 1 || intent.refusal.turn_id !== intent.turn_id
+            || !Number.isFinite(Date.parse(intent.not_before)))
+            conflict("Retained capacity chain is incomplete or belongs to another policy/task");
+        const previous = refusals.find(item => item.ordinal === intent.refusal.ordinal);
+        if (previous && JSON.stringify(previous) !== JSON.stringify(intent.refusal))
+            conflict("Retained failure observation changed");
+        if (!previous)
+            refusals.push(intent.refusal);
+    }
+    refusals.sort((a, b) => a.ordinal - b.ordinal);
+    for (let index = 0; index < refusals.length; index++) {
+        const refusal = refusals[index];
+        if (refusal.ordinal !== index || refusal.policy !== CAPACITY_POLICY || refusal.session_id !== input.sessionId || !refusal.turn_id
+            || refusals.slice(0, index).some(item => item.turn_id === refusal.turn_id) || !Number.isFinite(Date.parse(refusal.first_observed_at)))
+            conflict("Retained refusal chain is invalid");
+    }
+    // Reconcile a retained dispatch before preparing another. Native absence is
+    // the only safe way to reuse a prepared intent; a lost reply is not absence.
+    for (const intent of history) {
+        if (intent.ordinal < ordinal)
+            continue;
+        if (intent.predecessor !== operationId || intent.ordinal !== ordinal || intent.operation_id !== `${rootId}:capacity:${ordinal}:session.prompt`)
+            conflict("Retained successor skips its predecessor");
+        if (intent.prompt_sha256 !== promptSha || !intent.turn_id || !Number.isFinite(Date.parse(intent.not_before)))
+            throw error;
+        let observed;
+        try {
+            observed = await observe(() => input.inspectOperation(intent.operation_id), intent.operation_id);
+        }
+        catch (cause) {
+            let code = cause;
+            for (let depth = 0; depth < 4 && code?.code === "harness_adapter_failed"; depth++)
+                code = code.details?.cause;
+            if (code?.code !== "operation_not_found")
+                throw cause;
+            break;
+        }
+        if (observed.operation_id !== intent.operation_id || observed.session_id !== input.sessionId || observed.operation !== "session.prompt")
+            conflict("Retained native successor identity changed");
+        if (observed.primary_error)
+            throw observed.primary_error;
+        while (["prepared", "dispatching", "running", "awaiting_provider", "outcome_unknown"].includes(String(observed.state))) {
+            input.signal?.throwIfAborted();
+            input.assertCurrent();
+            const boundary = await observe(async () => await input.decision?.() ?? "continue", intent.operation_id);
+            if (boundary === "completed" || boundary === "paused")
+                return { provider_session_id: input.sessionId, capacity_decision: boundary };
+            if (boundary === "reconciliation_blocked")
+                throw new AppError("capacity_continuation_blocked", "Retained native successor has an unresolved owner boundary", 1, { operation_id: intent.operation_id, primary_error: error });
+            if (input.deadline !== undefined && Date.now() >= input.deadline)
+                throw new AppError("capacity_owner_deadline_exceeded", "Retained successor exhausted its original deadline", 1);
+            await delay(250, undefined, { signal: input.signal });
+            observed = await observe(() => input.inspectOperation(intent.operation_id), intent.operation_id);
+            input.signal?.throwIfAborted();
+            input.assertCurrent();
+            if (observed.operation_id !== intent.operation_id || observed.session_id !== input.sessionId || observed.operation !== "session.prompt")
+                conflict("Retained native successor identity changed");
+            if (observed.primary_error)
+                throw observed.primary_error;
+        }
+        if (observed.state === "completed" && observed.result && typeof observed.result === "object") {
+            input.signal?.throwIfAborted();
+            input.assertCurrent();
+            const boundary = await observe(async () => await input.decision?.() ?? "continue", intent.operation_id);
+            input.signal?.throwIfAborted();
+            input.assertCurrent();
+            if (boundary === "reconciliation_blocked")
+                throw new AppError("capacity_continuation_blocked", "Retained completion cannot be reconciled with its owner boundary", 1);
+            return boundary === "paused" || boundary === "completed" ? { provider_session_id: input.sessionId, capacity_decision: boundary } : observed.result;
+        }
+        if (observed.state === "failed" && observed.error && !terminalProviderOverload(observed.error, input.sessionId))
+            throw observed.error;
+        if (observed.state !== "failed" || !terminalProviderOverload(observed.error, input.sessionId))
+            throw new AppError("operation_observation_lost", "Retained successor outcome is not proven; do not resend", 1, { operation_id: intent.operation_id, primary_error: error, observed });
+        operationId = intent.operation_id;
+        error = observed.error;
+        ordinal += 1;
+    }
+    const decide = async (observedOperationId = operationId) => {
+        input.signal?.throwIfAborted();
+        input.assertCurrent();
+        const normalized = normalizeProviderFailure(error, input.sessionId);
+        const candidate = normalized?.request_proof.kind === "stream_request" ? normalized.primary_error.details?.recovered_error_boundary : undefined;
+        const decision = await observe(async () => await input.decision?.(candidate?.receipt) ?? "continue", observedOperationId);
+        input.signal?.throwIfAborted();
+        input.assertCurrent();
+        if (normalized?.request_proof.kind === "stream_request" && ["completed", "paused"].includes(decision) && !candidate?.receipt)
+            throw blocked("Stream refusal has no ordered evidence of recovery before the committed boundary");
+        if (decision === "reconciliation_blocked")
+            throw new AppError("capacity_continuation_blocked", "Native refusal cannot be reconciled with its durable owner boundary", 1, { primary_error: error, root_operation_id: rootId, provider_session_id: input.sessionId });
+        if (decision === "continue" && input.deadline !== undefined && Date.now() >= input.deadline)
+            throw new AppError("capacity_owner_deadline_exceeded", "Continuation exhausted its original owner deadline", 1, { primary_error: error, deadline: input.deadline, root_operation_id: rootId, provider_session_id: input.sessionId });
+        return decision === "continue" ? null : { provider_session_id: input.sessionId, capacity_decision: decision };
+    };
+    for (;; ordinal += 1) {
+        if (!Number.isSafeInteger(ordinal))
+            conflict("Continuation ordinal overflow");
+        if (!terminalProviderOverload(error, input.sessionId)) {
+            if (ordinal === 1)
+                return null;
+            throw error;
+        }
+        if (normalizeProviderFailure(error, input.sessionId)?.request_proof.kind !== "stream_request") {
+            const completed = await decide();
+            if (completed)
+                return completed;
+        }
+        const prior = await observe(() => input.inspectOperation(operationId));
+        const failure = prior.error;
+        const confirmed = normalizeProviderFailure(failure, input.sessionId);
+        const turnId = confirmed?.turn_id;
+        input.signal?.throwIfAborted();
+        input.assertCurrent();
+        if (prior.primary_error)
+            throw prior.primary_error;
+        if (prior.operation_id !== operationId || prior.operation !== "session.prompt" || prior.session_id !== input.sessionId
+            || prior.state !== "failed" || prior.primary_error || !turnId || !terminalProviderOverload(failure, input.sessionId, turnId)
+            || confirmed?.request_proof.kind === "stream_request" && confirmed.request_proof.operation_id !== operationId) {
+            throw blocked("Retained predecessor does not prove the exact terminal overloaded Turn");
+        }
+        let pendingClock;
+        const readiness = async () => {
+            input.signal?.throwIfAborted();
+            input.assertCurrent();
+            const seen = await observe(() => input.inspectSession(confirmed?.request_proof.kind === "stream_request" ? undefined : turnId));
+            input.signal?.throwIfAborted();
+            input.assertCurrent();
+            // An unloaded exact-Turn snapshot is not conflicting tool evidence.
+            // Preserve the bound failure receipt, but never hide a pending item.
+            const freshItems = seen.native_turn_items;
+            const stream = confirmed?.request_proof.kind === "stream_request";
+            const streamMetadata = seen.terminal_metadata;
+            const exactFresh = stream ? streamRequestKey(streamMetadata?.request_proof) === turnId : seen.terminal_turn_id === turnId && seen.terminal_status === "failed";
+            const items = (exactFresh
+                && !(freshItems?.observed === false && freshItems.pending !== true) ? freshItems : confirmed?.native_turn_items);
+            if (confirmed?.request_proof.kind === "stream_request") {
+                const metadata = seen.terminal_metadata;
+                if (streamRequestKey(metadata?.request_proof) !== turnId)
+                    return "unproven";
+            }
+            const current = capacityReadiness(seen, input.sessionId, operationId, turnId, items, input.allowSettledDescendants, confirmed?.request_proof);
+            if (current === "inspection_pending") {
+                const saved = seen.native_settlement.observation_clock;
+                pendingClock ??= new ObservationClock({ timeoutMs: saved.timeout_ms, saved, snapshotIsLive: true });
+                if (pendingClock.sample(saved.cursor, saved.observed_at ?? undefined))
+                    return "unproven";
+            }
+            return current;
+        };
+        // Pending is a bound observer, not a refusal or an intent to write. Resolve
+        // it before retaining a productive retry. Its producer owns the old clock.
+        const waitForReadiness = async (initial) => {
+            let current = initial;
+            while (current === "owned_children_live" || current === "inspection_pending") {
+                await delay(250, undefined, { signal: input.signal });
+                const boundary = await decide();
+                if (boundary)
+                    return boundary;
+                current = await readiness();
+            }
+            if (current !== "ready")
+                throw blocked("Native predecessor readiness became unproven");
+            return null;
+        };
+        let ready = await readiness();
+        while (ready === "inspection_pending") {
+            await delay(250, undefined, { signal: input.signal });
+            const boundary = await decide();
+            if (boundary)
+                return boundary;
+            ready = await readiness();
+        }
+        if (ready === "unproven")
+            throw blocked("Native predecessor readiness is unproven; do not cancel or replay to manufacture proof");
+        if (ready === "ready") {
+            const completed = await decide();
+            if (completed)
+                return completed;
+            if (confirmed?.request_proof.kind === "stream_request" && confirmed.primary_error.details?.recovered_error_boundary)
+                throw blocked("Recovered native error does not match its current durable owner boundary");
+        }
+        const native = normalizeProviderFailure(failure, input.sessionId, turnId);
+        let refusal = refusals.find(item => item.ordinal === ordinal - 1);
+        if (refusal && (refusal.turn_id !== turnId || refusal.session_id !== input.sessionId || refusal.policy !== CAPACITY_POLICY))
+            conflict("Failure ordinal identifies a different native Turn");
+        if (refusal) {
+            const expected = refusalObservation(failure, ordinal - 1, refusal.first_observed_at);
+            if (refusal.native_terminal_at !== expected.native_terminal_at || refusal.native_terminal_basis !== expected.native_terminal_basis)
+                conflict("Retained refusal time differs from its native receipt");
+        }
+        if (!refusal) {
+            if (refusals.some(item => item.turn_id === turnId))
+                conflict("A cached native Turn cannot be a second refusal");
+            // A cached legacy failure cannot be assigned a new time during reattach.
+            if (ordinal > 1 && parsed.ordinal > 0 && !native.observed_at)
+                conflict("Retained continuation has no original failure time");
+            const observedAt = typeof native.observed_at === "string" && Number.isFinite(Date.parse(native.observed_at)) ? native.observed_at : new Date().toISOString();
+            refusal = refusalObservation(failure, ordinal - 1, observedAt);
+            if (!input.atomicIntent)
+                input.retainFailure?.(operationId, refusal);
+            refusals.push(refusal);
+        }
+        const exhausted = overloadExhausted(refusals);
+        if (exhausted) {
+            if (input.atomicIntent)
+                input.retainFailure?.(operationId, refusal);
+            throw new AppError("provider_unavailable", "Provider overload retry episode exhausted", 1, { ...exhausted, root_operation_id: rootId, provider_session_id: input.sessionId, continuation_count: ordinal - 1, primary_error: refusals.length ? input.error : failure, last_error: failure });
+        }
+        const nextId = `${rootId}:capacity:${ordinal}:session.prompt`;
+        const existing = history.find(item => item.operation_id === nextId);
+        if (existing && (existing.predecessor !== operationId || existing.turn_id !== turnId || existing.prompt_sha256 !== promptSha || existing.ordinal !== ordinal))
+            throw error;
+        let wait;
+        try {
+            wait = existing ? Math.max(0, Date.parse(existing.not_before) - Date.now()) : capacityDelay(failure, ordinal);
+        }
+        catch (cause) {
+            if (input.atomicIntent)
+                input.retainFailure?.(operationId, refusal);
+            const metadata = cause;
+            throw new AppError(metadata.code ?? "capacity_retry_metadata_invalid", metadata.message ?? "Invalid provider retry metadata", 1, { ...metadata.details, policy: CAPACITY_POLICY, attempts: refusals.length, root_operation_id: rootId, primary_error: input.error, last_error: failure });
+        }
+        if (!Number.isFinite(wait) || Date.now() + wait > 8.64e15 || input.deadline !== undefined && Date.now() + wait >= input.deadline) {
+            if (input.atomicIntent)
+                input.retainFailure?.(operationId, refusal);
+            throw new AppError("capacity_owner_deadline_exceeded", "Continuation wait exceeds its original owner deadline", 1, { primary_error: failure, deadline: input.deadline ?? null });
+        }
+        if (!existing) {
+            const intent = { predecessor: operationId, operation_id: nextId, turn_id: turnId, request_key: native.request_key, request_proof: native.request_proof, ordinal, prompt_sha256: promptSha, not_before: new Date(Date.now() + wait).toISOString(), policy: CAPACITY_POLICY, root_operation_id: rootId, session_id: input.sessionId, deadline: input.deadline ?? null, refusal };
+            input.retainIntent(intent);
+            history.push(intent);
+        }
+        // Intent/refusal are durable before the first wait; no permit or writer is
+        // held here, and live siblings are never cancelled as a settlement proof.
+        const childBoundary = await waitForReadiness(ready);
+        if (childBoundary)
+            return childBoundary;
+        // Node timers truncate large delays. Bounded slices preserve cancellation
+        // and owner checks; monotonic elapsed time does not renew a Retry-After.
+        const until = performance.now() + Math.max(0, Date.parse(history.find(item => item.operation_id === nextId).not_before) - Date.now());
+        while (performance.now() < until) {
+            await delay(Math.min(250, Math.max(0, until - performance.now())), undefined, { signal: input.signal });
+            const boundary = await decide();
+            if (boundary)
+                return boundary;
+        }
+        const afterWait = await decide();
+        if (afterWait)
+            return afterWait;
+        const refreshedBoundary = await waitForReadiness(await readiness());
+        if (refreshedBoundary)
+            return refreshedBoundary;
+        const beforeDispatch = await decide();
+        if (beforeDispatch)
+            return beforeDispatch;
+        try {
+            return await observe(() => input.prompt(nextId, text, ordinal), nextId);
+        }
+        catch (nextError) {
+            // A task can complete/pause while the daemon queues for its permit.
+            // Admission then refuses the stale prompt; accept that owner decision
+            // before treating the transport's rejection as another provider failure.
+            const finished = await decide(nextId);
+            if (finished)
+                return finished;
+            error = nextError;
+            operationId = nextId;
+        }
+    }
+}

package/dist/services/recovery-observation-budget.js

@@ -0,0 +1,121 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+import { AppError } from "../shared/errors.js";
+import { validateObservationDuration } from "../harness-runtime/lib/observation-clock.mjs";
+export const recoveryObservationBudgetMs = 120_000;
+const activeObservation = new AsyncLocalStorage();
+export const withRecoveryObservation = (budget, action, retainActive = false) => activeObservation.run(retainActive ? activeObservation.getStore() ?? budget : budget, action);
+export const recoveryRpcTimeout = (maximum) => {
+    if (maximum != null)
+        validateObservationDuration(maximum);
+    const observation = activeObservation.getStore();
+    // An unbounded productive transport wait is still bounded while recovery is
+    // actively reconciling. Outside recovery, its owner policy decides when to
+    // stop rather than a hidden adapter timer.
+    if (maximum === null)
+        return observation ? observation.timeout() : null;
+    return observation?.timeout(maximum ?? 30_000) ?? maximum ?? 30_000;
+};
+/** Call only while holding the existing owner-claim transaction. A replay of
+ * an explicit request never replenishes time, even after another request. */
+export function renewRecoveryObservation(snapshot, requestId) {
+    if (typeof requestId !== "string" || !requestId.trim() || requestId.length > 200)
+        throw new AppError("usage", "Reconciliation requires a nonempty request ID of at most 200 characters", 2);
+    const previous = snapshot.recovery_observation;
+    const ids = previous?.reconciliation_ids ?? [];
+    if (!Array.isArray(ids) || ids.some(id => typeof id !== "string"))
+        throw new AppError("recovery_observation_invalid", "Retained reconciliation identities are malformed", 1);
+    if (ids.includes(requestId))
+        return snapshot;
+    return { ...snapshot, recovery_observation: { policy_id: "settlement-inactivity@1", observer_started: false, budget_ms: recoveryObservationBudgetMs, remaining_ms: recoveryObservationBudgetMs, progress_markers: [], observation_gaps: 0, reconciliation_ids: [...ids, requestId], poll_index: 0 } };
+}
+/**
+ * New episodes measure silence between confirmed retirement facts; legacy
+ * receipts keep cumulative active-observer accounting. A gap never proves
+ * native death, and repeated gaps exhaust observation rather than grant time.
+ * Remaining time and seen proofs survive owner replacement.
+ */
+export function recoveryObservationBudget(saved, monotonic = () => performance.now()) {
+    const prior = saved && typeof saved === "object" ? saved.recovery_observation : null;
+    if (prior?.policy_id !== undefined && prior.policy_id !== "settlement-inactivity@1")
+        throw new AppError("recovery_observation_invalid", "Unknown retained observation policy", 1);
+    const sliding = !prior || prior.policy_id === "settlement-inactivity@1";
+    if (prior?.progress_markers !== undefined && (!Array.isArray(prior.progress_markers) || prior.progress_markers.some(marker => typeof marker !== "string"))
+        || prior?.observation_gaps !== undefined && (!Number.isSafeInteger(prior.observation_gaps) || prior.observation_gaps < 0)
+        || prior?.observer_started !== undefined && typeof prior.observer_started !== "boolean")
+        throw new AppError("recovery_observation_invalid", "Retained observation progress is malformed", 1);
+    const markers = new Set(prior?.progress_markers ?? []);
+    let gaps = prior?.observation_gaps ?? 0;
+    let remaining = typeof prior?.remaining_ms === "number" && Number.isFinite(prior.remaining_ms)
+        ? Math.max(0, Math.min(recoveryObservationBudgetMs, prior.remaining_ms))
+        : prior == null ? recoveryObservationBudgetMs : 0;
+    // A restored observer cannot know what happened while it was absent. Retain
+    // one bounded uncertainty episode; repeated reconstruction never grants time.
+    if (sliding && prior && prior.observer_started !== false && ++gaps > 1)
+        remaining = 0;
+    let last = monotonic();
+    let pollIndex = Number.isSafeInteger(prior?.poll_index) && prior.poll_index >= 0 ? prior.poll_index : 0;
+    const observation = {
+        exhausted: () => {
+            const current = monotonic();
+            const elapsed = current - last;
+            last = current;
+            // A gap means this process was not observing; it never proves progress
+            // or death and therefore does not consume the reconciliation budget.
+            if (elapsed >= 0 && elapsed <= 60_000)
+                remaining = Math.max(0, remaining - elapsed);
+            else if (sliding && ++gaps > 1)
+                remaining = 0;
+            return remaining === 0;
+        },
+        observe: snapshot => {
+            observation.exhausted();
+            if (!sliding || remaining === 0)
+                return;
+            const proof = settlementProgressMarkers(snapshot);
+            const advanced = proof.some(marker => !markers.has(marker));
+            proof.forEach(marker => markers.add(marker));
+            if (advanced) {
+                remaining = recoveryObservationBudgetMs;
+                gaps = 0;
+            }
+        },
+        state: () => ({ ...(sliding ? { policy_id: "settlement-inactivity@1", observer_started: true, progress_markers: [...markers].sort(), observation_gaps: gaps } : {}), budget_ms: recoveryObservationBudgetMs, remaining_ms: Math.ceil(remaining), reconciliation_ids: prior?.reconciliation_ids ?? [], poll_index: pollIndex }),
+        timeout: (maximum = 30_000) => {
+            validateObservationDuration(maximum);
+            if (observation.exhausted())
+                throw new AppError("recovery_observation_budget_exhausted", "Recovery observation budget exhausted; explicit reconciliation is required", 1);
+            return Math.max(1, Math.min(maximum, Math.ceil(remaining)));
+        },
+        nextDelay: () => Math.min([1000, 2000, 5000, 10000][Math.min(pollIndex++, 3)], Math.ceil(remaining)),
+    };
+    return observation;
+}
+/** Only confirmed settlement facts advance recovery. Polls, errors, timestamps,
+ * and native model activity cannot replenish physical retirement observation. */
+export function settlementProgressMarkers(snapshot) {
+    const result = [];
+    const visit = (value, prefix, depth) => {
+        if (!value || typeof value !== "object" || depth > 12)
+            return;
+        if (Array.isArray(value)) {
+            for (const item of value)
+                visit(item, prefix, depth + 1);
+            return;
+        }
+        const object = value;
+        const identity = ["operation_id", "process_id", "session_id", "run_id", "control_id", "scope_id", "id"].map(key => typeof object[key] === "string" ? `${key}:${object[key]}` : "").filter(Boolean).join("|");
+        const location = `${prefix}/${identity}`;
+        for (const key of ["physical_settled", "native_settled", "settled", "retired", "published", "committed"])
+            if (object[key] === true)
+                result.push(`${location}/${key}`);
+        if (["completed", "released", "sealed", "verified", "retired"].includes(String(object.status)))
+            result.push(`${location}/status:${String(object.status)}`);
+        if (object.state === "settled")
+            result.push(`${location}/state:settled`);
+        for (const [key, item] of Object.entries(object))
+            if (key !== "recovery_observation" && key !== "error" && key !== "metadata" && key !== "capture_error")
+                visit(item, `${location}/${key}`, depth + 1);
+    };
+    visit(snapshot, "", 0);
+    return [...new Set(result)].sort();
+}