chati-dev 4.5.28 → 4.5.29

package/src/terminal/run-council-gate.js

@@ -0,0 +1,338 @@
+#!/usr/bin/env node
+import { withUpgradeMaintenance } from '../upgrade/maintenance.js';
+
+import { fileURLToPath } from 'node:url';
+import { randomUUID } from 'node:crypto';
+import { resolve } from 'node:path';
+import { buildSpawnCommand, getTerminalLaunchAuthorityReceipt, getTerminalCompletionAuthorityReceipt, killTerminal, spawnTerminal, waitForTerminal } from './spawner.js';
+import { checkProviderReadiness } from './provider-preflight.js';
+import { getRateLimiter } from './rate-limiter.js';
+import {
+  buildCouncilGatePrompt,
+  cleanupCouncilGateWorkspace,
+  createCouncilGatePlan,
+  createCouncilGateWorkspace,
+  persistCouncilGateDecision,
+  assertCouncilGateApproved,
+} from '../orchestrator/council-gate-runtime.js';
+import { resolveRuntimeExecutableBindingV2 } from '../orchestrator/runtime-installation-v2.js';
+
+function fail(code, message, details = {}) {
+  const error = new Error(message);
+  error.code = code;
+  error.details = details;
+  throw error;
+}
+
+function terminalOutput(handle) {
+  return [...(handle.stdout || []), ...(handle.stderr || [])].join('');
+}
+
+function parseCouncilPayload(stdout, launch_nonce, provider) {
+  // Grok's machine-readable transport carries the final response in top-level
+  // text. Never scan the raw wrapper (escaped JSON), metadata or echoed prompts
+  // for a vote, and never fall back to text if the requested transport failed.
+  if (provider === 'grok') {
+    let transport;
+    try { transport = JSON.parse(stdout); }
+    catch { return fail('COUNCIL_GATE_TRANSPORT_INVALID', 'Grok council output is not valid transport JSON'); }
+    if (!transport || Array.isArray(transport) || typeof transport.text !== 'string') {
+      return fail('COUNCIL_GATE_TRANSPORT_INVALID', 'Grok council output requires a top-level text response');
+    }
+    stdout = transport.text;
+  }
+  const matches = [...String(stdout).matchAll(/<CHATI_COUNCIL_REVIEW>\s*([\s\S]*?)\s*<\/CHATI_COUNCIL_REVIEW>/g)];
+  if (matches.length === 0) return fail('COUNCIL_GATE_OUTPUT_MISSING', 'council member did not return the required result envelope');
+  if (matches.length !== 1) return fail('COUNCIL_GATE_OUTPUT_AMBIGUOUS', 'council member output contains more than one result envelope');
+  let payload;
+  try { payload = JSON.parse(matches[0][1]); }
+  catch { return fail('COUNCIL_GATE_OUTPUT_INVALID', 'council member result envelope is not valid JSON'); }
+  if (payload?.launch_nonce !== launch_nonce) {
+    return fail('COUNCIL_GATE_OUTPUT_NONCE_MISMATCH', 'council member result is not bound to its authority launch nonce');
+  }
+  return payload;
+}
+
+async function executeMember({ projectDir, workspace, plan, member, language, timeout, artifact, spawn, kill, readinessCheck, limiterFor, control, transport }) {
+  const providerExecutableBinding = resolveRuntimeExecutableBindingV2({
+    artifact,
+    provider_id: member.provider_id,
+    harness_id: member.harness_id,
+  });
+  const readiness = readinessCheck(member.harness_id, {
+    workingDir: workspace,
+    executableBinding: providerExecutableBinding,
+  });
+  if (!readiness.ready) {
+    fail('COUNCIL_MEMBER_UNAVAILABLE', `selected council member ${member.provider_id}/${member.model_pin.model_id} is unavailable: ${readiness.detail}`, {
+      member_id: member.member_id,
+      provider_id: member.provider_id,
+      code: readiness.code,
+    });
+  }
+  const authorityContext = {
+    authority_kind: 'council_gate',
+    invocation_id: transport?.invocationId ?? `council-${plan.gate_id}-${member.member_id}`,
+    launch_nonce: transport?.launchNonce ?? randomUUID(),
+    provider_id: member.provider_id,
+  };
+  const config = {
+    agent: `council-${plan.stage}`,
+    taskId: plan.gate_id,
+    provider: member.harness_id,
+    providerId: member.provider_id,
+    model: member.model_pin.model_id,
+    reasoningConfiguration: member.model_pin.reasoning_configuration,
+    catalogSnapshotRef: member.model_pin.catalog_snapshot_ref,
+    providerExecutableBinding,
+    prompt: transport?.prompt ?? buildCouncilGatePrompt({ plan, member, material: control.material, launch_nonce: authorityContext.launch_nonce, language }),
+    workingDir: workspace,
+    timeout,
+    writeScope: [],
+    auditProjectDir: projectDir,
+    maskedReadPaths: [resolve(projectDir)],
+    councilToolIsolation: true,
+    // A council receives a private frozen workspace. Use the same authority
+    // containment as RAIL review. The source project is hidden from reviewers;
+    // on Darwin the workspace is retained because scoped group completion does
+    // not prove detached descendants have ended.
+    authorityContext,
+  };
+  const built = buildSpawnCommand(config);
+  if (built.actualProvider !== config.provider || built.actualModel !== config.model
+    || built.actualReasoningConfiguration !== config.reasoningConfiguration) {
+    fail('COUNCIL_MEMBER_ROUTE_MISMATCH', 'terminal adapter changed the selected council binding', { member_id: member.member_id });
+  }
+  await limiterFor(member.harness_id).waitForSlot();
+  if (control.aborted) {
+    fail('COUNCIL_MEMBER_CANCELLED', `council member ${member.member_id} was cancelled after another member failed`, {
+      member_id: member.member_id,
+    });
+  }
+  const handle = spawn(config);
+  control.handles.add(handle);
+  try {
+    await confirmCouncilMemberCompletion(control, handle);
+    if (handle.exitCode !== 0) {
+      fail('COUNCIL_MEMBER_FAILED', `council member ${member.member_id} exited with status ${handle.exitCode}`, {
+        member_id: member.member_id,
+        output: terminalOutput(handle).slice(0, 10_000),
+      });
+    }
+    if (transport) {
+      // Reuse containment, not council semantics. The caller must parse and
+      // attest its distinct protocol before any authorization can be consumed.
+      return Object.freeze({ stdout: (handle.stdout || []).join(''), stderr: (handle.stderr || []).join(''),
+        launchReceipt: getTerminalLaunchAuthorityReceipt(handle),
+        completionReceipt: getTerminalCompletionAuthorityReceipt(handle) });
+    }
+    const payload = parseCouncilPayload((handle.stdout || []).join(''), authorityContext.launch_nonce, member.harness_id);
+    return {
+      member_id: member.member_id,
+      reviewer_identity: member.reviewer_identity,
+      verdict: payload.verdict,
+      findings: payload.findings ?? [],
+    };
+  } finally {
+    if (handle.status === 'running' || (control.darwin && !control.completed.has(handle))) {
+      await terminateCouncilMember(control, handle, kill);
+    }
+    control.handles.delete(handle);
+  }
+}
+
+async function confirmCouncilMemberCompletion(control, handle) {
+  await control.wait(handle);
+  if (control.darwin) getTerminalCompletionAuthorityReceipt(handle);
+  control.completed.add(handle);
+}
+
+async function terminateCouncilMember(control, handle, kill) {
+  const prior = control.terminations.get(handle);
+  if (prior) return prior;
+  const termination = (async () => {
+    if (handle.status === 'running') await kill(handle);
+    if (handle.status === 'running') {
+      fail('COUNCIL_MEMBER_TERMINATION_UNCONFIRMED', 'council member process did not confirm termination before workspace cleanup');
+    }
+    if (control.darwin && !control.completed.has(handle)) await confirmCouncilMemberCompletion(control, handle);
+  })();
+  control.terminations.set(handle, termination);
+  return termination;
+}
+
+async function cancelCouncilMembers(control, kill) {
+  control.aborted = true;
+  if (!control.cancellation) {
+    const active = [...control.handles];
+    control.cancellation = Promise.allSettled(active.map((handle) => terminateCouncilMember(control, handle, kill))).then((settled) => {
+      const failed = settled.find((result) => result.status === 'rejected');
+      if (failed) {
+        control.workspaceCleanupSafe = false;
+        fail('COUNCIL_MEMBER_TERMINATION_UNCONFIRMED', 'a council member could not be confirmed stopped, so its private workspace was retained', {
+          cause_code: failed.reason?.code,
+        });
+      }
+    });
+  }
+  await control.cancellation;
+}
+
+/** Single no-tools reviewer transport, with no vote parsing or gate persistence. */
+export async function executeCouncilReviewTransport(options = {}) {
+  return withUpgradeMaintenance(options.projectDir, async () => {
+    const { projectDir, current, member, prompt, invocationId, launchNonce, timeout = 600_000,
+      spawn = spawnTerminal, wait = waitForTerminal, kill = killTerminal,
+      readinessCheck = checkProviderReadiness, limiterFor = getRateLimiter } = options;
+    if (!current?.artifact || !current?.plan || !member?.member_id
+      || !current.plan.selection?.members?.some(item => item.member_id === member.member_id
+        && JSON.stringify(item) === JSON.stringify(member))
+      || typeof prompt !== 'string' || !prompt || typeof invocationId !== 'string' || !invocationId
+      || typeof launchNonce !== 'string' || !launchNonce
+      || !Number.isInteger(timeout) || timeout < 1 || timeout > 1_800_000) {
+      fail('COUNCIL_REVIEW_TRANSPORT_INVALID', 'Single reviewer transport requires an exact selected member, prompt, invocation and finite timeout.');
+    }
+    const workspace = createCouncilGateWorkspace(current);
+    const control = { aborted: false, darwin: process.platform === 'darwin', wait,
+      handles: new Set(), completed: new Set(), terminations: new Map(), cancellation: null,
+      workspaceCleanupSafe: process.platform !== 'darwin', material: current.material };
+    try {
+      return await executeMember({ projectDir, workspace, plan: current.plan, member,
+        timeout, artifact: current.artifact, spawn, kill, readinessCheck, limiterFor, control,
+        transport: { prompt, invocationId, launchNonce } });
+    } catch (error) {
+      await cancelCouncilMembers(control, kill);
+      throw error;
+    } finally {
+      if (control.handles.size) await cancelCouncilMembers(control, kill);
+      if (control.workspaceCleanupSafe) cleanupCouncilGateWorkspace(workspace);
+    }
+  });
+}
+
+/**
+ * Executes every selected council member against the same private frozen
+ * workspace. It persists only the normalized decision, never raw transcripts.
+ */
+export async function executeCouncilGate(options = {}) {
+  return withUpgradeMaintenance(options?.projectDir, () => executeCouncilGateUnderMaintenance(options));
+}
+
+async function executeCouncilGateUnderMaintenance({
+  projectDir,
+  stage,
+  artifact,
+  builder_identity,
+  policy,
+  handoff,
+  language = 'en',
+  timeout = 600_000,
+  clock = () => new Date(),
+  spawn = spawnTerminal,
+  wait = waitForTerminal,
+  kill = killTerminal,
+  readinessCheck = checkProviderReadiness,
+  limiterFor = getRateLimiter,
+} = {}) {
+  const current = createCouncilGatePlan({
+    projectDir, stage, artifact, builder_identity, policy,
+    ...(handoff === undefined ? {} : { handoff }),
+    clock,
+  });
+  const workspace = createCouncilGateWorkspace(current);
+  const control = {
+    aborted: false,
+    darwin: process.platform === 'darwin',
+    wait,
+    handles: new Set(),
+    completed: new Set(),
+    terminations: new Map(),
+    cancellation: null,
+    workspaceCleanupSafe: process.platform !== 'darwin',
+    material: current.material,
+  };
+  let memberPromises = [];
+  try {
+    memberPromises = current.plan.selection.members.map((member) => executeMember({
+      projectDir,
+      workspace,
+      plan: current.plan,
+      member,
+      language,
+      timeout,
+      artifact: current.artifact,
+      spawn,
+      kill,
+      readinessCheck,
+      limiterFor,
+      control,
+    }).catch(async (error) => {
+      await cancelCouncilMembers(control, kill);
+      throw error;
+    }));
+    const reviews = await Promise.all(memberPromises);
+    const persisted = persistCouncilGateDecision({ projectDir, plan: current.plan, reviews });
+    const approved = assertCouncilGateApproved({
+      projectDir,
+      stage,
+      artifact: current.artifact,
+      ...(policy ? { policy } : {}),
+      ...(handoff === undefined ? {} : { handoff }),
+      clock,
+    });
+    if (approved.plan.gate_id !== current.plan.gate_id) {
+      fail('COUNCIL_GATE_DECISION_STALE', 'council artifacts changed while member reviews were running');
+    }
+    return Object.freeze({
+      ok: approved.decision.verdict === 'accept',
+      action: 'council_gate_recorded',
+      stage: current.plan.stage,
+      gate_id: current.plan.gate_id,
+      member_count: current.plan.selection.member_count,
+      decision: approved.decision,
+      decision_path: persisted.path,
+      next_required_action: approved.decision.verdict === 'accept' ? 'advance_pipeline' : 'planning_rework_required',
+    });
+  } catch (error) {
+    await cancelCouncilMembers(control, kill);
+    await Promise.allSettled(memberPromises);
+    throw error;
+  } finally {
+    if (control.handles.size > 0) {
+      await cancelCouncilMembers(control, kill);
+      await Promise.allSettled(memberPromises);
+    }
+    if (control.workspaceCleanupSafe) cleanupCouncilGateWorkspace(workspace);
+  }
+}
+
+function parseArgs(argv) {
+  const args = {};
+  for (let index = 2; index < argv.length; index += 1) {
+    const token = argv[index];
+    if (!token.startsWith('--')) continue;
+    const next = argv[index + 1];
+    if (next && !next.startsWith('--')) { args[token.slice(2)] = next; index += 1; }
+    else args[token.slice(2)] = true;
+  }
+  return args;
+}
+
+async function main() {
+  const args = parseArgs(process.argv);
+  try {
+    const result = await executeCouncilGate({
+      projectDir: resolve(args['project-dir'] || process.cwd()),
+      stage: args.stage,
+      ...(args.language ? { language: args.language } : {}),
+      ...(args.timeout ? { timeout: Number(args.timeout) } : {}),
+    });
+    process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
+    if (!result.ok) process.exitCode = 1;
+  } catch (error) {
+    process.stdout.write(`${JSON.stringify({ ok: false, action: 'error', code: error.code || 'COUNCIL_GATE_FAILED', error: error.message })}\n`);
+    process.exitCode = 1;
+  }
+}
+
+if (process.argv[1] && resolve(process.argv[1]) === resolve(fileURLToPath(import.meta.url))) await main();

package/src/terminal/run-planning-reconciliation.js

@@ -0,0 +1,49 @@
+import { withUpgradeMaintenance } from '../upgrade/maintenance.js';
+import { loadSession, reservePlanningReconciliation, startPlanningReconciliation,
+  settlePlanningReconciliation, recordPlanningReconciliationFailure } from '../orchestrator/session-manager.js';
+import { pinPlanningReconciliation, buildPlanningReconciliationPrompt, planningReconciliationInvocationId,
+  persistPlanningReconciliationDecision, readPlanningReconciliationDecision } from '../orchestrator/planning-reconciliation.js';
+import { executeCouncilReviewTransport } from './run-council-gate.js';
+
+/** One bounded installed reviewer, durable one-shot admission and atomic QA reservation. */
+export async function executePlanningReconciliation({ projectDir, match, timeout = 600_000,
+  reviewTransport = executeCouncilReviewTransport } = {}) {
+  if (!Number.isInteger(timeout) || timeout < 1 || timeout > 1_800_000) return {
+    settled: false, code: 'PLANNING_RECONCILIATION_TIMEOUT_INVALID', error: 'Reconciliation requires a finite timeout of at most 30 minutes.' };
+  return withUpgradeMaintenance(projectDir, async () => {
+    const session = loadSession(projectDir).session;
+    const previous = session?.agents?.['qa-planning']?.planning_reconciliations?.find(entry =>
+      entry.request.held_qa.claim_id === match?.claimId && entry.request.held_qa.finished_at === match?.finishedAt
+      && entry.request.source_gate_id === match?.gateId && entry.authorization);
+    if (previous) return settlePlanningReconciliation(projectDir, { ...match,
+      reconciliationId: previous.request.reconciliation_id, launchNonce: previous.launch_nonce });
+    const reserved = reservePlanningReconciliation(projectDir, match);
+    if (!reserved.reserved) return { ...reserved, settled: false };
+    const entry = reserved.reservation;
+    const exact = { ...match, reconciliationId: entry.request.reconciliation_id, launchNonce: entry.launch_nonce };
+    const subject = pinPlanningReconciliation(projectDir, match, loadSession(projectDir).session);
+    try {
+      // A crash after persisting authenticated output needs no new inference.
+      const recorded = readPlanningReconciliationDecision(subject, { allowMissing: true });
+      if (recorded) return settlePlanningReconciliation(projectDir, exact);
+      if (entry.status !== 'reserved') return { settled: false,
+        code: entry.status === 'outcome_unknown' ? 'PLANNING_RECONCILIATION_OUTCOME_UNKNOWN' : 'PLANNING_RECONCILIATION_IN_FLIGHT',
+        error: 'This assessment already started. No retry or replacement is authorized without its authenticated completed result.' };
+      const started = startPlanningReconciliation(projectDir, exact);
+      if (!started.started) return { ...started, settled: false };
+      try {
+        subject.assertCurrent();
+        const transport = await reviewTransport({ projectDir, current: subject, member: subject.request.reviewer,
+          prompt: buildPlanningReconciliationPrompt({ request: subject.request, material: subject.material, launchNonce: entry.launch_nonce }),
+          invocationId: planningReconciliationInvocationId(subject.request), launchNonce: entry.launch_nonce, timeout });
+        subject.assertCurrent();
+        persistPlanningReconciliationDecision({ ...transport, subject });
+        return settlePlanningReconciliation(projectDir, exact);
+      } catch (error) {
+        const failure = recordPlanningReconciliationFailure(projectDir, exact, error.code);
+        return { settled: false, code: error.code || 'PLANNING_RECONCILIATION_FAILED', error: error.message,
+          outcome_unknown: true, failure_recorded: failure.recorded === true };
+      }
+    } finally { subject.close(); }
+  });
+}

package/src/terminal/run-rail-adjudication.js

@@ -1,4 +1,5 @@
 #!/usr/bin/env node
+import { withUpgradeMaintenance } from '../upgrade/maintenance.js';
 
 import { fileURLToPath } from 'node:url';
 import { resolve } from 'node:path';
@@ -10,6 +11,7 @@ import {
   buildSpawnCommand, getTerminalCompletionAuthorityReceipt, getTerminalLaunchAuthorityReceipt,
   isTransientFailure, killTerminal, spawnTerminal, waitForTerminal,
 } from './spawner.js';
+import { waitForObservedRailProcess } from './rail-process-observer.js';
 import { checkProviderReadiness } from './provider-preflight.js';
 import { getRateLimiter } from './rate-limiter.js';
 import { startClaimHeartbeat } from './run-rail-task.js';
@@ -135,7 +137,7 @@ async function invokeAdjudicator({ projectDir, artifact, context, plan, review_r
   };
   let { handle, exactInvocation } = await spawnAdjudicationAttempt(0);
   let retries = 0;
-  await waitForTerminal(handle);
+  await waitForObservedRailProcess({ handle, projectDir, attempt_id: context.attempt.attempt_id });
   assertRailReadOnlyCommitWorkspaceUnchanged({ workspace: reviewWorkspace, label: 'RAIL adjudication' });
   while (handle.exitCode !== 0 && retries < max_retries && isTransientFailure(handle.exitCode, [...handle.stdout, ...handle.stderr])) {
     assertLease();
@@ -143,7 +145,7 @@ async function invokeAdjudicator({ projectDir, artifact, context, plan, review_r
     await new Promise((done) => setTimeout(done, 2_000 * (2 ** (retries - 1))));
     assertLease();
     ({ handle, exactInvocation } = await spawnAdjudicationAttempt(retries));
-    await waitForTerminal(handle);
+    await waitForObservedRailProcess({ handle, projectDir, attempt_id: context.attempt.attempt_id });
     assertRailReadOnlyCommitWorkspaceUnchanged({ workspace: reviewWorkspace, label: 'RAIL adjudication retry' });
   }
   const adjudicationOutput = (handle.stdout || []).join('');
@@ -162,7 +164,11 @@ function unanimous(entries) {
   return adjudicationEntriesAgree(entries);
 }
 
-export async function executeRailAdjudication({
+export async function executeRailAdjudication(options = {}) {
+  return withUpgradeMaintenance(options?.projectDir, () => executeRailAdjudicationUnderMaintenance(options));
+}
+
+async function executeRailAdjudicationUnderMaintenance({
   projectDir, handoff_id, attempt_id,
   timeout = 30 * 60 * 1000, max_retries = 2, lease_ttl_ms = 5 * 60 * 1000,
 } = {}) {
@@ -180,6 +186,9 @@ export async function executeRailAdjudication({
   context = ownership.context;
   const renewed = ownership.renewed;
   let activeHandle = null;
+  // No local process needs reconciliation initially (including observation-only
+  // resume). Each returned handle invalidates this until its completion is read.
+  let invocationCompletionProven = true;
   let attemptClosed = false;
   let reviewWorkspace = null;
   try {
@@ -224,11 +233,14 @@ export async function executeRailAdjudication({
           review_ref: plan.review_ref, priorDecisions, round, timeout, max_retries,
           reviewWorkspace,
           reviewSubject: immutableReviewSubject,
-          onHandle: (handle) => { activeHandle = handle; },
+          onHandle: (handle) => { activeHandle = handle; invocationCompletionProven = false; },
           assertLease: () => {
             if (heartbeat.evidence().failure) fail('RAIL_LEASE_HEARTBEAT_FAILED', 'adjudication lost its RAIL claim heartbeat');
           },
         });
+        // invokeAdjudicator only returns after reading the trusted completion.
+        // Preserve that proof separately from the heartbeat's active handle.
+        invocationCompletionProven = true;
         activeHandle = null;
         recordRailAdjudicationObservation({
           projectDir, handoff_id, attempt_id, review_ref: plan.review_ref, entry,
@@ -274,12 +286,22 @@ export async function executeRailAdjudication({
     next_required_action: next,
   });
   } finally {
-    let cleanupProven = !activeHandle;
+    // A Darwin process group is not recursive descendant quiescence. Keep the
+    // readonly workspace and require private completion before claim release.
+    const retainWorkspace = process.platform === 'darwin';
+    let cleanupProven = !activeHandle && (!retainWorkspace || invocationCompletionProven);
     try {
-      if (activeHandle) { await killTerminal(activeHandle); cleanupProven = true; }
+      if (activeHandle) {
+        await killTerminal(activeHandle);
+        if (retainWorkspace) {
+          await waitForTerminal(activeHandle);
+          getTerminalCompletionAuthorityReceipt(activeHandle);
+        }
+        cleanupProven = true;
+      }
     } finally {
       try {
-        if (cleanupProven && reviewWorkspace) cleanupRailReadOnlyCommitWorkspace(reviewWorkspace);
+        if (cleanupProven && reviewWorkspace && !retainWorkspace) cleanupRailReadOnlyCommitWorkspace(reviewWorkspace);
       } finally {
         if (cleanupProven && !attemptClosed) {
           relinquishRailTask({

package/src/terminal/run-rail-recovery.js

@@ -0,0 +1,97 @@
+import { observeRailInterruptedExecution, preserveRailInterruptedExecution, restoreRailInterruptedExecution, reconcileRailObservedExecution, resolveRailAttemptContext } from '../orchestrator/rail-runtime.js';
+import { executeReadyRailTask } from './run-rail-task.js';
+import { executeRailRework } from './run-rail-rework.js';
+import { attemptJournalRevision } from '@chati/rail';
+import { withUpgradeMaintenance } from '../upgrade/maintenance.js';
+import { reviewNextRequiredAction } from './run-rail-review.js';
+
+const REFRESH_AFTER_RECONCILIATION = Symbol('refresh-after-reconciliation');
+
+/** Resume the canonical stage, never blindly replay a completed initial build. */
+export async function recoverAndResumeRailTask(options = {}) {
+  return withUpgradeMaintenance(options?.projectDir, () => recoverUnderMaintenance(options));
+}
+
+async function recoverUnderMaintenance({ projectDir, handoff_id, attempt_id, operator_id, termination_attestation, ...options } = {}) {
+  // An explicit operator attestation only feeds the interruption observation
+  // (Darwin same-boot recovery); it never reaches dispatch or rework options.
+  const observation = termination_attestation ? { termination_attestation } : {};
+  let changed;
+  // Retry only when the engine proves the observed attempt revision changed,
+  // before any ownership write. Unchanged errors never cause another launch.
+  // Observation, checkpoint and restoration each append one canonical fact.
+  // Leave bounded room for those steps plus concurrent revision refreshes.
+  for (let refresh = 0; refresh < 6; refresh += 1) {
+    try {
+      const result = await resumeFromSnapshot({ projectDir, handoff_id, attempt_id, operator_id, observation, ...options });
+      if (result !== REFRESH_AFTER_RECONCILIATION) return result;
+    }
+    catch (error) { if (error.code !== 'RECOVERY_CONTEXT_CHANGED') throw error; changed = error; }
+  }
+  throw changed ?? Object.assign(new Error('attempt kept changing during completion reconciliation'), { code: 'RECOVERY_CONTEXT_CHANGED' });
+}
+
+async function resumeFromSnapshot({ projectDir, handoff_id, attempt_id, operator_id, observation = {}, ...options }) {
+  const context = resolveRailAttemptContext({ projectDir, handoff_id, attempt_id });
+  const records = context.records.filter((record) => record.payload?.attempt_id === attempt_id);
+  const executions = records.filter((record) => record.type === 'execution_recorded').map((record) => record.payload.execution_evidence);
+  const completedNonces = new Set(executions.map((entry) => entry.receipt_nonce));
+  const notStartedNonces = new Set(records.filter((record) => record.type === 'attempt_recovered')
+    .flatMap((record) => [...(record.payload.reconciliation?.execution_admissions ?? []),
+      ...(record.payload.reconciliation?.execution_dispatches ?? [])])
+    .filter((entry) => ['not_started', 'interrupted_restored'].includes(entry.status)).map((entry) => entry.launch_nonce));
+  const pending = records.filter((record) => ['execution_launch_reserved', 'rework_launch_reserved', 'execution_dispatched'].includes(record.type)
+    && !completedNonces.has(record.payload.invocation.launch_nonce)
+    && !notStartedNonces.has(record.payload.invocation.launch_nonce));
+  const successful = executions.filter((entry) => entry.execution.exit_status === 0 && typeof entry.result_commit_ref === 'string');
+  const control = records.filter((record) => ['review_decided', 'adjudication_decided'].includes(record.type)).at(-1);
+  const source = control?.type === 'review_decided' ? control.payload.review_ref : control?.payload.adjudication_ref;
+  const reworkAuthorized = control?.payload.decision?.decision === 'normal-rework'
+    || control?.payload.decision?.outcome === 'bounded-correction';
+  const reworkNeeded = reworkAuthorized && !successful.some((entry) => entry.source_decision_ref === source);
+  if (context.state !== 'active') {
+    const next = { ready_to_merge: 'rail_merge_required', merged: 'rail_release_readiness',
+      cancelled: 'planning_handoff_revision_required', blocked: 'rail_authority_resolution_required' };
+    return Object.freeze({ ok: true, action: 'rail_attempt_state_observed', handoff_id, attempt_id,
+      task_id: context.task.task_id, state: context.state, next_required_action: next[context.state] });
+  }
+  const pendingRework = pending.at(-1)?.type === 'rework_launch_reserved'
+    || pending.at(-1)?.payload.run_kind === 'rework';
+  const expected_attempt_event_id = attemptJournalRevision(context.records, attempt_id);
+  const observedDispatch = pending.find((record) => record.type === 'execution_dispatched'
+    && records.some((entry) => entry.type === 'provider_process_observed'
+      && entry.payload.dispatch_event_id === record.event_id && entry.payload.terminal_output));
+  if (observedDispatch) {
+    reconcileRailObservedExecution({ projectDir, handoff_id, attempt_id,
+      launch_nonce: observedDispatch.payload.invocation.launch_nonce, expected_attempt_event_id });
+    return REFRESH_AFTER_RECONCILIATION;
+  }
+  const interruptedDispatch = pending.find((record) => record.type === 'execution_dispatched');
+  if (interruptedDispatch && !records.some((record) => record.type === 'execution_worktree_restored'
+    && record.payload.dispatch_event_id === interruptedDispatch.event_id)) {
+    const observed = observeRailInterruptedExecution({ projectDir, handoff_id, attempt_id,
+      launch_nonce: interruptedDispatch.payload.invocation.launch_nonce, expected_attempt_event_id, ...observation });
+    if (!observed.replay) return REFRESH_AFTER_RECONCILIATION;
+    const checkpoint = preserveRailInterruptedExecution({ projectDir, handoff_id, attempt_id,
+      launch_nonce: interruptedDispatch.payload.invocation.launch_nonce, expected_attempt_event_id });
+    if (!checkpoint.replay) return REFRESH_AFTER_RECONCILIATION;
+    restoreRailInterruptedExecution({ projectDir, handoff_id, attempt_id,
+      launch_nonce: interruptedDispatch.payload.invocation.launch_nonce, expected_attempt_event_id });
+    return REFRESH_AFTER_RECONCILIATION;
+  }
+  if (pendingRework || (pending.length === 0 && reworkNeeded)) {
+    return executeRailRework({ projectDir, handoff_id, attempt_id, operator_id, ...options, expected_attempt_event_id });
+  }
+  if (pending.length || successful.length === 0) {
+    return executeReadyRailTask({ projectDir, handoff_id, attempt_id, operator_id,
+      task_id: context.task.task_id, ...options, recover_expired: true, expected_attempt_event_id });
+  }
+  const observed = successful.at(-1);
+  const next = control?.type === 'review_decided' && !reworkAuthorized
+    ? reviewNextRequiredAction(control.payload.decision.decision)
+    : control?.payload.decision?.outcome === 'blocked-authority' ? 'human_authority_required' : 'independent_review';
+  return Object.freeze({ ok: true, action: 'rail_execution_already_recorded', handoff_id, attempt_id,
+    task_id: context.task.task_id, result_commit_ref: observed.result_commit_ref,
+    execution_evidence_ref: `rail-execution://${handoff_id}/${attempt_id}/${observed.evidence_id}`,
+    next_required_action: next });
+}