elanous 0.2.40 → 0.2.42

package/scripts/release-loop/resume-release.ts

@@ -8,17 +8,21 @@ import { tmpdir } from 'node:os';
 import { dirname, join, resolve } from 'node:path';
 import { debug } from '../../src/debug/log.js';
 import { effectiveInstanceRoot, releaseLedgerRoot } from '../../src/instance/resolve.js';
+import { acquireLockAsync } from '../../src/storage/file-lock.js';
 import { lastJsonObject, runGraph, verifyReleaseCommandRoot, verifyReleaseRecipeSnapshot, type GraphRunState } from '../../src/graph-runner/runner.js';
 import { withOpSeatFlock } from '../../src/steward/resync-tick.js';
 import { releaseRunUniverse } from './release-universe.js';
 import { isReleaseVersion } from './release-version.js';
-import { changedFilesBetween, clearPartialPlan, collectExecutedPodShardImages, planPartialRegate, planStalledShardRecheck, writePartialPlan, writeRecheckPlan } from './gate-partial.js';
+import { changedFilesBetween, clearPartialPlan, collectExecutedPodShardImages, planPartialRegate, planSameCommitRecheck, planStalledShardRecheck, testFileOfId, writePartialPlan, writeRecheckPlan } from './gate-partial.js';
+import { gateAssignableTestsAtCommit, gateEnvFingerprint } from './gate-baseline-receipt.js';
+import { readGateShardManifest, readGateShardResult } from './gate-shard-receipts.js';
 import { gitClosureTree, testsReachingChanged, TRACEABLE_RELEASE_JSON, type ClosureTree } from './gate-closure.js';
 import { assertReleaseResumeApprovalBoundary, releaseResumeAcceptedRegressions, releaseResumeGateExecutor, releaseResumeGatePodPool, releaseResumeGateShardReroute, releaseResumeWaiver, type AcceptedRegression } from './resume.js';
 import { reserveReleaseResumeAttempt } from './resume-cap.js';
 import { cutReleaseBranch } from './cut-branch.js';
 import { releaseLoopConfig } from './unattended-release.js';
 import { trafficOptionsFromConfig } from './traffic-transition.js';
+import type { GateAutohealDecision } from './gate-autoheal.js';
 
 const GRAPH = join(import.meta.dir, '../../graphs/release/release-loop.yaml');
 
@@ -46,6 +50,8 @@ export interface ResumeReleaseDeps {
   ledgerRoot?: string;
   /** GATE-REUSE-BY-DIFF: the tip tree for the import closure — default read from git objects of the tip commit. */
   closureTree?: (commit: string) => ClosureTree;
+  /** Test seam for the pinned Pod gate selection; production reads the selected test universe at the cut commit. */
+  gateTestsAtCommit?: (commit: string) => string[] | null;
   /** Source observation before the first resumed node. */
   onSource?: (line: string) => void;
   /** Test seam; production resolves this from ELANOUS_RELEASE_TRAFFIC_FILE / release.loop.trafficFile. */
@@ -62,9 +68,36 @@ export interface ResumeReleaseOptions {
 export interface ResumeReleasePlan {
   runId: string; version: string; fromNode: string; commandRoot: string; tip: TipRefresh;
   partial: PartialOutcome; gatePlan: { mode: 'not-run' | 'partial' | 'full'; reason?: string };
+  /** Most recent failed gate's autoheal recommendation, carried into the resume plan for the caller. */
+  autoheal?: GateAutohealDecision;
+  autohealReroute?: string;
   nextAttempt: number; priorCountKnown: boolean; preflight: 'passed';
 }
 
+function priorGateAutoheal(run: GraphRunState): GateAutohealDecision | undefined {
+  const gate = lastJsonObject(run.nodes.find((node) => node.nodeId === 'gate')?.output);
+  const value = gate?.autoheal;
+  if (!value || typeof value !== 'object' || Array.isArray(value)) return undefined;
+  const candidate = value as Partial<GateAutohealDecision>;
+  if (!['reroute-shard', 'retry-shard', 'partial-resume', 'host-switch', 'hitl'].includes(String(candidate.action))) return undefined;
+  if (!['shard-pod-death', 'timeout', 'host-down', 'interference', 'true-regression', 'environment'].includes(String(candidate.cause))) return undefined;
+  if (typeof candidate.signature !== 'string' || typeof candidate.reason !== 'string'
+    || typeof candidate.shardElapsedMs !== 'number' || typeof candidate.overallElapsedMs !== 'number') return undefined;
+  return candidate as GateAutohealDecision;
+}
+
+function automaticReroute(run: GraphRunState, autoheal: GateAutohealDecision | undefined, tipChanged: boolean, from: string): string | undefined {
+  if (!autoheal || tipChanged || from !== 'gate' || typeof (run.input as Record<string, unknown> | undefined)?.gatePodPool !== 'string'
+    || (autoheal.action !== 'retry-shard' && autoheal.action !== 'reroute-shard' && autoheal.action !== 'host-switch')) return undefined;
+  const key = autoheal.shardKey;
+  if (autoheal?.action === 'host-switch') return 'auto';
+  if (typeof key === 'string' && /^pod-\d+$/.test(key)) return key;
+  if (typeof key === 'string' && /^\d+$/.test(key)) return `pod-${key}`;
+  const gate = lastJsonObject(run.nodes.find((node) => node.nodeId === 'gate')?.output);
+  const stalled = gate?.stalledShards;
+  return Array.isArray(stalled) && stalled.length > 0 ? 'auto' : undefined;
+}
+
 /** CLI entry files: a production file that only names one (help text, a goal's command line) does not run it. */
 const CLI_ENTRIES = ['bin/elanous.mjs', 'src/index.ts'] as const;
 /** The gate's own test-file set (gate-node sweep: `git ls-files '*.test.*'` filtered by this). */
@@ -293,14 +326,111 @@ function verifyResumePreflight(run: GraphRunState, opts: ResumeReleaseOptions, d
   }
 }
 
-/** GATE-PARTIAL outcome of a resume — applied (plan written) or the reason it fell back to a full gate. `auto` marks the
- *  default plan of `--from gate` (GATE-REUSE-BY-DIFF); `closure` is set when code changes were followed. */
+/** GATE-PARTIAL outcome of a resume — applied (with its exact remeasure selection) or the reason it fell back to a full gate.
+ *  `auto` marks the default plan of `--from gate` (GATE-REUSE-BY-DIFF); `closure` is set when code changes were followed. */
 export type PartialOutcome = { requested: false }
-  | { requested: true; applied: true; files: number; priorCommit: string; auto?: true; closure?: { reached: number; opaque: number } }
+  | { requested: true; applied: true; files: number; plannedFiles: string[]; priorCommit: string; auto?: true; closure?: { reached: number; opaque: number } }
   | { requested: true; applied: false; reason: string; auto?: true };
 
 /** Build (or clear) the partial re-gate plan for the new tip. Fail-closed: any doubt leaves a full gate.
  *  GATE-REUSE-BY-DIFF: `--from gate` on a moved tip plans by default (`partial: undefined`); `partial: false` opts out. */
+type StalledGateEvidence = {
+  stalledShards: Array<{ shard: number; files: string[]; reason: string }>;
+  partialSummary: { pass: number; fail: number; errors: number; ran: number; files: number };
+  partialFailures: { failures: string[]; errors: string[] };
+  preservedShardCount: number;
+};
+
+/** Recover an interrupted Pod sweep from its atomic shard receipts. No receipt or identity doubt authorizes a partial gate. */
+function stalledGateEvidence(input: { version: string; commit: string; repo: string; roots: string[];
+  gateOutput?: Record<string, unknown>; expectedBase40?: string; expectedTests?: (commit: string) => string[] | null }):
+  { ok: true; evidence: StalledGateEvidence } | { ok: false; reason: string } {
+  const fail = (reason: string) => ({ ok: false as const, reason });
+  if (input.gateOutput && (input.gateOutput.outcome !== 'error' || input.gateOutput.commit !== input.commit))
+    return fail('saved gate result is not an error for the recorded cut');
+  if (!/^[a-f0-9]{40}$/i.test(input.expectedBase40 ?? ''))
+    return fail('saved cutoff baseline is absent or invalid');
+  const fingerprint = gateEnvFingerprint(process.env);
+  let receiptDir: string | undefined;
+  for (const root of input.roots) {
+    const candidate = join(root, 'release', input.version, 'gate-logs', 'cut', 'receipts');
+    if (existsSync(candidate)) { receiptDir = candidate; break; }
+  }
+  if (!receiptDir) return fail('stalled gate receipts are absent');
+  const names = readdirSync(receiptDir).filter((name) => /^pod-\d+\.receipt\.json$/.test(name)).sort((a, b) =>
+    Number(/^pod-(\d+)/.exec(a)![1]) - Number(/^pod-(\d+)/.exec(b)![1]));
+  if (!names.length || names.some((name, index) => Number(/^pod-(\d+)/.exec(name)![1]) !== index))
+    return fail('stalled gate receipt shard set is incomplete');
+  const allFiles: string[] = [];
+  const failures: string[] = [];
+  const errors: string[] = [];
+  const stalledShards: StalledGateEvidence['stalledShards'] = [];
+  const imageDigests = new Set<string>();
+  let pass = 0, failCount = 0, errorCount = 0, ran = 0, files = 0, preservedShardCount = 0;
+  let base40: string | undefined;
+  for (const name of names) {
+    const id = name.slice(0, -'.receipt.json'.length);
+    const receipt = readGateShardManifest(receiptDir, id);
+    if (!receipt || receipt.head40 !== input.commit || !/^[a-f0-9]{40}$/i.test(receipt.base40)
+      || !receipt.files.length
+      || receipt.files.some((file) => typeof file !== 'string'
+        || !/^(?:[\w.-]+\/)*[\w.-]+\.test\.(?:tsx?|jsx?|mts|cts)$/.test(file) || file.split('/').includes('..'))
+      || typeof receipt.envKey !== 'string' || typeof receipt.host !== 'string')
+      return fail(`stalled gate receipt identity or file list is invalid: ${id}`);
+    if (receipt.base40 !== input.expectedBase40)
+      return fail(`stalled gate receipt baseline differs from saved cutoff baseline: ${id}`);
+    if (base40 && receipt.base40 !== base40) return fail(`stalled gate receipts disagree on baseline: ${id}`);
+    base40 = receipt.base40;
+    allFiles.push(...receipt.files);
+    if (receipt.status === 'unmeasured') {
+      stalledShards.push({ shard: Number(id.slice(4)), files: [...receipt.files], reason: 'incomplete' });
+      continue;
+    }
+    if (!receipt.envKey.startsWith(`${fingerprint}:`)) return fail(`stalled gate receipt environment mismatch: ${id}`);
+    const imageDigest = receipt.envKey.slice(fingerprint.length + 1);
+    if (!/^sha256:[a-f0-9]{64}$/i.test(imageDigest)) return fail(`stalled gate receipt image identity is missing: ${id}`);
+    imageDigests.add(imageDigest.toLowerCase());
+    const result = readGateShardResult(receiptDir, { shard: { id, files: receipt.files }, head40: input.commit,
+      base40: receipt.base40, envKey: receipt.envKey });
+    if (!result) return fail(`stalled gate receipt payload is incomplete or corrupt: ${id}`);
+    const totals = result.runs.reduce((sum, run) => ({ pass: sum.pass + run.pass, fail: sum.fail + run.fail,
+      errors: sum.errors + run.errors, ran: sum.ran + run.ran, files: sum.files + run.files }),
+    { pass: 0, fail: 0, errors: 0, ran: 0, files: 0 });
+    if (!Object.values(totals).every(Number.isSafeInteger) || receipt.failureIds.length !== totals.fail
+      || receipt.errorIds.length !== totals.errors)
+      return fail(`stalled gate receipt counts do not reconcile: ${id}`);
+    pass += totals.pass; failCount += totals.fail; errorCount += totals.errors; ran += totals.ran; files += totals.files;
+    failures.push(...receipt.failureIds); errors.push(...receipt.errorIds); preservedShardCount++;
+  }
+  if (!stalledShards.length) return fail('no incomplete shard receipt identifies the interrupted work');
+  if (new Set(allFiles).size !== allFiles.length) return fail('stalled gate receipt file assignments overlap');
+  if (imageDigests.size !== 1) return fail('stalled gate completed shards do not share one executed image');
+  const expected = (input.expectedTests ?? ((commit) => gateAssignableTestsAtCommit(input.repo, commit)))(input.commit);
+  if (!expected || JSON.stringify([...expected].sort()) !== JSON.stringify([...allFiles].sort()))
+    return fail('stalled gate receipts do not cover the pinned test selection');
+  const partialSummary = { pass, fail: failCount, errors: errorCount, ran, files };
+  const partialFailures = { failures: [...new Set(failures)].sort(), errors: [...new Set(errors)].sort() };
+  const gate = input.gateOutput;
+  if (gate) {
+    const savedSummary = gate.partialSummary as Partial<StalledGateEvidence['partialSummary']> | undefined;
+    const savedFailures = gate.partialFailures as Partial<StalledGateEvidence['partialFailures']> | undefined;
+    if (savedSummary && JSON.stringify(savedSummary) !== JSON.stringify(partialSummary))
+      return fail('saved gate summary disagrees with shard receipts');
+    if (savedFailures && (JSON.stringify([...((savedFailures.failures as string[] | undefined) ?? [])].sort()) !== JSON.stringify(partialFailures.failures)
+      || JSON.stringify([...((savedFailures.errors as string[] | undefined) ?? [])].sort()) !== JSON.stringify(partialFailures.errors)))
+      return fail('saved gate failures disagree with shard receipts');
+    if (gate.preservedShardCount !== undefined && gate.preservedShardCount !== preservedShardCount)
+      return fail('saved preserved shard count disagrees with shard receipts');
+    if (Array.isArray(gate.stalledShards)) {
+      const savedStalled = gate.stalledShards.flatMap((item) => item && typeof item === 'object'
+        && Array.isArray((item as { files?: unknown }).files) ? (item as { files: string[] }).files : []).sort();
+      if (JSON.stringify(savedStalled) !== JSON.stringify(stalledShards.flatMap((item) => item.files).sort()))
+        return fail('saved stalled shard list disagrees with shard receipts');
+    }
+  }
+  return { ok: true, evidence: { stalledShards, partialSummary, partialFailures, preservedShardCount } };
+}
+
 function preparePartial(tip: TipRefresh, opts: { from: string; partial?: boolean; gateExecutor?: string }, deps: ResumeReleaseDeps, dryRun = false): PartialOutcome {
   const root = deps.root ?? effectiveInstanceRoot();
   // Any resume starts from no plan: one without a partial plan (from whatever node) never inherits an older plan.
@@ -326,6 +456,42 @@ function preparePartial(tip: TipRefresh, opts: { from: string; partial?: boolean
     const saved = JSON.parse(readFileSync(tip.statePath, 'utf8')) as GraphRunState;
     const gateRecord = saved.nodes.find((node) => node.nodeId === 'gate');
     const previousGate = gateRecord ? lastJsonObject(gateRecord.output) as Record<string, unknown> | undefined : undefined;
+    // `loadFlaky` is also a partial-resume cause, but it is emitted separately from
+    // shard-only interference. Carry both sets into the same affected-file recheck.
+    const interference = [...new Set([
+      ...(Array.isArray(previousGate?.interferenceErrors) ? previousGate.interferenceErrors : []),
+      ...(Array.isArray(previousGate?.loadFlaky) ? previousGate.loadFlaky : []),
+    ])];
+    if (previousGate?.outcome === 'error' && previousGate.commit === tip.to && previousGate.autoheal
+      && typeof previousGate.autoheal === 'object' && (previousGate.autoheal as { action?: unknown }).action === 'partial-resume') {
+      if (!Array.isArray(interference) || !interference.length || !interference.every((id) => typeof id === 'string'))
+        return fallback('partial-resume has no complete interference error record');
+      let closure: ClosureTree;
+      try { closure = (deps.closureTree ?? ((commit: string) => gitClosureTree(deps.repo ?? process.cwd(), commit)))(tip.to); }
+      catch (error) { return fallback(`tip tree unreadable: ${String(error instanceof Error ? error.message : error).slice(0, 200)}`); }
+      const tracked = new Set(closure.files);
+      const planned = planSameCommitRecheck({ version: tip.version, measuredCommit: tip.to, forCommit: tip.to,
+        failures: [], errors: interference as string[], existsAtCommit: (file) => tracked.has(file) });
+      if (!planned.ok) return fallback(planned.reason);
+      // The gate consumer only executes same-cut partial work for the stalled-only recheck shape.
+      // Interference files are likewise unmeasured, so mark exactly this set as the recheck boundary.
+      const partialFailures = previousGate.partialFailures as { failures?: unknown; errors?: unknown } | undefined;
+      if (!Array.isArray(partialFailures?.failures) || !partialFailures.failures.every((id) => typeof id === 'string')
+        || !Array.isArray(partialFailures.errors) || !partialFailures.errors.every((id) => typeof id === 'string'))
+        return fallback('partial-resume has no complete prior failure record');
+      let priorFailures: string[];
+      let priorErrors: string[];
+      try {
+        const affected = new Set(planned.plan.files);
+        priorFailures = (partialFailures.failures as string[]).filter((id) => !affected.has(testFileOfId(id)));
+        priorErrors = (partialFailures.errors as string[]).filter((id) => !affected.has(testFileOfId(id)));
+      } catch { return fallback('partial-resume prior failure record is malformed'); }
+      const recheckPlan = { ...planned.plan, priorFailures, priorErrors, stalledFiles: planned.plan.files };
+      const path = dryRun ? undefined : writeRecheckPlan(root, recheckPlan);
+      debug.log('release-loop.resume', 'interference-recheck-planned', { version: tip.version, commit: tip.to,
+        remeasureFileCount: recheckPlan.files.length, fallbackReason: null, ...(path ? { path } : {}), dryRun });
+      return { requested: true, applied: true, files: recheckPlan.files.length, priorCommit: tip.to, ...(auto ? { auto: true as const } : {}) };
+    }
     const stalled = previousGate?.stalledShards;
     const summary = previousGate?.partialSummary as { pass?: unknown; fail?: unknown; errors?: unknown; ran?: unknown; files?: unknown } | undefined;
     const measured = previousGate?.partialFailures as { failures?: unknown; errors?: unknown } | undefined;
@@ -356,9 +522,11 @@ function preparePartial(tip: TipRefresh, opts: { from: string; partial?: boolean
     debug.log('release-loop.resume', 'stalled-recheck-planned', { version: tip.version, commit: tip.to,
       planKind: planned.plan.kind, remeasureFileCount: planned.plan.files.length,
       preservedShardCount: planned.plan.preservedShardCount ?? 0, fallbackReason: null, path });
-    return { requested: true, applied: true, files: planned.plan.files.length, priorCommit: tip.to, ...(auto ? { auto: true as const } : {}) };
+    return { requested: true, applied: true, files: planned.plan.files.length, plannedFiles: [...planned.plan.files], priorCommit: tip.to,
+      ...(auto ? { auto: true as const } : {}) };
   }
   let prior: { commit?: unknown; failures?: unknown; errors?: unknown } | undefined;
+  let movedStalledEvidence: StalledGateEvidence | undefined;
   for (const location of [...new Set([deps.ledgerRoot ?? releaseLedgerRoot(), root])]) {
     const path = join(location, 'release', tip.version, 'gate-failures.json');
     if (!existsSync(path)) continue;
@@ -368,7 +536,19 @@ function preparePartial(tip: TipRefresh, opts: { from: string; partial?: boolean
     if (record?.commit === tip.from) { prior = record; break; }
   }
   if (!prior || !Array.isArray(prior.failures) || !(prior.errors === undefined || Array.isArray(prior.errors))) {
-    return fallback(`no complete gate record for ${tip.from.slice(0, 12)} (a stalled or errored gate keeps none)`);
+    const gateRecord = saved.nodes.find((node) => node.nodeId === 'gate');
+    const gateOutput = gateRecord ? lastJsonObject(gateRecord.output) as Record<string, unknown> | undefined : undefined;
+    const cutoffRecord = saved.nodes.find((node) => node.nodeId === 'cutoff');
+    const cutoffOutput = cutoffRecord ? lastJsonObject(cutoffRecord.output) as Record<string, unknown> | undefined : undefined;
+    const expectedBase40 = (cutoffOutput?.baseline as { sha?: unknown } | undefined)?.sha;
+    const evidence = stalledGateEvidence({ version: tip.version, commit: tip.from, repo: deps.repo ?? process.cwd(),
+      roots: [...new Set([deps.ledgerRoot ?? releaseLedgerRoot(), root])], gateOutput,
+      ...(typeof expectedBase40 === 'string' ? { expectedBase40 } : {}),
+      expectedTests: deps.gateTestsAtCommit });
+    if (!evidence.ok) return fallback(`no complete gate record for ${tip.from.slice(0, 12)}; ${evidence.reason}`);
+    movedStalledEvidence = evidence.evidence;
+    prior = { commit: tip.from, failures: movedStalledEvidence.partialFailures.failures,
+      errors: movedStalledEvidence.partialFailures.errors };
   }
   // Both sides of a rename count (`--no-renames`): a source moved to a test/Markdown name is still a source change.
   let changed: string[];
@@ -404,10 +584,11 @@ function preparePartial(tip: TipRefresh, opts: { from: string; partial?: boolean
   const savedInput = saved.input as { gateExecutor?: unknown; gatePodPool?: unknown };
   const podGate = (opts.gateExecutor ?? savedInput.gateExecutor ?? (savedInput.gatePodPool ? 'pod' : undefined)) === 'pod';
   const priorShards = priorCutLogs ? collectExecutedPodShardImages(priorCutLogs, tip.from) : undefined;
-  const planned = planPartialRegate({ version: tip.version, priorCommit: tip.from, forCommit: tip.to, changedFiles: changed,
+  const stalledFiles = movedStalledEvidence?.stalledShards.flatMap((shard) => shard.files) ?? [];
+  const planned = planPartialRegate({ version: tip.version, priorCommit: tip.from, forCommit: tip.to, changedFiles: [...changed, ...stalledFiles],
     priorFailures: prior.failures as string[], priorErrors: (prior.errors ?? []) as string[],
     existsAtTip: (file) => tracked.has(file),
-    ...(podGate ? { priorShards: priorShards ?? [] } : {}),
+    ...((podGate || movedStalledEvidence) ? { priorShards: priorShards ?? [] } : {}),
     closure: (changedCode) => testsReachingChanged(tree, changedCode, tests, { cliEntries: CLI_ENTRIES }) });
   if (!planned.ok) return fallback(planned.reason);
   if (planned.plan.files.length >= tests.length) return fallback(`every test is re-run (${tests.length}) — full gate`);
@@ -427,7 +608,7 @@ function preparePartial(tip: TipRefresh, opts: { from: string; partial?: boolean
   const path = dryRun ? undefined : writePartialPlan(root, planned.plan);
   debug.log('release-loop.resume', 'partial-planned', { version: tip.version, priorCommit: tip.from, commit: tip.to, files: planned.plan.files.length,
     tests: tests.length, auto, ...(path ? { path } : {}), dryRun, ...(planned.closure ? { reached: planned.closure.reached, opaque: planned.closure.opaque } : {}) });
-  return { requested: true, applied: true, files: planned.plan.files.length, priorCommit: tip.from,
+  return { requested: true, applied: true, files: planned.plan.files.length, plannedFiles: [...planned.plan.files], priorCommit: tip.from,
     ...(auto ? { auto: true as const } : {}), ...(planned.closure ? { closure: planned.closure } : {}) };
 }
 
@@ -483,19 +664,25 @@ async function planResumeReleaseRunLocked(opts: ResumeReleaseOptions, deps: Resu
     waiverCommit = recorded;
   }
   const tip = refreshReleaseBranchTipInternal(opts.runId, deps, opts.from, waiverCommit, true);
+  const fromIndex = savedRun.path.indexOf(opts.from);
+  const gateIndex = savedRun.path.indexOf('gate');
+  const gateWillRun = fromIndex >= 0 && gateIndex >= fromIndex;
   if (opts.rerouteShard !== undefined && tip.changed) throw new Error('shard reroute requires the original cut head; moved tip needs a full re-gate');
   verifyResumeRecipePin(opts.commandRoot ?? process.cwd(), savedRun.graphSnapshot, tip.version);
   const commandRoot = verifyReleaseCommandRoot(opts.commandRoot ?? process.cwd(), tip.to, deps.git);
   verifyResumePreflight(savedRun, opts, deps, tip.version, tip.from, tip.to, commandRoot);
-  const partial = preparePartial(tip, opts.rerouteShard !== undefined ? { ...opts, partial: false } : opts, deps, true);
-  const fromIndex = savedRun.path.indexOf(opts.from);
-  const gateIndex = savedRun.path.indexOf('gate');
-  const gateWillRun = fromIndex >= 0 && gateIndex >= fromIndex;
+  const autoheal = gateWillRun ? priorGateAutoheal(savedRun) : undefined;
+  const autoReroute = opts.rerouteShard === undefined ? automaticReroute(savedRun, autoheal, tip.changed, opts.from) : undefined;
+  const automaticShardReroute = autoReroute ? releaseResumeGateShardReroute(savedRun, { from: opts.from, shard: autoReroute }) : undefined;
+  const effectiveOpts = automaticShardReroute ? { ...opts, rerouteShard: automaticShardReroute.set.gateRerouteShard } : opts;
+  const partial = preparePartial(tip, effectiveOpts.rerouteShard !== undefined ? { ...effectiveOpts, partial: false } : effectiveOpts, deps, true);
   const gatePlan = !gateWillRun ? { mode: 'not-run' as const }
     : partial.requested && partial.applied ? { mode: 'partial' as const }
       : { mode: 'full' as const, reason: partial.requested ? partial.reason : opts.partial === false ? 'partial recheck disabled' : 'no partial plan applies' };
+  if (autoheal) debug.log('release-loop.resume', 'gate-autoheal-plan', { runId: opts.runId, version: tip.version, appliedReroute: automaticShardReroute?.set.gateRerouteShard ?? null, ...autoheal });
   return { runId: opts.runId, version: tip.version, fromNode: opts.from, commandRoot, tip, partial,
-    gatePlan, nextAttempt: attempt.attempt, priorCountKnown: attempt.priorCountKnown, preflight: 'passed' };
+    gatePlan, ...(autoheal ? { autoheal } : {}), ...(automaticShardReroute ? { autohealReroute: automaticShardReroute.set.gateRerouteShard } : {}),
+    nextAttempt: attempt.attempt, priorCountKnown: attempt.priorCountKnown, preflight: 'passed' };
 }
 
 async function resumeReleaseRunLocked(opts: ResumeReleaseOptions, deps: ResumeReleaseDeps): Promise<{ tip: TipRefresh; partial: PartialOutcome; state: GraphRunState }> {
@@ -550,7 +737,12 @@ async function resumeReleaseRunLocked(opts: ResumeReleaseOptions, deps: ResumeRe
   const commandRoot = verifyReleaseCommandRoot(opts.commandRoot ?? process.cwd(), previewTip.to, deps.git);
   verifyResumePreflight(savedRun, opts, deps, previewTip.version, previewTip.from, previewTip.to, commandRoot);
   const tip = refreshReleaseBranchTipInternal(opts.runId, deps, opts.from, waiverCommit, false, previewTip.to);
-  const partial = preparePartial(tip, shardReroute ? { ...opts, partial: false } : opts, deps);
+  const autoheal = priorGateAutoheal(savedRun);
+  const autoReroute = !shardReroute ? automaticReroute(savedRun, autoheal, tip.changed, opts.from) : undefined;
+  const automaticShardReroute = autoReroute ? releaseResumeGateShardReroute(savedRun, { from: opts.from, shard: autoReroute }) : undefined;
+  const partial = preparePartial(tip, shardReroute || automaticShardReroute
+    ? { ...opts, rerouteShard: (shardReroute ?? automaticShardReroute)!.set.gateRerouteShard, partial: false } : opts, deps);
+  if (autoheal) debug.log('release-loop.resume', 'gate-autoheal-plan', { runId: opts.runId, version: tip.version, appliedReroute: automaticShardReroute?.set.gateRerouteShard ?? null, ...autoheal });
   // A failed preflight does not consume an observed release resume attempt.
   const attempt = reserveReleaseResumeAttempt(savedRun, deps.root!, { now: deps.now });
   debug.log('release-loop.resume', 'attempt', { runId: opts.runId, attempt: attempt.attempt,
@@ -565,8 +757,8 @@ async function resumeReleaseRunLocked(opts: ResumeReleaseOptions, deps: ResumeRe
     ...(acceptedRegressions ? { releaseAcceptedRegressions: { requested: opts.acceptedRegressions!, acceptedRegressions } } : {}),
     ...(gateExecutor ? { releaseGateExecutor: { executor: gateExecutor.set.gateExecutor, ...(opts.gateHost !== undefined ? { host: opts.gateHost } : {}) } } : {}),
     ...(gatePodPool ? { releaseGatePodPool: gatePodPool.set.gatePodPool } : {}),
-    ...(shardReroute ? { releaseGateShardReroute: { shard: shardReroute.set.gateRerouteShard,
-      ...(shardReroute.set.gateShardHost ? { host: shardReroute.set.gateShardHost } : {}),
+    ...(shardReroute || automaticShardReroute ? { releaseGateShardReroute: { shard: (shardReroute ?? automaticShardReroute)!.set.gateRerouteShard,
+      ...((shardReroute ?? automaticShardReroute)!.set.gateShardHost ? { host: (shardReroute ?? automaticShardReroute)!.set.gateShardHost } : {}),
       ...(opts.shardOs ? { os: opts.shardOs } : {}) } } : {}),
     deps: { root: deps.root, ...(deps.runBash ? { runBash: deps.runBash } : {}) } });
   return { tip, partial, state };
@@ -604,6 +796,23 @@ export interface RecoverReleaseResult {
   resume?: { status: string; tip: TipRefresh; partial: PartialOutcome };
 }
 
+export interface RecoverReleaseLandingOptions {
+  runId: string;
+  /** The landed repair commit on main; this is the idempotency key for a landing notification. */
+  mergeSha: string;
+  from?: string;
+  commandRoot?: string;
+  partial?: boolean;
+  repoRoot?: string;
+}
+
+export type RecoverReleaseLandingResult = RecoverReleaseResult & { duplicate: boolean };
+
+/** Commander represents `--no-partial` as false and its default (flag omitted) as true. */
+export function recoverLandingPartialOption(optionValue?: boolean): boolean | undefined {
+  return optionValue === false ? false : undefined;
+}
+
 export interface RecoverReleaseDeps extends ResumeReleaseDeps {
   /** `release cut-branch --append` — tests record the plan instead of pushing the branch. */
   cutBranch?: typeof cutReleaseBranch;
@@ -619,11 +828,19 @@ export class ReleaseRecoverStopped extends Error {
   }
 }
 
+function readRecoverJson(path: string, label: string): unknown {
+  try { return JSON.parse(readFileSync(path, 'utf8')) as unknown; }
+  catch (error) {
+    const detail = error instanceof Error ? error.message : String(error);
+    throw new Error(`release ${label} JSON을 읽을 수 없다: ${path} (${detail}) — 파일을 보존하고 원장을 올바른 JSON으로 복구한 뒤 재시도하라.`);
+  }
+}
+
 /** Append one stage result to the resume-attempts ledger. `attempts[]` keeps its shape — the stage results are added. */
 function recordRecoverStage(entry: { root: string; runId: string; version: string; priorCountKnown: boolean; stage: RecoverStageRecord }): void {
   const path = join(entry.root, 'release', entry.version, 'resume-attempts', `${entry.runId}.json`);
   const saved = existsSync(path)
-    ? JSON.parse(readFileSync(path, 'utf8')) as { attempts?: unknown; recoveries?: unknown; priorCountKnown?: unknown }
+    ? readRecoverJson(path, 'resume attempts 원장') as { attempts?: unknown; recoveries?: unknown; priorCountKnown?: unknown }
     : undefined;
   if (saved !== undefined && (!saved || typeof saved !== 'object' || Array.isArray(saved) || !Array.isArray(saved.attempts)
     || (saved.recoveries !== undefined && !Array.isArray(saved.recoveries)))) throw new Error('release resume attempts ledger invalid');
@@ -637,16 +854,39 @@ function recordRecoverStage(entry: { root: string; runId: string; version: strin
   renameSync(temporary, path);
 }
 
+async function acquireReleaseRecoveryLock(root: string, runId: string) {
+  const lockDir = join(root, 'release', 'recovery-locks');
+  mkdirSync(lockDir, { recursive: true });
+  // A full-gate subprocess can block this process's event loop, delaying the async lock heartbeat.
+  // The observed release gate reached 2h20m; leave a finite four-hour stale/wait window for one recovery.
+  const lock = await acquireLockAsync(join(lockDir, `${runId}.lock`), {
+    staleMs: 4 * 60 * 60_000, retryBusyMs: 1_000, maxTries: 14_400,
+  });
+  if (!lock.stillHeld()) {
+    lock.release();
+    throw new Error(`release recovery lock was lost for ${runId} — retry after inspecting the recovery ledger`);
+  }
+  return lock;
+}
+
 /**
  * Repair a failed release run in one command: append dry-run (plan only) → append (fast-forward release/<v> onto the
  * picked repairs) → resume --from export-check (preflight plan, then the graph). A failing stage stops the command:
  * the later stages never run, and the stage results are already in the ledger for the human to read.
  */
 export async function recoverReleaseRun(opts: RecoverReleaseOptions, deps: RecoverReleaseDeps = {}): Promise<RecoverReleaseResult> {
-  if (!Array.isArray(opts.pick) || !opts.pick.length) throw new Error('release recover requires at least one --pick commit');
   const root = releaseRunUniverse({ root: deps.root });
+  const runId = safeRunId(opts.runId);
+  const lock = await acquireReleaseRecoveryLock(root, runId);
+  try {
+    return await recoverReleaseRunLocked(opts, deps, root);
+  } finally { lock.release(); }
+}
+
+async function recoverReleaseRunLocked(opts: RecoverReleaseOptions, deps: RecoverReleaseDeps, root: string): Promise<RecoverReleaseResult> {
+  if (!Array.isArray(opts.pick) || !opts.pick.length) throw new Error('release recover requires at least one --pick commit');
   const statePath = join(root, 'graph-runs', 'release-loop', `${safeRunId(opts.runId)}.json`);
-  const savedRun = JSON.parse(readFileSync(statePath, 'utf8')) as GraphRunState;
+  const savedRun = readRecoverJson(statePath, 'release run 상태') as GraphRunState;
   if (savedRun.graphId !== 'release-loop' || savedRun.runId !== opts.runId) throw new Error(`run identity mismatch: ${statePath}`);
   // The saved version-release record names what to repair: the release branch and the commit the run cut.
   const versionRecord = savedRun.nodes.find((node) => node.nodeId === 'version-release');
@@ -707,8 +947,9 @@ export async function recoverReleaseRun(opts: RecoverReleaseOptions, deps: Recov
   const now = (): string => (deps.now?.() ?? new Date()).toISOString();
   const stages: RecoverStageRecord[] = [];
   const record = (stage: RecoverStageRecord): void => {
-    stages.push(stage);
-    recordRecoverStage({ root, runId: opts.runId, version, priorCountKnown: !savedRun.resume, stage });
+    const withPick = { ...stage, pick: stage.pick ?? [...opts.pick] };
+    stages.push(withPick);
+    recordRecoverStage({ root, runId: opts.runId, version, priorCountKnown: !savedRun.resume, stage: withPick });
   };
   const stop = (stage: RecoverStageName, error: unknown): never => {
     throw new ReleaseRecoverStopped(stage, stages, RECOVER_STAGES.slice(RECOVER_STAGES.indexOf(stage) + 1),
@@ -733,7 +974,7 @@ export async function recoverReleaseRun(opts: RecoverReleaseOptions, deps: Recov
   };
   const ledgerPath = join(root, 'release', version, 'resume-attempts', `${opts.runId}.json`);
   const recoveries: RecoverStageRecord[] = existsSync(ledgerPath)
-    ? ((JSON.parse(readFileSync(ledgerPath, 'utf8')) as { recoveries?: unknown[] }).recoveries ?? []) as RecoverStageRecord[]
+    ? ((readRecoverJson(ledgerPath, 'resume attempts 원장') as { recoveries?: unknown[] }).recoveries ?? []) as RecoverStageRecord[]
     : [];
   let appendedCommit: string | undefined;
   for (let index = recoveries.length - 1; index >= 0; index -= 1) {
@@ -843,3 +1084,69 @@ export async function recoverReleaseRun(opts: RecoverReleaseOptions, deps: Recov
     }
   }
 }
+
+/**
+ * Manual landing-event entrypoint. The event's merge SHA is the dedupe key; once that repair has completed its
+ * resume stage, a redelivered event returns the recorded result without appending or running the graph again.
+ * The event producer can call this after its own hook lands; this function does not subscribe to PR-land events.
+ */
+export async function recoverReleaseRunFromLanding(opts: RecoverReleaseLandingOptions,
+  deps: RecoverReleaseDeps = {}): Promise<RecoverReleaseLandingResult> {
+  if (!/^[0-9a-f]{40}$/i.test(opts.mergeSha)) throw new Error('release recover-landing requires a full 40-hex merge SHA');
+  const root = releaseRunUniverse({ root: deps.root });
+  const runId = safeRunId(opts.runId);
+  // Share one run-scoped lock with manual `release recover`: the dedupe read, append, and resume must not race.
+  const landingLock = await acquireReleaseRecoveryLock(root, runId);
+  try {
+    return await recoverReleaseRunFromLandingLocked(opts, deps, root, runId);
+  } finally { landingLock.release(); }
+}
+
+async function recoverReleaseRunFromLandingLocked(opts: RecoverReleaseLandingOptions, deps: RecoverReleaseDeps,
+  root: string, runId: string): Promise<RecoverReleaseLandingResult> {
+  const pick = [opts.mergeSha.toLowerCase()];
+  const statePath = join(root, 'graph-runs', 'release-loop', `${runId}.json`);
+  if (existsSync(statePath)) {
+    const savedRun = readRecoverJson(statePath, 'release run 상태') as GraphRunState;
+    const versionNode = savedRun.nodes.find((node) => node.nodeId === 'version-release');
+    const recorded = lastJsonObject(versionNode?.output);
+    const version = typeof recorded?.version === 'string' ? recorded.version : '';
+    if (savedRun.graphId === 'release-loop' && savedRun.runId === runId && isReleaseVersion(version)) {
+      const ledgerPath = join(root, 'release', version, 'resume-attempts', `${runId}.json`);
+      if (existsSync(ledgerPath)) {
+        const ledger = readRecoverJson(ledgerPath, 'resume attempts 원장') as { recoveries?: unknown };
+        if (ledger.recoveries !== undefined && !Array.isArray(ledger.recoveries)) throw new Error(`release resume attempts ledger invalid: ${ledgerPath}`);
+        const records = (ledger.recoveries ?? []) as RecoverStageRecord[];
+        const matchesPick = (entry: RecoverStageRecord): boolean => Array.isArray(entry.pick)
+          && entry.pick.some((sha) => typeof sha === 'string' && sha.toLowerCase() === pick[0]);
+        let completedIndex = -1;
+        for (let index = records.length - 1; index >= 0; index -= 1) {
+          const entry = records[index]!;
+          if (matchesPick(entry) && entry.stage === 'resume' && entry.outcome === 'ok') {
+            completedIndex = index;
+            break;
+          }
+        }
+        if (completedIndex >= 0) {
+          let cycleStart = completedIndex;
+          for (let index = completedIndex; index >= 0; index -= 1) {
+            const entry = records[index]!;
+            if (matchesPick(entry) && entry.stage === 'append-dry-run') {
+              cycleStart = index;
+              break;
+            }
+          }
+          const completed = records[completedIndex]!;
+          return { runId, version, branch: `release/${version}`, fromNode: completed.from ?? opts.from ?? RECOVER_FIRST_NODE,
+            stages: records.slice(cycleStart, completedIndex + 1).filter(matchesPick), duplicate: true };
+        }
+      }
+    }
+  }
+  const result = await recoverReleaseRunLocked({ runId, pick,
+    ...(opts.from === undefined ? {} : { from: opts.from }),
+    ...(opts.commandRoot === undefined ? {} : { commandRoot: opts.commandRoot }),
+    ...(opts.partial === undefined ? {} : { partial: opts.partial }),
+    ...(opts.repoRoot === undefined ? {} : { repoRoot: opts.repoRoot }) }, deps, root);
+  return { ...result, duplicate: false };
+}

package/scripts/release-loop/traffic-node.ts

@@ -42,7 +42,10 @@ export async function runTrafficNode(node = process.argv[2], env: NodeJS.Process
       return finishNode(node!, version, { outcome: 'fail', verdict: 'fail', summary });
     }
     const result = await applyReleaseTrafficEvent(context, event, options);
-    if (!result.applied) {
+    // Replaying the terminal restore after this run already restored its pre-cut traffic is a successful no-op.
+    const alreadyRestored = event === 'failed' && result.reason === 'already-restored';
+    const applied = result.applied || alreadyRestored;
+    if (!applied) {
       const consequence = event === 'failed' && result.reason === 'published-keeps-full'
         ? 'npm 발행은 완료됐으므로 전체 트래픽을 유지합니다. 후속 발행 단계 실패를 확인하세요.'
         : event === 'verify-complete' || event === 'npm-published'
@@ -52,8 +55,8 @@ export async function runTrafficNode(node = process.argv[2], env: NodeJS.Process
               : 'gate는 성공했으나 트래픽 감속이 적용되지 않았습니다.';
       (deps.report ?? reportFailure)(`RELEASE-POSTPUB-TRAFFIC ${version} ${runId || 'unknown'} ${node} 무시 — ${result.reason ?? 'traffic state unchanged'}. ${consequence}`);
     }
-    return finishNode(node!, version, { outcome: result.applied ? 'ok' : 'fail', verdict: result.applied ? 'pass' : 'fail',
-      summary: `${event} ${result.applied ? 'applied' : result.reason ?? 'ignored'}` });
+    return finishNode(node!, version, { outcome: applied ? 'ok' : 'fail', verdict: applied ? 'pass' : 'fail',
+      summary: `${event} ${alreadyRestored ? 'already-restored' : applied ? 'applied' : result.reason ?? 'ignored'}` });
   } catch (error) {
     const detail = errorResult(error).summary;
     const isLockTimeout = detail.startsWith('traffic lock timeout:');

package/scripts/release-loop/unattended-release.ts

@@ -49,6 +49,7 @@ export interface ReleaseLoopConfig {
 export interface ReleaseRunInput extends ReleaseLoopConfig {
   version: string;
   previousVersion: string;
+  triggerEvent?: { id: string; type: 'release.cut.due'; version: string; cutAt: string; at: string };
   cutCommit?: string;
   /** Required for the pod executor (the default); a host-executor run may have none. */
   gatePodPool?: string;
@@ -143,7 +144,7 @@ export function buildReleaseRunInput(version: string, deps: UnattendedReleaseDep
 
 /** Fail closed at the entry boundary, before any graph node can change a release. */
 export async function runUnattendedRelease(
-  opts: { version: string; dryRun?: boolean; cutCommit?: string; forceFreeze?: boolean; forceNew?: boolean; mainCut?: boolean; prerelease?: PrereleaseKind; publishAt?: string; gateExecutor?: string; gateHost?: string },
+  opts: { version: string; dryRun?: boolean; cutCommit?: string; forceFreeze?: boolean; forceNew?: boolean; mainCut?: boolean; prerelease?: PrereleaseKind; publishAt?: string; gateExecutor?: string; gateHost?: string; triggerEvent?: ReleaseRunInput['triggerEvent'] },
   deps: UnattendedReleaseDeps = {},
 ): Promise<{ input: ReleaseRunInput; dryRun: boolean; state?: GraphRunState }> {
   // A dry run only builds the input and mutates nothing, so it neither creates nor waits on the OP-seat lock.
@@ -154,7 +155,7 @@ export async function runUnattendedRelease(
 }
 
 async function runUnattendedReleaseLocked(
-  opts: { version: string; dryRun?: boolean; cutCommit?: string; forceFreeze?: boolean; forceNew?: boolean; mainCut?: boolean; prerelease?: PrereleaseKind; publishAt?: string; gateExecutor?: string; gateHost?: string },
+  opts: { version: string; dryRun?: boolean; cutCommit?: string; forceFreeze?: boolean; forceNew?: boolean; mainCut?: boolean; prerelease?: PrereleaseKind; publishAt?: string; gateExecutor?: string; gateHost?: string; triggerEvent?: ReleaseRunInput['triggerEvent'] },
   deps: UnattendedReleaseDeps,
 ): Promise<{ input: ReleaseRunInput; dryRun: boolean; state?: GraphRunState }> {
   // RELEASE-BRANCH (10-06): the default cut is a release branch, so main keeps landing and the run no longer needs a
@@ -171,6 +172,7 @@ async function runUnattendedReleaseLocked(
   }
   const version = resolveRunVersion(opts.version, opts.prerelease, deps);
   const input = buildReleaseRunInput(version, deps, opts.cutCommit, { executor: opts.gateExecutor, host: opts.gateHost });
+  if (opts.triggerEvent) input.triggerEvent = opts.triggerEvent;
   if (opts.publishAt !== undefined) {
     const invalid = publishAtError(opts.publishAt);
     if (invalid) throw new Error(invalid);

package/scripts/shadow-authoring-probe.ts

@@ -0,0 +1,20 @@
+/**
+ * SHADOW-MODEL-ARMS(0.2.44 칸) · 저작 단계 그림자 «1팔» probe — 명시 실행 전용 스크립트.
+ *
+ * 저작(goal-author) 단계 그림자 1팔을 `planShadowArms` 일일 상한 안에서 «실제 호출»해
+ * 난이도·모델·예상/실제 비용·결과를 한 줄 원장 형식으로 stdout 에 출력한다.
+ * ⛔ 관측만 한다 — 라이브 LLM(Claude 구독) 호출 0 · 골/PR/착지 파일 쓰기 0.
+ * ⛔ 자동 런 훅과 다른 단계(implement·review·judge) 팔은 후속 조각이다.
+ *
+ * 사용(저장소 루트에서): bun scripts/shadow-authoring-probe.ts
+ */
+import { probeAuthoringShadowArm } from '../src/self-implement/goal-author.js';
+
+/** 명시 실행 probe — 생산 저작 probe 경로(probeAuthoringShadowArm ⊕ probeShadowArm ⊕ planShadowArms)를 그대로 지나 한 줄을 낸다. */
+export function runShadowAuthoringProbe(authorRunId = `shadow-authoring-probe-${new Date().toISOString().slice(0, 10)}`): string {
+  return probeAuthoringShadowArm(authorRunId).line;
+}
+
+if (import.meta.main) {
+  console.log(runShadowAuthoringProbe());
+}