elanous 0.2.34 → 0.2.36

package/scripts/release-loop/pwa-remote.ts

@@ -0,0 +1,166 @@
+// RELEASE-PWA-NODE-RESILIENCE ① (0.2.36 · 대표 10-20 «PWA 단계도 원격에서»): the release `pwa` node can run its
+// scenario suite on an SSH host (`release.loop.pwaRemote`, e.g. node-b) instead of the busy release machine.
+// 0.2.34 (10-20 18:50~19:13): at 1-min load 100+ on mbp the CDP attach went past 5s and T1b failed twice on reruns,
+// while the same commit alone passed — the release machine's load, not the PWA, decided the verdict.
+//
+// Same shape as the gate's host executor (`--gate-executor host` · `gateRemote` · `gateRemoteMirror`):
+//   ① ensure the bare mirror on the host  ② push the commit to `refs/elanous/pwa/<sha>`
+//   ③ one ssh call: clone the mirror at <sha> into a temp dir, install, run THAT commit's
+//      `pwa-scenarios-node.ts` locally there, print its JSON verdict, remove the temp dir.
+// ⛔ No silent fallback: when any step fails the node reports `error` with the stage and the host's own words
+// and exits 2. It never reruns the suite on the release machine.
+import { spawnSync } from 'node:child_process';
+import { debug } from '../../src/debug/log.js';
+
+/** `[user@]host`. A leading `-` is refused separately (ssh/git would read it as an option — `-oProxyCommand=…`). */
+export const PWA_REMOTE_HOST_PATTERN = /^(?:[\w.-]+@)?[\w.-]+$/;
+export const DEFAULT_PWA_REMOTE_MIRROR = '~/mirror/elanous-agent.git';
+/** Build (≤15m) + boot + warm-up + suite + reruns on the host; the local ssh deadline is only a backstop. */
+export const PWA_REMOTE_TIMEOUT_MS = 75 * 60_000;
+/** The host kills its own work first (shorter than the local backstop) so the temp clone and the ref are cleaned there. */
+export const PWA_REMOTE_LIMIT_SECONDS = 70 * 60;
+/** rc of a run the Perl supervisor stopped after its deadline. */
+const LIMIT_RCS = new Set([124]);
+
+export type PwaRemoteStage = 'input' | 'mirror' | 'push' | 'run' | 'timeout' | 'parse';
+
+export interface PwaRemoteOptions {
+  host: string;
+  commit: string;
+  withCostly: boolean;
+  reruns: number;
+  /** Bare mirror on the host (absolute or `~/…`). Default = the gate's mirror. */
+  mirror?: string;
+  /** Local repository that holds `commit` (pushed from here). */
+  repo: string;
+  timeoutMs?: number;
+}
+
+export interface PwaRemoteResult {
+  /** Exit code for the graph node: 0 pass/flaky · 1 fail · 2 could not run. */
+  exitCode: 0 | 1 | 2;
+  output: Record<string, unknown>;
+}
+
+interface Spawned { rc: number; stdout: string; stderr: string; error?: string }
+export type PwaRemoteSpawn = (cmd: string, args: string[], opts: { cwd: string; timeoutMs: number }) => Spawned;
+
+const defaultSpawn: PwaRemoteSpawn = (cmd, args, opts) => {
+  const env = Object.fromEntries(Object.entries(process.env).filter(([k]) => !/^(https?|all)_proxy$/i.test(k)));
+  const r = spawnSync(cmd, args, { cwd: opts.cwd, env, encoding: 'utf8', timeout: opts.timeoutMs, killSignal: 'SIGKILL', maxBuffer: 64 * 1024 * 1024 });
+  return { rc: r.status ?? 255, stdout: r.stdout ?? '', stderr: r.stderr ?? '', ...(r.error ? { error: String(r.error) } : {}) };
+};
+
+export const shQuote = (value: string): string => `'${value.replace(/'/g, `'\\''`)}'`;
+
+/** Validate the host and mirror before anything is spawned. */
+export function checkPwaRemote(host: unknown, mirror: unknown): { host: string; mirror: string } {
+  if (typeof host === 'string' && host.trim().startsWith('-')) throw new Error(`invalid pwa remote host: ${host.trim()} — a leading «-» would be read as an ssh/git option`);
+  if (typeof host !== 'string' || !PWA_REMOTE_HOST_PATTERN.test(host.trim())) throw new Error(`invalid pwa remote host: ${String(host)}`);
+  const m = mirror === undefined ? DEFAULT_PWA_REMOTE_MIRROR : mirror;
+  if (typeof m !== 'string' || !/^(?:\/|~\/)[\w./-]+$/.test(m) || m.split('/').includes('..')) throw new Error(`invalid pwa remote mirror path: ${String(m)}`);
+  return { host: host.trim(), mirror: m };
+}
+
+/** `~/x` stays unquoted at the front so the remote shell expands it; the rest is quoted. */
+function remotePath(path: string): string {
+  return path.startsWith('~/') ? `"$HOME"/${shQuote(path.slice(2))}` : shQuote(path);
+}
+
+/**
+ * The single remote script for step ③ — `sh -c` so the remote login shell (zsh on node-b) does not change its meaning.
+ * - Limit: Perl supervises a separate process group at PWA_REMOTE_LIMIT_SECONDS.
+ *   On timeout it kills the group, including the shell's children, then exits 124.
+ * - Cleanup: traps on EXIT · HUP · INT · TERM remove the temp clone and this run's mirror ref `refs/elanous/pwa/<sha>`
+ *   on success and failure alike — the ref only had to live until the clone fetched it, and deleting «our own» ref
+ *   (not a sweep of `refs/elanous/pwa/*`) cannot race another run on a different commit.
+ */
+export function remoteRunScript(opts: { commit: string; mirror: string; withCostly: boolean; reruns: number; limitSeconds?: number }): string {
+  const sha = shQuote(opts.commit);
+  const ref = shQuote(`refs/elanous/pwa/${opts.commit}`);
+  const limit = String(opts.limitSeconds ?? PWA_REMOTE_LIMIT_SECONDS);
+  const nodeArgs = ['--commit', opts.commit, '--json', '--reruns', String(opts.reruns), ...(opts.withCostly ? ['--with-costly'] : [])].map(shQuote).join(' ');
+  const body = [
+    `git clone --quiet --shared --no-checkout ${remotePath(opts.mirror)} "$1/repo" >&2 || exit 2`,
+    `cd "$1/repo" && git fetch --quiet origin ${ref} >&2 && git checkout --quiet --detach ${sha} >&2 || exit 2`,
+    'bun install --frozen-lockfile >&2 || exit 2',
+    '(cd apps/pwa && bun install --frozen-lockfile) >&2 || exit 2',
+    `exec bun scripts/release-loop/pwa-scenarios-node.ts ${nodeArgs}`,
+  ].join('\n');
+  // Arm the deadline only after the child confirms its new group exists. Otherwise a busy host
+  // can deliver SIGALRM between fork and setpgrp, making kill(-pid) miss every descendant.
+  const supervisor = [
+    'my $seconds = shift;',
+    'pipe(my $ready_r, my $ready_w) or exit 127;',
+    'my $pid = fork(); defined($pid) or exit 127;',
+    'if ($pid == 0) { close $ready_r; defined(setpgrp(0, 0)) or exit 127; syswrite($ready_w, "1") == 1 or exit 127; close $ready_w; exec @ARGV or exit 127; }',
+    'close $ready_w; my $ready = ""; unless (read($ready_r, $ready, 1) == 1) { kill 9, -$pid; kill 9, $pid; waitpid($pid, 0); exit 127; } close $ready_r;',
+    'local $SIG{ALRM} = sub { kill 9, -$pid; waitpid($pid, 0); exit 124; };',
+    'alarm $seconds; waitpid($pid, 0); alarm 0;',
+    'exit(($? & 127) ? 128 + ($? & 127) : ($? >> 8));',
+  ].join(' ');
+  return [
+    'PATH=$HOME/.bun/bin:/opt/homebrew/bin:/usr/local/bin:$PATH; export PATH',
+    'command -v perl >/dev/null 2>&1 || { echo "pwa remote requires perl" >&2; exit 127; }',
+    'D=$(mktemp -d /tmp/pwa-node-remote-XXXXXX) || exit 2',
+    `cleanup() { cd /tmp; rm -rf "$D"; git --git-dir=${remotePath(opts.mirror)} update-ref -d ${ref} 2>/dev/null; }`,
+    `trap 'cleanup' EXIT`,
+    `trap 'cleanup; trap - EXIT; exit 129' HUP`,
+    `trap 'cleanup; trap - EXIT; exit 130' INT`,
+    `trap 'cleanup; trap - EXIT; exit 143' TERM`,
+    `lim() { perl -e ${shQuote(supervisor)} ${limit} "$@"; }`,
+    `lim sh -c ${shQuote(body)} pwa-node "$D"`,
+  ].join('\n');
+}
+
+const COMMIT_40 = /^[0-9a-f]{40}$/;
+
+const tail = (text: string, n = 600): string => text.trim().slice(-n);
+
+function lastJson(stdout: string): Record<string, unknown> | undefined {
+  const line = stdout.split('\n').reverse().find((l) => l.trim().startsWith('{'));
+  if (!line) return undefined;
+  try {
+    const value: unknown = JSON.parse(line);
+    return value && typeof value === 'object' && !Array.isArray(value) ? value as Record<string, unknown> : undefined;
+  } catch { return undefined; }
+}
+
+/** Run the PWA scenario node on `host`. Never falls back to the local machine. */
+export function runPwaRemote(opts: PwaRemoteOptions, spawn: PwaRemoteSpawn = defaultSpawn): PwaRemoteResult {
+  const { host, mirror } = checkPwaRemote(opts.host, opts.mirror);
+  if (!COMMIT_40.test(opts.commit)) {
+    const error = `remote ${host} input refused: commit must be a full 40-hex sha (got ${JSON.stringify(opts.commit)}) — a short sha can name a different commit on the host mirror`;
+    debug.log('release-loop.pwa', 'remote-failed', { host, stage: 'input', rc: null, detail: error }, { level: 'warn' });
+    return { exitCode: 2, output: { verdict: 'error', commit: opts.commit, withCostly: opts.withCostly, executor: 'host', host, cells: [], error, remote: { stage: 'input', rc: null } } };
+  }
+  const timeoutMs = opts.timeoutMs ?? PWA_REMOTE_TIMEOUT_MS;
+  const base = { commit: opts.commit, withCostly: opts.withCostly, executor: 'host', host };
+  const started = Date.now();
+  debug.log('release-loop.pwa', 'remote-start', { host, mirror, commit: opts.commit, withCostly: opts.withCostly, reruns: opts.reruns });
+  const fail = (stage: PwaRemoteStage, run: Spawned | undefined, detail: string): PwaRemoteResult => {
+    const error = `remote ${host} ${stage} failed${run ? ` (rc=${run.rc})` : ''}: ${detail || 'no output'} — 원격 실패는 mbp 로 폴백하지 않는다`;
+    debug.log('release-loop.pwa', 'remote-failed', { host, stage, rc: run?.rc ?? null, detail: detail.slice(0, 600), elapsedMs: Date.now() - started }, { level: 'warn' });
+    return { exitCode: 2, output: { verdict: 'error', ...base, cells: [], error, remote: { stage, rc: run?.rc ?? null } } };
+  };
+
+  // ① mirror — a bare repo the commit can be pushed to (the gate uses the same one).
+  const ensure = `test -d ${remotePath(mirror)} || { mkdir -p "$(dirname ${remotePath(mirror)})" && git init --quiet --bare ${remotePath(mirror)}; }`;
+  const m = spawn('ssh', ['--', host, `sh -c ${shQuote(ensure)}`], { cwd: '/tmp', timeoutMs: 120_000 });
+  if (m.rc !== 0) return fail('mirror', m, tail(`${m.stderr}\n${m.error ?? ''}`));
+
+  // ② push the commit from the release machine.
+  const pushTarget = mirror.startsWith('~/') ? mirror.slice(2) : mirror;
+  const p = spawn('git', ['push', '--quiet', '--force', '--', `${host}:${pushTarget}`, `${opts.commit}:refs/elanous/pwa/${opts.commit}`], { cwd: opts.repo, timeoutMs: 600_000 });
+  if (p.rc !== 0) return fail('push', p, tail(`${p.stderr}\n${p.error ?? ''}`));
+
+  // ③ one call: clone · install · run the commit's own node there.
+  const script = remoteRunScript({ commit: opts.commit, mirror, withCostly: opts.withCostly, reruns: opts.reruns });
+  const r = spawn('ssh', ['--', host, `sh -c ${shQuote(script)}`], { cwd: '/tmp', timeoutMs });
+  const data = lastJson(r.stdout);
+  if (!data) return fail(r.rc === 0 ? 'parse' : LIMIT_RCS.has(r.rc) ? 'timeout' : 'run', r, tail(`${r.stderr}\n${r.error ?? ''}`));
+  const verdict = data.verdict;
+  const exitCode: 0 | 1 | 2 = verdict === 'pass' || verdict === 'flaky' ? (r.rc === 0 ? 0 : 2) : verdict === 'fail' ? 1 : 2;
+  debug.log('release-loop.pwa', 'remote-result', { host, verdict: typeof verdict === 'string' ? verdict : null, rc: r.rc, exitCode, elapsedMs: Date.now() - started });
+  return { exitCode, output: { ...data, ...base, ...(exitCode === 2 && typeof data.error !== 'string' ? { error: `remote ${host} run rc=${r.rc} verdict=${String(verdict)}: ${tail(r.stderr, 300)}` } : {}) } };
+}

package/scripts/release-loop/pwa-scenarios-node.ts

@@ -3,6 +3,10 @@
  * Release-loop ④ — run the PWA scenario suite on one commit, in isolation, and print one verdict.
  *
  *   bun scripts/release-loop/pwa-scenarios-node.ts --commit <sha> [--with-costly] [--reruns 2] [--json]
+ *     [--remote <ssh-host> [--remote-mirror <path>]]
+ *
+ * `--remote` (release.loop.pwaRemote) runs the same steps on that host — see pwa-remote.ts.
+ * A remote failure is reported as `error` (exit 2); it never falls back to this machine.
  *
  * Steps: temp worktree at <sha> → `nexus build` → isolated nexus (`--test`, leased test port)
  * → warm-up pass (discarded — the first page load after boot is slow and made T4a/N5a fail on the
@@ -18,14 +22,17 @@ import { tmpdir } from 'node:os';
 import { join, resolve } from 'node:path';
 import { decideVerdict, exitCodeFor, idsToRerun, type CellRun, type NodeVerdict } from './pwa-node-verdict';
 import { envLiteral } from '../../src/platform/env-literal.js';
+import { checkPwaRemote, runPwaRemote } from './pwa-remote';
 
-interface Options { commit: string; withCostly: boolean; reruns: number; json: boolean }
+interface Options { commit: string; withCostly: boolean; reruns: number; json: boolean; remote?: string; remoteMirror?: string }
 
-function parseArgs(argv: readonly string[]): Options {
+export function parseArgs(argv: readonly string[]): Options {
   const opts: Options = { commit: '', withCostly: false, reruns: 2, json: false };
   for (let i = 0; i < argv.length; i++) {
     const arg = argv[i];
     if (arg === '--commit') opts.commit = argv[++i] ?? '';
+    else if (arg === '--remote') opts.remote = argv[++i] ?? '';
+    else if (arg === '--remote-mirror') opts.remoteMirror = argv[++i] ?? '';
     else if (arg === '--with-costly') opts.withCostly = true;
     else if (arg === '--json') opts.json = true;
     else if (arg === '--reruns') opts.reruns = Number(argv[++i] ?? '2');
@@ -33,6 +40,8 @@ function parseArgs(argv: readonly string[]): Options {
   }
   if (!/^[0-9a-f]{7,40}$/i.test(opts.commit)) throw new Error('--commit <sha> is required');
   if (!Number.isInteger(opts.reruns) || opts.reruns < 0 || opts.reruns > 5) throw new Error('--reruns must be 0..5');
+  if (opts.remoteMirror !== undefined && opts.remote === undefined) throw new Error('--remote-mirror requires --remote');
+  if (opts.remote !== undefined) checkPwaRemote(opts.remote, opts.remoteMirror);
   return opts;
 }
 
@@ -65,6 +74,13 @@ async function main(): Promise<number> {
   let opts: Options;
   try { opts = parseArgs(process.argv.slice(2)); } catch (e) { log(String(e)); return 2; }
   const repo = resolve('.');
+  if (opts.remote !== undefined) {
+    log(`remote ${opts.remote} · ${opts.commit.slice(0, 10)}`);
+    const remote = runPwaRemote({ host: opts.remote, ...(opts.remoteMirror ? { mirror: opts.remoteMirror } : {}), commit: opts.commit, withCostly: opts.withCostly, reruns: opts.reruns, repo });
+    if (opts.json) console.log(JSON.stringify(remote.output));
+    else console.log(`pwa-node: ${String(remote.output.verdict)} · ${opts.commit.slice(0, 10)} · host ${opts.remote}${typeof remote.output.error === 'string' ? `\n  error: ${remote.output.error}` : ''}`);
+    return remote.exitCode;
+  }
   const work = mkdtempSync(join(tmpdir(), 'pwa-node-'));
   const worktree = join(work, 'tree');
   let nexusPid: number | undefined;
@@ -137,4 +153,4 @@ async function main(): Promise<number> {
   return verdict ? exitCodeFor(verdict.verdict) : 2;
 }
 
-process.exit(await main());
+if (import.meta.main) process.exit(await main());

package/scripts/release-loop/resume-cap.ts

@@ -1,56 +1,36 @@
 import { randomUUID } from 'node:crypto';
 import { existsSync, mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from 'node:fs';
 import { dirname, join } from 'node:path';
-import { DecisionLedger } from '../../src/decisions/decision-ledger.js';
 import type { GraphRunState } from '../../src/graph-runner/runner.js';
 import { isReleaseVersion } from './release-version.js';
 
-export const RELEASE_RESUME_LIMIT = 2;
-
 /** Called under the OP seat flock, before any branch-tip or graph-run mutation. */
 export function reserveReleaseResumeAttempt(state: GraphRunState, root: string, options: {
   now?: () => Date;
-  openCard?: (version: string, runId: string, reason: string) => string;
-  /** Under the OP flock, check capacity before tip preflight without consuming an attempt. */
+  /** Under the OP flock, preview the next observed count before tip preflight. */
   checkOnly?: boolean;
-  /** A dry-run reports the cap without creating an OP decision card. */
-  noDecision?: boolean;
-} = {}): { attempt: number; path: string } {
+} = {}): { attempt: number; path: string; priorCountKnown: boolean } {
   const version = (state.input as { version?: unknown } | undefined)?.version;
   if (typeof version !== 'string' || !isReleaseVersion(version) || !/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(state.runId))
-    throw new Error('release resume cap requires a valid saved version and run id');
+    throw new Error('release resume count requires a valid saved version and run id');
   const path = join(root, 'release', version, 'resume-attempts', `${state.runId}.json`);
-  const card = options.openCard ?? ((v: string, id: string, reason: string) => {
-    const ref = `release-resume-cap:${v}:${id}`;
-    return new DecisionLedger({ stateDir: root, now: options.now }).raiseOnce({
-      title: `${v} release resume limit: ${id}`, category: 'scope',
-      scqa: { s: `Release ${v} run ${id} has reached its two-resume limit.`, c: reason,
-        q: 'Should this release be deferred or moved to the next version?' },
-      options: [{ key: 'a', label: 'Defer this release', consequence: 'Stop this run and reschedule the release.' },
-        { key: 'b', label: 'Move to next version', consequence: 'Stop this run and carry the work to a new release.' }],
-      recommendation: { skipped: true, reason: 'OP must choose the release disposition.' },
-      raisedBy: { agent: 'release-resume-cap', track: 'TC' }, refs: [ref],
-    }, ref).id;
-  });
-  const blocked = (reason: string): never => {
-    if (options.noDecision) throw new Error(`release resume limit reached: ${reason}`);
-    throw new Error(`release resume limit reached — decision ${card(version, state.runId, reason)}: ${reason}`);
-  };
-  if (options.noDecision && !options.checkOnly) throw new Error('noDecision requires checkOnly');
-  if (!existsSync(path) && state.resume) blocked('saved run was resumed before attempt counting; prior count is unknown');
-  const prior = existsSync(path) ? JSON.parse(readFileSync(path, 'utf8')) as unknown : { version, runId: state.runId, attempts: [] };
+  const hadLedger = existsSync(path);
+  const prior = hadLedger ? JSON.parse(readFileSync(path, 'utf8')) as unknown : { version, runId: state.runId, attempts: [] };
   if (!prior || typeof prior !== 'object' || Array.isArray(prior)) throw new Error('release resume attempts ledger invalid');
-  const saved = prior as { version?: unknown; runId?: unknown; attempts?: unknown };
+  const saved = prior as { version?: unknown; runId?: unknown; attempts?: unknown; priorCountKnown?: unknown };
   if (saved.version !== version || saved.runId !== state.runId || !Array.isArray(saved.attempts)
-    || saved.attempts.some(value => typeof value !== 'string' || !Number.isFinite(Date.parse(value))))
+    || saved.attempts.some(value => typeof value !== 'string' || !Number.isFinite(Date.parse(value)))
+    || (saved.priorCountKnown !== undefined && typeof saved.priorCountKnown !== 'boolean'))
     throw new Error('release resume attempts ledger invalid');
-  if (saved.attempts.length >= RELEASE_RESUME_LIMIT) blocked(`${saved.attempts.length} resumes already recorded`);
-  if (options.checkOnly) return { attempt: saved.attempts.length + 1, path };
+  // Existing ledgers predate this field and began before the first resume.
+  // A legacy resumed run with no ledger must carry its unknown history forward.
+  const priorCountKnown = hadLedger ? saved.priorCountKnown !== false : !state.resume;
+  if (options.checkOnly) return { attempt: saved.attempts.length + 1, path, priorCountKnown };
   const at = (options.now ?? (() => new Date()))().toISOString();
-  const next = { version, runId: state.runId, attempts: [...saved.attempts, at] };
+  const next = { version, runId: state.runId, priorCountKnown, attempts: [...saved.attempts, at] };
   mkdirSync(dirname(path), { recursive: true });
   const temp = `${path}.${process.pid}.${randomUUID()}.tmp`;
   try { writeFileSync(temp, `${JSON.stringify(next)}\n`, { mode: 0o600 }); renameSync(temp, path); }
   finally { try { unlinkSync(temp); } catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; } }
-  return { attempt: next.attempts.length, path };
+  return { attempt: next.attempts.length, path, priorCountKnown };
 }

package/scripts/release-loop/resume-release.ts

@@ -2,15 +2,16 @@
 // The saved version-release output is re-pointed at the new branch tip (fast-forward only, state backed up first), so
 // every later node reads the repaired commit without anyone hand-editing the run state.
 import { spawnSync } from 'node:child_process';
+import { createHash } from 'node:crypto';
 import { constants, copyFileSync, existsSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
 import { join } from 'node:path';
 import { debug } from '../../src/debug/log.js';
 import { effectiveInstanceRoot, releaseLedgerRoot } from '../../src/instance/resolve.js';
-import { lastJsonObject, runGraph, verifyReleaseCommandRoot, type GraphRunState } from '../../src/graph-runner/runner.js';
+import { lastJsonObject, runGraph, verifyReleaseCommandRoot, verifyReleaseRecipeSnapshot, type GraphRunState } from '../../src/graph-runner/runner.js';
 import { withOpSeatFlock } from '../../src/steward/resync-tick.js';
 import { releaseRunUniverse } from './release-universe.js';
 import { isReleaseVersion } from './release-version.js';
-import { changedFilesBetween, clearPartialPlan, planPartialRegate, writePartialPlan } from './gate-partial.js';
+import { changedFilesBetween, clearPartialPlan, planPartialRegate, planStalledShardRecheck, writePartialPlan, writeRecheckPlan } from './gate-partial.js';
 import { gitClosureTree, testsReachingChanged, TRACEABLE_RELEASE_JSON, type ClosureTree } from './gate-closure.js';
 import { releaseResumeAcceptedRegressions, releaseResumeGateExecutor, releaseResumeGatePodPool, releaseResumeWaiver, type AcceptedRegression } from './resume.js';
 import { reserveReleaseResumeAttempt } from './resume-cap.js';
@@ -47,7 +48,7 @@ export interface ResumeReleaseOptions {
 export interface ResumeReleasePlan {
   runId: string; version: string; fromNode: string; commandRoot: string; tip: TipRefresh;
   partial: PartialOutcome; gatePlan: { mode: 'not-run' | 'partial' | 'full'; reason?: string };
-  nextAttempt: number; attemptLimit: 2; preflight: 'passed';
+  nextAttempt: number; priorCountKnown: boolean; preflight: 'passed';
 }
 
 /** CLI entry files: a production file that only names one (help text, a goal's command line) does not run it. */
@@ -84,7 +85,7 @@ export function refreshReleaseBranchTip(runId: string, deps: ResumeReleaseDeps =
   return refreshReleaseBranchTipInternal(runId, deps, from, waiverCommit, false);
 }
 
-function refreshReleaseBranchTipInternal(runId: string, deps: ResumeReleaseDeps, from: string | undefined, waiverCommit: string | undefined, dryRun: boolean): TipRefresh {
+function refreshReleaseBranchTipInternal(runId: string, deps: ResumeReleaseDeps, from: string | undefined, waiverCommit: string | undefined, dryRun: boolean, expectedTip?: string): TipRefresh {
   const root = deps.root ?? effectiveInstanceRoot();
   const statePath = join(root, 'graph-runs', 'release-loop', `${safeRunId(runId)}.json`);
   const state = JSON.parse(readFileSync(statePath, 'utf8')) as GraphRunState;
@@ -108,6 +109,7 @@ function refreshReleaseBranchTipInternal(runId: string, deps: ResumeReleaseDeps,
   const git = deps.git ?? gitRunner(deps.repo ?? process.cwd());
   const advertised = git(['ls-remote', '--heads', 'origin', `refs/heads/${branch}`]).split(/\s+/)[0] ?? '';
   if (!/^[0-9a-f]{40}$/i.test(advertised)) throw new Error(`${branch} is not on origin — this run was not cut as a release branch`);
+  if (expectedTip && advertised.toLowerCase() !== expectedTip.toLowerCase()) throw new Error(`${branch} moved during pin preflight (${expectedTip} → ${advertised}) — retry resume`);
   let to: string;
   if (dryRun) {
     // Use the advertised object id directly: a preflight must not create/update local remote-tracking refs.
@@ -152,6 +154,26 @@ function refreshReleaseBranchTipInternal(runId: string, deps: ResumeReleaseDeps,
   return { version, branch, from: recorded, to, changed: true, statePath, backup };
 }
 
+/** Check the saved graph and recipes before fetch, backup, or run-ledger writes. */
+function verifyResumeRecipePin(commandRoot: string, snapshot: GraphRunState['graphSnapshot'], version: string): void {
+  try { verifyReleaseRecipeSnapshot(commandRoot, snapshot); }
+  catch (error) {
+    if (!snapshot) throw error;
+    const mismatches: string[] = [];
+    for (const [name, savedSha] of [
+      ['release-loop.yaml', snapshot.graphSha], ['recipes.yaml', snapshot.recipesSha],
+    ] as const) {
+      const file = join(commandRoot, 'graphs', 'release', name);
+      let currentSha: string;
+      try { currentSha = createHash('sha256').update(readFileSync(file)).digest('hex'); }
+      catch { continue; }
+      if (currentSha !== savedSha) mismatches.push(`${file} saved=${savedSha} current=${currentSha}`);
+    }
+    if (!mismatches.length) throw error;
+    throw new Error(`release recipe source differs from saved pin: ${mismatches.join('; ')}\nremedy: start a new pinned run with eln release run --version ${version}`, { cause: error });
+  }
+}
+
 /** GATE-PARTIAL outcome of a resume — applied (plan written) or the reason it fell back to a full gate. `auto` marks the
  *  default plan of `--from gate` (GATE-REUSE-BY-DIFF); `closure` is set when code changes were followed. */
 export type PartialOutcome = { requested: false }
@@ -173,13 +195,50 @@ function preparePartial(tip: TipRefresh, opts: { from: string; partial?: boolean
   const reachesGate = (opts.from === 'gate' || opts.from === 'export-check') && fromIndex >= 0 && gateIndex >= fromIndex;
   // The default applies only where it can help: a gate re-run on a tip that moved. An unchanged tip re-uses its own
   // shard results by commit already.
-  if (opts.partial === false || (auto && (!reachesGate || !tip.changed))) return { requested: false };
+  if (opts.partial === false || (auto && (!reachesGate || (!tip.changed && opts.from !== 'gate')))) return { requested: false };
   const fallback = (reason: string): PartialOutcome => {
-    debug.log('release-loop.resume', 'partial-fallback', { version: tip.version, reason, auto });
+    debug.log('release-loop.resume', 'partial-fallback', { version: tip.version, reason, auto,
+      planKind: 'full', remeasureFileCount: 0, preservedShardCount: 0, fallbackReason: reason });
     return { requested: true, applied: false, reason, ...(auto ? { auto: true as const } : {}) };
   };
   if (!reachesGate) return fallback('--partial requires a restart that reaches gate');
-  if (!tip.changed) return fallback('release branch tip unchanged — nothing new to re-check');
+  if (!tip.changed) {
+    if (opts.from !== 'gate') return fallback('same-commit stalled-shard recheck requires --from gate');
+    const saved = JSON.parse(readFileSync(tip.statePath, 'utf8')) as GraphRunState;
+    const gateRecord = saved.nodes.find((node) => node.nodeId === 'gate');
+    const previousGate = gateRecord ? lastJsonObject(gateRecord.output) as Record<string, unknown> | undefined : undefined;
+    const stalled = previousGate?.stalledShards;
+    const summary = previousGate?.partialSummary as { pass?: unknown; fail?: unknown; errors?: unknown; ran?: unknown; files?: unknown } | undefined;
+    const measured = previousGate?.partialFailures as { failures?: unknown; errors?: unknown } | undefined;
+    if (previousGate?.outcome !== 'error' || previousGate.commit !== tip.to || !Array.isArray(stalled) || !stalled.length || !summary || !measured
+      || !['pass', 'fail', 'errors', 'ran', 'files'].every((key) => Number.isSafeInteger(summary[key as keyof typeof summary]) && Number(summary[key as keyof typeof summary]) >= 0)
+      || !Array.isArray(measured.failures) || !measured.failures.every((id) => typeof id === 'string')
+      || !Array.isArray(measured.errors) || !measured.errors.every((id) => typeof id === 'string')
+      || measured.failures.length !== Number(summary.fail) || measured.errors.length !== Number(summary.errors))
+      return fallback('same-commit gate has no complete stalled-shard measurement record');
+    const stalledFiles: string[] = [];
+    for (const item of stalled) {
+      if (!item || typeof item !== 'object' || !Array.isArray((item as { files?: unknown }).files)
+        || !(item as { files: unknown[] }).files.every((file) => typeof file === 'string')
+        || !['incomplete', 'no-output', 'job-failed', 'exit-code', 'unattributed', 'timeout'].includes(String((item as { reason?: unknown }).reason)))
+        return fallback('stalled-shard record is malformed');
+      stalledFiles.push(...(item as { files: string[] }).files);
+    }
+    const closure = (deps.closureTree ?? ((commit: string) => gitClosureTree(deps.repo ?? process.cwd(), commit)))(tip.to);
+    const tracked = new Set(closure.files);
+    const preservedShardCount = Number.isSafeInteger(previousGate?.preservedShardCount)
+      && Number(previousGate?.preservedShardCount) >= 0 ? Number(previousGate?.preservedShardCount) : undefined;
+    const planned = planStalledShardRecheck({ version: tip.version, measuredCommit: tip.to, forCommit: tip.to,
+      failures: measured.failures as string[], errors: measured.errors as string[], stalledFiles,
+      ...(preservedShardCount === undefined ? {} : { preservedShardCount }),
+      existsAtCommit: (file) => tracked.has(file) });
+    if (!planned.ok) return fallback(planned.reason);
+    const path = writeRecheckPlan(root, planned.plan);
+    debug.log('release-loop.resume', 'stalled-recheck-planned', { version: tip.version, commit: tip.to,
+      planKind: planned.plan.kind, remeasureFileCount: planned.plan.files.length,
+      preservedShardCount: planned.plan.preservedShardCount ?? 0, fallbackReason: null, path });
+    return { requested: true, applied: true, files: planned.plan.files.length, priorCommit: tip.to, ...(auto ? { auto: true as const } : {}) };
+  }
   let prior: { commit?: unknown; failures?: unknown; errors?: unknown } | undefined;
   for (const location of [...new Set([deps.ledgerRoot ?? releaseLedgerRoot(), root])]) {
     const path = join(location, 'release', tip.version, 'gate-failures.json');
@@ -281,7 +340,7 @@ async function planResumeReleaseRunLocked(opts: ResumeReleaseOptions, deps: Resu
   const waiver = original && (opts.waive !== undefined || opts.reason !== undefined) ? releaseResumeWaiver(original, opts) : undefined;
   if (original && opts.acceptedRegressions !== undefined && !waiver)
     releaseResumeAcceptedRegressions(original, opts.acceptedRegressions, opts.from);
-  const attempt = reserveReleaseResumeAttempt(savedRun, deps.root!, { now: deps.now, checkOnly: true, noDecision: true });
+  const attempt = reserveReleaseResumeAttempt(savedRun, deps.root!, { now: deps.now, checkOnly: true });
   let waiverCommit: string | undefined;
   if (waiver) {
     const output = lastJsonObject(savedRun.nodes.find((node) => node.nodeId === 'version-release')?.output);
@@ -294,6 +353,7 @@ async function planResumeReleaseRunLocked(opts: ResumeReleaseOptions, deps: Resu
     waiverCommit = recorded;
   }
   const tip = refreshReleaseBranchTipInternal(opts.runId, deps, opts.from, waiverCommit, true);
+  verifyResumeRecipePin(opts.commandRoot ?? process.cwd(), savedRun.graphSnapshot, tip.version);
   const commandRoot = verifyReleaseCommandRoot(opts.commandRoot ?? process.cwd(), tip.to, deps.git);
   const partial = preparePartial(tip, opts, deps, true);
   const fromIndex = savedRun.path.indexOf(opts.from);
@@ -303,7 +363,7 @@ async function planResumeReleaseRunLocked(opts: ResumeReleaseOptions, deps: Resu
     : partial.requested && partial.applied ? { mode: 'partial' as const }
       : { mode: 'full' as const, reason: partial.requested ? partial.reason : opts.partial === false ? 'partial recheck disabled' : 'no partial plan applies' };
   return { runId: opts.runId, version: tip.version, fromNode: opts.from, commandRoot, tip, partial,
-    gatePlan, nextAttempt: attempt.attempt, attemptLimit: 2, preflight: 'passed' };
+    gatePlan, nextAttempt: attempt.attempt, priorCountKnown: attempt.priorCountKnown, preflight: 'passed' };
 }
 
 async function resumeReleaseRunLocked(opts: ResumeReleaseOptions, deps: ResumeReleaseDeps): Promise<{ tip: TipRefresh; partial: PartialOutcome; state: GraphRunState }> {
@@ -331,7 +391,7 @@ async function resumeReleaseRunLocked(opts: ResumeReleaseOptions, deps: ResumeRe
   const acceptedRegressions = original && opts.acceptedRegressions !== undefined && !waiver
     ? releaseResumeAcceptedRegressions(original, opts.acceptedRegressions, opts.from) : undefined;
   if (savedRun.graphId !== 'release-loop' || savedRun.runId !== opts.runId) throw new Error(`run identity mismatch: ${statePath}`);
-  // Validate request arguments before the cap, then block the third attempt before tip refresh.
+  // Validate request arguments before the tip refresh; the count is observational.
   reserveReleaseResumeAttempt(savedRun, deps.root!, { now: deps.now, checkOnly: true });
   // A waived verdict may reuse its original commit. A moved tip still requires a full re-gate.
   if (waiver) {
@@ -346,13 +406,16 @@ async function resumeReleaseRunLocked(opts: ResumeReleaseOptions, deps: ResumeRe
     if (advertised?.toLowerCase() !== recorded.toLowerCase()) throw new Error('release branch moved — re-gate the new tip before waiving a node');
     waiverCommit = recorded;
   }
-  // Check the refreshed tip before refresh writes a new version-release record or backup.
-  const tip = refreshReleaseBranchTip(opts.runId, deps, opts.from, waiverCommit);
-  const commandRoot = verifyReleaseCommandRoot(opts.commandRoot ?? process.cwd(), tip.to, deps.git);
+  // The read-only tip and recipe pin preflight must pass before fetch or a saved-run rewrite.
+  const previewTip = refreshReleaseBranchTipInternal(opts.runId, deps, opts.from, waiverCommit, true);
+  verifyResumeRecipePin(opts.commandRoot ?? process.cwd(), savedRun.graphSnapshot, previewTip.version);
+  const commandRoot = verifyReleaseCommandRoot(opts.commandRoot ?? process.cwd(), previewTip.to, deps.git);
+  const tip = refreshReleaseBranchTipInternal(opts.runId, deps, opts.from, waiverCommit, false, previewTip.to);
   const partial = preparePartial(tip, opts, deps);
-  // A failed preflight does not consume one of the two release resume attempts.
+  // A failed preflight does not consume an observed release resume attempt.
   const attempt = reserveReleaseResumeAttempt(savedRun, deps.root!, { now: deps.now });
-  debug.log('release-loop.resume', 'attempt', { runId: opts.runId, attempt: attempt.attempt, limit: 2, ledger: attempt.path });
+  debug.log('release-loop.resume', 'attempt', { runId: opts.runId, attempt: attempt.attempt,
+    priorCountKnown: attempt.priorCountKnown, ledger: attempt.path });
   // Resume the graph at the path it was started from (its snapshot is used); only tests inject another path.
   const graphPath = deps.graphPath ?? savedGraphPath(tip.statePath) ?? GRAPH;
   const savedSource = JSON.parse(readFileSync(tip.statePath, 'utf8')) as GraphRunState;

package/scripts/release-loop/traffic-node.ts

@@ -0,0 +1,59 @@
+#!/usr/bin/env bun
+import { spawnSync } from 'node:child_process';
+import { debug } from '../../src/debug/log.js';
+import { SPAWN_SYNC_MAX_BUFFER } from '../../src/util/spawn-sync-output.js';
+import { releaseLoopConfig } from './unattended-release.js';
+import { errorResult, finishNode, readGraphContext } from './node-verdict.js';
+import { applyReleaseTrafficEvent, trafficOptionsFromConfig, type ReleaseTrafficEvent, type TrafficTransitionOptions } from './traffic-transition.js';
+
+const EVENTS: Record<string, ReleaseTrafficEvent> = {
+  'traffic-gate-passed': 'gate-passed',
+  'traffic-verify-complete': 'verify-complete',
+};
+
+export interface TrafficNodeDeps {
+  options?: TrafficTransitionOptions;
+  report?: (body: string) => void;
+}
+
+function reportFailure(body: string): void {
+  const result = spawnSync('eln', ['msg', 'post', '--from', 'MK', '--to', 'OP', '--kind', 'report', '--body', body], { encoding: 'utf8', timeout: 10_000, maxBuffer: SPAWN_SYNC_MAX_BUFFER });
+  if (result.status !== 0 || result.error) process.stderr.write(`traffic OP mail failed: ${result.stderr || result.error || result.status}\n`);
+}
+
+/** Traffic transitions are warning-only for the release graph, but their node result remains truthful. */
+export async function runTrafficNode(node = process.argv[2], env: NodeJS.ProcessEnv = process.env, deps: TrafficNodeDeps = {}): Promise<number> {
+  let version = '';
+  let runId = '';
+  try {
+    const context = readGraphContext(env);
+    version = context.input.version;
+    runId = context.runId ?? '';
+    const event = EVENTS[node ?? ''];
+    if (!event || context.nodeId !== node) throw new Error(`traffic node context mismatch: ${node ?? 'missing'}`);
+    const options = deps.options ?? trafficOptionsFromConfig(env, releaseLoopConfig());
+    if (!options.trafficFile) {
+      const summary = `traffic-file-unset: ${event} skipped because trafficFile is not configured`;
+      debug.log('release-loop.traffic', 'traffic-file-unset', { runId, version, event, node, reason: summary });
+      (deps.report ?? reportFailure)(`RELEASE-POSTPUB-TRAFFIC ${version} ${runId || 'unknown'} ${node} ${summary}. Release traffic was not changed.`);
+      return finishNode(node!, version, { outcome: 'fail', verdict: 'fail', summary });
+    }
+    const result = await applyReleaseTrafficEvent(context, event, options);
+    if (!result.applied) {
+      const consequence = event === 'verify-complete'
+        ? '릴리스 검증은 성공했으나 트래픽 복귀가 적용되지 않았습니다.'
+        : 'gate는 성공했으나 트래픽 감속이 적용되지 않았습니다.';
+      (deps.report ?? reportFailure)(`RELEASE-POSTPUB-TRAFFIC ${version} ${runId || 'unknown'} ${node} 무시 — ${result.reason ?? 'traffic state unchanged'}. ${consequence}`);
+    }
+    return finishNode(node!, version, { outcome: 'ok', verdict: 'pass', summary: `${event} ${result.applied ? 'applied' : result.reason ?? 'ignored'}` });
+  } catch (error) {
+    const detail = errorResult(error).summary;
+    const isLockTimeout = detail.startsWith('traffic lock timeout:');
+    const failure = isLockTimeout ? `traffic-lock-timeout · stale 의심 경로 ${detail.slice('traffic lock timeout:'.length).trim()}` : detail;
+    debug.log('release-loop.traffic', 'non-blocking-failure', { node: node ?? 'missing', version, reason: detail });
+    (deps.report ?? reportFailure)(`RELEASE-POSTPUB-TRAFFIC ${version || 'unknown'} ${runId || 'unknown'} ${node ?? 'unknown'} 실패 — ${failure}. 릴리스 판정은 계속 진행한다.`);
+    return finishNode(node ?? 'traffic', version, { outcome: 'fail', verdict: 'fail', summary: `traffic transition failed; release graph continues with warning: ${failure}` });
+  }
+}
+
+if (import.meta.main) process.exitCode = await runTrafficNode();