elanous 0.2.0 → 0.2.4

package/src/graph-runner/runner.ts

@@ -0,0 +1,457 @@
+import { createHash, randomUUID } from 'node:crypto';
+import { execFile } from 'node:child_process';
+import { linkSync, mkdirSync, readFileSync, readdirSync, renameSync, rmdirSync, statSync, unlinkSync, writeFileSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { parse as parseYaml, stringify as stringifyYaml } from 'yaml';
+import { debug } from '../debug/log.js';
+import { effectiveInstanceRoot } from '../instance/resolve.js';
+import { loadNodeCatalog } from '../self-implement/graph-catalog.js';
+import { parseGraphTemplateYaml, type GraphEdgeSpec, type GraphNodeSpec } from '../self-implement/graph-yaml.js';
+import { executeBashNode } from '../workflow-runtime/nodes/bash.js';
+import type { BashNode, NodeExecContext, WorkflowDeps } from '../workflow-runtime/types.js';
+
+type BashRun = WorkflowDeps['runBash'];
+
+export interface GraphRunState {
+  graphId: string;
+  runId: string;
+  startedAt?: string;
+  finishedAt?: string;
+  status: 'running' | 'done' | 'failed' | 'budget-exceeded' | 'awaiting-approval';
+  path: string[];
+  nodes: Array<{ nodeId: string; ok: boolean; exit: number | null; executed: boolean; output?: unknown; error?: string; decidedBy?: string; decidedAt?: string }>;
+  input?: unknown;
+  pending?: { nodeId: string; message: string; since: string; notifiedAt?: string; decision?: 'approved' | 'rejected'; decidedBy?: string; decidedAt?: string };
+  approvalSourceHash?: string;
+  /** Hash of the graph and recipes used for this run; required to safely restart a failed path. */
+  sourceHash?: string;
+  resume?: { from: string; at: string; previousStatus: 'failed' };
+  executed: number;
+  dryRun: boolean;
+  statePath: string;
+}
+
+export interface GraphRunOptions {
+  input?: unknown;
+  dryRun?: boolean;
+  runId?: string;
+  resumeRunId?: string;
+  fromNodeId?: string;
+  deps?: { root?: string; runBash?: BashRun; log?: (event: string, data: Record<string, unknown>) => void };
+}
+
+function safeSegment(value: string): string {
+  if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(value) || value === '.' || value === '..') {
+    throw new Error(`invalid graph run identifier: ${value}`);
+  }
+  return value;
+}
+
+type CommandRecipe = { command: string; timeout_ms?: number };
+type Recipe = CommandRecipe | { approval: string };
+
+/** 접두 없는 recipe 가 카탈로그 역할이고, 그래프 옆 recipes.yaml 에 같은 키의 `{ command }` 가 있을 때만 cmd 처럼 실행한다. */
+function roleCommand(recipe: string, recipes: Record<string, Recipe>, catalogRoles: ReadonlySet<string>): CommandRecipe | undefined {
+  if (!recipe || recipe === 'none' || recipe.includes(':') || !catalogRoles.has(recipe)) return undefined;
+  const entry = recipes[recipe];
+  return entry && 'command' in entry ? entry : undefined;
+}
+
+function resolveNodeRecipe(node: GraphNodeSpec, recipes: Record<string, Recipe>, catalogRoles: ReadonlySet<string>): { command?: CommandRecipe; approval?: { approval: string } } {
+  if (node.recipe === 'none') return {};
+  if (node.recipe.startsWith('cmd:')) {
+    const id = node.recipe.slice(4);
+    const entry = recipes[id];
+    return entry && 'command' in entry ? { command: entry } : {};
+  }
+  if (node.recipe.startsWith('approval:')) {
+    const id = node.recipe.slice(9);
+    const entry = recipes[id];
+    return entry && 'approval' in entry ? { approval: entry } : {};
+  }
+  const command = roleCommand(node.recipe, recipes, catalogRoles);
+  return command ? { command } : {};
+}
+
+function approvalMessage(template: string, input: unknown): string {
+  if (!template.includes('{{input.version}}')) return template;
+  const version = input && typeof input === 'object' && 'version' in input ? (input as { version: unknown }).version : undefined;
+  if (typeof version !== 'string' || !/^(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)$/.test(version)) {
+    throw new Error('approval input.version must be a release version');
+  }
+  return template.replaceAll('{{input.version}}', version);
+}
+
+function recipesFor(path: string, recipeSource: string): Record<string, Recipe> {
+  const file = join(dirname(path), 'recipes.yaml');
+  const parsed: unknown = parseYaml(recipeSource);
+  if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) throw new Error(`invalid recipes: ${file}`);
+  const recipes: Record<string, Recipe> = Object.create(null);
+  for (const [id, raw] of Object.entries(parsed)) {
+    if (!raw || typeof raw !== 'object' || Array.isArray(raw)) throw new Error(`invalid recipe: ${id}`);
+    const item = raw as Record<string, unknown>;
+    if (Object.hasOwn(item, 'approval')) {
+      if (typeof item.approval !== 'string' || !item.approval.trim() || Object.keys(item).length !== 1) throw new Error(`invalid recipe: ${id}`);
+      recipes[id] = { approval: item.approval };
+    } else {
+      if (typeof item.command !== 'string' || !item.command.trim() ||
+        (item.timeout_ms !== undefined && (!Number.isSafeInteger(item.timeout_ms) || (item.timeout_ms as number) <= 0))) {
+        throw new Error(`invalid recipe: ${id}`);
+      }
+      recipes[id] = { command: item.command, ...(item.timeout_ms === undefined ? {} : { timeout_ms: item.timeout_ms as number }) };
+    }
+  }
+  return recipes;
+}
+
+const realRunBash: BashRun = (body, opts) => new Promise((resolve, reject) => {
+  execFile('/bin/bash', ['-c', body], { cwd: opts.cwd, env: opts.env, timeout: opts.timeoutMs, signal: opts.signal, maxBuffer: 10 * 1024 * 1024 }, (error, stdout, stderr) => {
+    if (error && typeof (error as NodeJS.ErrnoException).code !== 'number') {
+      reject(error);
+      return;
+    }
+    resolve({ stdout, stderr, exitCode: error ? (error as unknown as { code: number }).code : 0 });
+  });
+});
+
+function graphRunPath(graphId: string, runId: string, root: string): string {
+  return join(root, 'graph-runs', safeSegment(graphId), `${safeSegment(runId)}.json`);
+}
+
+function approvalDecisionPath(state: GraphRunState): string {
+  return `${state.statePath}.${state.path.length}.decision.json`;
+}
+
+function applyRecordedDecision(state: GraphRunState): GraphRunState {
+  if (!state.pending) return state;
+  let recorded: { nodeId: string; decision: 'approved' | 'rejected'; decidedBy?: string; decidedAt: string };
+  try { recorded = JSON.parse(readFileSync(approvalDecisionPath(state), 'utf8')); }
+  catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'ENOENT') return state;
+    throw error;
+  }
+  if (recorded.nodeId !== state.pending.nodeId || !['approved', 'rejected'].includes(recorded.decision) ||
+    typeof recorded.decidedAt !== 'string' || (recorded.decidedBy !== undefined && typeof recorded.decidedBy !== 'string') ||
+    (state.pending.decision && state.pending.decision !== recorded.decision)) {
+    throw new Error(`approval decision mismatch: ${state.graphId}/${state.runId}`);
+  }
+  state.pending = { ...state.pending, ...recorded };
+  return state;
+}
+
+function readGraphRun(graphId: string, runId: string, root: string): GraphRunState {
+  const file = graphRunPath(graphId, runId, root);
+  const state = JSON.parse(readFileSync(file, 'utf8')) as GraphRunState;
+  if (state.graphId !== graphId || state.runId !== runId) throw new Error(`run identity mismatch: ${graphId}/${runId}`);
+  return applyRecordedDecision({ ...state, statePath: file });
+}
+
+function persistGraphRun(state: GraphRunState): void {
+  mkdirSync(dirname(state.statePath), { recursive: true });
+  const temporary = `${state.statePath}.${process.pid}.tmp`;
+  writeFileSync(temporary, JSON.stringify(state, null, 2) + '\n');
+  renameSync(temporary, state.statePath);
+}
+
+export function decideGraphApproval(graphId: string, runId: string, decision: 'approved' | 'rejected', by?: string, root = effectiveInstanceRoot()): GraphRunState {
+  const state = readGraphRun(graphId, runId, root);
+  if (state.status !== 'awaiting-approval' || !state.pending || state.pending.decision) {
+    throw new Error(`run is not awaiting an undecided approval: ${graphId}/${runId}`);
+  }
+  if (by !== undefined && !by.trim()) throw new Error('approver name must not be empty');
+  const recorded = { nodeId: state.pending.nodeId, decision, ...(by === undefined ? {} : { decidedBy: by }), decidedAt: new Date().toISOString() };
+  const decisionPath = approvalDecisionPath(state);
+  const temporary = `${decisionPath}.${randomUUID()}.tmp`;
+  writeFileSync(temporary, JSON.stringify(recorded) + '\n', { flag: 'wx' });
+  try {
+    // Hard-link creation is atomic across processes: only the first approver can claim this visit.
+    linkSync(temporary, decisionPath);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'EEXIST') {
+      throw new Error(`run is not awaiting an undecided approval: ${graphId}/${runId}`);
+    }
+    throw error;
+  } finally {
+    unlinkSync(temporary);
+  }
+  return applyRecordedDecision(state);
+}
+
+/** 노드 stdout 의 «마지막 비어 있지 않은 줄»이 JSON 객체면 그것을 구조 산출로 본다(아니면 없음). */
+export function lastJsonObject(stdout: unknown): Record<string, unknown> | undefined {
+  if (typeof stdout !== 'string') return undefined;
+  const lines = stdout.split('\n').map((line) => line.trim()).filter(Boolean);
+  const last = lines.at(-1);
+  if (!last || !last.startsWith('{')) return undefined;
+  try {
+    const parsed: unknown = JSON.parse(last);
+    return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed as Record<string, unknown> : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+/** 이름 붙은 결과: 산출의 `outcome` 이 그 간선 map 의 키면 그 목적지 — 아니면 종전 ok/fail(exit) → fallback. */
+function nextNode(edges: readonly GraphEdgeSpec[], nodeId: string, exitOutcome: 'ok' | 'fail', namedOutcome?: string): string | undefined {
+  const edge = edges.find((e) => e.from === nodeId);
+  if (!edge) return undefined;
+  if (edge.to !== undefined) return edge.to;
+  if (edge.on !== 'outcome') throw new Error(`unsupported edge condition: ${edge.on}`);
+  if (namedOutcome !== undefined && edge.map && Object.hasOwn(edge.map, namedOutcome)) return edge.map[namedOutcome];
+  return edge.map?.[exitOutcome] ?? edge.fallback?.[0]?.node;
+}
+
+export async function runGraph(path: string, options: GraphRunOptions = {}): Promise<GraphRunState> {
+  const source = readFileSync(path, 'utf8');
+  // The shared YAML parser requires a recipe string. A command-less node in
+  // this runner is represented internally as `none`, without changing that parser.
+  const raw: unknown = parseYaml(source);
+  if (!raw || typeof raw !== 'object' || Array.isArray(raw)) throw new Error(`invalid graph: ${path}`);
+  const document = raw as Record<string, unknown>;
+  if (Array.isArray(document.nodes)) {
+    document.nodes = document.nodes.map((node: unknown) => node && typeof node === 'object' && !Array.isArray(node) && !('recipe' in node)
+      ? { ...node, recipe: 'none' } : node);
+  }
+  const parsed = parseGraphTemplateYaml(stringifyYaml(document), path);
+  if (!parsed.template || parsed.errors.length) throw new Error(parsed.errors.map((e) => `${e.path}: ${e.message}`).join('\n'));
+  const graph = parsed.template;
+  for (const terminal of graph.terminalNodes) {
+    if (terminal !== 'done' && terminal !== 'failed') {
+      throw new Error(`unsupported terminal node: ${terminal} (expected done or failed)`);
+    }
+  }
+  const recipeSource = readFileSync(join(dirname(path), 'recipes.yaml'), 'utf8');
+  const recipes = recipesFor(path, recipeSource);
+  const catalogRoles = new Set(loadNodeCatalog().roles.keys());
+  // Bind a human decision to the exact graph and recipe contents, not just its id or approval prompt.
+  const approvalSourceHash = createHash('sha256').update(source).update('\0').update(recipeSource).digest('hex');
+  // Resolve all commands before executing any node; an unknown recipe must never produce a partial publication.
+  for (const node of graph.nodes) {
+    if (node.recipe === 'none') continue;
+    const resolved = resolveNodeRecipe(node, recipes, catalogRoles);
+    if (!resolved.command && !resolved.approval) throw new Error(`unknown command recipe for ${node.nodeId}: ${node.recipe}`);
+  }
+  const graphId = safeSegment(graph.graphId);
+  if (options.resumeRunId && options.runId) throw new Error('runId and resumeRunId cannot be combined');
+  if (options.fromNodeId && !options.resumeRunId) throw new Error('--from requires --resume');
+  const runId = safeSegment(options.resumeRunId ?? options.runId ?? randomUUID());
+  const statePath = graphRunPath(graphId, runId, options.deps?.root ?? effectiveInstanceRoot());
+  // A resume owns the run from the first state read through its last write and command.
+  // Fail closed on a concurrent (or interrupted) owner rather than replaying a command.
+  const resumeLock = `${statePath}.resume.lock`;
+  if (options.resumeRunId) {
+    mkdirSync(dirname(statePath), { recursive: true });
+    try { mkdirSync(resumeLock); }
+    catch (error) {
+      if ((error as NodeJS.ErrnoException).code === 'EEXIST') throw new Error(`run is already being resumed: ${graphId}/${runId}`);
+      throw error;
+    }
+  }
+  try {
+  const state: GraphRunState = options.resumeRunId
+    ? readGraphRun(graphId, runId, options.deps?.root ?? effectiveInstanceRoot())
+    : { graphId, runId, startedAt: new Date().toISOString(), status: 'running', path: [], nodes: [], sourceHash: approvalSourceHash, ...(options.input === undefined ? {} : { input: options.input }), executed: 0, dryRun: options.dryRun === true, statePath };
+  if (options.fromNodeId) {
+    if (state.status !== 'failed' || state.pending || state.dryRun) throw new Error('--from requires a failed, non-dry run without pending approval');
+    if (!state.sourceHash || state.sourceHash !== approvalSourceHash) throw new Error('graph or recipes changed since failed run');
+    const from = state.path.indexOf(options.fromNodeId);
+    if (from < 0 || state.path.lastIndexOf(options.fromNodeId) !== from || graph.terminalNodes.includes(options.fromNodeId)) throw new Error(`--from node is not unique on the saved executable path: ${options.fromNodeId}`);
+    if (state.path.length !== state.nodes.length || state.nodes.some((record, i) => record.nodeId !== state.path[i]) ||
+      state.executed !== state.nodes.filter((record) => record.executed).length) {
+      throw new Error('saved run path and node records do not match');
+    }
+    for (let i = 0; i < state.path.length; i++) {
+      const recorded = state.nodes[i]!;
+      const spec = graph.nodes.find((node) => node.nodeId === recorded.nodeId);
+      if (!spec) throw new Error(`saved path has undeclared node: ${recorded.nodeId}`);
+      if (i > 0) {
+        const prior = state.nodes[i - 1]!;
+        const reported = lastJsonObject(prior.output)?.outcome;
+        const next = nextNode(graph.edges, prior.nodeId, prior.ok ? 'ok' : 'fail', typeof reported === 'string' ? reported : undefined);
+        if (next !== recorded.nodeId) throw new Error('saved run path does not follow graph edges');
+      } else if (recorded.nodeId !== graph.entryNode) throw new Error('saved run path does not start at entry node');
+      if (i < from && !recorded.ok) throw new Error('cannot preserve a failed predecessor when resuming');
+      const approved = resolveNodeRecipe(spec, recipes, catalogRoles).approval;
+      if (approved && i >= from) throw new Error('cannot restart across an approval decision');
+      if (approved && i < from) {
+        // A JSON state alone is not evidence of approval: require the immutable decision claim for this visit.
+        const decisionFile = `${statePath}.${i + 1}.decision.json`;
+        let decision: { nodeId?: string; decision?: string; decidedAt?: string };
+        try { decision = JSON.parse(readFileSync(decisionFile, 'utf8')); }
+        catch { throw new Error(`missing approval decision for ${recorded.nodeId}`); }
+        if (decision.nodeId !== recorded.nodeId || decision.decision !== 'approved' || decision.decidedAt !== recorded.decidedAt || !recorded.ok) {
+          throw new Error(`invalid approval decision for ${recorded.nodeId}`);
+        }
+      }
+    }
+    if (resolveNodeRecipe(graph.nodes.find((node) => node.nodeId === options.fromNodeId)!, recipes, catalogRoles).approval) {
+      throw new Error('--from cannot restart an approval node');
+    }
+    state.path = state.path.slice(0, from);
+    state.nodes = state.nodes.slice(0, from);
+    state.executed = state.nodes.filter((node) => node.executed).length;
+    state.status = 'running';
+    state.resume = { from: options.fromNodeId, at: new Date().toISOString(), previousStatus: 'failed' };
+    // Persist the restart boundary before executing it, so previous outputs remain durable.
+    persistGraphRun(state);
+    debug.log('graph.run', 'resume', { graphId, runId, from: options.fromNodeId });
+  }
+  if (options.resumeRunId && state.status === 'running' && state.pending) {
+    throw new Error(`run is not awaiting approval: ${graphId}/${runId}`);
+  }
+  if (options.resumeRunId && state.status === 'awaiting-approval' && !state.pending) {
+    throw new Error(`run is not awaiting approval: ${graphId}/${runId}`);
+  }
+  if (options.resumeRunId && state.status !== 'awaiting-approval' && state.status !== 'running') {
+    throw new Error(`run is not awaiting approval: ${graphId}/${runId}`);
+  }
+  if (options.resumeRunId && (state.graphId !== graphId || state.runId !== runId)) {
+    throw new Error(`run is not awaiting approval: ${graphId}/${runId}`);
+  }
+  if (options.resumeRunId && options.dryRun !== undefined && options.dryRun !== state.dryRun) throw new Error('cannot change dryRun when resuming');
+  if (options.resumeRunId && state.status === 'awaiting-approval' && state.approvalSourceHash !== approvalSourceHash) {
+    throw new Error(`approval source changed since run was paused: ${graphId}/${runId}`);
+  }
+  const persist = () => {
+    if (state.status === 'done' || state.status === 'failed' || state.status === 'budget-exceeded') state.finishedAt ??= new Date().toISOString();
+    persistGraphRun(state);
+  };
+  const log = options.deps?.log ?? ((event: string, data: Record<string, unknown>) => debug.log('graph.runner', event, data));
+  const visits = new Map<string, number>();
+  for (const nodeId of state.path) visits.set(nodeId, (visits.get(nodeId) ?? 0) + 1);
+  let current: string | undefined = options.fromNodeId ?? state.pending?.nodeId ?? state.path.at(-1) ?? graph.entryNode;
+  if (!options.resumeRunId) {
+    mkdirSync(dirname(statePath), { recursive: true });
+    if (readdirSync(dirname(statePath)).some((file) => file.startsWith(`${runId}.json.`) && file.endsWith('.decision.json'))) {
+      throw new Error(`run id already exists: ${graphId}/${runId}`);
+    }
+    try { writeFileSync(statePath, JSON.stringify(state, null, 2) + '\n', { flag: 'wx' }); }
+    catch (error) {
+      if ((error as NodeJS.ErrnoException).code === 'EEXIST') throw new Error(`run id already exists: ${graphId}/${runId}`);
+      throw error;
+    }
+  }
+  while (current !== undefined) {
+    const node = graph.nodes.find((n) => n.nodeId === current);
+    if (!node) throw new Error(`undeclared node: ${current}`);
+    const resumingPending = state.pending?.nodeId === current;
+    const completed = [...state.nodes].reverse().find((recorded) => recorded.nodeId === current && recorded.executed);
+    const resumingCompleted = options.resumeRunId !== undefined && !options.fromNodeId && !!completed && state.path.at(-1) === current && !resumingPending;
+    if (!resumingPending && !resumingCompleted && (visits.get(current) ?? 0) >= node.maxVisits) {
+      state.status = 'budget-exceeded';
+      log('budget-exceeded', { graphId, runId, nodeId: current });
+      persist();
+      break;
+    }
+    if (!resumingPending && !resumingCompleted) {
+      visits.set(current, (visits.get(current) ?? 0) + 1);
+      state.path.push(current);
+    }
+    const resolved = resolveNodeRecipe(node, recipes, catalogRoles);
+    const command = resolved.command;
+    const approval = resolved.approval ? { approval: approvalMessage(resolved.approval.approval, state.input) } : undefined;
+    if (resumingPending && (!approval || approval.approval !== state.pending?.message)) {
+      throw new Error(`pending approval no longer matches graph: ${current}`);
+    }
+    // Graphs may opt into previewing the approval boundary in dry runs.
+    if (approval && (!state.dryRun || document.dry_run_await_approval === true) && !state.pending?.decision) {
+      if (!state.pending) {
+        state.pending = { nodeId: current, message: approval.approval, since: new Date().toISOString() };
+        state.approvalSourceHash = approvalSourceHash;
+      }
+      state.status = 'awaiting-approval';
+      if (!state.dryRun && !state.pending.notifiedAt) {
+        try {
+          log('approval-pending', { graphId, runId, nodeId: current, message: state.pending.message, since: state.pending.since });
+          state.pending = { ...state.pending, notifiedAt: new Date().toISOString() };
+        } catch (error) {
+          persist();
+          throw error;
+        }
+      }
+      persist();
+      break;
+    }
+    state.status = 'running';
+    const startedAt = performance.now();
+    console.error(`[graph] ${current} start (0.00s)`);
+    debug.log('graph.run', 'node', { graphId, runId, nodeId: current, phase: 'start', dryRun: state.dryRun });
+    log('node-start', { graphId, runId, nodeId: current, visit: visits.get(current), dryRun: state.dryRun });
+    persist();
+    let ok = approval && !state.dryRun ? state.pending?.decision === 'approved' : true;
+    let exit: number | null = null;
+    let error: string | undefined;
+    let output: unknown;
+    if (command && !state.dryRun && !resumingCompleted) {
+      const outputs: Record<string, unknown> = Object.create(null);
+      // 구조 산출(마지막 JSON 줄)이 있으면 그것을, 없으면 원문을 준다 — 다음 노드가 판단을 «값»으로 받는다.
+      for (const previous of state.nodes) outputs[previous.nodeId] = lastJsonObject(previous.output) ?? previous.output ?? null;
+      const contextPath = join(`${statePath}.contexts`, `${state.path.length}.json`);
+      mkdirSync(dirname(contextPath), { recursive: true });
+      writeFileSync(contextPath, JSON.stringify({ graphId, runId, nodeId: current, input: state.input ?? null, outputs }, null, 2) + '\n');
+      let code: number | null = null;
+      const runBash: BashRun = async (body, opts) => {
+        const result = await (options.deps?.runBash ?? realRunBash)(body, opts);
+        code = result.exitCode;
+        return result;
+      };
+      const ctx = { arguments: '', artifactsDir: dirname(statePath), outputs: {}, resolvedProvider: undefined, resolvedModel: undefined, toolPolicy: {}, env: { ...process.env, ELANOUS_GRAPH_CONTEXT: contextPath, ELANOUS_GRAPH_DIR: dirname(resolve(path)) } } as NodeExecContext;
+      const result = await executeBashNode({ id: current, type: 'bash', bash: command.command, ...(command.timeout_ms ? { idle_timeout: command.timeout_ms } : {}) } as BashNode, ctx, { runBash } as WorkflowDeps);
+      output = result.output;
+      ok = result.ok;
+      exit = code;
+      error = result.error;
+      state.executed++;
+    }
+    if (resumingCompleted && completed) {
+      ok = completed.ok;
+      exit = completed.exit;
+      output = completed.output;
+      error = completed.error;
+    } else {
+      // 승인 노드도 결과를 «값»으로 남긴다 — 뒤 노드가 문맥(outputs)에서 승인 여부를 확인할 수 있게(발행 노드가 승인 없이 돌지 않도록).
+      if (approval && !state.dryRun && !command) {
+        output = JSON.stringify({ outcome: ok ? 'approved' : 'rejected', ...(state.pending?.decidedBy === undefined ? {} : { decidedBy: state.pending.decidedBy }), decidedAt: state.pending?.decidedAt ?? null });
+      }
+      state.nodes.push({ nodeId: current, ok, exit, executed: !!command && !state.dryRun, ...((command || approval) && !state.dryRun && output !== undefined ? { output } : {}), ...(error ? { error } : {}),
+        ...(approval && state.pending ? { ...(state.pending.decidedBy === undefined ? {} : { decidedBy: state.pending.decidedBy }), decidedAt: state.pending.decidedAt } : {}) });
+    }
+    if (approval) {
+      delete state.pending;
+      delete state.approvalSourceHash;
+    }
+    const reported = command && !state.dryRun ? lastJsonObject(resumingCompleted ? completed?.output : output)?.outcome : undefined;
+    const namedOutcome = typeof reported === 'string' ? reported : undefined;
+    const seconds = Number(((performance.now() - startedAt) / 1000).toFixed(2));
+    console.error(`[graph] ${current} ${ok ? 'ok' : 'fail'} (${seconds.toFixed(2)}s)`);
+    debug.log('graph.run', 'node', { graphId, runId, nodeId: current, phase: ok ? 'ok' : 'fail', seconds, exit });
+    log('node-end', { graphId, runId, nodeId: current, ok, exit, ...(namedOutcome === undefined ? {} : { outcome: namedOutcome }), dryRun: state.dryRun });
+    persist();
+    if (graph.terminalNodes.includes(current)) {
+      state.status = current === 'failed' || !ok ? 'failed' : 'done';
+      persist();
+      break;
+    }
+    current = nextNode(graph.edges, current, state.dryRun ? 'ok' : ok ? 'ok' : 'fail', state.dryRun ? undefined : namedOutcome);
+    if (current === undefined) {
+      state.status = 'failed';
+      persist();
+    }
+  }
+  return state;
+  } finally {
+    if (options.resumeRunId) rmdirSync(resumeLock);
+  }
+}
+
+export function latestGraphRun(graphId: string, root = effectiveInstanceRoot()): GraphRunState | null {
+  const dir = join(root, 'graph-runs', safeSegment(graphId));
+  let files: string[];
+  try { files = readdirSync(dir).filter((file) => file.endsWith('.json') && !file.endsWith('.decision.json')); }
+  catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null;
+    throw error;
+  }
+  const latest = files.sort((a, b) => statSync(join(dir, b)).mtimeMs - statSync(join(dir, a)).mtimeMs)[0];
+  return latest ? readGraphRun(graphId, latest.slice(0, -5), root) : null;
+}

package/src/grok/credential.ts

@@ -26,7 +26,7 @@
 //   「Using auth.json for API Access」 절이 유일한 문서다. ⇒ 조용히 깨질 수 있으니
 //   호출자는 실패를 fail-soft 로 접고 API 키 경로로 떨어질 수 있어야 한다.
 
-import { existsSync, readFileSync } from 'node:fs';
+import { chmodSync, existsSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
 import { homedir } from 'node:os';
 import { join } from 'node:path';
 
@@ -136,6 +136,15 @@ function readSubscriptionToken(
   return best ? { token: best.token, expiresAt: best.expiresAt, userId: best.userId } : null;
 }
 
+/** 호스트 발급 문이 응답에 실을 access. user id 는 반환하지 않는다. */
+export function readGrokSubscriptionAccess(
+  home?: string,
+): { key: string; expiresAt: string | null } | null {
+  const sub = readSubscriptionToken(grokAuthFilePath(home));
+  if (!sub) return null;
+  return { key: sub.token, expiresAt: sub.expiresAt };
+}
+
 /** 프록시 필수 헤더. ⛔ `x-grok-client-version` 을 빼면 426 이다(실측). */
 export function grokSubscriptionHeaders(opts: {
   model?: string;
@@ -155,12 +164,121 @@ export function grokSubscriptionHeaders(opts: {
   return headers;
 }
 
+/** Pod 가 호스트에 새 access 를 묻는 창. 이 안이면 URL 이 있을 때만 다시 받는다. */
+export const POD_CREDENTIAL_REFRESH_WINDOW_MS = 15 * 60 * 1000;
+export const POD_CREDENTIAL_URL_ENV = 'ELANOUS_POD_CREDENTIAL_URL';
+export const POD_CREDENTIAL_TOKEN_ENV = 'ELANOUS_POD_CREDENTIAL_TOKEN';
+
+interface PodCredentialRefresh {
+  readonly ok: boolean;
+  readonly reason: 'refreshed' | 'not-expiring' | 'no-env' | 'request-failed' | 'bad-body' | 'write-failed';
+}
+
+function observePodCredentialRefresh(ok: boolean, reason: PodCredentialRefresh['reason']): void {
+  try {
+    // eslint-disable-next-line @typescript-eslint/no-require-imports
+    const { debug } = require('../debug/log.js') as typeof import('../debug/log.js');
+    debug.log('llm.grok', 'pod-credential-refresh', { ok, reason });
+  } catch { /* fail-open — 관측이 자격 해석을 막지 않는다 */ }
+}
+
+/** auth.json 의 access 만 갈아 끼운다. refresh·계정 id 는 건드리지 않고, 실패하면 원본 바이트를 남긴다. */
+function rewriteAuthAccessKey(path: string, key: string, expiresAt: string): boolean {
+  let original: string;
+  try { original = readFileSync(path, 'utf-8'); } catch { return false; }
+  let parsed: unknown;
+  try { parsed = JSON.parse(original); } catch { return false; }
+  if (!parsed || typeof parsed !== 'object') return false;
+  const record = parsed as Record<string, unknown>;
+  let target: string | null = null;
+  let bestMs = Number.NEGATIVE_INFINITY;
+  for (const [name, scope] of Object.entries(record)) {
+    if (!scope || typeof scope !== 'object') continue;
+    const s = scope as GrokAuthScope;
+    if (typeof s.key !== 'string' || s.key.length === 0) continue;
+    const ms = typeof s.expires_at === 'string' ? Date.parse(s.expires_at) : Number.NEGATIVE_INFINITY;
+    const finite = Number.isNaN(ms) ? Number.NEGATIVE_INFINITY : ms;
+    if (target === null || finite > bestMs) { target = name; bestMs = finite; }
+  }
+  if (target === null) return false;
+  const scope = record[target];
+  if (!scope || typeof scope !== 'object') return false;
+  const next = { ...(scope as Record<string, unknown>), key, expires_at: expiresAt };
+  const body = JSON.stringify({ ...record, [target]: next });
+  const tmp = `${path}.${process.pid}.tmp`;
+  try {
+    writeFileSync(tmp, body, { encoding: 'utf-8', mode: 0o600 });
+    renameSync(tmp, path);
+    chmodSync(path, 0o600);
+    return true;
+  } catch {
+    try { writeFileSync(path, original, { encoding: 'utf-8', mode: 0o600 }); } catch { /* 원본이 그대로면 충분 */ }
+    return false;
+  }
+}
+
+/**
+ * Pod 안에서만 — 두 env 가 있고 access 가 15분 안이면 호스트에 새 key 를 받아 auth.json 을 0600 으로 다시 쓴다.
+ * env 가 없거나, 창 밖이거나, 요청이 실패하면 기존 바이트를 유지한다. refresh 는 보내지도 저장하지도 않는다.
+ */
+async function maybeRefreshPodCredential(opts: {
+  env: NodeJS.ProcessEnv;
+  home?: string;
+  now?: () => number;
+  fetchImpl?: typeof fetch;
+}): Promise<void> {
+  const url = opts.env[POD_CREDENTIAL_URL_ENV];
+  const token = opts.env[POD_CREDENTIAL_TOKEN_ENV];
+  if (!url || !token) return;
+  const path = grokAuthFilePath(opts.home);
+  const sub = readSubscriptionToken(path);
+  if (!sub?.expiresAt) {
+    observePodCredentialRefresh(false, 'not-expiring');
+    return;
+  }
+  const ms = Date.parse(sub.expiresAt);
+  if (!Number.isFinite(ms) || (opts.now ?? Date.now)() + POD_CREDENTIAL_REFRESH_WINDOW_MS < ms) {
+    observePodCredentialRefresh(false, 'not-expiring');
+    return;
+  }
+  let res: Response;
+  try {
+    res = await (opts.fetchImpl ?? fetch)(url, {
+      method: 'POST',
+      headers: { authorization: `Bearer ${token}` },
+    });
+  } catch {
+    observePodCredentialRefresh(false, 'request-failed');
+    return;
+  }
+  if (!res.ok) {
+    observePodCredentialRefresh(false, 'request-failed');
+    return;
+  }
+  let body: unknown;
+  try { body = await res.json(); } catch {
+    observePodCredentialRefresh(false, 'bad-body');
+    return;
+  }
+  const key = body && typeof body === 'object' ? (body as { key?: unknown }).key : undefined;
+  const expiresAt = body && typeof body === 'object' ? (body as { expires_at?: unknown }).expires_at : undefined;
+  if (typeof key !== 'string' || key.length === 0 || typeof expiresAt !== 'string' || expiresAt.length === 0) {
+    observePodCredentialRefresh(false, 'bad-body');
+    return;
+  }
+  if (!rewriteAuthAccessKey(path, key, expiresAt)) {
+    observePodCredentialRefresh(false, 'write-failed');
+    return;
+  }
+  observePodCredentialRefresh(true, 'refreshed');
+}
+
 /** ⭐ 단일 규칙 — 구독이 있으면 구독, 없을 때만 API 키.
  *
  *  ⛔ 만료를 여기서 «막지 않는다** — access token 이 지나도 refresh_token 이
  *  살아 있으면 grok 바이너리가 갱신한다. 만료를 불가용으로 접으면 멀쩡한 구독을
  *  API 키로 떨어뜨려 «지갑이 열린다**. 401 이 실제로 나면 호출자가 강등한다. */
-export function resolveGrokCredential(opts: {
+function readGrokCredential(opts: {
   env?: NodeJS.ProcessEnv;
   home?: string;
   model?: string;
@@ -197,7 +315,42 @@ export function resolveGrokCredential(opts: {
   return null;
 }
 
-/** 갱신 유도 결과. ⛔ 「갱신됐다」와 「안 됐다」와 「못 했다」를 다른 값으로. */
+export interface ResolveGrokCredentialOpts {
+  env?: NodeJS.ProcessEnv;
+  home?: string;
+  model?: string;
+  cliVersion?: string;
+  /** 구독을 «건너뛴다** — 401 을 맞은 뒤 호출자가 강등할 때만 쓴다. */
+  skipSubscription?: boolean;
+  now?: () => number;
+  fetchImpl?: typeof fetch;
+}
+
+/**
+ * 자격 해석.
+ * 두 Pod env 가 없으면 동기 `GrokCredential | null` — 종전 호출이 받는 값과 같다.
+ * 둘 다 있으면 Promise. access 가 15분 안일 때만 호스트 URL 의 새 key 로 auth.json 을
+ * 0600(tmp→rename) 다시 쓴 뒤 읽는다. 창 밖·실패는 파일을 그대로 두고 옛 키를 resolve 한다.
+ * 호스트 프로세스는 두 env 를 갖지 않으므로 이 오버로드의 Promise 갈래를 타지 않는다.
+ */
+export function resolveGrokCredential(opts: ResolveGrokCredentialOpts & {
+  env: NodeJS.ProcessEnv & Record<typeof POD_CREDENTIAL_URL_ENV, string> & Record<typeof POD_CREDENTIAL_TOKEN_ENV, string>;
+}): Promise<GrokCredential | null>;
+export function resolveGrokCredential(opts?: ResolveGrokCredentialOpts): GrokCredential | null;
+export function resolveGrokCredential(opts: ResolveGrokCredentialOpts = {}): GrokCredential | null | Promise<GrokCredential | null> {
+  const env = opts.env ?? process.env;
+  if (opts.skipSubscription === true || !env[POD_CREDENTIAL_URL_ENV] || !env[POD_CREDENTIAL_TOKEN_ENV]) {
+    return readGrokCredential(opts);
+  }
+  return maybeRefreshPodCredential({
+    env,
+    ...(opts.home !== undefined ? { home: opts.home } : {}),
+    ...(opts.now ? { now: opts.now } : {}),
+    ...(opts.fetchImpl ? { fetchImpl: opts.fetchImpl } : {}),
+  }).then(() => readGrokCredential(opts));
+}
+
+/** 갱신 유도 결과. ⛔ 「갱신됐다」와 «안 됐다»와 «못 했다»를 다른 값으로. */
 export type GrokRefreshOutcome = 'refreshed' | 'unchanged' | 'failed' | 'no-credential';
 
 /** 토큰의 만료 시각만 뽑는다(비교용). ⛔ 토큰 값은 안 낸다. */
@@ -352,7 +505,7 @@ export interface ResolveFreshGrokOpts {
  */
 export function resolveFreshGrokCredential(opts: ResolveFreshGrokOpts = {}): GrokCredential | null {
   const { execImpl, now, ...resolveOpts } = opts;
-  const cred = resolveGrokCredential(resolveOpts);
+  const cred = readGrokCredential(resolveOpts);
   // API 키는 만료 개념이 없다 · 자격이 없으면 갱신할 것도 없다.
   if (!cred || cred.kind !== 'subscription') return cred;
 
@@ -392,7 +545,7 @@ export function resolveFreshGrokCredential(opts: ResolveFreshGrokOpts = {}): Gro
     blockedMs: clock() - startedAt, timeoutMs: GROK_REFRESH_TIMEOUT_MS,
   });
   // ⛔ 「돌았다」가 아니라 「값이 변했다」로 판정한다 — refreshGrokSubscriptionToken 의 계약 그대로.
-  if (outcome === 'refreshed') return resolveGrokCredential(resolveOpts);
+  if (outcome === 'refreshed') return readGrokCredential(resolveOpts);
 
   // ⛔⭐⭐ 갱신이 실패해도 ***구독을 유지한다*** — 여기서 API 키로 강등하지 «않는다».
   //