elanous 0.2.34 → 0.2.36

package/src/task-orchestrator/surfaces/pod-command-job.ts

@@ -5,7 +5,7 @@
 // codex·elanous 자격 파일은 이 Job 에 없다. 격리 관문(호스트 넥서스에 닿으면 종료)은 기존 Job 과 같다.
 // 명령 인자는 셸 문자열로 이어붙이지 않고 `"$@"` 로 따로 인용한다.
 
-import { randomBytes } from 'node:crypto';
+import { createHash, randomBytes } from 'node:crypto';
 import { spawnSync } from 'node:child_process';
 import { mkdirSync, writeFileSync } from 'node:fs';
 import { isAbsolute, resolve } from 'node:path';
@@ -17,9 +17,10 @@ import { getDefaultLogStore, type LogStore } from '../../mss/logging/log-store.j
 import { podBunCacheVolume } from './pod-bun-cache.js';
 import { POD_JOB_TTL_AFTER_FINISHED_SECONDS } from './pod-lease.js';
 import { podHostDirsCheckScript, podInstallSlotsVolume, prepareHostDirsOnce, type HostDirsPrepResult, podHostDirsScript } from './pod-install-slots.js';
-import { type PodPoolMember, PodPoolScheduler, resolvePodPoolSpec, parsePodPool, checkPodPool, syncPoolImages, type PoolKubectl } from './pod-pool.js';
+import { type PodPoolMember, PodPoolScheduler, resolvePodPoolSpec, parsePodPool, checkPodPool, syncPoolImages, registryImageRef, type PoolKubectl } from './pod-pool.js';
 import { podSkillsDigest, readSkillEnvFiles, resolvePodSkills } from './pod-skills.js';
 import { podSourceScript, type PodSource } from './pod-source-receive.js';
+import { podProvenanceMetadata, type PodProvenance } from './pod-provenance.js';
 import { seedMirrorOnce, type MirrorSeedResult } from './pod-mirror-seed.js';
 import {
   POD_CHILD_REQUESTS,
@@ -201,6 +202,8 @@ if [ -d "$HOME/outbox" ]; then
 fi`;
 
 export interface PodCommandJobInput {
+  provenance?: Partial<PodProvenance>;
+  provenanceGate?: boolean;
   /** POD1 — this launch's own label; cleanup selects by it, never by name alone. */
   launch?: string;
   name: string;
@@ -226,18 +229,37 @@ export interface PodCommandJobInput {
   /** W5 — emptyDir{medium: Memory} sizeLimit mounted at {@link POD_TMPFS_MOUNT_PATH} ⊕ `TMPDIR` (release gate shards only). Omitted = image `/tmp`. */
   tmpfsSizeLimit?: string;
   source?: PodSource;
+  /** Verified producer files, delivered into this Job's own source tree. */
+  warmFiles?: readonly string[];
   deadlineSeconds: number;
   runId?: string;
   /** GATE-WARM-BOOT-NO-INIT fallback: the node-side hand-over did not run, so a root init container hands the hostPath dirs to uid 1000. */
   hostDirsInit?: boolean;
 }
 
+function warmSourceName(name: string, launch: string): string {
+  return `${name.slice(0, 50).replace(/-+$/, '')}-${launch}`;
+}
+
 function bashSingleQuote(value: string): string {
   return `'${value.replace(/'/g, `'\\''`)}'`;
 }
 
+function assertGateProvenanceException(gate: boolean | undefined, provenance: Partial<PodProvenance> | undefined,
+  clone: boolean | undefined, source: PodSource | undefined): void {
+  if (!gate) return;
+  const shardEntrance = process.env.ELANOUS_RELEASE_ENTRANCE?.trim() || 'release-run';
+  const shard = provenance?.kind === 'gate-shard' && provenance.entrance === shardEntrance;
+  const recheck = provenance?.kind === 'gate-recheck' && provenance.entrance === 'release-recheck';
+  if (!(shard || recheck) || clone !== true || source?.kind !== 'commit') {
+    throw new Error('pod provenance gate exception requires a release shard/recheck commit Job');
+  }
+}
+
 /** 명령 Job 매니페스트. `skills`·`llm` 을 안 주면 Secret 키를 하나도 참조하지 않는다. */
 export function podCommandJobManifest(o: PodCommandJobInput): Record<string, unknown> {
+  assertGateProvenanceException(o.provenanceGate, o.provenance, o.clone, o.source);
+  const provenance = podProvenanceMetadata(o.provenance, o.provenanceGate);
   const skillNames = [...new Set(o.skills.map((s) => s.trim()).filter(Boolean))];
   for (const name of skillNames) {
     if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/u.test(name)) throw new Error(`pod command skill name rejected: ${name}`);
@@ -266,6 +288,10 @@ export function podCommandJobManifest(o: PodCommandJobInput): Record<string, unk
     ...(o.clone
       ? ['export GH_TOKEN="$(cat /creds/gh-token)" && gh auth setup-git', podSourceScript(o.source ?? { kind: 'default' }, o.repoUrl)]
       : ['mkdir -p ~/work && cd ~/work']),
+    ...(o.warmFiles?.length ? [
+      `if [ "$(git rev-parse HEAD)" != ${bashSingleQuote(o.source?.kind === 'commit' ? o.source.sha : '')} ]; then echo '[warm-source] commit mismatch'; exit 5; fi`,
+      ...o.warmFiles.map((path, i) => `install -m 644 -D /warm-source/file-${i} ${bashSingleQuote(`./${path}`)} || exit 5\nprintf 'ELANOUS_WARM_SOURCE %s %s %s\\n' ${bashSingleQuote(path)} "$(sha256sum ${bashSingleQuote(`./${path}`)} | cut -d ' ' -f1)" "$(git rev-parse HEAD)"`),
+    ] : []),
     ...(hostDirMounts.length ? [podHostDirsCheckScript(hostDirMounts)] : []),
     ...(bunCache ? [bunCache.shellPrefix] : []),
     `set -- ${quotedArgs}`,
@@ -284,13 +310,13 @@ export function podCommandJobManifest(o: PodCommandJobInput): Record<string, unk
   return {
     apiVersion: 'batch/v1',
     kind: 'Job',
-    metadata: { name: o.name, namespace: o.namespace, labels: { 'elanous.substrate': 'pod', 'elanous.job': o.name, 'elanous.kind': 'command', ...(o.launch ? { 'elanous.launch': o.launch } : {}) } },
+    metadata: { name: o.name, namespace: o.namespace, labels: { 'elanous.substrate': 'pod', 'elanous.job': o.name, 'elanous.kind': 'command', ...(o.launch ? { 'elanous.launch': o.launch } : {}), ...provenance?.labels }, ...(provenance ? { annotations: provenance.annotations } : {}) },
     spec: {
       backoffLimit: 0,
       ttlSecondsAfterFinished: POD_JOB_TTL_AFTER_FINISHED_SECONDS,
       activeDeadlineSeconds: o.deadlineSeconds,
       template: {
-        metadata: { labels: { 'elanous.job': o.name, 'elanous.kind': 'command', ...(o.launch ? { 'elanous.launch': o.launch } : {}) } },
+        metadata: { labels: { 'elanous.job': o.name, 'elanous.kind': 'command', ...(o.launch ? { 'elanous.launch': o.launch } : {}), ...provenance?.labels }, ...(provenance ? { annotations: provenance.annotations } : {}) },
         spec: {
           restartPolicy: 'Never',
           securityContext: { runAsUser: 1000, fsGroup: 1000 },
@@ -306,16 +332,19 @@ export function podCommandJobManifest(o: PodCommandJobInput): Record<string, unk
               ...(o.runId ? [{ name: 'ELANOUS_RUN_ID', value: o.runId }] : []),
               { name: 'ELANOUS_SUBSTRATE', value: 'pod' },
               { name: 'ELANOUS_POD_NAME', valueFrom: { fieldRef: { fieldPath: 'metadata.name' } } },
+              ...(provenance?.env ?? []),
               ...(tmpfs ? [{ name: 'TMPDIR', value: POD_TMPFS_MOUNT_PATH }] : []),
             ],
-            ...(secretKeys || o.hostMirror || hostDirMounts.length || tmpfs ? { volumeMounts: [
+            ...(secretKeys || o.hostMirror || hostDirMounts.length || tmpfs || o.warmFiles?.length ? { volumeMounts: [
+              ...(o.warmFiles?.length ? [{ name: 'warm-source', mountPath: '/warm-source', readOnly: true }] : []),
               ...(secretKeys ? [{ name: 'creds', mountPath: '/creds', readOnly: true }] : []),
               ...(o.hostMirror ? [{ name: 'host-mirror', mountPath: '/host-mirror', readOnly: true }] : []),
               ...hostDirMounts,
               ...tmpfsMounts,
             ] } : {}),
           }],
-          ...(secretKeys || o.hostMirror || hostDirMounts.length || tmpfs ? { volumes: [
+          ...(secretKeys || o.hostMirror || hostDirMounts.length || tmpfs || o.warmFiles?.length ? { volumes: [
+            ...(o.warmFiles?.length ? [{ name: 'warm-source', configMap: { name: warmSourceName(o.name, o.launch ?? 'source') } }] : []),
             ...(secretKeys ? [{ name: 'creds', secret: { secretName: `${o.name}-creds`, defaultMode: 0o400 } }] : []),
             ...(o.hostMirror ? [{ name: 'host-mirror', hostPath: { path: o.hostMirror, type: 'Directory' } }] : []),
             ...(bunCache ? [bunCache.volume] : []),
@@ -382,6 +411,9 @@ export interface PodCommandResult {
 }
 
 export interface RunPodCommandOptions {
+  provenance?: Partial<PodProvenance>;
+  /** Release gate keeps availability: missing required identity becomes unknown and emits a warning. */
+  provenanceGate?: boolean;
   command: readonly string[];
   /** pod run opt-in; other runPodCommand callers keep their existing Job script. */
   returnLogs?: boolean;
@@ -402,6 +434,8 @@ export interface RunPodCommandOptions {
   /** POD7: 호출부가 «네트워크 스킬만 쓰는 명령»이라고 말할 때만 lite 이미지 ⊕ 2Gi. */
   lite?: boolean;
   source?: PodSource;
+  /** Opt-in handoff from an existing producer Pod in the same namespace. */
+  warmProducer?: { pod: string; podUid: string; context: string; image: string; commit: string; files: readonly [string, string] };
   ghToken?: () => string;
   deadlineSeconds?: number;
   namespace?: string;
@@ -434,6 +468,9 @@ export interface RunPodCommandOptions {
   checkPool?: typeof checkPodPool;
   syncImages?: typeof syncPoolImages;
   imageCommit?: string | null;
+  /** Shelf: require this exact registry tag; never fall back to a mutable local image. */
+  shelfImage?: string;
+  shelfLookup?: typeof registryImageRef;
   readSkillEnv?: (skills: readonly string[]) => Record<string, string>;
   grokCredentials?: () => { grokAuth?: string; grokApiKey?: string; ghToken: string };
   artifactsRoot?: string;
@@ -510,6 +547,9 @@ function awaitPodCommandAbort<T>(pending: Promise<T>, signal?: AbortSignal): Pro
 
 /** 풀 자리 · 이미지 판 · Job 적용 · 완료 대기 · 산출 회수 · Secret 정리. */
 export async function runPodCommand(options: RunPodCommandOptions): Promise<PodCommandResult> {
+  // A non-gate Job must have its owner before a pool lease, Secret, or Job is created.
+  assertGateProvenanceException(options.provenanceGate, options.provenance, options.clone, options.source);
+  const provenance = podProvenanceMetadata(options.provenance, options.provenanceGate);
   const nowMs = options.nowMs ?? Date.now;
   const ensureLive = () => {
     if (options.signal?.aborted) throw options.signal.reason ?? new Error('pod command aborted');
@@ -521,11 +561,21 @@ export async function runPodCommand(options: RunPodCommandOptions): Promise<PodC
   const skills = [...new Set((options.skills ?? []).map((s) => s.trim()).filter(Boolean))];
   const llm = options.llm;
   const log = options.log ?? ((c, e, d) => debug.log(c, e, d));
+  if (provenance.missing.length) log('pod.provenance', 'missing-gate-fields', { missing: provenance.missing });
   const baseKubectl = options.kubectl ?? ((args: readonly string[], input?: string) =>
     defaultKubectl(args, input, options.signal ? POD_COMMAND_CANCEL_KUBECTL_TIMEOUT_MS : undefined));
   const sleep = options.sleep ?? ((ms: number) => new Promise<void>((r) => setTimeout(r, ms)));
   const namespace = options.namespace ?? 'elanous-test';
   const image = options.image ?? podCommandImageFor(skills, options.clone === true, options.lite === true);
+  if (options.warmProducer && (!options.clone || options.source?.kind !== 'commit' || options.source.sha !== options.warmProducer.commit || options.image !== options.warmProducer.image || options.imageCommit !== options.warmProducer.commit)) {
+    throw new Error('pod warm source: consumer image and source commit must match producer');
+  }
+  if (options.warmProducer && (!/^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/.test(options.warmProducer.pod) || !options.warmProducer.podUid || typeof options.warmProducer.context !== 'string' || !/^[0-9a-f]{40}$/.test(options.warmProducer.commit) || options.warmProducer.files.length !== 2 || options.warmProducer.files.some((path) => !path || path.startsWith('/') || path.split('/').some((part) => !part || part === '.' || part === '..' || !/^[\w.-]+$/.test(part))))) {
+    throw new Error('pod warm source: invalid producer identity, commit or file path');
+  }
+  if (options.shelfImage && (options.image !== options.shelfImage || !options.imageCommit || !options.clone || options.source?.kind !== 'commit' || options.source.sha !== options.imageCommit)) {
+    throw new Error('pod shelf: image and source commit must match');
+  }
   const memoryLimit = options.memoryLimit ?? (image === POD_LITE_IMAGE ? POD_LITE_MEMORY_LIMIT : undefined);
   const repoUrl = options.repoUrl ?? 'https://github.com/ElanvitalAI/elanous';
   const deadlineSeconds = options.deadlineSeconds ?? POD_COMMAND_DEADLINE_SECONDS;
@@ -579,11 +629,15 @@ export async function runPodCommand(options: RunPodCommandOptions): Promise<PodC
     if (member && pool) pool.release(member);
     ensureLive();
   }
+  if (options.shelfImage && !member) throw new Error('pod shelf: remote pool member required');
   const currentContext = member || options.kubectl ? null : baseKubectl(['config', 'current-context']);
   const context = member?.context ?? (currentContext?.status === 0 ? currentContext.stdout.trim() : '');
   const kubectlRaw: Kubectl = (args, stdin) => baseKubectl(context ? ['--context', context, ...args] : [...args], stdin);
   const kubectl: Kubectl = (args, stdin) => { ensureLive(); const result = kubectlRaw(args, stdin); ensureLive(); return result; };
   const cleanupSecret = () => { kubectlRaw(['-n', namespace, 'delete', 'secret', '-l', own, '--ignore-not-found']); };
+  const cleanupWarmSource = () => {
+    if (options.warmProducer) kubectlRaw(['-n', namespace, 'delete', 'configmap', '-l', own, '--ignore-not-found']);
+  };
   const artifactRoot = options.artifactsRoot ?? `${effectiveInstanceRoot()}/pod-artifacts`;
   const artifactsDir = `${artifactRoot}/${name}`;
   let jobApplied = false;
@@ -617,10 +671,20 @@ export async function runPodCommand(options: RunPodCommandOptions): Promise<PodC
       const localSkillsDigest = options.kubectl ? null : image === POD_LITE_IMAGE
         ? podSkillsDigest(POD_NETWORK_SKILLS, resolve(import.meta.dir, '../../../skills')).digest
         : podSkillsDigest(resolvePodSkills(env).skills).digest;
-      const syncs = await awaitPodCommandAbort((options.syncImages ?? syncPoolImages)([member], image, imageCommit, { localSkillsDigest }), options.signal);
-      const synced = syncs.get(member.context);
-      if (synced && !synced.ok) throw new Error(`pod command: 이미지 판을 못 맞췄다: ${synced.detail}`);
-      if (synced?.imageRef) member = { ...member, imageRef: synced.imageRef };
+      if (options.warmProducer) {
+        // The leased producer is already running this immutable image on the selected pool.
+        // Do not replace it with the host's independently synchronized commit tag.
+        member = { ...member, imageRef: options.warmProducer.image };
+      } else if (options.shelfImage) {
+        const found = (options.shelfLookup ?? registryImageRef)(member, imageCommit);
+        if (found !== options.shelfImage) throw new Error(`pod shelf: selected image unavailable on ${member.context}`);
+        member = { ...member, imageRef: found };
+      } else {
+        const syncs = await awaitPodCommandAbort((options.syncImages ?? syncPoolImages)([member], image, imageCommit, { localSkillsDigest }), options.signal);
+        const synced = syncs.get(member.context);
+        if (synced && !synced.ok) throw new Error(`pod command: 이미지 판을 못 맞췄다: ${synced.detail}`);
+        if (synced?.imageRef) member = { ...member, imageRef: synced.imageRef };
+      }
     }
     const imageRef = member?.imageRef;
     if (!imageRef) log('pod.command-job', 'image-fallback-local', { reason: !member ? 'no-pool-member' : !imageCommit ? 'image-commit-unavailable' : 'registry-tag-unavailable' });
@@ -634,6 +698,33 @@ export async function runPodCommand(options: RunPodCommandOptions): Promise<PodC
         .catch((error: unknown) => ({ host: member!.sshHost!, ok: false, ms: 0, detail: String(error).slice(0, 300) })), options.signal);
       log('pod.command-job', 'mirror-seed', { host: member.sshHost, commit: options.source.sha, ok: seed?.ok ?? false, ms: seed?.ms ?? 0, ...(seed?.detail ? { detail: seed.detail } : {}) });
     }
+    // Only a matching, live producer may hand its two source files to this Job; no other Pod's tree is trusted.
+    const warmHashes: Record<string, string> = {};
+    if (options.warmProducer) {
+      const producer = options.warmProducer;
+      if (context !== producer.context || (imageRef && imageRef !== producer.image)) throw new Error('pod warm source: selected pool or image differs from producer');
+      const got = kubectl(['-n', namespace, 'get', 'pod', producer.pod, '-o', 'json']);
+      if (got.status !== 0) throw new Error('pod warm source: producer Pod unavailable');
+      let pod: { metadata?: { name?: string; uid?: string }; spec?: { containers?: Array<{ name?: string; image?: string }> } };
+      try { pod = JSON.parse(got.stdout); } catch { throw new Error('pod warm source: invalid producer Pod'); }
+      if (pod.metadata?.name !== producer.pod || pod.metadata?.uid !== producer.podUid || pod.spec?.containers?.find((container) => container.name === 'child')?.image !== producer.image) {
+        throw new Error('pod warm source: producer Pod identity or image mismatch');
+      }
+      // kubectl exec starts in the image working directory; podSourceScript cloned
+      // `repo` there, then changed directory only in the original Job shell.
+      const head = kubectl(['-n', namespace, 'exec', producer.pod, '-c', 'child', '--', 'bash', '-c', 'git -C repo rev-parse HEAD']);
+      if (head.status !== 0 || head.stdout.trim() !== producer.commit) throw new Error('pod warm source: producer commit mismatch');
+      const binaryData: Record<string, string> = {};
+      for (const [i, path] of producer.files.entries()) {
+        const read = kubectl(['-n', namespace, 'exec', producer.pod, '-c', 'child', '--', 'bash', '-c', `base64 -w0 ${bashSingleQuote(`repo/${path}`)}`]);
+        if (read.status !== 0 || !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(read.stdout.trim())) throw new Error(`pod warm source: producer file unavailable: ${path}`);
+        binaryData[`file-${i}`] = read.stdout.trim();
+        warmHashes[path] = createHash('sha256').update(Buffer.from(binaryData[`file-${i}`]!, 'base64')).digest('hex');
+      }
+      const config = { apiVersion: 'v1', kind: 'ConfigMap', metadata: { name: warmSourceName(name, launch), namespace, labels: { 'elanous.job': name, 'elanous.launch': launch } }, binaryData };
+      const applied = kubectl(['apply', '-f', '-'], JSON.stringify(config));
+      if (applied.status !== 0) throw new Error(applied.stderr.trim() || 'pod warm source: ConfigMap apply failed');
+    }
     // GATE-WARM-BOOT-NO-INIT (10-15): the hostPath dirs the uid-1000 child writes are handed over on the node itself, once per
     // member·paths in this process (no init container). Best effort: a failure only logs; the child then falls back.
     const hostDirPaths = [...(bunCache ? [bunCache] : []), ...(installSlots ? [installSlots] : [])];
@@ -661,9 +752,10 @@ export async function runPodCommand(options: RunPodCommandOptions): Promise<PodC
     }
     const manifest = podCommandJobManifest({
       name, namespace, launch, image: imageRef ?? image,
+      provenance: options.provenance, provenanceGate: options.provenanceGate,
       ...(imageRef ? { imagePullPolicy: 'IfNotPresent' as const } : {}),
       repoUrl, command, skills, ...(options.returnLogs ? { returnLogs: true } : {}), ...(llm ? { llm } : {}), ...(options.clone ? { clone: true } : {}),
-      ...(options.source ? { source: options.source } : {}), ...(hostMirror ? { hostMirror } : {}), ...(bunCache ? { bunCache } : {}), ...(installSlots ? { installSlots } : {}), ...(memoryLimit ? { memoryLimit } : {}), ...(options.cpu ? { cpu: options.cpu } : {}), ...(options.tmpfsSizeLimit ? { tmpfsSizeLimit: options.tmpfsSizeLimit } : {}), deadlineSeconds,
+      ...(options.source ? { source: options.source } : {}), ...(options.warmProducer ? { warmFiles: options.warmProducer.files } : {}), ...(hostMirror ? { hostMirror } : {}), ...(bunCache ? { bunCache } : {}), ...(installSlots ? { installSlots } : {}), ...(memoryLimit ? { memoryLimit } : {}), ...(options.cpu ? { cpu: options.cpu } : {}), ...(options.tmpfsSizeLimit ? { tmpfsSizeLimit: options.tmpfsSizeLimit } : {}), deadlineSeconds,
       ...(options.runId ? { runId: options.runId } : {}),
       ...(hostDirsInit ? { hostDirsInit: true } : {}),
     });
@@ -680,6 +772,12 @@ export async function runPodCommand(options: RunPodCommandOptions): Promise<PodC
     let markerRc: number | null = null;
     const appliedAt = Date.now();
     for (;;) {
+      // PRLAND-SHADOW-HOST-TIMEOUT: a Pending Pod poll never ends by itself — the host wall-clock deadline ends the
+      // loop here (before another status read) and leaves exactly one timeout record (deadline time ⊕ final state).
+      if (options.hostDeadlineAtMs !== undefined && nowMs() >= options.hostDeadlineAtMs) {
+        log('pod.command', 'host-timeout', { job: name, context, hostDeadlineAtMs: options.hostDeadlineAtMs, nowMs: nowMs(), finalState: 'timeout', waitedMs: nowMs() - appliedAt });
+        throw new Error('pod shadow: host timeout');
+      }
       ensureLive();
       const got = kubectl(['-n', namespace, 'get', 'job', name, '-o', 'jsonpath={.status.conditions[*].type}']);
       const finished = jobFinished(got.stdout);
@@ -706,6 +804,18 @@ export async function runPodCommand(options: RunPodCommandOptions): Promise<PodC
       await awaitPodCommandAbort(sleep(options.pollMs ?? 15_000), options.signal);
     }
     ensureLive();
+    if (options.warmProducer) {
+      const found = kubectl(['-n', namespace, 'get', 'pods', '-l', `job-name=${name}`, '-o', 'json']);
+      let pods: { items?: Array<{ metadata?: { name?: string }; spec?: { containers?: Array<{ name?: string; image?: string }> } }> };
+      try { pods = JSON.parse(found.stdout); } catch { throw new Error('pod warm source: consumer Pod unavailable'); }
+      // A retried Job can leave an older Pod in the list. Never attribute the
+      // warm-source proof to an arbitrary Pod when the consumer is ambiguous.
+      const consumer = pods.items?.length === 1 ? pods.items[0] : undefined;
+      if (found.status !== 0 || !consumer?.metadata?.name || consumer.metadata.name === options.warmProducer.pod || consumer.spec?.containers?.find((container) => container.name === 'child')?.image !== (imageRef ?? image)) {
+        throw new Error('pod warm source: consumer Pod identity or image mismatch');
+      }
+      log('pod.command-job', 'warm-source-linked', { producerPod: options.warmProducer.pod, consumerPod: consumer.metadata.name, image: imageRef ?? image, commit: options.warmProducer.commit, files: [...options.warmProducer.files] });
+    }
     let logsResult = kubectl(['-n', namespace, 'logs', `job/${name}`, '-c', 'child']);
     // Marker path returns before the Job condition — a full-log read can hit the same transient window the tail read
     // already survived (container log rotation · API hiccup). Retry it a little before calling the artifacts lost.
@@ -723,6 +833,13 @@ export async function runPodCommand(options: RunPodCommandOptions): Promise<PodC
     }
     const logsFailed = logsResult.status !== 0;
     const logs = logsFailed ? '' : splitPodResultMarker(logsResult.stdout, launch).logs;
+    if (options.warmProducer) {
+      if (logsFailed) throw new Error(`pod warm source: consumer logs unavailable: ${logsResult.stderr.trim() || logsResult.status}`);
+      for (const path of options.warmProducer.files) {
+        const marker = `ELANOUS_WARM_SOURCE ${path} ${warmHashes[path]} ${options.warmProducer.commit}`;
+        if (logs.split(/\r?\n/).filter((line) => line === marker).length !== 1) throw new Error(`pod warm source: consumer file or commit mismatch: ${path}`);
+      }
+    }
     const artifactDir = artifactRoot;
     let artifacts: PodArtifactCollection;
     if (logsFailed) {
@@ -758,6 +875,7 @@ export async function runPodCommand(options: RunPodCommandOptions): Promise<PodC
     // Marker path: the rc the child printed (the Pod's terminated state may lag like the condition). Condition path: unchanged.
     const exitCode = markerRc ?? exitCodeFromPod(kubectl, namespace, name, state);
     cleanupSecret();
+    cleanupWarmSource();
     log('pod.command', 'job-finished', {
       job: name,
       exitCode,
@@ -777,6 +895,7 @@ export async function runPodCommand(options: RunPodCommandOptions): Promise<PodC
       catch { /* preserve the cancellation reason */ }
     }
     cleanupSecret();
+    cleanupWarmSource();
     throw err;
   } finally {
     if (member && pool) pool.release(member);

package/src/task-orchestrator/surfaces/pod-failure-reason.ts

@@ -96,7 +96,7 @@ export function podTerminalRow(row: Record<string, unknown>): {
   return nestedPodResult(row);
 }
 
-export const POD_EXIT_CLASSES = ['oom-killed', 'deadline-exceeded', 'gh-login-failed', 'pod-secret', 'ask-reprompt', 'child-unconverged', 'child-error', 'no-result-line', 'unknown'] as const;
+export const POD_EXIT_CLASSES = ['oom-killed', 'deadline-exceeded', 'gh-login-failed', 'pod-secret', 'ask-reprompt', 'provider-error', 'child-unconverged', 'child-error', 'no-result-line', 'unknown'] as const;
 
 export type PodExitClass = typeof POD_EXIT_CLASSES[number];
 
@@ -108,6 +108,51 @@ const isPodErrorLine = (line: string): boolean =>
 
 const normalizePodLogLine = (line: string): string => line.replace(/\/home\/[^/\s]+\//gu, '~/').slice(0, 240);
 
+/**
+ * PROVIDER-ERROR-PRIORITY: 모델 호출 쪽이 죽은 줄(자격증명·쿼터·과부하·업스트림 5xx).
+ * 자식이 스스로 수렴 못 한 것과 사유가 다르므로 generic 실패(child-error/unknown)보다 먼저 분류한다.
+ */
+const POD_PROVIDER_ERROR_LINE =
+  /\bllm\.[a-z0-9_-]+ (?:credential|response|stream)\b.*\b(?:error|fail(?:ed|ure)?|denied|unavailable|overloaded|exhausted)\b|\b(?:rate[_ -]?limit|quota|credits?)\b.*\b(?:exceeded|exhausted|insufficient|hit)\b|\b(?:provider|upstream)\b.*\b(?:error|overloaded|unavailable|timed? out|5\d\d)\b|(?:프로바이더|제공자|모델)\s*(?:오류|호출 실패|한도 초과)/iu;
+
+/** provider error 줄인가 — bookkeeping 줄과 «파싱되는 JSON 줄»은 세지 않는다(classifyPodExit 의 줄 기준과 같다). */
+function isPodProviderErrorLine(line: string): boolean {
+  if (!line || isPodBookkeepingLine(line)) return false;
+  if (line.startsWith('{')) {
+    try {
+      const parsed: unknown = JSON.parse(line);
+      if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) return false;
+    } catch { /* 일반 줄이 중괄호로 시작할 수 있다. */ }
+  }
+  return POD_PROVIDER_ERROR_LINE.test(line);
+}
+
+/**
+ * POD-FAIL-EVIDENCE-CAPTURE-FU — duplicate-cap 귀속: 마지막 «canonical» 자식 terminal 하나에만 실패를 붙인다.
+ * canonical 접두(`self`/`external`)가 아닌 lookalike kind 행은 건너뛰고, 마지막 canonical terminal 이 성공이거나
+ * 읽을 오류 줄이 bookkeeping 뿐이면 null — 더 오래된(다른 terminal 의) 실패를 재사용하지 않는다.
+ */
+export function podChildFailureAttribution(logs: string): { stage: string; error: string } | null {
+  for (const line of logs.split('\n').reverse()) {
+    if (!line.trimStart().startsWith('{')) continue;
+    let parsed: unknown;
+    try { parsed = JSON.parse(line); } catch { continue; }
+    if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) continue;
+    const row = parsed as Record<string, unknown>;
+    const terminal = podTerminalRow(row);
+    if (!terminal) continue;
+    if (terminal.ok !== false || typeof terminal.error !== 'string') return null;
+    const errorLines = terminal.error.replace(/\/home\/[^/\s]+\//gu, '~/').split(/\r\n|\n|\r/u)
+      .map((part) => part.trim())
+      .filter((part) => part.length > 0 && !isPodBookkeepingLine(part));
+    if (!errorLines.length) return null;
+    const first = errorLines[0]!.slice(0, 240);
+    const last = errorLines.at(-1)!.slice(0, 240);
+    return { stage: terminal.stage, error: first === last ? first : `${first}\n${last}` };
+  }
+  return null;
+}
+
 /** Classify a failed Pod using its terminal metadata and child output, without side effects. */
 export function classifyPodExit(input: {
   errorCode?: string | null;
@@ -131,12 +176,19 @@ export function classifyPodExit(input: {
 } {
   const logLines = input.logs.split(/\r\n|\n|\r/u).map((line) => line.trim()).filter(Boolean);
   let terminalResult: ReturnType<typeof podTerminalRow> = null;
-  for (const line of logLines) {
+  let lastTerminalIndex = -1;
+  let previousTerminalIndex = -1;
+  for (const [index, line] of logLines.entries()) {
     if (!line.startsWith('{')) continue;
     let parsed: unknown;
     try { parsed = JSON.parse(line); } catch { continue; }
     if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) continue;
-    terminalResult = podTerminalRow(parsed as Record<string, unknown>) ?? terminalResult;
+    const terminal = podTerminalRow(parsed as Record<string, unknown>);
+    if (!terminal) continue;
+    // 마지막 canonical terminal 만 유지 — 앞선 terminal 의 실패를 재사용하지 않는다(duplicate-cap).
+    previousTerminalIndex = lastTerminalIndex;
+    terminalResult = terminal;
+    lastTerminalIndex = index;
   }
   const hasResultLine = terminalResult !== null;
   if (!terminalResult && input.result) terminalResult = podTerminalRow(input.result);
@@ -148,12 +200,19 @@ export function classifyPodExit(input: {
     try { const parsed = JSON.parse(line); return !!parsed && typeof parsed === 'object' && !Array.isArray(parsed); } catch { return false; }
   };
   const hasErrorLine = logLines.some((line) => !isPodBookkeepingLine(line) && !isParsedJsonLine(line) && isPodErrorLine(line));
+  // 동일 terminal 직전만: 마지막 canonical terminal 에 딸린 구간(직전 terminal 다음 ~ 마지막 terminal)의 줄만
+  // provider error 근거로 쓴다 — 오래된 terminal 의 실패를 재사용하지 않는다(duplicate-cap).
+  const attributionLines = lastTerminalIndex === -1
+    ? logLines
+    : logLines.slice(previousTerminalIndex + 1, lastTerminalIndex);
+  const providerError = attributionLines.some((line) => isPodProviderErrorLine(line));
   let exitClass: PodExitClass;
   if (input.errorCode === 'pod-secret') exitClass = 'pod-secret';
   else if (input.jobReason === 'DeadlineExceeded') exitClass = 'deadline-exceeded';
   else if (input.containerReason === 'OOMKilled') exitClass = 'oom-killed';
   else if (input.containerExitCode === 7) exitClass = 'gh-login-failed';
   else if (askReprompt) exitClass = 'ask-reprompt';
+  else if (providerError) exitClass = 'provider-error';
   else if (terminalResult?.stage && terminalResult.ok === false) exitClass = 'child-unconverged';
   else if (!hasResultLine && hasErrorLine) exitClass = 'child-error';
   else if (!hasResultLine) exitClass = 'no-result-line';
@@ -245,3 +304,156 @@ export function extractPodFailureReason(input: {
   const why = input.logTailReason?.trim() || (input.logs.trim() ? '로그에 읽을 수 있는 오류 줄 없음' : '자식 로그 비어 있음');
   return `사유 못 읽음: ${normalizePodLogLine(why)}`;
 }
+
+// JOB-FAILURE-TERMINATION-CAPTURE — the Kubernetes facts of a failed Job, read once at the failure verdict
+//   (Job Failed condition · result marker early exit) before GC removes the Job and its Pod. Unobserved stays
+//   `unknown` — `no-result-line` is never promoted to OOM or deadline here. Only these scalar fields are kept:
+//   no Pod env, Secret, or describe output.
+export type PodTerminationPath = 'job-failed-condition' | 'result-marker';
+export type PodTerminationUnknown = 'unknown';
+
+export interface PodTermination {
+  job: string;
+  path: PodTerminationPath;
+  observedAt: string;
+  /** `ok` only when the Job read names this launch's uid · `unavailable`: failed (NotFound after GC · API error),
+   *  a different or missing uid, or no launch uid to attribute it to. */
+  jobLookup: 'ok' | 'unavailable';
+  /** This launch's Job uid (the attribution key) — `unknown` when the launcher had none, and then nothing is attributed. */
+  jobUid: string | PodTerminationUnknown;
+  jobConditionReason: string | PodTerminationUnknown;
+  /** The Job's configured `spec.activeDeadlineSeconds` — a setting, not evidence the deadline fired. */
+  activeDeadlineSeconds: number | PodTerminationUnknown;
+  /** From the Job Failed condition reason `DeadlineExceeded` only. */
+  deadlineExceeded: boolean | PodTerminationUnknown;
+  /** `terminated`: the latest child container of this uid has a terminated state · without one, the Pod phase
+   *  (`running` · `pending`) or `unknown` (any other or unread phase) · `absent`: no Pod of this uid. */
+  podLookup: 'terminated' | 'running' | 'pending' | 'unknown' | 'absent' | 'unavailable';
+  containerReason: string | PodTerminationUnknown;
+  exitCode: number | PodTerminationUnknown;
+  finishedAt: string | PodTerminationUnknown;
+  /** From `state.terminated.reason === 'OOMKilled'` only (never from exit code 137). */
+  oomKilled: boolean | PodTerminationUnknown;
+}
+
+/** Job Failed condition reason first: the classification takes column one, the capture all three, from one read. */
+export const POD_TERMINATION_JOB_JSONPATH = '{.status.conditions[?(@.type=="Failed")].reason}{"\\t"}{.spec.activeDeadlineSeconds}{"\\t"}{.metadata.uid}';
+/**
+ * One row per Pod of this Job name. The first three columns keep the classification read's layout
+ * (`creation · terminated.reason · terminated.exitCode`, empty when the child has no status yet), so the
+ * Failed-condition path reads Pods once for both; `finishedAt` follows, then `|phase · owner Job uid`.
+ */
+export const POD_TERMINATION_PODS_JSONPATH = '{range .items[*]}{.metadata.creationTimestamp}{"\\t"}{range .status.containerStatuses[?(@.name=="child")]}{.state.terminated.reason}{"\\t"}{.state.terminated.exitCode}{"\\t"}{.state.terminated.finishedAt}{end}{"\\t"}{"\\t"}{"\\t"}{"|"}{.status.phase}{"\\t"}{.metadata.ownerReferences[?(@.kind=="Job")].uid}{"\\n"}{end}';
+
+type PodTerminationKubectl = (args: readonly string[]) => { status: number | null; stdout: string; stderr: string };
+
+const POD_TERMINATION_WORD = /^[A-Za-z][A-Za-z0-9_.:-]{0,63}$/;
+const JOB_UID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/iu;
+const terminationWord = (value: string | undefined): string => {
+  const text = value?.trim() ?? '';
+  return POD_TERMINATION_WORD.test(text) ? text : 'unknown';
+};
+const terminationCount = (value: string | undefined): number | PodTerminationUnknown => {
+  const text = value?.trim() ?? '';
+  return /^\d{1,10}$/.test(text) ? Number(text) : 'unknown';
+};
+const terminationInstant = (value: string | undefined): string => {
+  const text = value?.trim() ?? '';
+  return /^\d{4}-\d{2}-\d{2}T[\d:.]+Z$/.test(text) && Number.isFinite(Date.parse(text)) ? text : 'unknown';
+};
+
+/** Two reads (the Job · its Pods), each fail-soft: a failed or thrown read is `unavailable`, never an inferred value. */
+export function readPodTermination(input: { kubectl: PodTerminationKubectl; namespace: string; job: string; path: PodTerminationPath;
+  /** This launch's Job uid — the only attribution key. A same-name Job of another round (OOM retry · relaunch)
+   *  is not this verdict's state; without it every Job/Pod fact stays unobserved. */
+  jobUid: string | undefined;
+  /** The Failed-condition path's own Job read of `POD_TERMINATION_JOB_JSONPATH` (null: it threw) — not read again,
+   *  so a GC between the verdict and the capture cannot turn an observed reason into `unknown`. */
+  jobRead?: { status: number | null; stdout: string } | null;
+  /** The Failed-condition path's own Pods read of `POD_TERMINATION_PODS_JSONPATH` (null: it threw) — not read again. */
+  podsRead?: { status: number | null; stdout: string } | null;
+  now?: () => Date }): PodTermination {
+  const { kubectl, namespace, job, path } = input;
+  const read = (args: readonly string[]): { status: number | null; stdout: string } | null => {
+    try { return kubectl(args); } catch { return null; }
+  };
+  const jobRead = 'jobRead' in input ? input.jobRead ?? null
+    : read(['-n', namespace, 'get', 'job', job, '-o', `jsonpath=${POD_TERMINATION_JOB_JSONPATH}`]);
+  const [reasonText, deadlineText, uidText] = jobRead?.status === 0 ? jobRead.stdout.replace(/\r?\n$/, '').split('\t') : [];
+  const readUid = JOB_UID.test(uidText?.trim() ?? '') ? uidText!.trim() : undefined;
+  const jobUid = input.jobUid && JOB_UID.test(input.jobUid) ? input.jobUid : undefined;
+  // A read without a uid, or with another uid, is not this launch's Job.
+  const jobOk = jobRead?.status === 0 && jobUid !== undefined && readUid === jobUid;
+  const jobConditionReason = jobOk ? terminationWord(reasonText) : 'unknown';
+  const podsRead = 'podsRead' in input ? input.podsRead ?? null
+    : read(['-n', namespace, 'get', 'pods', '-l', `job-name=${job}`, '-o', `jsonpath=${POD_TERMINATION_PODS_JSONPATH}`]);
+  let podLookup: PodTermination['podLookup'] = 'unavailable';
+  let containerReason = 'unknown';
+  let exitCode: PodTermination['exitCode'] = 'unknown';
+  let finishedAt = 'unknown';
+  // Without a uid the rows cannot be attributed to this Job — they stay unread rather than borrowed from another round.
+  if (podsRead?.status === 0 && jobUid) {
+    // The latest Pod owned by this Job uid only.
+    const rows = podsRead.stdout.split('\n').map((line) => {
+      const cut = line.lastIndexOf('\t|');
+      const [created, reason, exit, finished] = (cut >= 0 ? line.slice(0, cut) : line).split('\t');
+      const [phase, owner] = cut >= 0 ? line.slice(cut + 2).split('\t') : [];
+      return { created: created ?? '', reason, exit, finished, phase: phase?.trim(), owner: owner?.trim() };
+    });
+    const owned = rows.filter((row) => row.created && Number.isFinite(Date.parse(row.created)) && row.owner === jobUid)
+      .sort((a, b) => Date.parse(b.created) - Date.parse(a.created));
+    const latest = owned[0];
+    // Two Pods of this uid created at the same instant (a Job retry): which one is «this» failure is not knowable.
+    const tied = latest !== undefined && owned.length > 1 && Date.parse(owned[1]!.created) === Date.parse(latest.created);
+    if (!latest) podLookup = 'absent';
+    else if (tied) podLookup = 'unknown';
+    else {
+      const { reason, exit, finished, phase } = latest;
+      containerReason = terminationWord(reason);
+      exitCode = terminationCount(exit);
+      finishedAt = terminationInstant(finished);
+      podLookup = containerReason !== 'unknown' || exitCode !== 'unknown' ? 'terminated'
+        : phase === 'Running' ? 'running' : phase === 'Pending' ? 'pending' : 'unknown';
+    }
+  }
+  return {
+    job, path, observedAt: (input.now?.() ?? new Date()).toISOString(),
+    jobLookup: jobOk ? 'ok' : 'unavailable',
+    jobUid: jobUid ?? 'unknown',
+    jobConditionReason,
+    activeDeadlineSeconds: jobOk ? terminationCount(deadlineText) : 'unknown',
+    deadlineExceeded: jobConditionReason === 'unknown' ? 'unknown' : jobConditionReason === 'DeadlineExceeded',
+    podLookup, containerReason, exitCode, finishedAt,
+    oomKilled: podLookup === 'terminated' && containerReason !== 'unknown' ? containerReason === 'OOMKilled' : 'unknown',
+  };
+}
+
+/** Ledger → incident: only a well-formed record of `job` (the current attempt's Job) is carried; anything else is dropped. */
+export function parsePodTermination(value: unknown, job: string): PodTermination | undefined {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) return undefined;
+  const row = value as Record<string, unknown>;
+  const oneOf = (v: unknown, allowed: readonly unknown[]) => allowed.includes(v);
+  const wordOrUnknown = (v: unknown) => typeof v === 'string' && (v === 'unknown' || POD_TERMINATION_WORD.test(v));
+  const numberOrUnknown = (v: unknown) => v === 'unknown' || (typeof v === 'number' && Number.isSafeInteger(v) && v >= 0);
+  const boolOrUnknown = (v: unknown) => v === 'unknown' || typeof v === 'boolean';
+  if (row.job !== job || !oneOf(row.path, ['job-failed-condition', 'result-marker'])
+    || typeof row.observedAt !== 'string' || !Number.isFinite(Date.parse(row.observedAt))
+    || !oneOf(row.jobLookup, ['ok', 'unavailable']) || !wordOrUnknown(row.jobConditionReason)
+    || !(row.jobUid === 'unknown' || (typeof row.jobUid === 'string' && JOB_UID.test(row.jobUid)))
+    || !numberOrUnknown(row.activeDeadlineSeconds) || !boolOrUnknown(row.deadlineExceeded)
+    || !oneOf(row.podLookup, ['terminated', 'running', 'pending', 'unknown', 'absent', 'unavailable'])
+    || !wordOrUnknown(row.containerReason) || !numberOrUnknown(row.exitCode)
+    || !(row.finishedAt === 'unknown' || (typeof row.finishedAt === 'string' && terminationInstant(row.finishedAt) !== 'unknown'))
+    || !boolOrUnknown(row.oomKilled)) return undefined;
+  // Facts are attributed through the uid only: a record claiming Job or Pod facts without one is contradictory.
+  if (row.jobUid === 'unknown' && (row.jobLookup !== 'unavailable' || row.podLookup !== 'unavailable')) return undefined;
+  return {
+    job, path: row.path as PodTerminationPath, observedAt: row.observedAt,
+    jobLookup: row.jobLookup as PodTermination['jobLookup'], jobUid: row.jobUid as string, jobConditionReason: row.jobConditionReason as string,
+    activeDeadlineSeconds: row.activeDeadlineSeconds as PodTermination['activeDeadlineSeconds'],
+    deadlineExceeded: row.deadlineExceeded as PodTermination['deadlineExceeded'],
+    podLookup: row.podLookup as PodTermination['podLookup'], containerReason: row.containerReason as string,
+    exitCode: row.exitCode as PodTermination['exitCode'], finishedAt: row.finishedAt as string,
+    oomKilled: row.oomKilled as PodTermination['oomKilled'],
+  };
+}