elanous 0.2.34 → 0.2.35

package/src/task-orchestrator/surfaces/pod-command-job.ts

@@ -17,9 +17,10 @@ import { getDefaultLogStore, type LogStore } from '../../mss/logging/log-store.j
 import { podBunCacheVolume } from './pod-bun-cache.js';
 import { POD_JOB_TTL_AFTER_FINISHED_SECONDS } from './pod-lease.js';
 import { podHostDirsCheckScript, podInstallSlotsVolume, prepareHostDirsOnce, type HostDirsPrepResult, podHostDirsScript } from './pod-install-slots.js';
-import { type PodPoolMember, PodPoolScheduler, resolvePodPoolSpec, parsePodPool, checkPodPool, syncPoolImages, type PoolKubectl } from './pod-pool.js';
+import { type PodPoolMember, PodPoolScheduler, resolvePodPoolSpec, parsePodPool, checkPodPool, syncPoolImages, registryImageRef, type PoolKubectl } from './pod-pool.js';
 import { podSkillsDigest, readSkillEnvFiles, resolvePodSkills } from './pod-skills.js';
 import { podSourceScript, type PodSource } from './pod-source-receive.js';
+import { podProvenanceMetadata, type PodProvenance } from './pod-provenance.js';
 import { seedMirrorOnce, type MirrorSeedResult } from './pod-mirror-seed.js';
 import {
   POD_CHILD_REQUESTS,
@@ -201,6 +202,8 @@ if [ -d "$HOME/outbox" ]; then
 fi`;
 
 export interface PodCommandJobInput {
+  provenance?: Partial<PodProvenance>;
+  provenanceGate?: boolean;
   /** POD1 — this launch's own label; cleanup selects by it, never by name alone. */
   launch?: string;
   name: string;
@@ -236,8 +239,21 @@ function bashSingleQuote(value: string): string {
   return `'${value.replace(/'/g, `'\\''`)}'`;
 }
 
+function assertGateProvenanceException(gate: boolean | undefined, provenance: Partial<PodProvenance> | undefined,
+  clone: boolean | undefined, source: PodSource | undefined): void {
+  if (!gate) return;
+  const shardEntrance = process.env.ELANOUS_RELEASE_ENTRANCE?.trim() || 'release-run';
+  const shard = provenance?.kind === 'gate-shard' && provenance.entrance === shardEntrance;
+  const recheck = provenance?.kind === 'gate-recheck' && provenance.entrance === 'release-recheck';
+  if (!(shard || recheck) || clone !== true || source?.kind !== 'commit') {
+    throw new Error('pod provenance gate exception requires a release shard/recheck commit Job');
+  }
+}
+
 /** 명령 Job 매니페스트. `skills`·`llm` 을 안 주면 Secret 키를 하나도 참조하지 않는다. */
 export function podCommandJobManifest(o: PodCommandJobInput): Record<string, unknown> {
+  assertGateProvenanceException(o.provenanceGate, o.provenance, o.clone, o.source);
+  const provenance = podProvenanceMetadata(o.provenance, o.provenanceGate);
   const skillNames = [...new Set(o.skills.map((s) => s.trim()).filter(Boolean))];
   for (const name of skillNames) {
     if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/u.test(name)) throw new Error(`pod command skill name rejected: ${name}`);
@@ -284,13 +300,13 @@ export function podCommandJobManifest(o: PodCommandJobInput): Record<string, unk
   return {
     apiVersion: 'batch/v1',
     kind: 'Job',
-    metadata: { name: o.name, namespace: o.namespace, labels: { 'elanous.substrate': 'pod', 'elanous.job': o.name, 'elanous.kind': 'command', ...(o.launch ? { 'elanous.launch': o.launch } : {}) } },
+    metadata: { name: o.name, namespace: o.namespace, labels: { 'elanous.substrate': 'pod', 'elanous.job': o.name, 'elanous.kind': 'command', ...(o.launch ? { 'elanous.launch': o.launch } : {}), ...provenance?.labels }, ...(provenance ? { annotations: provenance.annotations } : {}) },
     spec: {
       backoffLimit: 0,
       ttlSecondsAfterFinished: POD_JOB_TTL_AFTER_FINISHED_SECONDS,
       activeDeadlineSeconds: o.deadlineSeconds,
       template: {
-        metadata: { labels: { 'elanous.job': o.name, 'elanous.kind': 'command', ...(o.launch ? { 'elanous.launch': o.launch } : {}) } },
+        metadata: { labels: { 'elanous.job': o.name, 'elanous.kind': 'command', ...(o.launch ? { 'elanous.launch': o.launch } : {}), ...provenance?.labels }, ...(provenance ? { annotations: provenance.annotations } : {}) },
         spec: {
           restartPolicy: 'Never',
           securityContext: { runAsUser: 1000, fsGroup: 1000 },
@@ -306,6 +322,7 @@ export function podCommandJobManifest(o: PodCommandJobInput): Record<string, unk
               ...(o.runId ? [{ name: 'ELANOUS_RUN_ID', value: o.runId }] : []),
               { name: 'ELANOUS_SUBSTRATE', value: 'pod' },
               { name: 'ELANOUS_POD_NAME', valueFrom: { fieldRef: { fieldPath: 'metadata.name' } } },
+              ...(provenance?.env ?? []),
               ...(tmpfs ? [{ name: 'TMPDIR', value: POD_TMPFS_MOUNT_PATH }] : []),
             ],
             ...(secretKeys || o.hostMirror || hostDirMounts.length || tmpfs ? { volumeMounts: [
@@ -382,6 +399,9 @@ export interface PodCommandResult {
 }
 
 export interface RunPodCommandOptions {
+  provenance?: Partial<PodProvenance>;
+  /** Release gate keeps availability: missing required identity becomes unknown and emits a warning. */
+  provenanceGate?: boolean;
   command: readonly string[];
   /** pod run opt-in; other runPodCommand callers keep their existing Job script. */
   returnLogs?: boolean;
@@ -434,6 +454,9 @@ export interface RunPodCommandOptions {
   checkPool?: typeof checkPodPool;
   syncImages?: typeof syncPoolImages;
   imageCommit?: string | null;
+  /** Shelf: require this exact registry tag; never fall back to a mutable local image. */
+  shelfImage?: string;
+  shelfLookup?: typeof registryImageRef;
   readSkillEnv?: (skills: readonly string[]) => Record<string, string>;
   grokCredentials?: () => { grokAuth?: string; grokApiKey?: string; ghToken: string };
   artifactsRoot?: string;
@@ -510,6 +533,9 @@ function awaitPodCommandAbort<T>(pending: Promise<T>, signal?: AbortSignal): Pro
 
 /** 풀 자리 · 이미지 판 · Job 적용 · 완료 대기 · 산출 회수 · Secret 정리. */
 export async function runPodCommand(options: RunPodCommandOptions): Promise<PodCommandResult> {
+  // A non-gate Job must have its owner before a pool lease, Secret, or Job is created.
+  assertGateProvenanceException(options.provenanceGate, options.provenance, options.clone, options.source);
+  const provenance = podProvenanceMetadata(options.provenance, options.provenanceGate);
   const nowMs = options.nowMs ?? Date.now;
   const ensureLive = () => {
     if (options.signal?.aborted) throw options.signal.reason ?? new Error('pod command aborted');
@@ -521,11 +547,15 @@ export async function runPodCommand(options: RunPodCommandOptions): Promise<PodC
   const skills = [...new Set((options.skills ?? []).map((s) => s.trim()).filter(Boolean))];
   const llm = options.llm;
   const log = options.log ?? ((c, e, d) => debug.log(c, e, d));
+  if (provenance.missing.length) log('pod.provenance', 'missing-gate-fields', { missing: provenance.missing });
   const baseKubectl = options.kubectl ?? ((args: readonly string[], input?: string) =>
     defaultKubectl(args, input, options.signal ? POD_COMMAND_CANCEL_KUBECTL_TIMEOUT_MS : undefined));
   const sleep = options.sleep ?? ((ms: number) => new Promise<void>((r) => setTimeout(r, ms)));
   const namespace = options.namespace ?? 'elanous-test';
   const image = options.image ?? podCommandImageFor(skills, options.clone === true, options.lite === true);
+  if (options.shelfImage && (options.image !== options.shelfImage || !options.imageCommit || !options.clone || options.source?.kind !== 'commit' || options.source.sha !== options.imageCommit)) {
+    throw new Error('pod shelf: image and source commit must match');
+  }
   const memoryLimit = options.memoryLimit ?? (image === POD_LITE_IMAGE ? POD_LITE_MEMORY_LIMIT : undefined);
   const repoUrl = options.repoUrl ?? 'https://github.com/ElanvitalAI/elanous';
   const deadlineSeconds = options.deadlineSeconds ?? POD_COMMAND_DEADLINE_SECONDS;
@@ -579,6 +609,7 @@ export async function runPodCommand(options: RunPodCommandOptions): Promise<PodC
     if (member && pool) pool.release(member);
     ensureLive();
   }
+  if (options.shelfImage && !member) throw new Error('pod shelf: remote pool member required');
   const currentContext = member || options.kubectl ? null : baseKubectl(['config', 'current-context']);
   const context = member?.context ?? (currentContext?.status === 0 ? currentContext.stdout.trim() : '');
   const kubectlRaw: Kubectl = (args, stdin) => baseKubectl(context ? ['--context', context, ...args] : [...args], stdin);
@@ -617,10 +648,16 @@ export async function runPodCommand(options: RunPodCommandOptions): Promise<PodC
       const localSkillsDigest = options.kubectl ? null : image === POD_LITE_IMAGE
         ? podSkillsDigest(POD_NETWORK_SKILLS, resolve(import.meta.dir, '../../../skills')).digest
         : podSkillsDigest(resolvePodSkills(env).skills).digest;
-      const syncs = await awaitPodCommandAbort((options.syncImages ?? syncPoolImages)([member], image, imageCommit, { localSkillsDigest }), options.signal);
-      const synced = syncs.get(member.context);
-      if (synced && !synced.ok) throw new Error(`pod command: 이미지 판을 못 맞췄다: ${synced.detail}`);
-      if (synced?.imageRef) member = { ...member, imageRef: synced.imageRef };
+      if (options.shelfImage) {
+        const found = (options.shelfLookup ?? registryImageRef)(member, imageCommit);
+        if (found !== options.shelfImage) throw new Error(`pod shelf: selected image unavailable on ${member.context}`);
+        member = { ...member, imageRef: found };
+      } else {
+        const syncs = await awaitPodCommandAbort((options.syncImages ?? syncPoolImages)([member], image, imageCommit, { localSkillsDigest }), options.signal);
+        const synced = syncs.get(member.context);
+        if (synced && !synced.ok) throw new Error(`pod command: 이미지 판을 못 맞췄다: ${synced.detail}`);
+        if (synced?.imageRef) member = { ...member, imageRef: synced.imageRef };
+      }
     }
     const imageRef = member?.imageRef;
     if (!imageRef) log('pod.command-job', 'image-fallback-local', { reason: !member ? 'no-pool-member' : !imageCommit ? 'image-commit-unavailable' : 'registry-tag-unavailable' });
@@ -661,6 +698,7 @@ export async function runPodCommand(options: RunPodCommandOptions): Promise<PodC
     }
     const manifest = podCommandJobManifest({
       name, namespace, launch, image: imageRef ?? image,
+      provenance: options.provenance, provenanceGate: options.provenanceGate,
       ...(imageRef ? { imagePullPolicy: 'IfNotPresent' as const } : {}),
       repoUrl, command, skills, ...(options.returnLogs ? { returnLogs: true } : {}), ...(llm ? { llm } : {}), ...(options.clone ? { clone: true } : {}),
       ...(options.source ? { source: options.source } : {}), ...(hostMirror ? { hostMirror } : {}), ...(bunCache ? { bunCache } : {}), ...(installSlots ? { installSlots } : {}), ...(memoryLimit ? { memoryLimit } : {}), ...(options.cpu ? { cpu: options.cpu } : {}), ...(options.tmpfsSizeLimit ? { tmpfsSizeLimit: options.tmpfsSizeLimit } : {}), deadlineSeconds,

package/src/task-orchestrator/surfaces/pod-image-source.ts

@@ -14,9 +14,12 @@
  *   이미지는 표준판(elanous-harness)만 — 이 발사 경로는 lite 를 쓰지 않는다.
  */
 import { spawnSync } from 'node:child_process';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
 import { SPAWN_SYNC_MAX_BUFFER } from '../../util/spawn-sync-output.js';
 import { debug } from '../../debug/log.js';
-import { memberHasRegistry, registryImageRef, type RemoteRun } from './pod-pool.js';
+import { memberHasRegistry, registryImageRef, syncPoolImages, parsePodPool, type PodPoolMember, type RemoteRun } from './pod-pool.js';
 
 const POD_IMAGE = 'elanous-harness';
 
@@ -59,6 +62,91 @@ export function resolvePodImageSource(
   return { kind: 'pinned', ref: podImageRef(registry, commit), commit, registry };
 }
 
+export interface PodShelfImage { readonly commit: string; readonly image: string; readonly action: 'fresh' | 'built' }
+
+export function resolvePodShelfRef(ref: string, cwd = process.cwd(), exec = spawnSync): string {
+  const requested = ref.trim();
+  if (!requested || requested.startsWith('-') || /[\s\x00-\x1f]/u.test(requested)) throw new Error('pod shelf: invalid ref');
+  const spec = /^#?(\d+)$/u.exec(requested);
+  if (spec) {
+    const pr = exec('gh', ['pr', 'view', spec[1]!, '--json', 'headRefOid', '--jq', '.headRefOid'], { cwd, encoding: 'utf8', timeout: 30_000 });
+    const sha = String(pr.stdout ?? '').trim().toLowerCase();
+    if (pr.status !== 0 || !/^[0-9a-f]{40}$/u.test(sha)) throw new Error(`pod shelf: PR head unavailable: ${requested}`);
+    return sha;
+  }
+  const git = exec('git', ['rev-parse', '--verify', `${requested}^{commit}`], { cwd, encoding: 'utf8', timeout: 30_000 });
+  const sha = String(git.stdout ?? '').trim().toLowerCase();
+  if (git.status !== 0 || !/^[0-9a-f]{40}$/u.test(sha)) throw new Error(`pod shelf: ref unavailable: ${requested}`);
+  return sha;
+}
+
+/** Checkout only the requested commit; syncPoolImages retains its per-host build lock and SHA12 registry check. */
+export async function putPodShelfImage(ref: string, pool: string, deps: {
+  resolve?: (ref: string) => string;
+  sync?: typeof syncPoolImages;
+  checkout?: (commit: string, dir: string) => void;
+  tempDir?: () => string;
+  remove?: (dir: string) => void;
+  hasRegistry?: (member: PodPoolMember) => boolean;
+} = {}): Promise<PodShelfImage> {
+  const members = parsePodPool(pool);
+  if (members.length !== 1 || !members[0]!.sshHost) throw new Error('pod shelf: one remote registry pool member is required');
+  const member: PodPoolMember = members[0]!;
+  if (!(deps.hasRegistry ?? memberHasRegistry)(member)) throw new Error('pod shelf: remote registry unavailable');
+  const commit = (deps.resolve ?? resolvePodShelfRef)(ref);
+  if (!/^[0-9a-f]{40}$/u.test(commit)) throw new Error('pod shelf: full commit required');
+  const registry = member.registry ?? podImageRegistry();
+  const image = podImageRef(registry, commit);
+  if (!deps.sync && registryImageRef(member, commit) === image) return { commit, image, action: 'fresh' };
+  const dir = (deps.tempDir ?? (() => mkdtempSync(join(tmpdir(), 'pod-shelf-'))))();
+  try {
+    if (deps.checkout) deps.checkout(commit, dir);
+    else {
+      const clone = spawnSync('git', ['clone', '-q', '--no-checkout', process.cwd(), dir], { encoding: 'utf8', timeout: 120_000, maxBuffer: SPAWN_SYNC_MAX_BUFFER });
+      if (clone.status !== 0) throw new Error(`pod shelf: clone failed: ${clone.stderr}`);
+      const fetch = spawnSync('git', ['-C', dir, 'fetch', '-q', 'origin', commit], { encoding: 'utf8', timeout: 120_000, maxBuffer: SPAWN_SYNC_MAX_BUFFER });
+      if (fetch.status !== 0) {
+        const remote = spawnSync('git', ['remote', 'get-url', 'origin'], { encoding: 'utf8', timeout: 10_000, maxBuffer: SPAWN_SYNC_MAX_BUFFER });
+        if (remote.status !== 0) throw new Error(`pod shelf: fetch failed: ${fetch.stderr}`);
+        const fromOrigin = spawnSync('git', ['-C', dir, 'fetch', '-q', String(remote.stdout).trim(), commit], { encoding: 'utf8', timeout: 120_000, maxBuffer: SPAWN_SYNC_MAX_BUFFER });
+        if (fromOrigin.status !== 0) throw new Error(`pod shelf: fetch failed: ${fromOrigin.stderr}`);
+      }
+      const checkout = spawnSync('git', ['-C', dir, 'checkout', '--detach', commit], { encoding: 'utf8', timeout: 120_000, maxBuffer: SPAWN_SYNC_MAX_BUFFER });
+      if (checkout.status !== 0) throw new Error(`pod shelf: checkout failed: ${checkout.stderr}`);
+    }
+    const sync = deps.sync ?? syncPoolImages;
+    const results = await sync([member], 'elanous-harness:local', commit, {
+      buildScript: join(dir, 'docker/harness/build.sh'),
+      ...(deps.sync ? {} : { ship: () => ({ ok: false, action: 'failed' as const, detail: 'shelf build failed; local image fallback prohibited' }) }),
+      remoteBuild: deps.sync ? undefined : async (host, cluster, registry) => {
+        const { spawn } = await import('node:child_process');
+        return new Promise((done) => {
+          const child = spawn('bash', [join(dir, 'docker/harness/build.sh')], { env: { ...process.env, ELANOUS_BUILD_REMOTE: host, ELANOUS_L2_CLUSTER: cluster, ...(registry ? { ELANOUS_BUILD_REGISTRY: registry } : {}) }, stdio: ['ignore', 'pipe', 'pipe'] });
+          let output = '';
+          child.stdout.on('data', (chunk) => { output += chunk; });
+          child.stderr.on('data', (chunk) => { output += chunk; });
+          child.on('close', (code) => done({ ok: code === 0, detail: output.slice(-300) }));
+          child.on('error', (error) => done({ ok: false, detail: error.message }));
+        });
+      },
+    });
+    const result = results.get(member.context);
+    if (!result?.ok || result.imageRef !== image) throw new Error(`pod shelf: image unavailable: ${result?.detail ?? image}`);
+    return { commit, image, action: result.action === 'fresh' ? 'fresh' : 'built' };
+  } finally { (deps.remove ?? ((path) => rmSync(path, { recursive: true, force: true })))(dir); }
+}
+
+export function listPodShelfImages(pool: string, run: RemoteRun): string[] {
+  const member = parsePodPool(pool)[0];
+  if (!member?.sshHost) throw new Error('pod shelf: remote pool member required');
+  const registry = member.registry ?? podImageRegistry();
+  const port = registry.slice(registry.lastIndexOf(':') + 1);
+  const result = run(member.sshHost, `curl -fsS --max-time 5 http://localhost:${port}/v2/elanous-harness/tags/list 2>/dev/null`);
+  if (result.status !== 0) throw new Error(`pod shelf: registry unavailable: ${result.stderr}`);
+  const tags: unknown = JSON.parse(result.stdout).tags;
+  return Array.isArray(tags) ? tags.filter((tag): tag is string => typeof tag === 'string' && /^[0-9a-f]{12}$/u.test(tag)).map((tag) => `${registry}/elanous-harness:${tag}`) : [];
+}
+
 export function podImageMissingMessage(ref: string): string {
   return `기준 이미지 ${ref} 가 레지스트리에 없다 — 레일이 굽기 전이다`;
 }

package/src/task-orchestrator/surfaces/pod-lease.ts

@@ -236,6 +236,10 @@ export interface ExternalGitHubProbe {
   githubHttps: boolean | null;
   measuredAt: string | null;
   reason: string | null;
+  /** Name of the disposable Pod for matching this reading with its Kubernetes logs. */
+  podName?: string;
+  /** Unmodified stdout from `kubectl logs` for this Pod (only present when readable). */
+  podLog?: string;
 }
 const DNS_CACHE_MS = 10 * 60_000;
 const DNS_STALE_OK_MS = 30 * 60_000;
@@ -261,9 +265,11 @@ export function probePoolGitHub(context: string, kubectl: PoolKubectl = leaseKub
       if (typeof record?.at !== 'number' || now() < record.at || now() - record.at >= 60_000) return null;
       if (typeof record.measuredAt !== 'string' || ![true, false].includes(record.externalDns as boolean) ||
           ![true, false, null].includes(record.githubHttps as boolean | null) ||
-          !(record.reason === null || typeof record.reason === 'string')) return null;
+          !(record.reason === null || typeof record.reason === 'string') ||
+          typeof record.podName !== 'string' || typeof record.podLog !== 'string') return null;
       return { externalDns: record.externalDns as boolean, githubHttps: record.githubHttps as boolean | null,
-        measuredAt: record.measuredAt, reason: record.reason as string | null };
+        measuredAt: record.measuredAt, reason: record.reason as string | null,
+        podName: record.podName, podLog: record.podLog };
     } catch { return null; }
   };
   const fresh = cached();
@@ -285,6 +291,7 @@ export function probePoolGitHub(context: string, kubectl: PoolKubectl = leaseKub
     const shared = cached();
     if (shared) return shared;
     const result = runPoolGitHubProbe(context, kubectl, new Date(now()).toISOString());
+    debug.log('pod.lease', 'github-egress-probe', { context, ...result });
     if (result.measuredAt !== null) {
       const temp = `${cache}.${randomUUID()}.tmp`;
       try { writeFileSync(temp, JSON.stringify({ ...result, at: now() }), { mode: 0o600 }); renameSync(temp, cache); }
@@ -300,16 +307,18 @@ export function probePoolGitHub(context: string, kubectl: PoolKubectl = leaseKub
 function runPoolGitHubProbe(context: string, kubectl: PoolKubectl, measuredAt: string): ExternalGitHubProbe {
   const name = `elanous-egress-${randomUUID().slice(0, 12)}`;
   const base = ['--context', context, '--request-timeout=15s'];
-  const unknown = (reason: string): ExternalGitHubProbe => ({ externalDns: null, githubHttps: null, measuredAt: null, reason });
   let created = false;
+  const unknown = (reason: string): ExternalGitHubProbe => ({ externalDns: null, githubHttps: null, measuredAt: null, reason,
+    ...(created ? { podName: name } : {}) });
   try {
     const manifest = JSON.stringify({ apiVersion: 'v1', kind: 'Pod', metadata: { name, namespace: 'elanous-test', labels: { 'elanous.probe': 'github-egress' } }, spec: {
       restartPolicy: 'Never', activeDeadlineSeconds: 35, automountServiceAccountToken: false,
       containers: [{ name: 'probe', image: 'busybox:1.36', command: ['sh', '-c',
-        'if nslookup github.com >/dev/null 2>&1; then echo EXTERNAL_DNS_OK; if wget -q -T 8 -O /dev/null https://github.com/; then echo GITHUB_HTTPS_OK; else echo GITHUB_HTTPS_FAILED; fi; else echo EXTERNAL_DNS_FAILED; fi'],
+        'if nslookup github.com 2>&1; then echo EXTERNAL_DNS_OK; if wget -T 8 -O /dev/null https://github.com/ 2>&1; then echo GITHUB_HTTPS_OK; else echo GITHUB_HTTPS_FAILED; fi; else echo EXTERNAL_DNS_FAILED; fi'],
       resources: { requests: { cpu: '0', memory: '16Mi' }, limits: { cpu: '100m', memory: '64Mi' } } }],
     } });
-    if (kubectl([...base, 'create', '-f', '-'], manifest).status !== 0) return unknown('probe create failed');
+    const createdPod = kubectl([...base, 'create', '-f', '-'], manifest);
+    if (createdPod.status !== 0) return unknown(`probe create failed${createdPod.stderr.trim() ? `: ${createdPod.stderr.trim().split('\n').pop()!.slice(0, 160)}` : ''}`);
     created = true;
     const waited = kubectl([...base, '-n', 'elanous-test', 'wait', '--for=jsonpath={.status.phase}=Succeeded', `pod/${name}`, '--timeout=25s']);
     const podResult = kubectl([...base, '-n', 'elanous-test', 'get', `pod/${name}`, '-o', 'json']);
@@ -327,10 +336,10 @@ function runPoolGitHubProbe(context: string, kubectl: PoolKubectl, measuredAt: s
     const httpsOk = markers.includes('GITHUB_HTTPS_OK');
     const httpsFail = markers.includes('GITHUB_HTTPS_FAILED');
     if (dnsFail && !dnsOk && !httpsOk && !httpsFail)
-      return { externalDns: false, githubHttps: null, measuredAt, reason: 'external DNS failed' };
+      return { externalDns: false, githubHttps: null, measuredAt, reason: 'external DNS failed', podName: name, podLog: logs.stdout };
     if (dnsOk && !dnsFail && httpsOk !== httpsFail)
-      return { externalDns: true, githubHttps: httpsOk, measuredAt, reason: httpsFail ? 'GitHub HTTPS failed' : null };
-    return unknown(waited.status === 0 ? 'probe output unavailable' : 'probe timeout');
+      return { externalDns: true, githubHttps: httpsOk, measuredAt, reason: httpsFail ? 'GitHub HTTPS failed' : null, podName: name, podLog: logs.stdout };
+    return { ...unknown(waited.status === 0 ? 'probe output unavailable' : 'probe timeout'), podLog: logs.stdout };
   } catch { return unknown('probe failed'); }
   finally {
     if (created) try { kubectl([...base, '-n', 'elanous-test', 'delete', `pod/${name}`, '--ignore-not-found=true', '--wait=false']); }

package/src/task-orchestrator/surfaces/pod-manifest-test-normalize.ts

@@ -1,6 +1,12 @@
 import { BUDGET_SNAPSHOT_ENV } from '../../execution-loop/launch-gate.js';
 
-// Preserve the manifest contract except for two values generated per launch.
+/** The Pod push fence embeds a different six-hex run key in each independent launch. */
+export function stablePodRunGuard(text: string): string {
+  return text.replace(/pr-create-r[0-9a-f]{6}-/g, 'pr-create-r<run>-')
+    .replace(/-r[0-9a-f]{6}(?=[-|*])/g, '-r<run>');
+}
+
+// Preserve the manifest contract except for the budget timestamp and run identities generated per launch.
 export function stablePodManifestText(applied: Array<Record<string, any>>): string {
   const manifests = structuredClone(applied.filter((entry) => entry.kind === 'Job' || entry.kind === 'Secret'));
   let snapshots = 0;
@@ -19,5 +25,5 @@ export function stablePodManifestText(applied: Array<Record<string, any>>): stri
     }
   }
   if (snapshots === 0) throw new Error('manifest comparison requires a budget snapshot');
-  return JSON.stringify(manifests).replace(/run-[0-9a-f-]{36}/g, 'run-<id>');
+  return stablePodRunGuard(JSON.stringify(manifests).replace(/run-[0-9a-f-]{36}/g, 'run-<id>'));
 }

package/src/task-orchestrator/surfaces/pod-provenance.ts

@@ -0,0 +1,76 @@
+import { redactSecretText } from '../../debug/log.js';
+
+/** The work that owns a Pod. Job metadata is the current view; pod.provenance events retain history. */
+export interface PodProvenance {
+  actorKind: string;
+  actor: string;
+  kind: string;
+  run: string;
+  phase: string;
+  requestedAt: string;
+  character?: string;
+  launchHost?: string;
+  entrance?: string;
+  graphNode?: string;
+  cell?: string;
+  version?: string;
+  commit?: string;
+  shard?: string;
+  reason?: string;
+  pool?: string;
+  cpuReq?: string;
+  memReq?: string;
+  warmReused?: boolean;
+  argv?: string;
+}
+
+const required = ['actorKind', 'actor', 'kind', 'run', 'phase', 'requestedAt'] as const;
+const names: Record<keyof PodProvenance, string> = {
+  actorKind: 'actor-kind', actor: 'actor', kind: 'kind', run: 'run', phase: 'phase', requestedAt: 'requested-at',
+  character: 'character', launchHost: 'launch-host', entrance: 'entrance', graphNode: 'graph-node', cell: 'cell',
+  version: 'version', commit: 'commit', shard: 'shard', reason: 'reason',
+  pool: 'pool', cpuReq: 'cpu-req', memReq: 'mem-req',
+  warmReused: 'warm-reused', argv: 'argv',
+};
+
+const labelValue = (value: string): string => {
+  const cleaned = value.toLowerCase().replace(/[^a-z0-9._-]/g, '-').replace(/^[^a-z0-9]+|[^a-z0-9]+$/g, '');
+  return cleaned.slice(0, 63).replace(/[^a-z0-9]+$/g, '') || 'unknown';
+};
+
+export interface PodProvenanceMetadata {
+  labels: Record<string, string>;
+  annotations: Record<string, string>;
+  env: Array<{ name: string; value: string }>;
+  missing: string[];
+}
+
+/** A gate reports missing identity as `unknown`; every other launch refuses it before kubectl apply. */
+export function podProvenanceMetadata(input: Partial<PodProvenance> | undefined, gate = false): PodProvenanceMetadata {
+  const missing = required.filter((key) => typeof input?.[key] !== 'string' || !input?.[key]?.trim());
+  if (missing.length && !gate) throw new Error(`pod provenance missing: ${missing.map((key) => names[key]).join(', ')}`);
+  const values: Record<string, string> = {};
+  for (const [key, name] of Object.entries(names) as Array<[keyof PodProvenance, string]>) {
+    const raw = input?.[key];
+    if (raw === undefined || raw === null || raw === '') continue;
+    const value = String(raw).trim();
+    if (!value) continue;
+    // Metadata and env are observable by many users. Never carry a raw secret through an identity field.
+    const redacted = redactSecretText(value);
+    if (redacted !== value && key !== 'reason' && key !== 'argv') throw new Error(`pod provenance secret in ${name}`);
+    values[name] = redacted;
+  }
+  for (const key of missing) values[names[key]] = 'unknown';
+  // The launch reason is observable even when the caller cannot supply one.
+  values.reason ??= 'unknown';
+  const labels: Record<string, string> = {};
+  const annotations: Record<string, string> = {};
+  const env: Array<{ name: string; value: string }> = [];
+  for (const [name, value] of Object.entries(values)) {
+    const label = name === 'run' || name === 'commit' ? value.slice(0, 8) : value;
+    if (name !== 'reason' && name !== 'argv') labels[`elanous.ai/${name}`] = labelValue(label);
+    annotations[`elanous.ai/${name}`] = value;
+    env.push({ name: `ELANOUS_POD_${name.toUpperCase().replaceAll('-', '_')}`, value });
+  }
+  return { labels, annotations, env, missing: missing.map((key) => names[key]) };
+}

package/src/task-orchestrator/surfaces/pod-source-receive.ts

@@ -179,6 +179,24 @@ export function podSourceScript(source: PodSource, repoUrl: string, mirrorPath =
       'if ! [[ "$pr_sha" =~ ^[0-9a-f]{40}$ ]]; then exit 5; fi',
       `if ! git cat-file -e "$pr_sha^{commit}" 2>/dev/null; then git fetch --depth 50 origin pull/${n}/head || exit 5; fi`,
       `git checkout -B pr-${n} "$pr_sha" || exit 5`,
+      // A pinned image starts at one shallow main commit. Fetching a PR tip at
+      // depth 50 can leave that pin and the PR in disconnected shallow spans.
+      // Deepen only after merge-base fails; the gate still reports unavailable
+      // if ancestry cannot be obtained.
+      'remote_info=$(git ls-remote --symref origin HEAD 2>/dev/null) || { printf \'ELANOUS_POD_COMPARISON_BASE unavailable\\n\'; exit 5; }',
+      `base_branch=$(printf '%s\\n' "$remote_info" | awk '$1=="ref:" && $3=="HEAD" && $2 ~ /^refs\\/heads\\// {sub(/^refs\\/heads\\//, "", $2); print $2}')`,
+      'if [ -n "$base_branch" ] && git check-ref-format --branch "$base_branch" >/dev/null 2>&1; then',
+      '  base_ref="refs/remotes/origin/$base_branch"',
+      '  git fetch --depth 50 origin "+refs/heads/$base_branch:$base_ref" || { printf \'ELANOUS_POD_COMPARISON_BASE unavailable\\n\'; exit 5; }',
+      '  git symbolic-ref refs/remotes/origin/HEAD "$base_ref" || exit 5',
+      '  if ! git merge-base HEAD "$base_ref" >/dev/null 2>&1 && [ "$(git rev-parse --is-shallow-repository)" = true ]; then',
+      `    git fetch --deepen=100 origin "+refs/heads/$base_branch:$base_ref" +refs/pull/${n}/head:refs/remotes/origin/pr-${n}-head || printf 'ELANOUS_POD_COMPARISON_BASE deepen-failed\\n'`,
+      '  fi',
+      '  if ! git merge-base HEAD "$base_ref" >/dev/null 2>&1 && [ "$(git rev-parse --is-shallow-repository)" = true ]; then',
+      `    git fetch --unshallow origin "+refs/heads/$base_branch:$base_ref" +refs/pull/${n}/head:refs/remotes/origin/pr-${n}-head || { printf 'ELANOUS_POD_COMPARISON_BASE unavailable\\n'; exit 5; }`,
+      '  fi',
+      "  if ! git merge-base HEAD \"$base_ref\" >/dev/null 2>&1; then printf 'ELANOUS_POD_COMPARISON_BASE unavailable\\n'; fi",
+      "else printf 'ELANOUS_POD_COMPARISON_BASE unavailable\\n'; exit 5; fi",
       'head=$(git rev-parse HEAD)',
       "printf 'ELANOUS_POD_SOURCE pr %s\\n' \"$head\"",
     ].join('\n');

package/src/task-orchestrator/surfaces/pod-warm-provenance.ts

@@ -0,0 +1,81 @@
+import { spawnSync } from 'node:child_process';
+import type { Kubectl } from './self-implement-pod.js';
+import { podProvenanceMetadata } from './pod-provenance.js';
+import type { WarmWorkerCommand } from './pod-warm-worker-protocol.js';
+import type { WarmPodSpec } from './pod-warm-worker-pod.js';
+
+const PREFIX = 'elanous.ai/';
+const STATIC = new Set(['pool', 'image-digest', 'launch-host']);
+const workKey = (key: string) => key.startsWith(PREFIX) && !STATIC.has(key.slice(PREFIX.length));
+
+export function warmLeaseMetadata(command: WarmWorkerCommand) {
+  return podProvenanceMetadata({ actorKind: 'loop', actor: 'release-gate', character: 'gate-warm-worker',
+    entrance: 'release-gate-warm', kind: 'warm', run: command.runId, phase: 'release-gate',
+    requestedAt: command.requestedAt || 'unknown', reason: 'gate shadow replay', commit: command.cutSha,
+    shard: command.attemptId, warmReused: true });
+}
+
+export function warmExecEnv(command: WarmWorkerCommand): string[] {
+  return [...warmLeaseMetadata(command).env,
+    { name: 'ELANOUS_POD_LEASE', value: command.leaseId },
+    { name: 'ELANOUS_POD_WORKER', value: command.workerId },
+  ].map(({ name, value }) => `${name}=${value}`);
+}
+
+/** One UID/resourceVersion-guarded metadata mutation. A failed patch never admits a lease. */
+export function patchWarmLeaseMetadata(spec: WarmPodSpec, command: WarmWorkerCommand,
+  next: 'leased' | 'idle' | 'quarantined', injected?: Kubectl): { ok: boolean; reason: string } {
+  const fail = (reason: string) => ({ ok: false, reason });
+  if (!command.podUid || !command.requestedAt) return fail('lease-identity-missing');
+  const kubectl: Kubectl = injected ?? ((args, input) => {
+    const env = { ...process.env };
+    for (const name of ['HTTPS_PROXY', 'https_proxy', 'HTTP_PROXY', 'http_proxy', 'ALL_PROXY', 'all_proxy']) delete env[name];
+    const result = spawnSync('kubectl', [...args], { encoding: 'utf8', input, env,
+      timeout: 120_000, maxBuffer: 2 * 1024 * 1024 });
+    return { status: result.status, stdout: result.stdout ?? '', stderr: result.stderr ?? '' };
+  });
+  const prefix = ['--context', spec.kubectlContext, '-n', spec.namespace];
+  const observed = kubectl([...prefix, 'get', 'pod', spec.name, '-o', 'json']);
+  if (observed.status !== 0) return fail('pod-read-failed');
+  let pod: any;
+  try { pod = JSON.parse(observed.stdout); } catch { return fail('pod-read-invalid'); }
+  const metadata = pod?.metadata;
+  if (metadata?.name !== spec.name || metadata?.namespace !== spec.namespace
+    || metadata?.uid !== command.podUid || typeof metadata?.resourceVersion !== 'string'
+    || !metadata.resourceVersion
+    || metadata.annotations?.['elanous.dev/warm-worker-id'] !== spec.workerId
+    || metadata.annotations?.['elanous.dev/warm-cut-sha'] !== spec.cutSha) return fail('pod-identity-drift');
+  const labels = { ...(metadata.labels ?? {}) } as Record<string, string>;
+  const annotations = { ...(metadata.annotations ?? {}) } as Record<string, string>;
+  if (next === 'leased' && labels[`${PREFIX}state`] !== undefined
+    && labels[`${PREFIX}state`] !== 'idle') return fail('lease-state-not-idle');
+  if (next === 'leased' && labels[`${PREFIX}state`] === undefined
+    && [...Object.keys(labels), ...Object.keys(annotations)].some(workKey)) return fail('stale-work-metadata');
+  if (next === 'idle' && (labels[`${PREFIX}state`] !== 'leased'
+    || labels[`${PREFIX}lease`] !== command.leaseId)) return fail('lease-state-mismatch');
+  if (next === 'quarantined' && labels[`${PREFIX}state`] === 'leased'
+    && labels[`${PREFIX}lease`] !== command.leaseId) return fail('lease-state-mismatch');
+  for (const key of Object.keys(labels)) if (workKey(key)) delete labels[key];
+  for (const key of Object.keys(annotations)) if (workKey(key)) delete annotations[key];
+  if (next === 'leased') {
+    const work = warmLeaseMetadata(command);
+    Object.assign(labels, work.labels, { [`${PREFIX}state`]: 'leased', [`${PREFIX}lease`]: command.leaseId });
+    Object.assign(annotations, work.annotations, { [`${PREFIX}lease`]: command.leaseId });
+  } else labels[`${PREFIX}state`] = next;
+  const patch = [
+    { op: 'test', path: '/metadata/uid', value: command.podUid },
+    { op: 'test', path: '/metadata/resourceVersion', value: metadata.resourceVersion },
+    { op: 'add', path: '/metadata/labels', value: labels },
+    { op: 'add', path: '/metadata/annotations', value: annotations },
+  ];
+  const updated = kubectl([...prefix, 'patch', 'pod', spec.name, '--type=json', '-p', JSON.stringify(patch), '-o', 'json']);
+  if (updated.status !== 0) return fail('conditional-patch-failed');
+  try {
+    const after = JSON.parse(updated.stdout);
+    const matched = after?.metadata?.uid === command.podUid
+      && after?.metadata?.labels?.[`${PREFIX}state`] === next
+      && (next !== 'leased' ? after.metadata.labels[`${PREFIX}lease`] === undefined
+        : after.metadata.labels[`${PREFIX}lease`] === command.leaseId);
+    return matched ? { ok: true, reason: 'applied' } : fail('patch-result-mismatch');
+  } catch { return fail('patch-result-invalid'); }
+}

package/src/task-orchestrator/surfaces/pod-warm-worker-pod.ts

@@ -11,6 +11,7 @@ import { elanousStateRoot } from '../../autopilot/state-paths.js';
 import { debug } from '../../debug/log.js';
 import { parseWarmWorkerReply, type WarmWorkerCommand, type WarmWorkerKey, type WarmWorkerReply } from './pod-warm-worker-protocol.js';
 import type { WarmWorkerOutcome } from './pod-warm-worker-runtime.js';
+import { warmExecEnv } from './pod-warm-provenance.js';
 
 const SHA = /^[a-f0-9]{40}$/;
 const DIGEST = /^sha256:[a-f0-9]{64}$/;
@@ -424,7 +425,9 @@ export class WarmPodTransport {
     try { nodeArch = node.status === 0 ? JSON.parse(node.stdout)?.metadata?.labels?.['kubernetes.io/arch'] : null; }
     catch { nodeArch = null; }
     if (nodeArch !== this.spec.key.architecture) return fallback('pod-unready');
-    if (this.podUid && uid !== this.podUid) { this.quarantined = true; return fallback('pod-replaced'); }
+    if ((command.podUid && uid !== command.podUid) || (this.podUid && uid !== this.podUid)) {
+      this.quarantined = true; return fallback('pod-replaced');
+    }
     this.podUid = uid;
     const ledgerKey = sha256([this.spec.kubectlContext, this.spec.namespace, this.spec.name, uid].join('\0'));
     const ledgerPath = this.ledgerDir ? join(this.ledgerDir, ledgerKey) : null;
@@ -434,7 +437,8 @@ export class WarmPodTransport {
       this.quarantined = true;
       return fallback('worker-quarantined');
     }
-    const exec = this.kubectl([...prefix, 'exec', '-i', this.spec.name, '-c', 'worker', '--', 'bun', '-e', WARM_POD_EXEC], JSON.stringify(command));
+    const exec = this.kubectl([...prefix, 'exec', '-i', this.spec.name, '-c', 'worker', '--',
+      'env', ...warmExecEnv(command), 'bun', '-e', WARM_POD_EXEC], JSON.stringify(command));
     if (exec.status !== 0) { this.quarantined = true; return fallback('exec-failed'); }
     // UID alone is insufficient: ephemeral containers can be added to a live Pod under the same UID.
     const after = this.kubectl([...prefix, 'get', 'pod', this.spec.name, '-o', 'json']);

package/src/task-orchestrator/surfaces/pod-warm-worker-protocol.ts

@@ -20,6 +20,9 @@ export interface WarmWorkerCommand {
   leaseId: string;
   generation: number;
   workerId: string;
+  /** Acquired Pod identity and lease timestamp, carried only by the host control path. */
+  podUid?: string;
+  requestedAt?: string;
   key: WarmWorkerKey;
   deadlineAt: number;
   files: readonly string[];