elanous 0.2.22 → 0.2.24

package/src/task-orchestrator/surfaces/pod-image-source.ts

@@ -0,0 +1,212 @@
+/**
+ * IMAGE-ONE-SOURCE (대표 2026-10-09 «이미지도 공통 사용·기준값 관리 — 멀티 서버가 같은 곳을 보게»).
+ *
+ * `--substrate pod` 발사는 지금 «이 트리 HEAD ≠ 노드 이미지»면 그 자리에서 굽는다 — 부모 여럿이 같은 커밋을 동시에 굽고,
+ * 노드마다 `:local` 판이 갈렸다. 운영 config `harness.imageCommit`(커밋 12자리 이상)이 있으면 발사는 굽지 않고
+ * 레지스트리의 «기준 이미지»(커밋 태그)를 당긴다. 없으면 종전 경로(HEAD 비교·굽기) 그대로다.
+ *
+ * ⛔ 레지스트리 주소·태그 규칙은 docker/harness/build.sh 의 것이다(새 규칙을 만들지 않는다):
+ *   REGISTRY="${ELANOUS_BUILD_REGISTRY:-k3d-elanous-registry:${ELANOUS_FLEET_REGISTRY_PORT:-5050}}" · TAG="${COMMIT:0:12}".
+ *   풀 멤버가 `#k3d-레지스트리:포트` 를 명시했으면 그 멤버는 그 주소다(syncPoolImages 가 build.sh 에 넘기는 것과 같다).
+ *   조회·Job ref·오류 문구는 전부 «멤버마다 정해진 그 한 주소»를 쓴다.
+ * 레지스트리가 있는 풀 멤버는 원격이면 SSH, 로컬이면 이 기계에서 기준 태그를 찾는다.
+ * 레지스트리가 없는 멤버(node-c 등)와 조회할 수 없는 로컬 레지스트리는 종전 경로로 간다 — 레지스트리 미러는 범위 밖.
+ *   이미지는 표준판(elanous-harness)만 — 이 발사 경로는 lite 를 쓰지 않는다.
+ */
+import { spawnSync } from 'node:child_process';
+import { debug } from '../../debug/log.js';
+import { memberHasRegistry, registryImageRef, type RemoteRun } from './pod-pool.js';
+
+const POD_IMAGE = 'elanous-harness';
+
+export type PodImageSource =
+  | { readonly kind: 'pinned'; readonly ref: string; readonly commit: string; readonly registry: string }
+  | { readonly kind: 'head' };
+
+export type PodImageSourceOutcome = 'pinned-ready' | 'pinned-waited' | 'pinned-missing' | 'head';
+
+/** 기준 이미지가 레지스트리에 올라오기를 기다리는 기본 한도(초). */
+export const POD_IMAGE_PIN_WAIT_SEC = 600;
+const POD_IMAGE_PIN_POLL_MS = 15_000;
+
+const COMMIT = /^[0-9a-f]{12,40}$/u;
+
+/** `harness.imageCommit` 이 유효한 값인가(커밋 hex 12~40자리). */
+export function isPodImageCommit(value: unknown): value is string {
+  return typeof value === 'string' && COMMIT.test(value.trim().toLowerCase());
+}
+
+/** build.sh:274 와 같은 규칙 — `ELANOUS_BUILD_REGISTRY` 다음 `k3d-elanous-registry:${ELANOUS_FLEET_REGISTRY_PORT:-5050}`. */
+export function podImageRegistry(env: NodeJS.ProcessEnv = process.env): string {
+  // bash `${X:-d}` 는 빈 문자열도 «없음»으로 본다 — `||` 로 맞춘다.
+  return env.ELANOUS_BUILD_REGISTRY || `k3d-elanous-registry:${env.ELANOUS_FLEET_REGISTRY_PORT || '5050'}`;
+}
+
+/** build.sh:276 — 태그 = 커밋 앞 12자리. */
+export function podImageRef(registry: string, commit: string): string {
+  return `${registry}/${POD_IMAGE}:${commit.slice(0, 12)}`;
+}
+
+export function resolvePodImageSource(
+  cfg: { harness?: { imageCommit?: unknown } } | null | undefined,
+  deps: { env?: NodeJS.ProcessEnv } = {},
+): PodImageSource {
+  const raw = cfg?.harness?.imageCommit;
+  if (!isPodImageCommit(raw)) return { kind: 'head' };
+  const commit = raw.trim().toLowerCase();
+  const registry = podImageRegistry(deps.env ?? process.env);
+  return { kind: 'pinned', ref: podImageRef(registry, commit), commit, registry };
+}
+
+export function podImageMissingMessage(ref: string): string {
+  return `기준 이미지 ${ref} 가 레지스트리에 없다 — 레일이 굽기 전이다`;
+}
+
+export interface PodImageMember { readonly context: string; readonly sshHost?: string; readonly registry?: string }
+
+/** 이 멤버가 기준 이미지를 찾을 레지스트리 — 멤버 명시값, 없으면 build.sh 규칙. */
+export function memberPinRegistry(member: PodImageMember, source: Extract<PodImageSource, { kind: 'pinned' }>): string {
+  return member.registry ?? source.registry;
+}
+
+/** 실제 조회 — 원격은 pod-pool 의 SSH 판정, 로컬은 이 기계의 docker/curl 로 조회한다. */
+export function podPoolImageLookup(run?: RemoteRun, localRun: (cmd: string, args: readonly string[]) => { status: number | null; stdout: string } = (cmd, args) => {
+  const r = spawnSync(cmd, [...args], { encoding: 'utf8', timeout: 10_000 });
+  return { status: r.status, stdout: r.stdout ?? '' };
+}): {
+  hasRegistry: (member: PodImageMember & { context: string }, registry: string) => boolean;
+  lookupTag: (member: PodImageMember & { context: string }, registry: string, commit: string) => string | null;
+} {
+  return {
+    hasRegistry: (member, registry) => {
+      if (member.sshHost) return memberHasRegistry({ capacity: 0, k3dCluster: '', ...member, registry }, run);
+      try { return localRun('docker', ['inspect', registry.split(':')[0]!]).status === 0; }
+      catch { return false; }
+    },
+    lookupTag: (member, registry, commit) => {
+      if (member.sshHost) return registryImageRef({ capacity: 0, k3dCluster: '', ...member, registry }, commit, run, `${POD_IMAGE}:local`);
+      const port = registry.slice(registry.lastIndexOf(':') + 1);
+      const tag = commit.slice(0, 12);
+      try {
+        const r = localRun('curl', ['-fsS', '--max-time', '5', `http://localhost:${port}/v2/${POD_IMAGE}/tags/list`]);
+        if (r.status !== 0) return null;
+        const tags = (JSON.parse(r.stdout) as { tags?: string[] }).tags;
+        return Array.isArray(tags) && tags.includes(tag) ? podImageRef(registry, commit) : null;
+      } catch { return null; }
+    },
+  };
+}
+
+export interface PreparePodImageDeps<M extends PodImageMember, H> {
+  readonly cfg: { harness?: { imageCommit?: unknown } } | null | undefined;
+  readonly env?: NodeJS.ProcessEnv;
+  /** 풀 멤버(풀이 없으면 빈 배열 — 이 기계의 컨텍스트 하나). */
+  readonly members: readonly M[];
+  /** 종전 경로(HEAD 비교·굽기·노드 동기화). 기준값이 없으면 전 멤버로, 있으면 «기준 이미지를 못 쓰는» 멤버로만 부른다. */
+  readonly headPath: (members: readonly M[]) => Promise<H>;
+  /** 이 멤버의 노드에 그 레지스트리가 있나. */
+  readonly hasRegistry: (member: M, registry: string) => boolean;
+  /** 이 멤버의 그 레지스트리에 그 커밋 태그가 있으면 Pod 가 당길 ref, 없으면 null. */
+  readonly lookupTag: (member: M, registry: string, commit: string) => string | null;
+  readonly maxWaitMs?: number;
+  readonly pollMs?: number;
+  readonly sleep?: (ms: number) => Promise<void>;
+  readonly now?: () => number;
+  readonly onWait?: (pending: readonly M[], waitedMs: number) => void;
+}
+
+export interface PreparedPodImage<M extends PodImageMember, H> {
+  readonly source: PodImageSource;
+  readonly outcome: PodImageSourceOutcome;
+  /** 기준 이미지를 당기는 멤버 — `imageRef` 가 Job 이미지다. */
+  readonly pinned: ReadonlyArray<M & { imageRef: string }>;
+  /** 레지스트리는 있는데 한도 안에 태그가 안 올라온 멤버와 그 멤버가 찾던 ref. */
+  readonly missing: ReadonlyArray<{ member: M; ref: string }>;
+  /** 종전 경로를 탄 멤버(레지스트리 없음 · 로컬 태그 조회 실패). */
+  readonly unpinned: readonly M[];
+  /** 종전 경로의 결과 — 부르지 않았으면 null. */
+  readonly head: H | null;
+  readonly waitedMs: number;
+}
+
+/**
+ * 발사 이미지를 준비한다. 기준값이 없으면 `headPath(members)` 하나만 부른다(종전 동작 그대로).
+ * 기준값이 있으면: 레지스트리가 있는 멤버는 태그를 기다려 당기고(굽지 않는다), 나머지 멤버만 종전 경로로 보낸다.
+ * 원격 레지스트리 멤버 «전부» 태그가 끝내 없으면 outcome `pinned-missing` — 종전 경로도 부르지 않는다(호출부가 exit 2).
+ */
+export async function preparePodImage<M extends PodImageMember, H>(deps: PreparePodImageDeps<M, H>): Promise<PreparedPodImage<M, H>> {
+  const source = resolvePodImageSource(deps.cfg, { env: deps.env });
+  if (source.kind === 'head') {
+    return { source, outcome: 'head', pinned: [], missing: [], unpinned: deps.members, head: await deps.headPath(deps.members), waitedMs: 0 };
+  }
+  const localRefs = new Map<string, string>();
+  const registryMembers = deps.members.filter((m) => {
+    const registry = memberPinRegistry(m, source);
+    if (!deps.hasRegistry(m, registry)) return false;
+    if (m.sshHost) return true;
+    // A local registry without the requested tag (or an unreadable one) can still use the old build path.
+    const ref = deps.lookupTag(m, registry, source.commit);
+    if (!ref) return false;
+    localRefs.set(m.context, ref);
+    return true;
+  });
+  const unpinned = deps.members.filter((m) => !registryMembers.includes(m));
+  if (registryMembers.length === 0) {
+    // 기준 이미지를 당길 멤버가 없다(풀 없음 · 레지스트리 없는 멤버뿐) — 종전 경로.
+    return { source, outcome: 'head', pinned: [], missing: [], unpinned, head: await deps.headPath(unpinned), waitedMs: 0 };
+  }
+  const now = deps.now ?? Date.now;
+  const sleep = deps.sleep ?? ((ms: number) => new Promise<void>((done) => setTimeout(done, ms)));
+  const maxWaitMs = Math.max(0, deps.maxWaitMs ?? POD_IMAGE_PIN_WAIT_SEC * 1000);
+  const pollMs = Math.max(1, deps.pollMs ?? POD_IMAGE_PIN_POLL_MS);
+  const found = new Map<string, string>();
+  const t0 = now();
+  let pending = registryMembers;
+  let slept = false;
+  for (;;) {
+    pending = pending.filter((m) => {
+      const ref = localRefs.get(m.context) ?? deps.lookupTag(m, memberPinRegistry(m, source), source.commit);
+      if (ref) found.set(m.context, ref);
+      return !ref;
+    });
+    const waited = now() - t0;
+    if (pending.length === 0 || waited >= maxWaitMs) break;
+    deps.onWait?.(pending, waited);
+    slept = true;
+    await sleep(Math.min(pollMs, maxWaitMs - waited));
+  }
+  const waitedMs = now() - t0;
+  const pinned = registryMembers.filter((m) => found.has(m.context)).map((m) => ({ ...m, imageRef: found.get(m.context)! }));
+  const missing = pending.map((member) => ({ member, ref: podImageRef(memberPinRegistry(member, source), source.commit) }));
+  if (pinned.length === 0) {
+    return { source, outcome: 'pinned-missing', pinned, missing, unpinned, head: null, waitedMs };
+  }
+  return {
+    source,
+    outcome: slept ? 'pinned-waited' : 'pinned-ready',
+    pinned,
+    missing,
+    unpinned,
+    head: unpinned.length > 0 ? await deps.headPath(unpinned) : null,
+    waitedMs,
+  };
+}
+
+/**
+ * 기준값 경로의 `image-source` 사건 — 발사 호출부(index.ts)가 부르는 단 하나의 기록처.
+ * 이 기계의 로컬 멤버(sshHost 없음)가 기준 이미지를 당겼으면 `local: true` 를 싣는다.
+ */
+export function logPinnedImageSource<M extends PodImageMember, H>(
+  prepared: PreparedPodImage<M, H>,
+  ctx: { readonly ref: string; readonly headCommit: string | null },
+): void {
+  if (prepared.source.kind !== 'pinned') return;
+  const local = prepared.pinned.some((m) => !m.sshHost);
+  debug.log('self-implement.pod', 'image-source', {
+    kind: prepared.source.kind, ref: ctx.ref, pinRef: prepared.source.ref, headCommit: ctx.headCommit, waitedMs: prepared.waitedMs, outcome: prepared.outcome,
+    ...(local ? { local: true } : {}),
+    pinned: prepared.pinned.map((m) => ({ context: m.context, imageRef: m.imageRef })),
+    missing: prepared.missing.map((x) => ({ context: x.member.context, ref: x.ref })),
+    unpinned: prepared.unpinned.map((m) => m.context),
+  });
+}

package/src/task-orchestrator/surfaces/pod-lease.ts

@@ -8,7 +8,14 @@ import { dlopen } from 'bun:ffi';
 import { hostLeaseBaseDir, type PendingJobIdentity } from '../../pod-lease/host-lease.js';
 
 /** The harness Pod's container requests (self-implement-pod Job manifest). Lives here so admission reuses the real request. */
-export const POD_CHILD_REQUESTS = { cpu: '1', memory: '6Gi' } as const;
+// POD-CPU-REQUEST-RIGHTSIZE(10-09 · OP 실측): 하니스 Pod 실사용은 대부분 100m 미만(LLM 응답 대기)인데 요청 1 CPU 를
+//   잡아 node-b 32코어가 «요청 31/32»로 막혔다(실사용 4.6코어 · 14%). 요청은 실사용 근처로, 한도(4)는 그대로 — 몰릴 때는 한도까지 쓴다.
+//   게이트·pod-command 기본은 pod-command-job.ts 의 POD_COMMAND_CPU_REQUEST_DEFAULT(1)로 따로 둔다(테스트 조각은 CPU 를 실제로 쓴다).
+export const POD_CHILD_REQUESTS = { cpu: '250m', memory: '6Gi' } as const;
+/** Finished harness and command Jobs release their Pod slots after 30 minutes. */
+export const POD_JOB_TTL_AFTER_FINISHED_SECONDS = 1800;
+/** 게이트가 돌 때 하니스 «한 자리»로 셈하는 CPU — 요청(250m)보다 보수적으로 1 코어를 잡아 게이트 조각의 CPU 를 지킨다(GATE-RESERVE-AUTO 셈은 그대로). */
+export const HARNESS_SLOT_CPU_BESIDE_GATE = '1';
 /** POD-ADMIT-BY-USAGE — observed usage is scaled by this before it reserves memory. */
 const POD_USAGE_HEADROOM = 1.5;
 /** POD-ADMIT-BY-USAGE — a node keeps at least this share of allocatable memory free (real usage and bookkeeping). */
@@ -33,6 +40,8 @@ export function leaseKubectl(args: readonly string[], input?: string): { status:
 export interface PodLeaseMember {
   context: string;
   capacity: number;
+  /** POD-POOL-UNCAPPED-SPEC — `capacity` came from the spec default (`:N` absent), not from the operator. */
+  capacityDefaulted?: boolean;
   running: number | null;
   pending: number | null;
   /** Job names of Pending harness Pods already included in pending (used to avoid counting the same Job's host lease twice). */
@@ -87,7 +96,12 @@ export interface PoolLeaseMeasure {
 
 export interface PoolLeaseRecommendation {
   recommended: number | null;
-  limitedBy: 'capacity' | 'memory' | null;
+  /**
+   * What stopped admission: `memory` = node memory headroom · `capacity` = the pool slot budget ·
+   * `pool-cap` = POD-POOL-UNCAPPED-SPEC — a member whose cap came from the spec default (`:N` absent → 2) is
+   * already at/over that cap, so the pool spec itself (not memory, not a configured budget) is the bound.
+   */
+  limitedBy: 'capacity' | 'memory' | 'pool-cap' | null;
   reason: string | null;
   capacitySlots: number | null;
   memorySlots: number | null;
@@ -207,6 +221,7 @@ function runningLimit(spec: Record<string, unknown>): number | null {
 /** A short-lived Pod proves cluster DNS from inside the same namespace as harness Jobs. Image pull failures are not DNS failures. */
 export type PoolDnsProbe = 'ready' | 'dns' | 'image' | 'unknown';
 const DNS_CACHE_MS = 10 * 60_000;
+const DNS_STALE_OK_MS = 30 * 60_000;
 const DNS_LOCK_WAIT_MS = 6 * 60_000;
 // flock is released by the kernel on process exit, including a crash before any cache write.
 // The lock file is never unlinked: unlinking it would let waiters lock different inodes.
@@ -232,6 +247,17 @@ export function probePoolDns(context: string, kubectl: PoolKubectl = leaseKubect
     return record?.result === 'ready' && typeof record.at === 'number' &&
       now() >= record.at && now() - record.at < DNS_CACHE_MS;
   };
+  const withStaleReady = (result: PoolDnsProbe): PoolDnsProbe => {
+    if (result !== 'unknown') return result;
+    const record = readRecord(cache);
+    const readyAgeMs = typeof record?.at === 'number' ? now() - record.at : NaN;
+    if (record?.result === 'ready' &&
+        Number.isFinite(readyAgeMs) && readyAgeMs >= 0 && readyAgeMs < DNS_STALE_OK_MS) {
+      debug.log('pod.lease', 'dns-probe-stale-ready', { context, readyAgeMs });
+      return 'ready';
+    }
+    return result;
+  };
   if (cached()) return 'ready';
   const previous = readRecord(outcome);
   try { mkdirSync(dir, { recursive: true, mode: 0o700 }); } catch { return 'unknown'; }
@@ -251,31 +277,34 @@ export function probePoolDns(context: string, kubectl: PoolKubectl = leaseKubect
     while (!held) {
       if (cached()) return 'ready';
       const shared = sharedFailure();
-      if (shared) return shared;
+      if (shared) return withStaleReady(shared);
       if (dnsFlock(fd, 2 | 4) === 0) { held = true; break; } // LOCK_EX | LOCK_NB
       if (Date.now() >= deadline) return 'unknown';
       Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 100);
     }
     if (cached()) return 'ready';
     const shared = sharedFailure();
-    if (shared) return shared;
-    const result = runPoolDnsProbe(context, kubectl);
+    if (shared) return withStaleReady(shared);
+    const result = runPoolDnsProbe(context, kubectl, now);
     const destination = result === 'ready' ? cache : outcome;
     const temp = `${destination}.${randomUUID()}.tmp`;
     try {
       writeFileSync(temp, JSON.stringify({ result, at: now(), generation: randomUUID() }), { mode: 0o600 });
       renameSync(temp, destination);
     } catch { try { rmSync(temp, { force: true }); } catch { /* best-effort cleanup */ } }
-    return result;
+    return withStaleReady(result);
   } finally {
     if (held) dnsFlock(fd, 8); // LOCK_UN
     closeSync(fd);
   }
 }
 
-function runPoolDnsProbe(context: string, kubectl: PoolKubectl): PoolDnsProbe {
+const DNS_PROBE_READY = /\bName:\s*kubernetes\.default\.svc\.cluster\.local\b/u;
+
+function runPoolDnsProbe(context: string, kubectl: PoolKubectl, now: () => number): PoolDnsProbe {
   const name = `elanous-dns-${randomUUID().slice(0, 12)}`;
   const base = ['--context', context, '--request-timeout=10s'];
+  let cleanup = false;
   try {
     const core = kubectl([...base, '-n', 'kube-system', 'get', 'deployment', 'coredns', '-o', 'json']);
     if (core.status !== 0) return 'unknown';
@@ -283,12 +312,61 @@ function runPoolDnsProbe(context: string, kubectl: PoolKubectl): PoolDnsProbe {
     const spec = object(deployment?.spec);
     const status = object(deployment?.status);
     if (typeof spec?.replicas !== 'number' || spec.replicas < 1 || status?.readyReplicas !== spec.replicas) return 'dns';
+    const listed = kubectl([...base, '-n', 'elanous-test', 'get', 'pods', '-l', 'elanous.probe=dns', '-o', 'json']);
+    if (listed.status !== 0) return 'unknown';
+    const listedPods = items(JSON.parse(listed.stdout));
+    if (!listedPods) return 'unknown';
+    const probes = listedPods.filter((pod) => {
+      const metadata = object(pod.metadata);
+      return object(metadata?.labels)?.['elanous.probe'] === 'dns' &&
+        (metadata?.namespace === undefined || metadata.namespace === 'elanous-test');
+    });
+    let reaped = 0;
+    for (const probe of probes) {
+      const metadata = object(probe.metadata);
+      const name = metadata?.name;
+      const deletedAt = metadata?.deletionTimestamp;
+      if (typeof name !== 'string' || !/^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/.test(name) ||
+          typeof deletedAt !== 'string' || !Number.isFinite(Date.parse(deletedAt)) ||
+          now() - Date.parse(deletedAt) <= 60_000) continue;
+      try {
+        if (kubectl([...base, '-n', 'elanous-test', 'delete', 'pods', '-l', 'elanous.probe=dns', `--field-selector=metadata.name=${name}`, '--grace-period=0', '--force', '--ignore-not-found=true', '--wait=false']).status === 0) reaped++;
+      } catch { /* Reaping is best-effort; the crowded reading still prevents a new probe. */ }
+    }
+    if (reaped) debug.log('pod.lease', 'dns-probe-stale-reaped', { context, count: reaped });
+    // 🩸 10-09 22:1x: node-b had 20 Completed probes left over from older releases (and 7 Terminating). Counting every
+    //   labelled Pod would make the node «crowded» forever and admission would stay 0 — only live probes are crowding.
+    //   Finished probes are removed here so older leftovers drain too.
+    let finished = 0;
+    const live = probes.filter((probe) => {
+      const metadata = object(probe.metadata);
+      if (metadata?.deletionTimestamp !== undefined) return false;
+      const phase = object(probe.status)?.phase;
+      if (phase !== 'Succeeded' && phase !== 'Failed') return true;
+      const name = metadata?.name;
+      if (typeof name === 'string' && /^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/.test(name)) {
+        try {
+          if (kubectl([...base, '-n', 'elanous-test', 'delete', 'pods', '-l', 'elanous.probe=dns', `--field-selector=metadata.name=${name}`, '--ignore-not-found=true', '--wait=false']).status === 0) finished++;
+        } catch { /* best-effort: a finished probe never crowds */ }
+      }
+      return false;
+    });
+    if (finished) debug.log('pod.lease', 'dns-probe-finished-reaped', { context, count: finished });
+    if (live.length >= 3) {
+      debug.log('pod.lease', 'dns-probe-skipped-crowded', { context, existing: live.length });
+      return 'unknown';
+    }
+    cleanup = true;
     const manifest = JSON.stringify({ apiVersion: 'v1', kind: 'Pod', metadata: { name, namespace: 'elanous-test', labels: { 'elanous.probe': 'dns' } }, spec: {
-      restartPolicy: 'Never', activeDeadlineSeconds: 40, automountServiceAccountToken: false,
-      containers: [{ name: 'dns', image: 'busybox:1.36', command: ['nslookup', 'kubernetes.default.svc.cluster.local'], resources: { requests: { cpu: '10m', memory: '16Mi' }, limits: { cpu: '100m', memory: '64Mi' } } }],
+      restartPolicy: 'Never', activeDeadlineSeconds: 120, automountServiceAccountToken: false,
+      // 🩸 10-09 18:2x: 노드 CPU «요청» 100%(32/32) 이면 10m 탐침도 못 떠 측정이 «dns probe unknown» → 승인 추천 null → 전 발사 정지(교착).
+      //   탐침은 CPU 요청 0(명시 · 안 쓰면 k8s 가 한도 100m 를 요청으로 복사)으로 «꽉 찬 노드에도 뜨게» 한다.
+      containers: [{ name: 'dns', image: 'busybox:1.36', command: ['nslookup', 'kubernetes.default.svc.cluster.local'], resources: { requests: { cpu: '0', memory: '16Mi' }, limits: { cpu: '100m', memory: '64Mi' } } }],
     } });
     if (kubectl([...base, 'create', '-f', '-'], manifest).status !== 0) return 'unknown';
-    const waited = kubectl([...base, '-n', 'elanous-test', 'wait', '--for=jsonpath={.status.phase}=Succeeded', `pod/${name}`, '--timeout=30s']);
+    // 🩸 10-09 19:3x: 부하 104 노드에선 «배치됨»(PodScheduled=True) 뒤 컨테이너 생성만 30초를 넘겼다 → 30초 wait 가 Pending 을 보고 unknown.
+    //   기다림 90초 · Pod 기한 120초. 성공은 DNS_CACHE_MS 동안 공유 캐시라 매 측정이 이 시간을 물지 않는다.
+    const waited = kubectl([...base, '-n', 'elanous-test', 'wait', '--for=jsonpath={.status.phase}=Succeeded', `pod/${name}`, '--timeout=90s']);
     const pod = kubectl([...base, '-n', 'elanous-test', 'get', `pod/${name}`, '-o', 'json']);
     if (pod.status !== 0) return 'unknown';
     const state = object(JSON.parse(pod.stdout));
@@ -296,14 +374,24 @@ function runPoolDnsProbe(context: string, kubectl: PoolKubectl): PoolDnsProbe {
     const statuses = object(state?.status)?.containerStatuses;
     if (Array.isArray(statuses) && statuses.some((c) =>
       ['ErrImagePull', 'ImagePullBackOff', 'InvalidImageName'].includes(String(object(object(object(c)?.state)?.waiting)?.reason)))) return 'image';
+    // 🩸 10-09 18:5x: 부하 걸린 노드(load 78)에선 kubelet 이 «끝난» 탐침의 phase 를 30초 넘게 Running 으로 둔다
+    //   (nslookup 이 답을 찍은 뒤에도 · `timeout 8 nslookup` 도 46초 Running). phase 만 보면 측정이 «unknown» → 승인 0.
+    //   Running 이면 로그를 읽어 해석 성공(이름·주소 줄)이 이미 찍혔는지 본다 — 찍혔으면 DNS 는 된다.
+    if (phase === 'Running') {
+      const early = kubectl([...base, '-n', 'elanous-test', 'logs', `pod/${name}`]);
+      if (early.status === 0 && DNS_PROBE_READY.test(early.stdout) && /\bAddress(?:es)?:\s*\S+/u.test(early.stdout)) {
+        debug.log('pod.lease', 'dns-probe-ready-by-log', { context, phase });
+        return 'ready';
+      }
+    }
     if (phase !== 'Succeeded' && phase !== 'Failed') return 'unknown';
     if (waited.status !== 0 && phase !== 'Failed') return 'unknown';
     const log = kubectl([...base, '-n', 'elanous-test', 'logs', `pod/${name}`]);
     if (log.status !== 0) return 'unknown';
     if (phase === 'Failed') return /can't resolve|server can't find|connection timed out|no servers could be reached|NXDOMAIN|SERVFAIL/iu.test(log.stdout) ? 'dns' : 'unknown';
-    return /\bName:\s*kubernetes\.default\.svc\.cluster\.local\b/u.test(log.stdout) && /\bAddress(?:es)?:\s*\S+/u.test(log.stdout) ? 'ready' : 'dns';
+    return DNS_PROBE_READY.test(log.stdout) && /\bAddress(?:es)?:\s*\S+/u.test(log.stdout) ? 'ready' : 'dns';
   } catch { return 'unknown'; }
-  finally { try { kubectl([...base, '-n', 'elanous-test', 'delete', `pod/${name}`, '--ignore-not-found=true', '--wait=false']); } catch { /* best-effort cleanup */ } }
+  finally { if (cleanup) { try { kubectl([...base, '-n', 'elanous-test', 'delete', `pod/${name}`, '--ignore-not-found=true', '--wait=false']); } catch { /* best-effort cleanup */ } } }
 }
 
 /** Each cluster reads resource reservations; DNS requires a bounded disposable Pod. */
@@ -311,7 +399,7 @@ export function measurePoolLease(members: readonly PodPoolMember[], deps: { kube
   const kubectl = deps.kubectl ?? leaseKubectl;
   return { members: members.map((member) => {
     const base = ['--context', member.context, '--request-timeout=10s'];
-    const result: PodLeaseMember = { context: member.context, capacity: member.capacity, running: null, pending: null, unleasedRunning: null, memoryLimitBytes: null, allocatableMemoryBytes: null, allocatableCpuMillicores: null, availableMemoryByNodeBytes: null, reason: null };
+    const result: PodLeaseMember = { context: member.context, capacity: member.capacity, ...(member.capacityDefaulted ? { capacityDefaulted: true } : {}), running: null, pending: null, unleasedRunning: null, memoryLimitBytes: null, allocatableMemoryBytes: null, allocatableCpuMillicores: null, availableMemoryByNodeBytes: null, reason: null };
     try {
       const nodes = kubectl([...base, 'get', 'nodes', '-o', 'json']);
       const jobs = kubectl([...base, '-n', 'elanous-test', 'get', 'jobs', '-l', 'elanous.substrate=pod', '-o', 'json']);
@@ -534,7 +622,7 @@ export function measurePoolLease(members: readonly PodPoolMember[], deps: { kube
  */
 export function harnessCpuSlotsBesideGate(measure: PoolLeaseMeasure): number | null | undefined {
   if (!measure.members.some((m) => (m.gateJobs ?? 0) > 0)) return undefined;
-  const perPod = quantity(POD_CHILD_REQUESTS.cpu, 'cpu')!;
+  const perPod = quantity(HARNESS_SLOT_CPU_BESIDE_GATE, 'cpu')!;
   let slots = 0;
   for (const m of measure.members) {
     if (m.gateJobs == null || m.gateUnboundJobs == null || m.cpuFreeByNodeMillicores == null || m.pendingHarnessCpuMillicores == null) return null;
@@ -554,7 +642,7 @@ export function recommendConcurrency(measure: PoolLeaseMeasure, options: { capac
   if (goalBytes === null || goalBytes <= 0) return { ...empty, reason: '측정 불가: perGoalMemory' };
   const admission = admissionByUsage(measure, goalBytes, options.perGoalMemory !== undefined);
   if (!measure.members.length || measure.members.some((m) => m.running === null || m.pending === null || m.unleasedRunning === null || m.allocatableMemoryBytes === null || m.memoryLimitBytes === null || m.allocatableCpuMillicores === null || m.availableMemoryByNodeBytes === null)) {
-    debug.log('pod-lease', 'admit-by-usage', { samples: admission.samples, p95Bytes: admission.p95Bytes, admitBytes: admission.admitBytes, fallback: admission.fallback, recommended: null, limitedBy: null });
+    debug.log('pod-lease', 'admit-by-usage', { samples: admission.samples, p95Bytes: admission.p95Bytes, admitBytes: admission.admitBytes, fallback: admission.fallback, recommended: null, limitedBy: null, capacity: options.capacity, capacitySlots: null, memorySlots: null, running: null, pending: null });
     return { ...empty, reason: `측정 불가: cluster${measure.members.map((m) => m.reason ? ` ${m.context}: ${m.reason}` : '').join('')}` };
   }
   if (!Number.isSafeInteger(options.capacity) || options.capacity < 0) return { ...empty, reason: '측정 불가: capacity' };
@@ -599,9 +687,14 @@ export function recommendConcurrency(measure: PoolLeaseMeasure, options: { capac
     Math.floor((m.allocatableMemoryBytes! - m.memoryLimitBytes!) / liteBytes))), 0);
   const memoryBound = Math.min(memorySlots, placeableSlots);
   const recommended = Math.max(0, Math.min(capacitySlots, memoryBound) - eligibleUnleasedRunning);
-  const limitedBy = memoryBound < capacitySlots ? 'memory' : 'capacity';
+  // POD-POOL-UNCAPPED-SPEC — `capacity ≤ occupied` on a member whose cap is the spec default (`:N` absent → 2) means
+  // the «uncapped» pool spec is what admits nothing. Name it `pool-cap`, not the generic `capacity`, so a launch that
+  // sits on `recommended=0` says in its first admit line that the pool spec never declared a cap of its own.
+  const defaultCapFull = capacitySlots <= 0 && measure.members.some((m) => m.capacityDefaulted === true
+    && m.reason !== 'dns' && m.running !== null && m.pending !== null && m.running + m.pending >= m.capacity);
+  const limitedBy: PoolLeaseRecommendation['limitedBy'] = memoryBound < capacitySlots ? 'memory' : defaultCapFull ? 'pool-cap' : 'capacity';
   if (unleasedRunning > 0) debug.log('pod-lease', 'unleased', { unleasedRunning, running, recommended });
-  debug.log('pod-lease', 'admit-by-usage', { samples: admission.samples, p95Bytes: admission.p95Bytes, admitBytes: admission.admitBytes, fallback: admission.fallback, recommended, limitedBy });
+  debug.log('pod-lease', 'admit-by-usage', { samples: admission.samples, p95Bytes: admission.p95Bytes, admitBytes: admission.admitBytes, fallback: admission.fallback, recommended, limitedBy, capacity: options.capacity, capacitySlots, memorySlots, running, pending });
   return { recommended, limitedBy, reason: measure.members.some((m) => m.reason === 'dns') ? 'dns' : null, capacitySlots, memorySlots, liteMemorySlots, placeableSlots, accountSlots, running, pending, pendingJobs, unleasedRunning, admission };
 }
 

package/src/task-orchestrator/surfaces/pod-ledger-collect.ts

@@ -1,7 +1,9 @@
 import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { dirname } from 'node:path';
 import { gunzipSync } from 'node:zlib';
 import { debug } from '../../debug/log.js';
-import { runLedgerDir, runLedgerPath } from '../../self-implement/run-ledger.js';
+import { recordFailureEvent } from '../../self-implement/heal-intake.js';
+import { loadRunLedger, runLedgerDir, runLedgerPath, TRAILING_BOOKKEEPING_EVENTS } from '../../self-implement/run-ledger.js';
 import { ledgerLineToLogEvent } from './pod-ledger-events.js';
 import { defaultPodReemit } from './pod-ledger-prod-sink.js';
 
@@ -12,6 +14,8 @@ type ParseResult = Ledger[] | { error: Incomplete[]; ledgers: Ledger[] };
 const NON_PROGRESS_EVENTS = new Set(['progress-delivery-outcome']);
 const REEMIT_EXCLUDED_PREFIX = 'progress-delivery';
 type Emit = (category: string, event: string, data: Record<string, unknown>) => void;
+/** Heal-loop starter seam — defaults to the real `startHealLoop` inside `recordFailureEvent`. */
+type StartHealLoop = Parameters<typeof recordFailureEvent>[2];
 
 /** Reemit only complete ledger records; offsets name the first byte of each line in the Pod ledger. */
 function reemitLedgerLines(jsonl: string, runId: string, startOffset: number, emit: Emit): number {
@@ -91,15 +95,37 @@ export function parsePodLedgerChunks(logs: string): ParseResult {
   return error.length ? { error, ledgers } : ledgers;
 }
 
+function intakeCollectedFailure(runId: string, dir: string, log: Emit, startHealLoop?: StartHealLoop): void {
+  let ledger: ReturnType<typeof loadRunLedger>;
+  try { ledger = loadRunLedger(runId, dir); }
+  catch { return; }
+  if (!ledger) return;
+  let terminalIndex = ledger.length - 1;
+  while (terminalIndex >= 0 && ledger[terminalIndex]?.event !== 'run-status') terminalIndex -= 1;
+  const terminal = ledger[terminalIndex];
+  if (terminal?.data.runStatus !== 'failed' || !terminal.timestamp || !Number.isFinite(Date.parse(terminal.timestamp))
+    || ledger.slice(terminalIndex + 1).some(entry => !TRAILING_BOOKKEEPING_EVENTS.includes(entry.event as typeof TRAILING_BOOKKEEPING_EVENTS[number]))) return;
+  try {
+    // POD-HEAL-HOST — the recovered Pod failure is recorded into the HOST universe (`dirname(dir)`),
+    // so the host heal loop triages it (harvest + narrowed relaunch) instead of dying inside the Pod.
+    recordFailureEvent({ source: 'harness-run', kind: 'pod-run', ref: runId,
+      summary: `Pod run ${runId} failed`, at: terminal.timestamp }, dirname(dir), startHealLoop);
+  } catch (error) {
+    log('self-implement.pod', 'heal-intake-failed', { runId, reason: error instanceof Error ? error.message : String(error) });
+  }
+}
+
 /** Never replace a host ledger, even if another collector races this one —
  *  except a ledger that THIS run's live follower created (`replace`): that one is a partial copy and the final is complete. */
 export function collectPodLedgers(
   logs: string,
-  { dir = runLedgerDir(), log = defaultPodReemit(), emit = defaultPodReemit(), replace = new Set<string>() }: {
+  { dir = runLedgerDir(), log = defaultPodReemit(), emit = defaultPodReemit(), replace = new Set<string>(), startHealLoop }: {
     dir?: string;
     log?: (category: string, event: string, data: Record<string, unknown>) => void;
     emit?: Emit;
     replace?: ReadonlySet<string>;
+    /** Which starter receives a recovered Pod failure (POD-HEAL-HOST 판정선 measurement); default = real `startHealLoop`. */
+    startHealLoop?: StartHealLoop;
   } = {},
 ): void {
   const parsed = parsePodLedgerChunks(logs);
@@ -116,13 +142,16 @@ export function collectPodLedgers(
       const newLines = Buffer.from(jsonl).subarray(previousBytes).toString('utf8');
       reemitLedgerLines(newLines, runId, previousBytes, emit);
       log('self-implement.pod', 'ledger-collected', { runId, lines: jsonl.split('\n').filter(Boolean).length, bytes: Buffer.byteLength(jsonl) });
+      intakeCollectedFailure(runId, dir, log, startHealLoop);
     } catch (e) {
       if ((e as NodeJS.ErrnoException).code === 'EEXIST') {
         // A reattached Job may have already delivered this exact complete snapshot.
         // An unrelated host ledger (or a partial copy) must still remain untouched.
         try {
           const existing = readFileSync(runLedgerPath(runId, dir));
-          log('self-implement.pod', existing.equals(Buffer.from(jsonl)) ? 'ledger-collect-already-complete' : 'ledger-collect-skipped', { runId, reason: 'exists' });
+          const identical = existing.equals(Buffer.from(jsonl));
+          log('self-implement.pod', identical ? 'ledger-collect-already-complete' : 'ledger-collect-skipped', { runId, reason: 'exists' });
+          if (identical) intakeCollectedFailure(runId, dir, log, startHealLoop);
         } catch (readError) {
           log('self-implement.pod', 'ledger-collect-incomplete', { runId, reason: readError instanceof Error ? readError.message : String(readError) });
         }