elanous 0.2.22 → 0.2.23

package/src/task-orchestrator/surfaces/pod-image-source.ts

@@ -0,0 +1,212 @@
+/**
+ * IMAGE-ONE-SOURCE (대표 2026-10-09 «이미지도 공통 사용·기준값 관리 — 멀티 서버가 같은 곳을 보게»).
+ *
+ * `--substrate pod` 발사는 지금 «이 트리 HEAD ≠ 노드 이미지»면 그 자리에서 굽는다 — 부모 여럿이 같은 커밋을 동시에 굽고,
+ * 노드마다 `:local` 판이 갈렸다. 운영 config `harness.imageCommit`(커밋 12자리 이상)이 있으면 발사는 굽지 않고
+ * 레지스트리의 «기준 이미지»(커밋 태그)를 당긴다. 없으면 종전 경로(HEAD 비교·굽기) 그대로다.
+ *
+ * ⛔ 레지스트리 주소·태그 규칙은 docker/harness/build.sh 의 것이다(새 규칙을 만들지 않는다):
+ *   REGISTRY="${ELANOUS_BUILD_REGISTRY:-k3d-elanous-registry:${ELANOUS_FLEET_REGISTRY_PORT:-5050}}" · TAG="${COMMIT:0:12}".
+ *   풀 멤버가 `#k3d-레지스트리:포트` 를 명시했으면 그 멤버는 그 주소다(syncPoolImages 가 build.sh 에 넘기는 것과 같다).
+ *   조회·Job ref·오류 문구는 전부 «멤버마다 정해진 그 한 주소»를 쓴다.
+ * 레지스트리가 있는 풀 멤버는 원격이면 SSH, 로컬이면 이 기계에서 기준 태그를 찾는다.
+ * 레지스트리가 없는 멤버(node-c 등)와 조회할 수 없는 로컬 레지스트리는 종전 경로로 간다 — 레지스트리 미러는 범위 밖.
+ *   이미지는 표준판(elanous-harness)만 — 이 발사 경로는 lite 를 쓰지 않는다.
+ */
+import { spawnSync } from 'node:child_process';
+import { debug } from '../../debug/log.js';
+import { memberHasRegistry, registryImageRef, type RemoteRun } from './pod-pool.js';
+
+const POD_IMAGE = 'elanous-harness';
+
+export type PodImageSource =
+  | { readonly kind: 'pinned'; readonly ref: string; readonly commit: string; readonly registry: string }
+  | { readonly kind: 'head' };
+
+export type PodImageSourceOutcome = 'pinned-ready' | 'pinned-waited' | 'pinned-missing' | 'head';
+
+/** 기준 이미지가 레지스트리에 올라오기를 기다리는 기본 한도(초). */
+export const POD_IMAGE_PIN_WAIT_SEC = 600;
+const POD_IMAGE_PIN_POLL_MS = 15_000;
+
+const COMMIT = /^[0-9a-f]{12,40}$/u;
+
+/** `harness.imageCommit` 이 유효한 값인가(커밋 hex 12~40자리). */
+export function isPodImageCommit(value: unknown): value is string {
+  return typeof value === 'string' && COMMIT.test(value.trim().toLowerCase());
+}
+
+/** build.sh:274 와 같은 규칙 — `ELANOUS_BUILD_REGISTRY` 다음 `k3d-elanous-registry:${ELANOUS_FLEET_REGISTRY_PORT:-5050}`. */
+export function podImageRegistry(env: NodeJS.ProcessEnv = process.env): string {
+  // bash `${X:-d}` 는 빈 문자열도 «없음»으로 본다 — `||` 로 맞춘다.
+  return env.ELANOUS_BUILD_REGISTRY || `k3d-elanous-registry:${env.ELANOUS_FLEET_REGISTRY_PORT || '5050'}`;
+}
+
+/** build.sh:276 — 태그 = 커밋 앞 12자리. */
+export function podImageRef(registry: string, commit: string): string {
+  return `${registry}/${POD_IMAGE}:${commit.slice(0, 12)}`;
+}
+
+export function resolvePodImageSource(
+  cfg: { harness?: { imageCommit?: unknown } } | null | undefined,
+  deps: { env?: NodeJS.ProcessEnv } = {},
+): PodImageSource {
+  const raw = cfg?.harness?.imageCommit;
+  if (!isPodImageCommit(raw)) return { kind: 'head' };
+  const commit = raw.trim().toLowerCase();
+  const registry = podImageRegistry(deps.env ?? process.env);
+  return { kind: 'pinned', ref: podImageRef(registry, commit), commit, registry };
+}
+
+export function podImageMissingMessage(ref: string): string {
+  return `기준 이미지 ${ref} 가 레지스트리에 없다 — 레일이 굽기 전이다`;
+}
+
+export interface PodImageMember { readonly context: string; readonly sshHost?: string; readonly registry?: string }
+
+/** 이 멤버가 기준 이미지를 찾을 레지스트리 — 멤버 명시값, 없으면 build.sh 규칙. */
+export function memberPinRegistry(member: PodImageMember, source: Extract<PodImageSource, { kind: 'pinned' }>): string {
+  return member.registry ?? source.registry;
+}
+
+/** 실제 조회 — 원격은 pod-pool 의 SSH 판정, 로컬은 이 기계의 docker/curl 로 조회한다. */
+export function podPoolImageLookup(run?: RemoteRun, localRun: (cmd: string, args: readonly string[]) => { status: number | null; stdout: string } = (cmd, args) => {
+  const r = spawnSync(cmd, [...args], { encoding: 'utf8', timeout: 10_000 });
+  return { status: r.status, stdout: r.stdout ?? '' };
+}): {
+  hasRegistry: (member: PodImageMember & { context: string }, registry: string) => boolean;
+  lookupTag: (member: PodImageMember & { context: string }, registry: string, commit: string) => string | null;
+} {
+  return {
+    hasRegistry: (member, registry) => {
+      if (member.sshHost) return memberHasRegistry({ capacity: 0, k3dCluster: '', ...member, registry }, run);
+      try { return localRun('docker', ['inspect', registry.split(':')[0]!]).status === 0; }
+      catch { return false; }
+    },
+    lookupTag: (member, registry, commit) => {
+      if (member.sshHost) return registryImageRef({ capacity: 0, k3dCluster: '', ...member, registry }, commit, run, `${POD_IMAGE}:local`);
+      const port = registry.slice(registry.lastIndexOf(':') + 1);
+      const tag = commit.slice(0, 12);
+      try {
+        const r = localRun('curl', ['-fsS', '--max-time', '5', `http://localhost:${port}/v2/${POD_IMAGE}/tags/list`]);
+        if (r.status !== 0) return null;
+        const tags = (JSON.parse(r.stdout) as { tags?: string[] }).tags;
+        return Array.isArray(tags) && tags.includes(tag) ? podImageRef(registry, commit) : null;
+      } catch { return null; }
+    },
+  };
+}
+
+export interface PreparePodImageDeps<M extends PodImageMember, H> {
+  readonly cfg: { harness?: { imageCommit?: unknown } } | null | undefined;
+  readonly env?: NodeJS.ProcessEnv;
+  /** 풀 멤버(풀이 없으면 빈 배열 — 이 기계의 컨텍스트 하나). */
+  readonly members: readonly M[];
+  /** 종전 경로(HEAD 비교·굽기·노드 동기화). 기준값이 없으면 전 멤버로, 있으면 «기준 이미지를 못 쓰는» 멤버로만 부른다. */
+  readonly headPath: (members: readonly M[]) => Promise<H>;
+  /** 이 멤버의 노드에 그 레지스트리가 있나. */
+  readonly hasRegistry: (member: M, registry: string) => boolean;
+  /** 이 멤버의 그 레지스트리에 그 커밋 태그가 있으면 Pod 가 당길 ref, 없으면 null. */
+  readonly lookupTag: (member: M, registry: string, commit: string) => string | null;
+  readonly maxWaitMs?: number;
+  readonly pollMs?: number;
+  readonly sleep?: (ms: number) => Promise<void>;
+  readonly now?: () => number;
+  readonly onWait?: (pending: readonly M[], waitedMs: number) => void;
+}
+
+export interface PreparedPodImage<M extends PodImageMember, H> {
+  readonly source: PodImageSource;
+  readonly outcome: PodImageSourceOutcome;
+  /** 기준 이미지를 당기는 멤버 — `imageRef` 가 Job 이미지다. */
+  readonly pinned: ReadonlyArray<M & { imageRef: string }>;
+  /** 레지스트리는 있는데 한도 안에 태그가 안 올라온 멤버와 그 멤버가 찾던 ref. */
+  readonly missing: ReadonlyArray<{ member: M; ref: string }>;
+  /** 종전 경로를 탄 멤버(레지스트리 없음 · 로컬 태그 조회 실패). */
+  readonly unpinned: readonly M[];
+  /** 종전 경로의 결과 — 부르지 않았으면 null. */
+  readonly head: H | null;
+  readonly waitedMs: number;
+}
+
+/**
+ * 발사 이미지를 준비한다. 기준값이 없으면 `headPath(members)` 하나만 부른다(종전 동작 그대로).
+ * 기준값이 있으면: 레지스트리가 있는 멤버는 태그를 기다려 당기고(굽지 않는다), 나머지 멤버만 종전 경로로 보낸다.
+ * 원격 레지스트리 멤버 «전부» 태그가 끝내 없으면 outcome `pinned-missing` — 종전 경로도 부르지 않는다(호출부가 exit 2).
+ */
+export async function preparePodImage<M extends PodImageMember, H>(deps: PreparePodImageDeps<M, H>): Promise<PreparedPodImage<M, H>> {
+  const source = resolvePodImageSource(deps.cfg, { env: deps.env });
+  if (source.kind === 'head') {
+    return { source, outcome: 'head', pinned: [], missing: [], unpinned: deps.members, head: await deps.headPath(deps.members), waitedMs: 0 };
+  }
+  const localRefs = new Map<string, string>();
+  const registryMembers = deps.members.filter((m) => {
+    const registry = memberPinRegistry(m, source);
+    if (!deps.hasRegistry(m, registry)) return false;
+    if (m.sshHost) return true;
+    // A local registry without the requested tag (or an unreadable one) can still use the old build path.
+    const ref = deps.lookupTag(m, registry, source.commit);
+    if (!ref) return false;
+    localRefs.set(m.context, ref);
+    return true;
+  });
+  const unpinned = deps.members.filter((m) => !registryMembers.includes(m));
+  if (registryMembers.length === 0) {
+    // 기준 이미지를 당길 멤버가 없다(풀 없음 · 레지스트리 없는 멤버뿐) — 종전 경로.
+    return { source, outcome: 'head', pinned: [], missing: [], unpinned, head: await deps.headPath(unpinned), waitedMs: 0 };
+  }
+  const now = deps.now ?? Date.now;
+  const sleep = deps.sleep ?? ((ms: number) => new Promise<void>((done) => setTimeout(done, ms)));
+  const maxWaitMs = Math.max(0, deps.maxWaitMs ?? POD_IMAGE_PIN_WAIT_SEC * 1000);
+  const pollMs = Math.max(1, deps.pollMs ?? POD_IMAGE_PIN_POLL_MS);
+  const found = new Map<string, string>();
+  const t0 = now();
+  let pending = registryMembers;
+  let slept = false;
+  for (;;) {
+    pending = pending.filter((m) => {
+      const ref = localRefs.get(m.context) ?? deps.lookupTag(m, memberPinRegistry(m, source), source.commit);
+      if (ref) found.set(m.context, ref);
+      return !ref;
+    });
+    const waited = now() - t0;
+    if (pending.length === 0 || waited >= maxWaitMs) break;
+    deps.onWait?.(pending, waited);
+    slept = true;
+    await sleep(Math.min(pollMs, maxWaitMs - waited));
+  }
+  const waitedMs = now() - t0;
+  const pinned = registryMembers.filter((m) => found.has(m.context)).map((m) => ({ ...m, imageRef: found.get(m.context)! }));
+  const missing = pending.map((member) => ({ member, ref: podImageRef(memberPinRegistry(member, source), source.commit) }));
+  if (pinned.length === 0) {
+    return { source, outcome: 'pinned-missing', pinned, missing, unpinned, head: null, waitedMs };
+  }
+  return {
+    source,
+    outcome: slept ? 'pinned-waited' : 'pinned-ready',
+    pinned,
+    missing,
+    unpinned,
+    head: unpinned.length > 0 ? await deps.headPath(unpinned) : null,
+    waitedMs,
+  };
+}
+
+/**
+ * 기준값 경로의 `image-source` 사건 — 발사 호출부(index.ts)가 부르는 단 하나의 기록처.
+ * 이 기계의 로컬 멤버(sshHost 없음)가 기준 이미지를 당겼으면 `local: true` 를 싣는다.
+ */
+export function logPinnedImageSource<M extends PodImageMember, H>(
+  prepared: PreparedPodImage<M, H>,
+  ctx: { readonly ref: string; readonly headCommit: string | null },
+): void {
+  if (prepared.source.kind !== 'pinned') return;
+  const local = prepared.pinned.some((m) => !m.sshHost);
+  debug.log('self-implement.pod', 'image-source', {
+    kind: prepared.source.kind, ref: ctx.ref, pinRef: prepared.source.ref, headCommit: ctx.headCommit, waitedMs: prepared.waitedMs, outcome: prepared.outcome,
+    ...(local ? { local: true } : {}),
+    pinned: prepared.pinned.map((m) => ({ context: m.context, imageRef: m.imageRef })),
+    missing: prepared.missing.map((x) => ({ context: x.member.context, ref: x.ref })),
+    unpinned: prepared.unpinned.map((m) => m.context),
+  });
+}

package/src/task-orchestrator/surfaces/pod-lease.ts

@@ -8,7 +8,14 @@ import { dlopen } from 'bun:ffi';
 import { hostLeaseBaseDir, type PendingJobIdentity } from '../../pod-lease/host-lease.js';
 
 /** The harness Pod's container requests (self-implement-pod Job manifest). Lives here so admission reuses the real request. */
-export const POD_CHILD_REQUESTS = { cpu: '1', memory: '6Gi' } as const;
+// POD-CPU-REQUEST-RIGHTSIZE(10-09 · OP 실측): 하니스 Pod 실사용은 대부분 100m 미만(LLM 응답 대기)인데 요청 1 CPU 를
+//   잡아 node-b 32코어가 «요청 31/32»로 막혔다(실사용 4.6코어 · 14%). 요청은 실사용 근처로, 한도(4)는 그대로 — 몰릴 때는 한도까지 쓴다.
+//   게이트·pod-command 기본은 pod-command-job.ts 의 POD_COMMAND_CPU_REQUEST_DEFAULT(1)로 따로 둔다(테스트 조각은 CPU 를 실제로 쓴다).
+export const POD_CHILD_REQUESTS = { cpu: '250m', memory: '6Gi' } as const;
+/** Finished harness and command Jobs release their Pod slots after 30 minutes. */
+export const POD_JOB_TTL_AFTER_FINISHED_SECONDS = 1800;
+/** 게이트가 돌 때 하니스 «한 자리»로 셈하는 CPU — 요청(250m)보다 보수적으로 1 코어를 잡아 게이트 조각의 CPU 를 지킨다(GATE-RESERVE-AUTO 셈은 그대로). */
+export const HARNESS_SLOT_CPU_BESIDE_GATE = '1';
 /** POD-ADMIT-BY-USAGE — observed usage is scaled by this before it reserves memory. */
 const POD_USAGE_HEADROOM = 1.5;
 /** POD-ADMIT-BY-USAGE — a node keeps at least this share of allocatable memory free (real usage and bookkeeping). */
@@ -207,6 +214,7 @@ function runningLimit(spec: Record<string, unknown>): number | null {
 /** A short-lived Pod proves cluster DNS from inside the same namespace as harness Jobs. Image pull failures are not DNS failures. */
 export type PoolDnsProbe = 'ready' | 'dns' | 'image' | 'unknown';
 const DNS_CACHE_MS = 10 * 60_000;
+const DNS_STALE_OK_MS = 30 * 60_000;
 const DNS_LOCK_WAIT_MS = 6 * 60_000;
 // flock is released by the kernel on process exit, including a crash before any cache write.
 // The lock file is never unlinked: unlinking it would let waiters lock different inodes.
@@ -232,6 +240,17 @@ export function probePoolDns(context: string, kubectl: PoolKubectl = leaseKubect
     return record?.result === 'ready' && typeof record.at === 'number' &&
       now() >= record.at && now() - record.at < DNS_CACHE_MS;
   };
+  const withStaleReady = (result: PoolDnsProbe): PoolDnsProbe => {
+    if (result !== 'unknown') return result;
+    const record = readRecord(cache);
+    const readyAgeMs = typeof record?.at === 'number' ? now() - record.at : NaN;
+    if (record?.result === 'ready' &&
+        Number.isFinite(readyAgeMs) && readyAgeMs >= 0 && readyAgeMs < DNS_STALE_OK_MS) {
+      debug.log('pod.lease', 'dns-probe-stale-ready', { context, readyAgeMs });
+      return 'ready';
+    }
+    return result;
+  };
   if (cached()) return 'ready';
   const previous = readRecord(outcome);
   try { mkdirSync(dir, { recursive: true, mode: 0o700 }); } catch { return 'unknown'; }
@@ -251,31 +270,34 @@ export function probePoolDns(context: string, kubectl: PoolKubectl = leaseKubect
     while (!held) {
       if (cached()) return 'ready';
       const shared = sharedFailure();
-      if (shared) return shared;
+      if (shared) return withStaleReady(shared);
       if (dnsFlock(fd, 2 | 4) === 0) { held = true; break; } // LOCK_EX | LOCK_NB
       if (Date.now() >= deadline) return 'unknown';
       Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 100);
     }
     if (cached()) return 'ready';
     const shared = sharedFailure();
-    if (shared) return shared;
-    const result = runPoolDnsProbe(context, kubectl);
+    if (shared) return withStaleReady(shared);
+    const result = runPoolDnsProbe(context, kubectl, now);
     const destination = result === 'ready' ? cache : outcome;
     const temp = `${destination}.${randomUUID()}.tmp`;
     try {
       writeFileSync(temp, JSON.stringify({ result, at: now(), generation: randomUUID() }), { mode: 0o600 });
       renameSync(temp, destination);
     } catch { try { rmSync(temp, { force: true }); } catch { /* best-effort cleanup */ } }
-    return result;
+    return withStaleReady(result);
   } finally {
     if (held) dnsFlock(fd, 8); // LOCK_UN
     closeSync(fd);
   }
 }
 
-function runPoolDnsProbe(context: string, kubectl: PoolKubectl): PoolDnsProbe {
+const DNS_PROBE_READY = /\bName:\s*kubernetes\.default\.svc\.cluster\.local\b/u;
+
+function runPoolDnsProbe(context: string, kubectl: PoolKubectl, now: () => number): PoolDnsProbe {
   const name = `elanous-dns-${randomUUID().slice(0, 12)}`;
   const base = ['--context', context, '--request-timeout=10s'];
+  let cleanup = false;
   try {
     const core = kubectl([...base, '-n', 'kube-system', 'get', 'deployment', 'coredns', '-o', 'json']);
     if (core.status !== 0) return 'unknown';
@@ -283,12 +305,61 @@ function runPoolDnsProbe(context: string, kubectl: PoolKubectl): PoolDnsProbe {
     const spec = object(deployment?.spec);
     const status = object(deployment?.status);
     if (typeof spec?.replicas !== 'number' || spec.replicas < 1 || status?.readyReplicas !== spec.replicas) return 'dns';
+    const listed = kubectl([...base, '-n', 'elanous-test', 'get', 'pods', '-l', 'elanous.probe=dns', '-o', 'json']);
+    if (listed.status !== 0) return 'unknown';
+    const listedPods = items(JSON.parse(listed.stdout));
+    if (!listedPods) return 'unknown';
+    const probes = listedPods.filter((pod) => {
+      const metadata = object(pod.metadata);
+      return object(metadata?.labels)?.['elanous.probe'] === 'dns' &&
+        (metadata?.namespace === undefined || metadata.namespace === 'elanous-test');
+    });
+    let reaped = 0;
+    for (const probe of probes) {
+      const metadata = object(probe.metadata);
+      const name = metadata?.name;
+      const deletedAt = metadata?.deletionTimestamp;
+      if (typeof name !== 'string' || !/^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/.test(name) ||
+          typeof deletedAt !== 'string' || !Number.isFinite(Date.parse(deletedAt)) ||
+          now() - Date.parse(deletedAt) <= 60_000) continue;
+      try {
+        if (kubectl([...base, '-n', 'elanous-test', 'delete', 'pods', '-l', 'elanous.probe=dns', `--field-selector=metadata.name=${name}`, '--grace-period=0', '--force', '--ignore-not-found=true', '--wait=false']).status === 0) reaped++;
+      } catch { /* Reaping is best-effort; the crowded reading still prevents a new probe. */ }
+    }
+    if (reaped) debug.log('pod.lease', 'dns-probe-stale-reaped', { context, count: reaped });
+    // 🩸 10-09 22:1x: node-b had 20 Completed probes left over from older releases (and 7 Terminating). Counting every
+    //   labelled Pod would make the node «crowded» forever and admission would stay 0 — only live probes are crowding.
+    //   Finished probes are removed here so older leftovers drain too.
+    let finished = 0;
+    const live = probes.filter((probe) => {
+      const metadata = object(probe.metadata);
+      if (metadata?.deletionTimestamp !== undefined) return false;
+      const phase = object(probe.status)?.phase;
+      if (phase !== 'Succeeded' && phase !== 'Failed') return true;
+      const name = metadata?.name;
+      if (typeof name === 'string' && /^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/.test(name)) {
+        try {
+          if (kubectl([...base, '-n', 'elanous-test', 'delete', 'pods', '-l', 'elanous.probe=dns', `--field-selector=metadata.name=${name}`, '--ignore-not-found=true', '--wait=false']).status === 0) finished++;
+        } catch { /* best-effort: a finished probe never crowds */ }
+      }
+      return false;
+    });
+    if (finished) debug.log('pod.lease', 'dns-probe-finished-reaped', { context, count: finished });
+    if (live.length >= 3) {
+      debug.log('pod.lease', 'dns-probe-skipped-crowded', { context, existing: live.length });
+      return 'unknown';
+    }
+    cleanup = true;
     const manifest = JSON.stringify({ apiVersion: 'v1', kind: 'Pod', metadata: { name, namespace: 'elanous-test', labels: { 'elanous.probe': 'dns' } }, spec: {
-      restartPolicy: 'Never', activeDeadlineSeconds: 40, automountServiceAccountToken: false,
-      containers: [{ name: 'dns', image: 'busybox:1.36', command: ['nslookup', 'kubernetes.default.svc.cluster.local'], resources: { requests: { cpu: '10m', memory: '16Mi' }, limits: { cpu: '100m', memory: '64Mi' } } }],
+      restartPolicy: 'Never', activeDeadlineSeconds: 120, automountServiceAccountToken: false,
+      // 🩸 10-09 18:2x: 노드 CPU «요청» 100%(32/32) 이면 10m 탐침도 못 떠 측정이 «dns probe unknown» → 승인 추천 null → 전 발사 정지(교착).
+      //   탐침은 CPU 요청 0(명시 · 안 쓰면 k8s 가 한도 100m 를 요청으로 복사)으로 «꽉 찬 노드에도 뜨게» 한다.
+      containers: [{ name: 'dns', image: 'busybox:1.36', command: ['nslookup', 'kubernetes.default.svc.cluster.local'], resources: { requests: { cpu: '0', memory: '16Mi' }, limits: { cpu: '100m', memory: '64Mi' } } }],
     } });
     if (kubectl([...base, 'create', '-f', '-'], manifest).status !== 0) return 'unknown';
-    const waited = kubectl([...base, '-n', 'elanous-test', 'wait', '--for=jsonpath={.status.phase}=Succeeded', `pod/${name}`, '--timeout=30s']);
+    // 🩸 10-09 19:3x: 부하 104 노드에선 «배치됨»(PodScheduled=True) 뒤 컨테이너 생성만 30초를 넘겼다 → 30초 wait 가 Pending 을 보고 unknown.
+    //   기다림 90초 · Pod 기한 120초. 성공은 DNS_CACHE_MS 동안 공유 캐시라 매 측정이 이 시간을 물지 않는다.
+    const waited = kubectl([...base, '-n', 'elanous-test', 'wait', '--for=jsonpath={.status.phase}=Succeeded', `pod/${name}`, '--timeout=90s']);
     const pod = kubectl([...base, '-n', 'elanous-test', 'get', `pod/${name}`, '-o', 'json']);
     if (pod.status !== 0) return 'unknown';
     const state = object(JSON.parse(pod.stdout));
@@ -296,14 +367,24 @@ function runPoolDnsProbe(context: string, kubectl: PoolKubectl): PoolDnsProbe {
     const statuses = object(state?.status)?.containerStatuses;
     if (Array.isArray(statuses) && statuses.some((c) =>
       ['ErrImagePull', 'ImagePullBackOff', 'InvalidImageName'].includes(String(object(object(object(c)?.state)?.waiting)?.reason)))) return 'image';
+    // 🩸 10-09 18:5x: 부하 걸린 노드(load 78)에선 kubelet 이 «끝난» 탐침의 phase 를 30초 넘게 Running 으로 둔다
+    //   (nslookup 이 답을 찍은 뒤에도 · `timeout 8 nslookup` 도 46초 Running). phase 만 보면 측정이 «unknown» → 승인 0.
+    //   Running 이면 로그를 읽어 해석 성공(이름·주소 줄)이 이미 찍혔는지 본다 — 찍혔으면 DNS 는 된다.
+    if (phase === 'Running') {
+      const early = kubectl([...base, '-n', 'elanous-test', 'logs', `pod/${name}`]);
+      if (early.status === 0 && DNS_PROBE_READY.test(early.stdout) && /\bAddress(?:es)?:\s*\S+/u.test(early.stdout)) {
+        debug.log('pod.lease', 'dns-probe-ready-by-log', { context, phase });
+        return 'ready';
+      }
+    }
     if (phase !== 'Succeeded' && phase !== 'Failed') return 'unknown';
     if (waited.status !== 0 && phase !== 'Failed') return 'unknown';
     const log = kubectl([...base, '-n', 'elanous-test', 'logs', `pod/${name}`]);
     if (log.status !== 0) return 'unknown';
     if (phase === 'Failed') return /can't resolve|server can't find|connection timed out|no servers could be reached|NXDOMAIN|SERVFAIL/iu.test(log.stdout) ? 'dns' : 'unknown';
-    return /\bName:\s*kubernetes\.default\.svc\.cluster\.local\b/u.test(log.stdout) && /\bAddress(?:es)?:\s*\S+/u.test(log.stdout) ? 'ready' : 'dns';
+    return DNS_PROBE_READY.test(log.stdout) && /\bAddress(?:es)?:\s*\S+/u.test(log.stdout) ? 'ready' : 'dns';
   } catch { return 'unknown'; }
-  finally { try { kubectl([...base, '-n', 'elanous-test', 'delete', `pod/${name}`, '--ignore-not-found=true', '--wait=false']); } catch { /* best-effort cleanup */ } }
+  finally { if (cleanup) { try { kubectl([...base, '-n', 'elanous-test', 'delete', `pod/${name}`, '--ignore-not-found=true', '--wait=false']); } catch { /* best-effort cleanup */ } } }
 }
 
 /** Each cluster reads resource reservations; DNS requires a bounded disposable Pod. */
@@ -534,7 +615,7 @@ export function measurePoolLease(members: readonly PodPoolMember[], deps: { kube
  */
 export function harnessCpuSlotsBesideGate(measure: PoolLeaseMeasure): number | null | undefined {
   if (!measure.members.some((m) => (m.gateJobs ?? 0) > 0)) return undefined;
-  const perPod = quantity(POD_CHILD_REQUESTS.cpu, 'cpu')!;
+  const perPod = quantity(HARNESS_SLOT_CPU_BESIDE_GATE, 'cpu')!;
   let slots = 0;
   for (const m of measure.members) {
     if (m.gateJobs == null || m.gateUnboundJobs == null || m.cpuFreeByNodeMillicores == null || m.pendingHarnessCpuMillicores == null) return null;

package/src/task-orchestrator/surfaces/pod-ledger-collect.ts

@@ -1,7 +1,9 @@
 import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { dirname } from 'node:path';
 import { gunzipSync } from 'node:zlib';
 import { debug } from '../../debug/log.js';
-import { runLedgerDir, runLedgerPath } from '../../self-implement/run-ledger.js';
+import { recordFailureEvent } from '../../self-implement/heal-intake.js';
+import { loadRunLedger, runLedgerDir, runLedgerPath, TRAILING_BOOKKEEPING_EVENTS } from '../../self-implement/run-ledger.js';
 import { ledgerLineToLogEvent } from './pod-ledger-events.js';
 import { defaultPodReemit } from './pod-ledger-prod-sink.js';
 
@@ -91,6 +93,24 @@ export function parsePodLedgerChunks(logs: string): ParseResult {
   return error.length ? { error, ledgers } : ledgers;
 }
 
+function intakeCollectedFailure(runId: string, dir: string, log: Emit): void {
+  let ledger: ReturnType<typeof loadRunLedger>;
+  try { ledger = loadRunLedger(runId, dir); }
+  catch { return; }
+  if (!ledger) return;
+  let terminalIndex = ledger.length - 1;
+  while (terminalIndex >= 0 && ledger[terminalIndex]?.event !== 'run-status') terminalIndex -= 1;
+  const terminal = ledger[terminalIndex];
+  if (terminal?.data.runStatus !== 'failed' || !terminal.timestamp || !Number.isFinite(Date.parse(terminal.timestamp))
+    || ledger.slice(terminalIndex + 1).some(entry => !TRAILING_BOOKKEEPING_EVENTS.includes(entry.event as typeof TRAILING_BOOKKEEPING_EVENTS[number]))) return;
+  try {
+    recordFailureEvent({ source: 'harness-run', kind: 'pod-run', ref: runId,
+      summary: `Pod run ${runId} failed`, at: terminal.timestamp }, dirname(dir));
+  } catch (error) {
+    log('self-implement.pod', 'heal-intake-failed', { runId, reason: error instanceof Error ? error.message : String(error) });
+  }
+}
+
 /** Never replace a host ledger, even if another collector races this one —
  *  except a ledger that THIS run's live follower created (`replace`): that one is a partial copy and the final is complete. */
 export function collectPodLedgers(
@@ -116,13 +136,16 @@ export function collectPodLedgers(
       const newLines = Buffer.from(jsonl).subarray(previousBytes).toString('utf8');
       reemitLedgerLines(newLines, runId, previousBytes, emit);
       log('self-implement.pod', 'ledger-collected', { runId, lines: jsonl.split('\n').filter(Boolean).length, bytes: Buffer.byteLength(jsonl) });
+      intakeCollectedFailure(runId, dir, log);
     } catch (e) {
       if ((e as NodeJS.ErrnoException).code === 'EEXIST') {
         // A reattached Job may have already delivered this exact complete snapshot.
         // An unrelated host ledger (or a partial copy) must still remain untouched.
         try {
           const existing = readFileSync(runLedgerPath(runId, dir));
-          log('self-implement.pod', existing.equals(Buffer.from(jsonl)) ? 'ledger-collect-already-complete' : 'ledger-collect-skipped', { runId, reason: 'exists' });
+          const identical = existing.equals(Buffer.from(jsonl));
+          log('self-implement.pod', identical ? 'ledger-collect-already-complete' : 'ledger-collect-skipped', { runId, reason: 'exists' });
+          if (identical) intakeCollectedFailure(runId, dir, log);
         } catch (readError) {
           log('self-implement.pod', 'ledger-collect-incomplete', { runId, reason: readError instanceof Error ? readError.message : String(readError) });
         }

package/src/task-orchestrator/surfaces/pod-pool.ts

@@ -23,7 +23,7 @@ import { debug } from '../../debug/log.js';
 import { PodLeaseAdmission, type PodLeasePredecessor, type PodLeaseRelease } from '../../pod-lease/admission.js';
 import { predecessorState, type PredecessorState } from '../../pod-lease/dependency-state.js';
 import { HostPoolLease, hostLeaseBaseDir, leaseHasPendingPod } from '../../pod-lease/host-lease.js';
-import { acquireLockSync } from '../../storage/file-lock.js';
+import { acquireLockAsync, acquireLockSync } from '../../storage/file-lock.js';
 import { harnessCpuSlotsBesideGate, measurePoolLease, recommendConcurrency, type PodLeaseMember, type PoolDnsProbe, type PoolLeaseRecommendation } from './pod-lease.js';
 
 export interface PodPoolMember {
@@ -99,6 +99,12 @@ export function registryImageRef(member: PodPoolMember, commit: string | null, r
   } catch { return null; }
 }
 
+/** IMAGE-ONE-SOURCE: 이 멤버의 노드에 레지스트리 컨테이너가 있나(syncPoolImages 의 판정과 같은 명령). 로컬 멤버는 false. */
+export function memberHasRegistry(member: PodPoolMember, run: RemoteRun = defaultRemoteRun): boolean {
+  if (!member.sshHost) return false;
+  return run(member.sshHost, `docker inspect ${memberRegistry(member).split(':')[0]} >/dev/null 2>&1`).status === 0;
+}
+
 /** 노드 이미지의 스킬 해시 라벨(`elanous.pod-skills`). */
 export function remoteImageSkillsDigest(member: PodPoolMember, image: string, run: RemoteRun = defaultRemoteRun): string | null {
   if (!member.sshHost) return null;
@@ -276,7 +282,14 @@ export class PodPoolScheduler {
       const decision = measured.recommended === null ? measured : { ...measured, recommended: Math.max(0, measured.recommended - others + (transferIsOther ? 1 : 0)) };
       // An unknown reading cannot erase permits already granted: on recovery,
       // the first healthy recommendation must still account for their slots.
-      if (decision.recommended === null) return decision;
+      // GATE-ADMIT-UNMEASURED(10-09 대표 «판 발행 문제없게»): 측정이 «?»(예: 부하 노드의 DNS 탐침)여도 게이트 조각은 하나씩 들어간다 —
+      //   게이트가 측정 복구를 기다리며 조각마다 승인 시간 초과를 물지 않게. 실제로 자리가 없으면 스케줄러가 Pending 으로 둔다(잃는 것 없음).
+      if (decision.recommended === null) {
+        if (!gate) return decision;
+        const active = this.gateAdmission.snapshot().active;
+        debug.log('pod.pool', 'gate-admit-unmeasured', { active, reason: decision.reason ?? null });
+        return { ...decision, recommended: active + 1 };
+      }
       // A new launch consumes a measured additional slot. Compare consecutive
       // measurements so repeated stale readings cannot grant that slot again.
       this.budget = Math.max(0, this.budget + decision.recommended - (this.initialized ? this.lastFree : 0));
@@ -324,7 +337,7 @@ export class PodPoolScheduler {
         const pendingJobs = this.lastRaw?.pendingJobs ?? [];
         const lease = transferred ?? (free !== null && free > 0 ? this.hostLease.tryReserve((others) =>
           others - this.hostLease.live().filter((r) => leaseHasPendingPod(r, pendingJobs) &&
-            (r.pid !== process.pid || r.startedAt !== this.hostLease.selfStart())).length < free) : null);
+            (r.pid !== process.pid || r.startedAt !== this.hostLease.selfStart())).length < free) : (gate && free === null ? this.hostLease.tryReserve(() => true) : null));
         if (lease) {
           let hostReleased = false;
           const releaseHost = () => { if (!hostReleased) { hostReleased = true; lease(); } };
@@ -719,7 +732,7 @@ function defaultRemoteBuild(script: string, image: string): RemoteBuild {
  *           📏 커밋이 바뀐 뒤 두 노드를 올리는 데 벽시계 55초(종전: 다시 굽기 1분 40초 ⊕ 4GB 차례 전송 5분 48초).
  *   실패하면 = 종전의 통째 전송(syncPoolImage)으로 떨어진다.
  */
-export async function syncPoolImages(members: readonly PodPoolMember[], image: string, localCommit: string | null, deps: { run?: RemoteRun; remoteBuild?: RemoteBuild; buildScript?: string | null; ship?: typeof syncPoolImage; localSkillsDigest?: string | null; waitIntervalMs?: number; waitMaxMs?: number; now?: () => number; sleep?: (ms: number) => Promise<void>; inspect?: LocalImageInspect; transfer?: ImageShip; pack?: PoolPack | null } = {}): Promise<Map<string, PoolImageSync>> {
+export async function syncPoolImages(members: readonly PodPoolMember[], image: string, localCommit: string | null, deps: { run?: RemoteRun; remoteBuild?: RemoteBuild; buildScript?: string | null; ship?: typeof syncPoolImage; localSkillsDigest?: string | null; waitIntervalMs?: number; waitMaxMs?: number; now?: () => number; sleep?: (ms: number) => Promise<void>; inspect?: LocalImageInspect; transfer?: ImageShip; pack?: PoolPack | null; lock?: ImageShipLockOptions } = {}): Promise<Map<string, PoolImageSync>> {
   const run = deps.run ?? defaultRemoteRun;
   const script = deps.buildScript === undefined ? podImageBuildScript() : deps.buildScript;
   const remoteBuild = deps.remoteBuild ?? (script ? defaultRemoteBuild(script, image) : null);
@@ -734,22 +747,61 @@ export async function syncPoolImages(members: readonly PodPoolMember[], image: s
     const t0 = Date.now();
     if (!m.sshHost) { results.set(m.context, { ok: true, action: 'local', detail: 'this machine', ms: 0 }); return; }
     const before = remoteImageCommit(m, image, run);
-    // ⭐ «같은 판» = 커밋 ⊕ 스킬 해시 ⊕ (레지스트리 노드면) 레지스트리에 그 커밋 태그가 있다 — 스킬만 바뀌어도 다시 굽는다(대표 2026-09-26 «스킬 셋트 싱크»).
     const skillsNow = deps.localSkillsDigest;
-    const skillsSame = skillsNow == null || remoteImageSkillsDigest(m, image, run) === skillsNow;
     const refBefore = registryImageRef(m, localCommit, run, image);
     const hasRegistry = run(m.sshHost, `docker inspect ${memberRegistry(m).split(':')[0]} >/dev/null 2>&1`).status === 0;
-    if (localCommit && before === localCommit && skillsSame && (!hasRegistry || refBefore)) {
-      results.set(m.context, { ok: true, action: 'fresh', detail: before.slice(0, 12), ms: Date.now() - t0, ...(refBefore ? { imageRef: refBefore } : {}) });
+    const registrySkillsSame = (ref: string) => skillsNow == null || remoteImageSkillsDigest(m, `localhost:${memberRegistry(m).split(':')[1]}/${ref.slice(ref.indexOf('/') + 1)}`, run) === skillsNow;
+    const skillsSame = hasRegistry && refBefore ? registrySkillsSame(refBefore) : skillsNow == null || remoteImageSkillsDigest(m, image, run) === skillsNow;
+    if (hasRegistry && refBefore && skillsSame) {
+      debug.log('pod.pool', 'image-fresh-registry-tag', { context: m.context, host: m.sshHost, commit: localCommit, localLabel: before, imageRef: refBefore, waitedMs: 0 });
+      results.set(m.context, { ok: true, action: 'fresh', detail: localCommit!.slice(0, 12), ms: Date.now() - t0, imageRef: refBefore });
+      return;
+    }
+    if (!hasRegistry && localCommit && before === localCommit && skillsSame) {
+      results.set(m.context, { ok: true, action: 'fresh', detail: before.slice(0, 12), ms: Date.now() - t0 });
       return;
     }
     if (remoteBuild) {
-      // 같은 판이 아닌 멤버가 처음 여기 닿을 때 pack 한다(전부 fresh 면 pack 도 없다).
-      const shared = sharePack ? await (packed ??= sharePack()) : null;
-      if (shared && !shared.ok) debug.log('self-implement.pod', 'pool-image-pack-failed', { context: m.context, detail: shared.detail });
-      const b = shared && !shared.ok
-        ? { ok: false, detail: shared.detail }
-        : await remoteBuild(m.sshHost, m.k3dCluster, m.registry, shared?.tgz);
+      // Build and publish under a separate per-host lock: a failed build can then enter ship's synchronous lock.
+      const lockOpts = deps.lock ?? {};
+      const lockDir = lockOpts.dir ?? join(hostLeaseBaseDir(), 'image-ship');
+      const lockPath = `${imageShipLockPath(m.sshHost, lockDir)}.build`;
+      const retryMs = lockOpts.retryMs ?? 2_000;
+      const staleMs = lockOpts.staleMs ?? IMAGE_SHIP_TIMEOUT_MS + REMOTE_RUN_TIMEOUT_MS + 120_000;
+      const lockStarted = Date.now();
+      let b: { ok: boolean; detail: string };
+      try {
+        mkdirSync(lockDir, { recursive: true });
+        // Probe atomically rather than testing path existence: another process can acquire between that test and us.
+        let lock;
+        let waited = false;
+        try { lock = await acquireLockAsync(lockPath, { staleMs, retryBusyMs: 1, maxTries: 1 }); }
+        catch (error) {
+          if (!(error instanceof Error) || !error.message.startsWith('acquireLockAsync: timed out')) throw error;
+          waited = true;
+          debug.log('pod.pool', 'image-build-lock-wait', { context: m.context, host: m.sshHost, commit: localCommit, localLabel: before, imageRef: refBefore, waitedMs: Date.now() - lockStarted });
+          lock = await acquireLockAsync(lockPath, { staleMs, retryBusyMs: retryMs, maxTries: lockOpts.maxTries ?? Math.ceil(staleMs / retryMs) + 10 });
+        }
+        try {
+          if (waited && hasRegistry) {
+            const refNow = registryImageRef(m, localCommit, run, image);
+            if (refNow && registrySkillsSame(refNow)) {
+              debug.log('pod.pool', 'image-build-skipped-after-wait', { context: m.context, host: m.sshHost, commit: localCommit, localLabel: before, imageRef: refNow, waitedMs: Date.now() - lockStarted });
+              results.set(m.context, { ok: true, action: 'fresh', imageRef: refNow, detail: `${localCommit?.slice(0, 12)} (built by another process while waiting)`, ms: Date.now() - t0 });
+              return;
+            }
+          }
+          // 같은 판이 아닌 멤버가 처음 여기 닿을 때 pack 한다(전부 fresh 면 pack 도 없다).
+          const shared = sharePack ? await (packed ??= sharePack()) : null;
+          if (shared && !shared.ok) debug.log('self-implement.pod', 'pool-image-pack-failed', { context: m.context, detail: shared.detail });
+          b = shared && !shared.ok
+            ? { ok: false, detail: shared.detail }
+            : lock.stillHeld() ? await remoteBuild(m.sshHost, m.k3dCluster, m.registry, shared?.tgz) : { ok: false, detail: 'image build lock lost' };
+          if (!lock.stillHeld()) b = { ok: false, detail: 'image build lock lost' };
+        } finally { lock.release(); }
+      } catch (error) {
+        b = { ok: false, detail: `image build lock: ${error instanceof Error ? error.message : String(error)}` };
+      }
       const after = remoteImageCommit(m, image, run);
       const refAfter = registryImageRef(m, localCommit ?? after, run, image);
       // ⭐ 레지스트리 노드면 «이 판 커밋 태그가 레지스트리에 있나»로 판정한다 — Pod 는 그 커밋 태그를 pull 한다.