elanous 0.2.10 → 0.2.11

package/src/task-orchestrator/surfaces/pod-lease.ts

@@ -1,13 +1,22 @@
 import type { PodPoolMember, PoolKubectl } from './pod-pool.js';
+import { debug } from '../../debug/log.js';
 import { spawnSync } from 'node:child_process';
+import { createHash, randomUUID } from 'node:crypto';
+import { closeSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { dlopen } from 'bun:ffi';
+import { hostLeaseBaseDir } from '../../pod-lease/host-lease.js';
+
+/** Persisted on a Job admitted through the host lease; the file lease is released after apply. */
+export const POD_HOST_LEASE_ANNOTATION = 'elanous.dev/host-lease-admitted';
 
 // `get pods --all-namespaces -o json` on a busy cluster is several MB — the default spawnSync
 // buffer fails with ENOBUFS (10-03 node-b measured). Same proxy stripping as the Pod launch kubectl.
 const LEASE_KUBECTL_MAX_BUFFER = 256 * 1024 * 1024;
-export function leaseKubectl(args: readonly string[]): { status: number | null; stdout: string; stderr: string } {
+export function leaseKubectl(args: readonly string[], input?: string): { status: number | null; stdout: string; stderr: string } {
   const env = { ...process.env };
   for (const k of ['HTTPS_PROXY', 'https_proxy', 'HTTP_PROXY', 'http_proxy', 'ALL_PROXY', 'all_proxy']) delete env[k];
-  const r = spawnSync('kubectl', [...args], { encoding: 'utf8', env, timeout: 120_000, maxBuffer: LEASE_KUBECTL_MAX_BUFFER });
+  const r = spawnSync('kubectl', [...args], { encoding: 'utf8', input, env, timeout: 120_000, maxBuffer: LEASE_KUBECTL_MAX_BUFFER });
   return { status: r.status, stdout: r.stdout ?? '', stderr: (r.stderr ?? '') + (r.error ? String(r.error) : '') };
 }
 
@@ -16,6 +25,8 @@ export interface PodLeaseMember {
   capacity: number;
   running: number | null;
   pending: number | null;
+  /** Running harness Pods whose Jobs predate host lease admission. */
+  unleasedRunning: number | null;
   memoryLimitBytes: number | null;
   /** All-node allocatable memory, before existing reservations. */
   allocatableMemoryBytes: number | null;
@@ -35,10 +46,14 @@ export interface PoolLeaseRecommendation {
   reason: string | null;
   capacitySlots: number | null;
   memorySlots: number | null;
+  /** POD7 — the same memory headroom counted in lite goals (2Gi) · observation only: `recommended` stays on the standard size. */
+  liteMemorySlots?: number | null;
   placeableSlots: number | null;
   accountSlots: number | null;
   running: number | null;
   pending: number | null;
+  /** Absent only for injected legacy status fixtures; real measurements always report a number or null. */
+  unleasedRunning?: number | null;
 }
 
 /** Kubernetes quantity to bytes (memory) or millicores (CPU). */
@@ -110,12 +125,114 @@ function runningLimit(spec: Record<string, unknown>): number | null {
   return total;
 }
 
-/** Read only: each cluster probe is kubectl get. Missing reservations cannot become zero. */
-export function measurePoolLease(members: readonly PodPoolMember[], deps: { kubectl?: PoolKubectl } = {}): PoolLeaseMeasure {
+/** A short-lived Pod proves cluster DNS from inside the same namespace as harness Jobs. Image pull failures are not DNS failures. */
+export type PoolDnsProbe = 'ready' | 'dns' | 'image' | 'unknown';
+const DNS_CACHE_MS = 10 * 60_000;
+const DNS_LOCK_WAIT_MS = 6 * 60_000;
+// flock is released by the kernel on process exit, including a crash before any cache write.
+// The lock file is never unlinked: unlinking it would let waiters lock different inodes.
+const dnsFlock = process.platform === 'linux' || process.platform === 'darwin'
+  ? dlopen(process.platform === 'linux' ? 'libc.so.6' : 'libSystem.B.dylib', {
+    flock: { args: ['int', 'int'], returns: 'int' },
+  }).symbols.flock : null;
+
+/** The cache and lock are shared by CLI processes, keyed by context rather than pool composition. */
+export function probePoolDns(context: string, kubectl: PoolKubectl = leaseKubectl, options: { dir?: string; now?: () => number } = {}): PoolDnsProbe {
+  const dir = options.dir ?? hostLeaseBaseDir();
+  const now = options.now ?? Date.now;
+  const key = createHash('sha256').update(context).digest('hex');
+  const cache = join(dir, `dns-${key}.cache`);
+  const outcome = join(dir, `dns-${key}.outcome`);
+  const lock = join(dir, `dns-${key}.flock`);
+  const readRecord = (path: string): Record<string, unknown> | null => {
+    try { return object(JSON.parse(readFileSync(path, 'utf8'))); }
+    catch { return null; }
+  };
+  const cached = (): boolean => {
+    const record = readRecord(cache);
+    return record?.result === 'ready' && typeof record.at === 'number' &&
+      now() >= record.at && now() - record.at < DNS_CACHE_MS;
+  };
+  if (cached()) return 'ready';
+  const previous = readRecord(outcome);
+  try { mkdirSync(dir, { recursive: true, mode: 0o700 }); } catch { return 'unknown'; }
+  if (!dnsFlock) return 'unknown';
+  let fd: number;
+  try { fd = openSync(lock, 'a', 0o600); } catch { return 'unknown'; }
+  const deadline = Date.now() + DNS_LOCK_WAIT_MS;
+  let held = false;
+  try {
+    const sharedFailure = (): PoolDnsProbe | null => {
+      const record = readRecord(outcome);
+      // Only a caller that started before this generation was written can reuse it.
+      if (record?.generation !== undefined && record.generation !== previous?.generation &&
+          ['dns', 'image', 'unknown'].includes(String(record.result))) return record.result as PoolDnsProbe;
+      return null;
+    };
+    while (!held) {
+      if (cached()) return 'ready';
+      const shared = sharedFailure();
+      if (shared) return shared;
+      if (dnsFlock(fd, 2 | 4) === 0) { held = true; break; } // LOCK_EX | LOCK_NB
+      if (Date.now() >= deadline) return 'unknown';
+      Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 100);
+    }
+    if (cached()) return 'ready';
+    const shared = sharedFailure();
+    if (shared) return shared;
+    const result = runPoolDnsProbe(context, kubectl);
+    const destination = result === 'ready' ? cache : outcome;
+    const temp = `${destination}.${randomUUID()}.tmp`;
+    try {
+      writeFileSync(temp, JSON.stringify({ result, at: now(), generation: randomUUID() }), { mode: 0o600 });
+      renameSync(temp, destination);
+    } catch { try { rmSync(temp, { force: true }); } catch { /* best-effort cleanup */ } }
+    return result;
+  } finally {
+    if (held) dnsFlock(fd, 8); // LOCK_UN
+    closeSync(fd);
+  }
+}
+
+function runPoolDnsProbe(context: string, kubectl: PoolKubectl): PoolDnsProbe {
+  const name = `elanous-dns-${randomUUID().slice(0, 12)}`;
+  const base = ['--context', context, '--request-timeout=10s'];
+  try {
+    const core = kubectl([...base, '-n', 'kube-system', 'get', 'deployment', 'coredns', '-o', 'json']);
+    if (core.status !== 0) return 'unknown';
+    const deployment = object(JSON.parse(core.stdout));
+    const spec = object(deployment?.spec);
+    const status = object(deployment?.status);
+    if (typeof spec?.replicas !== 'number' || spec.replicas < 1 || status?.readyReplicas !== spec.replicas) return 'dns';
+    const manifest = JSON.stringify({ apiVersion: 'v1', kind: 'Pod', metadata: { name, namespace: 'elanous-test', labels: { 'elanous.probe': 'dns' } }, spec: {
+      restartPolicy: 'Never', activeDeadlineSeconds: 40, automountServiceAccountToken: false,
+      containers: [{ name: 'dns', image: 'busybox:1.36', command: ['nslookup', 'kubernetes.default.svc.cluster.local'], resources: { requests: { cpu: '10m', memory: '16Mi' }, limits: { cpu: '100m', memory: '64Mi' } } }],
+    } });
+    if (kubectl([...base, 'create', '-f', '-'], manifest).status !== 0) return 'unknown';
+    const waited = kubectl([...base, '-n', 'elanous-test', 'wait', '--for=jsonpath={.status.phase}=Succeeded', `pod/${name}`, '--timeout=30s']);
+    const pod = kubectl([...base, '-n', 'elanous-test', 'get', `pod/${name}`, '-o', 'json']);
+    if (pod.status !== 0) return 'unknown';
+    const state = object(JSON.parse(pod.stdout));
+    const phase = object(state?.status)?.phase;
+    const statuses = object(state?.status)?.containerStatuses;
+    if (Array.isArray(statuses) && statuses.some((c) =>
+      ['ErrImagePull', 'ImagePullBackOff', 'InvalidImageName'].includes(String(object(object(object(c)?.state)?.waiting)?.reason)))) return 'image';
+    if (phase !== 'Succeeded' && phase !== 'Failed') return 'unknown';
+    if (waited.status !== 0 && phase !== 'Failed') return 'unknown';
+    const log = kubectl([...base, '-n', 'elanous-test', 'logs', `pod/${name}`]);
+    if (log.status !== 0) return 'unknown';
+    if (phase === 'Failed') return /can't resolve|server can't find|connection timed out|no servers could be reached|NXDOMAIN|SERVFAIL/iu.test(log.stdout) ? 'dns' : 'unknown';
+    return /\bName:\s*kubernetes\.default\.svc\.cluster\.local\b/u.test(log.stdout) && /\bAddress(?:es)?:\s*\S+/u.test(log.stdout) ? 'ready' : 'dns';
+  } catch { return 'unknown'; }
+  finally { try { kubectl([...base, '-n', 'elanous-test', 'delete', `pod/${name}`, '--ignore-not-found=true', '--wait=false']); } catch { /* best-effort cleanup */ } }
+}
+
+/** Each cluster reads resource reservations; DNS requires a bounded disposable Pod. */
+export function measurePoolLease(members: readonly PodPoolMember[], deps: { kubectl?: PoolKubectl; dns?: (context: string) => PoolDnsProbe } = {}): PoolLeaseMeasure {
   const kubectl = deps.kubectl ?? leaseKubectl;
   return { members: members.map((member) => {
     const base = ['--context', member.context, '--request-timeout=10s'];
-    const result: PodLeaseMember = { context: member.context, capacity: member.capacity, running: null, pending: null, memoryLimitBytes: null, allocatableMemoryBytes: null, allocatableCpuMillicores: null, availableMemoryByNodeBytes: null, reason: null };
+    const result: PodLeaseMember = { context: member.context, capacity: member.capacity, running: null, pending: null, unleasedRunning: null, memoryLimitBytes: null, allocatableMemoryBytes: null, allocatableCpuMillicores: null, availableMemoryByNodeBytes: null, reason: null };
     try {
       const nodes = kubectl([...base, 'get', 'nodes', '-o', 'json']);
       const jobs = kubectl([...base, '-n', 'elanous-test', 'get', 'jobs', '-l', 'elanous.substrate=pod', '-o', 'json']);
@@ -165,6 +282,7 @@ export function measurePoolLease(members: readonly PodPoolMember[], deps: { kube
       if (!podItems) reasons.push(`cluster pods: ${pods.status === 0 ? 'invalid response' : pods.stderr.trim().split('\n').pop() || `rc=${pods.status}`}`);
       if (podItems && jobItems) {
         const jobNames = new Set<string>();
+        const leasedJobs = new Set<string>();
         let jobsValid = true;
         for (const job of jobItems) {
           const meta = object(job.metadata);
@@ -172,9 +290,10 @@ export function measurePoolLease(members: readonly PodPoolMember[], deps: { kube
           if (object(meta.labels)?.['elanous.substrate'] !== 'pod') continue;
           if (typeof meta.name !== 'string' || !meta.name) { jobsValid = false; continue; }
           jobNames.add(meta.name);
+          if (object(meta.annotations)?.[POD_HOST_LEASE_ANNOTATION] === 'true') leasedJobs.add(meta.name);
         }
         if (!jobsValid) reasons.push('cluster jobs: missing name/labels');
-        let running = 0, pending = 0, limits = 0, limitsValid = true, reservationsValid = nodeItems !== null && nodeNames.size === nodeItems.length;
+        let running = 0, pending = 0, unleasedRunning = 0, limits = 0, limitsValid = true, reservationsValid = nodeItems !== null && nodeNames.size === nodeItems.length;
         let unknownPhase = false;
         const seenPods = new Set<string>();
         for (const pod of podItems) {
@@ -190,7 +309,7 @@ export function measurePoolLease(members: readonly PodPoolMember[], deps: { kube
             if (seenPods.has(identity)) { reservationsValid = false; continue; }
             seenPods.add(identity);
           } else reservationsValid = false;
-          const harness = metadata?.namespace === 'elanous-test' &&
+          const harness = metadata?.namespace === 'elanous-test' && labels?.['elanous.probe'] !== 'dns' &&
             (labels?.['elanous.substrate'] === 'pod' || jobNames.has(String(labels?.['elanous.job'] ?? '')));
           const spec = object(pod.spec);
           let limit: number | null = null;
@@ -198,6 +317,8 @@ export function measurePoolLease(members: readonly PodPoolMember[], deps: { kube
             if (phase === 'Pending') pending++;
             else {
               running++;
+              const jobName = labels?.['elanous.job'] ?? labels?.['job-name'];
+              if (typeof jobName !== 'string' || !leasedJobs.has(jobName)) unleasedRunning++;
               limit = spec ? runningLimit(spec) : null;
               if (limit === null) limitsValid = false;
               else limits += limit;
@@ -212,15 +333,21 @@ export function measurePoolLease(members: readonly PodPoolMember[], deps: { kube
           if (nodeFree.has(nodeName)) nodeFree.set(nodeName, nodeFree.get(nodeName)! - Math.max(requested, limit ?? 0));
         }
         if (unknownPhase) reasons.push('cluster pods: missing/Unknown phase');
-        if (jobsValid && !unknownPhase) { result.running = running; result.pending = pending; }
+        if (jobsValid && !unknownPhase) { result.running = running; result.pending = pending; result.unleasedRunning = unleasedRunning; }
         if (limitsValid && jobsValid && !unknownPhase) result.memoryLimitBytes = limits;
         else if (!limitsValid) reasons.push('cluster pods: Running Pod memory limit missing/invalid');
         if (reservationsValid && limitsValid && jobsValid && !unknownPhase) result.availableMemoryByNodeBytes = [...nodeFree.values()];
         else if (!reservationsValid && !unknownPhase) reasons.push('cluster reservations: node assignment or memory request/limit missing/invalid');
       }
       result.reason = reasons.length ? reasons.join('; ').slice(0, 240) : null;
+      if (nodeItems?.length && jobItems && podItems) {
+        let dnsState: PoolDnsProbe = 'unknown';
+        try { dnsState = (deps.dns ?? ((context) => probePoolDns(context, kubectl)))(member.context); } catch { /* probe could not be measured */ }
+        if (dnsState === 'dns') { result.capacity = 0; result.availableMemoryByNodeBytes = []; result.reason = 'dns'; }
+        else if (dnsState !== 'ready') { result.availableMemoryByNodeBytes = null; result.reason = `측정 불가: dns probe ${dnsState}`; }
+      }
     } catch (error) {
-      result.running = null; result.pending = null; result.memoryLimitBytes = null;
+      result.running = null; result.pending = null; result.unleasedRunning = null; result.memoryLimitBytes = null;
       result.allocatableMemoryBytes = null; result.allocatableCpuMillicores = null; result.availableMemoryByNodeBytes = null;
       result.reason = `cluster: ${error instanceof Error ? error.message : String(error)}`.split('\n')[0]!.slice(0, 240);
     }
@@ -229,32 +356,42 @@ export function measurePoolLease(members: readonly PodPoolMember[], deps: { kube
 }
 
 /** Per-node memory and per-member slots cannot be exchanged across nodes/clusters. */
-export function recommendConcurrency(measure: PoolLeaseMeasure, options: { capacity: number; perGoalMemory?: string; accounts: number; perAccount: number }): PoolLeaseRecommendation {
-  const empty: PoolLeaseRecommendation = { recommended: null, limitedBy: null, reason: null, capacitySlots: null, memorySlots: null, placeableSlots: null, accountSlots: null, running: null, pending: null };
+export function recommendConcurrency(measure: PoolLeaseMeasure, options: { capacity: number; perGoalMemory?: string; liteGoalMemory?: string; accounts: number; perAccount: number }): PoolLeaseRecommendation {
+  const empty: PoolLeaseRecommendation = { recommended: null, limitedBy: null, reason: null, capacitySlots: null, memorySlots: null, liteMemorySlots: null, placeableSlots: null, accountSlots: null, running: null, pending: null, unleasedRunning: null };
   const goalBytes = quantity(options.perGoalMemory ?? '16Gi', 'memory');
   if (goalBytes === null || goalBytes <= 0) return { ...empty, reason: '측정 불가: perGoalMemory' };
-  if (!measure.members.length || measure.members.some((m) => m.running === null || m.pending === null || m.allocatableMemoryBytes === null || m.memoryLimitBytes === null || m.allocatableCpuMillicores === null || m.availableMemoryByNodeBytes === null)) {
+  if (!measure.members.length || measure.members.some((m) => m.running === null || m.pending === null || m.unleasedRunning === null || m.allocatableMemoryBytes === null || m.memoryLimitBytes === null || m.allocatableCpuMillicores === null || m.availableMemoryByNodeBytes === null)) {
     return { ...empty, reason: `측정 불가: cluster${measure.members.map((m) => m.reason ? ` ${m.context}: ${m.reason}` : '').join('')}` };
   }
   if (!Number.isSafeInteger(options.capacity) || options.capacity < 0) return { ...empty, reason: '측정 불가: capacity' };
   const running = measure.members.reduce((sum, m) => sum + m.running!, 0);
   const pending = measure.members.reduce((sum, m) => sum + m.pending!, 0);
-  const capacitySlots = Math.max(0, Math.min(options.capacity - running - pending,
-    measure.members.reduce((sum, m) => sum + Math.max(0, m.capacity - m.running! - m.pending!), 0)));
+  const unleasedRunning = measure.members.reduce((sum, m) => sum + m.unleasedRunning!, 0);
+  // Pool-wide free = sum of per-member free slots: a member over its cap counts as full, never negative,
+  // so it cannot zero another member's free slot (and its unleased Pods only consume its own free slots).
+  const memberFree = measure.members.map((m) => Math.max(0, m.capacity - m.running! - m.pending!));
+  const eligibleUnleasedRunning = measure.members.reduce((sum, m, i) => sum + (m.reason === 'dns' ? 0 : Math.min(m.unleasedRunning!, memberFree[i]!)), 0);
+  const eligibleOccupied = measure.members.reduce((sum, m) => sum + (m.reason === 'dns' ? 0 : Math.min(m.capacity, m.running! + m.pending!)), 0);
+  const capacitySlots = Math.max(0, Math.min(options.capacity - eligibleOccupied,
+    memberFree.reduce((sum, free) => sum + free, 0)));
   const memberMemorySlots = measure.members.map((m) => m.availableMemoryByNodeBytes!.reduce(
     (sum, bytes) => sum + Math.max(0, Math.floor(bytes / goalBytes)), 0));
   // Preserve the requested allocatable-minus-harness-limit metric. The node-local
   // placement bound additionally accounts for every namespace's reservations.
-  const memorySlots = measure.members.reduce((sum, m) => sum + Math.max(0,
-    Math.floor((m.allocatableMemoryBytes! - m.memoryLimitBytes!) / goalBytes)), 0);
+  const memorySlots = measure.members.reduce((sum, m) => sum + (m.reason === 'dns' ? 0 : Math.max(0,
+    Math.floor((m.allocatableMemoryBytes! - m.memoryLimitBytes!) / goalBytes))), 0);
   const placeableSlots = measure.members.reduce((sum, m, i) => sum + Math.min(
     Math.max(0, m.capacity - m.running! - m.pending!), memberMemorySlots[i]!,
   ), 0);
   // Account count is not a concurrency limit (same IP · own accounts, 10-03 decision) — observed only, never a bound.
   const accountSlots = [options.accounts, options.perAccount].every((n) => Number.isSafeInteger(n) && n >= 0)
     ? Math.max(0, options.accounts * options.perAccount - running) : null;
+  const liteBytes = quantity(options.liteGoalMemory ?? '2Gi', 'memory');
+  const liteMemorySlots = liteBytes === null || liteBytes <= 0 ? null : measure.members.reduce((sum, m) => sum + (m.reason === 'dns' ? 0 : Math.max(0,
+    Math.floor((m.allocatableMemoryBytes! - m.memoryLimitBytes!) / liteBytes))), 0);
   const memoryBound = Math.min(memorySlots, placeableSlots);
-  const recommended = Math.min(capacitySlots, memoryBound);
-  const limitedBy = recommended === memoryBound && memoryBound < capacitySlots ? 'memory' : 'capacity';
-  return { recommended, limitedBy, reason: null, capacitySlots, memorySlots, placeableSlots, accountSlots, running, pending };
+  const recommended = Math.max(0, Math.min(capacitySlots, memoryBound) - eligibleUnleasedRunning);
+  const limitedBy = memoryBound < capacitySlots ? 'memory' : 'capacity';
+  if (unleasedRunning > 0) debug.log('pod-lease', 'unleased', { unleasedRunning, running, recommended });
+  return { recommended, limitedBy, reason: measure.members.some((m) => m.reason === 'dns') ? 'dns' : null, capacitySlots, memorySlots, liteMemorySlots, placeableSlots, accountSlots, running, pending, unleasedRunning };
 }

package/src/task-orchestrator/surfaces/pod-pool.ts

@@ -1,21 +1,26 @@
 /**
  * ☸️ Pod 풀 — 여러 k8s 클러스터(k3d)를 «우선순위 ⊕ 노드별 상한»으로 묶는다 (대표 2026-09-25).
  *
- * 스펙: `ELANOUS_POD_POOL` 또는 `--pod-pool` = `컨텍스트[@ssh호스트][:상한]` 을 쉼표로, «앞이 우선».
- *   예) `pool-node-b@node-b:8`
+ * 스펙: `ELANOUS_POD_POOL` 또는 `--pod-pool` = `컨텍스트[@ssh호스트][:상한][#레지스트리주소]` 을 쉼표로, «앞이 우선».
+ *   예) `pool-node-b@node-b:8#k3d-elanous-registry:5051,pool-node-c@node-c:2#k3d-elanous-registry:5052`
  *   ⭐ 운영 권장(대표 2026-09-25): M3 Ultra(node-b) «전용» — Pod 한 개가 6Gi 를 넘겨(OOM) 한도를 12Gi 로 올렸고, node-c(OrbStack VM 16GB)는 그 한 개도 빠듯해 기본 풀에서 뺀다(필요할 때만 명시) ·
  *      이 맥(mbp)은 브라우저·편집 프로그램으로 메모리가 모자라기 쉬워 «기본 풀에서 뺀다»(필요할 때만 `k3d-elanous-h1:1` 을 명시).
  *   - 컨텍스트  kubectl 컨텍스트 이름(이 맥의 kubeconfig)
  *   - @ssh호스트 원격 클러스터면 그 기계 — 이미지 판 대조·반입(`docker load` ⊕ `k3d image import`)에 쓴다
  *   - 상한      이 노드에 동시에 둘 Job 수(기본 2)
  * Job 마다 앞 순위부터 «진행 중 < 상한»인 노드를 고른다. 다 차면 자리가 날 때까지 기다린다.
- * ⛔ 진행 중 수는 «이 프로세스가 띄운 것»만 센다 — 다른 호스트가 같은 클러스터에 띄운 Job 은 안 센다(알려진 한계).
+ * 노드 선택의 진행 중 수는 로컬 프로세스 기준이다. 발사 허가는 별도의 lease 측정으로
+ * 다른 호스트의 Running/Pending 및 메모리 예약까지 고려한다.
  * 풀을 안 주면 종전처럼 «현재 컨텍스트» 하나다(동작 불변).
  */
 import { spawn, spawnSync } from 'node:child_process';
 import { existsSync } from 'node:fs';
 import { join, resolve } from 'node:path';
 import { debug } from '../../debug/log.js';
+import { PodLeaseAdmission, type PodLeasePredecessor, type PodLeaseRelease } from '../../pod-lease/admission.js';
+import { predecessorState, type PredecessorState } from '../../pod-lease/dependency-state.js';
+import { HostPoolLease } from '../../pod-lease/host-lease.js';
+import { measurePoolLease, recommendConcurrency, type PoolDnsProbe, type PoolLeaseRecommendation } from './pod-lease.js';
 
 export interface PodPoolMember {
   readonly context: string;
@@ -26,11 +31,17 @@ export interface PodPoolMember {
   readonly k3dCluster: string;
   /** 노드 로컬 레지스트리의 이미지(커밋 태그) — 있으면 Pod 가 이것을 pull 한다(델타 · 대표 2026-09-26). 없으면 반입된 로컬 이미지. */
   readonly imageRef?: string;
+  /** 클러스터 안 레지스트리 주소(host:port). 호스트 push/조회는 같은 포트의 localhost. */
+  readonly registry?: string;
 }
 
 /** 노드 로컬 레지스트리(`scripts/fleet/node-setup.sh` 5b) — 클러스터 안에서 부르는 이름. */
 export const NODE_REGISTRY = `k3d-elanous-registry:${process.env.ELANOUS_FLEET_REGISTRY_PORT ?? '5050'}`;
 
+function memberRegistry(member: PodPoolMember): string {
+  return member.registry ?? NODE_REGISTRY;
+}
+
 /** 노드 쪽 빌드가 실패했을 때, 같은 판을 굽는 다른 발사의 레지스트리 태그를 기다리는 한도. */
 const CONCURRENT_BUILD_WAIT_MS = 5 * 60_000;
 const CONCURRENT_BUILD_POLL_MS = 15_000;
@@ -41,15 +52,15 @@ const sleep = (ms: number): Promise<void> => new Promise((done) => setTimeout(do
  * 노드 쪽 빌드가 부딪혀 실패한 뒤, 같은 판을 굽는 다른 발사가 레지스트리에 이 커밋 태그를 올리는지 본다.
  * 생기면 그 imageRef. 한도까지 없으면 null. 간격은 시험이 짧게 주입한다.
  */
-async function waitForConcurrentBuild(member: PodPoolMember, commit: string | null, opts: { intervalMs?: number; maxMs?: number; now?: () => number; sleep?: (ms: number) => Promise<void>; run?: RemoteRun } = {}): Promise<{ imageRef: string | null; waitedMs: number }> {
+async function waitForConcurrentBuild(member: PodPoolMember, commit: string | null, opts: { intervalMs?: number; maxMs?: number; now?: () => number; sleep?: (ms: number) => Promise<void>; run?: RemoteRun; image?: string } = {}): Promise<{ imageRef: string | null; waitedMs: number }> {
   const intervalMs = opts.intervalMs ?? CONCURRENT_BUILD_POLL_MS;
   const maxMs = opts.maxMs ?? CONCURRENT_BUILD_WAIT_MS;
-  if (maxMs <= 0) return { imageRef: registryImageRef(member, commit, opts.run), waitedMs: 0 };
+  if (maxMs <= 0) return { imageRef: registryImageRef(member, commit, opts.run, opts.image), waitedMs: 0 };
   const now = opts.now ?? Date.now;
   const pause = opts.sleep ?? sleep;
   const started = now();
   for (;;) {
-    const imageRef = registryImageRef(member, commit, opts.run);
+    const imageRef = registryImageRef(member, commit, opts.run, opts.image);
     const waitedMs = now() - started;
     if (imageRef || waitedMs >= maxMs) return { imageRef, waitedMs };
     const remaining = maxMs - waitedMs;
@@ -70,15 +81,17 @@ function defaultLocalImageInspect(image: string): { status: number | null; stdou
 }
 
 /** 이 커밋의 이미지가 노드 레지스트리에 있나 — 있으면 클러스터 안 이미지 주소. */
-export function registryImageRef(member: PodPoolMember, commit: string | null, run: RemoteRun = defaultRemoteRun): string | null {
+export function registryImageRef(member: PodPoolMember, commit: string | null, run: RemoteRun = defaultRemoteRun, image = 'elanous-harness:local'): string | null {
   if (!member.sshHost || !commit) return null;
+  const imageName = image === 'elanous-harness-lite:local' ? 'elanous-harness-lite' : 'elanous-harness';
   const tag = commit.slice(0, 12);
-  const port = process.env.ELANOUS_FLEET_REGISTRY_PORT ?? '5050';
-  const r = run(member.sshHost, `curl -fsS http://localhost:${port}/v2/elanous-harness/tags/list 2>/dev/null`);
+  const registry = memberRegistry(member);
+  const port = registry.slice(registry.lastIndexOf(':') + 1);
+  const r = run(member.sshHost, `curl -fsS --max-time 5 http://localhost:${port}/v2/${imageName}/tags/list 2>/dev/null`);
   if (r.status !== 0) return null;
   try {
     const tags = (JSON.parse(r.stdout) as { tags?: string[] }).tags ?? [];
-    return tags.includes(tag) ? `${NODE_REGISTRY}/elanous-harness:${tag}` : null;
+    return tags.includes(tag) ? `${registry}/${imageName}:${tag}` : null;
   } catch { return null; }
 }
 
@@ -90,17 +103,18 @@ export function remoteImageSkillsDigest(member: PodPoolMember, image: string, ru
   return v && v !== '<no value>' ? v : null;
 }
 
-const MEMBER = /^([A-Za-z0-9][A-Za-z0-9._-]*)(?:@([A-Za-z0-9][A-Za-z0-9._-]*))?(?::(\d+))?$/u;
+const MEMBER = /^([A-Za-z0-9][A-Za-z0-9._-]*)(?:@([A-Za-z0-9][A-Za-z0-9._-]*))?(?::(\d+))?(?:#(k3d-[A-Za-z0-9][A-Za-z0-9._-]*):([1-9]\d{0,4}))?$/u;
 
 export function parsePodPool(spec: string): PodPoolMember[] {
   const members = spec.split(',').map((s) => s.trim()).filter(Boolean).map((part) => {
     const m = MEMBER.exec(part);
-    if (!m) throw new Error(`--pod-pool: 못 읽는 노드 「${part}」 — 형식 컨텍스트[@ssh호스트][:상한]`);
+    if (!m) throw new Error(`--pod-pool: 못 읽는 노드 「${part}」 — 형식 컨텍스트[@ssh호스트][:상한][#k3d-레지스트리:포트]`);
     const capacity = m[3] === undefined ? 2 : Number(m[3]);
     if (!Number.isInteger(capacity) || capacity < 1) throw new Error(`--pod-pool: 상한은 1 이상 — 「${part}」`);
+    if (m[5] && Number(m[5]) > 65535) throw new Error(`--pod-pool: 레지스트리 포트는 65535 이하 — 「${part}」`);
     const context = m[1]!;
     const k3dCluster = context.startsWith('k3d-') ? context.slice(4) : 'elanous-pool';
-    return { context, capacity, k3dCluster, ...(m[2] ? { sshHost: m[2] } : {}) };
+    return { context, capacity, k3dCluster, ...(m[2] ? { sshHost: m[2] } : {}), ...(m[4] ? { registry: `${m[4]}:${m[5]}` } : {}) };
   });
   if (members.length === 0) throw new Error('--pod-pool: 노드가 없다');
   if (new Set(members.map((m) => m.context)).size !== members.length) throw new Error('--pod-pool: 컨텍스트가 겹친다');
@@ -110,6 +124,7 @@ export function parsePodPool(spec: string): PodPoolMember[] {
 /** 스펙 해석 순서: 명시 인자 → `ELANOUS_POD_POOL` → 없음(null = 현재 컨텍스트 하나). */
 export function resolvePodPoolSpec(explicit: string | undefined, env: NodeJS.ProcessEnv = process.env, configPool: () => string | undefined = defaultConfigPool): string | null {
   // ⭐ «--substrate pod 만 써도 분배»(대표 2026-09-26) — 인자 → 환경 → 설정 `pod.pool` 순. 셋 다 없으면 현재 컨텍스트 하나.
+  //   harness 입구는 `harness.podPool` 을 먼저 보는 자기 해석(하니스 실행 칸 기본값 해석)으로 풀을 «인자»로 넘긴다 — 범용 경로 계약은 그대로.
   const v = explicit?.trim() || env.ELANOUS_POD_POOL?.trim() || configPool()?.trim();
   return v ? v : null;
 }
@@ -118,14 +133,97 @@ function defaultConfigPool(): string | undefined {
   try {
     // eslint-disable-next-line @typescript-eslint/no-require-imports
     const { getUserConfig } = require('../../user-config.js') as typeof import('../../user-config.js');
-    return getUserConfig().pod?.pool;
+    return genericConfigPodPool(getUserConfig());
   } catch { return undefined; }
 }
 
+/** The generic Pod path reads only `pod.pool` (harness launches resolve `harness.podPool` themselves and pass it explicitly). */
+export function genericConfigPodPool(config: { pod?: { pool?: string }; harness?: { podPool?: string } }): string | undefined {
+  return config.pod?.pool;
+}
+
 /** 노드 자리 배분 — 우선순위 순서로 첫 빈 자리. 순수(시각·대기는 호출자가). */
 export class PodPoolScheduler {
   private readonly inflight = new Map<string, number>();
-  constructor(readonly members: readonly PodPoolMember[]) {}
+  private readonly admission: PodLeaseAdmission;
+  private readonly outstanding = new Set<PodLeaseRelease>();
+  private lastFree = 0;
+  private budget = 0;
+  private initialized = false;
+  private readonly measure: () => Promise<PoolLeaseRecommendation>;
+  private lastRaw: PoolLeaseRecommendation | null = null;
+  private readonly hostLease: HostPoolLease;
+  private readonly pollMs: number;
+  constructor(readonly members: readonly PodPoolMember[], options: { status?: () => PoolLeaseRecommendation | Promise<PoolLeaseRecommendation>; kubectl?: PoolKubectl; dns?: (context: string) => PoolDnsProbe; pollMs?: number; hostLease?: HostPoolLease; dependencyMerged?: (after: PodLeasePredecessor, signal?: AbortSignal) => PredecessorState | boolean | Promise<PredecessorState | boolean> } = {}) {
+    this.pollMs = options.pollMs ?? 15_000;
+    // One lease directory per pool (contexts, not caps) — every CLI process launching into the same clusters shares it.
+    this.hostLease = options.hostLease ?? new HostPoolLease([...members.map((m) => `${m.context}@${m.sshHost ?? ''}`)].sort().join(','));
+    const measureStatus = options.status ?? (() => {
+      const measure = measurePoolLease(members, { ...(options.kubectl ? { kubectl: options.kubectl } : {}), ...(options.dns ? { dns: options.dns } : {}) });
+      return recommendConcurrency(measure, { capacity: members.reduce((n, m) => n + m.capacity, 0), accounts: 0, perAccount: 0 });
+    });
+    this.measure = async () => measureStatus();
+    this.admission = new PodLeaseAdmission({ status: async () => {
+      const measured = await measureStatus();
+      this.lastRaw = measured;
+      // Launches admitted by other processes on this host but not yet visible as Jobs hold free slots too.
+      const others = measured.recommended === null ? 0 : this.hostLease.othersPending();
+      const decision = measured.recommended === null ? measured : { ...measured, recommended: Math.max(0, measured.recommended - others) };
+      // An unknown reading cannot erase permits already granted: on recovery,
+      // the first healthy recommendation must still account for their slots.
+      if (decision.recommended === null) return decision;
+      // A new launch consumes a measured additional slot. Compare consecutive
+      // measurements so repeated stale readings cannot grant that slot again.
+      this.budget = Math.max(0, this.budget + decision.recommended - (this.initialized ? this.lastFree : 0));
+      this.initialized = true;
+      this.lastFree = decision.recommended;
+      return { ...decision, recommended: this.admission.snapshot().active + this.budget };
+    }, ...(options.pollMs ? { pollMs: options.pollMs } : {}), dependencyMerged: options.dependencyMerged ?? ((after, signal) => predecessorState(after, {}, signal)) });
+  }
+  /** FIFO gate within this process, then a host lease so independent CLI processes cannot over-admit the same pool. */
+  async acquireAdmission(signal?: AbortSignal, after?: PodLeasePredecessor): Promise<PodLeaseRelease & { applied: () => void }> {
+    const local = await this.acquireLocalAdmission(signal, after);
+    try {
+      for (;;) {
+        if (signal?.aborted) throw new Error('pod lease admission aborted');
+        const free = this.lastRaw?.recommended ?? null;
+        // Same check as the local gate, but atomic across processes: other unapplied leases must stay under the free slots.
+        const lease = free !== null && free > 0 ? this.hostLease.tryReserve((others) => others < free) : null;
+        if (lease) {
+          let hostReleased = false;
+          const releaseHost = () => { if (!hostReleased) { hostReleased = true; lease(); } };
+          // Once the Job is applied the cluster measurement counts it — the host lease would double count.
+          return Object.assign(() => { releaseHost(); local(); }, { applied: releaseHost });
+        }
+        await new Promise<void>((resolve, reject) => {
+          const timer = setTimeout(() => { signal?.removeEventListener('abort', onAbort); resolve(); }, this.pollMs);
+          const onAbort = () => { clearTimeout(timer); reject(new Error('pod lease admission aborted')); };
+          signal?.addEventListener('abort', onAbort, { once: true });
+        });
+        try { this.lastRaw = await this.measure(); } catch { this.lastRaw = null; }
+      }
+    } catch (error) {
+      local();
+      throw error;
+    }
+  }
+  private acquireLocalAdmission(signal?: AbortSignal, after?: PodLeasePredecessor): Promise<PodLeaseRelease> {
+    return this.admission.acquire(signal, after).then((release) => {
+      this.budget = Math.max(0, this.budget - 1);
+      this.outstanding.add(release);
+      return () => {
+        if (!this.outstanding.delete(release)) return;
+        // A release only restores a locally reserved slot while the latest
+        // healthy measurement still reports free capacity. If the pool fell
+        // from N=2 to N=0, a release cannot manufacture a new permit.
+        if (this.lastFree > 0) this.budget = Math.min(this.lastFree, this.budget + 1);
+        release();
+      };
+    });
+  }
+  admissionSnapshot(): ReturnType<PodLeaseAdmission['snapshot']> {
+    return this.admission.snapshot();
+  }
   /** 자리가 있으면 그 노드를 잡고 돌려준다. 없으면 null. */
   tryAcquire(): PodPoolMember | null {
     for (const m of this.members) {
@@ -192,7 +290,7 @@ export function syncPoolImage(member: PodPoolMember, image: string, localCommit:
     : { ok: false, action: 'failed', detail: `보낸 뒤 판이 ${after ?? '없음'} — 기대 ${localCommit ?? '?'}` };
 }
 
-export type PoolKubectl = (args: readonly string[]) => { status: number | null; stdout: string; stderr: string };
+export type PoolKubectl = (args: readonly string[], input?: string) => { status: number | null; stdout: string; stderr: string };
 
 /**
  * 발사 전 점검 — 노드마다 «컨텍스트가 닿나 · elanous-test 네임스페이스가 있나».
@@ -220,11 +318,14 @@ export function podImageBuildScript(cwd: string = process.cwd()): string | null
   return null;
 }
 
-type RemoteBuild = (host: string, cluster: string) => Promise<{ ok: boolean; detail: string }>;
+type RemoteBuild = (host: string, cluster: string, registry?: string) => Promise<{ ok: boolean; detail: string }>;
 
-function defaultRemoteBuild(script: string): RemoteBuild {
-  return (host, cluster) => new Promise((done) => {
-    const child = spawn('bash', [script], { env: { ...process.env, ELANOUS_BUILD_REMOTE: host, ELANOUS_L2_CLUSTER: cluster }, stdio: ['ignore', 'pipe', 'pipe'] });
+function defaultRemoteBuild(script: string, image: string): RemoteBuild {
+  return (host, cluster, registry) => new Promise((done) => {
+    const env: NodeJS.ProcessEnv = { ...process.env, ELANOUS_BUILD_REMOTE: host, ELANOUS_L2_CLUSTER: cluster, ELANOUS_BUILD_IMAGE: image === 'elanous-harness-lite:local' ? 'lite' : 'full' };
+    delete env.ELANOUS_BUILD_REGISTRY;
+    if (registry) env.ELANOUS_BUILD_REGISTRY = registry;
+    const child = spawn('bash', [script], { env, stdio: ['ignore', 'pipe', 'pipe'] });
     let out = '';
     child.stdout.on('data', (d) => { out += d; }); child.stderr.on('data', (d) => { out += d; });
     const timer = setTimeout(() => child.kill('SIGKILL'), 1_800_000);
@@ -241,7 +342,7 @@ function defaultRemoteBuild(script: string): RemoteBuild {
 export async function syncPoolImages(members: readonly PodPoolMember[], image: string, localCommit: string | null, deps: { run?: RemoteRun; remoteBuild?: RemoteBuild; buildScript?: string | null; ship?: typeof syncPoolImage; localSkillsDigest?: string | null; waitIntervalMs?: number; waitMaxMs?: number; now?: () => number; sleep?: (ms: number) => Promise<void>; inspect?: LocalImageInspect; transfer?: ImageShip } = {}): Promise<Map<string, PoolImageSync>> {
   const run = deps.run ?? defaultRemoteRun;
   const script = deps.buildScript === undefined ? podImageBuildScript() : deps.buildScript;
-  const remoteBuild = deps.remoteBuild ?? (script ? defaultRemoteBuild(script) : null);
+  const remoteBuild = deps.remoteBuild ?? (script ? defaultRemoteBuild(script, image) : null);
   const ship = deps.ship ?? ((m: PodPoolMember, img: string, commit: string | null, remote: RemoteRun) => syncPoolImage(m, img, commit, { run: remote, inspect: deps.inspect, transfer: deps.transfer }));
   const results = new Map<string, PoolImageSync>();
   await Promise.all(members.map(async (m) => {
@@ -251,16 +352,16 @@ export async function syncPoolImages(members: readonly PodPoolMember[], image: s
     // ⭐ «같은 판» = 커밋 ⊕ 스킬 해시 ⊕ (레지스트리 노드면) 레지스트리에 그 커밋 태그가 있다 — 스킬만 바뀌어도 다시 굽는다(대표 2026-09-26 «스킬 셋트 싱크»).
     const skillsNow = deps.localSkillsDigest;
     const skillsSame = skillsNow == null || remoteImageSkillsDigest(m, image, run) === skillsNow;
-    const refBefore = registryImageRef(m, localCommit, run);
-    const hasRegistry = run(m.sshHost, 'docker inspect k3d-elanous-registry >/dev/null 2>&1').status === 0;
+    const refBefore = registryImageRef(m, localCommit, run, image);
+    const hasRegistry = run(m.sshHost, `docker inspect ${memberRegistry(m).split(':')[0]} >/dev/null 2>&1`).status === 0;
     if (localCommit && before === localCommit && skillsSame && (!hasRegistry || refBefore)) {
       results.set(m.context, { ok: true, action: 'fresh', detail: before.slice(0, 12), ms: Date.now() - t0, ...(refBefore ? { imageRef: refBefore } : {}) });
       return;
     }
     if (remoteBuild) {
-      const b = await remoteBuild(m.sshHost, m.k3dCluster);
+      const b = await remoteBuild(m.sshHost, m.k3dCluster, m.registry);
       const after = remoteImageCommit(m, image, run);
-      const refAfter = registryImageRef(m, localCommit ?? after, run);
+      const refAfter = registryImageRef(m, localCommit ?? after, run, image);
       // ⭐ 레지스트리 노드면 «이 판 커밋 태그가 레지스트리에 있나»로 판정한다 — Pod 는 그 커밋 태그를 pull 한다.
       //   🩸 2026-09-26(🅣 실측): 두 발사가 같은 노드를 동시에 구우면 가변 태그 `:local` 을 서로 덮어, 빌드는 성공했는데
       //   `:local` 의 판이 남의 커밋이라 «못 맞췄다»로 노드를 뺐다(런이 시작도 못 함). 커밋 태그는 덮이지 않는다.
@@ -271,7 +372,7 @@ export async function syncPoolImages(members: readonly PodPoolMember[], image: s
       if (b.ok && after && (!localCommit || after === localCommit)) { results.set(m.context, { ok: true, action: 'built', ...(refAfter ? { imageRef: refAfter } : {}), detail: `${before?.slice(0, 12) ?? '없음'} → ${after.slice(0, 12)} (노드 쪽 빌드)`, ms: Date.now() - t0 }); return; }
       // 빌드가 성공하면 위에서 즉시 끝난다. 실패했을 때만, 통째 전송 전에 같은 판을 굽는 다른 발사의 태그를 기다린다.
       if (!b.ok) {
-        const waited = await waitForConcurrentBuild(m, localCommit, { intervalMs: deps.waitIntervalMs, maxMs: deps.waitMaxMs, now: deps.now, sleep: deps.sleep, run });
+        const waited = await waitForConcurrentBuild(m, localCommit, { intervalMs: deps.waitIntervalMs, maxMs: deps.waitMaxMs, now: deps.now, sleep: deps.sleep, run, image });
         debug.log('self-implement.pod', 'pool-image-wait-concurrent', { context: m.context, commit: localCommit, waitedMs: waited.waitedMs, outcome: waited.imageRef ? 'built' : 'timeout' });
         if (waited.imageRef) {
           results.set(m.context, { ok: true, action: 'built', imageRef: waited.imageRef, detail: '동시 빌드 결과 사용', ms: Date.now() - t0 });