elanous 0.2.17 → 0.2.19

package/src/task-orchestrator/surfaces/pod-command-job.ts

@@ -37,6 +37,54 @@ export const POD_LITE_IMAGE = 'elanous-harness-lite:local';
 export const POD_LITE_MEMORY_LIMIT = '2Gi';
 
 /** 요청은 한도를 넘을 수 없다(쿠버네티스가 Job 을 거부한다) — lite 2Gi 처럼 한도가 기본 요청보다 작으면 한도로 맞춘다. */
+/** The command Job container's CPU limit when the caller names none (GATE-SPEED A3①: unchanged default). */
+export const POD_COMMAND_CPU_LIMIT_DEFAULT = '4';
+
+/** Container CPU request/limit override. Omitted fields keep the defaults (request `POD_CHILD_REQUESTS.cpu` · limit 4). */
+export interface PodCpu { request?: string; limit?: string }
+
+/** Kubernetes CPU quantity → millicores (`1` · `0.5` · `500m`); null when it is not a positive quantity we accept. */
+export function podCpuMillis(value: string): number | null {
+  const m = /^(?:(\d+(?:\.\d{1,3})?)|(\d+)m)$/.exec(value.trim());
+  if (!m) return null;
+  const millis = m[1] !== undefined ? Math.round(Number(m[1]) * 1000) : Number(m[2]);
+  return Number.isSafeInteger(millis) && millis > 0 ? millis : null;
+}
+
+/** Resolve a CPU override against the defaults; throws when a value is malformed or the request exceeds the limit (k8s would refuse the Job). */
+export function resolvePodCpu(cpu: PodCpu | undefined): { request: string; limit: string } {
+  const request = cpu?.request?.trim() || POD_CHILD_REQUESTS.cpu;
+  const limit = cpu?.limit?.trim() || POD_COMMAND_CPU_LIMIT_DEFAULT;
+  const r = podCpuMillis(request), l = podCpuMillis(limit);
+  if (r === null) throw new Error(`pod cpu request is not a positive CPU quantity: ${request}`);
+  if (l === null) throw new Error(`pod cpu limit is not a positive CPU quantity: ${limit}`);
+  if (r > l) throw new Error(`pod cpu request ${request} exceeds limit ${limit}`);
+  return { request, limit };
+}
+
+/** Config value → CPU override (release gate `release.loop.gatePodCpu`). A scalar (`2` · `"1500m"`) sets request = limit;
+ *  `{ request, limit }` sets either side. Absent/empty = undefined (request 1 / limit 4). Malformed values throw. */
+export function parsePodCpu(value: unknown, label = 'cpu'): PodCpu | undefined {
+  const text = (v: unknown): string | undefined => {
+    if (v === undefined || v === null || v === '') return undefined;
+    if (typeof v !== 'number' && typeof v !== 'string') throw new Error(`${label} must be a CPU quantity or { request, limit }`);
+    return String(v).trim() || undefined;
+  };
+  let cpu: PodCpu;
+  if (value !== null && typeof value === 'object' && !Array.isArray(value)) {
+    const { request, limit, ...rest } = value as Record<string, unknown>;
+    if (Object.keys(rest).length) throw new Error(`${label} has unknown keys: ${Object.keys(rest).join(', ')}`);
+    const r = text(request), l = text(limit);
+    cpu = { ...(r ? { request: r } : {}), ...(l ? { limit: l } : {}) };
+  } else {
+    const both = text(value);
+    cpu = both ? { request: both, limit: both } : {};
+  }
+  if (!cpu.request && !cpu.limit) return undefined;
+  resolvePodCpu(cpu);
+  return cpu;
+}
+
 export function memoryRequestWithin(limit: string | undefined): string {
   const gi = (value: string | undefined) => { const m = /^(\d+)Gi$/.exec(value ?? ''); return m ? Number(m[1]) : null; };
   const request = gi(POD_CHILD_REQUESTS.memory);
@@ -130,6 +178,8 @@ export interface PodCommandJobInput {
   bunCache?: string;
   /** 컨테이너 메모리 한도(기본 16Gi) — 게이트가 파일 하나 격리 Job 에 올린다. */
   memoryLimit?: string;
+  /** GATE-SPEED A3①: CPU request/limit override (release gate shards only). Omitted = request 1 / limit 4. */
+  cpu?: PodCpu;
   source?: PodSource;
   deadlineSeconds: number;
   runId?: string;
@@ -147,6 +197,7 @@ export function podCommandJobManifest(o: PodCommandJobInput): Record<string, unk
   }
   const grok = o.llm === 'grok';
   const bunCache = o.bunCache ? podBunCacheVolume(o.bunCache) : undefined;
+  const cpu = resolvePodCpu(o.cpu);
   const quotedArgs = o.command.map(bashSingleQuote).join(' ');
   const credLines = [
     ...(grok ? ['mkdir -p ~/.grok && install -m 600 /creds/grok-auth.json ~/.grok/auth.json'] : []),
@@ -185,7 +236,7 @@ export function podCommandJobManifest(o: PodCommandJobInput): Record<string, unk
           initContainers: [{ name: 'isolation-gate', image: o.image, imagePullPolicy: o.imagePullPolicy ?? 'Never', command: ['bash', '-c'], args: [GATE] }],
           containers: [{
             name: 'child', image: o.image, imagePullPolicy: o.imagePullPolicy ?? 'Never',
-            resources: { requests: { ...POD_CHILD_REQUESTS, memory: memoryRequestWithin(o.memoryLimit) }, limits: { memory: o.memoryLimit ?? '16Gi', cpu: '4' } },
+            resources: { requests: { ...POD_CHILD_REQUESTS, cpu: cpu.request, memory: memoryRequestWithin(o.memoryLimit) }, limits: { memory: o.memoryLimit ?? '16Gi', cpu: cpu.limit } },
             command: ['bash', '-c'], args: [script],
             env: [
               ...(o.runId ? [{ name: 'ELANOUS_RUN_ID', value: o.runId }] : []),
@@ -272,6 +323,8 @@ export interface RunPodCommandOptions {
   bunCache?: string;
   /** 컨테이너 메모리 한도(기본 16Gi · lite 면 2Gi). */
   memoryLimit?: string;
+  /** GATE-SPEED A3①: CPU request/limit override — only the release gate passes it. Omitted = request 1 / limit 4. */
+  cpu?: PodCpu;
   /** POD7: 호출부가 «네트워크 스킬만 쓰는 명령»이라고 말할 때만 lite 이미지 ⊕ 2Gi. */
   lite?: boolean;
   source?: PodSource;
@@ -284,6 +337,7 @@ export interface RunPodCommandOptions {
   name?: string;
   pollMs?: number;
   kubectl?: Kubectl;
+  /** A caller-owned scheduler is self-capped: acquires count only its own Jobs, never cluster occupancy (POOLGLOBAL-REG). */
   poolScheduler?: PodPoolScheduler;
   checkPool?: typeof checkPodPool;
   syncImages?: typeof syncPoolImages;
@@ -356,6 +410,8 @@ export async function runPodCommand(options: RunPodCommandOptions): Promise<PodC
   if (hostMirror && !isAbsolute(hostMirror)) throw new Error('pod.hostMirror must be an absolute directory path');
   const bunCache = options.bunCache?.trim() || undefined;
   if (bunCache && !isAbsolute(bunCache)) throw new Error('pod.bunCache must be an absolute directory path');
+  // Fail before any Secret/Job is applied — a bad quantity must not reach the cluster.
+  if (options.cpu) resolvePodCpu(options.cpu);
 
   let member: PodPoolMember | null = null;
   let pool = options.poolScheduler;
@@ -366,12 +422,16 @@ export async function runPodCommand(options: RunPodCommandOptions): Promise<PodC
       const members = parsePodPool(spec);
       const checked = (options.checkPool ?? checkPodPool)(members, baseKubectl as PoolKubectl);
       if (!checked.ok) throw new Error('pod command: 풀의 노드가 하나도 준비되지 않았다');
-      pool = new PodPoolScheduler(checked.ready);
+      // Occupancy reads go through the same kubectl as the readiness check (an injected one included).
+      pool = new PodPoolScheduler(checked.ready, { kubectl: baseKubectl as PoolKubectl });
     }
   }
   if (pool) {
+    // POOLGLOBAL-REG: a caller that hands in its own scheduler (the release gate fans its shards over one) holds its own
+    // concurrency cap — its Jobs count against the spec caps, not the cluster's goal Pods (#24253 → 0.2.15 gate ran 1 shard).
+    const selfCapped = options.poolScheduler !== undefined;
     for (;;) {
-      member = await pool.tryAcquire();
+      member = await pool.tryAcquire(undefined, undefined, { selfCapped });
       if (member) break;
       await sleep(options.pollMs ?? 15_000);
     }
@@ -426,7 +486,7 @@ export async function runPodCommand(options: RunPodCommandOptions): Promise<PodC
       name, namespace, launch, image: imageRef ?? image,
       ...(imageRef ? { imagePullPolicy: 'IfNotPresent' as const } : {}),
       repoUrl, command, skills, ...(llm ? { llm } : {}), ...(options.clone ? { clone: true } : {}),
-      ...(options.source ? { source: options.source } : {}), ...(hostMirror ? { hostMirror } : {}), ...(bunCache ? { bunCache } : {}), ...(memoryLimit ? { memoryLimit } : {}), deadlineSeconds,
+      ...(options.source ? { source: options.source } : {}), ...(hostMirror ? { hostMirror } : {}), ...(bunCache ? { bunCache } : {}), ...(memoryLimit ? { memoryLimit } : {}), ...(options.cpu ? { cpu: options.cpu } : {}), deadlineSeconds,
       ...(options.runId ? { runId: options.runId } : {}),
     });
     kubectl(['-n', namespace, 'delete', 'job', '-l', own, '--ignore-not-found']);

package/src/task-orchestrator/surfaces/pod-failure-reason.ts

@@ -33,6 +33,8 @@ export function extractPodFailureReason(input: {
   containerReason?: string | null;
   jobReason?: string;
   deadlineSeconds?: number;
+  /** POD-NORESULT: the stage and PR from the child's parsed result row (disposition) — used only when no log line gives a readable reason. */
+  result?: { stage?: string | null; prUrl?: string | null; prNumber?: number | null } | null;
 }): string {
   if (input.jobReason === 'DeadlineExceeded') {
     return `DeadlineExceeded: Job 수명 상한 ${input.deadlineSeconds ?? 'unknown'}초 초과`;
@@ -72,6 +74,14 @@ export function extractPodFailureReason(input: {
   const errorLine = [...meaningful].reverse().find((line) => line.terminal || isError(line.text));
   const reason = errorLine?.text.replace(/\/home\/[^/\s]+\//gu, '~/').slice(0, 240);
   if (reason) return reason;
+  // POD-NORESULT: an unconverged child (exit 2 → BackoffLimitExceeded under backoffLimit 0) often leaves a result row
+  // with a stage and PR but no error text — report that instead of «unreadable».
+  const stage = input.result?.stage?.trim();
+  if (stage) {
+    const prNumber = input.result?.prNumber;
+    const pr = Number.isSafeInteger(prNumber) && prNumber! > 0 ? `PR #${prNumber}` : input.result?.prUrl ? `PR ${input.result.prUrl}` : '';
+    return pr ? `수확 가능(${stage}) · ${pr}` : `수렴 못 함(단계 ${stage})`;
+  }
   const why = input.logTailReason?.trim() || (input.logs.trim() ? '로그에 읽을 수 있는 오류 줄 없음' : '자식 로그 비어 있음');
   return `사유 못 읽음: ${why.replace(/\/home\/[^/\s]+\//gu, '~/').slice(0, 240)}`;
 }

package/src/task-orchestrator/surfaces/pod-lease.ts

@@ -7,6 +7,13 @@ import { join } from 'node:path';
 import { dlopen } from 'bun:ffi';
 import { hostLeaseBaseDir, type PendingJobIdentity } from '../../pod-lease/host-lease.js';
 
+/** The harness Pod's container requests (self-implement-pod Job manifest). Lives here so admission reuses the real request. */
+export const POD_CHILD_REQUESTS = { cpu: '1', memory: '6Gi' } as const;
+/** POD-ADMIT-BY-USAGE — observed usage is scaled by this before it reserves memory. */
+const POD_USAGE_HEADROOM = 1.5;
+/** POD-ADMIT-BY-USAGE — a node keeps at least this share of allocatable memory free (real usage and bookkeeping). */
+const POD_NODE_FREE_FLOOR_RATIO = 0.1;
+
 /** Persisted on a Job admitted through the host lease; the file lease remains until its Pod is observed Running or terminal. */
 export const POD_HOST_LEASE_ANNOTATION = 'elanous.dev/host-lease-admitted';
 
@@ -39,6 +46,27 @@ export interface PodLeaseMember {
   /** Node-by-node memory left after all assigned Pod requests and harness Running limits. */
   availableMemoryByNodeBytes: number[] | null;
   reason: string | null;
+  /**
+   * POD-ADMIT-BY-USAGE (only when measured with `measureUsage`): observed memory of Running harness Pods
+   * (`kubectl top pods`). null = metrics unreadable (see usageReason) — every Pod is then reserved at its limit, as before.
+   */
+  harnessUsageBytes?: number[] | null;
+  usageReason?: string | null;
+  /** Pool-wide harness reservation: Σ min(limit, max(request, usage×1.5)), or max(request, limit) when the Pod's usage is unmeasured. */
+  memoryReservedBytes?: number | null;
+  /** Parallel to availableMemoryByNodeBytes: each schedulable node's allocatable memory. */
+  allocatableMemoryByNodeBytes?: number[] | null;
+  /** Parallel to availableMemoryByNodeBytes: each node's real memory use (`kubectl top nodes`), null when unmeasured. */
+  usedMemoryByNodeBytes?: Array<number | null> | null;
+}
+
+/** POD-ADMIT-BY-USAGE — how much memory one new goal is counted at, and why. */
+export interface PodAdmissionByUsage {
+  samples: number;
+  p95Bytes: number | null;
+  admitBytes: number;
+  /** Why admission stayed on the conservative per-goal size; null when usage drove it. */
+  fallback: string | null;
 }
 
 export interface PoolLeaseMeasure {
@@ -61,6 +89,28 @@ export interface PoolLeaseRecommendation {
   pendingJobs?: PendingJobIdentity[];
   /** Absent only for injected legacy status fixtures; real measurements always report a number or null. */
   unleasedRunning?: number | null;
+  /** POD-ADMIT-BY-USAGE — the per-goal size this recommendation used. */
+  admission?: PodAdmissionByUsage;
+}
+
+/** `kubectl top … --no-headers` → name → memory bytes; null when any line cannot be read (fail conservative). */
+function topMemory(stdout: string, column: number): Map<string, number> | null {
+  const out = new Map<string, number>();
+  for (const line of stdout.split('\n')) {
+    if (!line.trim()) continue;
+    const cols = line.trim().split(/\s+/u);
+    const bytes = cols.length > column ? quantity(cols[column]!, 'memory') : null;
+    if (bytes === null || out.has(cols[0]!)) return null;
+    out.set(cols[0]!, bytes);
+  }
+  return out;
+}
+
+/** Nearest-rank p95. */
+function p95(values: readonly number[]): number | null {
+  if (!values.length) return null;
+  const sorted = [...values].sort((a, b) => a - b);
+  return sorted[Math.max(0, Math.ceil(sorted.length * 0.95) - 1)]!;
 }
 
 /** Kubernetes quantity to bytes (memory) or millicores (CPU). */
@@ -238,7 +288,7 @@ function runPoolDnsProbe(context: string, kubectl: PoolKubectl): PoolDnsProbe {
 }
 
 /** Each cluster reads resource reservations; DNS requires a bounded disposable Pod. */
-export function measurePoolLease(members: readonly PodPoolMember[], deps: { kubectl?: PoolKubectl; dns?: (context: string) => PoolDnsProbe; /** Read-only snapshots must not create the disposable DNS Pod or update its cache. */ skipDnsProbe?: boolean } = {}): PoolLeaseMeasure {
+export function measurePoolLease(members: readonly PodPoolMember[], deps: { kubectl?: PoolKubectl; dns?: (context: string) => PoolDnsProbe; /** Read-only snapshots must not create the disposable DNS Pod or update its cache. */ skipDnsProbe?: boolean; /** POD-ADMIT-BY-USAGE — also read `kubectl top` so Running harness Pods reserve by observed usage. */ measureUsage?: boolean } = {}): PoolLeaseMeasure {
   const kubectl = deps.kubectl ?? leaseKubectl;
   return { members: members.map((member) => {
     const base = ['--context', member.context, '--request-timeout=10s'];
@@ -249,6 +299,26 @@ export function measurePoolLease(members: readonly PodPoolMember[], deps: { kube
       // All namespaces are necessary: unrelated workloads also reserve memory on these nodes.
       const pods = kubectl([...base, 'get', 'pods', '--all-namespaces', '-o', 'json']);
       const reasons: string[] = [];
+      // POD-ADMIT-BY-USAGE: metrics are optional — a failed `top` leaves every reservation at today's limit.
+      let podUsage: Map<string, number> | null = null;
+      let nodeUsage: Map<string, number> | null = null;
+      if (deps.measureUsage) {
+        const top = kubectl([...base, '-n', 'elanous-test', 'top', 'pods', '--no-headers']);
+        podUsage = top.status === 0 ? topMemory(top.stdout, 2) : null;
+        result.usageReason = podUsage ? null
+          : `top pods unavailable: ${top.status === 0 ? 'unparseable output' : top.stderr.trim().split('\n').pop() || `rc=${top.status}`}`.slice(0, 160);
+        if (podUsage) {
+          const topNodes = kubectl([...base, 'top', 'nodes', '--no-headers']);
+          nodeUsage = topNodes.status === 0 ? topMemory(topNodes.stdout, 3) : null;
+          // The node real-usage floor needs node metrics: without them stay on today's limit reservations.
+          if (!nodeUsage) {
+            podUsage = null;
+            result.usageReason = `top nodes unavailable: ${topNodes.status === 0 ? 'unparseable output' : topNodes.stderr.trim().split('\n').pop() || `rc=${topNodes.status}`}`.slice(0, 160);
+          }
+        }
+      }
+      const harnessUsage: number[] = [];
+      const nodeAllocatable = new Map<string, number>();
       const parsedItems = (response: ReturnType<PoolKubectl>): Record<string, unknown>[] | null => {
         if (response.status !== 0) return null;
         try { return items(JSON.parse(response.stdout)); } catch { return null; }
@@ -280,12 +350,15 @@ export function measurePoolLease(members: readonly PodPoolMember[], deps: { kube
           }
           nodeNames.add(name);
           const repels = taintList.some((t) => t!.effect === 'NoSchedule' || t!.effect === 'NoExecute');
-          if (spec?.unschedulable !== true && !repels && object(ready[0])?.status === 'True') nodeFree.set(name, mem);
+          if (spec?.unschedulable !== true && !repels && object(ready[0])?.status === 'True') { nodeFree.set(name, mem); nodeAllocatable.set(name, mem); }
           memory += mem; cpu += cores;
         }
         if (valid) { result.allocatableMemoryBytes = memory; result.allocatableCpuMillicores = cpu; }
-        else { nodeFree.clear(); nodeNames.clear(); reasons.push('cluster nodes: invalid name/allocatable memory/cpu/readiness'); }
+        else { nodeFree.clear(); nodeAllocatable.clear(); nodeNames.clear(); reasons.push('cluster nodes: invalid name/allocatable memory/cpu/readiness'); }
       }
+      // POD-ADMIT-BY-USAGE: every schedulable node needs a real-usage reading, or the member stays on limits.
+      const unreadNode = podUsage && nodeUsage ? [...nodeFree.keys()].find((n) => !nodeUsage!.has(n)) : undefined;
+      if (unreadNode !== undefined) { podUsage = null; result.usageReason = `top nodes: no reading for ${unreadNode}`.slice(0, 160); }
       const jobItems = parsedItems(jobs);
       const podItems = parsedItems(pods);
       if (!jobItems) reasons.push(`cluster jobs: ${jobs.status === 0 ? 'invalid response' : jobs.stderr.trim().split('\n').pop() || `rc=${jobs.status}`}`);
@@ -303,7 +376,7 @@ export function measurePoolLease(members: readonly PodPoolMember[], deps: { kube
           if (object(meta.annotations)?.[POD_HOST_LEASE_ANNOTATION] === 'true') leasedJobs.add(meta.name);
         }
         if (!jobsValid) reasons.push('cluster jobs: missing name/labels');
-        let running = 0, pending = 0, unleasedRunning = 0, limits = 0, limitsValid = true, reservationsValid = nodeItems !== null && nodeNames.size === nodeItems.length;
+        let running = 0, pending = 0, unleasedRunning = 0, limits = 0, reserved = 0, limitsValid = true, reservationsValid = nodeItems !== null && nodeNames.size === nodeItems.length;
         const pendingJobs = new Map<string, PendingJobIdentity>();
         let unknownPhase = false;
         const seenPods = new Set<string>();
@@ -324,6 +397,7 @@ export function measurePoolLease(members: readonly PodPoolMember[], deps: { kube
             (labels?.['elanous.substrate'] === 'pod' || jobNames.has(String(labels?.['elanous.job'] ?? '')));
           const spec = object(pod.spec);
           let limit: number | null = null;
+          let reserve: number | null = null;
           if (harness) {
             if (phase === 'Pending') {
               pending++;
@@ -338,7 +412,15 @@ export function measurePoolLease(members: readonly PodPoolMember[], deps: { kube
               if (typeof jobName !== 'string' || !leasedJobs.has(jobName)) unleasedRunning++;
               limit = spec ? runningLimit(spec) : null;
               if (limit === null) limitsValid = false;
-              else limits += limit;
+              else {
+                limits += limit;
+                // Measured usage: max(request, usage×1.5) capped at the limit. Unmeasured: max(request, limit) (today's node reservation).
+                const used = typeof metadata.name === 'string' ? podUsage?.get(metadata.name) : undefined;
+                const request = spec ? requestedMemory(spec) : null;
+                if (used !== undefined) harnessUsage.push(used);
+                reserve = used !== undefined && request !== null ? Math.min(limit, Math.max(request, Math.ceil(used * POD_USAGE_HEADROOM))) : Math.max(request ?? 0, limit);
+                reserved += reserve;
+              }
             }
           }
           // Pending Pods with no assigned node have no node reservation yet. They already occupy a pool slot.
@@ -347,13 +429,22 @@ export function measurePoolLease(members: readonly PodPoolMember[], deps: { kube
               requestedMemory(spec) !== null) continue;
           const requested = spec ? requestedMemory(spec) : null;
           if (typeof nodeName !== 'string' || !nodeNames.has(nodeName) || requested === null) { reservationsValid = false; continue; }
-          if (nodeFree.has(nodeName)) nodeFree.set(nodeName, nodeFree.get(nodeName)! - Math.max(requested, limit ?? 0));
+          if (nodeFree.has(nodeName)) nodeFree.set(nodeName, nodeFree.get(nodeName)! - Math.max(requested, reserve ?? limit ?? 0));
         }
         if (unknownPhase) reasons.push('cluster pods: missing/Unknown phase');
         if (jobsValid && !unknownPhase) { result.running = running; result.pending = pending; result.pendingJobs = [...pendingJobs.values()]; result.unleasedRunning = unleasedRunning; }
-        if (limitsValid && jobsValid && !unknownPhase) result.memoryLimitBytes = limits;
+        if (limitsValid && jobsValid && !unknownPhase) {
+          result.memoryLimitBytes = limits;
+          if (deps.measureUsage) { result.memoryReservedBytes = reserved; result.harnessUsageBytes = podUsage ? harnessUsage : null; }
+        }
         else if (!limitsValid) reasons.push('cluster pods: Running Pod memory limit missing/invalid');
-        if (reservationsValid && limitsValid && jobsValid && !unknownPhase) result.availableMemoryByNodeBytes = [...nodeFree.values()];
+        if (reservationsValid && limitsValid && jobsValid && !unknownPhase) {
+          result.availableMemoryByNodeBytes = [...nodeFree.values()];
+          if (deps.measureUsage) {
+            result.allocatableMemoryByNodeBytes = [...nodeFree.keys()].map((n) => nodeAllocatable.get(n)!);
+            result.usedMemoryByNodeBytes = [...nodeFree.keys()].map((n) => nodeUsage?.get(n) ?? null);
+          }
+        }
         else if (!reservationsValid && !unknownPhase) reasons.push('cluster reservations: node assignment or memory request/limit missing/invalid');
       }
       result.reason = reasons.length ? reasons.join('; ').slice(0, 240) : null;
@@ -366,6 +457,7 @@ export function measurePoolLease(members: readonly PodPoolMember[], deps: { kube
     } catch (error) {
       result.running = null; result.pending = null; result.unleasedRunning = null; result.memoryLimitBytes = null;
       result.allocatableMemoryBytes = null; result.allocatableCpuMillicores = null; result.availableMemoryByNodeBytes = null;
+      if (deps.measureUsage) { result.memoryReservedBytes = null; result.harnessUsageBytes = null; result.allocatableMemoryByNodeBytes = null; result.usedMemoryByNodeBytes = null; }
       result.reason = `cluster: ${error instanceof Error ? error.message : String(error)}`.split('\n')[0]!.slice(0, 240);
     }
     return result;
@@ -377,7 +469,9 @@ export function recommendConcurrency(measure: PoolLeaseMeasure, options: { capac
   const empty: PoolLeaseRecommendation = { recommended: null, limitedBy: null, reason: null, capacitySlots: null, memorySlots: null, liteMemorySlots: null, placeableSlots: null, accountSlots: null, running: null, pending: null, unleasedRunning: null };
   const goalBytes = quantity(options.perGoalMemory ?? '16Gi', 'memory');
   if (goalBytes === null || goalBytes <= 0) return { ...empty, reason: '측정 불가: perGoalMemory' };
+  const admission = admissionByUsage(measure, goalBytes, options.perGoalMemory !== undefined);
   if (!measure.members.length || measure.members.some((m) => m.running === null || m.pending === null || m.unleasedRunning === null || m.allocatableMemoryBytes === null || m.memoryLimitBytes === null || m.allocatableCpuMillicores === null || m.availableMemoryByNodeBytes === null)) {
+    debug.log('pod-lease', 'admit-by-usage', { samples: admission.samples, p95Bytes: admission.p95Bytes, admitBytes: admission.admitBytes, fallback: admission.fallback, recommended: null, limitedBy: null });
     return { ...empty, reason: `측정 불가: cluster${measure.members.map((m) => m.reason ? ` ${m.context}: ${m.reason}` : '').join('')}` };
   }
   if (!Number.isSafeInteger(options.capacity) || options.capacity < 0) return { ...empty, reason: '측정 불가: capacity' };
@@ -392,12 +486,25 @@ export function recommendConcurrency(measure: PoolLeaseMeasure, options: { capac
   const eligibleOccupied = measure.members.reduce((sum, m) => sum + (m.reason === 'dns' ? 0 : Math.min(m.capacity, m.running! + m.pending!)), 0);
   const capacitySlots = Math.max(0, Math.min(options.capacity - eligibleOccupied,
     memberFree.reduce((sum, free) => sum + free, 0)));
-  const memberMemorySlots = measure.members.map((m) => m.availableMemoryByNodeBytes!.reduce(
-    (sum, bytes) => sum + Math.max(0, Math.floor(bytes / goalBytes)), 0));
+  // The goal size may fall back to the conservative size pool-wide; the node floor is separate and follows each
+  // member: once its Running Pods reserve by usage (usage read), its nodes keep 10% free in bookkeeping and real use.
+  const admitBytes = admission.admitBytes;
+  const usageRead = (m: PodLeaseMember) => Array.isArray(m.harnessUsageBytes);
+  const memberMemorySlots = measure.members.map((m) => m.availableMemoryByNodeBytes!.reduce((sum, bytes, i) => {
+    if (!usageRead(m)) return sum + Math.max(0, Math.floor(bytes / admitBytes));
+    const allocatable = m.allocatableMemoryByNodeBytes?.[i];
+    const used = m.usedMemoryByNodeBytes?.[i];
+    // A usage-read member always carries both per-node readings; anything missing admits nothing on that node.
+    if (typeof allocatable !== 'number' || typeof used !== 'number') return sum;
+    const floor = allocatable * POD_NODE_FREE_FLOOR_RATIO;
+    const slots = Math.min(Math.floor((bytes - floor) / admitBytes), Math.floor((allocatable - used - floor) / admitBytes));
+    return sum + Math.max(0, slots);
+  }, 0));
   // Preserve the requested allocatable-minus-harness-limit metric. The node-local
   // placement bound additionally accounts for every namespace's reservations.
-  const memorySlots = measure.members.reduce((sum, m) => sum + (m.reason === 'dns' ? 0 : Math.max(0,
-    Math.floor((m.allocatableMemoryBytes! - m.memoryLimitBytes!) / goalBytes))), 0);
+  const memorySlots = measure.members.reduce((sum, m) => sum + (m.reason === 'dns' ? 0 : Math.max(0, usageRead(m)
+    ? Math.floor((m.allocatableMemoryBytes! * (1 - POD_NODE_FREE_FLOOR_RATIO) - (m.memoryReservedBytes ?? m.memoryLimitBytes!)) / admitBytes)
+    : Math.floor((m.allocatableMemoryBytes! - m.memoryLimitBytes!) / admitBytes))), 0);
   const placeableSlots = measure.members.reduce((sum, m, i) => sum + Math.min(
     Math.max(0, m.capacity - m.running! - m.pending!), memberMemorySlots[i]!,
   ), 0);
@@ -411,5 +518,27 @@ export function recommendConcurrency(measure: PoolLeaseMeasure, options: { capac
   const recommended = Math.max(0, Math.min(capacitySlots, memoryBound) - eligibleUnleasedRunning);
   const limitedBy = memoryBound < capacitySlots ? 'memory' : 'capacity';
   if (unleasedRunning > 0) debug.log('pod-lease', 'unleased', { unleasedRunning, running, recommended });
-  return { recommended, limitedBy, reason: measure.members.some((m) => m.reason === 'dns') ? 'dns' : null, capacitySlots, memorySlots, liteMemorySlots, placeableSlots, accountSlots, running, pending, pendingJobs, unleasedRunning };
+  debug.log('pod-lease', 'admit-by-usage', { samples: admission.samples, p95Bytes: admission.p95Bytes, admitBytes: admission.admitBytes, fallback: admission.fallback, recommended, limitedBy });
+  return { recommended, limitedBy, reason: measure.members.some((m) => m.reason === 'dns') ? 'dns' : null, capacitySlots, memorySlots, liteMemorySlots, placeableSlots, accountSlots, running, pending, pendingJobs, unleasedRunning, admission };
+}
+
+/**
+ * POD-ADMIT-BY-USAGE — one new goal counts at max(the Pod's memory request, p95 of Running harness Pod usage × 1.5),
+ * capped at the per-goal size. No samples (metrics unreadable, nothing running, or an explicit per-goal size such as an
+ * OOM retry tier) keeps the conservative per-goal size.
+ */
+function admissionByUsage(measure: PoolLeaseMeasure, goalBytes: number, explicit: boolean): PodAdmissionByUsage {
+  const live = measure.members.filter((m) => m.reason !== 'dns');
+  const samples = live.flatMap((m) => m.harnessUsageBytes ?? []);
+  const p95Bytes = p95(samples);
+  const requestBytes = quantity(POD_CHILD_REQUESTS.memory, 'memory')!;
+  // Any live member whose usage was not read keeps the whole pool on today's per-goal size (fail conservative).
+  const fallback = explicit ? 'explicit perGoalMemory'
+    : live.every((m) => m.harnessUsageBytes === undefined) ? 'usage not measured'
+      : live.some((m) => m.harnessUsageBytes == null) ? (live.map((m) => m.harnessUsageBytes == null ? `${m.context}: ${m.usageReason ?? 'usage not measured'}` : '').filter(Boolean).join('; '))
+        : p95Bytes === null ? 'no Running harness Pod usage samples'
+          : null;
+  const admitBytes = fallback !== null || p95Bytes === null ? goalBytes
+    : Math.min(goalBytes, Math.max(requestBytes, Math.ceil(p95Bytes * POD_USAGE_HEADROOM)));
+  return { samples: samples.length, p95Bytes, admitBytes, fallback };
 }

package/src/task-orchestrator/surfaces/pod-pool.ts

@@ -193,7 +193,8 @@ export class PodPoolScheduler {
     // One lease directory per pool (contexts, not caps) — every CLI process launching into the same clusters shares it.
     this.hostLease = options.hostLease ?? podPoolHostLease(members);
     const measureStatus = options.status ?? (() => {
-      const measure = measurePoolLease(members, { ...(options.kubectl ? { kubectl: options.kubectl } : {}), ...(options.dns ? { dns: options.dns } : {}) });
+      // POD-ADMIT-BY-USAGE: admission reads observed Pod usage (`kubectl top`); tryAcquire's occupancy read does not.
+      const measure = measurePoolLease(members, { ...(options.kubectl ? { kubectl: options.kubectl } : {}), ...(options.dns ? { dns: options.dns } : {}), measureUsage: true });
       return recommendConcurrency(measure, { capacity: members.reduce((n, m) => n + m.capacity, 0), accounts: 0, perAccount: 0 });
     });
     this.measure = async () => measureStatus();
@@ -297,11 +298,18 @@ export class PodPoolScheduler {
    * POOL-SPREAD (10-06): comparing absolute free slots always picked the big member (node-b 25 vs node-c 4), so the small
    * member idled until the big one was full. Comparing the free share fills members in proportion to their capacity.
    * A member whose occupancy could not be read falls back to inflight-only (never treated as zero occupied).
+   * `selfCapped` (POOLGLOBAL-REG 10-07): a caller with its own concurrency cap (the release gate) counts only its own Jobs
+   * — free = capacity − this scheduler's inflight, summed over members as the pool total — and reads no cluster occupancy.
+   * #24253 made every acquire subtract all Running+Pending harness goal Pods, so with ~24 goal Pods on the gate's 25-slot
+   * member the 0.2.15 gate launched its shards one at a time.
    * Ties keep the configured member order. Returns null when every readable member is full.
    */
-  async tryAcquire(skip?: ReadonlySet<string>, onEmpty?: (reason: string) => void): Promise<PodPoolMember | null> {
+  async tryAcquire(skip?: ReadonlySet<string>, onEmpty?: (reason: string) => void, mode: { selfCapped?: boolean } = {}): Promise<PodPoolMember | null> {
+    const selfCapped = mode.selfCapped === true;
     let reading: Record<string, MemberOccupancy>;
-    try { reading = await this.occupancy(); }
+    // A self-capped caller's own Jobs are its inflight; other workloads' Pods are not its slots (no cluster read).
+    if (selfCapped) reading = Object.fromEntries(this.members.map((m) => [m.context, { occupied: 0 }]));
+    else try { reading = await this.occupancy(); }
     catch (error) {
       const reason = error instanceof Error ? error.message : String(error);
       reading = Object.fromEntries(this.members.map((m) => [m.context, { occupied: null, reason }]));
@@ -329,12 +337,12 @@ export class PodPoolScheduler {
       if (better(cand)) best = cand;
     }
     if (!best) {
-      onEmpty?.(`no free slot · free=${JSON.stringify(free)} · measured=${JSON.stringify(measured)}${skip?.size ? ` · skipped=${[...skip].join(',')}` : ''}`);
+      onEmpty?.(`no free slot · free=${JSON.stringify(free)} · measured=${JSON.stringify(measured)}${selfCapped ? ' · selfCapped' : ''}${skip?.size ? ` · skipped=${[...skip].join(',')}` : ''}`);
       return null;
     }
     this.inflight.set(best.member.context, (this.inflight.get(best.member.context) ?? 0) + 1);
     const share = Object.fromEntries(this.members.map((m) => [m.context, Math.round(Math.max(0, free[m.context] ?? 0) / m.capacity * 100) / 100]));
-    debug.log('pod.pool', 'member-selected', { member: best.member.context, free, measured, share });
+    debug.log('pod.pool', 'member-selected', { member: best.member.context, free, measured, share, ...(selfCapped ? { selfCapped: true } : {}) });
     return best.member;
   }
   release(member: PodPoolMember): void {