elanous 0.2.32 → 0.2.34

package/src/task-orchestrator/surfaces/pod-command-job.ts

@@ -592,7 +592,8 @@ export async function runPodCommand(options: RunPodCommandOptions): Promise<PodC
     if (!context && !options.kubectl) throw new Error('pod command: kubectl context 를 확인할 수 없다');
     let grokAuth: string | undefined;
     if (llm === 'grok') {
-      const creds = (options.grokCredentials ?? (() => hostGrokCredentials({ env, apiKeyOptIn: false })))();
+      // POD-GROK-VALIDITY-FIT — this Job's deadline is not fitted to the token, so the copy must outlive the whole Job.
+      const creds = (options.grokCredentials ?? (() => hostGrokCredentials({ env, apiKeyOptIn: false, minValidityMs: deadlineSeconds * 1000 })))();
       if (!creds.grokAuth) throw new Error('grok: 구독 자격 없음 · API 키 opt-in 꺼짐 또는 키 없음');
       let parsed: unknown;
       try { parsed = JSON.parse(creds.grokAuth); } catch { throw new Error('grok: Pod 구독 자격 JSON 이 아니다'); }

package/src/task-orchestrator/surfaces/pod-lifecycle-stages.ts

@@ -14,7 +14,7 @@ import { debug } from '../../debug/log.js';
 
 export const POD_LIFECYCLE_STAGES = ['create', 'schedule', 'clone', 'mirror', 'deps', 'test', 'delete'] as const;
 /** si-task Pod 은 시험이 아니라 `self implement`(또는 harness ask/say) 본체를 돈다 — 그 칸은 `test` 가 아니라 `run` 이다. */
-export const SI_TASK_POD_LIFECYCLE_STAGES = ['create', 'schedule', 'clone', 'mirror', 'deps', 'run', 'delete'] as const;
+export const SI_TASK_POD_LIFECYCLE_STAGES = ['create', 'schedule', 'start', 'clone', 'mirror', 'deps', 'run', 'delete'] as const;
 export type PodLifecycleStageName = typeof POD_LIFECYCLE_STAGES[number] | typeof SI_TASK_POD_LIFECYCLE_STAGES[number];
 export type PodLifecycleClock = 'host' | 'k8s' | 'pod' | 'mixed';
 export type PodLifecycleKind = 'gate' | 'si-task';
@@ -29,6 +29,16 @@ export interface PodLifecycleEvent {
   stages: PodLifecycleStage[];
   outcome: string;
   missing: PodLifecycleStageName[];
+  /** si-task: the Pod whose named logs supplied the stage markers. */
+  podUid?: string | null;
+  podName?: string | null;
+  /** Original Kubernetes timestamps; null means the API did not provide one. */
+  k8s?: { created: string | null; scheduled: string | null; started: string | null; clock: 'k8s' };
+  /** A skipped install is a state with a Pod-clock time, never a fabricated duration. */
+  deps?: { status: 'skipped'; at: number; clock: 'pod'; reason: string } | { status: 'missing'; reason: string };
+  missingReasons?: Partial<Record<PodLifecycleStageName, string>>;
+  podSelectionReason?: string | null;
+  watch?: { start: number; end: number; clock: 'host' };
   [extra: string]: unknown;
 }
 
@@ -118,9 +128,40 @@ export function siTaskPodStageShell(stage: SiTaskMarkedStage, edge: 'start' | 'e
   return `printf 'ELANOUS_POD_STAGE ${stage} ${edge} %s\\n' "$(date +%s)" || true`;
 }
 const SI_TASK_STAGE_LINE = /^ELANOUS_POD_STAGE (clone|mirror|run) (start|end) (\d{1,12})\r?$/gm;
+const SI_TASK_DEPS_SKIPPED_LINE = /^ELANOUS_POD_DEPS skipped (\d{1,12}) (image-prepared-no-install)\r?$/m;
+
+export function parseSiTaskDepsSkipped(logs: string | undefined): { at: number; reason: string } | undefined {
+  const match = logs?.match(SI_TASK_DEPS_SKIPPED_LINE);
+  return match ? { at: Number(match[1]), reason: match[2]! } : undefined;
+}
+
+/** The API timestamps belong to one Pod UID; retries must never combine two Pods' clocks. */
+export function siTaskPodK8sTiming(json: string | undefined, childRunLabel: string, podJob: string, jobUid: string | undefined): { podName?: string; podUid?: string; podCreated?: string; podScheduled?: string; containerStarted?: string; podSelectionReason?: string } {
+  try {
+    const items = (JSON.parse(json ?? '') as { items?: unknown[] }).items;
+    if (!Array.isArray(items)) return {};
+    const matching = items.filter((item): item is Record<string, any> => !!item && typeof item === 'object')
+      .filter((item) => item.metadata?.labels?.['elanous.child-run'] === childRunLabel);
+    // `logs job/<name>` may choose either Pod. No identity can safely be joined when
+    // more than one Pod belongs to this child run, even if one was created later.
+    if (matching.length > 1) return { podSelectionReason: 'ambiguous-pod-attempts' };
+    const pod = matching[0];
+    if (!pod || typeof pod.metadata?.name !== 'string' || typeof pod.metadata?.uid !== 'string') return {};
+    const jobLabels = [pod.metadata.labels?.['batch.kubernetes.io/job-name'], pod.metadata.labels?.['job-name']].filter((value): value is string => typeof value === 'string');
+    const owned = pod.metadata.ownerReferences?.some((owner: any) => owner?.kind === 'Job' && owner?.name === podJob && owner?.uid === jobUid);
+    if (!jobUid || jobLabels.length === 0 || jobLabels.some((value) => value !== podJob) || !owned) return { podSelectionReason: 'pod-job-unverified' };
+    const scheduled = pod.status?.conditions?.find((condition: any) => condition?.type === 'PodScheduled' && condition.status === 'True')?.lastTransitionTime;
+    const child = pod.status?.containerStatuses?.find((status: any) => status?.name === 'child');
+    const started = child?.state?.running?.startedAt ?? child?.state?.terminated?.startedAt;
+    return { podName: pod.metadata.name, podUid: pod.metadata.uid,
+      ...(isoMs(pod.metadata.creationTimestamp) !== undefined ? { podCreated: pod.metadata.creationTimestamp } : {}),
+      ...(isoMs(scheduled) !== undefined ? { podScheduled: scheduled } : {}),
+      ...(isoMs(started) !== undefined ? { containerStarted: started } : {}) };
+  } catch { return {}; }
+}
 
 /**
- * Job 로그에서 단계 표지를 읽는다 → `{ 'clone-start': 초, 'clone-end': 초, … }` (gate 의 podStages 와 같은 꼴).
+ * 선택된 Pod 로그에서 단계 표지를 읽는다 → `{ 'clone-start': 초, 'clone-end': 초, … }` (gate 의 podStages 와 같은 꼴).
  * 같은 표지가 두 번 나오면 «첫» 값 — 셸 표지는 모두 자식 출력(`cat /tmp/si.out`)보다 앞에 찍히므로, 자식이 같은 모양의 줄을
  * 내도 셸 값을 덮지 못한다(리뷰 should-fix). 모르는 단계·꼴은 버린다(지어내지 않는다).
  */
@@ -135,12 +176,12 @@ export function parseSiTaskPodStages(logs: string | undefined): Record<string, n
 }
 
 /**
- * si-task Pod — Pod 셸이 찍은 단계 표지(`parseSiTaskPodStages`)로 clone · mirror · deps · run 을 채운다(pod 시계).
+ * si-task Pod — Kubernetes 원시 시각으로 create · schedule · start, Pod 셸 표지로 clone · mirror · run 을 채운다.
  * - clone  = 소스 받기(`podSourceScript`) 시작 → 그 스크립트 전체가 끝난 뒤(`cd repo` 포함 · 어느 줄에서든 exit 하면 끝이 없다 → missing)
  * - mirror = 호스트 미러 `clone --shared` 시도 → 성공(미러를 건너뛰면 끝이 없다 → missing)
- * - deps   = si-task Job 셸엔 의존성 설치 단계가 없다 → 표지를 읽지 않는다(`deps` 표지 줄이 있어도 늘 missing)
+ * - deps   = 현재 이미지는 의존성을 미리 포함한다. 자식이 install 생략 시각과 이유를 별도 상태로 기록한다.
  * - run    = 본체(`self implement` · `harness ask`/`say`) 띄움 → `wait` 가 돌아옴(SIGTERM 이면 끝이 없다 → missing)
- * - create · schedule · delete = 호스트가 잴 값이 없다 → missing.
+ * - delete = 이 경로에서 관측하지 않는다 → missing.
  * 호스트가 지켜본 구간은 단계가 아닌 `watch`(host 시계)로 따로 싣는다 — 단계로 지어내지 않는다.
  */
 export function siTaskPodLifecycleEvent(input: {
@@ -149,16 +190,53 @@ export function siTaskPodLifecycleEvent(input: {
   host?: string | null;
   watchStart?: number;
   watchEnd?: number;
+  createStart?: number;
   podStages?: Readonly<Record<string, number>>;
+  podUid?: string;
+  podName?: string;
+  podSelectionReason?: string;
+  podCreated?: string;
+  podScheduled?: string;
+  containerStarted?: string;
+  depsSkipped?: { at: number; reason: string };
   outcome: string;
 }): PodLifecycleEvent {
   const pod = input.podStages ?? {};
-  const measured: Measured = {};
+  const created = isoMs(input.podCreated);
+  const scheduled = isoMs(input.podScheduled);
+  const started = isoMs(input.containerStarted);
+  const measured: Measured = {
+    create: { start: input.createStart, end: created, clock: 'mixed' },
+    schedule: { start: created, end: scheduled, clock: 'k8s' },
+    start: { start: scheduled, end: started, clock: 'k8s' },
+  };
   for (const name of SI_TASK_MARKED_STAGES) measured[name] = { start: podSecondsMs(pod[`${name}-start`]), end: podSecondsMs(pod[`${name}-end`]), clock: 'pod' };
   const { stages, missing } = splitPodLifecycleStages(measured, SI_TASK_POD_LIFECYCLE_STAGES);
-  const watched = input.watchStart !== undefined && input.watchEnd !== undefined && Number.isFinite(input.watchStart) && Number.isFinite(input.watchEnd) && input.watchEnd >= input.watchStart;
+  const watchStart = input.watchStart;
+  const watchEnd = input.watchEnd;
+  let watch: PodLifecycleEvent['watch'];
+  if (watchStart !== undefined && watchEnd !== undefined && Number.isFinite(watchStart) && Number.isFinite(watchEnd) && watchEnd >= watchStart) {
+    watch = { start: watchStart, end: watchEnd, clock: 'host' };
+  }
+  const missingReason = (stage: PodLifecycleStageName): string => {
+    if (input.podSelectionReason && stage !== 'delete') return input.podSelectionReason;
+    if (stage === 'deps') return input.depsSkipped ? `skipped:${input.depsSkipped.reason}` : 'child-deps-marker-unavailable';
+    if (stage === 'delete') return 'not-observed';
+    if (stage === 'create' && input.createStart === undefined) return 'host-apply-start-unavailable';
+    if (stage === 'create' || stage === 'schedule' || stage === 'start') {
+      const timestamps = stage === 'create' ? [created] : stage === 'schedule' ? [created, scheduled] : [scheduled, started];
+      return timestamps.some((time) => time === undefined) ? 'k8s-timestamp-unavailable' : 'clock-order-invalid';
+    }
+    return 'stage-marker-unavailable';
+  };
   return { runId: input.runId ?? null, podJob: input.podJob, host: input.host ?? null, kind: 'si-task', stages, outcome: input.outcome, missing,
-    ...(watched ? { watch: { start: input.watchStart, end: input.watchEnd, clock: 'host' as const } } : {}) };
+    podUid: input.podUid ?? null, podName: input.podName ?? null,
+    k8s: { created: input.podCreated ?? null, scheduled: input.podScheduled ?? null, started: input.containerStarted ?? null, clock: 'k8s' },
+    deps: input.depsSkipped ? { status: 'skipped', at: input.depsSkipped.at * 1000, clock: 'pod', reason: input.depsSkipped.reason }
+      : { status: 'missing', reason: input.podSelectionReason ?? 'child-deps-marker-unavailable' },
+    podSelectionReason: input.podSelectionReason ?? null,
+    missingReasons: Object.fromEntries(missing.map((stage) => [stage, missingReason(stage)])),
+    ...(watch ? { watch } : {}) };
 }
 
 export type PodLifecycleLog = (category: string, event: string, data: Record<string, unknown>) => void;

package/src/task-orchestrator/surfaces/pod-pool.ts

@@ -294,6 +294,8 @@ export class PodPoolScheduler {
    * GATE-RESERVE-AUTO CPU bound, which exists to protect the gate from harness Pods, not from itself.
    */
   private readonly gateAdmission: PodLeaseAdmission;
+  /** A launch with an explicit memory limit must be measured with that limit, not the pool's 16Gi default. */
+  private readonly memoryAdmissions = new Map<string, { admission: PodLeaseAdmission; reading: () => PoolLeaseRecommendation | null; measure: () => Promise<PoolLeaseRecommendation>; take: () => void; release: () => void }>();
   private readonly outstanding = new Set<PodLeaseRelease>();
   private lastFree = 0;
   private budget = 0;
@@ -301,6 +303,8 @@ export class PodPoolScheduler {
   private seenActive = 0;
   private initialized = false;
   private readonly measure: () => Promise<PoolLeaseRecommendation>;
+  private readonly measureForMemory: (memoryLimit: string) => Promise<PoolLeaseRecommendation>;
+  private readonly dependencyMerged: (after: PodLeasePredecessor, signal?: AbortSignal) => PredecessorState | boolean | Promise<PredecessorState | boolean>;
   private lastRaw: PoolLeaseRecommendation | null = null;
   /** GATE-ADMIT-EXEMPT — the same reading before the gate CPU cap (memory/limitedBy unchanged); gate callers read it. */
   private lastUncapped: PoolLeaseRecommendation | null = null;
@@ -308,18 +312,18 @@ export class PodPoolScheduler {
   private readonly pollMs: number;
   private readonly occupancy: () => Record<string, MemberOccupancy> | Promise<Record<string, MemberOccupancy>>;
   private readonly memberMemory: () => Array<{ context: string; allocatableMemoryBytes: number | null }> | Promise<Array<{ context: string; allocatableMemoryBytes: number | null }>>;
-  constructor(readonly members: readonly PodPoolMember[], options: { status?: () => PoolLeaseRecommendation | Promise<PoolLeaseRecommendation>; kubectl?: PoolKubectl; dns?: (context: string) => PoolDnsProbe; pollMs?: number; hostLease?: HostPoolLease; memberMemory?: () => Array<{ context: string; allocatableMemoryBytes: number | null }> | Promise<Array<{ context: string; allocatableMemoryBytes: number | null }>>; occupancy?: () => Record<string, MemberOccupancy> | Promise<Record<string, MemberOccupancy>>; dependencyMerged?: (after: PodLeasePredecessor, signal?: AbortSignal) => PredecessorState | boolean | Promise<PredecessorState | boolean> } = {}) {
+  constructor(readonly members: readonly PodPoolMember[], options: { status?: (memoryLimit?: string) => PoolLeaseRecommendation | Promise<PoolLeaseRecommendation>; kubectl?: PoolKubectl; dns?: (context: string) => PoolDnsProbe; pollMs?: number; hostLease?: HostPoolLease; memberMemory?: () => Array<{ context: string; allocatableMemoryBytes: number | null }> | Promise<Array<{ context: string; allocatableMemoryBytes: number | null }>>; occupancy?: () => Record<string, MemberOccupancy> | Promise<Record<string, MemberOccupancy>>; dependencyMerged?: (after: PodLeasePredecessor, signal?: AbortSignal) => PredecessorState | boolean | Promise<PredecessorState | boolean> } = {}) {
     this.pollMs = options.pollMs ?? 15_000;
     // One lease directory per pool (contexts, not caps) — every CLI process launching into the same clusters shares it.
     this.hostLease = options.hostLease ?? podPoolHostLease(members);
     // GATE-RESERVE-AUTO: the gate reservation is returned beside the recommendation, never folded into it — the
     // delta budget below must not see a gate start/end as a free-slot change (that made the result order-dependent).
-    const measureStatus: () => Promise<{ status: PoolLeaseRecommendation; gateSlots: number | null | undefined }> = options.status
-      ? async () => ({ status: await options.status!(), gateSlots: undefined })
-      : async () => {
+    const measureStatus: (memoryLimit?: string) => Promise<{ status: PoolLeaseRecommendation; gateSlots: number | null | undefined }> = options.status
+      ? async (memoryLimit) => ({ status: await options.status!(memoryLimit), gateSlots: undefined })
+      : async (memoryLimit) => {
         // POD-ADMIT-BY-USAGE: admission reads observed Pod usage (`kubectl top`); tryAcquire's occupancy read does not.
         const measure = measurePoolLease(members, { ...(options.kubectl ? { kubectl: options.kubectl } : {}), ...(options.dns ? { dns: options.dns } : {}), measureUsage: true });
-        return { status: recommendConcurrency(measure, { capacity: members.reduce((n, m) => n + m.capacity, 0), accounts: 0, perAccount: 0 }), gateSlots: harnessCpuSlotsBesideGate(measure) };
+        return { status: recommendConcurrency(measure, { capacity: members.reduce((n, m) => n + m.capacity, 0), ...(memoryLimit ? { perGoalMemory: memoryLimit } : {}), accounts: 0, perAccount: 0 }), gateSlots: harnessCpuSlotsBesideGate(measure) };
       };
     /** Cluster-level additional slots for the cross-process host lease: the measured free slots, capped by the gate's CPU. */
     const capped = ({ status, gateSlots }: { status: PoolLeaseRecommendation; gateSlots: number | null | undefined }): PoolLeaseRecommendation =>
@@ -327,6 +331,7 @@ export class PodPoolScheduler {
         : gateSlots === null ? { ...status, recommended: null, reason: '측정 불가: gate CPU' }
           : { ...status, recommended: Math.min(status.recommended, gateSlots) };
     this.measure = async () => { const reading = await measureStatus(); this.lastUncapped = reading.status; return capped(reading); };
+    this.measureForMemory = async (memoryLimit) => capped(await measureStatus(memoryLimit));
     // An injected status (tests, callers with their own measurement) without memberMemory never shells out to kubectl here.
     this.memberMemory = options.memberMemory ?? (options.status ? (() => []) : () => measurePoolLease(members, { ...(options.kubectl ? { kubectl: options.kubectl } : {}), ...(options.dns ? { dns: options.dns } : {}) }).members
       .map((m: PodLeaseMember) => ({ context: m.context, allocatableMemoryBytes: m.allocatableMemoryBytes })));
@@ -379,28 +384,74 @@ export class PodPoolScheduler {
         ? { ...decision, recommended, reason: '측정 불가: gate CPU' }
         : { ...decision, recommended };
     };
-    const dependencyMerged = options.dependencyMerged ?? ((after: PodLeasePredecessor, signal?: AbortSignal) => predecessorState(after, {}, signal));
-    this.admission = new PodLeaseAdmission({ status: () => decide(false), ...(options.pollMs ? { pollMs: options.pollMs } : {}), dependencyMerged });
-    this.gateAdmission = new PodLeaseAdmission({ status: () => decide(true), ...(options.pollMs ? { pollMs: options.pollMs } : {}), dependencyMerged });
+    this.dependencyMerged = options.dependencyMerged ?? ((after: PodLeasePredecessor, signal?: AbortSignal) => predecessorState(after, {}, signal));
+    this.admission = new PodLeaseAdmission({ status: () => decide(false), ...(options.pollMs ? { pollMs: options.pollMs } : {}), dependencyMerged: this.dependencyMerged });
+    this.gateAdmission = new PodLeaseAdmission({ status: () => decide(true), ...(options.pollMs ? { pollMs: options.pollMs } : {}), dependencyMerged: this.dependencyMerged });
+  }
+  private memoryAdmission(memoryLimit: string): { admission: PodLeaseAdmission; reading: () => PoolLeaseRecommendation | null; measure: () => Promise<PoolLeaseRecommendation>; take: () => void; release: () => void } {
+    const existing = this.memoryAdmissions.get(memoryLimit);
+    if (existing) return existing;
+    let last: PoolLeaseRecommendation | null = null;
+    let admission: PodLeaseAdmission;
+    let lastFree = 0;
+    let budget = 0;
+    let initialized = false;
+    const measure = async () => { last = await this.measureForMemory(memoryLimit); return last; };
+    admission = new PodLeaseAdmission({
+      status: async () => {
+        const raw = await measure();
+        if (raw.recommended === null) return raw;
+        const ownStart = this.hostLease.selfStart();
+        const pendingJobs = raw.pendingJobs ?? [];
+        const transferId = process.env.ELANOUS_POD_RESERVED_LEASE;
+        const transferIsOther = !!transferId && this.hostLease.reservationIsOther(transferId);
+        const others = this.hostLease.live().filter((r) =>
+          (r.pid !== process.pid || r.startedAt !== ownStart) && !leaseHasPendingPod(r, pendingJobs)).length;
+        const available = Math.max(0, raw.recommended - others + (transferIsOther ? 1 : 0));
+        // A repeated stale reading does not mint another permit before an admitted Job is observed.
+        // Other tiers and processes share the host lease, which enforces the pool cap before Job apply.
+        budget = Math.max(0, budget + available - (initialized ? lastFree : 0));
+        initialized = true;
+        lastFree = available;
+        return { ...raw, recommended: admission.snapshot().active + budget };
+      },
+      pollMs: this.pollMs,
+      dependencyMerged: this.dependencyMerged,
+    });
+    const entry = { admission, reading: () => last, measure,
+      take: () => { budget = Math.max(0, budget - 1); },
+      release: () => { if (lastFree > 0) budget = Math.min(lastFree, budget + 1); },
+    };
+    this.memoryAdmissions.set(memoryLimit, entry);
+    return entry;
   }
   /** FIFO gate within this process, then a host lease so independent CLI processes cannot over-admit the same pool. */
-  async acquireAdmission(signal?: AbortSignal, after?: PodLeasePredecessor, opts: { gate?: boolean } = {}): Promise<PodLeaseRelease & { applied: (job: string, context: string, namespace: string) => void; observed: () => void }> {
+  async acquireAdmission(signal?: AbortSignal, after?: PodLeasePredecessor, opts: { gate?: boolean; memoryLimit?: string } = {}): Promise<PodLeaseRelease & { applied: (job: string, context: string, namespace: string) => void; observed: () => void }> {
     const gate = opts.gate === true;
-    const local = await this.acquireLocalAdmission(signal, after, gate);
+    const lane = !gate && opts.memoryLimit && opts.memoryLimit !== '16Gi' ? this.memoryAdmission(opts.memoryLimit) : null;
+    const local = lane ? await lane.admission.acquire(signal, after).then((release) => {
+      lane.take();
+      let done = false;
+      return () => { if (done) return; done = true; lane.release(); release(); };
+    }) : await this.acquireLocalAdmission(signal, after, gate);
     try {
       for (;;) {
         if (signal?.aborted) throw new Error('pod lease admission aborted');
         // GATE-ADMIT-EXEMPT: a gate caller is not bounded by its own gate's CPU reservation (memory still bounds it).
-        const free = (gate ? this.lastUncapped : this.lastRaw)?.recommended ?? null;
+        const reading = lane?.reading() ?? (gate ? this.lastUncapped : this.lastRaw);
+        const free = reading?.recommended ?? null;
         // Same check as the local gate, but atomic across processes: other authoring and pre-Running leases hold slots.
         const reservedId = process.env.ELANOUS_POD_RESERVED_LEASE;
         const transferred = reservedId ? this.hostLease.claim(reservedId) : null;
         if (reservedId && !transferred) throw new Error('pod lease: authoring reservation could not be transferred');
         if (transferred) delete process.env.ELANOUS_POD_RESERVED_LEASE;
-        const pendingJobs = this.lastRaw?.pendingJobs ?? [];
+        const pendingJobs = reading?.pendingJobs ?? [];
         const lease = transferred ?? (free !== null && free > 0 ? this.hostLease.tryReserve((others) =>
           others - this.hostLease.live().filter((r) => leaseHasPendingPod(r, pendingJobs) &&
-            (r.pid !== process.pid || r.startedAt !== this.hostLease.selfStart())).length < free) : (gate && free === null ? this.hostLease.tryReserve(() => true) : null));
+            ((lane || this.memoryAdmissions.size > 0) || r.pid !== process.pid || r.startedAt !== this.hostLease.selfStart())).length < free,
+          // The legacy queue accounts for its own permits locally. Mixed memory tiers have separate queues,
+          // so their host leases must count together under the same pool cap.
+          !!lane || this.memoryAdmissions.size > 0) : (gate && free === null ? this.hostLease.tryReserve(() => true) : null));
         if (lease) {
           let hostReleased = false;
           const releaseHost = () => { if (!hostReleased) { hostReleased = true; lease(); } };
@@ -421,7 +472,8 @@ export class PodPoolScheduler {
           const onAbort = () => { clearTimeout(timer); reject(new Error('pod lease admission aborted')); };
           signal?.addEventListener('abort', onAbort, { once: true });
         });
-        try { this.lastRaw = await this.measure(); } catch { this.lastRaw = null; this.lastUncapped = null; }
+        try { if (lane) await lane.measure(); else this.lastRaw = await this.measure(); }
+        catch { if (!lane) { this.lastRaw = null; this.lastUncapped = null; } }
       }
     } catch (error) {
       local();
@@ -454,14 +506,18 @@ export class PodPoolScheduler {
       ? [{ context: r.context, allocatableMemoryBytes: r.allocatableMemoryBytes }] : []);
   }
   /** Why admission is still waiting: the pool-wide lease measurement (null before any reading). Slot waits get their reason from tryAcquire's onEmpty. */
-  waitReason(): string | null {
-    const r = this.lastRaw;
+  waitReason(memoryLimit?: string): string | null {
+    const r = memoryLimit && memoryLimit !== '16Gi' ? this.memoryAdmissions.get(memoryLimit)?.reading() : this.lastRaw;
     if (!r) return null;
     return `recommended=${r.recommended ?? 'unknown'} limitedBy=${r.limitedBy ?? '-'} memorySlots=${r.memorySlots ?? '?'}${r.reason ? ` reason=${r.reason}` : ''}`;
   }
   /** The harness queue by default; `{ gate: true }` reads the gate-caller queue (GATE-ADMIT-EXEMPT). */
   admissionSnapshot(opts: { gate?: boolean } = {}): ReturnType<PodLeaseAdmission['snapshot']> {
-    return (opts.gate ? this.gateAdmission : this.admission).snapshot();
+    if (opts.gate) return this.gateAdmission.snapshot();
+    const snapshots = [this.admission.snapshot(), ...[...this.memoryAdmissions.values()].map((lane) => lane.admission.snapshot())];
+    return { active: snapshots.reduce((n, item) => n + item.active, 0), queued: snapshots.reduce((n, item) => n + item.queued, 0),
+      recommended: Math.min(this.members.reduce((n, item) => n + item.capacity, 0), snapshots.reduce((n, item) => n + item.recommended, 0)),
+      waiting: snapshots.flatMap((item) => item.waiting) };
   }
   /**
    * Pick the member with the largest free share: (capacity − measured cluster occupancy − this process's inflight) / capacity.

package/src/task-orchestrator/surfaces/pod-source-receive.ts

@@ -56,13 +56,48 @@ function assertRemoteBase(repoUrl: string, base: string): void {
 /** Job 스크립트의 원천 단계. 성공 갈래는 `ELANOUS_POD_SOURCE <kind> <HEAD>` 로 끝난다.
  *  `stageMarkers` — si-task Job 만: 호스트 미러 `clone --shared` 시도의 시작·성공을 `ELANOUS_POD_STAGE mirror start|end` 로 찍는다
  *  (건너뛰거나 실패하면 끝 표지가 없다 → 호스트가 missing 으로 읽는다). 기본은 꺼짐 — gate·pod-command Job 의 바이트는 그대로다. */
-export function podSourceScript(source: PodSource, repoUrl: string, mirrorPath = '/host-mirror', options: { stageMarkers?: boolean } = {}): string {
+export function podSourceScript(source: PodSource, repoUrl: string, mirrorPath = '/host-mirror', options: { stageMarkers?: boolean; imageSourceDir?: string; imageCommitFile?: string; imageGitDir?: string } = {}): string {
   const mark = (edge: 'start' | 'end'): string[] => options.stageMarkers ? [siTaskPodStageShell('mirror', edge)] : [];
   const quotedUrl = `'${repoUrl.replace(/'/g, `'\\''`)}'`;
   const quotedMirror = `'${mirrorPath.replace(/'/g, `'\\''`)}'`;
+  const quotedImageDir = `'${(options.imageSourceDir ?? '/opt/elanous-source').replace(/'/g, `'\\''`)}'`;
+  const quotedCommitFile = `'${(options.imageCommitFile ?? '/opt/elanous-source-commit').replace(/'/g, `'\\''`)}'`;
+  const quotedImageGitDir = `'${(options.imageGitDir ?? '/opt/elanous-pin.git').replace(/'/g, `'\\''`)}'`;
   const clone = `git clone -q --depth 50 ${quotedUrl} repo`;
   const start = [
     'if [ -e repo ]; then exit 5; fi',
+    // POD-MIRROR-IO: the image's pinned tree is a shared read-only lower layer.
+    // Git metadata is tiny and private to this Pod; alternates point first at
+    // the image's shallow pin objects, then optionally at the host mirror.
+    'source_via=',
+    `if [ -f ${quotedCommitFile} ] && [ -d ${quotedImageDir} ] && [ -d ${quotedImageGitDir} ]; then`,
+    // Older pinned images can contain macOS AppleDouble pack sidecars. Git may
+    // interpret `._pack-*.idx` as an index and fail even when the real pack is valid.
+    `  if [ -d ${quotedImageGitDir}/objects/pack ]; then find ${quotedImageGitDir}/objects/pack -maxdepth 1 -type f -name '._*' -delete || { printf 'ELANOUS_POD_SOURCE_IMAGE_FAIL appledouble-cleanup\n'; exit 5; }; fi`,
+    `  image_commit=$(cat ${quotedCommitFile})`,
+    `  if [[ "$image_commit" =~ ^[0-9a-f]{40}$ ]] && git -C ${quotedImageGitDir} cat-file -e "$image_commit^{commit}" 2>/dev/null && [ -f ${quotedImageGitDir}/shallow ] && grep -Fxq "$image_commit" ${quotedImageGitDir}/shallow; then`,
+    `    ln -s ${quotedImageDir} repo || exit 5`,
+    `    image_objects=$(git -C ${quotedImageGitDir} rev-parse --path-format=absolute --git-path objects)`,
+    `    if git -C repo init -q -b main && mkdir -p repo/.git/objects/info && printf '%s\\n' "$image_objects" > repo/.git/objects/info/alternates; then`,
+    `      if [ -d ${quotedMirror} ]; then mirror_objects=$(git -C ${quotedMirror} rev-parse --path-format=absolute --git-path objects 2>/dev/null || :); [ -z "$mirror_objects" ] || printf '%s\\n' "$mirror_objects" >> repo/.git/objects/info/alternates; fi`,
+    `      if [ -f ${quotedImageGitDir}/shallow ]; then cp ${quotedImageGitDir}/shallow repo/.git/shallow || exit 5; fi`,
+    `      if git -C repo update-ref refs/heads/main "$image_commit" && git -C repo read-tree "$image_commit" && git -C repo update-index --refresh && git -C repo diff-files --quiet && extra=$(git -C repo ls-files --others) && [ -z "$extra" ]; then`,
+    `        git -C repo remote add origin ${quotedUrl} || exit 5`,
+    '      source_via=image',
+    '      printf \'ELANOUS_POD_SOURCE_IMAGE %s\\n\' "$image_commit"',
+    '      else',
+    '        printf \'ELANOUS_POD_SOURCE_IMAGE_SKIP source-tree-mismatch\\n\'',
+    '        rm -- repo || exit 5',
+    '      fi',
+    '    else',
+    '      printf \'ELANOUS_POD_SOURCE_IMAGE_FAIL bootstrap-failed\\n\'',
+    '      exit 5',
+    '    fi',
+    '  else',
+    '    printf \'ELANOUS_POD_SOURCE_IMAGE_SKIP pin-objects-mismatch\\n\'',
+    '  fi',
+    'fi',
+    'if [ "$source_via" != image ]; then',
     // TA-POD-WARMBOOT-PARITY (10-16): the mirror fallback used to be silent — 157 HARD gate Pods all said
     // `VIA github` and nothing said why (the Job had no /host-mirror mount). Each failed step now names itself.
     'mirror_skip=',
@@ -81,6 +116,7 @@ export function podSourceScript(source: PodSource, repoUrl: string, mirrorPath =
     `  ${clone} || exit 5`,
     '  source_via=github',
     'fi',
+    'fi',
     "printf 'ELANOUS_POD_SOURCE_VIA %s\\n' \"$source_via\"",
     'cd repo || exit 5',
   ];
@@ -100,13 +136,27 @@ export function podSourceScript(source: PodSource, repoUrl: string, mirrorPath =
     }
     return [
       ...start,
-      'if [ "$source_via" = mirror ]; then',
-      '  remote_head=$(git ls-remote origin HEAD | cut -f1) || exit 5',
+      'if [ "$source_via" != github ]; then',
+      '  remote_info=$(git ls-remote --symref origin HEAD) || exit 5',
+      `  remote_head=$(printf '%s\\n' "$remote_info" | awk '$2=="HEAD" && $1!="ref:" {print $1}')`,
+      `  remote_branch=$(printf '%s\\n' "$remote_info" | awk '$1=="ref:" && $3=="HEAD" && $2 ~ /^refs\\/heads\\// {sub(/^refs\\/heads\\//, "", $2); print $2}')`,
       '  if ! [[ "$remote_head" =~ ^[0-9a-f]{40}$ ]]; then exit 5; fi',
       '  if [ "$remote_head" != "$(git rev-parse HEAD)" ]; then',
-      '    git fetch --depth 50 origin HEAD && git reset --hard FETCH_HEAD || exit 5',
+      `    if [ "$source_via" = image ] && git -C ${quotedMirror} cat-file -e "$remote_head^{commit}" 2>/dev/null; then`,
+      '      git reset --hard "$remote_head" || exit 5',
+      "      printf 'ELANOUS_POD_SOURCE_DELTA mirror\\n'",
+      '    else',
+      '      git fetch --depth 50 origin HEAD && git reset --hard FETCH_HEAD || exit 5',
+      "      printf 'ELANOUS_POD_SOURCE_DELTA origin\\n'",
+      '    fi',
       '  fi',
       'fi',
+      'if [ "$source_via" = image ] && [ -n "$remote_branch" ] && git check-ref-format --branch "$remote_branch" >/dev/null 2>&1; then',
+      '  if [ "$remote_branch" != main ]; then git branch -m "$remote_branch" || exit 5; fi',
+      '  git update-ref "refs/remotes/origin/$remote_branch" "$(git rev-parse HEAD)" || exit 5',
+      '  git symbolic-ref refs/remotes/origin/HEAD "refs/remotes/origin/$remote_branch" || exit 5',
+      '  git branch --set-upstream-to="origin/$remote_branch" "$remote_branch" >/dev/null || exit 5',
+      'fi',
       'head=$(git rev-parse HEAD)',
       "printf 'ELANOUS_POD_SOURCE default %s\\n' \"$head\"",
     ].join('\n');

package/src/task-orchestrator/surfaces/pod-warm-worker-pod.ts

@@ -8,6 +8,7 @@ import { isAbsolute, join, normalize, resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import type { Kubectl } from './self-implement-pod.js';
 import { elanousStateRoot } from '../../autopilot/state-paths.js';
+import { debug } from '../../debug/log.js';
 import { parseWarmWorkerReply, type WarmWorkerCommand, type WarmWorkerKey, type WarmWorkerReply } from './pod-warm-worker-protocol.js';
 import type { WarmWorkerOutcome } from './pod-warm-worker-runtime.js';
 
@@ -43,6 +44,8 @@ export interface WarmPodSpec {
   nodeName?: string;
   cutSha: string;
   key: WarmWorkerKey;
+  /** Resident worker tier. A lite Pod is capped at 2 GiB, including each request exec. */
+  memoryLimit?: '2Gi' | '16Gi';
   /** Caller resolves the live host daemon endpoint; no guessed port. */
   hostProbeUrl: string;
   /** Cluster service endpoint to prove production network isolation. */
@@ -187,6 +190,7 @@ function validateSpec(spec: WarmPodSpec): void {
       || normalize(spec.sourceMirrorHostPath) !== spec.sourceMirrorHostPath || !spec.nodeName))
     || !probeValid(spec.hostProbeUrl, 'host.orb.internal')
     || !probeValid(spec.prodProbeUrl, 'core.elanous-prod')
+    || (spec.memoryLimit !== undefined && spec.memoryLimit !== '2Gi' && spec.memoryLimit !== '16Gi')
     || (spec.nodeName !== undefined && !DNS.test(spec.nodeName.replace(/\./g, '-')))) throw new Error('invalid warm Pod spec');
 }
 
@@ -223,9 +227,11 @@ export function warmWorkerPodManifest(spec: WarmPodSpec): Record<string, unknown
           ...(mirror ? [{ name: 'source-mirror', mountPath: '/source-mirror', readOnly: true }] : [])] },
       { name: 'install', image: spec.image, imagePullPolicy: 'IfNotPresent',
         securityContext: { runAsUser: 0, allowPrivilegeEscalation: false },
+        resources: { limits: { memory: spec.memoryLimit ?? '16Gi' }, requests: { memory: '512Mi' } },
         command: ['bun', '-e', WARM_POD_INSTALL],
         volumeMounts: [{ name: 'workspace', mountPath: '/workspace' }] }],
       containers: [{ name: 'worker', image: spec.image, imagePullPolicy: 'IfNotPresent',
+        resources: { limits: { memory: spec.memoryLimit ?? '16Gi' }, requests: { memory: '512Mi' } },
         securityContext: { runAsUser: 0, allowPrivilegeEscalation: false,
           capabilities: { drop: ['ALL'], add: ['CHOWN', 'SETUID', 'SETGID', 'KILL'] } },
         command: ['sleep', 'infinity'], env: [
@@ -240,7 +246,7 @@ export function warmWorkerPodManifest(spec: WarmPodSpec): Record<string, unknown
           { name: 'WARM_HOST_PROBE_URL', value: spec.hostProbeUrl },
           { name: 'WARM_PROD_PROBE_URL', value: spec.prodProbeUrl },
         ],
-        volumeMounts: [{ name: 'workspace', mountPath: '/workspace/repo', subPath: 'repo', readOnly: true },
+        volumeMounts: [{ name: 'workspace', mountPath: '/workspace/repo', subPath: 'repo' },
           { name: 'runtime-data', mountPath: '/workspace/data' }] }],
       volumes: [{ name: 'workspace', emptyDir: {} },
         { name: 'runtime-data', emptyDir: {} },
@@ -252,14 +258,26 @@ export function warmWorkerPodManifest(spec: WarmPodSpec): Record<string, unknown
 export interface WarmPodTransportResult {
   /** Every non-complete response leaves the existing one-shot Job authoritative. */
   status: 'observed' | 'fallback';
-  reason: 'complete' | 'key-drift' | 'pod-unready' | 'pod-replaced' | 'exec-failed' | 'invalid-result' | 'worker-quarantined';
+  reason: 'complete' | 'key-drift' | 'pod-unready' | 'pod-replaced' | 'exec-failed' | 'invalid-result' | 'worker-quarantined'
+    | 'no-candidate' | 'lease-unavailable' | 'context-unavailable';
   reply: WarmWorkerReply | null;
+  /** Original local kubectl error when a remote pool context is absent; no Pod was contacted. */
+  kubectlStderr?: string;
   artifactsDir?: string;
   /** Base64 preserves the exact bytes copied from the same Pod. Keys are artifact suffixes. */
   artifacts?: readonly { file: string; data: Readonly<Record<string, string>> }[];
 }
 const fallback = (reason: WarmPodTransportResult['reason']): WarmPodTransportResult => ({ status: 'fallback', reason, reply: null });
 
+/** A remote pool's internal k3d context must be available locally before acquiring its worker lease. */
+export function probeWarmPodContext(context: string, kubectl: Kubectl = warmKubectl): WarmPodTransportResult | null {
+  if (!/^k3d-[a-z0-9-]+$/.test(context)) return null;
+  const observed = kubectl(['config', 'get-contexts', context, '-o', 'name']);
+  if (observed.status === 0 && observed.stdout.split(/\r?\n/).includes(context)) return null;
+  debug.log('pod.warm', 'context-unavailable', { context, kubectlStderr: observed.stderr }, { level: 'warn' });
+  return { status: 'fallback', reason: 'context-unavailable', reply: null, kubectlStderr: observed.stderr };
+}
+
 function canonicalJson(value: unknown): unknown {
   if (Array.isArray(value)) return value.map(canonicalJson);
   if (value !== null && typeof value === 'object') return Object.fromEntries(
@@ -330,7 +348,12 @@ function matchesPodManifest(pod: any, spec: WarmPodSpec): boolean {
       // API-server defaults may add resources/termination fields; execution-bearing fields stay exact.
       for (const field of ['name', 'image', 'imagePullPolicy', 'command', 'args', 'workingDir', 'env', 'envFrom',
         'volumeMounts', 'volumeDevices', 'ports', 'securityContext', 'stdin', 'stdinOnce', 'tty', 'lifecycle',
-        'startupProbe', 'livenessProbe', 'readinessProbe', 'resizePolicy', 'restartPolicy']) {
+        'startupProbe', 'livenessProbe', 'readinessProbe', 'resizePolicy', 'restartPolicy', 'resources']) {
+        // Kubernetes materializes an omitted resource stanza as {} on init containers.
+        // Only that empty default is equivalent; limits and requests still must match exactly.
+        if (field === 'resources' && want?.resources === undefined && got?.resources
+          && typeof got.resources === 'object' && !Array.isArray(got.resources)
+          && Object.keys(got.resources).length === 0) continue;
         if (!sameJson(got?.[field], want?.[field])) return false;
       }
     }