elanous 0.2.19 → 0.2.21

package/src/task-orchestrator/surfaces/pod-command-job.ts

@@ -13,7 +13,9 @@ import { getUserConfig } from '../../user-config.js';
 import { debug } from '../../debug/log.js';
 import { effectiveInstanceRoot } from '../../instance/resolve.js';
 import { collectPodArtifacts } from './pod-artifact-return.js';
+import { getDefaultLogStore, type LogStore } from '../../mss/logging/log-store.js';
 import { podBunCacheVolume } from './pod-bun-cache.js';
+import { podHostDirsInit, podInstallSlotsVolume } from './pod-install-slots.js';
 import { type PodPoolMember, PodPoolScheduler, resolvePodPoolSpec, parsePodPool, checkPodPool, syncPoolImages, type PoolKubectl } from './pod-pool.js';
 import { podSkillsDigest, readSkillEnvFiles, resolvePodSkills } from './pod-skills.js';
 import { podSourceScript, type PodSource } from './pod-source-receive.js';
@@ -135,6 +137,19 @@ for i in $(seq 1 60); do
 done
 echo "[gate] ISOLATION NOT ENFORCED within 30s"; exit 1`;
 
+/** Export the command's log store into the existing outbox return channel without changing the command exit status. */
+const COMMAND_LOG_EXPORT = `if mkdir -p "$HOME/outbox/pod-logs" && elanous --test="$HOME/.elanous-test" logs --instance prod --since 12h --limit 20000 --json --json-data > "$HOME/outbox/pod-logs/logs.jsonl"; then
+  logs_size=$(( $(wc -c < "$HOME/outbox/pod-logs/logs.jsonl") ))
+  if [ "$logs_size" -gt 5242880 ]; then
+    tail -c 5242880 "$HOME/outbox/pod-logs/logs.jsonl" | tail -n +2 > "$HOME/outbox/pod-logs/logs.jsonl.tail" && mv -f "$HOME/outbox/pod-logs/logs.jsonl.tail" "$HOME/outbox/pod-logs/logs.jsonl"
+    printf 'ELANOUS_POD_LOGS_TRUNCATED %s %s\\n' "$logs_size" "$(( $(wc -c < "$HOME/outbox/pod-logs/logs.jsonl") ))"
+  fi
+else
+  logs_rc=$?
+  rm -f "$HOME/outbox/pod-logs/logs.jsonl"
+  printf 'ELANOUS_POD_LOGS_UNAVAILABLE export-exit-%s\\n' "$logs_rc"
+fi`;
+
 /** 기존 Job 과 같은 `~/outbox` 조각 형식(`ELANOUS_POD_ARTIFACT`). 원장 회수는 하지 않는다. */
 const ARTIFACT_EMIT = `set -o pipefail
 artifact_bytes=0
@@ -171,11 +186,15 @@ export interface PodCommandJobInput {
   repoUrl: string;
   command: readonly string[];
   skills: readonly string[];
+  /** pod run opt-in: return this command's local log store with its outbox artifacts. */
+  returnLogs?: boolean;
   llm?: 'grok';
   /** 저장소를 clone 한다 — 비공개 저장소라 GitHub 토큰이 Secret 으로 간다(명시 opt-in). 기본은 clone 없이 `~/work` 에서 이미지의 `elanous` 로 돈다. */
   clone?: boolean;
   hostMirror?: string;
   bunCache?: string;
+  /** GATE-INSTALL-CACHE — hostPath of the node-wide install tokens (release gate shards only · `pod-install-slots.ts`). */
+  installSlots?: string;
   /** 컨테이너 메모리 한도(기본 16Gi) — 게이트가 파일 하나 격리 Job 에 올린다. */
   memoryLimit?: string;
   /** GATE-SPEED A3①: CPU request/limit override (release gate shards only). Omitted = request 1 / limit 4. */
@@ -197,6 +216,9 @@ export function podCommandJobManifest(o: PodCommandJobInput): Record<string, unk
   }
   const grok = o.llm === 'grok';
   const bunCache = o.bunCache ? podBunCacheVolume(o.bunCache) : undefined;
+  const installSlots = o.installSlots ? podInstallSlotsVolume(o.installSlots) : undefined;
+  // hostPath directories the uid-1000 child writes: a root init container hands them over (DirectoryOrCreate is root 0755).
+  const hostDirMounts = [...(bunCache ? [bunCache.volumeMount] : []), ...(installSlots ? [installSlots.volumeMount] : [])];
   const cpu = resolvePodCpu(o.cpu);
   const quotedArgs = o.command.map(bashSingleQuote).join(' ');
   const credLines = [
@@ -216,6 +238,8 @@ export function podCommandJobManifest(o: PodCommandJobInput): Record<string, unk
     `set -- ${quotedArgs}`,
     '"$@"',
     'rc=$?',
+    // A command may not end in LF; keep artifact markers on their own line for the existing parser.
+    ...(o.returnLogs ? ["printf '\\n'", COMMAND_LOG_EXPORT] : []),
     ARTIFACT_EMIT,
     'exit $rc',
   ].join('\n');
@@ -233,7 +257,10 @@ export function podCommandJobManifest(o: PodCommandJobInput): Record<string, unk
         spec: {
           restartPolicy: 'Never',
           securityContext: { runAsUser: 1000, fsGroup: 1000 },
-          initContainers: [{ name: 'isolation-gate', image: o.image, imagePullPolicy: o.imagePullPolicy ?? 'Never', command: ['bash', '-c'], args: [GATE] }],
+          initContainers: [
+            { name: 'isolation-gate', image: o.image, imagePullPolicy: o.imagePullPolicy ?? 'Never', command: ['bash', '-c'], args: [GATE] },
+            ...(hostDirMounts.length ? [podHostDirsInit(o.image, o.imagePullPolicy ?? 'Never', hostDirMounts)] : []),
+          ],
           containers: [{
             name: 'child', image: o.image, imagePullPolicy: o.imagePullPolicy ?? 'Never',
             resources: { requests: { ...POD_CHILD_REQUESTS, cpu: cpu.request, memory: memoryRequestWithin(o.memoryLimit) }, limits: { memory: o.memoryLimit ?? '16Gi', cpu: cpu.limit } },
@@ -243,16 +270,17 @@ export function podCommandJobManifest(o: PodCommandJobInput): Record<string, unk
               { name: 'ELANOUS_SUBSTRATE', value: 'pod' },
               { name: 'ELANOUS_POD_NAME', valueFrom: { fieldRef: { fieldPath: 'metadata.name' } } },
             ],
-            ...(secretKeys || o.hostMirror || bunCache ? { volumeMounts: [
+            ...(secretKeys || o.hostMirror || hostDirMounts.length ? { volumeMounts: [
               ...(secretKeys ? [{ name: 'creds', mountPath: '/creds', readOnly: true }] : []),
               ...(o.hostMirror ? [{ name: 'host-mirror', mountPath: '/host-mirror', readOnly: true }] : []),
-              ...(bunCache ? [bunCache.volumeMount] : []),
+              ...hostDirMounts,
             ] } : {}),
           }],
-          ...(secretKeys || o.hostMirror || bunCache ? { volumes: [
+          ...(secretKeys || o.hostMirror || hostDirMounts.length ? { volumes: [
             ...(secretKeys ? [{ name: 'creds', secret: { secretName: `${o.name}-creds`, defaultMode: 0o400 } }] : []),
             ...(o.hostMirror ? [{ name: 'host-mirror', hostPath: { path: o.hostMirror, type: 'Directory' } }] : []),
             ...(bunCache ? [bunCache.volume] : []),
+            ...(installSlots ? [installSlots.volume] : []),
           ] } : {}),
         },
       },
@@ -315,12 +343,16 @@ export interface PodCommandResult {
 
 export interface RunPodCommandOptions {
   command: readonly string[];
+  /** pod run opt-in; other runPodCommand callers keep their existing Job script. */
+  returnLogs?: boolean;
   pool?: string;
   skills?: readonly string[];
   llm?: 'grok';
   clone?: boolean;
   hostMirror?: string;
   bunCache?: string;
+  /** GATE-INSTALL-CACHE — hostPath of the node-wide install tokens (absolute). */
+  installSlots?: string;
   /** 컨테이너 메모리 한도(기본 16Gi · lite 면 2Gi). */
   memoryLimit?: string;
   /** GATE-SPEED A3①: CPU request/limit override — only the release gate passes it. Omitted = request 1 / limit 4. */
@@ -345,6 +377,8 @@ export interface RunPodCommandOptions {
   readSkillEnv?: (skills: readonly string[]) => Record<string, string>;
   grokCredentials?: () => { grokAuth?: string; grokApiKey?: string; ghToken: string };
   artifactsRoot?: string;
+  /** Test seam for the launching instance's log store. */
+  logStore?: LogStore | null;
   sleep?: (ms: number) => Promise<void>;
   log?: (category: string, event: string, data: Record<string, unknown>) => void;
   env?: NodeJS.ProcessEnv;
@@ -410,6 +444,8 @@ export async function runPodCommand(options: RunPodCommandOptions): Promise<PodC
   if (hostMirror && !isAbsolute(hostMirror)) throw new Error('pod.hostMirror must be an absolute directory path');
   const bunCache = options.bunCache?.trim() || undefined;
   if (bunCache && !isAbsolute(bunCache)) throw new Error('pod.bunCache must be an absolute directory path');
+  const installSlots = options.installSlots?.trim() || undefined;
+  if (installSlots && !isAbsolute(installSlots)) throw new Error('pod installSlots must be an absolute directory path');
   // Fail before any Secret/Job is applied — a bad quantity must not reach the cluster.
   if (options.cpu) resolvePodCpu(options.cpu);
 
@@ -440,7 +476,8 @@ export async function runPodCommand(options: RunPodCommandOptions): Promise<PodC
   const context = member?.context ?? (currentContext?.status === 0 ? currentContext.stdout.trim() : '');
   const kubectl: Kubectl = (args, stdin) => baseKubectl(context ? ['--context', context, ...args] : [...args], stdin);
   const cleanupSecret = () => { kubectl(['-n', namespace, 'delete', 'secret', '-l', own, '--ignore-not-found']); };
-  const artifactsDir = `${options.artifactsRoot ?? `${effectiveInstanceRoot()}/pod-artifacts`}/${name}`;
+  const artifactRoot = options.artifactsRoot ?? `${effectiveInstanceRoot()}/pod-artifacts`;
+  const artifactsDir = `${artifactRoot}/${name}`;
   try {
     if (!context && !options.kubectl) throw new Error('pod command: kubectl context 를 확인할 수 없다');
     let grokAuth: string | undefined;
@@ -485,8 +522,8 @@ export async function runPodCommand(options: RunPodCommandOptions): Promise<PodC
     const manifest = podCommandJobManifest({
       name, namespace, launch, image: imageRef ?? image,
       ...(imageRef ? { imagePullPolicy: 'IfNotPresent' as const } : {}),
-      repoUrl, command, skills, ...(llm ? { llm } : {}), ...(options.clone ? { clone: true } : {}),
-      ...(options.source ? { source: options.source } : {}), ...(hostMirror ? { hostMirror } : {}), ...(bunCache ? { bunCache } : {}), ...(memoryLimit ? { memoryLimit } : {}), ...(options.cpu ? { cpu: options.cpu } : {}), deadlineSeconds,
+      repoUrl, command, skills, ...(options.returnLogs ? { returnLogs: true } : {}), ...(llm ? { llm } : {}), ...(options.clone ? { clone: true } : {}),
+      ...(options.source ? { source: options.source } : {}), ...(hostMirror ? { hostMirror } : {}), ...(bunCache ? { bunCache } : {}), ...(installSlots ? { installSlots } : {}), ...(memoryLimit ? { memoryLimit } : {}), ...(options.cpu ? { cpu: options.cpu } : {}), deadlineSeconds,
       ...(options.runId ? { runId: options.runId } : {}),
     });
     kubectl(['-n', namespace, 'delete', 'job', '-l', own, '--ignore-not-found']);
@@ -505,19 +542,31 @@ export async function runPodCommand(options: RunPodCommandOptions): Promise<PodC
     const logsResult = kubectl(['-n', namespace, 'logs', `job/${name}`, '-c', 'child']);
     const logsFailed = logsResult.status !== 0;
     const logs = logsFailed ? '' : logsResult.stdout;
-    const artifactDir = options.artifactsRoot ?? `${effectiveInstanceRoot()}/pod-artifacts`;
+    const artifactDir = artifactRoot;
     let artifacts: PodArtifactCollection;
     if (logsFailed) {
       const reason = logsResult.stderr.trim() || `kubectl logs exit ${logsResult.status}`;
       artifacts = { files: null, names: [], error: reason };
       log('pod.command', 'artifact-collect-failed', { job: name, reason });
     } else {
+      const store = options.returnLogs ? (options.logStore === undefined ? getDefaultLogStore() : options.logStore) : null;
       collectPodArtifacts(logs, {
-        dir: artifactDir,
-        job: name,
-        log: (c, e, d) => log(c, e, d),
+        dir: artifactDir, job: name, log: (c, e, d) => log(c, e, d),
+        ...(options.returnLogs ? { logStore: store } : {}),
       });
       const names = artifactNames(logs);
+      // A plain command need not write JSONL. Make its child output queryable without changing the store schema.
+      if (options.returnLogs && store) {
+        try {
+          const output = logs.split(/\r?\n/).filter((line) => !line.startsWith('ELANOUS_POD_ARTIFACT ')).join('\n').trim();
+          if (output) store.insertPodBatch(name, [{ line: -1, surface: 'pod', rec: {
+            ts: new Date().toISOString(), category: 'pod.command', event: 'child-output', level: 'info',
+            data: { job: name, output: tailBytes(output, options.childLogMaxBytes ?? podChildLogMaxBytes()), origin: 'pod', podJob: name },
+          } }]);
+        } catch (error) {
+          log('pod.command', 'child-output-import-failed', { job: name, reason: error instanceof Error ? error.message : String(error) });
+        }
+      }
       artifacts = { files: names.length, names, error: null };
     }
     // 표지가 없어도 알리는 경로는 실제로 존재해야 한다. 자식 로그 전문은 child.log (상한 넘으면 끝부분).

package/src/task-orchestrator/surfaces/pod-failure-reason.ts

@@ -1,3 +1,7 @@
+/** STOP-RECORD stop_class values used by the card terminal classifier. */
+export const STOP_CLASS_POD_FAILURE = 'pod-failure' as const;
+export const STOP_CLASS_NO_LAUNCH = 'no-launch' as const;
+
 /** Executor kinds of the dev child's `--json` result line (`src/index.ts` writeStdoutJson · DevExecutor.kind). */
 const CHILD_RESULT_KINDS = new Set(['self', 'external']);
 

package/src/task-orchestrator/surfaces/pod-gate-deps.ts

@@ -0,0 +1,123 @@
+// GATE-IMAGE-DEPS (0.2.21 · GATE-SPEED) — the release-gate shards' dependencies come baked in the harness image.
+//
+// Measured on the 0.2.20 cut gate (~/.elanous/release/0.2.20/gate-logs/cut · 24 root shards · pod-<n>.log ⊕ .junit.xml):
+//   install-slot wait median 156 s (max 509 s) ⊕ `bun install` root+pwa median 104 s (max 160 s) = median ≈ 260 s of a
+//   ≈ 1200 s shard spent before the first test — and the shards that waited longest were the ones the 1200 s deadline cut
+//   (pod-1 · 3 · 11 · 21 · 22 waited 327–509 s and ran only 215–436 s of tests). Alone the install takes seconds; 24 at
+//   once thrash the node (GATE-INSTALL-CACHE). Not installing at all removes both the wait and the install.
+//
+// How: `docker/harness/build.sh` stages each install tree's lockfile ⊕ the install-relevant projection of its package.json
+// ({@link GATE_DEPS_PROJECTION_JS}) into `gate-deps/`; the Dockerfile installs them once into {@link GATE_DEPS_DIR}
+// (`--frozen-lockfile`). The layer's inputs change only when dependencies change, so the per-commit rebuild reuses it from
+// the build cache. A shard links `node_modules` to the baked tree only when its own lockfile and projection are
+// byte-identical to the baked ones; anything else — no baked tree (old image), a different lockfile, a different
+// projection, an existing node_modules, a failed link — falls back to the old path (install slot ⊕ `bun install`).
+// The link is a symlink: copying (or hard-linking across overlay layers) is the very I/O the install slots were throttling.
+// Module resolution follows the real path, and the baked layout mirrors the repo (`<dir>/node_modules`,
+// `<dir>/apps/pwa/node_modules`), so the parent lookup from a package inside the pwa tree is the same as before.
+
+import { createHash } from 'node:crypto';
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+/** Baked dependency root inside the full harness image (mirrors the repo layout). */
+export const GATE_DEPS_DIR = '/opt/elanous-gate-deps';
+/** The install trees, relative to the repo root, in install order (same as the old `bun install && (cd apps/pwa && bun install)`). */
+export const GATE_DEPS_TREES = ['.', 'apps/pwa'] as const;
+
+/**
+ * The part of a package.json that decides what `bun install` produces: dependency maps, overrides, trusted/patched
+ * dependencies and the install lifecycle scripts. `version`, `description` and every other script are dropped, so a
+ * release bump or a new script does not invalidate the baked layer. Run as `bun -e <this> <package.json>` (stdout).
+ * ⛔ build.sh (bake) and the shard (check) run this same text — changing it simply makes the next shards fall back once.
+ */
+export const GATE_DEPS_PROJECTION_JS = [
+  "const p=JSON.parse(require('fs').readFileSync(process.argv[1],'utf8'));const o={};",
+  "for(const k of ['name','dependencies','devDependencies','optionalDependencies','peerDependencies','trustedDependencies','overrides','resolutions','patchedDependencies','workspaces'])if(p[k]!==undefined)o[k]=p[k];",
+  "const s={};for(const k of ['preinstall','install','postinstall','prepare'])if(p.scripts&&p.scripts[k]!==undefined)s[k]=p.scripts[k];",
+  "if(Object.keys(s).length)o.scripts=s;process.stdout.write(JSON.stringify(o,null,2)+'\\n');",
+].join('');
+
+/** Same projection in-process (tests · build staging). */
+export function projectDepsManifest(manifest: Record<string, unknown>): Record<string, unknown> {
+  const out: Record<string, unknown> = {};
+  for (const key of ['name', 'dependencies', 'devDependencies', 'optionalDependencies', 'peerDependencies', 'trustedDependencies', 'overrides', 'resolutions', 'patchedDependencies', 'workspaces']) {
+    if (manifest[key] !== undefined) out[key] = manifest[key];
+  }
+  const scripts = manifest.scripts as Record<string, unknown> | undefined;
+  const lifecycle: Record<string, unknown> = {};
+  for (const key of ['preinstall', 'install', 'postinstall', 'prepare']) if (scripts && scripts[key] !== undefined) lifecycle[key] = scripts[key];
+  if (Object.keys(lifecycle).length) out.scripts = lifecycle;
+  return out;
+}
+
+/**
+ * build.sh staging: `<dest>/<tree>/bun.lock` (verbatim) ⊕ `<dest>/<tree>/package.json` (the projection) for every install
+ * tree. The digest (sha256 of the staged bytes, 12 hex) becomes the image label `elanous.gate-deps`.
+ */
+export function stageGateDeps(root: string, dest: string): { digest: string; trees: string[] } {
+  const hash = createHash('sha256');
+  const trees: string[] = [];
+  mkdirSync(dest, { recursive: true });
+  for (const tree of GATE_DEPS_TREES) {
+    const lock = join(root, tree, 'bun.lock');
+    const manifest = join(root, tree, 'package.json');
+    if (!existsSync(lock) || !existsSync(manifest)) continue;
+    const out = join(dest, tree);
+    mkdirSync(out, { recursive: true });
+    const lockBytes = readFileSync(lock);
+    const projected = `${JSON.stringify(projectDepsManifest(JSON.parse(readFileSync(manifest, 'utf8')) as Record<string, unknown>), null, 2)}\n`;
+    writeFileSync(join(out, 'bun.lock'), lockBytes, { mode: 0o644 });
+    writeFileSync(join(out, 'package.json'), projected, { mode: 0o644 });
+    hash.update(`${tree}\0`).update(lockBytes).update('\0').update(projected).update('\0');
+    trees.push(tree);
+  }
+  return { digest: hash.digest('hex').slice(0, 12), trees };
+}
+
+const sq = (value: string) => `'${value.replace(/'/g, `'\\''`)}'`;
+
+/**
+ * Bash that installs the shard's dependencies (cwd = the repo clone) and sets `irc` (0 = ready). Each tree is linked to
+ * the baked tree when it matches, else installed — only the installs take an install slot (`install_slot_acquire` /
+ * `install_slot_release` from `installSlotScript` must already be defined when `installSlots > 0`). Every decision is a
+ * `[gate] deps <tree> baked|miss (<why>)` line in `log` (the shard's install.log).
+ */
+export function gateDepsInstallScript(o: { log: string; installSlots: number; depsDir?: string }): string {
+  const deps = sq(o.depsDir ?? GATE_DEPS_DIR);
+  const log = o.log;
+  return [
+    'gate_deps_link() {',
+    `  local t="$1" d=${deps} want`,
+    '  [ "$t" = . ] || d="$d/$t"',
+    '  if [ ! -d "$d/node_modules" ]; then echo "[gate] deps $t miss (no baked deps)"; return 1; fi',
+    '  if [ -e "$t/node_modules" ] || [ -L "$t/node_modules" ]; then echo "[gate] deps $t miss (node_modules exists)"; return 1; fi',
+    '  if ! cmp -s "$t/bun.lock" "$d/bun.lock"; then echo "[gate] deps $t miss (bun.lock differs)"; return 1; fi',
+    `  want=$(bun -e ${sq(GATE_DEPS_PROJECTION_JS)} "$t/package.json" 2>/dev/null) || { echo "[gate] deps $t miss (package.json unreadable)"; return 1; }`,
+    '  if [ "$want" != "$(cat "$d/package.json" 2>/dev/null)" ]; then echo "[gate] deps $t miss (package.json differs)"; return 1; fi',
+    '  if ! ln -s "$d/node_modules" "$t/node_modules"; then echo "[gate] deps $t miss (link failed)"; return 1; fi',
+    '  echo "[gate] deps $t baked"',
+    '}',
+    'irc=0; gate_deps_need=',
+    `for t in ${GATE_DEPS_TREES.join(' ')}; do gate_deps_link "$t" >> ${log} 2>&1 || gate_deps_need="$gate_deps_need $t"; done`,
+    'if [ -n "$gate_deps_need" ]; then',
+    ...(o.installSlots > 0 ? [`  install_slot_acquire >> ${log} 2>&1`] : []),
+    `  for t in $gate_deps_need; do (cd "$t" && bun install) >> ${log} 2>&1; irc=$?; [ "$irc" -eq 0 ] || break; done`,
+    ...(o.installSlots > 0 ? ['  install_slot_release'] : []),
+    'fi',
+  ].join('\n');
+}
+
+/** Per-tree decisions from a shard's install log (`baked` · `miss:<why>`), for the gate's `pod-shard` observation. */
+export function parseGateDeps(log: string): Record<string, string> {
+  const out: Record<string, string> = {};
+  for (const match of log.matchAll(/^\[gate\] deps (\S+) (baked|miss \(([^)]*)\))$/gm)) out[match[1]!] = match[2] === 'baked' ? 'baked' : `miss:${match[3]}`;
+  return out;
+}
+
+if (import.meta.main) {
+  const [cmd, dest] = process.argv.slice(2);
+  if (cmd !== 'stage' || !dest) { console.error('usage: pod-gate-deps.ts stage <dir>'); process.exit(2); }
+  const staged = stageGateDeps(process.cwd(), dest);
+  console.log(`digest=${staged.digest} trees=${staged.trees.join(',') || '-'}`);
+}

package/src/task-orchestrator/surfaces/pod-install-slots.ts

@@ -0,0 +1,89 @@
+// GATE-INSTALL-CACHE (0.2.20) — a node-wide throttle for the release-gate shards' `bun install`.
+//
+// Cause, measured on the 0.2.19 gate (`elanous logs --category release-loop.gate --event pod-shard --all --include-test`):
+//   · no cache is «not» the cause — the harness image carries a warm Bun cache, and a Pod that installed alone
+//     (10-07 10:48–10:55 · 12:13 · 12:16) took 2.4–5.0 s with `pod-bun-cache source=none`;
+//   · the same install took 150–340 s when 24 shard Pods (or ~20 isolation Pods) started together on the one 32-CPU
+//     node-b node (09:3x round 1: 221–337 s · 11:4x round 3: 150–240 s · 10:34 isolation wave: 124–188 s).
+//   ⇒ concurrent installs thrash the node (24 × copying node_modules out of the image cache + CPU), so a shared
+//   hostPath Bun cache would not help (the bytes are already local) — fewer installs «at once» does.
+//
+// Concurrency safety: the hostPath holds only throttle tokens (`slot-<n>` directories), never packages. Each Pod still
+// installs into its own container (its own node_modules, its own image-local Bun cache), so correctness never depends on
+// the token: a lost token (Pod killed mid-install) only lowers throughput until it goes stale (`staleSeconds`), a doubled
+// token (two waiters reclaiming one stale slot in the same instant) only lets one extra install run, and a waiter that
+// cannot get a token within `waitSeconds` — or finds the directory unwritable — installs anyway (fail-open).
+// `mkdir` is atomic on one filesystem, and every Pod on a node sees the same hostPath directory.
+
+/** Default hostPath of the install tokens on each gate node. */
+export const POD_INSTALL_SLOTS_HOST_PATH = '/var/lib/elanous/gate-install-slots';
+/** Default concurrent installs per node (`release.loop.gatePodInstallSlots` · 0 = off). Alone ≈ 3–5 s ⇒ 24 shards ≈ 6 waves. */
+export const POD_INSTALL_SLOTS_DEFAULT = 4;
+export const POD_INSTALL_SLOTS_MOUNT = '/gate-install-slots';
+
+/** hostPath volume ⊕ mount for the tokens. `DirectoryOrCreate` makes a root-owned 0755 directory, so the Pod also needs
+ *  {@link podHostDirsInit} (the child runs as uid 1000). */
+export function podInstallSlotsVolume(hostPath: string = POD_INSTALL_SLOTS_HOST_PATH) {
+  return {
+    volume: { name: 'install-slots', hostPath: { path: hostPath, type: 'DirectoryOrCreate' } },
+    volumeMount: { name: 'install-slots', mountPath: POD_INSTALL_SLOTS_MOUNT, readOnly: false },
+  };
+}
+
+/**
+ * A root init container that hands the mounted hostPath directories to the child's uid. Without it a `DirectoryOrCreate`
+ * hostPath stays root:root 0755 and the child's `[ -w … ]` guard silently skips it (the Bun cache option was a no-op).
+ */
+export function podHostDirsInit(image: string, imagePullPolicy: string, mounts: ReadonlyArray<{ name: string; mountPath: string; readOnly: boolean }>) {
+  const paths = mounts.map((mount) => `'${mount.mountPath.replace(/'/g, `'\\''`)}'`).join(' ');
+  return {
+    name: 'host-dirs', image, imagePullPolicy,
+    securityContext: { runAsUser: 0 },
+    command: ['sh', '-c'],
+    // Never fails the Pod: an unchanged directory only means the child falls back (no cache · no throttle).
+    args: [`for d in ${paths}; do mkdir -p "$d" && chown 1000:1000 "$d" && chmod 0775 "$d" || echo "[host-dirs] could not prepare $d"; done; exit 0`],
+    volumeMounts: mounts.map((mount) => ({ ...mount })),
+  };
+}
+
+const int = (value: number, label: string) => {
+  if (!Number.isSafeInteger(value) || value < 1) throw new Error(`invalid ${label}: ${value}`);
+  return value;
+};
+
+/**
+ * Bash that defines `install_slot_acquire` / `install_slot_release`. Acquire waits for one of `slots` tokens under `dir`
+ * (polling every `pollSeconds`, default 2 s), reclaims a token older than `staleSeconds` (minute granularity), and gives up after `waitSeconds` (install runs anyway).
+ * It prints `[gate] install-slot <n> waited <s>s` or `[gate] install-slot none …` for the shard log.
+ */
+export function installSlotScript(o: { dir?: string; slots: number; waitSeconds?: number; staleSeconds?: number; pollSeconds?: number }): string {
+  const dir = `'${(o.dir ?? POD_INSTALL_SLOTS_MOUNT).replace(/'/g, `'\\''`)}'`;
+  const slots = int(o.slots, 'install slots');
+  const wait = int(o.waitSeconds ?? 600, 'install slot wait');
+  const poll = o.pollSeconds ?? 2;
+  if (!Number.isFinite(poll) || poll <= 0 || poll > 60) throw new Error(`invalid install slot poll: ${poll}`);
+  const staleMinutes = Math.max(1, Math.ceil(int(o.staleSeconds ?? 900, 'install slot stale age') / 60));
+  return [
+    'install_slot=',
+    'install_slot_acquire() {',
+    `  local d=${dir} t0 now n`,
+    '  t0=$(date +%s)',
+    '  if [ ! -d "$d" ] || [ ! -w "$d" ]; then echo "[gate] install-slot none (directory not writable)"; return 0; fi',
+    '  while :; do',
+    `    for n in $(seq 0 ${slots - 1}); do`,
+    '      if mkdir "$d/slot-$n" 2>/dev/null; then',
+    '        install_slot="$d/slot-$n"',
+    '        echo "[gate] install-slot $n waited $(( $(date +%s) - t0 ))s"; return 0',
+    '      fi',
+    // A token whose holder died (directory mtime older than the stale age · `find -mmin` is the same on GNU and BSD) is
+    // renamed away first (rename is atomic: one reclaimer wins) and the slot is tried again on the next pass.
+    `      if [ -n "$(find "$d/slot-$n" -maxdepth 0 -mmin +${staleMinutes} 2>/dev/null)" ]; then mv "$d/slot-$n" "$d/stale-$n-$$" 2>/dev/null && rm -rf "$d/stale-$n-$$"; fi`,
+    '    done',
+    '    now=$(date +%s)',
+    `    if [ $(( now - t0 )) -ge ${wait} ]; then echo "[gate] install-slot none (waited $(( now - t0 ))s)"; return 0; fi`,
+    `    sleep ${poll}`,
+    '  done',
+    '}',
+    'install_slot_release() { if [ -n "$install_slot" ]; then rm -rf "$install_slot"; install_slot=; fi; }',
+  ].join('\n');
+}

package/src/task-orchestrator/surfaces/pod-lease.ts

@@ -52,6 +52,18 @@ export interface PodLeaseMember {
    */
   harnessUsageBytes?: number[] | null;
   usageReason?: string | null;
+  /** GATE-RESERVE-AUTO — non-terminal release-gate shard Jobs (`gate-<uuid>`, kind=command). */
+  gateJobs?: number | null;
+  /** GATE-RESERVE-AUTO — of those, Jobs whose Pod is not bound to a node yet (not created, or Pending unscheduled). */
+  gateUnboundJobs?: number | null;
+  /** GATE-RESERVE-AUTO — CPU requested by the non-terminal gate Jobs' Pod templates (observability; Infinity: unreadable). */
+  gateCpuMillicores?: number | null;
+  /** GATE-RESERVE-AUTO — allocatable CPU of the schedulable (Ready, untainted, cordon-free) nodes. */
+  schedulableCpuMillicores?: number | null;
+  /** GATE-RESERVE-AUTO — per schedulable node: allocatable CPU − requests of every non-terminal Pod bound to it (gate Pods too). */
+  cpuFreeByNodeMillicores?: number[] | null;
+  /** GATE-RESERVE-AUTO — CPU requested by harness Pods still waiting for a node (they take CPU somewhere once bound). */
+  pendingHarnessCpuMillicores?: number | null;
   /** Pool-wide harness reservation: Σ min(limit, max(request, usage×1.5)), or max(request, limit) when the Pod's usage is unmeasured. */
   memoryReservedBytes?: number | null;
   /** Parallel to availableMemoryByNodeBytes: each schedulable node's allocatable memory. */
@@ -138,16 +150,19 @@ function object(value: unknown): Record<string, unknown> | null {
   return value !== null && typeof value === 'object' && !Array.isArray(value) ? value as Record<string, unknown> : null;
 }
 
-/** Effective scheduling reservation: max(sum of app requests, largest init request, Pod-level request) + overhead. */
-function requestedMemory(spec: Record<string, unknown>): number | null {
+/**
+ * Effective scheduling reservation: max(sum of app requests, largest init request, Pod-level request) + overhead.
+ * A container without a request reserves its limit (Kubernetes defaults the request to the limit), or 0 with neither.
+ */
+function requestedResource(spec: Record<string, unknown>, unit: 'memory' | 'cpu'): number | null {
   const request = (resources: unknown): number | null => {
     const r = object(resources);
     if (resources !== undefined && !r) return null;
     const requests = object(r?.requests);
     const limits = object(r?.limits);
     if ((r?.requests !== undefined && !requests) || (r?.limits !== undefined && !limits)) return null;
-    const raw = requests?.memory ?? limits?.memory;
-    return raw === undefined ? 0 : typeof raw === 'string' ? quantity(raw, 'memory') : null;
+    const raw = requests?.[unit] ?? limits?.[unit];
+    return raw === undefined ? 0 : typeof raw === 'string' ? quantity(raw, unit) : null;
   };
   const containers = spec.containers;
   if (!Array.isArray(containers) || !containers.length) return null;
@@ -167,12 +182,16 @@ function requestedMemory(spec: Record<string, unknown>): number | null {
     init = Math.max(init, v);
   }
   const pod = spec.resources === undefined ? 0 : request(spec.resources);
-  const overheadRaw = object(spec.overhead)?.memory;
+  const overheadRaw = object(spec.overhead)?.[unit];
   const overhead = spec.overhead !== undefined && !object(spec.overhead) ? null
-    : overheadRaw === undefined ? 0 : typeof overheadRaw === 'string' ? quantity(overheadRaw, 'memory') : null;
+    : overheadRaw === undefined ? 0 : typeof overheadRaw === 'string' ? quantity(overheadRaw, unit) : null;
   return pod === null || overhead === null ? null : Math.max(app, init, pod) + overhead;
 }
 
+const requestedMemory = (spec: Record<string, unknown>): number | null => requestedResource(spec, 'memory');
+/** GATE-RESERVE-AUTO — the CPU the scheduler reserves for a Pod (same rule as memory). */
+const requestedCpu = (spec: Record<string, unknown>): number | null => requestedResource(spec, 'cpu');
+
 function runningLimit(spec: Record<string, unknown>): number | null {
   if (!Array.isArray(spec.containers) || !spec.containers.length) return null;
   let total = 0;
@@ -325,6 +344,8 @@ export function measurePoolLease(members: readonly PodPoolMember[], deps: { kube
       };
       const nodeItems = parsedItems(nodes);
       const nodeFree = new Map<string, number>();
+      let schedulableCpu = 0;
+      const nodeCpuFree = new Map<string, number>();
       const nodeNames = new Set<string>();
       if (!nodeItems?.length) reasons.push(`cluster nodes: ${nodes.status === 0 ? 'invalid/empty response' : nodes.stderr.trim().split('\n').pop() || `rc=${nodes.status}`}`);
       else {
@@ -350,11 +371,11 @@ export function measurePoolLease(members: readonly PodPoolMember[], deps: { kube
           }
           nodeNames.add(name);
           const repels = taintList.some((t) => t!.effect === 'NoSchedule' || t!.effect === 'NoExecute');
-          if (spec?.unschedulable !== true && !repels && object(ready[0])?.status === 'True') { nodeFree.set(name, mem); nodeAllocatable.set(name, mem); }
+          if (spec?.unschedulable !== true && !repels && object(ready[0])?.status === 'True') { nodeFree.set(name, mem); nodeAllocatable.set(name, mem); nodeCpuFree.set(name, cores); schedulableCpu += cores; }
           memory += mem; cpu += cores;
         }
-        if (valid) { result.allocatableMemoryBytes = memory; result.allocatableCpuMillicores = cpu; }
-        else { nodeFree.clear(); nodeAllocatable.clear(); nodeNames.clear(); reasons.push('cluster nodes: invalid name/allocatable memory/cpu/readiness'); }
+        if (valid) { result.allocatableMemoryBytes = memory; result.allocatableCpuMillicores = cpu; result.schedulableCpuMillicores = schedulableCpu; }
+        else { nodeFree.clear(); nodeAllocatable.clear(); nodeCpuFree.clear(); nodeNames.clear(); reasons.push('cluster nodes: invalid name/allocatable memory/cpu/readiness'); }
       }
       // POD-ADMIT-BY-USAGE: every schedulable node needs a real-usage reading, or the member stays on limits.
       const unreadNode = podUsage && nodeUsage ? [...nodeFree.keys()].find((n) => !nodeUsage!.has(n)) : undefined;
@@ -366,6 +387,8 @@ export function measurePoolLease(members: readonly PodPoolMember[], deps: { kube
       if (podItems && jobItems) {
         const jobNames = new Set<string>();
         const leasedJobs = new Set<string>();
+        const gateJobs = new Map<string, number>();
+        const boundGateJobs = new Set<string>();
         let jobsValid = true;
         for (const job of jobItems) {
           const meta = object(job.metadata);
@@ -373,12 +396,27 @@ export function measurePoolLease(members: readonly PodPoolMember[], deps: { kube
           if (object(meta.labels)?.['elanous.substrate'] !== 'pod') continue;
           if (typeof meta.name !== 'string' || !meta.name) { jobsValid = false; continue; }
           jobNames.add(meta.name);
+          // GATE-RESERVE-AUTO: a release-gate shard Job (`gate-<uuid>`, kind=command) reserves its Pod template's CPU
+          // from creation until it is terminal — also while its Pod is still Pending or not created yet.
+          if (meta.name.startsWith('gate-') && object(meta.labels)?.['elanous.kind'] === 'command') {
+            const status = object(job.status);
+            const conditions = Array.isArray(status?.conditions) ? status.conditions as unknown[] : [];
+            const terminal = !!status?.completionTime || Number(status?.succeeded) > 0 ||
+              conditions.some((c) => (object(c)?.type === 'Failed' || object(c)?.type === 'Complete') && object(c)?.status === 'True');
+            if (!terminal) {
+              const spec = object(object(object(job.spec)?.template)?.spec);
+              // An unreadable request cannot be reserved as 0: it blocks harness admission until the gate ends.
+              gateJobs.set(meta.name, (spec ? requestedCpu(spec) : null) ?? Number.POSITIVE_INFINITY);
+            }
+          }
           if (object(meta.annotations)?.[POD_HOST_LEASE_ANNOTATION] === 'true') leasedJobs.add(meta.name);
         }
         if (!jobsValid) reasons.push('cluster jobs: missing name/labels');
         let running = 0, pending = 0, unleasedRunning = 0, limits = 0, reserved = 0, limitsValid = true, reservationsValid = nodeItems !== null && nodeNames.size === nodeItems.length;
         const pendingJobs = new Map<string, PendingJobIdentity>();
         let unknownPhase = false;
+        let pendingHarnessCpu = 0;
+        let cpuValid = true;
         const seenPods = new Set<string>();
         for (const pod of podItems) {
           const metadata = object(pod.metadata);
@@ -393,7 +431,9 @@ export function measurePoolLease(members: readonly PodPoolMember[], deps: { kube
             if (seenPods.has(identity)) { reservationsValid = false; continue; }
             seenPods.add(identity);
           } else reservationsValid = false;
-          const harness = metadata?.namespace === 'elanous-test' && labels?.['elanous.probe'] !== 'dns' &&
+          const ownerJob = String(labels?.['elanous.job'] ?? labels?.['job-name'] ?? '');
+          const gatePod = metadata?.namespace === 'elanous-test' && gateJobs.has(ownerJob);
+          const harness = metadata?.namespace === 'elanous-test' && labels?.['elanous.probe'] !== 'dns' && !gatePod &&
             (labels?.['elanous.substrate'] === 'pod' || jobNames.has(String(labels?.['elanous.job'] ?? '')));
           const spec = object(pod.spec);
           let limit: number | null = null;
@@ -425,6 +465,17 @@ export function measurePoolLease(members: readonly PodPoolMember[], deps: { kube
           }
           // Pending Pods with no assigned node have no node reservation yet. They already occupy a pool slot.
           const nodeName = spec?.nodeName;
+          // GATE-RESERVE-AUTO: CPU is placed per node (a free core split across two nodes is not one slot). Every Pod
+          // bound to a schedulable node — gate Pods too — takes its request there; harness Pods without a node yet
+          // take CPU somewhere once bound. A gate Pod bound to any node marks its Job as placed.
+          const boundNode = typeof nodeName === 'string' && nodeName !== '' ? nodeName : null;
+          if (gatePod && boundNode !== null && nodeNames.has(boundNode)) boundGateJobs.add(ownerJob);
+          if ((boundNode !== null && nodeCpuFree.has(boundNode)) || (boundNode === null && harness && phase === 'Pending')) {
+            const cpu = spec ? requestedCpu(spec) : null;
+            if (cpu === null) cpuValid = false;
+            else if (boundNode !== null) nodeCpuFree.set(boundNode, nodeCpuFree.get(boundNode)! - cpu);
+            else pendingHarnessCpu += cpu;
+          }
           if ((nodeName === undefined || nodeName === '') && phase === 'Pending' && spec && typeof metadata?.namespace === 'string' &&
               requestedMemory(spec) !== null) continue;
           const requested = spec ? requestedMemory(spec) : null;
@@ -432,6 +483,15 @@ export function measurePoolLease(members: readonly PodPoolMember[], deps: { kube
           if (nodeFree.has(nodeName)) nodeFree.set(nodeName, nodeFree.get(nodeName)! - Math.max(requested, reserve ?? limit ?? 0));
         }
         if (unknownPhase) reasons.push('cluster pods: missing/Unknown phase');
+        if (jobsValid && !unknownPhase && result.schedulableCpuMillicores !== undefined) {
+          result.gateJobs = gateJobs.size;
+          result.gateUnboundJobs = [...gateJobs.keys()].filter((job) => !boundGateJobs.has(job)).length;
+          result.gateCpuMillicores = [...gateJobs.values()].reduce((sum, cpu) => sum + cpu, 0);
+        }
+        if (cpuValid && !unknownPhase && result.schedulableCpuMillicores !== undefined) {
+          result.cpuFreeByNodeMillicores = [...nodeCpuFree.values()];
+          result.pendingHarnessCpuMillicores = pendingHarnessCpu;
+        }
         if (jobsValid && !unknownPhase) { result.running = running; result.pending = pending; result.pendingJobs = [...pendingJobs.values()]; result.unleasedRunning = unleasedRunning; }
         if (limitsValid && jobsValid && !unknownPhase) {
           result.memoryLimitBytes = limits;
@@ -464,6 +524,29 @@ export function measurePoolLease(members: readonly PodPoolMember[], deps: { kube
   }) };
 }
 
+/**
+ * GATE-RESERVE-AUTO — while a release-gate shard Job is not terminal, how many more harness Pods (each requesting
+ * `POD_CHILD_REQUESTS.cpu`) fit beside it. The gate gets its CPU first: while any gate shard is still waiting for a
+ * node, no harness Pod is admitted (the scheduler may place new harness Pods where the shard would have gone). Once
+ * every shard is bound its CPU sits on its node, and the answer is Σ per-node ⌊free ÷ request⌋ minus the harness Pods
+ * still waiting for a node. `undefined`: no gate Job is active (normal admission) · `null`: a gate is active but CPU
+ * could not be measured — a Pod's or a gate template's request is unreadable (admit nothing more). The count comes from this one reading, never from earlier ones.
+ */
+export function harnessCpuSlotsBesideGate(measure: PoolLeaseMeasure): number | null | undefined {
+  if (!measure.members.some((m) => (m.gateJobs ?? 0) > 0)) return undefined;
+  const perPod = quantity(POD_CHILD_REQUESTS.cpu, 'cpu')!;
+  let slots = 0;
+  for (const m of measure.members) {
+    if (m.gateJobs == null || m.gateUnboundJobs == null || m.cpuFreeByNodeMillicores == null || m.pendingHarnessCpuMillicores == null) return null;
+    // A gate Job whose Pod template CPU cannot be read is unmeasured even when its Pod is bound (fail closed).
+    if (m.gateCpuMillicores == null || !Number.isFinite(m.gateCpuMillicores)) return null;
+    if (m.gateUnboundJobs > 0) return 0;
+    const placeable = m.cpuFreeByNodeMillicores.reduce((sum, free) => sum + Math.max(0, Math.floor(free / perPod)), 0);
+    slots += Math.max(0, placeable - Math.ceil(m.pendingHarnessCpuMillicores / perPod));
+  }
+  return slots;
+}
+
 /** Per-node memory and per-member slots cannot be exchanged across nodes/clusters. */
 export function recommendConcurrency(measure: PoolLeaseMeasure, options: { capacity: number; perGoalMemory?: string; liteGoalMemory?: string; accounts: number; perAccount: number }): PoolLeaseRecommendation {
   const empty: PoolLeaseRecommendation = { recommended: null, limitedBy: null, reason: null, capacitySlots: null, memorySlots: null, liteMemorySlots: null, placeableSlots: null, accountSlots: null, running: null, pending: null, unleasedRunning: null };