elanous 0.2.38 → 0.2.41

package/src/task-orchestrator/surfaces/pod-ledger-events.ts

@@ -1,3 +1,4 @@
+import { createHash } from 'node:crypto';
 import { mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
 import { join } from 'node:path';
 
@@ -39,6 +40,29 @@ export function captureFailedPodJobs(input: {
   }
   const location = { host: first.host, context: first.context, namespace: first.namespace, childRunId: first.childRunId, jobUid: first.jobUid };
   const evidenceDir = join(artifactDir, 'pod-failure-capture', runId, job, first.childRunId as string);
+  // A failed kubectl/SSH read is itself the missing diagnostic. Keep its stderr in
+  // the private artifact directory and put only the path and exit status in the ledger.
+  const readFailure = (reason: string, result: { status: number | null; stderr: string }, extra: Record<string, unknown> = {}): void => {
+    let stderrEvidence: string | null = null;
+    let stderrCaptureStatus: 'saved' | 'redacted' | 'empty' | 'write-failed' = result.stderr ? 'saved' : 'empty';
+    if (result.stderr) {
+      try {
+        mkdirSync(evidenceDir, { recursive: true });
+        const tail = result.stderr.slice(-8192).split(/\r?\n/).map(line => {
+          if (!SECRET_VALUE.test(line)) return line;
+          stderrCaptureStatus = 'redacted';
+          return '[redacted credential line]';
+        }).join('\n');
+        const digest = createHash('sha256').update(tail).digest('hex').slice(0, 16);
+        stderrEvidence = join(evidenceDir, `${reason}-${digest}.stderr.log`);
+        try { writeFileSync(stderrEvidence, tail, { flag: 'wx', mode: 0o600 }); }
+        catch (error) {
+          if ((error as NodeJS.ErrnoException).code !== 'EEXIST' || readFileSync(stderrEvidence, 'utf8') !== tail) throw error;
+        }
+      } catch { stderrEvidence = null; stderrCaptureStatus = 'write-failed'; }
+    }
+    finish('failed', reason, { ...location, ...extra, kubectlStatus: result.status, stderrEvidence, stderrCaptureStatus });
+  };
   const previous = rows.find(row => row.event === 'pod-failure-capture' && row.data?.job === job && row.data?.outcome === 'collected'
     && row.data.childRunId === first.childRunId && row.data.jobUid === first.jobUid
     && row.data.host === first.host && row.data.context === first.context && row.data.namespace === first.namespace);
@@ -54,8 +78,8 @@ export function captureFailedPodJobs(input: {
   try {
     const jobResult = kubectl(first.host as string, [...args, 'get', 'job', job, '-o', 'json']);
     if (jobResult.status !== 0) {
-      finish(/\bNotFound\b|not found/i.test(jobResult.stderr) ? 'vanished' : 'failed',
-        /\bNotFound\b|not found/i.test(jobResult.stderr) ? 'job-not-found' : 'job-read-failed', location);
+      if (/\bNotFound\b|not found/i.test(jobResult.stderr)) finish('vanished', 'job-not-found', { ...location, kubectlStatus: jobResult.status });
+      else readFailure('job-read-failed', jobResult);
       return;
     }
     const jobBody = JSON.parse(jobResult.stdout) as { metadata?: { name?: string; uid?: string } };
@@ -78,7 +102,7 @@ export function captureFailedPodJobs(input: {
       } catch { return false; }
     });
     const podResult = kubectl(first.host as string, [...args, 'get', 'pods', '-l', `job-name=${job}`, '-o', 'json']);
-    if (podResult.status !== 0) { finish('failed', 'pods-read-failed', { ...location, jobEvidence }); return; }
+    if (podResult.status !== 0) { readFailure('pods-read-failed', podResult, { jobEvidence }); return; }
     const pods = JSON.parse(podResult.stdout) as { items?: Array<{ metadata?: { name?: string; ownerReferences?: Array<{ uid?: string }> } }> };
     if (!pods.items?.length || pods.items.some(p => !p.metadata?.ownerReferences?.some(owner => owner.uid === jobBody.metadata?.uid))) {
       finish('failed', 'pod-owner-mismatch', { ...location, jobEvidence }); return;
@@ -89,7 +113,8 @@ export function captureFailedPodJobs(input: {
       finish('failed', 'pod-ambiguous', { ...location, jobEvidence, podEvidence }); return;
     }
     const logs = kubectl(first.host as string, [...args, 'logs', `pod/${podName}`, '-c', 'child']);
-    if (logs.status !== 0) { finish('failed', 'logs-read-failed', { ...location, jobEvidence, podEvidence }); return; }
+    if (logs.status !== 0) { readFailure('logs-read-failed', logs, { jobEvidence, podEvidence }); return; }
+    if (!logs.stdout.trim()) { finish('failed', 'logs-empty', { ...location, jobEvidence, podEvidence }); return; }
     const logEvidence = save('child.log', logs.stdout);
     finish('collected', 'job-failed', { ...location, jobEvidence, podEvidence, logEvidence });
   } catch { finish('failed', 'capture-error', location); }

package/src/task-orchestrator/surfaces/pod-mirror-seed.ts

@@ -78,24 +78,27 @@ export function seedMirrorOnce(o: { host: string; commit: string; cwd: string; p
  * one sha when it does not — objects only, no ref is moved). Never throws: a miss returns `reason` and the caller skips
  * the seed (best effort — the Job keeps its GitHub path).
  */
-export async function resolveDefaultSourceHead(o: { repoUrl: string; base?: string; cwd: string; git?: Push; limitMs?: number }): Promise<{ sha?: string; reason?: string }> {
+export async function resolveDefaultSourceHead(o: { repoUrl: string; base?: string; cwd: string; git?: Push; limitMs?: number }): Promise<{ sha?: string; reason?: string; originCalls?: number }> {
   const git = o.git ?? gitWithin;
   const limitMs = o.limitMs ?? 30_000;
   const base = o.base || 'main';
-  if (base.startsWith('-') || !/^[\w./-]+$/.test(base) || o.repoUrl.startsWith('-')) return { reason: 'invalid-ref' };
+  if (base.startsWith('-') || !/^[\w./-]+$/.test(base) || o.repoUrl.startsWith('-')) return { reason: 'invalid-ref', originCalls: 0 };
+  let originCalls = 0;
   try {
+    originCalls += 1;
     const listed = await git(['ls-remote', '--', o.repoUrl, `refs/heads/${base}`], o.cwd, limitMs);
-    if (listed.rc !== 0) return { reason: 'ls-remote-failed' };
+    if (listed.rc !== 0) return { reason: 'ls-remote-failed', originCalls };
     // Only the exact ref asked for — never the first `refs/heads/*` line (a branch named `a/refs/heads/main` must not pass).
     const want = `refs/heads/${base}`;
     const sha = listed.output.split(/\r?\n/).map((line) => line.split('\t')).find(([hash, ref]) => ref === want && /^[0-9a-f]{40}$/.test(hash ?? ''))?.[0];
-    if (!sha) return { reason: 'ls-remote-no-sha' };
-    if ((await git(['cat-file', '-e', `${sha}^{commit}`], o.cwd, limitMs)).rc === 0) return { sha };
+    if (!sha) return { reason: 'ls-remote-no-sha', originCalls };
+    if ((await git(['cat-file', '-e', `${sha}^{commit}`], o.cwd, limitMs)).rc === 0) return { sha, originCalls };
+    originCalls += 1;
     const fetched = await git(['fetch', '--quiet', '--no-write-fetch-head', '--', o.repoUrl, sha], o.cwd, GATE_MIRROR_SEED_LIMIT_MS);
-    if (fetched.rc !== 0) return { reason: 'commit-fetch-failed' };
-    return (await git(['cat-file', '-e', `${sha}^{commit}`], o.cwd, limitMs)).rc === 0 ? { sha } : { reason: 'commit-not-local' };
+    if (fetched.rc !== 0) return { reason: 'commit-fetch-failed', originCalls };
+    return (await git(['cat-file', '-e', `${sha}^{commit}`], o.cwd, limitMs)).rc === 0 ? { sha, originCalls } : { reason: 'commit-not-local', originCalls };
   } catch {
-    return { reason: 'resolve-threw' };
+    return { reason: 'resolve-threw', originCalls };
   }
 }
 

package/src/task-orchestrator/surfaces/pod-si-warm-protocol.ts

@@ -103,7 +103,7 @@ export function parseSiWarmCommand(raw: unknown): SiWarmCommand | null {
     || !digest(row.executionKey) || !str(row.sourceSha) || !SHA.test(row.sourceSha)
     || !token(row.podUid) || !integer(row.generation) || row.generation < 1
     || !validKey(row.key) || row.envProfile !== 'si-standard-v1'
-    || !finite(row.deadlineAt) || !digest(row.payloadSha256)
+    || !finite(row.deadlineAt) || row.deadlineAt <= 0 || !digest(row.payloadSha256)
     || !digest(row.credentialBundleSha256)) return null;
   return raw as SiWarmCommand;
 }

package/src/task-orchestrator/surfaces/pod-source-receive.ts

@@ -5,7 +5,7 @@ import { execFileSync } from 'node:child_process';
 import { siTaskPodStageShell } from './pod-lifecycle-stages.js';
 
 export type PodSource =
-  | { kind: 'default'; base?: string }
+  | { kind: 'default'; base?: string; baseSha?: string }
   | { kind: 'commit'; sha: string }
   | { kind: 'pr'; number: number }
   | { kind: 'bundle'; bundlePath: string; sha256: string; sizeBytes: number; headCommit: string };
@@ -64,6 +64,24 @@ export function podSourceScript(source: PodSource, repoUrl: string, mirrorPath =
   const quotedCommitFile = `'${(options.imageCommitFile ?? '/opt/elanous-source-commit').replace(/'/g, `'\\''`)}'`;
   const quotedImageGitDir = `'${(options.imageGitDir ?? '/opt/elanous-pin.git').replace(/'/g, `'\\''`)}'`;
   const clone = `git clone -q --depth 50 ${quotedUrl} repo`;
+  // First observation slice: count source checkout work without changing its routing.
+  const ioStart = [
+    // Count calls performed by this Pod source stage. Branch validation via
+    // assertRemoteBase runs on the host before Job creation and is outside it.
+    'pod_source_io_origin_calls=0',
+    'pod_source_io_started=$(date +%s)',
+    'pod_source_io_reported=0',
+    'pod_source_io_measured=0',
+    'pod_source_io_measure() { pod_source_io_files=$(git ls-files -z | tr -cd "\\000" | wc -c | tr -d " "); pod_source_io_seconds=$(( $(date +%s) - pod_source_io_started )); pod_source_io_measured=1; }',
+    'pod_source_io_emit() {',
+    '  [ "$pod_source_io_reported" -eq 0 ] || return 0; pod_source_io_reported=1',
+    '  [ -n "${pod_source_io_seconds:-}" ] || pod_source_io_seconds=$(( $(date +%s) - pod_source_io_started ))',
+    '  [ -n "${pod_source_io_files:-}" ] || pod_source_io_files=null',
+    '  pod_source_io_status=incomplete; if [ "$pod_source_io_measured" -eq 1 ]; then pod_source_io_status=complete; fi',
+    '  printf \'ELANOUS_POD_SOURCE_IO {"originCalls":%s,"checkoutFiles":%s,"sourceVia":"%s","sourceSeconds":%s,"sourceStatus":"%s","pushCalls":null,"installRan":null,"installSeconds":null}\\n\' "$pod_source_io_origin_calls" "$pod_source_io_files" "${source_via:-unknown}" "$pod_source_io_seconds" "$pod_source_io_status"',
+    '}',
+    'trap pod_source_io_emit EXIT',
+  ];
   const start = [
     'if [ -e repo ]; then exit 5; fi',
     // POD-MIRROR-IO: the image's pinned tree is a shared read-only lower layer.
@@ -113,7 +131,11 @@ export function podSourceScript(source: PodSource, repoUrl: string, mirrorPath =
     'else',
     "  printf 'ELANOUS_POD_SOURCE_MIRROR_SKIP %s\\n' \"$mirror_skip\"",
     '  if [ -e repo ]; then rm -r -- repo || exit 5; fi',
-    `  ${clone} || exit 5`,
+    ...(source.kind === 'default' && source.baseSha ? [] : ['  pod_source_io_origin_calls=$((pod_source_io_origin_calls + 1))']),
+    ...(source.kind === 'default' && source.baseSha ? [
+      `  printf 'ELANOUS_POD_SOURCE_BASE_SHA_UNAVAILABLE %s\\n' '${source.baseSha}'`,
+      '  exit 5',
+    ] : [`  ${clone} || exit 5`]),
     '  source_via=github',
     'fi',
     'fi',
@@ -121,22 +143,50 @@ export function podSourceScript(source: PodSource, repoUrl: string, mirrorPath =
     'cd repo || exit 5',
   ];
   if (source.kind === 'default') {
+    if (source.baseSha) {
+      const sha = assertSha(source.baseSha, 'base sha');
+      const base = source.base ?? 'main';
+      if (base.startsWith('-') || !/^[\w./-]+$/.test(base)) throw new Error(`invalid Pod base ref: ${base}`);
+      const quote = (value: string) => `'${value.replace(/'/g, `'\\''`)}'`;
+      const quotedBase = quote(base);
+      const quotedTrackingRef = quote(`refs/remotes/origin/${base}`);
+      const quotedUpstream = quote(`origin/${base}`);
+      return [
+        ...ioStart,
+        ...start,
+        `if ! git cat-file -e '${sha}^{commit}' 2>/dev/null; then git fetch --quiet --no-write-fetch-head ${quotedMirror} '${sha}' || { printf 'ELANOUS_POD_SOURCE_BASE_SHA_UNAVAILABLE %s\\n' '${sha}'; exit 5; }; fi`,
+        `git checkout -q -B ${quotedBase} '${sha}' || exit 5`,
+        `git update-ref ${quotedTrackingRef} '${sha}' || exit 5`,
+        `git symbolic-ref refs/remotes/origin/HEAD ${quotedTrackingRef} || exit 5`,
+        `git branch --set-upstream-to=${quotedUpstream} ${quotedBase} >/dev/null || exit 5`,
+        'head=$(git rev-parse HEAD)',
+        'pod_source_io_measure',
+        "printf 'ELANOUS_POD_SOURCE default %s\\n' \"$head\"",
+        'pod_source_io_emit',
+      ].join('\n');
+    }
     if (source.base && source.base !== 'main') {
       assertRemoteBase(repoUrl, source.base);
       const base = source.base;
       const quote = (value: string) => `'${value.replace(/'/g, `'\\''`)}'`;
       const quotedTrackingRef = quote(`refs/remotes/origin/${base}`);
       return [
+        ...ioStart,
         ...start,
+        'pod_source_io_origin_calls=$((pod_source_io_origin_calls + 1))',
         `git fetch --depth 50 origin ${quote(`+refs/heads/${base}:refs/remotes/origin/${base}`)} || exit 5`,
         `git checkout -q -B ${quote(base)} ${quotedTrackingRef} || exit 5`,
         'head=$(git rev-parse HEAD)',
+        'pod_source_io_measure',
         "printf 'ELANOUS_POD_SOURCE default %s\\n' \"$head\"",
+        'pod_source_io_emit',
       ].join('\n');
     }
     return [
+      ...ioStart,
       ...start,
       'if [ "$source_via" != github ]; then',
+      '  pod_source_io_origin_calls=$((pod_source_io_origin_calls + 1))',
       '  remote_info=$(git ls-remote --symref origin HEAD) || exit 5',
       `  remote_head=$(printf '%s\\n' "$remote_info" | awk '$2=="HEAD" && $1!="ref:" {print $1}')`,
       `  remote_branch=$(printf '%s\\n' "$remote_info" | awk '$1=="ref:" && $3=="HEAD" && $2 ~ /^refs\\/heads\\// {sub(/^refs\\/heads\\//, "", $2); print $2}')`,
@@ -146,6 +196,7 @@ export function podSourceScript(source: PodSource, repoUrl: string, mirrorPath =
       '      git reset --hard "$remote_head" || exit 5',
       "      printf 'ELANOUS_POD_SOURCE_DELTA mirror\\n'",
       '    else',
+      '      pod_source_io_origin_calls=$((pod_source_io_origin_calls + 1))',
       '      git fetch --depth 50 origin HEAD && git reset --hard FETCH_HEAD || exit 5',
       "      printf 'ELANOUS_POD_SOURCE_DELTA origin\\n'",
       '    fi',
@@ -158,53 +209,68 @@ export function podSourceScript(source: PodSource, repoUrl: string, mirrorPath =
       '  git branch --set-upstream-to="origin/$remote_branch" "$remote_branch" >/dev/null || exit 5',
       'fi',
       'head=$(git rev-parse HEAD)',
+      'pod_source_io_measure',
       "printf 'ELANOUS_POD_SOURCE default %s\\n' \"$head\"",
+      'pod_source_io_emit',
     ].join('\n');
   }
   if (source.kind === 'commit') {
     const sha = assertSha(source.sha, 'sha');
     return [
+      ...ioStart,
       ...start,
-      `if ! git cat-file -e '${sha}^{commit}' 2>/dev/null; then git fetch --depth 1 origin ${sha} || exit 5; fi`,
+      `if ! git cat-file -e '${sha}^{commit}' 2>/dev/null; then pod_source_io_origin_calls=$((pod_source_io_origin_calls + 1)); git fetch --depth 1 origin ${sha} || exit 5; fi`,
       `git checkout --detach ${sha} || exit 5`,
       'head=$(git rev-parse HEAD)',
+      'pod_source_io_measure',
       "printf 'ELANOUS_POD_SOURCE commit %s\\n' \"$head\"",
+      'pod_source_io_emit',
     ].join('\n');
   }
   if (source.kind === 'pr') {
     const n = assertPositiveInt(source.number);
     return [
+      ...ioStart,
       ...start,
+      'pod_source_io_origin_calls=$((pod_source_io_origin_calls + 1))',
       `pr_sha=$(git ls-remote origin refs/pull/${n}/head | cut -f1) || exit 5`,
       'if ! [[ "$pr_sha" =~ ^[0-9a-f]{40}$ ]]; then exit 5; fi',
-      `if ! git cat-file -e "$pr_sha^{commit}" 2>/dev/null; then git fetch --depth 50 origin pull/${n}/head || exit 5; fi`,
+      `if ! git cat-file -e "$pr_sha^{commit}" 2>/dev/null; then pod_source_io_origin_calls=$((pod_source_io_origin_calls + 1)); git fetch --depth 50 origin pull/${n}/head || exit 5; fi`,
       `git checkout -B pr-${n} "$pr_sha" || exit 5`,
       // A pinned image starts at one shallow main commit. Fetching a PR tip at
       // depth 50 can leave that pin and the PR in disconnected shallow spans.
       // Deepen only after merge-base fails; the gate still reports unavailable
       // if ancestry cannot be obtained.
+      'pod_source_io_origin_calls=$((pod_source_io_origin_calls + 1))',
       'remote_info=$(git ls-remote --symref origin HEAD 2>/dev/null) || { printf \'ELANOUS_POD_COMPARISON_BASE unavailable\\n\'; exit 5; }',
       `base_branch=$(printf '%s\\n' "$remote_info" | awk '$1=="ref:" && $3=="HEAD" && $2 ~ /^refs\\/heads\\// {sub(/^refs\\/heads\\//, "", $2); print $2}')`,
       'if [ -n "$base_branch" ] && git check-ref-format --branch "$base_branch" >/dev/null 2>&1; then',
       '  base_ref="refs/remotes/origin/$base_branch"',
+      '  pod_source_io_origin_calls=$((pod_source_io_origin_calls + 1))',
       '  git fetch --depth 50 origin "+refs/heads/$base_branch:$base_ref" || { printf \'ELANOUS_POD_COMPARISON_BASE unavailable\\n\'; exit 5; }',
       '  git symbolic-ref refs/remotes/origin/HEAD "$base_ref" || exit 5',
       '  if ! git merge-base HEAD "$base_ref" >/dev/null 2>&1 && [ "$(git rev-parse --is-shallow-repository)" = true ]; then',
+      '    pod_source_io_origin_calls=$((pod_source_io_origin_calls + 1))',
       `    git fetch --deepen=100 origin "+refs/heads/$base_branch:$base_ref" +refs/pull/${n}/head:refs/remotes/origin/pr-${n}-head || printf 'ELANOUS_POD_COMPARISON_BASE deepen-failed\\n'`,
       '  fi',
       '  if ! git merge-base HEAD "$base_ref" >/dev/null 2>&1 && [ "$(git rev-parse --is-shallow-repository)" = true ]; then',
+      '    pod_source_io_origin_calls=$((pod_source_io_origin_calls + 1))',
       `    git fetch --unshallow origin "+refs/heads/$base_branch:$base_ref" +refs/pull/${n}/head:refs/remotes/origin/pr-${n}-head || { printf 'ELANOUS_POD_COMPARISON_BASE unavailable\\n'; exit 5; }`,
       '  fi',
       "  if ! git merge-base HEAD \"$base_ref\" >/dev/null 2>&1; then printf 'ELANOUS_POD_COMPARISON_BASE unavailable\\n'; fi",
       "else printf 'ELANOUS_POD_COMPARISON_BASE unavailable\\n'; exit 5; fi",
       'head=$(git rev-parse HEAD)',
+      'pod_source_io_measure',
       "printf 'ELANOUS_POD_SOURCE pr %s\\n' \"$head\"",
+      'pod_source_io_emit',
     ].join('\n');
   }
   const sha256 = assertSha256(source.sha256);
   const sizeBytes = assertSize(source.sizeBytes);
   const headCommit = assertSha(source.headCommit, 'headCommit');
   return [
+    ...ioStart,
+    'source_via=bundle',
     'ready=0',
     'for i in $(seq 1 300); do',
     '  if [ -f /tmp/source.ready ]; then ready=1; break; fi',
@@ -242,8 +308,10 @@ export function podSourceScript(source: PodSource, repoUrl: string, mirrorPath =
     '  git -C repo fetch /tmp/source.bundle "refs/elanous/pod-source/*:refs/remotes/bundle/*" || exit 5',
     'fi',
     `git -C repo checkout --detach ${headCommit} || exit 5`,
-    'head=$(git -C repo rev-parse HEAD) || exit 5',
-    "printf 'ELANOUS_POD_SOURCE bundle %s\\n' \"$head\"",
     'cd repo || exit 5',
+    'head=$(git rev-parse HEAD) || exit 5',
+    'pod_source_io_measure',
+    "printf 'ELANOUS_POD_SOURCE bundle %s\\n' \"$head\"",
+    'pod_source_io_emit',
   ].join('\n');
 }

package/src/task-orchestrator/surfaces/pod-warm-worker-pod.ts

@@ -389,6 +389,7 @@ export class WarmPodTransport {
       || command.workerId !== this.spec.workerId || command.cutSha !== this.spec.cutSha
       || !equalKey(command.key, this.spec.key)
       || ![command.runId, command.attemptId, command.leaseId, command.workerId].every(value => TOKEN.test(value))
+      || typeof command.podUid !== 'string' || !command.podUid
       || !Number.isSafeInteger(command.generation) || command.generation < 1) return fallback('key-drift');
     const prefix = ['--context', this.spec.kubectlContext, '-n', this.spec.namespace];
     const observed = this.kubectl([...prefix, 'get', 'pod', this.spec.name, '-o', 'json']);
@@ -425,7 +426,7 @@ export class WarmPodTransport {
     try { nodeArch = node.status === 0 ? JSON.parse(node.stdout)?.metadata?.labels?.['kubernetes.io/arch'] : null; }
     catch { nodeArch = null; }
     if (nodeArch !== this.spec.key.architecture) return fallback('pod-unready');
-    if ((command.podUid && uid !== command.podUid) || (this.podUid && uid !== this.podUid)) {
+    if (uid !== command.podUid || (this.podUid !== null && uid !== this.podUid)) {
       this.quarantined = true; return fallback('pod-replaced');
     }
     this.podUid = uid;

package/src/task-orchestrator/surfaces/pod-warm-worker-provider.ts

@@ -44,6 +44,8 @@ function waitForPrepareRetry(signal?: AbortSignal): Promise<void> {
 export type WarmReplayInput = Omit<WarmWorkerCommand, 'leaseId' | 'generation' | 'workerId' | 'key'>;
 export interface WarmReplayResult { command: WarmWorkerCommand | null; transport: WarmPodTransportResult }
 export interface WarmWorkerCandidate { host: string; tier: WarmPoolTier; spec: WarmPodSpec }
+/** The OS `flock(fd, operation)` binding the prepare lease waits on. */
+export type WarmPrepareFlock = (fd: number, operation: number) => number;
 export interface WarmPodProviderOptions {
   candidates: readonly WarmWorkerCandidate[];
   /** One pinned template per host/tier. Each expands to the measured profile quota. */
@@ -64,6 +66,8 @@ export interface WarmPodProviderOptions {
   prepareLockDirectory?: string;
   /** Cancels a prepare waiting for another process to finish the same host/worker slot. */
   prepareAbortSignal?: AbortSignal;
+  /** Injects the prepare-lease flock binding. Omitted keeps this platform's binding; null/false declares it unsupported. */
+  prepareFlock?: WarmPrepareFlock | null | false;
   /** Gate file this pool serves, used to bind cold prepare legs to their 1-file gate measurement key. */
   pairingGateFile?: string;
 }
@@ -164,7 +168,7 @@ const prepareLockLib = process.platform === 'linux' || process.platform === 'dar
     flock: { args: ['int', 'int'], returns: 'int' },
     [process.platform === 'linux' ? '__errno_location' : '__error']: { args: [], returns: 'ptr' },
   }) : null;
-const prepareFlock = prepareLockLib?.symbols.flock ?? null;
+const prepareFlock: WarmPrepareFlock | null = prepareLockLib?.symbols.flock ?? null;
 const prepareErrnoLocation = prepareLockLib?.symbols[process.platform === 'linux' ? '__errno_location' : '__error'];
 const readPrepareErrno = (): number => (prepareErrnoLocation ? read.i32(prepareErrnoLocation() as Pointer) : -1);
 const LOCK_EX = 2;
@@ -173,7 +177,7 @@ const LOCK_UN = 8;
 const PREPARE_LOCK_WAIT_MS = (POD_READY_WAIT_SECONDS * 1_000) + 30_000;
 
 async function withPrepareLease<T>(candidate: WarmWorkerCandidate, directory: string | undefined,
-  work: () => Promise<T>, signal?: AbortSignal): Promise<T | null> {
+  work: () => Promise<T>, signal?: AbortSignal, flock?: WarmPrepareFlock | null): Promise<T | null> {
   const uid = typeof process.getuid === 'function' ? process.getuid() : userInfo().uid;
   const root = directory ?? join(tmpdir(), `elanous-warm-prepare-${uid}`);
   const identity = createHash('sha256').update(`${candidate.host}/${candidate.spec.workerId}`).digest('hex');
@@ -186,11 +190,11 @@ async function withPrepareLease<T>(candidate: WarmWorkerCandidate, directory: st
     const info = lstatSync(lock);
     if (!info.isFile() || info.isSymbolicLink() || info.uid !== uid || (info.mode & 0o077) !== 0)
       throw new Error('unsafe warm prepare lock file');
-    if (!prepareFlock) throw new Error('warm prepare OS locks are unavailable on this platform');
+    if (!flock) throw new Error('warm prepare OS locks are unavailable on this platform');
     const lockDeadline = performance.now() + PREPARE_LOCK_WAIT_MS;
     while (!acquired) {
       if (signal?.aborted) throw new Error('warm prepare lock wait cancelled');
-      if (prepareFlock(fd, LOCK_EX | LOCK_NB) === 0) { acquired = true; break; }
+      if (flock(fd, LOCK_EX | LOCK_NB) === 0) { acquired = true; break; }
       const errno = readPrepareErrno();
       // EWOULDBLOCK/EAGAIN: Linux 11, Darwin 35. EINTR is safe to retry.
       if (errno !== 11 && errno !== 35 && errno !== 4)
@@ -209,7 +213,7 @@ async function withPrepareLease<T>(candidate: WarmWorkerCandidate, directory: st
       workerId: candidate.spec.workerId, reason: error instanceof Error ? error.message : String(error) }, { level: 'warn' });
     throw error;
   } finally {
-    if (acquired && fd !== undefined) prepareFlock?.(fd, LOCK_UN);
+    if (acquired && fd !== undefined) flock?.(fd, LOCK_UN);
     if (fd !== undefined) closeSync(fd);
   }
 }
@@ -223,7 +227,14 @@ export class WarmPodProvider {
   private readonly legs: WarmReuseTimeLegRow[] = [];
   /** One terminal transition per fenced lease, including concurrent callers. Failed transitions remain retryable. */
   private readonly terminalLeases = new Map<string, Promise<'sanitized' | 'quarantined'>>();
+  /** A gate fallback supersedes an in-flight sanitize even if its control reply is late. */
+  private readonly quarantineRequested = new Set<string>();
+  private readonly terminalEndLogged = new Set<string>();
+  /** While quarantine is settling, reject only that Pod incarnation after acquisition. */
+  private readonly quarantiningPods = new Set<string>();
   private readonly standingWorkers = new Map<string, Promise<boolean>>();
+  /** This instance's prepare-lease flock binding; null means the platform cannot lock and every slot fails. */
+  private readonly prepareFlock: WarmPrepareFlock | null;
   constructor(private readonly options: WarmPodProviderOptions) {
     const url = new URL(options.controlUrl);
     if (!['http:', 'https:'].includes(url.protocol) || url.username || url.password || url.search || url.hash
@@ -266,6 +277,9 @@ export class WarmPodProvider {
     }
     this.fetcher = options.fetch ?? fetch;
     this.now = options.now ?? Date.now;
+    // An omitted option keeps the platform binding; an explicit false/null declares flock unsupported.
+    this.prepareFlock = options.prepareFlock === undefined ? prepareFlock
+      : options.prepareFlock === false ? null : options.prepareFlock;
   }
 
   private async control(path: string, credential: string, body?: Record<string, unknown>): Promise<{ status: number; value: any }> {
@@ -347,7 +361,8 @@ export class WarmPodProvider {
     const inFlight = this.standingWorkers.get(key);
     if (inFlight) return inFlight;
     const preparation = withPrepareLease(configured, this.options.prepareLockDirectory,
-      () => this.prepareStandingWorker(configured, this.options.prepareAbortSignal), this.options.prepareAbortSignal)
+      () => this.prepareStandingWorker(configured, this.options.prepareAbortSignal), this.options.prepareAbortSignal,
+      this.prepareFlock)
       .then(result => result === true);
     this.standingWorkers.set(key, preparation);
     try { return await preparation; }
@@ -626,6 +641,12 @@ export class WarmPodProvider {
       podName: leasedCandidate.spec.name, podContext: leasedCandidate.spec.kubectlContext, image: leasedCandidate.spec.image,
       requestedAt: new Date(this.now()).toISOString(),
       workerId: row.name, key: leasedCandidate.spec.key };
+    // Acquisition is by pool key. A healthy sibling or replacement Pod must remain eligible.
+    // Never run transport on the old Pod while its previous lease is being quarantined.
+    if (this.quarantiningPods.has(JSON.stringify([command.workerId, command.podUid]))) {
+      await this.quarantine(command, 'prior-lease-quarantine-pending', false);
+      return fallback('lease-unavailable');
+    }
     const admitted = patchWarmLeaseMetadata(leasedCandidate.spec, command, 'leased', this.options.kubectl);
     if (!admitted.ok) {
       debug.log('pod.provenance', 'lease-patch-refused', { pod: leasedCandidate.spec.name,
@@ -685,7 +706,7 @@ export class WarmPodProvider {
     let outcome: 'sanitized' | 'quarantined';
     try { outcome = await terminal; }
     catch (error) { if (this.terminalLeases.get(key) === terminal) this.terminalLeases.delete(key); throw error; }
-    if (outcome === 'quarantined') throw new Error('warm sanitize unavailable');
+    if (outcome === 'quarantined' || this.quarantineRequested.has(key)) throw new Error('warm sanitize unavailable');
   }
 
   private async finalizeOnce(command: WarmWorkerCommand): Promise<'sanitized' | 'quarantined'> {
@@ -702,25 +723,39 @@ export class WarmPodProvider {
       await this.quarantineOnce(command, 'sanitize-unavailable', true);
       return 'quarantined';
     }
-    debug.log('pod.provenance', 'lease-end', { pod: candidate.spec.name, podUid: command.podUid,
-      leaseId: command.leaseId, runId: command.runId, attemptId: command.attemptId,
-      releasedAt: new Date(this.now()).toISOString(), result: 'sanitized' });
+    const key = JSON.stringify([command.workerId, command.leaseId, command.generation, command.podUid]);
+    if (!this.quarantineRequested.has(key)) {
+      debug.log('pod.provenance', 'lease-end', { pod: candidate.spec.name, podUid: command.podUid,
+        leaseId: command.leaseId, runId: command.runId, attemptId: command.attemptId,
+        releasedAt: new Date(this.now()).toISOString(), result: 'sanitized' });
+      this.terminalEndLogged.add(key);
+    }
     return 'sanitized';
   }
 
   /** `leaseStarted=false` marks a refusal before `lease-start`, so start/end history stays paired. */
   async quarantine(command: WarmWorkerCommand, reason: string, leaseStarted = true): Promise<void> {
     const key = JSON.stringify([command.workerId, command.leaseId, command.generation, command.podUid]);
+    const podKey = JSON.stringify([command.workerId, command.podUid]);
+    this.quarantiningPods.add(podKey);
     const prior = this.terminalLeases.get(key);
     if (prior) {
-      const outcome = await prior;
-      debug.log('pod.provenance', 'lease-terminal-suppressed', { workerId: command.workerId,
-        leaseId: command.leaseId, generation: command.generation, requested: 'quarantine', outcome });
+      this.quarantineRequested.add(key);
+      const terminal = prior.then(async outcome => {
+        if (outcome === 'sanitized') {
+          await this.quarantineOnce(command, reason, leaseStarted);
+          return 'quarantined' as const;
+        }
+        return outcome;
+      });
+      this.terminalLeases.set(key, terminal);
+      try { await terminal; this.quarantiningPods.delete(podKey); }
+      catch (error) { if (this.terminalLeases.get(key) === terminal) this.terminalLeases.delete(key); throw error; }
       return;
     }
     const terminal = this.quarantineOnce(command, reason, leaseStarted).then(() => 'quarantined' as const);
     this.terminalLeases.set(key, terminal);
-    try { await terminal; }
+    try { await terminal; this.quarantiningPods.delete(podKey); }
     catch (error) { if (this.terminalLeases.get(key) === terminal) this.terminalLeases.delete(key); throw error; }
   }
 
@@ -734,9 +769,11 @@ export class WarmPodProvider {
       if (!cleared.ok && reason !== 'provenance-unavailable') debug.log('pod.provenance', 'lease-patch-refused', { pod: candidate.spec.name,
         podUid: command.podUid, leaseId: command.leaseId, reason: cleared.reason }, { level: 'warn' });
     }
-    debug.log('pod.provenance', leaseStarted ? 'lease-end' : 'lease-refused', { podUid: command.podUid,
+    const key = JSON.stringify([command.workerId, command.leaseId, command.generation, command.podUid]);
+    debug.log('pod.provenance', this.terminalEndLogged.has(key) ? 'lease-late-quarantine' : leaseStarted ? 'lease-end' : 'lease-refused', { podUid: command.podUid,
       leaseId: command.leaseId, runId: command.runId, attemptId: command.attemptId, leaseStarted,
       releasedAt: new Date(this.now()).toISOString(), result: 'quarantined', reason });
+    this.terminalEndLogged.add(key);
   }
 
 
@@ -751,6 +788,10 @@ export function loadWarmPodProvider(path: string, pairingGateFile?: string): War
   const parsed: unknown = JSON.parse(readFileSync(path, 'utf8'));
   if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) throw new Error('invalid warm provider config');
   const options = parsed as WarmPodProviderOptions;
+  // JSON cannot carry a real OS binding: only the unsupported-platform declaration is meaningful here.
+  // Omitting the field keeps this platform's dlopen binding (undefined), so the default path is unchanged.
+  if (options.prepareFlock !== undefined && options.prepareFlock !== null && options.prepareFlock !== false)
+    throw new Error('invalid warm provider prepare flock');
   // The production entry points (--prepare/--reconcile/--serve) bind the 1-file gate key here.
   if (pairingGateFile !== undefined) {
     if (!pairingGateFile.trim()) throw new Error('warm provider pairing gate file must not be empty');