denpex-mcp 0.2.30 → 0.2.31

package/dist/index.js

@@ -1349,8 +1349,8 @@ function lo(e, t = 15e4) {
   Ct(o, 0, 47, 300, n.length), Ct(o, n.length - 96, n.length - 1, 800, n.length);
   let a = -1;
   for (let _ = 0; _ < n.length; _++) {
-    let g = n[_] ?? "", f = dp.test(g), E = np.test(g), C = lp.test(g), y = op.test(g) && ap.test(g);
-    a < 0 && (f || E || C) && (a = _), f ? Ct(o, _ - 3, _ + 3, 1e3, n.length) : C ? Ct(o, _ - 3, _ + 3, 940, n.length) : E && Ct(o, _ - 3, _ + 3, 900, n.length), y && Ct(o, _ - 1, _ + 1, 880, n.length), sp.test(g) && Ct(o, _ - 1, _ + 1, 700, n.length), cp.test(g) && o.set(_, Math.max(650, o.get(_) ?? 0));
+    let g = n[_] ?? "", f = dp.test(g), E = np.test(g), k = lp.test(g), y = op.test(g) && ap.test(g);
+    a < 0 && (f || E || k) && (a = _), f ? Ct(o, _ - 3, _ + 3, 1e3, n.length) : k ? Ct(o, _ - 3, _ + 3, 940, n.length) : E && Ct(o, _ - 3, _ + 3, 900, n.length), y && Ct(o, _ - 1, _ + 1, 880, n.length), sp.test(g) && Ct(o, _ - 1, _ + 1, 700, n.length), cp.test(g) && o.set(_, Math.max(650, o.get(_) ?? 0));
   }
   a >= 0 && Ct(o, a - 6, a + 8, 1100, n.length);
   let s = `${mp} input condensed by Denpex after exceeding ${t.toLocaleString()} characters. Environment, causal, rank, topology, middle, and final-tail evidence were selected.]`;
@@ -1493,9 +1493,9 @@ function Xi(e, t = 3) {
   for (let d of yp) {
     let l = 0, m = [];
     for (let [f, E] of n) {
-      let C = d.weights.get(f);
-      if (C) {
-        let y = E * C;
+      let k = d.weights.get(f);
+      if (k) {
+        let y = E * k;
         l += y, m.push([f, y]);
       }
     }
@@ -1732,8 +1732,8 @@ function Zp(e) {
     return !/^\d+$/.test(A) || !Number.isInteger(w) || w < 0 || w > 255;
   }), g = new Set(u.filter((A) => /^\d+$/.test(A) && Number(A) <= 255).map(Number)), f = [...t.matchAll(/\b(?:node(?:[_-]?name)?|host(?:[_-]?name)?|k8s\.node\.name)\s*[:=]\s*([a-z0-9][a-z0-9._-]{0,127})\b/gi), ...t.matchAll(/\b((?:node|host)-[a-z0-9][a-z0-9._-]{0,127})\b/gi)].map((A) => oi(A[1])).filter(Boolean), E = xo(r);
   E && f.push(oi(E));
-  let C = new Set(f.filter(Boolean)), y = [...t.matchAll(/\b(?:PCI(?:e)?(?:\s+(?:Bus|Bus Id))?|pci[_-]?bus[_-]?id|BDF)\s*[:=]?\s*((?:[0-9a-f]{4,8}:)?[0-9a-f]{2}:[0-9a-f]{2}(?:\.[0-7])?)\b/gi), ...t.matchAll(/\b((?:[0-9a-f]{4,8}:)[0-9a-f]{2}:[0-9a-f]{2}\.[0-7])\b/gi)], T = new Set(y.map((A) => Fo(A[1])).filter(Boolean));
-  return i || s || _ || h.size > 1 || g.size > 1 || C.size > 1 || T.size > 1;
+  let k = new Set(f.filter(Boolean)), y = [...t.matchAll(/\b(?:PCI(?:e)?(?:\s+(?:Bus|Bus Id))?|pci[_-]?bus[_-]?id|BDF)\s*[:=]?\s*((?:[0-9a-f]{4,8}:)?[0-9a-f]{2}:[0-9a-f]{2}(?:\.[0-7])?)\b/gi), ...t.matchAll(/\b((?:[0-9a-f]{4,8}:)[0-9a-f]{2}:[0-9a-f]{2}\.[0-7])\b/gi)], T = new Set(y.map((A) => Fo(A[1])).filter(Boolean));
+  return i || s || _ || h.size > 1 || g.size > 1 || k.size > 1 || T.size > 1;
 }
 function qt(e) {
   return { ...e };
@@ -2094,8 +2094,8 @@ function vu(e) {
       y === 0 ? (t.xidExplicitlyClean = true, t.telemetryPresent = true) : (t.xidCodes.push(y), t.positives.push(`Xid ${f[1]}: ${g.slice(0, 140)}`));
       continue;
     }
-    let C = g.match(/\bLink\s+(\d+)\b/i);
-    if (C && (i === "nvlink" || /nvlink/i.test(g)) && (n = Number(C[1])), i === "nvlink" || /nvlink/i.test(g)) {
+    let k = g.match(/\bLink\s+(\d+)\b/i);
+    if (k && (i === "nvlink" || /nvlink/i.test(g)) && (n = Number(k[1])), i === "nvlink" || /nvlink/i.test(g)) {
       f = g.match(/(?:Correctable|CRC|Replay|Recovery)\s*(?:Errors?|Count(?:er)?)?\s*:\s*([\d,]+)/i);
       let y = /uncorrectable/i.test(g);
       if (f && !y) {
@@ -2179,9 +2179,9 @@ function vu(e) {
   }
   for (let g of r) {
     if (!/(double[\s-]?bit|\bdbe\b|uncorrectable|contained ecc|unhomed ecc|uncontained ecc|row[\s-]?remap(?:ping)?|retired pages?)/i.test(g) || /(?:double[\s-]?bit|\bdbe\b|uncorrectable)[^;\n]{0,32}(?:ECC[^;\n]{0,16})?[:=]\s*0\b/i.test(g) || /:\s*[\d,]+\s*$/.test(g) || /ECC\s+(?:Correctable|Uncorrectable)\s*:\s*[\d,]+\s*$/i.test(g) || /\bno\b|\bnone\b|\b0\s*(?:errors?|events?)\b/i.test(g) || /errors?\s*:\s*$/i.test(g)) continue;
-    let f = [], E = /(double[\s-]?bit|\bdbe\b|uncorrectable|contained ecc|unhomed ecc|uncontained ecc|row[\s-]?remap(?:ping)?|retired pages?)/gi, C;
-    for (; (C = E.exec(g)) !== null; ) {
-      let T = g.slice(C.index + C[0].length, C.index + C[0].length + 32).match(/^[^\d\n]{0,24}?([\d,]+)/);
+    let f = [], E = /(double[\s-]?bit|\bdbe\b|uncorrectable|contained ecc|unhomed ecc|uncontained ecc|row[\s-]?remap(?:ping)?|retired pages?)/gi, k;
+    for (; (k = E.exec(g)) !== null; ) {
+      let T = g.slice(k.index + k[0].length, k.index + k[0].length + 32).match(/^[^\d\n]{0,24}?([\d,]+)/);
       T && f.push(Number(T[1].replace(/,/g, "")));
     }
     if (!(f.length > 0 && f.every((y) => y === 0))) {
@@ -2210,7 +2210,7 @@ function vu(e) {
     t.ibPortClean = g && t.ibPortActive !== false, t.ibPortClean ? t.negatives.push("IB port counters clean (symbol errors 0, link downs 0, retransmits 0, port ACTIVE)") : t.positives.push(`IB port counters elevated: symbol=${t.ibSymbolErrors}, linkDown=${t.ibLinkDownEvents}, retrans=${t.ibRetransmits}`);
   }
   if (t.checksumMismatch && (t.telemetryPresent = true, t.positives.push(`payload checksum mismatch during collective${t.checksumMismatch.bitsDiff !== null ? ` (${t.checksumMismatch.bitsDiff}-bit difference)` : ""}: ${t.checksumMismatch.context}`)), t.eccCorrectableByGpu.length > 0) {
-    let g = [...t.eccCorrectableByGpu].sort((E, C) => C.count - E.count);
+    let g = [...t.eccCorrectableByGpu].sort((E, k) => k.count - E.count);
     t.eccCorrectableMax = g[0].count;
     let f = g[1]?.count ?? 0;
     (g[0].count >= 1e5 || g[0].count >= 5e4 && g[0].count >= f * 10) && (t.eccCorrectableStorm = true, t.eccCorrectableStormGpu = g[0].gpu, t.positives.push(`corrected-ECC storm on GPU ${g[0].gpu ?? "?"}: ${g[0].count.toLocaleString()} correctable errors${f > 0 ? ` (peers \u2264 ${f.toLocaleString()})` : ""}. HBM degrading, a DBE precursor; drain-worthy even when not the incident's root cause`));
@@ -2219,16 +2219,16 @@ function vu(e) {
   for (let g = 0; g < r.length; g++) for (let f of p) {
     if (!f.re.test(r[g])) continue;
     let E = null;
-    for (let C = g; C <= Math.min(g + 6, r.length - 1); C++) {
-      if (h.test(r[C])) {
+    for (let k = g; k <= Math.min(g + 6, r.length - 1); k++) {
+      if (h.test(r[k])) {
         E = "moved";
         break;
       }
-      if (u.test(r[C])) {
+      if (u.test(r[k])) {
         E = "resolved";
         break;
       }
-      if (_.test(r[C])) {
+      if (_.test(r[k])) {
         E = "persisted";
         break;
       }
@@ -2280,17 +2280,17 @@ function Ko(e) {
   return { p50: +qo(t, 0.5).toFixed(1), p95: +qo(t, 0.95).toFixed(1) };
 }
 function Wo(e, t = Pr) {
-  let r = Array.isArray(e) ? e.filter((k) => k && typeof k == "object") : [], i = { ttftMs: Number.isFinite(t?.ttftMs) && t.ttftMs > 0 ? t.ttftMs : Pr.ttftMs, itlMs: Number.isFinite(t?.itlMs) && t.itlMs > 0 ? t.itlMs : Pr.itlMs }, n = mr(r, "ttftMs"), o = mr(r, "itlMs"), a = mr(r, "queueDepth"), s = mr(r, "kvCacheUsagePct"), c = mr(r, "preemptions"), d = mr(r, "runningReqs"), l = Ko(n), m = Ko(o), p = { verdict: "insufficient_data", bottleneck: "unknown", samples: r.length, ttftP50: l.p50, ttftP95: l.p95, itlP50: m.p50, itlP95: m.p95, ttftBreachRate: null, itlBreachRate: null, maxQueueDepth: a.length ? Math.max(...a) : null, maxKvUsagePct: s.length ? Math.max(...s) : null, totalPreemptions: c.length ? c.reduce((k, D) => k + D, 0) : null, wastedWorkRatio: null, findings: [], action: null, targets: i };
+  let r = Array.isArray(e) ? e.filter((C) => C && typeof C == "object") : [], i = { ttftMs: Number.isFinite(t?.ttftMs) && t.ttftMs > 0 ? t.ttftMs : Pr.ttftMs, itlMs: Number.isFinite(t?.itlMs) && t.itlMs > 0 ? t.itlMs : Pr.itlMs }, n = mr(r, "ttftMs"), o = mr(r, "itlMs"), a = mr(r, "queueDepth"), s = mr(r, "kvCacheUsagePct"), c = mr(r, "preemptions"), d = mr(r, "runningReqs"), l = Ko(n), m = Ko(o), p = { verdict: "insufficient_data", bottleneck: "unknown", samples: r.length, ttftP50: l.p50, ttftP95: l.p95, itlP50: m.p50, itlP95: m.p95, ttftBreachRate: null, itlBreachRate: null, maxQueueDepth: a.length ? Math.max(...a) : null, maxKvUsagePct: s.length ? Math.max(...s) : null, totalPreemptions: c.length ? c.reduce((C, D) => C + D, 0) : null, wastedWorkRatio: null, findings: [], action: null, targets: i };
   if (r.length < Qe.MIN_SAMPLES) return { ...p, findings: [`Only ${r.length} serving sample(s); need ${Qe.MIN_SAMPLES} before any SLO or bottleneck claim is meaningful.`] };
   if (n.length === 0 && o.length === 0) return { ...p, findings: ["No TTFT or ITL values in the telemetry. Enable per-request latency reporting on the engine (vLLM: the metrics endpoint; SGLang: --enable-metrics), queue depth alone cannot establish an SLO breach."] };
-  let h = n.length ? +(n.filter((k) => k > i.ttftMs).length / n.length).toFixed(3) : null, u = o.length ? +(o.filter((k) => k > i.itlMs).length / o.length).toFixed(3) : null, _ = [], g = p.maxQueueDepth, f = p.maxKvUsagePct, E = p.totalPreemptions, C = "none", y = null, T = g !== null && g > Qe.QUEUE_DOMINANT, A = f !== null && f >= Qe.KV_SATURATED_PCT, w = E !== null && E >= Qe.PREEMPTION_MATERIAL * r.length, S = null;
+  let h = n.length ? +(n.filter((C) => C > i.ttftMs).length / n.length).toFixed(3) : null, u = o.length ? +(o.filter((C) => C > i.itlMs).length / o.length).toFixed(3) : null, _ = [], g = p.maxQueueDepth, f = p.maxKvUsagePct, E = p.totalPreemptions, k = "none", y = null, T = g !== null && g > Qe.QUEUE_DOMINANT, A = f !== null && f >= Qe.KV_SATURATED_PCT, w = E !== null && E >= Qe.PREEMPTION_MATERIAL * r.length, O = null;
   if (E !== null && d.length > 0) {
-    let k = d.reduce((D, x) => D + x, 0);
-    S = k > 0 ? +Math.min(1, E / k).toFixed(3) : null;
+    let C = d.reduce((D, x) => D + x, 0);
+    O = C > 0 ? +Math.min(1, E / C).toFixed(3) : null;
   }
-  w && A ? (C = "preemption", _.push(`Preemption thrash: ${E} preemption(s) across ${r.length} samples with KV cache peaking at ${f}%. Every preemption discards computed KV state and re-prefills that sequence later, so the GPU stays busy producing tokens it already produced, utilisation and raw throughput both look healthy while goodput falls.` + (S !== null ? ` Estimated wasted decode capacity: ${Math.round(S * 100)}% (lower bound).` : "")), y = "Reduce admitted concurrency (--max-num-seqs / --max-running-requests) so the working set fits in KV cache. Admitting FEWER requests raises completed-tokens-per-second here, because the re-prefill waste disappears. Scaling out without this fix carries the waste onto every new replica.") : T ? (C = "queue", _.push(`Queue-dominated latency: queue depth peaked at ${g} while TTFT p95 is ${p.ttftP95 ?? "unreported"}ms against a ${i.ttftMs}ms budget. TTFT is queue wait plus prefill compute, and at this depth the queue term dominates, prefill tuning (chunked prefill, prefix caching) will produce no measurable improvement.`), y = "Shed load at the gateway with an explicit queue bound and a 429, then scale replicas out. Do not raise the concurrency ceiling to absorb the queue: that converts queueing delay into per-token slowdown for every in-flight request and usually raises p99 further.") : u !== null && u > Qe.BREACH_FRACTION ? (C = "decode", _.push(`Decode-bound: ITL p95 is ${p.itlP95}ms against a ${i.itlMs}ms budget with ${Math.round(u * 100)}% of samples breaching, and the queue is not deep (peak ${g ?? "unreported"}). Decode is memory-bandwidth bound, so per-token latency rises with batch size, aggregate throughput can be climbing while every individual stream degrades.`), y = "Cap --max-num-seqs at the concurrency where ITL stays inside your budget and scale out for volume. If ITL degraded as a step rather than a curve, check for a CUDA-graph ceiling (batches above the captured maximum fall back to eager) before treating it as a bandwidth limit.") : h !== null && h > Qe.BREACH_FRACTION && (C = "prefill", _.push(`Prefill-bound: TTFT p95 is ${p.ttftP95}ms against a ${i.ttftMs}ms budget with ${Math.round(h * 100)}% of samples breaching, but the queue is shallow (peak ${g ?? "unreported"}). The time is going into prefill compute itself, not into waiting.`), y = "Enable prefix caching and confirm it is actually hitting (send an identical long prefix twice and compare TTFT, no improvement means it is inactive whatever the flag says). Volatile fields such as timestamps or request ids inside the system prompt defeat prefix caching entirely and are the most common cause. Enable chunked prefill so long prompts do not block the decode batch."), A && C !== "preemption" && _.push(`KV cache peaked at ${f}%, at or above ${Qe.KV_SATURATED_PCT}% the scheduler cannot admit a new sequence without evicting one, so preemption is imminent even if none has been recorded yet.`);
-  let b = Math.max(h ?? 0, u ?? 0), O;
-  return b > Qe.BREACH_FRACTION || C === "preemption" ? O = "breaching" : b > 0 || A || T ? O = "at_risk" : O = "healthy", O === "healthy" && _.push(`Within SLO: TTFT p95 ${p.ttftP95 ?? "n/a"}ms (budget ${i.ttftMs}ms), ITL p95 ${p.itlP95 ?? "n/a"}ms (budget ${i.itlMs}ms), queue peak ${g ?? "n/a"}, KV peak ${f ?? "n/a"}%.`), { ...p, verdict: O, bottleneck: C, ttftBreachRate: h, itlBreachRate: u, wastedWorkRatio: S, findings: _, action: y };
+  w && A ? (k = "preemption", _.push(`Preemption thrash: ${E} preemption(s) across ${r.length} samples with KV cache peaking at ${f}%. Every preemption discards computed KV state and re-prefills that sequence later, so the GPU stays busy producing tokens it already produced, utilisation and raw throughput both look healthy while goodput falls.` + (O !== null ? ` Estimated wasted decode capacity: ${Math.round(O * 100)}% (lower bound).` : "")), y = "Reduce admitted concurrency (--max-num-seqs / --max-running-requests) so the working set fits in KV cache. Admitting FEWER requests raises completed-tokens-per-second here, because the re-prefill waste disappears. Scaling out without this fix carries the waste onto every new replica.") : T ? (k = "queue", _.push(`Queue-dominated latency: queue depth peaked at ${g} while TTFT p95 is ${p.ttftP95 ?? "unreported"}ms against a ${i.ttftMs}ms budget. TTFT is queue wait plus prefill compute, and at this depth the queue term dominates, prefill tuning (chunked prefill, prefix caching) will produce no measurable improvement.`), y = "Shed load at the gateway with an explicit queue bound and a 429, then scale replicas out. Do not raise the concurrency ceiling to absorb the queue: that converts queueing delay into per-token slowdown for every in-flight request and usually raises p99 further.") : u !== null && u > Qe.BREACH_FRACTION ? (k = "decode", _.push(`Decode-bound: ITL p95 is ${p.itlP95}ms against a ${i.itlMs}ms budget with ${Math.round(u * 100)}% of samples breaching, and the queue is not deep (peak ${g ?? "unreported"}). Decode is memory-bandwidth bound, so per-token latency rises with batch size, aggregate throughput can be climbing while every individual stream degrades.`), y = "Cap --max-num-seqs at the concurrency where ITL stays inside your budget and scale out for volume. If ITL degraded as a step rather than a curve, check for a CUDA-graph ceiling (batches above the captured maximum fall back to eager) before treating it as a bandwidth limit.") : h !== null && h > Qe.BREACH_FRACTION && (k = "prefill", _.push(`Prefill-bound: TTFT p95 is ${p.ttftP95}ms against a ${i.ttftMs}ms budget with ${Math.round(h * 100)}% of samples breaching, but the queue is shallow (peak ${g ?? "unreported"}). The time is going into prefill compute itself, not into waiting.`), y = "Enable prefix caching and confirm it is actually hitting (send an identical long prefix twice and compare TTFT, no improvement means it is inactive whatever the flag says). Volatile fields such as timestamps or request ids inside the system prompt defeat prefix caching entirely and are the most common cause. Enable chunked prefill so long prompts do not block the decode batch."), A && k !== "preemption" && _.push(`KV cache peaked at ${f}%, at or above ${Qe.KV_SATURATED_PCT}% the scheduler cannot admit a new sequence without evicting one, so preemption is imminent even if none has been recorded yet.`);
+  let b = Math.max(h ?? 0, u ?? 0), S;
+  return b > Qe.BREACH_FRACTION || k === "preemption" ? S = "breaching" : b > 0 || A || T ? S = "at_risk" : S = "healthy", S === "healthy" && _.push(`Within SLO: TTFT p95 ${p.ttftP95 ?? "n/a"}ms (budget ${i.ttftMs}ms), ITL p95 ${p.itlP95 ?? "n/a"}ms (budget ${i.itlMs}ms), queue peak ${g ?? "n/a"}, KV peak ${f ?? "n/a"}%.`), { ...p, verdict: S, bottleneck: k, ttftBreachRate: h, itlBreachRate: u, wastedWorkRatio: O, findings: _, action: y };
 }
 function Xo(e) {
   let t = typeof e == "string" ? e : "", r = [];
@@ -2416,8 +2416,8 @@ function Qo(e) {
   if (!R(t, /\bRoCE\b|\bPFC\b|\bECN\b|\bDCQCN\b|priority\s*flow\s*control/i)) return Xt(r, i);
   let o = Re(t, /rx_pause_ctrl_phy\s*[:=]?\s*(\d[\d,]*)/i), a = Re(t, /tx_pause_ctrl_phy\s*[:=]?\s*(\d[\d,]*)/i), s = Re(t, /rx_prio\d+_pause_duration\s*[:=]?\s*(\d[\d,]*)/i), c = Re(t, /np_cnp_sent\s*[:=]?\s*(\d[\d,]*)/i), d = Re(t, /rp_cnp_handled\s*[:=]?\s*(\d[\d,]*)/i), l = (o ?? 0) > 1e3 || (a ?? 0) > 1e3 || (s ?? 0) > 1e3, m = (c ?? 0) > 100 || (d ?? 0) > 100;
   if (!l && !m) return Xt(r, i);
-  let p = Re(t, /busbw[^0-9]*([0-9]+(?:\.[0-9]+)?)\s*GB\/s/i), h = Re(t, /baseline\s*([0-9]+(?:\.[0-9]+)?)\s*GB\/s/i), u = t.match(/rx_prio(\d+)_pause_duration/i)?.[1] || t.match(/(?:PFC\s+)?priority\s*(\d+)/i)?.[1], _ = t.match(/\b(mlx5_\d+|ib\d+|eth\d+)\b/i)?.[1] || "the RoCE interface", g = li(t), f = g ? ` on ${g}` : "", E = P(t, /rx_pause_ctrl_phy|tx_pause_ctrl_phy|pause_duration|np_cnp_sent|rp_cnp_handled|RoCE\s*v?\d?\s*enabled|busbw/i, 5), C = [o !== null ? `${o} received pause frames` : "", s !== null ? `${s} us of ${u ? `priority-${u} ` : ""}pause` : "", c !== null ? `${c} ECN congestion notifications sent` : ""].filter(Boolean).join(", "), y = p !== null ? `busbw sits at ${p} GB/s${h !== null ? ` against a ${h} GB/s baseline` : ""}` : "collective throughput has collapsed";
-  return { id: r, title: i, detected: true, severity: "critical", summary: `The RoCE fabric is applying backpressure to this host${f}: ${C}. That is why ${y}. Congestion control is working as designed, this is a queueing and buffer-tuning problem on the fabric, not a failed link, cable or transceiver.`, evidence: E, action: `Treat this as congestion, not a fault. In order:
+  let p = Re(t, /busbw[^0-9]*([0-9]+(?:\.[0-9]+)?)\s*GB\/s/i), h = Re(t, /baseline\s*([0-9]+(?:\.[0-9]+)?)\s*GB\/s/i), u = t.match(/rx_prio(\d+)_pause_duration/i)?.[1] || t.match(/(?:PFC\s+)?priority\s*(\d+)/i)?.[1], _ = t.match(/\b(mlx5_\d+|ib\d+|eth\d+)\b/i)?.[1] || "the RoCE interface", g = li(t), f = g ? ` on ${g}` : "", E = P(t, /rx_pause_ctrl_phy|tx_pause_ctrl_phy|pause_duration|np_cnp_sent|rp_cnp_handled|RoCE\s*v?\d?\s*enabled|busbw/i, 5), k = [o !== null ? `${o} received pause frames` : "", s !== null ? `${s} us of ${u ? `priority-${u} ` : ""}pause` : "", c !== null ? `${c} ECN congestion notifications sent` : ""].filter(Boolean).join(", "), y = p !== null ? `busbw sits at ${p} GB/s${h !== null ? ` against a ${h} GB/s baseline` : ""}` : "collective throughput has collapsed";
+  return { id: r, title: i, detected: true, severity: "critical", summary: `The RoCE fabric is applying backpressure to this host${f}: ${k}. That is why ${y}. Congestion control is working as designed, this is a queueing and buffer-tuning problem on the fabric, not a failed link, cable or transceiver.`, evidence: E, action: `Treat this as congestion, not a fault. In order:
 1. Confirm where the backpressure originates: \`ethtool -S ${_} | grep -E 'pause|prio'\` and \`mlnx_qos -i <netdev>\` to read the per-priority pause configuration${f}. Received pause frames mean the switch is telling this host to stop transmitting.
 2. On the switch, review the PFC headroom and buffer watermarks for ${u ? `priority ${u}` : "the RoCE priority"} and the ECN/DCQCN marking thresholds. ${c !== null && d !== null ? `${c} CNPs sent against ${d} handled says ECN is active and the senders are not backing off enough.` : "ECN marking that is too late leaves PFC as the only brake, which is what produces sustained pause."}
 3. Find the traffic pattern driving it, incast from a many-to-one collective, or a QoS class sharing the same queue as the RDMA traffic. Separate the classes if they are sharing.
@@ -2539,7 +2539,7 @@ function aa(e) {
 function sa(e) {
   let t = String(e || ""), r = "pcie-link-downgrade", i = "The PCIe link negotiated down and is throttling this GPU", n = me(t, /PCIe\s+Generation[^\n]*?Max\s*:?\s*(\d+)/i), o = me(t, /PCIe\s+Generation[^\n]*?Current\s*:?\s*(\d+)/i), a = me(t, /Link\s+Width[^\n]*?Max\s*:?\s*(\d+)x/i), s = me(t, /Link\s+Width[^\n]*?Current\s*:?\s*(\d+)x/i), c = /LnkSta:[^\n]*\(downgraded\)/i.test(t), d = n !== null && o !== null && o < n, l = a !== null && s !== null && s < a;
   if (!d && !l && !c) return pr(r, i);
-  let m = [...t.matchAll(/([0-9]+(?:\.[0-9]+)?)\s*GB\/s/gi)].map((C) => Number(C[1])), p = m.length ? Math.min(...m) : null, h = m.length ? Math.max(...m) : null, u = R(t, /Physical Layer|PCIe Bus Error|LnkSta[^\n]*downgraded/i), _ = t.match(/\b([0-9a-f]{2,4}:[0-9a-f]{2}\.[0-9a-f])\b/i)?.[1], g = t.match(/-i\s+(\d+)|GPU\s*(\d+)/i), f = Lr(t), E = f ? ` on ${f}` : "";
+  let m = [...t.matchAll(/([0-9]+(?:\.[0-9]+)?)\s*GB\/s/gi)].map((k) => Number(k[1])), p = m.length ? Math.min(...m) : null, h = m.length ? Math.max(...m) : null, u = R(t, /Physical Layer|PCIe Bus Error|LnkSta[^\n]*downgraded/i), _ = t.match(/\b([0-9a-f]{2,4}:[0-9a-f]{2}\.[0-9a-f])\b/i)?.[1], g = t.match(/-i\s+(\d+)|GPU\s*(\d+)/i), f = Lr(t), E = f ? ` on ${f}` : "";
   return { id: r, title: i, detected: true, severity: "critical", summary: `This is a physical link fault${E}, not a tuning problem. ${d ? `The link negotiated down to PCIe Gen${o} against a Gen${n} maximum` : "The link negotiated down"}${l ? ` and to x${s} of x${a} width` : ""}${u ? ", with corrected physical-layer errors on the port" : ""}. ${p !== null && h !== null && h > p ? `That is why this device moves ${p} GB/s against ${h} GB/s on its peers on the same host.` : "That is why this device is far slower than its peers on the same host."} A downgraded link does not crash and does not raise an Xid, it bleeds throughput quietly, so clean ECC and a passing diagnostic do not exonerate it.`, evidence: P(t, /PCIe Generation|Link Width|LnkSta|Physical Layer|PCIe Bus Error|GB\/s/i, 5), action: `Service the physical path, this will not be fixed by a setting:
 1. Schedule the node out of service, power it fully down (a warm reboot does not re-seat anything), then reseat the GPU${g ? ` (GPU ${g[1] || g[2]})` : ""} and its riser/cable${_ ? `, at ${_}` : ""}. Inspect the slot and the connector for debris or bent pins.
 2. If it still negotiates low, move the card to a known-good slot. If the fault follows the card, replace the card; if it stays with the slot, service the riser or the board.
@@ -2613,8 +2613,8 @@ function pa(e) {
 function ua(e) {
   let t = String(e || ""), r = "mixed-gpu-sku", i = "The job spans two GPU models, so the slower one sets the pace", n = new Set([...t.matchAll(/\b((?:H100|H200|A100|A800|V100|L40S?|B200|GB200|RTX\s*\d{4}|T4))\b/gi)].map((h) => h[1].toUpperCase().replace(/\s+/g, ""))), o = new Set([...t.matchAll(/(?:A100|H100|H200|V100|L40S?|B200|GB200|RTX\s*\d{4}|T4)[A-Za-z0-9-]*?-(\d{2,3})GB\b/gi)].map((h) => Number.parseInt(h[1], 10)).filter((h) => Number.isFinite(h) && h > 0));
   if (o.size >= 2) {
-    let h = [...o].sort((C, y) => C - y), u = h[0], _ = h[h.length - 1], g = /out of memory|OutOfMemoryError|CUDA_ERROR_OUT_OF_MEMORY/i.test(t);
-    return { id: "mixed-gpu-capacity", title: "The job spans GPUs of different memory sizes, so the smallest one sets the limit", detected: true, severity: "warning", confidence: g ? 88 : 76, causalClass: "paired-causal-owner", causalDiscriminator: true, summary: `This allocation mixes ${h.map((C) => `${C}GB`).join(" and ")} cards of the same family, and in data-parallel training every rank receives the same per-rank batch. The job is therefore limited by the ${u}GB card, not by the ${_}GB ones.
+    let h = [...o].sort((k, y) => k - y), u = h[0], _ = h[h.length - 1], g = /out of memory|OutOfMemoryError|CUDA_ERROR_OUT_OF_MEMORY/i.test(t);
+    return { id: "mixed-gpu-capacity", title: "The job spans GPUs of different memory sizes, so the smallest one sets the limit", detected: true, severity: "warning", confidence: g ? 88 : 76, causalClass: "paired-causal-owner", causalDiscriminator: true, summary: `This allocation mixes ${h.map((k) => `${k}GB`).join(" and ")} cards of the same family, and in data-parallel training every rank receives the same per-rank batch. The job is therefore limited by the ${u}GB card, not by the ${_}GB ones.
 
 ${g ? "That is why the out-of-memory error names the smaller card while the larger ones are fine. It is not a model that suddenly needs more memory, and it will recur on exactly that rank every run." : "Expect the smaller card to be the first to run out of memory, and to do so on the same rank every run."}
 
@@ -3644,10 +3644,10 @@ function js(e, t) {
   _ === null ? n.push("GPU utilization") : _ < 50 && (o.WORKLOAD_IMBALANCE += 40, r.push(`GPU utilization on the suspect rank is ${_}%, so the device is waiting for work rather than being slowed.`));
   let g = Ur(a.eccUncorrectable), f = Ur(a.xidCount);
   g === "positive" ? r.push(`Uncorrectable ECC errors present (${a.eccUncorrectable}).`) : g === "clean" ? i.push("Uncorrectable ECC counters read zero, which is evidence against failing memory.") : n.push("ECC counters"), f === "positive" ? r.push(`Xid events recorded (${a.xidCount}).`) : f === "clean" ? i.push("No Xid events were recorded during the window.") : n.push("Xid events");
-  let E = Object.entries(o).filter(([w]) => w !== "UNEXPLAINED").sort((w, S) => S[1] - w[1]), [C, y] = E[0];
+  let E = Object.entries(o).filter(([w]) => w !== "UNEXPLAINED").sort((w, O) => O[1] - w[1]), [k, y] = E[0];
   if (y === 0) return { cause: "UNEXPLAINED", confidence: 0, supporting: r, exonerating: i, notMeasured: n };
-  let A = C === "CLOCK_THROTTLING" || C === "NETWORK_DEGRADATION" ? Math.min(25, i.length * 8) : 0;
-  return { cause: C, confidence: Math.max(0, Math.min(95, y - A)), supporting: r, exonerating: i, notMeasured: n };
+  let A = k === "CLOCK_THROTTLING" || k === "NETWORK_DEGRADATION" ? Math.min(25, i.length * 8) : 0;
+  return { cause: k, confidence: Math.max(0, Math.min(95, y - A)), supporting: r, exonerating: i, notMeasured: n };
 }
 var nn = "throughput-regression-paste";
 var on = "Sustained throughput regression";
@@ -3716,7 +3716,7 @@ This is a real, sustained regression, not a spike: ${m.length} steps ran at a me
 WHAT THIS DOES NOT ESTABLISH. It does not say which node, rank or GPU is responsible, and it does not name a cause. The pasted log carries ${t.hasRankAttribution ? "a single rank" : "no per-rank attribution"} and none of these counters: ${u.absent.join(", ")}. Clock throttling, fabric degradation, collective desync and workload imbalance all present exactly like this at the run level, they need different fixes, and nothing here separates them. Naming one would be a guess dressed as a finding.
 
 The step number is the useful part right now: something changed at step ${l}. If you know what happened around then (a node replaced, a config change, another job landing on the same nodes, a checkpoint resumed), that is the strongest lead you have.`, action: vh, fallbackAction: `If the job cannot be re-run with instrumentation, compare step ${l} against your scheduler and cluster history: node allocation changes, a co-scheduled job starting, a driver or container image change, or a checkpoint resume at that step. A regression that begins at an exact step usually has an event at that step.`, verificationCommand: "nvidia-smi --query-gpu=index,clocks.sm,clocks.max.sm,clocks_throttle_reasons.active --format=csv -l 1", evidence: h, resumeFrom: null };
-  let g = t.samples.filter((w) => w.step >= l), f = Xs(g), E = g.filter((w) => w.rank === f.rank), C = g.filter((w) => w.rank !== f.rank), y = js(E, C), T = [f.gpuUuid ? `GPU ${f.gpuUuid}` : null, f.node ? `node ${f.node}` : null, f.rank !== null ? `rank ${f.rank}` : null].filter(Boolean).join(", ");
+  let g = t.samples.filter((w) => w.step >= l), f = Xs(g), E = g.filter((w) => w.rank === f.rank), k = g.filter((w) => w.rank !== f.rank), y = js(E, k), T = [f.gpuUuid ? `GPU ${f.gpuUuid}` : null, f.node ? `node ${f.node}` : null, f.rank !== null ? `rank ${f.rank}` : null].filter(Boolean).join(", ");
   h.push(`${t.rankCount} ranks compared. Slowest is rank ${f.rank} at ${f.slowestRankStepTimeMs}ms against a peer median of ${f.peerMedianStepTimeMs}ms.`), h.push(`Localization reached: ${f.depth}.`), y.supporting.length && h.push(`Supporting: ${y.supporting.join("; ")}.`), y.exonerating.length && h.push(`Measured and clean: ${y.exonerating.join("; ")}.`), y.notMeasured.length && h.push(`Not measured: ${y.notMeasured.join("; ")}.`);
   let A = y.cause !== "UNEXPLAINED" && y.confidence > 0;
   return { id: nn, title: on, detected: true, severity: "warning", confidence: A ? Math.min(88, 60 + y.confidence) : 74, causalClass: A ? "contracted-subsystem" : "terminal-symptom", causalDiscriminator: A, summary: `Training slowed by ${a.slowdownPercent}% at step ${l}. The slowest rank is ${T || `rank ${f.rank}`}.
@@ -4147,8 +4147,8 @@ function Nh(e) {
   for (let p of e.split(/\r?\n/)) {
     let h = p.trim(), u = h.match(/\brank\s*[:=]?\s*(\d+)/i) || h.match(/\[rank\s*(\d+)\]/i);
     if (!u) continue;
-    let _ = Number(u[1]), g = h.match(/\bstep\s*[:=]?\s*(\d+)/i) || h.match(/\biter(?:ation)?\s*[:=]?\s*(\d+)/i), f = g ? g[1] : "*", E = h.match(/(?:weight|param(?:eter)?|model|state)?[_ ]?(?:hash|checksum|md5|sha\d*|crc\d*)\s*[:=]\s*(0x[0-9a-f]+|[0-9a-f]{4,}|\d+)/i), C = h.match(/(?:reduced|global|all[- ]?reduced|synced|avg)[_ ]?grad(?:ient)?[_ ]?norm\s*[:=]?\s*([0-9]+(?:\.[0-9]+)?)/i);
-    (E || C) && r.push({ rank: _, step: f, line: h.slice(0, 200), hash: E ? E[1].toLowerCase() : void 0, grad: C ? Number(C[1]) : void 0 });
+    let _ = Number(u[1]), g = h.match(/\bstep\s*[:=]?\s*(\d+)/i) || h.match(/\biter(?:ation)?\s*[:=]?\s*(\d+)/i), f = g ? g[1] : "*", E = h.match(/(?:weight|param(?:eter)?|model|state)?[_ ]?(?:hash|checksum|md5|sha\d*|crc\d*)\s*[:=]\s*(0x[0-9a-f]+|[0-9a-f]{4,}|\d+)/i), k = h.match(/(?:reduced|global|all[- ]?reduced|synced|avg)[_ ]?grad(?:ient)?[_ ]?norm\s*[:=]?\s*([0-9]+(?:\.[0-9]+)?)/i);
+    (E || k) && r.push({ rank: _, step: f, line: h.slice(0, 200), hash: E ? E[1].toLowerCase() : void 0, grad: k ? Number(k[1]) : void 0 });
   }
   let i = sc(r, (p) => p.hash, (p, h) => p !== h), n = sc(r, (p) => p.grad, (p, h) => {
     let u = Number(p), _ = Number(h), g = Math.max(Math.abs(u), Math.abs(_), 1e-9);
@@ -4177,10 +4177,10 @@ function xh(e) {
 }
 function Uh(e) {
   let t = G(e, /Watchdog caught collective operation timeout|work.*timed out after|NCCL.*timeout|ProcessGroupNCCL.*timeout/i), r = G(e, /rank\s*\d+.*(?:stuck|slow|behind|not responding|lagging)/i), i = String.raw`(?:tokens?|samples?|steps?|images?|seq(?:uences?)?)\s*\/\s*s(?:ec(?:ond)?s?)?|it\s*\/\s*s|tflops?`, n = new RegExp(String.raw`(\d+(?:\.\d+)?)\s*(?:${i})\b`, "i"), o = new RegExp(String.raw`(?:${i})\s*[:=]?\s*(\d+(?:\.\d+)?)\b`, "i"), a = /(\d+(?:\.\d+)?)\s*(ms|milliseconds?|s|sec|secs|seconds?|us|µs|microseconds?|min|minutes?)\b/i, s = /* @__PURE__ */ new Map(), c = /* @__PURE__ */ new Map(), d = /* @__PURE__ */ new Map();
-  for (let k of e.split(/\r?\n/)) {
-    let D = k.match(/\brank\s*[:=#]?\s*(\d{1,5})\b/i);
+  for (let C of e.split(/\r?\n/)) {
+    let D = C.match(/\brank\s*[:=#]?\s*(\d{1,5})\b/i);
     if (!D) continue;
-    let x = Number(D[1]), B = k.replace(D[0], " "), z11 = k.trim().slice(0, 200);
+    let x = Number(D[1]), B = C.replace(D[0], " "), z11 = C.trim().slice(0, 200);
     d.has(x) || d.set(x, z11);
     let oe = B.match(n)?.[1] ?? B.match(o)?.[1];
     if (oe !== void 0) {
@@ -4195,34 +4195,34 @@ function Uh(e) {
     }
   }
   let l = /* @__PURE__ */ new Map();
-  if (/\b(?:relative|individual)\s+(?:section\s+|GPU\s+)?performance\s+scores?\b|\bperformance\s+scores?\s*:/i.test(e)) for (let k of e.matchAll(/\b(\d{1,5})\s*:\s*(0?\.\d+|1(?:\.0+)?)\s*[,}\n]/g)) {
-    let D = Number(k[1]), x = Number(k[2]);
+  if (/\b(?:relative|individual)\s+(?:section\s+|GPU\s+)?performance\s+scores?\b|\bperformance\s+scores?\s*:/i.test(e)) for (let C of e.matchAll(/\b(\d{1,5})\s*:\s*(0?\.\d+|1(?:\.0+)?)\s*[,}\n]/g)) {
+    let D = Number(C[1]), x = Number(C[2]);
     !Number.isFinite(x) || x <= 0 || x > 1 || (l.has(D) || l.set(D, []), l.get(D).push(x), d.has(D) || d.set(D, `rank ${D} performance score ${x}`));
   }
-  let m = (k) => {
-    let D = [...k].sort((B, z11) => B - z11), x = Math.floor(D.length / 2);
+  let m = (C) => {
+    let D = [...C].sort((B, z11) => B - z11), x = Math.floor(D.length / 2);
     return D.length % 2 ? D[x] : (D[x - 1] + D[x]) / 2;
-  }, p = (k) => [...k.entries()].map(([D, x]) => ({ rank: D, value: m(x) })), h = null, u = null, _ = null, g = p(s), f = p(c), E = p(l);
+  }, p = (C) => [...C.entries()].map(([D, x]) => ({ rank: D, value: m(x) })), h = null, u = null, _ = null, g = p(s), f = p(c), E = p(l);
   if (E.length >= 3) {
-    let k = E.reduce((x, B) => B.value < x.value ? B : x), D = E.reduce((x, B) => B.value > x.value ? B : x).value;
-    D > 0 && k.value * 2.5 < D && (_ = { rank: k.rank, score: k.value, line: d.get(k.rank) ?? "", best: D });
+    let C = E.reduce((x, B) => B.value < x.value ? B : x), D = E.reduce((x, B) => B.value > x.value ? B : x).value;
+    D > 0 && C.value * 2.5 < D && (_ = { rank: C.rank, score: C.value, line: d.get(C.rank) ?? "", best: D });
   }
   if (g.length >= 3) {
-    let k = g.reduce((B, z11) => z11.value > B.value ? z11 : B), D = g.filter((B) => B.rank !== k.rank).map((B) => B.value), x = m(D);
-    x > 0 && k.value > x * 2.5 && (h = { rank: k.rank, ms: k.value, line: d.get(k.rank) ?? "", median: x });
+    let C = g.reduce((B, z11) => z11.value > B.value ? z11 : B), D = g.filter((B) => B.rank !== C.rank).map((B) => B.value), x = m(D);
+    x > 0 && C.value > x * 2.5 && (h = { rank: C.rank, ms: C.value, line: d.get(C.rank) ?? "", median: x });
   } else if (f.length >= 3) {
-    let k = f.reduce((B, z11) => z11.value < B.value ? z11 : B), D = f.filter((B) => B.rank !== k.rank).map((B) => B.value), x = m(D);
-    x > 0 && k.value * 2.5 < x && (u = { rank: k.rank, rate: k.value, line: d.get(k.rank) ?? "", median: x });
+    let C = f.reduce((B, z11) => z11.value < B.value ? z11 : B), D = f.filter((B) => B.rank !== C.rank).map((B) => B.value), x = m(D);
+    x > 0 && C.value * 2.5 < x && (u = { rank: C.rank, rate: C.value, line: d.get(C.rank) ?? "", median: x });
   }
   if (!(r.length > 0 || h !== null || u !== null || _ !== null)) return q("straggler", "Straggler / gray failure");
   if (t.length > 0 && r.length === 0 && h === null && u === null && _ === null) {
-    let k = dg(e);
-    if (k === 1 || k === 2) return q("straggler", "Straggler / gray failure");
+    let C = dg(e);
+    if (C === 1 || C === 2) return q("straggler", "Straggler / gray failure");
   }
   let T = [...t, ...r];
   h && T.push(h.line), u && T.push(u.line), _ && T.push(_.line);
-  let A = (k) => k >= 1e3 ? `${(k / 1e3).toFixed(2)}s` : `${Math.round(k)}ms`, w = h ? `Rank ${h.rank} is taking ${A(h.ms)} against a median of ${A(h.median)} across the other ranks, roughly ${(h.ms / Math.max(h.median, 1e-9)).toFixed(1)}x slower. In a synchronous job every other rank waits at the next collective, so the whole job runs at this rank's pace.` : "", S = u ? `Rank ${u.rank} is running at ${u.rate} against a median of ${u.median} across the other ranks, roughly ${(u.median / Math.max(u.rate, 1e-9)).toFixed(1)}x slower. In a synchronous job every other rank waits at the next collective, so the whole job runs at this rank's pace.` : "", b = (k) => Number.isInteger(k) ? k.toFixed(1) : String(k), O = _ ? `Rank ${_.rank} has a performance score of ${b(_.score)} against ${b(_.best)} for the fastest rank, roughly ${(_.best / Math.max(_.score, 1e-9)).toFixed(1)}x slower. That reading is from the straggler report itself, not inferred from step times. In a synchronous job every other rank waits at the next collective, so the whole job runs at this rank's pace.` : "";
-  return { id: "straggler", title: "Straggler / gray failure", detected: true, severity: "warning", summary: w || S || O || "A collective watchdog timeout fired without a hard crash, one rank is lagging (gray failure) rather than failing outright.", evidence: T.slice(0, 5), action: "Identify the lagging rank/node, check for thermal throttling (nvidia-smi -q -d TEMPERATURE,CLOCK), a degraded NIC/cable, or a noisy neighbor. Drain and re-test the node with nccl-tests; exclude it from the job if busbw is below the fleet baseline." };
+  let A = (C) => C >= 1e3 ? `${(C / 1e3).toFixed(2)}s` : `${Math.round(C)}ms`, w = h ? `Rank ${h.rank} is taking ${A(h.ms)} against a median of ${A(h.median)} across the other ranks, roughly ${(h.ms / Math.max(h.median, 1e-9)).toFixed(1)}x slower. In a synchronous job every other rank waits at the next collective, so the whole job runs at this rank's pace.` : "", O = u ? `Rank ${u.rank} is running at ${u.rate} against a median of ${u.median} across the other ranks, roughly ${(u.median / Math.max(u.rate, 1e-9)).toFixed(1)}x slower. In a synchronous job every other rank waits at the next collective, so the whole job runs at this rank's pace.` : "", b = (C) => Number.isInteger(C) ? C.toFixed(1) : String(C), S = _ ? `Rank ${_.rank} has a performance score of ${b(_.score)} against ${b(_.best)} for the fastest rank, roughly ${(_.best / Math.max(_.score, 1e-9)).toFixed(1)}x slower. That reading is from the straggler report itself, not inferred from step times. In a synchronous job every other rank waits at the next collective, so the whole job runs at this rank's pace.` : "";
+  return { id: "straggler", title: "Straggler / gray failure", detected: true, severity: "warning", summary: w || O || S || "A collective watchdog timeout fired without a hard crash, one rank is lagging (gray failure) rather than failing outright.", evidence: T.slice(0, 5), action: "Identify the lagging rank/node, check for thermal throttling (nvidia-smi -q -d TEMPERATURE,CLOCK), a degraded NIC/cable, or a noisy neighbor. Drain and re-test the node with nccl-tests; exclude it from the job if busbw is below the fleet baseline." };
 }
 function Gh(e) {
   if (/pallas_call/i.test(e)) return q("weight_delta", "Pallas emulation is not a weight delta");
@@ -4255,12 +4255,12 @@ Routing: ${i.join(" | ")}` };
 function Bh(e, t = "auto") {
   let r = false, i = false, n = false, o = false, a = false, s = false, c = [], d = [], l = t === "light" || t === "auto", m = t === "topology" || t === "auto";
   if (l) {
-    let C = G(e, /driver\/library version mismatch|Failed to initialize NVML|NVML[^\n]*(?:init(?:ializ)?[^\n]*fail|mismatch)|couldn'?t communicate with the NVIDIA driver/i);
-    C.length && (c.push("NVIDIA driver/NVML failed to initialize (driver - library version mismatch), the GPU stack is not usable; reload the driver or reboot before launching."), d.push(...C));
+    let k = G(e, /driver\/library version mismatch|Failed to initialize NVML|NVML[^\n]*(?:init(?:ializ)?[^\n]*fail|mismatch)|couldn'?t communicate with the NVIDIA driver/i);
+    k.length && (c.push("NVIDIA driver/NVML failed to initialize (driver - library version mismatch), the GPU stack is not usable; reload the driver or reboot before launching."), d.push(...k));
     let y = G(e, /(?:CUDA\s+)?driver version is insufficient|cudaErrorInsufficientDriver|CUDA(?: initialization)? error(?: code)?\s*:?\s*35\b/i);
     if (y.length) {
-      let w = e.match(/Driver Version:\s*([\d.]+)/i)?.[1], S = e.match(/libcudart\.so\.(\d+)/i)?.[1] || e.match(/cuda[-_ ]?(\d+)\.\d+/i)?.[1], O = S ? { 13: "580", 12: "525", 11: "450" }[S] : void 0;
-      c.push(`The CUDA runtime in this image is newer than the installed driver${w ? ` (driver ${w}` : ""}${w && S ? `, CUDA ${S}.x runtime)` : w ? ")" : ""}. This is a version-compatibility failure, not a missing or faulty GPU, the device is visible. ` + (O ? `CUDA ${S}.x requires a ${O}-series or newer driver. Either upgrade the driver to >= ${O}, or roll the image back to a CUDA runtime the current driver supports.` : "Either upgrade the NVIDIA driver to the minimum branch documented for this CUDA release, or roll the image back to a CUDA runtime the current driver supports.") + " Drain or cordon the affected node before changing the driver, and re-check with `nvidia-smi` plus a trivial CUDA init after the change."), d.push(...y), r = true;
+      let w = e.match(/Driver Version:\s*([\d.]+)/i)?.[1], O = e.match(/libcudart\.so\.(\d+)/i)?.[1] || e.match(/cuda[-_ ]?(\d+)\.\d+/i)?.[1], S = O ? { 13: "580", 12: "525", 11: "450" }[O] : void 0;
+      c.push(`The CUDA runtime in this image is newer than the installed driver${w ? ` (driver ${w}` : ""}${w && O ? `, CUDA ${O}.x runtime)` : w ? ")" : ""}. This is a version-compatibility failure, not a missing or faulty GPU, the device is visible. ` + (S ? `CUDA ${O}.x requires a ${S}-series or newer driver. Either upgrade the driver to >= ${S}, or roll the image back to a CUDA runtime the current driver supports.` : "Either upgrade the NVIDIA driver to the minimum branch documented for this CUDA release, or roll the image back to a CUDA runtime the current driver supports.") + " Drain or cordon the affected node before changing the driver, and re-check with `nvidia-smi` plus a trivial CUDA init after the change."), d.push(...y), r = true;
     }
     let T = G(e, /no CUDA[- ]?capable device|no CUDA devices? (?:were )?(?:found|detected)|no devices? were found|nvidia-smi[^\n]*No devices were found/i);
     T.length && (c.push("No CUDA-capable device is visible to the host. GPUs are missing, the driver is not loaded, or CUDA_VISIBLE_DEVICES is hiding them."), d.push(...T));
@@ -4269,32 +4269,32 @@ function Bh(e, t = "auto") {
   }
   if (m) {
     for (let b of e.split(/\r?\n/)) {
-      let O = b.match(/busbw[^0-9]*([0-9]+(?:\.[0-9]+)?)\s*GB\/s/i);
-      if (O) {
-        let k = Number(O[1]);
-        if (d.push(b.trim().slice(0, 200)), k < 30) {
+      let S = b.match(/busbw[^0-9]*([0-9]+(?:\.[0-9]+)?)\s*GB\/s/i);
+      if (S) {
+        let C = Number(S[1]);
+        if (d.push(b.trim().slice(0, 200)), C < 30) {
           let D = e.match(/\b((?:node|host|gpu-worker|dgx)[-\w]*\d[\w-]*)\b/i)?.[1] || "", x = /NCCL_P2P_DISABLE\s*(?:=|:|\s+set by environment to\s+)\s*1/i.test(e), B = /NCCL_SHM_DISABLE\s*(?:=|:|\s+set by environment to\s+)\s*1/i.test(e), z11 = /via SHM/i.test(e);
-          x ? (i = true, c.push(`${D ? `${D}: ` : ""}all-reduce busbw is only ${k} GB/s because NCCL_P2P_DISABLE=1, peer-to-peer is switched off, so transfers fall back to ${z11 ? 'shared memory (the log shows "via SHM")' : "a slower path"} instead of NVLink. This is a configuration setting, not a hardware fault: unset NCCL_P2P_DISABLE (or set it to 0) and re-measure. Neither a smaller batch nor new hardware is the remedy here.`)) : B ? c.push(`all-reduce busbw is only ${k} GB/s and NCCL_SHM_DISABLE=1 is set, removing the shared-memory path. Unset it unless it was added to work around a specific defect, then re-measure.`) : c.push(`all-reduce busbw is only ${k} GB/s, far below a healthy NVLink/IB fabric; expect collectives to bottleneck.`);
+          x ? (i = true, c.push(`${D ? `${D}: ` : ""}all-reduce busbw is only ${C} GB/s because NCCL_P2P_DISABLE=1, peer-to-peer is switched off, so transfers fall back to ${z11 ? 'shared memory (the log shows "via SHM")' : "a slower path"} instead of NVLink. This is a configuration setting, not a hardware fault: unset NCCL_P2P_DISABLE (or set it to 0) and re-measure. Neither a smaller batch nor new hardware is the remedy here.`)) : B ? c.push(`all-reduce busbw is only ${C} GB/s and NCCL_SHM_DISABLE=1 is set, removing the shared-memory path. Unset it unless it was added to work around a specific defect, then re-measure.`) : c.push(`all-reduce busbw is only ${C} GB/s, far below a healthy NVLink/IB fabric; expect collectives to bottleneck.`);
         }
         break;
       }
     }
-    let C = /\b(?:mlx5_\d+|ibstat|ibportstate|infiniband|\bib\d+\b|NET\/IB|rdma)\b/i.test(e), y = C ? G(e, /(\b(?:mlx5_\d+|ib\d+)\b[^\n]*(?:State\s*:\s*Down|PORT_DOWN)|ibstat[^\n]*Down|Physical state\s*:\s*(?:Disabled|Polling)|port.*PORT_DOWN)/i) : [];
+    let k = /\b(?:mlx5_\d+|ibstat|ibportstate|infiniband|\bib\d+\b|NET\/IB|rdma)\b/i.test(e), y = k ? G(e, /(\b(?:mlx5_\d+|ib\d+)\b[^\n]*(?:State\s*:\s*Down|PORT_DOWN)|ibstat[^\n]*Down|Physical state\s*:\s*(?:Disabled|Polling)|port.*PORT_DOWN)/i) : [];
     /Persistence\s*Mode\s*:\s*Disabled/i.test(e) && (s = true, c.unshift("Persistence Mode is Disabled, the driver unloads whenever no client holds the GPU, so every new process pays full re-initialisation. This is startup cost only: steady-state step time is unaffected, which is why the job looks healthy once running. Enable it with `nvidia-smi -pm 1`, and enable the nvidia-persistenced service so it survives reboot. It matters most for short jobs, frequent restarts and elastic workers."), d.unshift(...G(e, /Persistence\s*Mode\s*:\s*Disabled/i).slice(0, 2)));
     let T = [...new Set((e.match(/\bmtu\s+(\d{3,5})\b/gi) || []).map((b) => b.replace(/\D/g, "")))], A = /Frag needed and DF set|message too long|EMSGSIZE/i.test(e);
-    !C && (T.length > 1 || A) && (a = true, c.unshift(`Inconsistent MTU across the fabric${T.length > 1 ? ` (${T.join(" vs ")})` : ""}${A ? ', the kernel reports "Frag needed and DF set", so oversized frames cannot cross' : ""}. This fails by SIZE, not at random: small collectives fit and succeed, large ones are dropped silently and hang with no error. Align every host AND its switch port to the same MTU (\`ip link show\`, then the switch side), and verify with \`ping -M do -s <mtu-28>\` between the hosts that disagree.`), d.unshift(...G(e, /\bmtu\s+\d{3,5}\b|Frag needed and DF set/i).slice(0, 4)));
+    !k && (T.length > 1 || A) && (a = true, c.unshift(`Inconsistent MTU across the fabric${T.length > 1 ? ` (${T.join(" vs ")})` : ""}${A ? ', the kernel reports "Frag needed and DF set", so oversized frames cannot cross' : ""}. This fails by SIZE, not at random: small collectives fit and succeed, large ones are dropped silently and hang with no error. Align every host AND its switch port to the same MTU (\`ip link show\`, then the switch side), and verify with \`ping -M do -s <mtu-28>\` between the hosts that disagree.`), d.unshift(...G(e, /\bmtu\s+\d{3,5}\b|Frag needed and DF set/i).slice(0, 4)));
     let w = G(e, /Slave Interface[^\n]*|MII Status\s*:\s*down|link status definitely down for interface\s+\S+|bond\d+:[^\n]*down/i).filter((b) => /down/i.test(b));
-    if (!C && w.length) {
+    if (!k && w.length) {
       let b = e.match(/link status definitely down for interface\s+(\S+)/i)?.[1] || e.match(/Slave Interface:\s*(\S+)[\s\S]{0,80}?MII Status\s*:\s*down/i)?.[1];
       o = true, c.unshift(`${b ? `Bond member ${b} is DOWN` : "A bond member is DOWN"}, 802.3ad/LACP degrades silently, so the host keeps running at a fraction of its egress with no error raised. Check \`cat /proc/net/bonding/<bond>\` for member MII status, then the cable, transceiver and switch port for ${b || "the failed member"}. Expect throughput to scale with the surviving members: losing one of two halves it.`), d.unshift(...w);
     }
     if (y.length) {
       n = true;
-      let b = e.match(/\b(?:mlx5_\d+|ib\w*\d+)\b(?=[^\n]*State\s*:\s*Down)/i)?.[0], O = /using\s+NET\/Socket|No device found for\s+(?:mlx5_\d+|ib\w*\d+)/i.test(e);
-      c.unshift(`${b ? `${b} ` : "An InfiniBand "}port is DOWN` + (O ? " and NCCL has already fallen back to NET/Socket (TCP), the job is still training, at a fraction of fabric speed." : ", collectives will fall back to TCP or hang.") + " Bring the link up (check the cable, `ibportstate` on the switch side, and `ibstat`), then confirm NCCL re-selects NET/IB. Do not pin NCCL_IB_DISABLE=1: that makes the degraded path permanent."), d.unshift(...y);
+      let b = e.match(/\b(?:mlx5_\d+|ib\w*\d+)\b(?=[^\n]*State\s*:\s*Down)/i)?.[0], S = /using\s+NET\/Socket|No device found for\s+(?:mlx5_\d+|ib\w*\d+)/i.test(e);
+      c.unshift(`${b ? `${b} ` : "An InfiniBand "}port is DOWN` + (S ? " and NCCL has already fallen back to NET/Socket (TCP), the job is still training, at a fraction of fabric speed." : ", collectives will fall back to TCP or hang.") + " Bring the link up (check the cable, `ibportstate` on the switch side, and `ibstat`), then confirm NCCL re-selects NET/IB. Do not pin NCCL_IB_DISABLE=1: that makes the degraded path permanent."), d.unshift(...y);
     }
-    let S = G(e, /\b(SYS|PHB|NODE)\b.*\b(SYS|PHB|NODE)\b/i).filter((b) => /GPU\d/i.test(b));
-    S.length && (c.push("GPU topology shows non-NVLink paths (SYS/PHB) between GPUs, cross-GPU bandwidth will be PCIe-limited."), d.push(...S));
+    let O = G(e, /\b(SYS|PHB|NODE)\b.*\b(SYS|PHB|NODE)\b/i).filter((b) => /GPU\d/i.test(b));
+    O.length && (c.push("GPU topology shows non-NVLink paths (SYS/PHB) between GPUs, cross-GPU bandwidth will be PCIe-limited."), d.push(...O));
   }
   if (c.length === 0) return q("preflight", "Preflight cluster health");
   let p = t === "light" ? "light host/GPU sanity" : t === "topology" ? "topology-aware fabric" : "host + topology", h = r && c.length === 1, u = i && !r && c.length === 1, _ = n && !h, g = o && !n && !h, f = a && !n && !o && !h, E = s && !n && !o && !a && !h;
@@ -4603,8 +4603,8 @@ CRITICAL, detection is not origin: rank ${I} only DETECTED the bad payload. In a
 1. Drain the node, then inspect the link: \`nvidia-smi -q -d PCIE\` (replay counter, link gen/width downgrade) and \`lspci -vvv\` for AER correctable/uncorrectable error records.
 2. Check the physical path: riser, retimer, and slot seating; verify the link hasn't renegotiated to a lower generation/width under load.
 3. Re-validate under load: \`dcgmi diag -r 3\`, then a bandwidth soak (\`nvbandwidth\` / \`p2pBandwidthLatencyTest\`) while watching the replay delta, any steady growth fails the node.`, fallback: (v) => "If replays continue after reseat, swap the GPU to a different slot/riser to see whether the counter follows the GPU or the slot, that decides GPU RMA vs motherboard/riser service. Check for AER 'completion timeout' entries in dmesg, and validate BIOS/retimer firmware.", emergency: (v) => "Exclude the node and resume from the last good checkpoint. If this node carried a NIC (GPUDirect path), also re-run the fabric preflight (`all_reduce_perf -c 1`) on the remaining set before committing a long run." };
-  let S = /nvls|multicast|failed to bind nvlink sharp|cumulticastbindmem/i.test(r), b = /node running low on memory|memory usage threshold of [\d.]+|killed due to memory pressure|raylet[^\n]*(?:died|killed|memory monitor)|ray\.exceptions\.OutOfMemoryError|\bMemoryError\b/i.test(r);
-  if ((/out of memory|\boom\b/.test(n) || b) && !S) {
+  let O = /nvls|multicast|failed to bind nvlink sharp|cumulticastbindmem/i.test(r), b = /node running low on memory|memory usage threshold of [\d.]+|killed due to memory pressure|raylet[^\n]*(?:died|killed|memory monitor)|ray\.exceptions\.OutOfMemoryError|\bMemoryError\b/i.test(r);
+  if ((/out of memory|\boom\b/.test(n) || b) && !O) {
     let v = /torch\.cuda\.outofmemory|cuda out of memory|cudamalloc|\bhbm\b|reserved .*by pytorch|gpu \d+;/i.test(r);
     if ((/oom-?kill|out of memory: killed process|anon-rss|slurmstepd|cgroup[^\n]*(?:out of memory|oom)|killed by the cgroup|\bsignal 9\b|return code -9|exit code 137|dataloader worker[^\n]*(?:exit|kill|die)/i.test(r) || b) && !v) {
       let ue = /cgroup|slurmstepd|--mem\b|exit code 137/i.test(r), $e = Vs(r);
@@ -4638,15 +4638,15 @@ ${Z.fingerprint}. `), Z.freeGB > 0 && Z.allocationGB > Z.freeGB && (Ve = `Alloca
     }
     return { key: "CUDA_OOM", label: "CUDA out-of-memory", primary: () => `The process ran out of GPU memory. Reduce the working set first: lower the (micro)batch size, enable activation/gradient checkpointing, and shorten max sequence length if applicable.${De}`, fallback: () => `Then tune the allocator (${Ve}). If this is a distributed job, shard optimizer/params state (ZeRO-3 / FSDP), or add CPU/NVMe offload or tensor parallelism. Confirm the true peak with \`torch.cuda.max_memory_allocated()\`.`, emergency: () => "Before resuming, verify that a checkpoint exists and passes the job's integrity checks. Only then resume from that verified checkpoint with a smaller global batch (or more gradient-accumulation steps) so the per-step activation memory fits. If integrity cannot be established, do not assume resuming is safe." };
   }
-  let O = [...l], k = O.length > 0 && O.every((v) => v === 13 || v === 31), D = /illegal memory access|unspecified launch failure|device-side assert|misaligned address|cuda error/i.test(o), x = (a.xidCodes.length > 0 || R(o, /\bxid\b/i) && !a.xidExplicitlyClean || et(r).length > 0) && !(k && D), B = /nvlink[^\n]*(?:fault|error)/i.test(o) && (a.nvlinkDegraded || !a.nvlinkParsed), z11 = /aer:|iommu|completion timeout|machine check|\bmce\b/i.test(o) || /nvrm/i.test(o) && !(k && D);
+  let S = [...l], C = S.length > 0 && S.every((v) => v === 13 || v === 31), D = /illegal memory access|unspecified launch failure|device-side assert|misaligned address|cuda error/i.test(o), x = (a.xidCodes.length > 0 || R(o, /\bxid\b/i) && !a.xidExplicitlyClean || et(r).length > 0) && !(C && D), B = /nvlink[^\n]*(?:fault|error)/i.test(o) && (a.nvlinkDegraded || !a.nvlinkParsed), z11 = /aer:|iommu|completion timeout|machine check|\bmce\b/i.test(o) || /nvrm/i.test(o) && !(C && D);
   if (/failed to bind nvlink sharp|cumulticastbindmem|multicast memory of size|\bNVLS\b[^\n]{0,60}(?:multicast|slot|memory|bind|out of)|multicast[^\n]{0,40}slots?[^\n]{0,20}(?:exhaust|max|128|out of)/i.test(r)) return { key: "NVLS_SLOT_EXHAUSTION", label: "NCCL NVLS multicast-slot exhaustion (not a GPU fault)", primary: () => 'NCCL exhausted the NVSwitch NVLS multicast slots (128 hardware limit): many small sub-communicators, e.g. FSDP\'s per-2-rank groups, each allocate an NVLS multicast group until the bind fails ("Failed to bind NVLink SHARP (NVLS) Multicast memory \u2026 out of memory / ncclUnhandledCudaError"). This is a config/scale limit, NOT a GPU hardware fault, do NOT drain or RMA the GPU. Disable NVLS across all ranks: `export NCCL_NVLS_ENABLE=0` and re-run.', fallback: () => "If you need NVLS for the large collectives, upgrade to an NCCL build that falls back gracefully on slot exhaustion (\u2265 2.30), or downgrade to 2.28.9 which fell back instead of crashing. Alternatively reduce sub-communicator count so you don't create an NVLS group per 2-rank shard.", emergency: () => "Set `NCCL_NVLS_ENABLE=0` in the launcher env for every rank and resume from the last checkpoint; the GPUs are healthy." };
-  if (k && D && /reproduc(?:es|ible)|multiple nodes?|node-[a-z][^\n]*node-[a-z]|extension[^\n]*(?:build|version|deployed|release)|custom kernel|fused_[a-z0-9_]+/i.test(r)) {
+  if (C && D && /reproduc(?:es|ible)|multiple nodes?|node-[a-z][^\n]*node-[a-z]|extension[^\n]*(?:build|version|deployed|release)|custom kernel|fused_[a-z0-9_]+/i.test(r)) {
     let v = r.match(/(?:extension|build|version)\s*(?:build\s*)?([\w.-]+)/i)?.[1], I = r.match(/\b([a-z][a-z0-9_]*(?:kernel|op)[a-z0-9_]*)\b/i)?.[1] || "the custom CUDA extension";
     return { key: "CUDA_SOFTWARE_KERNEL_ILLEGAL_ACCESS", label: "application CUDA kernel illegal-memory-access", primary: () => `The same ${I} illegal-memory-access reproduces across multiple nodes and only with the newly deployed extension${v ? ` (${v})` : ""}. Xid 13/31 reports the application page fault; clean ECC/DCGM evidence and cross-node reproduction exonerate the physical GPU. Roll back the extension to the last known-good build, then reproduce the failing input with \`compute-sanitizer --tool memcheck\` (or \`CUDA_LAUNCH_BLOCKING=1\`) to identify the first invalid access.`, fallback: () => `Rebuild the extension against the exact deployed PyTorch/CUDA ABI with debug line information, then run its smallest failing shape under compute-sanitizer. Validate pointer lifetime, bounds, strides/alignment, and stream synchronization around ${I}.`, emergency: () => "Keep the known-good extension build pinned and quarantine the failing input/shape until the kernel fix passes sanitizer and cross-node replay. No GPU drain or replacement is indicated by this evidence." };
   }
-  let oe = new RegExp([/double[- ]?bit ECC/, /(?:volatile|aggregate)?\s*uncorrectable(?:\s+ECC)?(?:\s+errors?)?\s*:?\s*[1-9]\d*/, /remapping failure occurred\s*:?\s*yes/, /row[- ]?remap(?:ping)?\s*(?:failure|error)\s*:?\s*(?:yes|[1-9]\d*)/, /pending page retirement\s*:?\s*(?:yes|[1-9]\d*)/, /fallen off the bus|gpu is lost/, /over ?temp|thermal (?:violation|slowdown)/, /machine check|\bmce\b|aer:\s|completion timeout/].map((v) => v.source).join("|"), "i"), ae = [48, 63, 64, 74, 79, 92, 94, 95, 122, 123, 140], se = oe.test(o) || O.some((v) => ae.includes(v)), ee = /\b(?:driver|firmware|kernel module)\s+regression\b|\bregression\s+(?:from|in|introduced|since)\b|\b(?:since|after)\s+(?:upgrading|updating|the upgrade)\s+to\b|\bworked\s+(?:fine\s+)?on\b[^\n]{0,60}\bbrok(?:e|en)\b|\bknown\s+(?:driver\s+)?(?:bug|issue|regression)\b/i, $ = /\b(?:driver|nvidia|nvrm|branch)?\s*\b\d{3}(?:\.\d+)*(?:\.x|\.\d+)?\b/i.test(o) || /\b\d{3}\.\d{2,3}(?:\.\d{2})?\b/.test(o), wr = /\bfully reproducible\b|\b100% reproducible\b|\breproducible\b|\breproduces?\s+(?:every|each|consistently|reliably|on demand|within)\b|\bevery\s+(?:single\s+)?(?:run|launch|time)\b/i.test(o);
+  let oe = new RegExp([/double[- ]?bit ECC/, /(?:volatile|aggregate)?\s*uncorrectable(?:\s+ECC)?(?:\s+errors?)?\s*:?\s*[1-9]\d*/, /remapping failure occurred\s*:?\s*yes/, /row[- ]?remap(?:ping)?\s*(?:failure|error)\s*:?\s*(?:yes|[1-9]\d*)/, /pending page retirement\s*:?\s*(?:yes|[1-9]\d*)/, /fallen off the bus|gpu is lost/, /over ?temp|thermal (?:violation|slowdown)/, /machine check|\bmce\b|aer:\s|completion timeout/].map((v) => v.source).join("|"), "i"), ae = [48, 63, 64, 74, 79, 92, 94, 95, 122, 123, 140], se = oe.test(o) || S.some((v) => ae.includes(v)), ee = /\b(?:driver|firmware|kernel module)\s+regression\b|\bregression\s+(?:from|in|introduced|since)\b|\b(?:since|after)\s+(?:upgrading|updating|the upgrade)\s+to\b|\bworked\s+(?:fine\s+)?on\b[^\n]{0,60}\bbrok(?:e|en)\b|\bknown\s+(?:driver\s+)?(?:bug|issue|regression)\b/i, $ = /\b(?:driver|nvidia|nvrm|branch)?\s*\b\d{3}(?:\.\d+)*(?:\.x|\.\d+)?\b/i.test(o) || /\b\d{3}\.\d{2,3}(?:\.\d{2})?\b/.test(o), wr = /\bfully reproducible\b|\b100% reproducible\b|\breproducible\b|\breproduces?\s+(?:every|each|consistently|reliably|on demand|within)\b|\bevery\s+(?:single\s+)?(?:run|launch|time)\b/i.test(o);
   if (x && !se && wr && ee.test(o) && $) {
-    let v = o.match(/\b(\d{3}(?:\.\d+)*(?:\.x)?)\b/)?.[1], I = O.length ? O.join("/") : "the reported Xid";
+    let v = o.match(/\b(\d{3}(?:\.\d+)*(?:\.x)?)\b/)?.[1], I = S.length ? S.join("/") : "the reported Xid";
     return { key: "GPU_DRIVER_REGRESSION", label: "NVIDIA driver regression (not a hardware fault)", primary: () => `This reproduces on demand and the incident names a specific driver branch${v ? ` (${v})` : ""} as the regression point, while ECC, row-remap and off-bus telemetry are all clean. Xid ${I} here is the driver reporting that it could not complete a context switch, it is the symptom of the regression, not evidence of failing silicon. Treat the GPU as healthy: no node-level hardware action is indicated by this evidence. Pin the driver to the last known-good branch and re-run the same workload to confirm the fault disappears; that single A/B is the whole diagnosis.`, fallback: () => `If a rollback is not available, bisect the driver branches between the last known-good version and ${v || "the current one"} to identify the first bad release, and upgrade to a branch where the fix has landed. Record the exact driver version alongside the workload so the regression is reproducible for an NVIDIA bug report.`, emergency: () => "Pin every affected node to the last known-good driver version and resume from the last checkpoint. The GPUs are healthy, no node action is indicated by this evidence." };
   }
   if (x || B || z11) {
@@ -4773,8 +4773,8 @@ function Br(e) {
       let f = gr(_c(u).key);
       if (f > n) {
         n = f;
-        let E = Lt(e, u), C = gr(E.key);
-        (!i || C > i.priority) && (i = { rank: g, line: u.trim().slice(0, 200), fault: E, priority: C });
+        let E = Lt(e, u), k = gr(E.key);
+        (!i || k > i.priority) && (i = { rank: g, line: u.trim().slice(0, 200), fault: E, priority: k });
       }
     }
   }
@@ -4956,13 +4956,13 @@ function lg(e) {
   return { id: "gpudirect_rdma_deadlock", title: "GPUDirect RDMA Deadlock (Network caused GPU drop)", detected: true, severity: "critical", summary: "An InfiniBand/mlx5 network fault flooded the PCIe bus with retries, causing a PCIe Completion Timeout (CmpltTO). This forced the Linux kernel to unbind the GPU, triggering an Xid 79. The root cause is the NETWORK fabric, not the GPU.", evidence: [...n, ...o], action: "Investigate the network link (IB/RoCE). Check the optical transceiver, switch port, and MTU settings. Do NOT replace the GPU." };
 }
 function mg(e) {
-  let t = String(e || ""), r = (S) => R(t, S), i = r(/\bibv_poll_cq\b[^.\n]*failed\s+with\s+error\s+(\d+)/i) || r(/\bibv_poll_cq\b/i) && r(/\berror\s+(\d+)\b/i), n = r(/CQE error[^.\n]*syndrome\s+0x([0-9a-fA-F]+)/i), o = r(/Local Length Error/i), a = r(/(frame size|mtu)\s*(\d{3,5})[^.\n]*exceeds\s*(?:MTU\s*)?(\d{3,5})/i) || r(/RoCE[^.\n]*MTU\s*mismatch/i), s = r(/vendor\s*err(?:or)?\s*(\d+)/i), c = r(/IBV_WC_RETRY_EXC_ERR|\bRETRY_EXC\b/i), d = r(/packet\s*drop(?:ped)?|retry(?:\s*limit)?\s*exceeded/i), l = r(/(?:mlx5_core[^\n]*\b(?:ib\d+)[^\n]*Link\s+down|Port\s+\d+[^\n]*(?:ACTIVE\s*->\s*DOWN|changed state[^\n]*DOWN)|\blink\s+flap)/i);
+  let t = String(e || ""), r = (O) => R(t, O), i = r(/\bibv_poll_cq\b[^.\n]*failed\s+with\s+error\s+(\d+)/i) || r(/\bibv_poll_cq\b/i) && r(/\berror\s+(\d+)\b/i), n = r(/CQE error[^.\n]*syndrome\s+0x([0-9a-fA-F]+)/i), o = r(/Local Length Error/i), a = r(/(frame size|mtu)\s*(\d{3,5})[^.\n]*exceeds\s*(?:MTU\s*)?(\d{3,5})/i) || r(/RoCE[^.\n]*MTU\s*mismatch/i), s = r(/vendor\s*err(?:or)?\s*(\d+)/i), c = r(/IBV_WC_RETRY_EXC_ERR|\bRETRY_EXC\b/i), d = r(/packet\s*drop(?:ped)?|retry(?:\s*limit)?\s*exceeded/i), l = r(/(?:mlx5_core[^\n]*\b(?:ib\d+)[^\n]*Link\s+down|Port\s+\d+[^\n]*(?:ACTIVE\s*->\s*DOWN|changed state[^\n]*DOWN)|\blink\s+flap)/i);
   if (!(i || n || o || a || s || c || d || l)) return q("ib_fabric", "Network fabric fault (IB / RoCE)");
   let p = t.match(/(?:frame size|mtu)\s*(\d{3,5})[^.\n]*exceeds\s*(?:MTU\s*)?(\d{3,5})/i), h = t.match(/syndrome\s+0x([0-9a-fA-F]+)/i), u = t.match(/vendor\s*err(?:or)?\s*(\d+)/i), _ = [], g = P(t, /\bibv_poll_cq\b|\bIBV_WC_RETRY_EXC_ERR\b|Local Length Error|frame size[^.\n]*exceeds|CQE error|Link\s+down|Port\s+\d+[^\n]*(?:ACTIVE\s*->\s*DOWN|changed state[^\n]*DOWN)|vendor\s*err/i, 5);
   _.push(...g);
-  let f = p ? `MTU ${p[1]} > path MTU ${p[2]}` : "no MTU pair detected", E = h ? `CQE syndrome 0x${h[1]}` : u ? `vendor err ${u[1]}` : "IB retry-exceeded / poll_cq failure", C = zr(t), y = C.node ? ` on ${C.node}` : "", T = C.pci ? ` (PCI ${C.pci})` : "", A = t.match(/\b(ib\d+|mlx5_\d+)\b/i)?.[1] || "the affected IB interface", w = t.match(/\bPort\s+(\d+)\b/i)?.[1];
+  let f = p ? `MTU ${p[1]} > path MTU ${p[2]}` : "no MTU pair detected", E = h ? `CQE syndrome 0x${h[1]}` : u ? `vendor err ${u[1]}` : "IB retry-exceeded / poll_cq failure", k = zr(t), y = k.node ? ` on ${k.node}` : "", T = k.pci ? ` (PCI ${k.pci})` : "", A = t.match(/\b(ib\d+|mlx5_\d+)\b/i)?.[1] || "the affected IB interface", w = t.match(/\bPort\s+(\d+)\b/i)?.[1];
   return l ? { id: "ib_fabric", title: "Network fabric fault (IB / RoCE)", detected: true, severity: "critical", causalDiscriminator: true, summary: `InfiniBand link ${A}${w ? ` / switch port ${w}` : ""} changed DOWN before the NCCL completion error${u ? ` (vendor err ${u[1]})` : ""}. The link/cable/transceiver/switch path is the initiating subsystem; the NCCL error is collateral.`, evidence: _, action: `Keep the affected path out of service and inspect it directly: run \`ibstat ${A}\`, \`iblinkinfo\`, and \`ibqueryerrors\`; correlate ${w ? `switch port ${w}` : "the mapped switch port"} with ${A} / the HCA port and check link-down, symbol, receive, and retry counters. Reseat or replace the cable/transceiver, test the switch port, and verify the port remains ACTIVE under \`ib_write_bw\` plus \`nccl-tests all_reduce_perf\` before re-admission. Extending a collective timeout will not restore a link that changed state first.`, fallbackAction: "If the link stays down after cable/transceiver replacement, move the HCA to a known-good switch port/rail and update the fabric manager/subnet manager mapping. If the fault follows the HCA, service the NIC; if it stays with the port, service the switch path.", emergencyAction: "Drain only nodes using the failed rail/path, relaunch on a healthy rail, and resume from the last verified checkpoint after every selected node passes the collective preflight." } : { id: "ib_fabric", title: "Network fabric fault (IB / RoCE)", detected: true, severity: "critical", causalDiscriminator: true, summary: `Layer-2 fabric failure${y}${T}: ${E}. ${f}. The downstream NCCL timeout / ProcessGroupNCCL error is collateral from dropped RDMA packets, not a software bug, fix the fabric, do NOT tune NCCL.`, evidence: _, action: `Failing fast is mandatory; extending NCCL_TIMEOUT on a physically broken cluster wastes thousands of dollars per hour in idle GPU compute. Execute in order:
-1. Confirm the MTU pair on ${C.node || "the affected node"}: \`ip link show ib0\` and \`cat /sys/class/net/ib0/device/netdev/*/mtu\`, both ends of the path must agree (typical RoCE: 9000 on host, 9216 on switch, OR 1500/1500; never 9000/1500).
+1. Confirm the MTU pair on ${k.node || "the affected node"}: \`ip link show ib0\` and \`cat /sys/class/net/ib0/device/netdev/*/mtu\`, both ends of the path must agree (typical RoCE: 9000 on host, 9216 on switch, OR 1500/1500; never 9000/1500).
 2. If the host NIC is jumbo-framed but the switch path is not: \`ip link set ib0 mtu 1500\` (or whichever the path carries).
 3. Verify fabric end-to-end: \`ibdiagnet -rtsu\` and \`ib_write_bw -x 1048576 -d mlx5_0 -p <port> --report_gib\`. A single-link CRC error storm demands a different ToR port.
 4. Re-test with the cluster preflight before relaunching training: \`nccl-tests/all_reduce_perf -b 1G -e 8G -f 2 -g 1\`.
@@ -5763,7 +5763,7 @@ async function Jc(e, t) {
     clearTimeout(l);
   }
   let p = s.confidence >= 50, h = Hg(r, `${s.title} ${m || ""}`), u = p && h >= 0.12, _ = (f) => {
-    let { score: E, _resolve: C, strong: y, resolution: T, ...A } = f;
+    let { score: E, _resolve: k, strong: y, resolution: T, ...A } = f;
     return A;
   }, g = { ..._(s), ...m ? { resolution: m, action: `The fix that worked:
 ${m}
@@ -6392,9 +6392,9 @@ function Wr(e, t = 120) {
       continue;
     }
     if (h_(h, a) || g_(h)) continue;
-    let u = Wt(h), _ = u.epochMs, g = { line: h.slice(0, 420), lineIndex: p, timestampMs: _, timestampLabel: u.matched ?? null, rank: d_(h), localRank: fd(h), node: l_(h), physicalGpuIndex: m_(h), gpuBusId: p_(h), xidCode: u_(h) }, f = o.find((C) => C.line.test(h));
+    let u = Wt(h), _ = u.epochMs, g = { line: h.slice(0, 420), lineIndex: p, timestampMs: _, timestampLabel: u.matched ?? null, rank: d_(h), localRank: fd(h), node: l_(h), physicalGpuIndex: m_(h), gpuBusId: p_(h), xidCode: u_(h) }, f = o.find((k) => k.line.test(h));
     f && n.push({ ...g, role: "initiator", family: f.family, marker: f.marker });
-    let E = c_.find((C) => C.line.test(h));
+    let E = c_.find((k) => k.line.test(h));
     E && n.push({ ...g, role: "victim", family: null, marker: E.marker });
   }
   let s = n.filter((p) => p.role === "initiator").sort(yn), c = n.filter((p) => p.role === "victim").sort(yn), d = Math.max(1, t) * 1e3, l = s.find((p) => c.some((h) => gd(p, h, d))) ?? null, m = l ? c.filter((p) => gd(l, p, d)) : [];
@@ -6764,66 +6764,75 @@ function Ti(e, t) {
   let r = /independent (?:mini)?batch/i.test(t) && /\b\w+\s*-=\s*[^\n;]*\.grad\b/.test(t) && /MISSING_ZERO_GRAD/i.test(String(e.failureType || ""));
   r && e.awaitingEvidence !== true && (e.action = "Manual action required: Clear the affected leaf tensor gradients before backward on each independent minibatch. For tensors updated directly with their .grad, set each affected tensor.grad = None before computing the next gradient. If an optimizer owns those parameters, use that existing optimizer to clear its gradients instead. Do not clear between intentionally accumulated microbatches.", e.verificationCommand = "Repeat multiple independent minibatches with different targets. Compare each gradient and parameter update with the per-minibatch reference from identical initial state. Confirm no previous batch gradient leaks into the next update; finite loss alone is insufficient.", e.fallbackAction = null, e.emergencyAction = null, e.codeFix = null, e.confidenceDisplay = null, e.repairVerified = false);
   let i = r && e.awaitingEvidence !== true, n = t.match(/MaxRSS\s*=\s*(\d+(?:\.\d+)?)G/i), o = t.match(/ReqMem\s*=\s*(\d+(?:\.\d+)?)G/i);
-  n && o && Number(n[1]) <= Number(o[1]) && /cgroup.*out of memory|State=OUT_OF_MEMORY/i.test(t) && (e.summary = "The scheduler and cgroup report a host-memory OOM kill. The reported MaxRSS does not exceed the requested memory value, so those two numbers alone do not prove the instantaneous limit breach. Check the enforced cgroup limit, memory peak and sampling scope before choosing new headroom.", i = true), /Xid\s*74/i.test(t) && /Xid\s*79/i.test(t) && /earlier|before/i.test(t) && !/confirmed initiator|isolated reproduction|verified causal/i.test(t) && (e.summary = "An earlier Xid 79 and inaccessible peer are a candidate initiating fault for the affected pair. The local Xid 74 may be secondary, but event timing alone does not confirm that causal relationship.", e.action = "Read-only evidence required: Map the GPU UUIDs, PCI bus IDs and affected NVLink or fabric path for the exact peer pair. Compare first-error timestamps, peer logs and link counters before assigning the initiating device. Preserve evidence and obtain owner approval for any scoped containment; do not reset or reboot a node based on chronology alone.", e.awaitingEvidence = true, e.clarifyingQuestions = ["Do the exact GPU identities and topology confirm that the inaccessible device is on the affected path, and do peer logs or link counters corroborate propagation?"], e.needsEvidence = e.clarifyingQuestions, e.fallbackAction = null, e.emergencyAction = null, i = true), /Xid\s*119/i.test(t) && /driver upgrade|after.*upgrade/i.test(t) && !/\b\d{3}\.\d{2}(?:\.\d+)?\b/.test(t) && (e.summary = "A GSP RPC timeout followed a driver change. This supports investigating the software or firmware change, but does not establish exact version compatibility or prove a physical GPU fault.", e.action = "Read-only compatibility review required: Supply the exact previous and installed driver, firmware and GPU architecture, plus a matched known-good workload result. Check documented compatibility and supported recovery for that tuple before an owner-approved version change. Do not disable GSP or roll back an unspecified driver from this evidence alone.", e.awaitingEvidence = true, e.clarifyingQuestions = ["What are the exact previous and installed driver and firmware versions and GPU architecture, and was the same workload checked on the known-good node?"], e.needsEvidence = e.clarifyingQuestions, e.fallbackAction = null, e.emergencyAction = null, i = true);
+  n && o && Number(n[1]) <= Number(o[1]) && /cgroup.*out of memory|State=OUT_OF_MEMORY/i.test(t) && (e.summary = "The scheduler and cgroup report a host-memory OOM kill. The reported MaxRSS does not exceed the requested memory value, so those two numbers alone do not prove the instantaneous limit breach. Check the enforced cgroup limit, memory peak and sampling scope before choosing new headroom.", i = true), /Xid\s*74/i.test(t) && /Xid\s*79/i.test(t) && /earlier|before/i.test(t) && !/confirmed initiator|isolated reproduction|verified causal/i.test(t) && (e.failureType = "GPU_PEER_LINK_UNATTRIBUTED", e.causationChain = [], e.causalDiscriminator = false, e.summary = "An earlier Xid 79 and inaccessible peer are a candidate initiating fault for the affected pair. The local Xid 74 may be secondary, but event timing alone does not confirm that causal relationship.", e.action = "Read-only evidence required: Map the GPU UUIDs, PCI bus IDs and affected NVLink or fabric path for the exact peer pair. Compare first-error timestamps, peer logs and link counters before assigning the initiating device. Preserve evidence and obtain owner approval for any scoped containment; do not reset or reboot a node based on chronology alone.", e.awaitingEvidence = true, e.clarifyingQuestions = ["Do the exact GPU identities and topology confirm that the inaccessible device is on the affected path, and do peer logs or link counters corroborate propagation?"], e.needsEvidence = e.clarifyingQuestions, e.fallbackAction = null, e.emergencyAction = null, i = true), /Xid\s*119/i.test(t) && /driver upgrade|after.*upgrade/i.test(t) && !/\b\d{3}\.\d{2}(?:\.\d+)?\b/.test(t) && (e.causationChain = [], e.causalDiscriminator = false, e.summary = "A GSP RPC timeout followed a driver change. This supports investigating the software or firmware change, but does not establish exact version compatibility or prove a physical GPU fault.", e.action = "Read-only compatibility review required: Supply the exact previous and installed driver, firmware and GPU architecture, plus a matched known-good workload result. Check documented compatibility and supported recovery for that tuple before an owner-approved version change. Do not disable GSP or roll back an unspecified driver from this evidence alone.", e.awaitingEvidence = true, e.clarifyingQuestions = ["What are the exact previous and installed driver and firmware versions and GPU architecture, and was the same workload checked on the known-good node?"], e.needsEvidence = e.clarifyingQuestions, e.fallbackAction = null, e.emergencyAction = null, i = true);
   let a = () => {
-    for (let _ of ["fallbackAction", "emergencyAction", "codeFix", "steps", "remediationScript", "recoveryScript", "nextAction"]) e[_] = null;
+    for (let E of ["fallbackAction", "emergencyAction", "codeFix", "steps", "remediationScript", "recoveryScript", "nextAction"]) e[E] = null;
     e.resumeFrom = null;
   }, s = Tn(t);
   /compile|Inductor/i.test(t) && /alpha\s*=|lerp/i.test(t) && /tl\.fma|generated.*arithmetic|drops?.*alpha|omits?.*alpha/i.test(t) && /eager/i.test(t) && /ratio\s*=\s*1\b|eager.*correct|eager.*clean/i.test(t) && (e.action = "Manual temporary workaround required: Run the affected optimizer update eagerly, using the same inputs and optimizer state as the successful eager control. Do not keep an arithmetic rewrite compiled and call it fixed without inspecting the generated arithmetic and checking outputs, gradients and state updates. A lerp rewrite can be affected by the same scalar-fusion defect. Confirm an upstream matcher fix is present in the installed build before re-enabling compilation.", e.verificationCommand = "Compare affected optimizer outputs and state updates with the eager reference using identical inputs and initial state, including nondefault alpha or lerp weights. Repeat multiple updates and check gradients and losses remain finite. Inspect generated arithmetic if compilation is restored.", a(), i = true), /mO\.strides\[0\]/i.test(t) && /MLA|prefill_context_chunk/i.test(t) && /FA4|flash.attn.*cute/i.test(t) && (e.action = "Manual compatibility check required: This failure concerns the FA4 MLA output layout during context-chunk prefill. Check the installed vLLM and attention dependency revisions for a fix that handles dynamic output strides. Alternatively, select a documented MLA prefill backend supported on the installed GPU and version, then rerun the failing request compositions. Disabling chunked prefill or changing dtype is not an established remedy for this evidence. Do not use an unverified boolean command-line syntax.", e.verificationCommand = "Rerun the original long-context requests and different prompt-length compositions with the same model and precision. Check valid outputs, worker survival and subsequent requests. Record the exact backend and dependency revisions; a short request succeeding is not recovery.", e.needsEvidence = ["Installed vLLM and attention dependency revisions", "Documented MLA prefill backend support for the GPU and installed version"], e.awaitingEvidence = true, a(), i = true), /layer.?norm/i.test(t) && /constant.input|all.ones|torch\.ones/i.test(t) && /non.?zero|saved.mean[^\n]{0,60}(?:low|wrong)|mean[^\n]{0,40}(?:ulp|error)|output[^\n]{0,30}must[^\n]{0,12}zero|normalization invariant failed/i.test(t) && (e.action = "Manual numerical workaround required: Preserve the original normalized dimension, epsilon and intended model computation. Test higher-precision normalization statistics or a centered reference implementation against the affected input on the installed stack. Padding the hidden dimension changes the normalization and increasing epsilon can hide rather than correct a mean error. Use an upstream kernel fix only after confirming it is present in the installed build. Measure the performance cost of any temporary reference implementation.", e.verificationCommand = "For the same shape, check constant-input output against the mathematical reference, then compare nonconstant forward outputs, all intended gradients and parameter updates with an independent high-precision reference over repeated steps. Check recurrence on the original workload. A constant-input forward check alone is insufficient.", a(), i = true), /SDPA|scaled_dot_product_attention/i.test(t) && /cuDNN|CUDNN/i.test(t) && /dQ|query.gradient|grad.*nan/i.test(t) && s && (e.summary = "The supplied backend comparison shows a nonfinite query gradient on cuDNN SDPA while the tested MATH or EFFICIENT control remains finite. The evidence supports a backend-specific discrepancy, not an unobserved sequence shape.", e.action = "Manual scoped backend workaround required: Use the MATH or EFFICIENT SDPA backend that the supplied comparison demonstrated has finite gradients for this exact mask, shape and dtype. Exclude cuDNN SDPA across forward, backward and any checkpoint recomputation. Preserve and restore the previous backend settings when leaving the affected workload. Do not substitute another backend without testing its mask support and numerical results, and do not assume the next stable version includes an upstream fix.", e.verificationCommand = "Repeat the failing mask and sequence shape. Compare outputs, dQ, dK, dV and optimizer updates with the finite reference backend, including checkpoint recomputation if used. Check finite losses and recurrence over repeated training steps, and confirm prior backend settings are restored.", a(), i = true), /compile|compiled/i.test(t) && /eager/i.test(t) && /(?:outputs?|results?|values?|gradients?)[^\n]{0,65}(?:differ|mismatch|unexpected)|(?:differ|mismatch)[^\n]{0,65}(?:outputs?|results?|values?|gradients?)|eager and compiled differ/i.test(t) && !/Triton|tl\.|mO\.strides|LayerNorm|scaled_dot_product_attention|alpha\s*=|lerp|clone\(|add_\(|mul_\(/i.test(t) && !/torch\s*[:=]?\s*\d+\.\d+|PyTorch\s+\d+\.\d+/i.test(t) && (e.summary = "The supplied evidence establishes a compiled versus eager discrepancy, but does not establish its cause.", e.action = "Read-only evidence required: Provide the exact framework and compiler versions, failing operation, shapes, dtypes, precision settings and a minimal reproducer using identical inputs and initial state. Include expected and actual values. Compare eager and compiled outputs and gradients before selecting a correction; input mutation, precision settings and compiler defects require different remedies.", e.awaitingEvidence = true, e.needsEvidence = ["Exact framework and compiler versions", "Minimal failing operation with shapes, dtypes, identical inputs and initial state", "Expected and actual outputs and gradients with precision settings"], e.verificationCommand = null, a(), i = true), /^(?:NCCL_TIMEOUT_UNATTRIBUTED|GLOO_COMMUNICATION_UNATTRIBUTED)$/.test(String(e.failureType || "")) && (e.awaitingEvidence = true, e.needsEvidence = ["First exception and last completed collective on every participating rank", "Matching scheduler exit reasons and transport or GPU events from the failure window"], e.resumeFrom = null, i = true), /checkpoint|inline_container/i.test(t) && /unexpected pos/i.test(t) && !/No space left on device|errno\D{0,8}28|quota exceeded|permission denied|read.only file system/i.test(t) && (e.awaitingEvidence = true, e.needsEvidence = ["Original write exception and errno before the unexpected-position error", "Available capacity, quota and writable mount status for the checkpoint destination"], e.action = "Read-only evidence required: Preserve the original write exception and errno, then check available capacity, quota and write capability on the checkpoint destination. An unexpected-position error alone does not establish disk exhaustion or a complete remedy. Do not delete existing checkpoints or treat an atomic-save change as proof of recovery.", e.fallbackAction = null, e.emergencyAction = null, e.resumeFrom = null, i = true), /UNEVEN_COLLECTIVE_PARTICIPATION/.test(String(e.failureType || "")) && /evaluation|evaluate\(/i.test(t) && /barrier_inside_each_iteration[^\n]*true|barrier\(\)/i.test(t) && (e.action = "Manual code change required: In the supplied uneven evaluation loop, move the explicit barrier outside the per-item loop so every rank reaches the same barrier once after local evaluation. Collect metrics only at that shared point. A DDP join context does not automatically shadow an arbitrary explicit barrier. If the model forward has other synchronized operations, review those separately before allowing uneven iteration counts.", e.verificationCommand = "Run the affected evaluation shard layout and verify every rank reaches the final collective, every intended sample contributes exactly once, and aggregated metrics match a single-process reference. A timeout disappearing after dropping samples is not successful recovery.", e.fallbackAction = null, e.emergencyAction = null, e.codeFix = null, i = true), /sparse/i.test(t) && /backward/i.test(t) && /out of memory/i.test(t) && /dense gradient|dense.*gradient matrix/i.test(t) && (e.action = "Manual implementation review required: The full dense gradient is the allocation that must be avoided. Use a sparse matmul backward implementation that computes value gradients only at the stored nonzero indices without materializing the full dense gradient matrix. Setting requires_grad only on the values is not, by itself, a verified memory fix for every PyTorch version. Validate the chosen implementation on the installed version with a small dense numerical reference for all intended gradients and measure peak memory before scaling up.", e.verificationCommand = "Compare forward values, gradients of sparse nonzero values and the other trainable operand, and parameter updates with a bounded dense reference. Measure peak allocated memory at the affected backward operation. Preserving only the forward output or disabling required gradients is not recovery.", e.fallbackAction = null, e.emergencyAction = null, e.codeFix = null, i = true);
-  let c = /* @__PURE__ */ new Set(), d = new Set((t.match(/unrecognized arguments:[^\n]*/gi) || []).flatMap((_) => _.match(/--[A-Za-z][A-Za-z0-9_-]*/g) || [])), l = new Set((t.match(/--[A-Za-z][A-Za-z0-9_-]*/g) || []).filter((_) => !d.has(_)));
-  for (let _ of ["action", "fallbackAction", "emergencyAction", "verificationCommand", "codeFix"]) {
-    let g = e[_];
-    if (_ !== "action" && g != null && [...c].some((T) => String(JSON.stringify(g) || "").includes(T))) {
-      e[_] = null, i = true;
+  let c = /* @__PURE__ */ new Set(), d = new Set((t.match(/unrecognized arguments:[^\n]*/gi) || []).flatMap((E) => E.match(/--[A-Za-z][A-Za-z0-9_-]*/g) || [])), l = new Set((t.match(/--[A-Za-z][A-Za-z0-9_-]*/g) || []).filter((E) => !d.has(E)));
+  for (let E of ["action", "fallbackAction", "emergencyAction", "verificationCommand", "codeFix"]) {
+    let k = e[E];
+    if (E !== "action" && k != null && [...c].some((O) => String(JSON.stringify(k) || "").includes(O))) {
+      e[E] = null, i = true;
       continue;
     }
-    if (typeof g != "string") continue;
-    let f = g.match(/\b(?:or\s+)?(?:run|launch)\b[^\n]{0,140}\b(?:the|your|this|existing)\s+script\b/i) || g.match(/\bpython(?:3)?\s+(?:"[^"]+\.py"|[^\s`]+\.py)/i);
-    if (!f) continue;
-    let C = (g.slice(f.index).match(/--[A-Za-z][A-Za-z0-9_-]*/g) || []).filter((T) => !l.has(T));
-    if (!C.length) continue;
-    for (let T of C) c.add(T);
-    let y = g.slice(0, f.index).replace(/[,;]\s*$/, "").trim();
-    _ === "action" && y.length > 20 && /\b(?:modify|replace|change|use|edit)\b/i.test(y) ? e.action = y + ". Apply the code edit at the failing call site. Exact launch flags for this customer script have not been verified; use its documented interface rather than guessing from Python attribute names." : _ === "action" ? (e.action = "Read-only evidence required: Provide the customer script's documented launch arguments before selecting a launch command. The proposed flags were not established by the submitted evidence.", e.awaitingEvidence = true, e.needsEvidence = ["Documented workload launch interface or optimizer parameter ownership and gradient scope"]) : e[_] = null, i = true;
-  }
-  for (let _ of ["steps", "remediationScript", "recoveryScript", "nextAction"]) e[_] != null && [...c].some((g) => String(JSON.stringify(e[_]) || "").includes(g)) && (e[_] = null);
-  /OutOfMemoryError|CUDA out of memory/i.test(t) && !/inactive_split_bytes[^\n]*[=:]\s*[1-9]|reserved[^\n]*(?:much larger|>>)[^\n]*allocated|fragmentation confirmed/i.test(t) && /expandable_segments|max_split_size_mb|allocator fragmentation/i.test(String(e.action || "")) && (e.action = "Manual evidence-led action required: Address the initiating allocation failure. Inspect live allocations, reserved and inactive split memory, other processes and the failing batch objective. Reduce activation demand only while preserving the intended effective batch and gradients. An OOM alone does not establish fragmentation; allocator tuning is not a confirmed remedy without supporting memory evidence and version compatibility.", e.fallbackAction = null, e.emergencyAction = null, e.codeFix = null, e.repairVerified = false, i = true);
-  let p = String(e.action || "");
-  /\b(?:roll back|rollback|resume|restore)\b[^.\n]*checkpoint/i.test(p) && !/verified|validate|checksum|integrity/i.test(p) && (e.action = p + " Before choosing a restore point, validate checkpoint integrity, required workload state and software compatibility. An earlier timestamp alone is not recovery proof. Preserve suspect artifacts and do not resume until the required checks pass.", e.repairVerified = false, i = true);
-  for (let _ of ["action", "fallbackAction", "emergencyAction", "codeFix"]) (typeof e[_] == "string" ? String(e[_]) : "").split(/(?<=[.!?])\s+|\n/).some((E) => /\b(?:discard|delete|wipe)\b[^.\n]*checkpoint/i.test(E) && !/\b(?:do not|don't|never|avoid)\b/i.test(E)) && (e[_] = _ === "action" ? "Manual evidence preservation required: Quarantine suspect checkpoints without deleting them. Validate integrity, required training state and a known-good reference before choosing a restore point. A hardware event alone does not prove every subsequent checkpoint is corrupt. Obtain owner approval for any retention cleanup after a usable recovery artifact is preserved." : null, e.repairVerified = false, i = true);
-  let h = /modified by an inplace operation|AUTOGRAD_INPLACE_MODIFICATION/i.test(t + " " + String(e.failureType || "")) && /optimizer\.step|opt(?:imizer)?_\w+\.step|optimizer.*modif/i.test(t), u = /generator.loss|generator_loss/i.test(t) && /critic.loss|critic_loss|discriminator.loss/i.test(t) || /shared (?:parameters|forward|graph)|objective_[a-z]|loss_[a-z]\.backward/i.test(t);
-  if (h && u) {
-    let _ = /"optimizer_parameters_overlap"\s*:\s*false/.test(t), g = /"backward_parameter_scope"\s*:\s*\{[^}]+\}/.test(t) && /"zeroing_order"\s*:\s*\[[^\]]+\]/.test(t), f = "Manual workload verification required: An exception disappearing is not proof of correct training. Check that each optimizer receives only its intended objective gradients, parameter updates match that objective, losses and gradients remain finite, and the failure does not recur. A different objective can accumulate gradients on another optimizer's parameters.";
-    e.verificationCommand = f, !_ || !g ? (e.action = "Read-only evidence required: The optimizer update before a pending backward pass explains the version error, but a reorder alone can mix objective gradients. Provide the loop including zero_grad, each backward call, each optimizer step, and the parameter sets owned by each optimizer before choosing a complete correction.", e.awaitingEvidence = true, e.needsEvidence = ["Documented workload launch interface or optimizer parameter ownership and gradient scope"], e.clarifyingQuestions = ["Which parameters does each optimizer own, and which objective backward passes write gradients to those parameters?", "Show the zero_grad, backward and step order, including any detach or parameter freezing."]) : (e.action = "Manual code change required: For the confirmed disjoint optimizer parameter sets, compute each objective gradient only with respect to the parameters owned by its optimizer, for example with torch.autograd.grad. Keep the shared forward weights unchanged until all required gradients have been computed. Assign only the intended objective gradients to each optimizer, then step. Do not accumulate a different objective into an optimizer update unless that is the documented intended objective. If ownership or the intended objective changes, review the loop again rather than applying this recipe unchanged.", e.awaitingEvidence = false, e.needsEvidence = []), e.fallbackAction = null, e.emergencyAction = null, e.codeFix = null, i = true;
+    if (typeof k != "string") continue;
+    let y = k.match(/\b(?:or\s+)?(?:run|launch)\b[^\n]{0,140}\b(?:the|your|this|existing)\s+script\b/i) || k.match(/\bpython(?:3)?\s+(?:"[^"]+\.py"|[^\s`]+\.py)/i);
+    if (!y) continue;
+    let A = (k.slice(y.index).match(/--[A-Za-z][A-Za-z0-9_-]*/g) || []).filter((O) => !l.has(O));
+    if (!A.length) continue;
+    for (let O of A) c.add(O);
+    let w = k.slice(0, y.index).replace(/[,;]\s*$/, "").trim();
+    E === "action" && w.length > 20 && /\b(?:modify|replace|change|use|edit)\b/i.test(w) ? e.action = w + ". Apply the code edit at the failing call site. Exact launch flags for this customer script have not been verified; use its documented interface rather than guessing from Python attribute names." : E === "action" ? (e.action = "Read-only evidence required: Provide the customer script's documented launch arguments before selecting a launch command. The proposed flags were not established by the submitted evidence.", e.awaitingEvidence = true, e.needsEvidence = ["Documented workload launch interface or optimizer parameter ownership and gradient scope"]) : e[E] = null, i = true;
+  }
+  for (let E of ["steps", "remediationScript", "recoveryScript", "nextAction"]) e[E] != null && [...c].some((k) => String(JSON.stringify(e[E]) || "").includes(k)) && (e[E] = null);
+  /OutOfMemoryError|CUDA out of memory/i.test(t) && !/inactive[ _]split[ _]bytes[^\n]*[=:]\s*[1-9]|reserved[^\n]*(?:much larger|>>)[^\n]*allocated|fragmentation confirmed/i.test(t) && /expandable_segments|max_split_size_mb|allocator fragmentation/i.test(String(e.action || "")) && (e.action = "Manual evidence-led action required: Address the initiating allocation failure. Inspect live allocations, reserved and inactive split memory, other processes and the failing batch objective. Reduce activation demand only while preserving the intended effective batch and gradients. An OOM alone does not establish fragmentation; allocator tuning is not a confirmed remedy without supporting memory evidence and version compatibility.", e.fallbackAction = null, e.emergencyAction = null, e.codeFix = null, e.repairVerified = false, i = true);
+  let p = t.match(/inactive[ _]split[ _]bytes\s*[=:]\s*(\d+(?:\.\d+)?)\s*(GiB|MiB|GB|MB|bytes|B)?/i), h = t.match(/(?:requested|tried to allocate)\s*(\d+(?:\.\d+)?)\s*(GiB|MiB|GB|MB|bytes|B)/i), u = (E, k = "B") => Number(E) * (/GiB/i.test(k) ? 1024 ** 3 : /MiB/i.test(k) ? 1024 ** 2 : /GB/i.test(k) ? 1e9 : /MB/i.test(k) ? 1e6 : 1);
+  if (p && h && u(p[1], p[2]) > u(h[1], h[2]) && /CUDA out of memory|CUDA.*(?:OOM|FRAGMENTATION)/i.test(t + " " + String(e.failureType || ""))) {
+    e.failureType = "CUDA_POOL_FRAGMENTATION_CANDIDATE", e.summary = "The allocation failed while the supplied snapshot reports inactive split memory larger than the requested allocation. That makes pool fragmentation a relevant competing explanation; absolute live-tensor capacity alone is not established by the reported device free memory.", e.action = "Read-only allocator compatibility check required: Reuse the supplied allocated, reserved and inactive split values. Confirm the installed PyTorch version, allocator backend and configuration before selecting a supported allocation strategy for variable shapes. Compare a controlled replay with identical input and effective batch; do not change the computation merely to remove the exception.";
+    let E = [/(?:PyTorch|torch(?:_version)?)["'\s:=]+(?:version[\s:=]+)?\d+\.\d/i.test(t) ? null : "PyTorch version", /allocator[ _]backend["'\s:=]+(?:native|cudaMallocAsync)\b/i.test(t) ? null : "allocator backend", /allocator[ _]config["'\s:=]+[^\n,}]|PYTORCH_CUDA_ALLOC_CONF\s*=/i.test(t) ? null : "allocator configuration"].filter(Boolean);
+    e.awaitingEvidence = E.length > 0, e.confidence = Math.min(60, Number(e.confidence) || 0), e.clarifyingQuestions = E.length ? ["Provide the missing allocator context: " + E.join(", ") + ". These determine which allocator remedy is supported; the supplied memory values need not be collected again."] : [], e.needsEvidence = e.clarifyingQuestions, e.codeFix = null, e.causationChain = [], e.fallbackAction = null, e.emergencyAction = null, i = true;
+  }
+  let _ = String(e.action || "");
+  /checkpoint/i.test(_) && /\b(?:roll back|rollback|resume|restore)\b|last safe checkpoint/i.test(_) && !(/integrity/i.test(_) && /(?:workload|optimizer|training) state/i.test(_) && /compatib/i.test(_)) && (e.action = _.replace(/\blast safe checkpoint\b/gi, "candidate checkpoint") + " Before choosing a restore point, validate checkpoint integrity, required workload state and software compatibility. An earlier timestamp alone is not recovery proof. Preserve suspect artifacts and do not resume until the required checks pass.", e.repairVerified = false, i = true), Array.isArray(e.clarifyingQuestions) && (e.clarifyingQuestions = e.clarifyingQuestions.map((E) => typeof E == "string" ? E.replace(/(?:last safe checkpoint|safe checkpoint boundary|safe checkpoint)/gi, "candidate checkpoint boundary, with integrity, required workload state and software compatibility checks still required") : E));
+  for (let E of ["action", "fallbackAction", "emergencyAction", "codeFix"]) (typeof e[E] == "string" ? String(e[E]) : "").split(/(?<=[.!?])\s+|\n/).some((T) => /\b(?:discard|delete|wipe)\b[^.\n]*checkpoint/i.test(T) && !/\b(?:do not|don't|never|avoid)\b/i.test(T)) && (e[E] = E === "action" ? "Manual evidence preservation required: Quarantine suspect checkpoints without deleting them. Validate integrity, required training state and a known-good reference before choosing a restore point. A hardware event alone does not prove every subsequent checkpoint is corrupt. Obtain owner approval for any retention cleanup after a usable recovery artifact is preserved." : null, e.repairVerified = false, i = true);
+  let g = /modified by an inplace operation|AUTOGRAD_INPLACE_MODIFICATION/i.test(t + " " + String(e.failureType || "")) && /optimizer\.step|opt(?:imizer)?_\w+\.step|optimizer.*modif/i.test(t), f = /generator.loss|generator_loss/i.test(t) && /critic.loss|critic_loss|discriminator.loss/i.test(t) || /shared (?:parameters|forward|graph)|objective_[a-z]|loss_[a-z]\.backward/i.test(t);
+  if (g && f) {
+    let E = /"optimizer_parameters_overlap"\s*:\s*false/.test(t), k = /"backward_parameter_scope"\s*:\s*\{[^}]+\}/.test(t) && /"zeroing_order"\s*:\s*\[[^\]]+\]/.test(t), y = "Manual workload verification required: An exception disappearing is not proof of correct training. Check that each optimizer receives only its intended objective gradients, parameter updates match that objective, losses and gradients remain finite, and the failure does not recur. A different objective can accumulate gradients on another optimizer's parameters.";
+    e.verificationCommand = y, !E || !k ? (e.action = "Read-only evidence required: The optimizer update before a pending backward pass explains the version error, but a reorder alone can mix objective gradients. Provide the loop including zero_grad, each backward call, each optimizer step, and the parameter sets owned by each optimizer before choosing a complete correction.", e.awaitingEvidence = true, e.needsEvidence = ["Documented workload launch interface or optimizer parameter ownership and gradient scope"], e.clarifyingQuestions = ["Which parameters does each optimizer own, and which objective backward passes write gradients to those parameters?", "Show the zero_grad, backward and step order, including any detach or parameter freezing."]) : (e.action = "Manual code change required: For the confirmed disjoint optimizer parameter sets, compute each objective gradient only with respect to the parameters owned by its optimizer, for example with torch.autograd.grad. Keep the shared forward weights unchanged until all required gradients have been computed. Assign only the intended objective gradients to each optimizer, then step. Do not accumulate a different objective into an optimizer update unless that is the documented intended objective. If ownership or the intended objective changes, review the loop again rather than applying this recipe unchanged.", e.awaitingEvidence = false, e.needsEvidence = []), e.fallbackAction = null, e.emergencyAction = null, e.codeFix = null, i = true;
   }
   if (i && (e.confidenceDisplay = null, e.awaitingEvidence === true && (e.confidence = Math.min(60, Number(e.confidence) || 0)), e.repairVerified = false), i && e.forensicReport && typeof e.forensicReport == "object") {
-    let _ = e.forensicReport;
-    delete _.remediationRunbook, delete _.automationReadiness, delete _.automationExecutionResult;
+    let E = e.forensicReport;
+    delete E.remediationRunbook, delete E.automationReadiness, delete E.automationExecutionResult;
   }
   return i;
 }
 function On(e) {
-  return /NEEDS_MORE|^NAN_LOSS$|UNATTRIBUTED|(?:^|_)UNKNOWN(?:_|$)|INSUFFICIENT|INCONCLUSIVE|AMBIGUOUS/i.test(String(e || ""));
+  return /NEEDS_MORE|^NAN_LOSS$|UNATTRIBUTED|UNOBSERVABLE|NODE_NOT_RESPONDING|(?:^|_)UNKNOWN(?:_|$)|INSUFFICIENT|INCONCLUSIVE|AMBIGUOUS/i.test(String(e || ""));
 }
 function Sn(e, t) {
-  if (!On(e.failureType)) return false;
-  let r = /observed_gradient|expected_gradient|reference_result_matched/i.test(t) && /backward|gradient|parameter/i.test(t), i = /NVRM|\bXid["'\s:=]*\d|\bOOMKilled\b|CUDA out of memory|NCCL[^\n]{0,80}(?:WARN|ERROR|timeout)|uncorrectable ECC/i.test(t);
-  if (!r || i) {
-    let s = Array.isArray(e.clarifyingQuestions) ? e.clarifyingQuestions.filter((l) => typeof l == "string" && l.trim().length > 0) : [], c = s.length ? s : In(t, String(e.failureType || ""));
-    if (!c.length) return false;
-    let d = e.awaitingEvidence !== true;
-    return e.awaitingEvidence = true, e.needsEvidence = c, e.clarifyingQuestions = c, e.repairVerified = false, d;
-  }
-  let n = ["Show the full affected training iteration, including parameter or leaf-tensor construction, gradient clearing, backward calls and updates. This distinguishes retained gradients from an incorrect objective or gradient hook.", "Provide the reference calculation using the same inputs and starting parameter state. This distinguishes a workload discrepancy from a comparison made against different state."], o = `Read-only evidence required: ${n.join(" ")} No complete correction is established yet.`, a = e.awaitingEvidence !== true || e.action !== o;
-  if (e.awaitingEvidence = true, e.action = o, e.clarifyingQuestions = n, e.needsEvidence = n, e.fallbackAction = null, e.emergencyAction = null, e.codeFix = null, e.confidenceDisplay = null, e.confidenceBand = "low", e.confidence = Math.min(60, Number(e.confidence) || 0), e.repairVerified = false, e.verificationCommand = "No complete correction is established. Once a correction is supported, rerun the affected iteration from the same starting state and compare intended outputs, gradients and updates with the reference.", e.forensicReport && typeof e.forensicReport == "object") {
-    let s = e.forensicReport;
-    delete s.remediationRunbook, delete s.automationReadiness, delete s.automationExecutionResult;
-  }
-  return a;
+  let r = /NCCL|GLOO|DISTRIBUTED|HANG|SLURM_NODE_FAIL/i.test(String(e.failureType || "")) && (Number(e.confidence) <= 60 || /insufficient|unknown|cannot.{0,30}(?:determin|isolat|establish|identify)|unable to determine|prevent distinguishing|not enough|lack of telemetry/i.test(String(e.summary || "")));
+  if (!On(e.failureType) && !r) return false;
+  let i = /observed_gradient|expected_gradient|reference_result_matched/i.test(t) && /backward|gradient|parameter/i.test(t), n = /NVRM|\bXid["'\s:=]*\d|\bOOMKilled\b|CUDA out of memory|NCCL[^\n]{0,80}(?:WARN|ERROR|timeout)|uncorrectable ECC/i.test(t);
+  if (!i || n) {
+    let c = Array.isArray(e.clarifyingQuestions) ? e.clarifyingQuestions.filter((m) => typeof m == "string" && m.trim().length > 0) : [], d = c.length ? c : In(t, String(e.failureType || ""));
+    if (!d.length) return false;
+    let l = e.awaitingEvidence !== true;
+    return e.awaitingEvidence = true, e.needsEvidence = d, e.clarifyingQuestions = d, e.repairVerified = false, l;
+  }
+  let o = ["Show the full affected training iteration, including parameter or leaf-tensor construction, gradient clearing, backward calls and updates. This distinguishes retained gradients from an incorrect objective or gradient hook.", "Provide the reference calculation using the same inputs and starting parameter state. This distinguishes a workload discrepancy from a comparison made against different state."], a = `Read-only evidence required: ${o.join(" ")} No complete correction is established yet.`, s = e.awaitingEvidence !== true || e.action !== a;
+  if (e.awaitingEvidence = true, e.action = a, e.clarifyingQuestions = o, e.needsEvidence = o, e.fallbackAction = null, e.emergencyAction = null, e.codeFix = null, e.confidenceDisplay = null, e.confidenceBand = "low", e.confidence = Math.min(60, Number(e.confidence) || 0), e.repairVerified = false, e.verificationCommand = "No complete correction is established. Once a correction is supported, rerun the affected iteration from the same starting state and compare intended outputs, gradients and updates with the reference.", e.forensicReport && typeof e.forensicReport == "object") {
+    let c = e.forensicReport;
+    delete c.remediationRunbook, delete c.automationReadiness, delete c.automationExecutionResult;
+  }
+  return s;
 }
 function In(e, t) {
   let r = [];
-  return /NCCL|GLOO|COLLECTIVE|TIMEOUT|UNATTRIBUTED/i.test(t) && (r.push("Collect the earliest exception and process exit code from every rank, with rank, host and timestamp. Include ranks that produced no output. This distinguishes a rank that failed first from peers reporting a secondary collective timeout."), /SeqNum\s*[=:]|collective_sequence|SequenceNumber/i.test(e) || r.push("Collect collective sequence numbers, operation names and tensor shapes from each rank. This distinguishes mismatched collective order or shapes from a stalled participant. If Flight Recorder was not enabled for this run, enable TORCH_NCCL_TRACE_BUFFER_SIZE and TORCH_NCCL_DUMP_ON_TIMEOUT before a controlled rerun; the old trace cannot be recovered."), /OOMKilled|ExitCode\s*[=:]|termination_reason\s*[=:]/i.test(e) || r.push("Provide sacct job State, ExitCode, Reason and MaxRSS, or the Kubernetes container termination reason and exit code. This distinguishes host memory exhaustion, preemption and process failure. A job ID alone does not establish why it exited. Do not assume a killed rank was a GPU memory failure.")), /OOM|MEMORY/i.test(t) && r.push("Capture per-rank allocated and reserved GPU memory at the failed operation, batch size, sequence length and other GPU processes. Compare against the last healthy run of the same workload. This distinguishes workload demand, retained allocations and contention; a current idle memory snapshot does not prove the state at failure."), /NCCL|GLOO|CUDA|XID|FABRIC|UNATTRIBUTED/i.test(t) && !/torch_version|PyTorch version|NCCL version|Driver Version/i.test(e) && r.push("Provide PyTorch, CUDA runtime, NVIDIA driver and NCCL versions on every participating node, including differences between nodes. This determines whether a compatibility remedy applies to your installed stack rather than another version."), /XID|ECC|FABRIC|NCCL|PCIE/i.test(t) && r.push("Collect nvidia-smi -q and relevant kernel Xid messages with the GPU UUID and PCI bus ID. Include available DCGM and link counters. This determines whether device or link evidence supports hardware escalation instead of changing training code; unavailable telemetry is not a healthy reading."), /AUTOGRAD|OPTIMIZER|GRADIENT|BACKWARD|NAN|NUMERIC|COMPILE|SDPA|ATTENTION/i.test(t) && r.push("Provide the smallest redacted failing operation, tensor shape and dtype, intended gradient paths, optimizer parameter ownership, and a last-good or reference result. This distinguishes a framework error from a computation or update that completes but changes the intended result. Do not include private data or credentials."), r.slice(0, 6);
+  /NODE_NOT_RESPONDING|SLURM_NODE_FAIL/i.test(t) && r.push("Ask the administrator for the affected node console, BMC system event log and scheduler heartbeat or exit reason from the failure window. This distinguishes host failure from power loss or a connectivity partition; scheduler silence alone does not identify the root cause."), (/NCCL|GLOO|COLLECTIVE|DISTRIBUTED|HANG/i.test(t) || /TIMEOUT|UNATTRIBUTED/i.test(t) && /NCCL|GLOO|collective|\brank[\s:=]+\d/i.test(e)) && (r.push("Collect the earliest exception and process exit code from every rank, with rank, host and timestamp. Include ranks that produced no output. This distinguishes a rank that failed first from peers reporting a secondary collective timeout."), /SeqNum\s*[=:]|collective_sequence|SequenceNumber/i.test(e) || r.push("Collect collective sequence numbers, operation names and tensor shapes from each rank. This distinguishes mismatched collective order or shapes from a stalled participant. If Flight Recorder was not enabled for this run, enable TORCH_NCCL_TRACE_BUFFER_SIZE and TORCH_NCCL_DUMP_ON_TIMEOUT before a controlled rerun; the old trace cannot be recovered."), /OOMKilled|ExitCode\s*[=:]|termination_reason\s*[=:]/i.test(e) || r.push("Provide sacct job State, ExitCode, Reason and MaxRSS, or the Kubernetes container termination reason and exit code. This distinguishes host memory exhaustion, preemption and process failure. A job ID alone does not establish why it exited. Do not assume a killed rank was a GPU memory failure.")), /OOM|MEMORY/i.test(t) && r.push("Capture per-rank allocated and reserved GPU memory at the failed operation, batch size, sequence length and other GPU processes. Compare against the last healthy run of the same workload. This distinguishes workload demand, retained allocations and contention; a current idle memory snapshot does not prove the state at failure."), /NCCL|GLOO|CUDA/i.test(t) && !/torch_version|PyTorch version|NCCL version|Driver Version/i.test(e) && r.push("Provide PyTorch, CUDA runtime, NVIDIA driver and NCCL versions on every participating node, including differences between nodes. This determines whether a compatibility remedy applies to your installed stack rather than another version."), /XID|ECC|FABRIC|NCCL|PCIE/i.test(t) && r.push("Collect nvidia-smi -q and relevant kernel Xid messages with the GPU UUID and PCI bus ID. Include available DCGM and link counters. This determines whether device or link evidence supports hardware escalation instead of changing training code; unavailable telemetry is not a healthy reading."), /AUTOGRAD|OPTIMIZER|GRADIENT|BACKWARD|NAN|NUMERIC|COMPILE|SDPA|ATTENTION/i.test(t) && r.push("Provide the smallest redacted failing operation, tensor shape and dtype, intended gradient paths, optimizer parameter ownership, and a last-good or reference result. This distinguishes a framework error from a computation or update that completes but changes the intended result. Do not include private data or credentials.");
+  let i = /permission denied|no cluster.admin|no sudo|only own rank|scheduler events require admin/i.test(e);
+  return r.slice(0, 6).map((n) => i && /every rank|each rank|every participating node|sacct|nvidia-smi|BMC/i.test(n) ? "Administrator handoff: " + n : n);
 }
 function G_(e, t) {
   let r = `${t}
@@ -6864,7 +6873,7 @@ function xd(e, t) {
   let r = String(e || "").split(/\r?\n/), i = t.getTime(), n = null, o = null, a = null, s = false, c = false, d = false, l = null, m = null;
   for (let p of r) {
     let h = Fd(p);
-    if (B_.test(p.replace(/(?:\/[\w.-]+)+/g, "")) && (c = true), Nd.test(p) && !Ld.test(p) && (d = true), !Ld.test(p)) continue;
+    if (B_.test(p.replace(/(?:\/[\w.-]+)+/g, "").replace(/\b(?:no|without|not)\s+(?:NaNs?|Infs?|infinity|non[- ]?finite)(?:\s+(?:or|and)\s+(?:NaNs?|Infs?|infinity|non[- ]?finite))*\b/gi, "")) && (c = true), Nd.test(p) && !Ld.test(p) && (d = true), !Ld.test(p)) continue;
     let u = V_(p) || n;
     u && (n = u), z_.test(p) && (s = true, m = u || m);
     let _ = Dd.test(p), g = d || h != null && Number.isFinite(i) && h > i;
@@ -6874,7 +6883,7 @@ function xd(e, t) {
     let p = m || l || n, h = s ? "explicit corruption or integrity-failure evidence" : c ? "non-finite or desynchronized state before the save" : "a write that began or completed after the initiating failure";
     return { lastSafeCheckpoint: o, confidence: 99, isSafe: false, message: `Checkpoint${p ? ` ${p}` : ""} has ${h}. Do not resume it.${o ? ` Use previously verified checkpoint ${o}.` : " Locate and verify an earlier checkpoint first."}`, status: s ? "CORRUPTED" : "TAINTED", latestCheckpointFound: n, safeRollbackStep: o ? Number(o) : null, safeCheckpointPath: o };
   }
-  return a ? { lastSafeCheckpoint: o, confidence: 95, isSafe: false, message: `Checkpoint ${a} was still being written when logging stopped. Do not resume it until checksum and manifest verification pass.${o ? ` The last explicitly verified checkpoint is ${o}.` : ""}`, status: "TAINTED", latestCheckpointFound: n, safeRollbackStep: o ? Number(o) : null, safeCheckpointPath: o } : o ? { lastSafeCheckpoint: o, confidence: 99, isSafe: true, message: `Integrity evidence in the submitted logs verifies checkpoint ${o}; it is safe to resume from that artifact.`, status: "SAFE", latestCheckpointFound: o, safeRollbackStep: Number(o), safeCheckpointPath: o } : { lastSafeCheckpoint: null, confidence: 0, isSafe: null, message: "No checkpoint integrity evidence was found in the provided logs; resume safety is unknown, not failed.", status: "UNVERIFIED", latestCheckpointFound: n, safeRollbackStep: null, safeCheckpointPath: null };
+  return a ? { lastSafeCheckpoint: o, confidence: 95, isSafe: false, message: `Checkpoint ${a} was still being written when logging stopped. Do not resume it until checksum and manifest verification pass.${o ? ` The last explicitly verified checkpoint is ${o}.` : ""}`, status: "TAINTED", latestCheckpointFound: n, safeRollbackStep: o ? Number(o) : null, safeCheckpointPath: o } : o ? { lastSafeCheckpoint: o, confidence: 99, isSafe: true, message: `Integrity evidence in the submitted logs verifies checkpoint ${o}; artifact integrity alone does not establish resume safety. Verify required workload state and software compatibility before a controlled resume.`, status: "SAFE", latestCheckpointFound: o, safeRollbackStep: Number(o), safeCheckpointPath: o } : { lastSafeCheckpoint: null, confidence: 0, isSafe: null, message: "No checkpoint integrity evidence was found in the provided logs; resume safety is unknown, not failed.", status: "UNVERIFIED", latestCheckpointFound: n, safeRollbackStep: null, safeCheckpointPath: null };
 }
 var q_ = /* @__PURE__ */ new Set([48, 94, 95]);
 var K_ = 64;
@@ -6924,18 +6933,18 @@ ${i}` : e, o = rr(n, /NVRM:\s*Xid|\bXid\s*\((?:PCI:)?[0-9a-fA-F:.]+\)|fallen off
 `).some(Dn), m = n.split(`
 `).some(($) => /SRAM threshold exceeded/i.test($) && !/\b(?:not|never|no)\b.{0,20}\bSRAM threshold exceeded\b/i.test($)), p = /dcgmi?\s*diag[\s\S]{0,120}?\bFAIL\b|Diagnostic\b.{0,60}\bFAIL\b/i.test(n), h = rr(n, /\bAER\b[^\n]*(?:fatal|uncorrectable)|PCIe[^\n]*(?:fatal|uncorrectable)/i, 20), u = a.filter(($) => q_.has($)), _ = s.filter(($) => /uncorrectable|double[- ]bit/i.test($)).length, g = a.length > 0 && a.every(($) => W_.has($)) && !l && _ === 0 && d === 0;
   if (!/(?:^GPU_(?:XID|OFF_BUS|ECC|HARDWARE)|ECC|ROW_?REMAP|MEMORY_ROW_REMAP|PCIE|AER|NVSWITCH|NVLINK.*(?:FATAL|HARDWARE)|HARDWARE)/i.test(t) && !g || !(o.length > 0 || s.length > 0 || h.length > 0 || d > 0 || l || m || p)) return null;
-  let C, y;
-  g ? (C = "NOT_RMA_WORTHY", y = `Only application-class Xid codes present (${a.join(", ")}, illegal address / app abort per the Xid catalog). This is almost always a software bug, not failing silicon. Do NOT RMA this GPU; debug the workload.`) : l || m || p || d >= 2 || u.length > 0 && _ >= 3 ? (C = "FIELD_DIAG_FIRST", y = "The submitted evidence points to a device that needs hardware investigation. Capture the bug report, isolate the GPU, run NVIDIA Field Diagnostic, and ask the vendor to determine RMA eligibility. DCGM and log patterns alone cannot confirm an RMA.") : d === 1 ? (C = "FIELD_DIAG_FIRST", y = "Single Xid 79 (GPU fell off the bus). Check PCIe seating, power, and thermals first. Collect a bug report and run NVIDIA Field Diagnostic if the fault persists.") : u.length > 0 || _ > 0 || a.includes(Ud) ? (C = "FIELD_DIAG_FIRST", y = a.includes(Ud) && u.length === 0 ? "Xid 63: a bad row was successfully remapped. This is the GPU healing itself, reset the GPU to apply the remap and keep monitoring. Not an RMA unless failures continue." : "Uncorrectable memory error observed. Capture a bug report before an approved reset, check row-remap status, then run diagnostics on an isolated GPU. The vendor determines RMA eligibility.") : (C = "FIELD_DIAG_FIRST", y = "Hardware-suspicious signals are present. Collect evidence and run the verification steps before asking the vendor to review a case.");
-  let T = fe(/\b(GPU-[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})\b/, n) || fe(/GPU UUID\s*:\s*([0-9a-fA-F-]{16,})/i, n), A = fe(/\bSerial Number\s*:\s*([A-Za-z0-9-]{4,})/i, n), w = fe(/Xid\s*\((?:PCI:)?([0-9a-fA-F:.]+)\)/i, n) || fe(/\b(?:GPU|device)\s+(0{4,8}:[0-9a-fA-F]{2}:[0-9a-fA-F]{2}(?:\.[0-9a-fA-F])?)/i, n), S = fe(/NVRM version: NVIDIA UNIX [\w]+ Kernel Module\s+([0-9.]+)/i, n) || fe(/Driver Version\s*:?\s*([0-9.]+)/i, n), b = fe(/VBIOS Version\s*:\s*([0-9a-fA-F.]+)/i, n), O = fe(/^(?:\[[^\]]*\]\s*|\w{3} +\d+ +[\d:]+ +)?([\w][\w.-]{1,63}) kernel:/m, n) || fe(/\b((?:gpu[-_]?worker|compute|worker|node|host)[-_]?[A-Za-z0-9._-]*\d[A-Za-z0-9._-]*)\s+(?:kernel|NVRM)\b/i, n) || (r?.hostname ? String(r.hostname).slice(0, 64) : null), k = fe(/(?:system|chassis|service tag|product) serial(?: number)?\s*:\s*([A-Za-z0-9-]{4,})/i, n) || fe(/\bService Tag\s*:?\s*([A-Za-z0-9]{5,})/i, n), D = n.match(/^(\[.*?\]|\w{3} +\d+ +\d+:\d+:\d+)/m), x = D ? D[1] : (/* @__PURE__ */ new Date()).toISOString(), B = "Unknown";
+  let k, y;
+  g ? (k = "NOT_RMA_WORTHY", y = `Only application-class Xid codes present (${a.join(", ")}, illegal address / app abort per the Xid catalog). This is almost always a software bug, not failing silicon. Do NOT RMA this GPU; debug the workload.`) : l || m || p || d >= 2 || u.length > 0 && _ >= 3 ? (k = "FIELD_DIAG_FIRST", y = "The submitted evidence points to a device that needs hardware investigation. Capture the bug report, isolate the GPU, run NVIDIA Field Diagnostic, and ask the vendor to determine RMA eligibility. DCGM and log patterns alone cannot confirm an RMA.") : d === 1 ? (k = "FIELD_DIAG_FIRST", y = "Single Xid 79 (GPU fell off the bus). Check PCIe seating, power, and thermals first. Collect a bug report and run NVIDIA Field Diagnostic if the fault persists.") : u.length > 0 || _ > 0 || a.includes(Ud) ? (k = "FIELD_DIAG_FIRST", y = a.includes(Ud) && u.length === 0 ? "Xid 63: a bad row was successfully remapped. This is the GPU healing itself, reset the GPU to apply the remap and keep monitoring. Not an RMA unless failures continue." : "Uncorrectable memory error observed. Capture a bug report before an approved reset, check row-remap status, then run diagnostics on an isolated GPU. The vendor determines RMA eligibility.") : (k = "FIELD_DIAG_FIRST", y = "Hardware-suspicious signals are present. Collect evidence and run the verification steps before asking the vendor to review a case.");
+  let T = fe(/\b(GPU-[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})\b/, n) || fe(/GPU UUID\s*:\s*([0-9a-fA-F-]{16,})/i, n), A = fe(/\bSerial Number\s*:\s*([A-Za-z0-9-]{4,})/i, n), w = fe(/Xid\s*\((?:PCI:)?([0-9a-fA-F:.]+)\)/i, n) || fe(/\b(?:GPU|device)\s+(0{4,8}:[0-9a-fA-F]{2}:[0-9a-fA-F]{2}(?:\.[0-9a-fA-F])?)/i, n), O = fe(/NVRM version: NVIDIA UNIX [\w]+ Kernel Module\s+([0-9.]+)/i, n) || fe(/Driver Version\s*:?\s*([0-9.]+)/i, n), b = fe(/VBIOS Version\s*:\s*([0-9a-fA-F.]+)/i, n), S = fe(/^(?:\[[^\]]*\]\s*|\w{3} +\d+ +[\d:]+ +)?([\w][\w.-]{1,63}) kernel:/m, n) || fe(/\b((?:gpu[-_]?worker|compute|worker|node|host)[-_]?[A-Za-z0-9._-]*\d[A-Za-z0-9._-]*)\s+(?:kernel|NVRM)\b/i, n) || (r?.hostname ? String(r.hostname).slice(0, 64) : null), C = fe(/(?:system|chassis|service tag|product) serial(?: number)?\s*:\s*([A-Za-z0-9-]{4,})/i, n) || fe(/\bService Tag\s*:?\s*([A-Za-z0-9]{5,})/i, n), D = n.match(/^(\[.*?\]|\w{3} +\d+ +\d+:\d+:\d+)/m), x = D ? D[1] : (/* @__PURE__ */ new Date()).toISOString(), B = "Unknown";
   /supermicro|\bSMC\b/i.test(n) ? B = "Supermicro" : /\bdell\b|poweredge|service tag/i.test(n) ? B = "Dell" : /\bHPE\b|proliant|apollo/i.test(n) ? B = "HPE" : /lenovo|thinksystem/i.test(n) ? B = "Lenovo" : (T || /NVRM/i.test(n)) && (B = "NVIDIA");
-  let z11 = [{ item: "GPU UUID", present: !!T, howToCollect: "nvidia-smi --query-gpu=uuid --format=csv,noheader" }, { item: "GPU serial number", present: !!A, howToCollect: "nvidia-smi --query-gpu=serial --format=csv,noheader" }, { item: "System / chassis serial (service tag)", present: !!k, howToCollect: "sudo dmidecode -s system-serial-number" }, { item: "Driver + VBIOS versions", present: !!(S && b), howToCollect: "nvidia-smi --query-gpu=driver_version,vbios_version --format=csv,noheader" }, { item: "Xid excerpt with timestamps", present: o.length > 0, howToCollect: 'dmesg -T | grep -iE "xid|nvrm" | tail -80' }, { item: "ECC / row-remap counters", present: s.length > 0, howToCollect: "nvidia-smi -q -d ECC,ROW_REMAPPER" }, { item: "nvidia-bug-report archive (NVIDIA delays RMAs without it)", present: false, howToCollect: "sudo nvidia-bug-report.sh   # run on the failing node, attach nvidia-bug-report.log.gz" }, { item: "DCGM diagnostic result", present: p, howToCollect: "dcgmi diag -r 3   # run on an isolated idle GPU" }, { item: "NVIDIA Field Diagnostic result", present: false, howToCollect: "Request the Field Diagnostic procedure and tool from NVIDIA or your hardware vendor" }, { item: "BMC system event log (server-vendor cases)", present: false, howToCollect: "ipmitool sel elist | tail -50" }], oe = fe(/nvidia-smi[^\n]*\s-i\s+(\d+)\b/i, n) || fe(/\bGPU\s*(?:index\s*)?(\d+)\b/i, n), ae = C === "NOT_RMA_WORTHY" ? ["CUDA_LAUNCH_BLOCKING=1 python train.py   # localize the first failing application kernel", "compute-sanitizer --tool memcheck python train.py   # prove the invalid access in workload code", "nvidia-smi -q -d ECC,ROW_REMAPPER   # confirm hardware counters remain clean during reproduction"] : ['nvidia-smi -q -d ROW_REMAPPER   # check "Remapping Failure Occurred" and pending remaps', "sudo nvidia-bug-report.sh        # capture evidence before any reset or power cycle", ...O ? [`Manual approval required before draining ${O}; isolate workloads before diagnostics.`] : [], ...oe ? [`Manual approval required before resetting GPU ${oe}; confirm that no workload owns it.`] : [], "Manual approval required before dcgmi diag -r 3; run it only on an isolated idle GPU.", "Manual approval required before dcgmi diag -r 4; use extended stress only on an isolated idle GPU.", "Request NVIDIA Field Diagnostic from the vendor before deciding on RMA eligibility."], se = z11.map(($) => `  [${$.present ? "x" : " "}] ${$.item}${$.present ? "" : `, collect via: ${$.howToCollect}`}`).join(`
-`), ee = [C === "FIELD_DIAG_FIRST" ? "GPU HARDWARE REVIEW DRAFT (denpex.rma.v2)" : "GPU FAULT DISPOSITION. SOFTWARE, NOT RMA (denpex.rma.v2)", "-----------------------------------------------------------", `Verdict: ${C}`, `Reason: ${y}`, "", `Vendor: ${B}`, `Host: ${O || "Unknown"}   System serial: ${k || "Not found, run dmidecode"}`, `GPU UUID: ${T || "Not found in logs"}`, `GPU serial: ${A || "Not found, run nvidia-smi --query-gpu=serial"}`, `PCI bus: ${w || "Not found"}   Driver: ${S || "?"}   VBIOS: ${b || "?"}`, `Time of failure: ${x}`, `Fault signature: ${t || "Uncorrectable hardware fault"}   Xid codes: ${a.join(", ") || "n/a"}`, "", "Evidence checklist:", se, "", o.length ? `Xid events:
+  let z11 = [{ item: "GPU UUID", present: !!T, howToCollect: "nvidia-smi --query-gpu=uuid --format=csv,noheader" }, { item: "GPU serial number", present: !!A, howToCollect: "nvidia-smi --query-gpu=serial --format=csv,noheader" }, { item: "System / chassis serial (service tag)", present: !!C, howToCollect: "sudo dmidecode -s system-serial-number" }, { item: "Driver + VBIOS versions", present: !!(O && b), howToCollect: "nvidia-smi --query-gpu=driver_version,vbios_version --format=csv,noheader" }, { item: "Xid excerpt with timestamps", present: o.length > 0, howToCollect: 'dmesg -T | grep -iE "xid|nvrm" | tail -80' }, { item: "ECC / row-remap counters", present: s.length > 0, howToCollect: "nvidia-smi -q -d ECC,ROW_REMAPPER" }, { item: "nvidia-bug-report archive (NVIDIA delays RMAs without it)", present: false, howToCollect: "sudo nvidia-bug-report.sh   # run on the failing node, attach nvidia-bug-report.log.gz" }, { item: "DCGM diagnostic result", present: p, howToCollect: "dcgmi diag -r 3   # run on an isolated idle GPU" }, { item: "NVIDIA Field Diagnostic result", present: false, howToCollect: "Request the Field Diagnostic procedure and tool from NVIDIA or your hardware vendor" }, { item: "BMC system event log (server-vendor cases)", present: false, howToCollect: "ipmitool sel elist | tail -50" }], oe = fe(/nvidia-smi[^\n]*\s-i\s+(\d+)\b/i, n) || fe(/\bGPU\s*(?:index\s*)?(\d+)\b/i, n), ae = k === "NOT_RMA_WORTHY" ? ["CUDA_LAUNCH_BLOCKING=1 python train.py   # localize the first failing application kernel", "compute-sanitizer --tool memcheck python train.py   # prove the invalid access in workload code", "nvidia-smi -q -d ECC,ROW_REMAPPER   # confirm hardware counters remain clean during reproduction"] : ['nvidia-smi -q -d ROW_REMAPPER   # check "Remapping Failure Occurred" and pending remaps', "sudo nvidia-bug-report.sh        # capture evidence before any reset or power cycle", ...S ? [`Manual approval required before draining ${S}; isolate workloads before diagnostics.`] : [], ...oe ? [`Manual approval required before resetting GPU ${oe}; confirm that no workload owns it.`] : [], "Manual approval required before dcgmi diag -r 3; run it only on an isolated idle GPU.", "Manual approval required before dcgmi diag -r 4; use extended stress only on an isolated idle GPU.", "Request NVIDIA Field Diagnostic from the vendor before deciding on RMA eligibility."], se = z11.map(($) => `  [${$.present ? "x" : " "}] ${$.item}${$.present ? "" : `, collect via: ${$.howToCollect}`}`).join(`
+`), ee = [k === "FIELD_DIAG_FIRST" ? "GPU HARDWARE REVIEW DRAFT (denpex.rma.v2)" : "GPU FAULT DISPOSITION. SOFTWARE, NOT RMA (denpex.rma.v2)", "-----------------------------------------------------------", `Verdict: ${k}`, `Reason: ${y}`, "", `Vendor: ${B}`, `Host: ${S || "Unknown"}   System serial: ${C || "Not found, run dmidecode"}`, `GPU UUID: ${T || "Not found in logs"}`, `GPU serial: ${A || "Not found, run nvidia-smi --query-gpu=serial"}`, `PCI bus: ${w || "Not found"}   Driver: ${O || "?"}   VBIOS: ${b || "?"}`, `Time of failure: ${x}`, `Fault signature: ${t || "Uncorrectable hardware fault"}   Xid codes: ${a.join(", ") || "n/a"}`, "", "Evidence checklist:", se, "", o.length ? `Xid events:
 ${o.slice(0, 12).map(($) => `  ${$}`).join(`
 `)}` : "", s.length ? `ECC / remap evidence:
 ${s.slice(0, 12).map(($) => `  ${$}`).join(`
-`)}` : "", "", C === "FIELD_DIAG_FIRST" ? "This is an evidence draft, not an RMA approval. Run the verification flow and ask the vendor to determine eligibility." : "Denpex assessment: NOT RMA-worthy. The evidence points at software, not silicon.", "", "Generated by Denpex ML Incident Intelligence."].filter(Boolean).join(`
+`)}` : "", "", k === "FIELD_DIAG_FIRST" ? "This is an evidence draft, not an RMA approval. Run the verification flow and ask the vendor to determine eligibility." : "Denpex assessment: NOT RMA-worthy. The evidence points at software, not silicon.", "", "Generated by Denpex ML Incident Intelligence."].filter(Boolean).join(`
 `);
-  return { schema: "denpex.rma.v2", verdict: C, verdictReason: y, vendor: B, gpuUuid: T, gpuSerial: A, pciBusId: w, driverVersion: S, vbiosVersion: b, hostname: O, systemSerial: k, faultTimestamp: x, faultSignature: t || "Uncorrectable Hardware Fault", xidCodes: a, xidEvents: o, eccEvidence: s, nvlinkEvidence: c, evidenceChecklist: z11, verificationCommands: ae, autoGeneratedTicket: ee };
+  return { schema: "denpex.rma.v2", verdict: k, verdictReason: y, vendor: B, gpuUuid: T, gpuSerial: A, pciBusId: w, driverVersion: O, vbiosVersion: b, hostname: S, systemSerial: C, faultTimestamp: x, faultSignature: t || "Uncorrectable Hardware Fault", xidCodes: a, xidEvents: o, eccEvidence: s, nvlinkEvidence: c, evidenceChecklist: z11, verificationCommands: ae, autoGeneratedTicket: ee };
 }
 function $_(e, t) {
   let r = t.toLowerCase();
@@ -7201,12 +7210,12 @@ function Mn(e, t) {
 function Yr(e, t) {
   Ti(e, t), Mn(e, t);
   let r = $d(e.evidence, t);
-  if (!r.some((C) => C.claimable)) {
-    let C = vy(e, t);
-    C.length && (e.matcherEvidence = e.evidence, e.evidence = C, r = $d(e.evidence, t));
+  if (!r.some((k) => k.claimable)) {
+    let k = vy(e, t);
+    k.length && (e.matcherEvidence = e.evidence, e.evidence = k, r = $d(e.evidence, t));
   }
   e.evidenceProvenance = r;
-  let i = Zd(e), n = Math.max(0, Math.min(100, Number(e.confidence) || 0)), o = String(e.sourceType || ""), a = String(e.failureType || "UNKNOWN"), s = e.falsificationReport, c = [...Nt(e.contradictions), ...Nt(s?.evidenceAgainst)], d = [...Nt(e.needsEvidence), ...Nt(s?.missingRequiredEvidence)], l = r.filter((C) => C.claimable).length, m = /^(?:generalization_causal_detector|unified_causal_precedence|negative-control)$/.test(o), p = l > 0 && (e.causalDiscriminator === true || m || s?.status === "CONFIRMED" || Ty(r)), h = Oy(e), u = [], _ = 100, g = /^(?:UNKNOWN|NEEDS_|INCOMPLETE_|INVALID_)/i.test(a) || /^(?:triage|timeout_fallback)$/.test(o) || e.evidenceFirst === true;
+  let i = Zd(e), n = Math.max(0, Math.min(100, Number(e.confidence) || 0)), o = String(e.sourceType || ""), a = String(e.failureType || "UNKNOWN"), s = e.falsificationReport, c = [...Nt(e.contradictions), ...Nt(s?.evidenceAgainst)], d = [...Nt(e.needsEvidence), ...Nt(s?.missingRequiredEvidence)], l = r.filter((k) => k.claimable).length, m = /^(?:generalization_causal_detector|unified_causal_precedence|negative-control)$/.test(o), p = l > 0 && (e.causalDiscriminator === true || m || s?.status === "CONFIRMED" || Ty(r)), h = Oy(e), u = [], _ = 100, g = /^(?:UNKNOWN|NEEDS_|INCOMPLETE_|INVALID_)/i.test(a) || /^(?:triage|timeout_fallback)$/.test(o) || e.evidenceFirst === true;
   !p && !g && (_ = Math.min(_, kr), u.push("no grounded causal discriminator")), l === 0 && !/^(?:NO_FAILURE_|NO_FAULT_|HEALTHY_)/i.test(a) && (_ = Math.min(_, 45), u.push("no claimable evidence span")), c.length > 0 && (_ = Math.min(_, 60), u.push("submitted evidence contradicts the selected owner")), d.length > 0 && (_ = Math.min(_, 60), u.push("required discriminator is missing")), h && (_ = Math.min(_, kr), u.push("multiple causal owners remain plausible")), !i.complete && !g && (_ = Math.min(_, kr), u.push("remediation contract is incomplete")), fy(a, t) && (_ = Math.min(_, kr), u.push("GPU hardware verdict rests on driver-class Xids with no hardware corroboration")), !g && ly(t) && (_ = Math.min(_, kr), u.push("no machine-generated evidence in the submission, the report is prose only")), uy(a, t) && (_ = Math.min(_, kr), u.push("storage stall asserted with no evidence that I/O blocked")), g && (_ = Math.min(_, 60));
   let f = Math.min(n, _);
   f < n && (e.ungovernedConfidence = n, e.confidence = f), Cy(e, f, p, ky(t)), Zd(e);
@@ -7310,16 +7319,16 @@ function Bn(e, t) {
   let p = zd(t.logs, String(e.failureType || ""), t.telemetry);
   if (t.allowRmaPayload && p) {
     r.vendorRmaPayload = p;
-    let w = Oi(p), S = Oi(p, { system: "jira" }), b = Oi(p, { system: "servicenow" });
-    r.vendorRmaPackage = { ready: p.verdict === "RMA_CONFIRMED", verdict: p.verdict, oemTarget: p.vendor, pcieBusId: p.pciBusId, gpuUuid: p.gpuUuid, gpuSerial: p.gpuSerial, hardwareXids: p.xidCodes, hardwareProofSummary: p.verdictReason, oneClickTicketMarkdown: p.autoGeneratedTicket, ticketDrafts: { generic: w, jira: S, serviceNow: b } }, e.vendorRmaPackage = r.vendorRmaPackage;
+    let w = Oi(p), O = Oi(p, { system: "jira" }), b = Oi(p, { system: "servicenow" });
+    r.vendorRmaPackage = { ready: p.verdict === "RMA_CONFIRMED", verdict: p.verdict, oemTarget: p.vendor, pcieBusId: p.pciBusId, gpuUuid: p.gpuUuid, gpuSerial: p.gpuSerial, hardwareXids: p.xidCodes, hardwareProofSummary: p.verdictReason, oneClickTicketMarkdown: p.autoGeneratedTicket, ticketDrafts: { generic: w, jira: O, serviceNow: b } }, e.vendorRmaPackage = r.vendorRmaPackage;
   } else delete r.vendorRmaPayload, delete r.vendorRmaPackage, delete e.vendorRmaPackage;
   let h = xd(t.logs, t.crashTimestamp ?? Md(t.logs));
   r.checkpointIntegrity = h, e.checkpointIntegrity = { latestCheckpointFound: h.latestCheckpointFound ?? null, status: h.status ?? "UNVERIFIED", reason: h.message, safeRollbackStep: h.safeRollbackStep ?? null, safeCheckpointPath: h.safeCheckpointPath ?? null, safeToResume: h.isSafe };
   let u = bi(t.logs);
   if (u) {
     e.initiatingCulprit = { nodeHostname: u.culpritNode, rank: u.culpritRank, localRank: u.culpritLocalRank, physicalGpuIndex: u.culpritPhysicalGpuIndex, pciBusId: u.culpritGpuBusId, hardwareXid: u.exactXidCode, xidDescription: u.exactXidCode === null ? null : `NVIDIA Xid ${u.exactXidCode}`, initiatingTimestamp: u.culpritTimestamp };
-    let w = Wr(t.logs), S = w.victims.map((b) => b.rank).filter((b) => b !== null).sort((b, O) => b - O);
-    e.causalChain = { patientZero: u.initiatingEvent, propagationDelaySeconds: u.propagationDelaySeconds, victimCount: u.victimCount, exoneratedVictims: [{ rankRange: S.length > 0 ? S.join(", ") : null, errorSeen: w.victims[0]?.line ?? "Downstream timeout or supervisor abort", reason: "This event followed the initiating fault inside the bounded causal window, so it is treated as a victim signal." }] }, r.initiatingCulprit = e.initiatingCulprit, r.causalChain = e.causalChain;
+    let w = Wr(t.logs), O = w.victims.map((b) => b.rank).filter((b) => b !== null).sort((b, S) => b - S);
+    e.causalChain = { patientZero: u.initiatingEvent, propagationDelaySeconds: u.propagationDelaySeconds, victimCount: u.victimCount, exoneratedVictims: [{ rankRange: O.length > 0 ? O.join(", ") : null, errorSeen: w.victims[0]?.line ?? "Downstream timeout or supervisor abort", reason: "This event followed the initiating fault inside the bounded causal window, so it is treated as a victim signal." }] }, r.initiatingCulprit = e.initiatingCulprit, r.causalChain = e.causalChain;
   }
   let _ = Vd(t.logs);
   (_.regressionDetected || _.delayedMemoryLeak) && (r.goodputAnalysis = _);
@@ -7329,9 +7338,9 @@ function Bn(e, t) {
   f.bottleneck !== "UNKNOWN" && (r.inferenceAnalysis = f);
   let E = Wd(t.logs);
   E.source !== "unknown" && (r.incidentOwner = E);
-  let C = Kd(t.logs);
-  if (C.supported && (r.minimalReproduction = C), u && !o && !/Cluster integration NVSentinel (?:node|GPUReset) observation:/i.test(t.logs) && (r.remediationRunbook = jd({ logs: t.logs, culpritNode: u.culpritNode, initiatingEvent: u.initiatingEvent, jobId: t.logs.match(/\bSLURM_JOB_ID\s*[=:]\s*(\d+)\b/i)?.[1] ?? null, safeCheckpoint: h.lastSafeCheckpoint })), e.awaitingEvidence === true) {
-    let w = Array.isArray(e.clarifyingQuestions) ? e.clarifyingQuestions.filter((S) => typeof S == "string") : [];
+  let k = Kd(t.logs);
+  if (k.supported && (r.minimalReproduction = k), u && !o && !/Cluster integration NVSentinel (?:node|GPUReset) observation:/i.test(t.logs) && (r.remediationRunbook = jd({ logs: t.logs, culpritNode: u.culpritNode, initiatingEvent: u.initiatingEvent, jobId: t.logs.match(/\bSLURM_JOB_ID\s*[=:]\s*(\d+)\b/i)?.[1] ?? null, safeCheckpoint: h.lastSafeCheckpoint })), e.awaitingEvidence === true) {
+    let w = Array.isArray(e.clarifyingQuestions) ? e.clarifyingQuestions.filter((O) => typeof O == "string") : [];
     e.clarifyingQuestions = [.../* @__PURE__ */ new Set([...In(t.logs, String(e.failureType || "")), ...w])].slice(0, 8);
   }
   By(e);
@@ -7342,7 +7351,7 @@ function Bn(e, t) {
 }
 function il(e) {
   if (/NVRM[^\n]*Xid\s*(?:\([^)]*\))?\s*[:=]?\s*(?:48|62|74|79|94|95|119|120|149)\b|uncorrectable ECC|Above 4G Decoding|BAR[^\n]*failed to assign|EXT4-fs[^\n]*(?:aborted journal|error|read-only)|groups missing|membership pruned|pip install[^\n]*(?:-U|--upgrade)[^\n]*torch|Memory cgroup out of memory|oom_kill|State=OUT_OF_MEMORY/i.test(e)) return null;
-  let t = e.split(/\r?\n/), r = (b, O, k, D, x, B = []) => ({ tier: 0, failureType: b, confidence: B.length ? 60 : 95, sourceType: "operation-evidence-contract", causalDiscriminator: true, summary: k, action: (B.length ? "Read-only evidence required: " : "Manual action required: ") + D, verificationCommand: "Manual verification required: " + x, evidence: t.filter((z11) => O.test(z11)).slice(0, 4), awaitingEvidence: B.length > 0, inviteFollowup: B.length > 0, clarifyingQuestions: B, needsEvidence: B, fallbackAction: null, emergencyAction: null, resumeFrom: null, repairVerified: false }), i = /(?:RuntimeError:)?[^\n]*view size is not compatible with input tensor.*stride/i;
+  let t = e.split(/\r?\n/), r = (b, S, C, D, x, B = []) => ({ tier: 0, failureType: b, confidence: B.length ? 60 : 95, sourceType: "operation-evidence-contract", causalDiscriminator: true, summary: C, action: (B.length ? "Read-only evidence required: " : "Manual action required: ") + D, verificationCommand: "Manual verification required: " + x, evidence: t.filter((z11) => S.test(z11)).slice(0, 4), awaitingEvidence: B.length > 0, inviteFollowup: B.length > 0, clarifyingQuestions: B, needsEvidence: B, fallbackAction: null, emergencyAction: null, resumeFrom: null, repairVerified: false }), i = /(?:RuntimeError:)?[^\n]*view size is not compatible with input tensor.*stride/i;
   if (i.test(e)) return r("NON_CONTIGUOUS_TENSOR_VIEW", i, "The requested view is incompatible with the input strides. This is a tensor layout contract, not evidence of a TensorRT quantization failure.", "Inspect shape, strides and intended logical element order. Use reshape, or contiguous followed by view, only when that order matches the intended computation. These operations may copy data; do not invent customer launch flags.", "Replay the exact operation and compare every output element with the intended logical-order reference. For training, also compare gradients and parameter updates. Absence of the exception alone is insufficient.");
   let n = /(?:RuntimeError:)?[^\n]*Expected all tensors to be on the same device/i;
   if (n.test(e)) return r("PT_DEVICE_MISMATCH", n, "Operands of the failing operation reside on different devices. The submitted exception does not imply TorchScript is in use.", /argument index|index\.device\s*=\s*cpu/i.test(e) ? "Put the index operand on the input tensor device at this call site, preserving its required integer dtype. Check restored buffers and dynamically created tensors. Do not relocate the entire workload without checking the exact operands." : "Inspect device and dtype of the exact operands, model buffers and restored optimizer state at the failing call. Place them deliberately on compatible devices without casting integer labels or changing the intended operation.", "Replay the same failing operation with the same values. Compare selected elements or outputs and, where applicable, gradients and parameter updates against an agreed reference.");
@@ -7373,8 +7382,8 @@ function il(e) {
   if (f.test(e) && /(?:allocatable|available)[^\n]*GPU|GPU[^\n]*(?:allocatable|available)/i.test(e)) return r("GANG_SCHEDULING_CAPACITY", f, "The gang cannot obtain its required simultaneous eligible capacity. Training has not yet started, so runtime GPU repair is not indicated.", "Compare all-member requirements, eligible topology and queue quota with current allocatable capacity. Request approved capacity or deliberately revise the parallel plan. Do not evict other tenants or silently reduce the gang.", "Confirm every required member schedules and runs the intended parallel workload with matching collective participation and correct outputs.");
   let E = /maximum context length[^\n]*(?:tokens|exceed)|max(?:imum)? seq len[^\n]*larger than[^\n]*KV cache/i;
   if (E.test(e)) return r(/KV cache/i.test(e) ? "INFERENCE_KV_CACHE_EXHAUSTION" : "INFERENCE_CONTEXT_CAPACITY", E, /KV cache/i.test(e) ? "The configured sequence length exceeds measured KV cache capacity." : "The request exceeds the server context policy. This does not by itself establish a GPU allocation failure.", "Fit the intended prompt and completion budget to the documented context and measured cache capacity. Use deliberate chunking/retrieval or supported capacity/parallelism when content must be preserved. Do not silently truncate required content or reduce a memory budget to increase cache capacity.", "Exercise representative prompt lengths and concurrency. Check required content coverage, expected output behavior and stable memory/progress before claiming serving recovery.");
-  let C = /CUDA error:\s*invalid device ordinal/i;
-  if (C.test(e)) return r("CUDA_VISIBLE_DEVICE_ORDINAL", C, "The selected CUDA device ordinal is unavailable in this process. CUDA_VISIBLE_DEVICES remaps physical devices into logical visible indices.", "Compare the visible logical device count with the selected local rank/index in the launching process. Select the intended visible ordinal rather than a physical GPU id. Do not change shared cluster visibility without authorization.", "Confirm logical-to-physical mapping and run a known operation on each intended rank/device with the expected result.");
+  let k = /CUDA error:\s*invalid device ordinal/i;
+  if (k.test(e)) return r("CUDA_VISIBLE_DEVICE_ORDINAL", k, "The selected CUDA device ordinal is unavailable in this process. CUDA_VISIBLE_DEVICES remaps physical devices into logical visible indices.", "Compare the visible logical device count with the selected local rank/index in the launching process. Select the intended visible ordinal rather than a physical GPU id. Do not change shared cluster visibility without authorization.", "Confirm logical-to-physical mapping and run a known operation on each intended rank/device with the expected result.");
   let y = /(?:RuntimeError:|CUDA error:)[^\n]*no kernel image is available for execution on the device/i;
   if (y.test(e)) return r("CUDA_KERNEL_ARCHITECTURE_UNSUPPORTED", y, "The selected binary has no executable kernel image for this target device. This does not establish that TGI or a specific serving launcher is installed.", "Compare the failing extension binary target architectures with the actual GPU compute capability and runtime. Use its documented compatible build or a supported reference backend. Do not guess launcher flags or upgrade unrelated packages.", "Execute the exact affected kernel and compare outputs and gradients with the supported reference on representative inputs.");
   let T = /(?:metrics|losses|history)\.append\(loss\)/i;
@@ -7383,8 +7392,8 @@ function il(e) {
   if (A.test(e) && /(?:sm_|compute capability|Target)[^\n]*\d/i.test(e)) return r("QUANTIZATION_BACKEND_SUPPORT", A, "The selected quantization backend rejects the target capability. A different model format or backend requires its own compatibility and quality checks.", "Check the installed backend support matrix and model artifact format. Select a documented supported backend/artifact or supported hardware. Do not assume a quantized checkpoint can be losslessly converted.", "Compare representative output quality and intended serving behavior before substituting a model format or backend.");
   let w = /Invalid __(?:global|shared)__ (?:read|write)/i;
   if (w.test(e) && /Compute Sanitizer|compute-sanitizer|========= /i.test(e)) return r("SANITIZER_INVALID_MEMORY_ACCESS", w, "The sanitizer identifies an invalid memory access in the application kernel. An accompanying Xid 43 is not sufficient to declare a physical hardware fault.", "Inspect the reported kernel location, index bounds, allocated buffer extent and lifetime. Correct the violating access in an isolated reproduction. Do not replace hardware on this application evidence alone.", "Require sanitizer-clean execution and independently correct numerical outputs and gradients for the original operation. A clean diagnostic exit alone is insufficient.");
-  let S = /tokenizers[^\n]*forked|Disabling parallelism to avoid deadlocks/i;
-  return S.test(e) && /(?:subsequent|all)[^\n]*batches complete|no hang|no process exit/i.test(e) && !/RuntimeError:|Traceback \(most recent|actual.*hang|stopped progressing/i.test(e) ? r("TOKENIZER_FORK_WARNING_ONLY", S, "The tokenizer reports a fork/parallelism warning, while the supplied observations show continued work. This does not establish an actual deadlock.", "No crash remedy is established. If changing initialization or TOKENIZERS_PARALLELISM, evaluate the supported policy for the installed version and preserve intended tokenization. Investigate only if progress or throughput actually regresses.", "Confirm continued representative batch progress and identical token outputs; do not label a warning-only run as a verified repair.") : /\bExitCode=0\b/i.test(e) && /completed\s+\d+\s+steps/i.test(e) && /gradients finite/i.test(e) && !/RuntimeError:|Traceback \(most recent|NCCL.*timeout|Xid\s*\d|OOMKilled|loss\s*=\s*(?:nan|inf)/i.test(e) ? { ...r("NO_FAILURE_OBSERVED", /ExitCode=0|completed\s+\d+\s+steps|gradients finite/i, "The supplied window shows completed steps, finite gradients and a successful exit. No specific failure is established in this window; this is not certification of every workload state.", "No repair is indicated by these observations. If another interval failed, submit its first exception or termination event with the affected operation.", "No repair has been applied or verified. Continued healthy work is separate from a recovery claim."), confidence: 90 } : null;
+  let O = /tokenizers[^\n]*forked|Disabling parallelism to avoid deadlocks/i;
+  return O.test(e) && /(?:subsequent|all)[^\n]*batches complete|no hang|no process exit/i.test(e) && !/RuntimeError:|Traceback \(most recent|actual.*hang|stopped progressing/i.test(e) ? r("TOKENIZER_FORK_WARNING_ONLY", O, "The tokenizer reports a fork/parallelism warning, while the supplied observations show continued work. This does not establish an actual deadlock.", "No crash remedy is established. If changing initialization or TOKENIZERS_PARALLELISM, evaluate the supported policy for the installed version and preserve intended tokenization. Investigate only if progress or throughput actually regresses.", "Confirm continued representative batch progress and identical token outputs; do not label a warning-only run as a verified repair.") : /\bExitCode=0\b/i.test(e) && /completed\s+\d+\s+steps/i.test(e) && /gradients finite/i.test(e) && !/RuntimeError:|Traceback \(most recent|NCCL.*timeout|Xid\s*\d|OOMKilled|loss\s*=\s*(?:nan|inf)/i.test(e) ? { ...r("NO_FAILURE_OBSERVED", /ExitCode=0|completed\s+\d+\s+steps|gradients finite/i, "The supplied window shows completed steps, finite gradients and a successful exit. No specific failure is established in this window; this is not certification of every workload state.", "No repair is indicated by these observations. If another interval failed, submit its first exception or termination event with the affected operation.", "No repair has been applied or verified. Continued healthy work is separate from a recovery claim."), confidence: 90 } : null;
 }
 var nl = "Enable PyTorch Flight Recorder (free, built into PyTorch >= 2.4) so the next hang is provable instead of inferred: set TORCH_NCCL_TRACE_BUFFER_SIZE=2000 and TORCH_NCCL_DUMP_ON_TIMEOUT=true on every rank. On the next watchdog timeout each rank writes a trace dump; paste the dumps here (or run `python -m torch.distributed.flight_recorder.fr_trace -j <dump_dir>`) and the desynced rank is identified deterministically.";
 function Si(e) {
@@ -7516,9 +7525,9 @@ function ol(e) {
   }
   return null;
 }
-var ir = "2026.10.09+checkpoint-history-evidence-r28";
-var Pb = Object.freeze(["worker/workload-instruction-safety.ts", "worker/remedy-applicability.ts", "worker/follow-up-context.ts", "worker/diagnosis-worker-optimized.ts", "worker/control-arm-falsification.ts", "worker/diagnosis-governance.ts", "worker/telemetry-parse.ts", "worker/keyword-match-tier.ts", "worker/detectors.ts", "worker/config-fault-detectors.ts", "worker/host-config-detectors.ts", "worker/environment-fault-detectors.ts", "worker/fleet-config-detectors.ts", "worker/infra-fault-detectors.ts", "worker/scheduler-network-detectors.ts", "worker/access-control-detectors.ts", "worker/fabric-config-detectors.ts", "worker/host-runtime-detectors.ts", "worker/pipeline-runtime-detectors.ts", "worker/host-capacity-detectors.ts", "worker/orchestration-precision-detectors.ts", "worker/capacity-transport-detectors.ts", "worker/throughput-runtime-detectors.ts", "worker/policy-configuration-detectors.ts", "worker/adversarial-causal-detectors.ts", "worker/detector-precedence.ts", "worker/incident-fact-ledger.ts", "worker/deep-reasoning-contract.ts", "worker/long-tail-causal-detectors.ts", "worker/precision-causal-detectors.ts", "worker/generalization-causal-detectors.ts", "worker/causal-timeline.ts", "worker/generalization-cycle-3-detectors.ts", "worker/generalization-cycle-4-detectors.ts", "worker/generalization-cycle-4-revealed-regressions.ts", "worker/generalization-cycle-5-detectors.ts", "worker/generalization-cycle-6-detectors.ts", "worker/generalization-cycle-7-revealed-regressions.ts", "worker/agent-challenge-causal-detectors.ts", "worker/clarifying-questions.ts", "worker/investigation-loop.ts", "worker/remediation-engine.ts", "worker/rma-generator.ts", "worker/negated-evidence.ts", "worker/healthy-baseline.ts", "worker/host-memory-evidence.ts", "worker/patterns/registry.ts", "worker/patterns/broad_taxonomy_patterns.ts", "worker/patterns/family-contracts.ts", "worker/patterns/seo_gap_patterns.ts", "worker/patterns/audit_gap_patterns.ts", "worker/patterns/cycle9_gap_patterns.ts", "worker/patterns/master_improvements.ts", "worker/patterns/remediation_gap_patterns.ts", "worker/patterns/silent_failures.ts", "worker/patterns/checkpoint_errors.ts", "worker/patterns/ddp_errors.ts", "worker/patterns/enterprise_core.ts", "worker/patterns/fallen_off_bus.ts", "worker/patterns/flash_attention.ts", "worker/patterns/fs_errors.ts", "worker/patterns/gpu_reset.ts", "worker/patterns/hung_rank.ts", "worker/patterns/hw_slowdown.ts", "worker/patterns/ib_errors.ts", "worker/patterns/index.ts", "worker/patterns/inference_serving.ts", "worker/patterns/init_errors.ts", "worker/patterns/jax_flax.ts", "worker/patterns/kernel_deadlock.ts", "worker/patterns/kmsg_gpu_errors.ts", "worker/patterns/nccl_hang_detector.ts", "worker/patterns/nccl_slow_node.ts", "worker/patterns/nvlink_check.ts", "worker/patterns/pcie_degradation.ts", "worker/patterns/prefilter.ts", "worker/patterns/straggler_detector.ts", "worker/patterns/sxid_detector.ts", "worker/patterns/thermal_check.ts", "worker/patterns/xid_core.ts", "worker/clock-skew.ts", "worker/serving-slo.ts", "worker/workload-class.ts", "worker/encyclopedia-name-evidence.ts", "worker/encyclopedia-negated-claim.ts", "worker/torchelastic-exitcode.ts", "worker/input-starvation.ts", "worker/cuda-driver-compat.ts", "worker/loss-scale-thrash.ts", "worker/transient-recovered.ts", "worker/throughput-paste-detector.ts", "worker/throughput-log-parse.ts", "worker/throughput-regression.ts", "worker/contracts.ts", "worker/finalize-diagnosis.ts", "worker/checkpoint-analyzer.ts", "worker/goodput.ts", "worker/straggler-detector.ts", "worker/inference-analysis.ts", "worker/fleet-risk.ts", "worker/remediation-scripts.ts", "worker/diagnostic-evidence-requests.ts", "worker/operation-evidence-contracts.ts", "worker/ownership.ts", "worker/mre-generator.ts", "worker/rma-ticket.ts", "worker/log-negation.ts", "worker/diagnostic-input-quality.ts", "worker/exoneration.ts", "worker/evidence-hierarchy.ts", "worker/failure-priors.ts", "worker/confidence-calibration.ts", "worker/encyclopedia-match.ts", "worker/bayesian-network.ts", "src/data/failures-matcher.ts", "src/lib/log-extract.ts", "src/lib/what-to-do.ts"]);
-var Hn = "28b6f5650b7b33e8b7655cfc1e9aef64d1cbcd9c9a72a06fe4d4edb22519f828";
+var ir = "2026.10.09+verified-recovery-boundaries-r32";
+var Pb = Object.freeze(["worker/workload-instruction-safety.ts", "worker/remedy-applicability.ts", "worker/follow-up-context.ts", "worker/diagnosis-worker-optimized.ts", "worker/control-arm-falsification.ts", "worker/diagnosis-governance.ts", "worker/telemetry-parse.ts", "worker/keyword-match-tier.ts", "worker/detectors.ts", "worker/config-fault-detectors.ts", "worker/host-config-detectors.ts", "worker/environment-fault-detectors.ts", "worker/fleet-config-detectors.ts", "worker/infra-fault-detectors.ts", "worker/scheduler-network-detectors.ts", "worker/access-control-detectors.ts", "worker/fabric-config-detectors.ts", "worker/host-runtime-detectors.ts", "worker/pipeline-runtime-detectors.ts", "worker/host-capacity-detectors.ts", "worker/orchestration-precision-detectors.ts", "worker/capacity-transport-detectors.ts", "worker/throughput-runtime-detectors.ts", "worker/policy-configuration-detectors.ts", "worker/adversarial-causal-detectors.ts", "worker/detector-precedence.ts", "worker/incident-fact-ledger.ts", "worker/deep-reasoning-contract.ts", "worker/long-tail-causal-detectors.ts", "worker/precision-causal-detectors.ts", "worker/generalization-causal-detectors.ts", "worker/causal-timeline.ts", "worker/generalization-cycle-3-detectors.ts", "worker/generalization-cycle-4-detectors.ts", "worker/generalization-cycle-4-revealed-regressions.ts", "worker/generalization-cycle-5-detectors.ts", "worker/generalization-cycle-6-detectors.ts", "worker/generalization-cycle-7-revealed-regressions.ts", "worker/agent-challenge-causal-detectors.ts", "worker/clarifying-questions.ts", "worker/investigation-loop.ts", "worker/remediation-engine.ts", "worker/rma-generator.ts", "worker/negated-evidence.ts", "worker/healthy-baseline.ts", "worker/host-memory-evidence.ts", "worker/patterns/registry.ts", "worker/patterns/broad_taxonomy_patterns.ts", "worker/patterns/family-contracts.ts", "worker/patterns/seo_gap_patterns.ts", "worker/patterns/audit_gap_patterns.ts", "worker/patterns/cycle9_gap_patterns.ts", "worker/patterns/master_improvements.ts", "worker/patterns/remediation_gap_patterns.ts", "worker/patterns/silent_failures.ts", "worker/patterns/checkpoint_errors.ts", "worker/patterns/ddp_errors.ts", "worker/patterns/enterprise_core.ts", "worker/patterns/fallen_off_bus.ts", "worker/patterns/flash_attention.ts", "worker/patterns/fs_errors.ts", "worker/patterns/gpu_reset.ts", "worker/patterns/hung_rank.ts", "worker/patterns/hw_slowdown.ts", "worker/patterns/ib_errors.ts", "worker/patterns/index.ts", "worker/patterns/inference_serving.ts", "worker/patterns/init_errors.ts", "worker/patterns/jax_flax.ts", "worker/patterns/kernel_deadlock.ts", "worker/patterns/kmsg_gpu_errors.ts", "worker/patterns/nccl_hang_detector.ts", "worker/patterns/nccl_slow_node.ts", "worker/patterns/nvlink_check.ts", "worker/patterns/pcie_degradation.ts", "worker/patterns/prefilter.ts", "worker/patterns/straggler_detector.ts", "worker/patterns/sxid_detector.ts", "worker/patterns/thermal_check.ts", "worker/patterns/xid_core.ts", "worker/clock-skew.ts", "worker/serving-slo.ts", "worker/workload-class.ts", "worker/encyclopedia-name-evidence.ts", "worker/encyclopedia-negated-claim.ts", "worker/torchelastic-exitcode.ts", "worker/input-starvation.ts", "worker/cuda-driver-compat.ts", "worker/loss-scale-thrash.ts", "worker/transient-recovered.ts", "worker/throughput-paste-detector.ts", "worker/throughput-log-parse.ts", "worker/throughput-regression.ts", "worker/contracts.ts", "worker/finalize-diagnosis.ts", "worker/checkpoint-analyzer.ts", "worker/goodput.ts", "worker/straggler-detector.ts", "worker/inference-analysis.ts", "worker/fleet-risk.ts", "worker/remediation-scripts.ts", "worker/diagnostic-evidence-requests.ts", "worker/operation-evidence-contracts.ts", "worker/ownership.ts", "worker/mre-generator.ts", "worker/rma-ticket.ts", "worker/log-negation.ts", "worker/diagnostic-input-quality.ts", "worker/diagnostic-request-admission.ts", "worker/exoneration.ts", "worker/evidence-hierarchy.ts", "worker/failure-priors.ts", "worker/confidence-calibration.ts", "worker/encyclopedia-match.ts", "worker/bayesian-network.ts", "src/data/failures-matcher.ts", "src/lib/log-extract.ts", "src/lib/what-to-do.ts"]);
+var Hn = "a4e9f6563b0de3c0a5344c80dd85add908c281340fc077d225f3f056ab88a037";
 var Ii = { sha: "0000000000000000000000000000000000000000", shortSha: "offline", branch: "offline-distribution", subject: "Canonical zero-egress diagnosis engine", committedAt: "1970-01-01T00:00:00.000Z", engineVersion: ir };
 var Nb = `canonical-offline@${ir}`;
 var Xy = { "immutable-negative-fact": ["paired-causal-owner", "contracted-subsystem", "generic-subsystem", "terminal-symptom"], "paired-causal-owner": ["contracted-subsystem", "generic-subsystem", "terminal-symptom"], "contracted-subsystem": ["generic-subsystem", "terminal-symptom"], "generic-subsystem": ["terminal-symptom"], "terminal-symptom": [] };
@@ -9247,12 +9256,12 @@ async function aR(e, t, r, i, n = true, o, a) {
     }, h = async (_) => _?.error ? (console.log(`[Diagnosis] provider ${e.name} returned an error body, rotating`), await _t(e, t), null) : p(um(_, false), pm(_));
     if ((m.headers.get("content-type") || "").toLowerCase().includes("application/json")) return h(await m.json());
     if (m.body) {
-      let _ = m.body.getReader(), g = new TextDecoder(), f = "", E = "", C = [], y = (T) => {
+      let _ = m.body.getReader(), g = new TextDecoder(), f = "", E = "", k = [], y = (T) => {
         if (!T.startsWith("data: ")) return;
         let A = T.slice(6);
         if (A !== "[DONE]") try {
-          let w = JSON.parse(A), S = pm(w);
-          S && C.push(S);
+          let w = JSON.parse(A), O = pm(w);
+          O && k.push(O);
           let b = um(w, true);
           b && (a && a(b), f += b);
         } catch {
@@ -9265,17 +9274,17 @@ async function aR(e, t, r, i, n = true, o, a) {
           if (T) break;
           let w = g.decode(A, { stream: true });
           E += w;
-          let S = E.split(`
+          let O = E.split(`
 `);
-          E = S.pop() || "";
-          for (let b of S) y(b.trimEnd());
+          E = O.pop() || "";
+          for (let b of O) y(b.trimEnd());
         }
         E += g.decode(), E.trim() && y(E.trimEnd());
       } catch (T) {
         if (console.log("[Diagnosis] stream read error:", T), T.name === "AbortError") return await _t(e, t), null;
       }
       if (f) {
-        let A = C.find((w) => !hm(w)) || C[C.length - 1] || null;
+        let A = k.find((w) => !hm(w)) || k[k.length - 1] || null;
         return p(f, A);
       } else {
         let T = E.trim();
@@ -9697,9 +9706,9 @@ function MR(e) {
     return isNaN(l) && (l = 0.6), l > 1 && (l = l / 100), Math.max(0, Math.min(1, l));
   }, n = DR(e);
   if (n) try {
-    let d = r(n.failureType || n.failure_type), l = i(n.confidence), m = String(n.rootCause || n.root_cause || n.summary || "").trim(), p = Array.isArray(n.evidence) && n.evidence.length ? n.evidence.map(String) : FR(n.evidenceRanking ?? n.forensicReport?.evidenceRanking), h = (k) => {
-      if (!k) return null;
-      let D = k.split(`
+    let d = r(n.failureType || n.failure_type), l = i(n.confidence), m = String(n.rootCause || n.root_cause || n.summary || "").trim(), p = Array.isArray(n.evidence) && n.evidence.length ? n.evidence.map(String) : FR(n.evidenceRanking ?? n.forensicReport?.evidenceRanking), h = (C) => {
+      if (!C) return null;
+      let D = C.split(`
 `), x = /* @__PURE__ */ new Set(), B = [];
       for (let z11 of D) {
         let oe = z11.trim();
@@ -9713,19 +9722,19 @@ function MR(e) {
 `);
     }, u = h(String(n.fix || n.action || "").trim()) || "", _ = n.primaryFix || n.primary_fix;
     if (_ && typeof _ == "object") {
-      let k = Array.isArray(_.commands) ? _.commands : [];
-      u = h(k.join(`
+      let C = Array.isArray(_.commands) ? _.commands : [];
+      u = h(C.join(`
 `) || String(_.description || u)) || u;
     }
-    let g = (k) => {
-      if (!k || typeof k != "object") return null;
-      let D = Array.isArray(k.commands) ? k.commands : [];
+    let g = (C) => {
+      if (!C || typeof C != "object") return null;
+      let D = Array.isArray(C.commands) ? C.commands : [];
       return h(D.join(`
-`) || String(k.description || "")) || null;
-    }, f = g(n.fallbackFix || n.fallback_fix), E = g(n.emergencyFix || n.emergency_fix), C = h(String(n.prevention || "").trim()) || null, y = h(String(n.verificationCommand || n.verification_command || "").trim()) || null, T = Wc(n.clarifyingQuestions || n.clarifying_questions), A = T.questions.slice(0, 3), w = T.choices, S = Array.isArray(n.causationChain) ? n.causationChain.map((k) => ({ role: String(k.role || "").toUpperCase(), event: String(k.event || ""), evidence: String(k.evidence || "") })).filter((k) => k.role && k.event) : [], b = n.evidenceBreakdown && typeof n.evidenceBreakdown == "object" ? { hardware: Number(n.evidenceBreakdown.hardware) || 0, cuda: Number(n.evidenceBreakdown.cuda) || 0, nccl: Number(n.evidenceBreakdown.nccl) || 0, framework: Number(n.evidenceBreakdown.framework) || 0, application: Number(n.evidenceBreakdown.application) || 0 } : null, O = UR(n.competingHypotheses ?? n.rankedHypotheses ?? n.hypotheses);
+`) || String(C.description || "")) || null;
+    }, f = g(n.fallbackFix || n.fallback_fix), E = g(n.emergencyFix || n.emergency_fix), k = h(String(n.prevention || "").trim()) || null, y = h(String(n.verificationCommand || n.verification_command || "").trim()) || null, T = Wc(n.clarifyingQuestions || n.clarifying_questions), A = T.questions.slice(0, 3), w = T.choices, O = Array.isArray(n.causationChain) ? n.causationChain.map((C) => ({ role: String(C.role || "").toUpperCase(), event: String(C.event || ""), evidence: String(C.evidence || "") })).filter((C) => C.role && C.event) : [], b = n.evidenceBreakdown && typeof n.evidenceBreakdown == "object" ? { hardware: Number(n.evidenceBreakdown.hardware) || 0, cuda: Number(n.evidenceBreakdown.cuda) || 0, nccl: Number(n.evidenceBreakdown.nccl) || 0, framework: Number(n.evidenceBreakdown.framework) || 0, application: Number(n.evidenceBreakdown.application) || 0 } : null, S = UR(n.competingHypotheses ?? n.rankedHypotheses ?? n.hypotheses);
     if (m || d !== "UNKNOWN" || A.length) {
-      let k = n.forensicReport, D = k && typeof k == "object" ? { executiveSummary: String(k.executiveSummary || "").trim() || void 0, incidentMetadata: ye(k.incidentMetadata), environment: ye(k.environment), diagnosticFacts: ER(k.diagnosticFacts), diagnosticInferences: RR(k.diagnosticInferences), diagnosticHypotheses: kR(k.diagnosticHypotheses), evidenceLedger: CR(k.evidenceLedger), unexplainedEvents: vR(k.unexplainedEvents), rootCauseLayers: bR(k.rootCauseLayers), timeline: fR(k.timeline), rankGraph: ye(k.rankGraph), eventDependencyTree: Lm(k.eventDependencyTree), failureTaxonomy: ye(k.failureTaxonomy), confidenceDetail: AR(k.confidenceDetail), negativeEvidence: ye(k.negativeEvidence), evidenceGaps: IR(k.evidenceGaps), missingEvidence: xe(k.missingEvidence)?.map(String), rootCauseBreakdown: ye(k.rootCauseBreakdown), mostSuspiciousEarlierEvent: ye(k.mostSuspiciousEarlierEvent), counterfactual: xe(k.counterfactual), blastRadius: ye(k.blastRadius), businessImpact: ye(k.businessImpact), recoveryDifficulty: ye(k.recoveryDifficulty), similarFailures: xe(k.similarFailures), assumptions: xe(k.assumptions)?.map(String), verificationPlan: PR(k.verificationPlan), recoveryPlan: _R(k.recoveryPlan), preventionPlan: yR(k.preventionPlan), whatWouldChange: LR(k.whatWouldChange), missingDiagnosticSignals: xe(k.missingDiagnosticSignals), executionRequests: xe(k.executionRequests), fixSimulation: ye(k.fixSimulation), iterationStatus: String(k.iterationStatus || "") || void 0, prioritizedFixPlan: xe(k.prioritizedFixPlan)?.map(String), postFixStatus: String(k.postFixStatus || "") || void 0, eventChain: xe(k.eventChain), hypothesisTable: xe(k.hypothesisTable), blastRadiusAnalysis: ye(k.blastRadiusAnalysis), whyNotSection: xe(k.whyNotSection), informationValue: ye(k.informationValue), historicalSimilarity: ye(k.historicalSimilarity), counterfactualTests: xe(k.counterfactualTests), firstAbnormalSignal: ye(k.firstAbnormalSignal), confidenceCalibration: ye(k.confidenceCalibration), safeRemediationPlan: xe(k.safeRemediationPlan), evidenceRanking: xe(k.evidenceRanking), timelineReconstruction: xe(k.timelineReconstruction), hardwareValidation: ye(k.hardwareValidation), recoveryDecisionTreeAscii: String(k.recoveryDecisionTreeAscii || "").trim() || void 0, checkpointDecision: ye(k.checkpointDecision) } : void 0, x = hR(n.falsePaths), B = gR(n.causalityDisclaimer), z11 = String(n.chatTitle || "").trim().slice(0, 120) || void 0;
-      return { failureType: d, confidence: Math.round(l * 100), summary: m || "Analysis complete.", action: u || "Check logs for details.", evidence: p, falsePaths: x && x.length ? x : void 0, causalityDisclaimer: B, chatTitle: z11, causationChain: S.length ? S : void 0, evidenceBreakdown: b || void 0, competingHypotheses: O.length ? O : void 0, rankedHypotheses: O.length ? O : void 0, fallbackAction: f, emergencyAction: E, prevention: C, verificationCommand: y, clarifyingQuestions: A, clarifyingChoices: w, forensicReport: D, rawContent: e };
+      let C = n.forensicReport, D = C && typeof C == "object" ? { executiveSummary: String(C.executiveSummary || "").trim() || void 0, incidentMetadata: ye(C.incidentMetadata), environment: ye(C.environment), diagnosticFacts: ER(C.diagnosticFacts), diagnosticInferences: RR(C.diagnosticInferences), diagnosticHypotheses: kR(C.diagnosticHypotheses), evidenceLedger: CR(C.evidenceLedger), unexplainedEvents: vR(C.unexplainedEvents), rootCauseLayers: bR(C.rootCauseLayers), timeline: fR(C.timeline), rankGraph: ye(C.rankGraph), eventDependencyTree: Lm(C.eventDependencyTree), failureTaxonomy: ye(C.failureTaxonomy), confidenceDetail: AR(C.confidenceDetail), negativeEvidence: ye(C.negativeEvidence), evidenceGaps: IR(C.evidenceGaps), missingEvidence: xe(C.missingEvidence)?.map(String), rootCauseBreakdown: ye(C.rootCauseBreakdown), mostSuspiciousEarlierEvent: ye(C.mostSuspiciousEarlierEvent), counterfactual: xe(C.counterfactual), blastRadius: ye(C.blastRadius), businessImpact: ye(C.businessImpact), recoveryDifficulty: ye(C.recoveryDifficulty), similarFailures: xe(C.similarFailures), assumptions: xe(C.assumptions)?.map(String), verificationPlan: PR(C.verificationPlan), recoveryPlan: _R(C.recoveryPlan), preventionPlan: yR(C.preventionPlan), whatWouldChange: LR(C.whatWouldChange), missingDiagnosticSignals: xe(C.missingDiagnosticSignals), executionRequests: xe(C.executionRequests), fixSimulation: ye(C.fixSimulation), iterationStatus: String(C.iterationStatus || "") || void 0, prioritizedFixPlan: xe(C.prioritizedFixPlan)?.map(String), postFixStatus: String(C.postFixStatus || "") || void 0, eventChain: xe(C.eventChain), hypothesisTable: xe(C.hypothesisTable), blastRadiusAnalysis: ye(C.blastRadiusAnalysis), whyNotSection: xe(C.whyNotSection), informationValue: ye(C.informationValue), historicalSimilarity: ye(C.historicalSimilarity), counterfactualTests: xe(C.counterfactualTests), firstAbnormalSignal: ye(C.firstAbnormalSignal), confidenceCalibration: ye(C.confidenceCalibration), safeRemediationPlan: xe(C.safeRemediationPlan), evidenceRanking: xe(C.evidenceRanking), timelineReconstruction: xe(C.timelineReconstruction), hardwareValidation: ye(C.hardwareValidation), recoveryDecisionTreeAscii: String(C.recoveryDecisionTreeAscii || "").trim() || void 0, checkpointDecision: ye(C.checkpointDecision) } : void 0, x = hR(n.falsePaths), B = gR(n.causalityDisclaimer), z11 = String(n.chatTitle || "").trim().slice(0, 120) || void 0;
+      return { failureType: d, confidence: Math.round(l * 100), summary: m || "Analysis complete.", action: u || "Check logs for details.", evidence: p, falsePaths: x && x.length ? x : void 0, causalityDisclaimer: B, chatTitle: z11, causationChain: O.length ? O : void 0, evidenceBreakdown: b || void 0, competingHypotheses: S.length ? S : void 0, rankedHypotheses: S.length ? S : void 0, fallbackAction: f, emergencyAction: E, prevention: k, verificationCommand: y, clarifyingQuestions: A, clarifyingChoices: w, forensicReport: D, rawContent: e };
     }
   } catch {
   }
@@ -9753,7 +9762,7 @@ function xR(e) {
   return ["Before restarting, identify who owns VRAM: `nvidia-smi --query-compute-apps=gpu_uuid,pid,process_name,used_memory --format=csv,noheader`. Terminate only a confirmed stale/unrelated PID; do not kill an active peer rank.", "Capture the allocator state at the failing step: log `torch.cuda.memory_summary(abbreviated=False)` inside the `except torch.cuda.OutOfMemoryError` handler, plus `torch.cuda.max_memory_allocated()` and `torch.cuda.max_memory_reserved()`.", t ? "The log contains a fragmentation signal. Retry with `export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True`; keep the workload unchanged for this one A/B test so the result actually isolates allocator fragmentation." : "If allocated memory is near device capacity, reduce per-GPU microbatch or maximum sequence/context length and recover effective global batch with gradient accumulation; enable activation checkpointing for activation-heavy training. Do not label this fragmentation without a reserved-versus-allocated gap.", r ? "Compare `torch.cuda.max_memory_allocated()` on every rank for the same step. If only one rank is high, inspect sequence-length/data sharding and FSDP/ZeRO partition boundaries before shrinking the global workload." : "Rerun the exact failing batch/sequence after one change. The fix is verified only if it completes and peak allocated/reserved memory leaves repeatable headroom; a restart alone is not proof."];
 }
 function rt(e, t) {
-  let r = Ke(e), i = r.hardEccEvent || r.rowRemapEvent || r.xidCodes.length > 0 || /NVRM:\s*Xid|GPU has fallen off the bus|Unable to query utilization|AER:|IOMMU|completion timeout/i.test(e) || /\bXid\b|double[- ]?bit|uncorrectable ECC/i.test(e) && !r.xidExplicitlyClean && r.eccClean !== true, n = r.checksumMismatch !== null && !r.hardEccEvent && r.xidCodes.length === 0, o = /mlx5_core.*link (?:down|up)|IB[^\n]{0,30}link (?:down|flap|up)|port.*state.*down|infiniband.*port.*down/i.test(e), a = /RoCE v2 frame size \d+ exceeds MTU \d+|syndrome.*0x123444|Local Length Error|CREATE_QP.*failed.*bad parameter/i.test(e), s = /CUDA error|unspecified launch failure|illegal memory access|device-side assert|Cuda failure/i.test(e), c = /NCCL.*timeout|Watchdog.*collective|ALLREDUCE|Proxy stalled/i.test(e), d = /symbol_errors\.+0|link_downed\.+0|port_rcv_errors\.+0|retransmissions\.+0/i.test(e), l = String(t || "").toUpperCase();
+  let r = Ke(e), i = r.hardEccEvent || r.rowRemapEvent || r.xidCodes.length > 0 || /NVRM:\s*Xid|GPU has fallen off the bus|Unable to query utilization|AER:|IOMMU|completion timeout/i.test(e) || /\bXid\b|double[- ]?bit|uncorrectable ECC/i.test(e.replace(/\b(?:No|without)\s+(?:ECC\s*\/\s*Xid|Xid\s*\/\s*ECC|ECC|Xid)\s+events?\b/gi, "")) && !r.xidExplicitlyClean && r.eccClean !== true, n = r.checksumMismatch !== null && !r.hardEccEvent && r.xidCodes.length === 0, o = /mlx5_core.*link (?:down|up)|IB[^\n]{0,30}link (?:down|flap|up)|port.*state.*down|infiniband.*port.*down/i.test(e), a = /RoCE v2 frame size \d+ exceeds MTU \d+|syndrome.*0x123444|Local Length Error|CREATE_QP.*failed.*bad parameter/i.test(e), s = /CUDA error|unspecified launch failure|illegal memory access|device-side assert|Cuda failure/i.test(e), c = /NCCL.*timeout|Watchdog.*collective|ALLREDUCE|Proxy stalled/i.test(e), d = /symbol_errors\.+0|link_downed\.+0|port_rcv_errors\.+0|retransmissions\.+0/i.test(e), l = String(t || "").toUpperCase();
   if (a) {
     let m = e.match(/RoCE v2 frame size\s+(\d+)\s+exceeds MTU\s+(\d+)/i), p = m ? `RoCE v2 frame size ${m[1]} exceeds MTU ${m[2]}` : "The submitted log contains an RDMA MTU/completion mismatch signature";
     return [{ theory: "Network fabric MTU mismatch (true root cause)", score: l.includes("MTU") || l.includes("NET") ? 96 : 88, supporting: `${p}. The NIC and end-to-end network path disagree on frame size, and the submitted QP/completion error is transport-level evidence.`, contradicting: "" }, { theory: "NCCL / GPU software bug", score: s ? 15 : 8, supporting: s ? "CUDA error appeared after the NCCL timeout." : "", contradicting: "NCCL timeout is downstream of the RDMA transport failure. The traceback begins with ibv_poll_cq completion errors, not a compute bug." }, { theory: "GPU hardware fault", score: 3, supporting: "No Xid, ECC, or GPU off-bus evidence is present.", contradicting: "All errors are RDMA transport-level; the GPU hardware is healthy." }];
@@ -9803,17 +9812,17 @@ function Ut(e) {
   if (Fm(e)) return { failureType: "NO_FAILURE_DETECTED", confidence: 0, summary: "These logs show a run that completed successfully, there is no failure to diagnose. Nothing here needs remediation.", action: `No action required. This run completed successfully and exited cleanly.
 
 If you expected a failure, the logs may be truncated or from the wrong run: paste the tail of the failing rank's log, or the output of \`dmesg -T | grep -iE "xid|nvrm"\` from the incident window.`, evidence: e.split(/\r?\n/).map((b) => b.trim()).filter((b) => Nm.test(b)).slice(0, 2), fallbackAction: null, emergencyAction: null, prevention: null, verificationCommand: null, clarifyingQuestions: [], inviteFollowup: true };
-  let n = r(/\b(nvidia|cuda|xid|gpu|nvrm|ecc)\b/i), o = r(/\b(xid|nvrm|ecc|row[_ -]?remap|dcgm[^\n]{0,40}fail|fallen off the bus|gpu (?:lost|reset)|pcie aer)\b/i), a = r(/\b(nccl|distributed|allreduce|all_reduce|collective|watchdog|world_size|local_rank)\b|\[rank\s*\d+\]|\brank\s*(?:[:=]\s*)?\d+\b/i), s = r(/\b(ibstatus|ibv_devinfo|rdma|roce|infiniband|mtu|ib0|mlx5)\b/i), c = r(/\b(custom (?:cuda )?(?:extension|kernel|operator|backend)|[A-Za-z][\w.-]+\s+extension|fused[_ .:-]?\w+|flash[_ -]?attn|triton(?:\s+(?:kernel|operator))?|\w+(?:_kernel|_op)|attention backend)\b/i), d = r(/\b(detokeniz\w*|tokenizer|request[_ -]?(?:id|key|state)|sequence[_ -]?id)\b/i), l = r(/\b(checkpoint|unpickling|state_dict|pytorchstreamreader|distcp|safetensors)\b/i), m = r(/\b(undefined symbol|no module named|importerror|glibc|cxxabi|abi|ptxas|unsupported gpu architecture|build failed|compile error)\b/i), p = r(/\b(dataloader|dataset|sample|parquet|tfrecord|jsondecodeerror|index out of range|too many values to unpack|not enough values to unpack)\b/i), h = r(/\b(oomkilled|evicted|crashloopbackoff|slurm|srun|sbatch|gres|preempt|node notready|pod sandbox)\b/i), u = r(/\b(no space left|disk quota|stale file handle|input\/output error|read-only file system|s3|lustre|gpfs|nfs)\b/i), _ = [/\bxid(?:\s*(?:code|errors?))?\s*[:=]?\s*0\b/i, /\bno\s+(?:current\s+)?xid(?:\s+errors?)?\b/i, /\b(?:volatile\s+)?(?:uncorrectable|correctable)?\s*ecc(?:\s+errors?)?\s*[:=]?\s*0\b/i, /\brow[_ -]?remap(?:per|ping)?(?:\s+(?:failures?|pending))?\s*[:=]?\s*0\b/i, /\bdcgm(?:\s+diagnostic)?\s*[:=]?\s*pass(?:ed)?\b/i].some((b) => b.test(e)), g = t.search(/\b(?:runtimeerror|valueerror|keyerror|assertionerror|indexerror|typeerror|filenotfounderror|modulenotfounderror|cuda error|device-side assert|illegal memory access|segmentation fault)\b/i), f = t.search(/\b(?:nccl|watchdog|collective)[^\n]{0,100}(?:timeout|timed out|aborted|failed|error)\b/i), E = a && g >= 0 && (f < 0 || g < f) && !/^.{0,80}\bnccl\b/i.test(t.slice(g, g + 160)), C = null;
-  if (d && (i.push("Request state: trace the failing request/sequence key through create \u2192 enqueue \u2192 decode \u2192 free, including worker id and monotonic timestamp; replay the same request stream with one worker and then the original concurrency. A missing key only in the concurrent run selects a lifecycle/race owner, not GPU hardware."), C = "Replay the same request stream once with one worker and once at the original concurrency, then paste the first create/free/lookup event for the missing request key."), c && (i.push("Optional component: replay the identical input once with the named extension/kernel/backend enabled and once disabled, using `CUDA_LAUNCH_BLOCKING=1` for the failing arm. Enabled-only failure isolates that path; failure in both arms falsifies it."), C ??= "`CUDA_LAUNCH_BLOCKING=1` with the identical input and the named optional component enabled, followed by the same input with it disabled."), l) {
+  let n = r(/\b(nvidia|cuda|xid|gpu|nvrm|ecc)\b/i), o = r(/\b(xid|nvrm|ecc|row[_ -]?remap|dcgm[^\n]{0,40}fail|fallen off the bus|gpu (?:lost|reset)|pcie aer)\b/i), a = r(/\b(nccl|distributed|allreduce|all_reduce|collective|watchdog|world_size|local_rank)\b|\[rank\s*\d+\]|\brank\s*(?:[:=]\s*)?\d+\b/i), s = r(/\b(ibstatus|ibv_devinfo|rdma|roce|infiniband|mtu|ib0|mlx5)\b/i), c = r(/\b(custom (?:cuda )?(?:extension|kernel|operator|backend)|[A-Za-z][\w.-]+\s+extension|fused[_ .:-]?\w+|flash[_ -]?attn|triton(?:\s+(?:kernel|operator))?|\w+(?:_kernel|_op)|attention backend)\b/i), d = r(/\b(detokeniz\w*|tokenizer|request[_ -]?(?:id|key|state)|sequence[_ -]?id)\b/i), l = r(/\b(checkpoint|unpickling|state_dict|pytorchstreamreader|distcp|safetensors)\b/i), m = r(/\b(undefined symbol|no module named|importerror|glibc|cxxabi|abi|ptxas|unsupported gpu architecture|build failed|compile error)\b/i), p = r(/\b(dataloader|dataset|sample|parquet|tfrecord|jsondecodeerror|index out of range|too many values to unpack|not enough values to unpack)\b/i), h = r(/\b(oomkilled|evicted|crashloopbackoff|slurm|srun|sbatch|gres|preempt|node notready|pod sandbox)\b/i), u = r(/\b(no space left|disk quota|stale file handle|input\/output error|read-only file system|s3|lustre|gpfs|nfs)\b/i), _ = [/\bxid(?:\s*(?:code|errors?))?\s*[:=]?\s*0\b/i, /\bno\s+(?:current\s+)?xid(?:\s+errors?)?\b/i, /\b(?:volatile\s+)?(?:uncorrectable|correctable)?\s*ecc(?:\s+errors?)?\s*[:=]?\s*0\b/i, /\brow[_ -]?remap(?:per|ping)?(?:\s+(?:failures?|pending))?\s*[:=]?\s*0\b/i, /\bdcgm(?:\s+diagnostic)?\s*[:=]?\s*pass(?:ed)?\b/i].some((b) => b.test(e)), g = t.search(/\b(?:runtimeerror|valueerror|keyerror|assertionerror|indexerror|typeerror|filenotfounderror|modulenotfounderror|cuda error|device-side assert|illegal memory access|segmentation fault)\b/i), f = t.search(/\b(?:nccl|watchdog|collective)[^\n]{0,100}(?:timeout|timed out|aborted|failed|error)\b/i), E = a && g >= 0 && (f < 0 || g < f) && !/^.{0,80}\bnccl\b/i.test(t.slice(g, g + 160)), k = null;
+  if (d && (i.push("Request state: trace the failing request/sequence key through create \u2192 enqueue \u2192 decode \u2192 free, including worker id and monotonic timestamp; replay the same request stream with one worker and then the original concurrency. A missing key only in the concurrent run selects a lifecycle/race owner, not GPU hardware."), k = "Replay the same request stream once with one worker and once at the original concurrency, then paste the first create/free/lookup event for the missing request key."), c && (i.push("Optional component: replay the identical input once with the named extension/kernel/backend enabled and once disabled, using `CUDA_LAUNCH_BLOCKING=1` for the failing arm. Enabled-only failure isolates that path; failure in both arms falsifies it."), k ??= "`CUDA_LAUNCH_BLOCKING=1` with the identical input and the named optional component enabled, followed by the same input with it disabled."), l) {
     let b = zR(e);
-    i.push(`Checkpoint: validate deserialization with \`python -c "import torch; torch.load('${b}', map_location='cpu')"\`, then compare its manifest/checksum and producer world-size/state-dict type with the last verified-good checkpoint before choosing a resume point.`), C ??= `\`python -c "import torch; torch.load('${b}', map_location='cpu'); print('checkpoint load OK')"\``;
+    i.push(`Checkpoint: validate deserialization with \`python -c "import torch; torch.load('${b}', map_location='cpu')"\`, then compare its manifest/checksum and producer world-size/state-dict type with the last verified-good checkpoint before choosing a resume point.`), k ??= `\`python -c "import torch; torch.load('${b}', map_location='cpu'); print('checkpoint load OK')"\``;
   }
-  (m || r(/\b(triton|ptx|recompile)\b/i)) && (i.push("Toolchain: record the exact framework, CUDA runtime, driver, compiler/ptxas, Triton and extension build tuple; rebuild only the failing artifact in a fresh cache, then run its minimal import/JIT reproducer before changing the cluster image."), C ??= "`python -m torch.utils.collect_env` plus the first compiler/linker error and the minimal import/JIT reproducer result."), p && (i.push("Data contract: log the failing sample id/path and its post-collation shapes/dtypes on the first failing rank; replay that exact sample with `num_workers=0`, then with the original worker count. This separates corrupt/input-contract data from multiprocessing or shared-memory failure."), C ??= "Replay the exact failing sample with `num_workers=0` and paste its sample id plus post-collation shapes/dtypes."), h && (i.push('Scheduler: preserve the pod/job termination reason and node event before retrying (`kubectl describe pod` plus `kubectl get events --sort-by=.lastTimestamp`, or `sacct -j "$SLURM_JOB_ID" -o JobID,State,ExitCode,Reason,NodeList`). A scheduler kill/preemption owns the rank loss when it precedes framework errors.'), C ??= "Paste the scheduler termination reason and timestamp for the first exited task/pod, aligned with the first framework error."), u && (i.push("Storage: test the exact failing path from the same node/container identity, record free bytes and inodes, and checksum the object/shard against a healthy reader. Retry only after the same path passes read/write and integrity checks."), C ??= "Paste `df -h` and `df -i` for the affected mount plus a checksum/read test for the exact failing object or shard."), r(/\b(out of memory|outofmemory|cuda oom|cannot allocate)\b/i) && (i.push("Memory ownership: capture `torch.cuda.memory_summary()` at failure and `nvidia-smi --query-compute-apps=pid,used_memory --format=csv`; compare allocated vs reserved and identify co-tenant PIDs before changing batch size or allocator settings."), C ??= "Paste `torch.cuda.memory_summary()` from the failing process and `nvidia-smi --query-compute-apps=pid,used_memory --format=csv` from the same timestamp."), r(/\b(nan|inf|loss scale|gradscaler|overflow)\b/i) && (i.push("Numerics: preserve the first non-finite tensor with `torch.autograd.detect_anomaly()` on the reproducing batch, compare fp32/bf16/fp16 on that same batch, and inspect inputs/labels before changing learning rate or clipping policy."), C ??= "Paste the first `detect_anomaly` forward/backward op and the same-batch fp32/bf16/fp16 result."), s && (i.push("Fabric: on the implicated endpoints, compare `ibv_devinfo`, link state/error counters, GID/MTU and NCCL-selected HCA/interface; prove the same collective fails on the fabric path and passes on a bounded socket fallback before assigning the network as owner."), C ??= "Paste the implicated endpoints\u2019 link/error counters and one same-collective fabric-versus-socket A/B result."), a && !E && !c && !d ? (i.push("NCCL: re-run with `NCCL_DEBUG=INFO NCCL_DEBUG_SUBSYS=COLL,NET` and align timestamps across ranks to find the first rank/op that stops progressing; timeout reporters waiting later are collateral until proven otherwise."), C ??= "Paste the earliest per-rank application or collective event before the watchdog timeout, with rank and timestamp.") : E && (i.push("Cascade ordering: preserve the earlier application exception and use aligned per-rank timestamps only to identify which rank emitted it first. Treat later NCCL watchdog/peer errors as collateral unless the application fault also occurs after they are removed."), C ??= "Paste the earliest source-located application exception and the first later timeout, each with rank and timestamp."), o && !_ ? (i.push('GPU evidence: collect `dmesg -T | grep -iE "xid|nvrm"` and `nvidia-smi -q -d ECC,ROW_REMAPPER` from the incident window on every node. Do not take disruptive hardware action until a non-zero/current Xid, ECC, row-remap, or failed diagnostic identifies the affected device.'), C ??= "Paste non-zero/current Xid, ECC, row-remap, or failed diagnostic output from the incident window; clean output rules hardware down, not up.") : n && _ && i.push("Hardware is currently exonerated by the submitted zero/clean Xid, ECC, row-remap, or DCGM evidence. Keep the GPU in service while isolating the software/request path; revisit hardware only if a non-zero incident-window counter or failed diagnostic appears."), i.length === 0 && (i.push('Capture the FIRST error, not the cascade: `grep -nEi "error|fail|abort|traceback" <logfile> | head`.'), i.push('Snapshot every node: `nvidia-smi -q > gpu_$(hostname).txt` and `dmesg -T | grep -iE "xid|nvrm|oom" | tail -50`.'), i.push("Re-run with tracing: `NCCL_DEBUG=INFO TORCH_DISTRIBUTED_DEBUG=DETAIL CUDA_LAUNCH_BLOCKING=1` to surface the exact failing op."));
+  (m || r(/\b(triton|ptx|recompile)\b/i)) && (i.push("Toolchain: record the exact framework, CUDA runtime, driver, compiler/ptxas, Triton and extension build tuple; rebuild only the failing artifact in a fresh cache, then run its minimal import/JIT reproducer before changing the cluster image."), k ??= "`python -m torch.utils.collect_env` plus the first compiler/linker error and the minimal import/JIT reproducer result."), p && (i.push("Data contract: log the failing sample id/path and its post-collation shapes/dtypes on the first failing rank; replay that exact sample with `num_workers=0`, then with the original worker count. This separates corrupt/input-contract data from multiprocessing or shared-memory failure."), k ??= "Replay the exact failing sample with `num_workers=0` and paste its sample id plus post-collation shapes/dtypes."), h && (i.push('Scheduler: preserve the pod/job termination reason and node event before retrying (`kubectl describe pod` plus `kubectl get events --sort-by=.lastTimestamp`, or `sacct -j "$SLURM_JOB_ID" -o JobID,State,ExitCode,Reason,NodeList`). A scheduler kill/preemption owns the rank loss when it precedes framework errors.'), k ??= "Paste the scheduler termination reason and timestamp for the first exited task/pod, aligned with the first framework error."), u && (i.push("Storage: test the exact failing path from the same node/container identity, record free bytes and inodes, and checksum the object/shard against a healthy reader. Retry only after the same path passes read/write and integrity checks."), k ??= "Paste `df -h` and `df -i` for the affected mount plus a checksum/read test for the exact failing object or shard."), r(/\b(out of memory|outofmemory|cuda oom|cannot allocate)\b/i) && (i.push("Memory ownership: capture `torch.cuda.memory_summary()` at failure and `nvidia-smi --query-compute-apps=pid,used_memory --format=csv`; compare allocated vs reserved and identify co-tenant PIDs before changing batch size or allocator settings."), k ??= "Paste `torch.cuda.memory_summary()` from the failing process and `nvidia-smi --query-compute-apps=pid,used_memory --format=csv` from the same timestamp."), r(/\b(nan|inf|loss scale|gradscaler|overflow)\b/i) && (i.push("Numerics: preserve the first non-finite tensor with `torch.autograd.detect_anomaly()` on the reproducing batch, compare fp32/bf16/fp16 on that same batch, and inspect inputs/labels before changing learning rate or clipping policy."), k ??= "Paste the first `detect_anomaly` forward/backward op and the same-batch fp32/bf16/fp16 result."), s && (i.push("Fabric: on the implicated endpoints, compare `ibv_devinfo`, link state/error counters, GID/MTU and NCCL-selected HCA/interface; prove the same collective fails on the fabric path and passes on a bounded socket fallback before assigning the network as owner."), k ??= "Paste the implicated endpoints\u2019 link/error counters and one same-collective fabric-versus-socket A/B result."), a && !E && !c && !d ? (i.push("NCCL: re-run with `NCCL_DEBUG=INFO NCCL_DEBUG_SUBSYS=COLL,NET` and align timestamps across ranks to find the first rank/op that stops progressing; timeout reporters waiting later are collateral until proven otherwise."), k ??= "Paste the earliest per-rank application or collective event before the watchdog timeout, with rank and timestamp.") : E && (i.push("Cascade ordering: preserve the earlier application exception and use aligned per-rank timestamps only to identify which rank emitted it first. Treat later NCCL watchdog/peer errors as collateral unless the application fault also occurs after they are removed."), k ??= "Paste the earliest source-located application exception and the first later timeout, each with rank and timestamp."), o && !_ ? (i.push('GPU evidence: collect `dmesg -T | grep -iE "xid|nvrm"` and `nvidia-smi -q -d ECC,ROW_REMAPPER` from the incident window on every node. Do not take disruptive hardware action until a non-zero/current Xid, ECC, row-remap, or failed diagnostic identifies the affected device.'), k ??= "Paste non-zero/current Xid, ECC, row-remap, or failed diagnostic output from the incident window; clean output rules hardware down, not up.") : n && _ && i.push("Hardware is currently exonerated by the submitted zero/clean Xid, ECC, row-remap, or DCGM evidence. Keep the GPU in service while isolating the software/request path; revisit hardware only if a non-zero incident-window counter or failed diagnostic appears."), i.length === 0 && (i.push('Capture the FIRST error, not the cascade: `grep -nEi "error|fail|abort|traceback" <logfile> | head`.'), i.push('Snapshot every node: `nvidia-smi -q > gpu_$(hostname).txt` and `dmesg -T | grep -iE "xid|nvrm|oom" | tail -50`.'), i.push("Re-run with tracing: `NCCL_DEBUG=INFO TORCH_DISTRIBUTED_DEBUG=DETAIL CUDA_LAUNCH_BLOCKING=1` to surface the exact failing op."));
   let y = e.split(/\r?\n/).map((b) => b.trim()).filter((b) => !/^\[(?:SYSTEM_WARNING|SYSTEM_NOTE|DENPEX|ENGINE)[^\]]*\]/i.test(b)).filter((b) => b.length > 0 && /error|fail|fatal|exception|traceback|timeout|killed|xid|oom|violated/i.test(b)).slice(0, 4), T = ji(e), A = Yi(e), w = T?.symbol ? `No detector claims this failure, so treat this as unconfirmed. The first fault this log reports names \`${T.symbol}\`, which is the most specific owner candidate available and is quoted in the evidence below.` + (A ? " Your log already contains a control isolating it, so that result is treated as evidence rather than re-requested." : "") + " Confirm it before acting." : "No single root cause was conclusive from these logs alone. Current evidence quality: LOW. Cannot safely assign ownership without more signals.";
   T?.symbol && i.unshift(`Start at \`${T.symbol}\`, the component named on the first fault line quoted in the evidence below.` + (A ? " The control result already present in your log supports it being the owner." : ""));
-  let S = _r(e);
-  return { failureType: "NEEDS_MORE_SIGNAL", confidence: 0, summary: w, action: i.map((b, O) => `${O + 1}. ${b}`).join(`
-`), evidence: y, fallbackAction: null, emergencyAction: null, prevention: null, verificationCommand: C ?? "Paste the earliest source-located error with its rank/host and timestamp, plus the result of one controlled run that changes only the most plausible owner.", clarifyingQuestions: S.map((b) => b.question), clarifyingChoices: S, inviteFollowup: true };
+  let O = _r(e);
+  return { failureType: "NEEDS_MORE_SIGNAL", confidence: 0, summary: w, action: i.map((b, S) => `${S + 1}. ${b}`).join(`
+`), evidence: y, fallbackAction: null, emergencyAction: null, prevention: null, verificationCommand: k ?? "Paste the earliest source-located error with its rank/host and timestamp, plus the result of one controlled run that changes only the most plausible owner.", clarifyingQuestions: O.map((b) => b.question), clarifyingChoices: O, inviteFollowup: true };
 }
 var HR = "deepseek-ai/deepseek-v4-flash-0731";
 var VR = /Xid[\s:()]*\d+|NVRM|dmesg|nvidia-smi|\becc\b|row.?remap|fallen off the bus|ibstat|ibv_devinfo|SXid/i;
@@ -9882,8 +9891,8 @@ function Mm(e, t) {
     }
     return;
   }
-  let E = typeof e.summary == "string" ? e.summary.trim() : "", C = i && e.failureType !== "UNKNOWN" && E.length >= 60 && !/^Best-supported hypothesis so far:/i.test(E) && !/don't contain the hard evidence/i.test(E) && !/below the decision threshold/i.test(E);
-  e.summary = C ? `${E} Confidence is below the decision threshold, so treat this as the best-supported explanation rather than a settled verdict. Run the ${m.length === 1 ? "check" : `${m.length} checks`} below and paste the output to confirm it.` : `${p}These logs don't contain the hard evidence needed to isolate ONE root cause safely. Run the ${m.length === 1 ? "check" : `${m.length} checks`} below and paste the output, that will pin it down.`, e.action = m.map((y, T) => `${T + 1}. ${y}`).join(`
+  let E = typeof e.summary == "string" ? e.summary.trim() : "", k = i && e.failureType !== "UNKNOWN" && E.length >= 60 && !/^Best-supported hypothesis so far:/i.test(E) && !/don't contain the hard evidence/i.test(E) && !/below the decision threshold/i.test(E);
+  e.summary = k ? `${E} Confidence is below the decision threshold, so treat this as the best-supported explanation rather than a settled verdict. Run the ${m.length === 1 ? "check" : `${m.length} checks`} below and paste the output to confirm it.` : `${p}These logs don't contain the hard evidence needed to isolate ONE root cause safely. Run the ${m.length === 1 ? "check" : `${m.length} checks`} below and paste the output, that will pin it down.`, e.action = m.map((y, T) => `${T + 1}. ${y}`).join(`
 `);
 }
 var Gi = 75;
@@ -10029,9 +10038,9 @@ function ZR(e, t, r = t, i) {
       e.summary = `${f} The earliest fault is quoted first in the evidence below.${g} Investigate that before acting on the following, which is downstream of it.
 
 ${String(e.summary || "")}`;
-      let E = u.symbol ? `\`${u.symbol}\`` : "the first fault line", C = String(e.action || "");
-      (u.symbol ? C.includes(u.symbol) : /first fault line/i.test(C)) || (e.action = `1. Start at ${E}, quoted first in the evidence below. Establish what produced that fault before treating the later failure as the cause.
-` + C);
+      let E = u.symbol ? `\`${u.symbol}\`` : "the first fault line", k = String(e.action || "");
+      (u.symbol ? k.includes(u.symbol) : /first fault line/i.test(k)) || (e.action = `1. Start at ${E}, quoted first in the evidence below. Establish what produced that fault before treating the later failure as the cause.
+` + k);
       let T = Array.isArray(e.evidence) ? e.evidence.map((A) => String(A)) : [];
       e.evidence = [u.line, ...T.filter((A) => A !== u.line)].slice(0, 6);
     }
@@ -10058,9 +10067,9 @@ The exit code alone does not discriminate between an application exception, a ho
   {
     let _ = { GPU_MEMORY_CONTENTION: /Process\s+\d+\s*\([^)]*\)\s*has\s+[\d.]+\s*(?:GiB|MiB)/i, CUDA_DRIVER_RUNTIME_MISMATCH: /driver version is insufficient|Driver Version:\s*[\d.]+|libcudart\.so\.\d+/i, NCCL_P2P_DISABLED_PERF: /NCCL_P2P_DISABLE|busbw/i, GPU_XID_74_NVLINK: /Replay Errors:\s*\d+|\bLink\s*\d+\b|nvlink\s+-e/i, GPU_XID_ECC: /Xid[^\n]*\b(?:48|63|64|94|95)\b|Remapping Failure|Pending\s*:\s*Yes|ROW_REMAPPER/i, GPU_MEMORY_ORPHANED: /No running processes found|<defunct>|fuser[^\n]*\/dev\/nvidia|Memory-Usage[^\n]*MiB/i, MIG_PARTITION_MISMATCH: /MIG\s*M\.?\s*:\s*Enabled|\b\d+g\.\d+gb\b|GiB total capacity/i, IB_PORT_DOWN: /State\s*:\s*Down|No device found for\s+(?:mlx5_\d+|ib\w*\d+)|using\s+NET\/Socket/i, PYTORCH_AMP_GRADSCALER_NAN_1: /Gradient overflow|loss scaler|grad_norm\s+inf|loss\s+nan/i, TORCHRUN_NNODES_MISMATCH: /world_size:\s*\d+,\s*worker_count:\s*\d+|only \d+ ranks joined|WORLD_SIZE=\d+|Failed to start/i, CHECKPOINT_TRUNCATED: /central directory|PytorchStreamReader|unexpected EOF|truncat\w*|invalid load key/i }[String(e.failureType || "")];
     if (_) {
-      let g = Array.isArray(e.evidence) ? e.evidence : [], f = new Set(g.map((C) => String(C).trim())), E = [];
-      for (let C of String(t).split(/\r?\n/)) {
-        let y = C.trim();
+      let g = Array.isArray(e.evidence) ? e.evidence : [], f = new Set(g.map((k) => String(k).trim())), E = [];
+      for (let k of String(t).split(/\r?\n/)) {
+        let y = k.trim();
         if (!(!y || f.has(y) || !_.test(y)) && (E.push(y.slice(0, 200)), E.length >= 3)) break;
       }
       E.length && (e.evidence = [...E, ...g].slice(0, 6));
@@ -10096,8 +10105,8 @@ Where to apply this: ${g}.`);
 ${g}`);
     }
     if (!e.telemetryEvidence) {
-      let g = Ke(t), f = (g.positives ?? []).slice(0, 6), E = /^NO_(?:FAULT|FAILURE)_/.test(String(e.failureType || "")) ? (Array.isArray(e.evidence) ? e.evidence : []).filter((y) => /completed|healthy|pass|verified|exit code 0/i.test(String(y))).map(String).slice(0, 6) : [], C = [...g.negatives ?? [], ...E].slice(0, 6);
-      (f.length || C.length) && (e.telemetryEvidence = { indicting: f, exonerating: C });
+      let g = Ke(t), f = (g.positives ?? []).slice(0, 6), E = /^NO_(?:FAULT|FAILURE)_/.test(String(e.failureType || "")) ? (Array.isArray(e.evidence) ? e.evidence : []).filter((y) => /completed|healthy|pass|verified|exit code 0/i.test(String(y))).map(String).slice(0, 6) : [], k = [...g.negatives ?? [], ...E].slice(0, 6);
+      (f.length || k.length) && (e.telemetryEvidence = { indicting: f, exonerating: k });
     }
   } catch {
   }
@@ -10113,13 +10122,13 @@ ${g}`);
       for (let f of String(t).split(/\r?\n/)) {
         let E = f.trim();
         if (!E || !_.test(E)) continue;
-        let C = E.match(/\b((?:node|host|gpu-worker|dgx)[-\w]*\d[\w-]*)\b/i)?.[1];
-        !C || C.toLowerCase() === u.toLowerCase() || g.has(C) || g.set(C, E.slice(0, 200));
+        let k = E.match(/\b((?:node|host|gpu-worker|dgx)[-\w]*\d[\w-]*)\b/i)?.[1];
+        !k || k.toLowerCase() === u.toLowerCase() || g.has(k) || g.set(k, E.slice(0, 200));
       }
       if (g.size) {
         let f = [...g.entries()].slice(0, 3);
         m.push({ summary: `A second, unrelated fault is present on ${f.map(([E]) => E).join(", ")}, it is NOT explained by the diagnosis above`, severity: "warning", action: `The primary verdict covers ${u}. These lines are from a different host and are not downstream of it, so fixing ${u} will not resolve them:
-` + f.map(([E, C]) => `  \u2022 ${E}: ${C}`).join(`
+` + f.map(([E, k]) => `  \u2022 ${E}: ${k}`).join(`
 `) + `
 Triage them as separate incidents. If they are genuinely the same event, the shared cause has to be established rather than assumed.` });
       }
@@ -10175,9 +10184,9 @@ async function Em(e, t, r, i, n) {
     o = { id: h, user_id: i, diagnosis_id: String(e.id), failure_type: e.failureType, root_cause: e.summary, fix_applied: e.action, fix_outcome: "unknown", cluster_id: l, node_id: c, rank: s, gpu_index: vo(s) ?? void 0, gpu_model: p, severity: e.forensicReport?.failureTaxonomy?.severity || "High", precursor_signals: JSON.stringify(_), false_hypotheses: JSON.stringify(f), resolution_time_ms: e.diagnosisTimeMs || 0 };
     let E = await xt(ad(r.denpex_db, i, { failureType: e.failureType, nodeId: c, clusterId: l, diagnosisId: String(e.id) }), { isRegression: false }, "regression-detection");
     E.isRegression && (e.priorIncident = { note: E.message, advisoryOnly: true, verifiedResolutionAvailable: !!E.pastResolutions?.[0] }, E.flag === "HARDWARE_REGRESSION" && (e.confidence = Math.max(e.confidence - 10, 50)));
-    let C = QR(e.failureType, c, n);
-    if (e.forensicReport = e.forensicReport || {}, e.forensicReport.emergentPatterns = C, C.length > 0) {
-      let y = C[0];
+    let k = QR(e.failureType, c, n);
+    if (e.forensicReport = e.forensicReport || {}, e.forensicReport.emergentPatterns = k, k.length > 0) {
+      let y = k[0];
       if (y.frequency > 1) {
         let T = [];
         c && T.push(c);
@@ -10214,14 +10223,14 @@ ${e}`);
 ${e}`);
   let u = "";
   if (o?.failedNodes) try {
-    let O = JSON.parse(o.failedNodes), k = s;
-    !k && o?.topologyMap && (k = JSON.parse(o.topologyMap));
-    let D = bn(O, k);
+    let S = JSON.parse(o.failedNodes), C = s;
+    !C && o?.topologyMap && (C = JSON.parse(o.topologyMap));
+    let D = bn(S, C);
     D.isTopologyFault && D.domain && (console.log(`[Diagnosis] Topology fault detected at ${D.domain.type}: ${D.domain.name}`), u = `[TOPOLOGY_OVERRIDE] A massive failure perfectly aligned with network topology boundary ${D.domain.type} (${D.domain.name}). This is a NETWORK_FABRIC failure, not a software failure.
 
 `);
-  } catch (O) {
-    console.warn("[Diagnosis] Failed to parse topology telemetry:", O);
+  } catch (S) {
+    console.warn("[Diagnosis] Failed to parse topology telemetry:", S);
   }
   let _ = Td(e);
   (_.hasLossSpike || _.hasNaN) && (console.log("[Diagnosis] Loss anomaly observed; cause is not established."), e = `[LOSS_ANOMALY_OBSERVED] Loss spike or NaN observed. This does not establish data corruption or rule out OOM. Evidence: ${_.evidence.join(" | ")}
@@ -10231,44 +10240,44 @@ ${e}`), e = u + e;
   if (g.isCppFault && (console.log(`[Diagnosis] C++ Stack Trace unwound: ${g.faultingLibrary}`), e = `[CPP_EXTENSION_FAULT] ${g.evidence.join(" ")}
 
 ${e}`), o?.profilerTrace) {
-    let O = Sd(o.profilerTrace);
-    O.isKernelStraggler && (console.log(`[Diagnosis] Trace Analysis: Straggler detected - ${O.stragglingKernel}`), e = `[KERNEL_STRAGGLER_DETECTED] ${O.evidence.join(" ")}
+    let S = Sd(o.profilerTrace);
+    S.isKernelStraggler && (console.log(`[Diagnosis] Trace Analysis: Straggler detected - ${S.stragglingKernel}`), e = `[KERNEL_STRAGGLER_DETECTED] ${S.evidence.join(" ")}
 
 ${e}`);
   }
   if (o?.hardwareCounters) try {
-    let O = JSON.parse(o.hardwareCounters), k = Od(O);
-    k.isPfcStorm && (console.log("[Diagnosis] PFC Storm Detected!"), e = `[NETWORK_CONGESTION_STORM] ${k.evidence.join(" ")}
+    let S = JSON.parse(o.hardwareCounters), C = Od(S);
+    C.isPfcStorm && (console.log("[Diagnosis] PFC Storm Detected!"), e = `[NETWORK_CONGESTION_STORM] ${C.evidence.join(" ")}
 
-${e}`), k.isDataLoaderBottleneck && (console.log("[Diagnosis] Host CPU/DataLoader Bottleneck Detected!"), e = `[DATALOADER_BOTTLENECK] ${k.evidence.join(" ")}
+${e}`), C.isDataLoaderBottleneck && (console.log("[Diagnosis] Host CPU/DataLoader Bottleneck Detected!"), e = `[DATALOADER_BOTTLENECK] ${C.evidence.join(" ")}
 
 ${e}`);
-  } catch (O) {
-    console.warn("[Diagnosis] Failed to parse hardware counters:", O);
+  } catch (S) {
+    console.warn("[Diagnosis] Failed to parse hardware counters:", S);
   }
-  let f = [l, o?.dmesg, o?.nvidia_smi, o?.ibstat].filter((O) => typeof O == "string" && O.trim().length > 0).join(`
+  let f = [l, o?.dmesg, o?.nvidia_smi, o?.ibstat].filter((S) => typeof S == "string" && S.trim().length > 0).join(`
 
-`), E = await ZE({ logs: l, plan: i, telemetry: o, codeContext: a, topologyMap: s, operatorAnswers: d, memoryScope: r === "anonymous" ? "anonymous" : r, modelReview: t.CLOUD_DIAGNOSIS_REVIEW === "required" && !Le(t) }), C = null, y = (O) => ZR(O, e, f, C), T = il(tt(f));
+`), E = await ZE({ logs: l, plan: i, telemetry: o, codeContext: a, topologyMap: s, operatorAnswers: d, memoryScope: r === "anonymous" ? "anonymous" : r, modelReview: t.CLOUD_DIAGNOSIS_REVIEW === "required" && !Le(t) }), k = null, y = (S) => ZR(S, e, f, k), T = il(tt(f));
   if (T) return { success: true, signature: E, outcome: y(T), cacheEligible: true };
-  if (/\b(?:explain|why|how|what|tell me)\b/i.test(l) && /\bno (?:runtime |incident |training )?logs (?:attached|provided|available)\b/i.test(l) && ![o?.dmesg, o?.nvidia_smi, o?.ibstat].some((O) => O?.trim()) && !/NVRM:\s*Xid|Traceback \(most recent|RuntimeError:|NCCL WARN/i.test(l)) return { success: true, signature: E, outcome: y({ tier: 0, failureType: "NEEDS_MORE_SIGNAL", confidence: 0, summary: "This is a configuration question without incident logs. It does not establish a runtime failure or its cause.", action: "Verify the premise against the documentation for your exact version. If a run actually hangs, provide the first runtime errors and all-rank progress traces before changing configuration.", evidence: [], sourceType: "input-boundary", resumeFrom: null }) };
+  if (/\b(?:explain|why|how|what|tell me)\b/i.test(l) && /\bno (?:runtime |incident |training )?logs (?:attached|provided|available)\b/i.test(l) && ![o?.dmesg, o?.nvidia_smi, o?.ibstat].some((S) => S?.trim()) && !/NVRM:\s*Xid|Traceback \(most recent|RuntimeError:|NCCL WARN/i.test(l)) return { success: true, signature: E, outcome: y({ tier: 0, failureType: "NEEDS_MORE_SIGNAL", confidence: 0, summary: "This is a configuration question without incident logs. It does not establish a runtime failure or its cause.", action: "Verify the premise against the documentation for your exact version. If a run actually hangs, provide the first runtime errors and all-rank progress traces before changing configuration.", evidence: [], sourceType: "input-boundary", resumeFrom: null }) };
   if (Fm(`${e}
 
 ${f}`)) {
-    let O = Ut(l);
-    return { success: true, outcome: y(O), signature: E };
+    let S = Ut(l);
+    return { success: true, outcome: y(S), signature: E };
   }
   let A = ct(f)?.line || "", w = A && !Ar(A) ? A : Om(f);
   try {
-    let O = Le(t) ? null : await t.DIAGNOSIS_CACHE?.get(`diag:${i}:${E}`);
-    if (O) {
+    let S = Le(t) ? null : await t.DIAGNOSIS_CACHE?.get(`diag:${i}:${E}`);
+    if (S) {
       console.log("[Diagnosis] Cache hit");
-      let k = JSON.parse(O), x = k.forensicReport?.finalization?.version === 1;
-      return { success: true, outcome: x ? k : y(k), signature: E, cached: true, finalized: x };
+      let C = JSON.parse(S), x = C.forensicReport?.finalization?.version === 1;
+      return { success: true, outcome: x ? C : y(C), signature: E, cached: true, finalized: x };
     }
-  } catch (O) {
-    console.log("[Diagnosis] Cache error:", O);
+  } catch (S) {
+    console.log("[Diagnosis] Cache error:", S);
   }
-  let S = new AbortController(), b = setTimeout(() => S.abort(), 31e4);
+  let O = new AbortController(), b = setTimeout(() => O.abort(), 31e4);
   try {
     {
       let N = tt(f), te = Br(N), Y = ct(N), L = async (F) => {
@@ -10289,7 +10298,7 @@ ${f}`)) {
       }, X = Ke(N), ce = vn(N, X), J = { ...X.positives.length || X.negatives.length ? { telemetryEvidence: { indicting: X.positives.slice(0, 6), exonerating: X.negatives.slice(0, 6) } } : {}, ...ce.originNote ? { originNote: ce.originNote } : {}, ...ce.resumeSafetyNote ? { resumeSafetyNote: ce.resumeSafetyNote } : {}, ...ce.secondaryHardwareNote ? { secondaryHardwareNote: ce.secondaryHardwareNote } : {} };
       if (!Y && !Ir(f) && !/Watchdog caught collective|collective operation timeout|NCCL.*\btimeout\b|timed out waiting/i.test(f) && /\bstep\s*=?\s*\d+[^\n]*\bloss\s*=/i.test(f) && /checkpoint[^\n]*(?:verified|sha256)[^\n]*(?:OK|PASS)/i.test(f) && /DCGM[^\n]*PASS/i.test(f) && X.xidExplicitlyClean && X.eccClean === true) return L({ tier: 0, failureType: "NO_FAILURE_DETECTED", confidence: 55, summary: "No crash or diagnosable fault is present in this excerpt. Training is progressing, the checkpoint verified successfully, and the supplied GPU health checks explicitly report clean ECC/Xid state.", action: "Continue monitoring. If an incident occurred outside this excerpt, include the first failing rank, its exception/traceback, the matching dmesg window, and per-rank timestamps. No disruptive hardware remediation is warranted from this healthy sample.", fallbackAction: null, emergencyAction: null, resumeFrom: "current healthy run", sourceType: "negative-control", evidence: f.split(/\r?\n/).filter((F) => /step.*loss|checkpoint.*(?:verified|sha256)|ECC\s*=\s*0|Xid\s*=\s*None|DCGM.*PASS/i.test(F)).slice(0, 4), ...J });
       let Et = /version magic|Invalid module format|\/dev\/nvidia-uvm[^\n]*(?:ABSENT|missing|no such)|nvidia[_-]uvm\s+NOT loaded/i.test(f), Rt = Y ? Lt(N, Y.line) : null, cr = Rt ? Qi(Rt.key, N || e) : null;
-      Rt && cr && (C = { candidate: Rt.key, arm: cr }, console.log(`[Diagnosis] tier-0 initiator ${Rt.key} stood down: the log shows the same hardware clean under a different software configuration ("${cr.line.slice(0, 120)}")`));
+      Rt && cr && (k = { candidate: Rt.key, arm: cr }, console.log(`[Diagnosis] tier-0 initiator ${Rt.key} stood down: the log shows the same hardware clean under a different software configuration ("${cr.line.slice(0, 120)}")`));
       let oo = Ir(N), zm = !oo && /CUDAGraph replay/i.test(f) && /succeeds with --enforce-eager/i.test(f) && /ECC\s*=\s*0|no Xid|DCGM\s+PASS/i.test(f), Or = Mr(f), Hm = /\bXid(?:\s*\([^)]*\))?\s*:?\s*[1-9]\d{0,2}\b|(?:double[\s-]?bit|uncorrectable)\s+ECC\s+error|GPU has fallen off the bus|AER:\s*(?:Uncorrected|Fatal)/i.test(f), ao = /* @__PURE__ */ new Set(["nvidia-blackwell-xid109-driver-regression", "nvidia-open-xid31-154-gsp-mmu-regression", "vllm-sm120-recurring-chip-wide-xid13", "cutlass-sm100-nvfp4-swiglu-prefill-xid13", "blackwell-nvlink-netir-link-down", "nvswitch-sxid-fatal-fabric-abort", "gb200-nvl72-uptime-nvlink-firmware-hang"]), Vm = Or.some((F) => ao.has(F.id)), qm = Or.length > 0 && (!Hm || Vm);
       if (!!!(Rt && Rt.key !== "NCCL" && !Q(Rt.key).rejected && !cr && !zm && !qm && (te.detected || !Et))) {
         if (Or.length > 0) {
@@ -10368,8 +10377,8 @@ ${F}` : e, H = ol(V);
       let Wi = f.split(/\r?\n/).filter((F) => /Watchdog caught collective|collective operation timeout|NCCL.*\btimeout\b|timed out waiting/i.test(F)).map((F) => F.trim()).filter(Boolean), ep = /Temporary failure in name resolution|socket\.gaierror|CoreDNS|rendezvous|DistStoreError|ibv_|mlx5|Link\s+down|vendor err|CQE|AER|IOMMU|NVLink|Xid\s*(?:\([^)]*\))?\s*:?\s*\d+|Lustre|Ceph|NFS|checkpoint[^\n]*(?:saving|loading|write|read|stall)|I\/O error|No space left|collective[^\n]*(?:mismatch|desync)|tensor[^\n]*shape[^\n]*mismatch|size mismatch for|unexpected keyword argument|Insufficient nvidia\.com\/(?:mig|gpu)|checkpoint was saved with world_size|ObjectStoreFullError|BackendCompilerFailed|PytorchStreamReader|Unexpected bus error|SIGBUS|shared memory|sample_id\s*=|target_length\s*=|undefined symbol|Must flatten tensors with uniform dtype|gres\/gpu count reported lower|state\s*=\s*INVAL|CrashLoopBackOff|op(?:eration)?[^\n]*mismatch|rank\s*\d+[^\n]*(?:slow|stuck|behind)|rank\s*\d+[^\n]*\d{3,}\s*ms|OOMKilled|out of memory|device-side assert|illegal memory access|CUDA error/i.test(f), tp = /TORCH_NCCL_TIMEOUT|NCCL_TIMEOUT/i.test(f) && /completes? in\s*\d+\s*s\b/i.test(f), rp = vr(f);
       if (Wi.length > 0 && !Y && !ep && !rp && !tp && !qi) return L({ tier: 0, failureType: "NCCL_TIMEOUT_UNATTRIBUTED", confidence: 52, summary: `${Wi.length} collective timeout line(s) are present, but the excerpt does not identify the initiating rank or subsystem. A watchdog timeout is the terminal symptom shared by GPU crashes, rank desynchronization, fabric/DNS/storage faults, and genuine collective hangs; assigning one of those causes from this evidence would be unsafe.`, action: 'Reproduce once with the PyTorch NCCL Flight Recorder enabled on every rank: `TORCH_NCCL_TRACE_BUFFER_SIZE=200000`, `TORCH_NCCL_DUMP_ON_TIMEOUT=1`, and `TORCH_NCCL_DESYNC_DEBUG=1`. Preserve the per-rank dump and compare the first collective/sequence number where ranks diverge. From the same time window collect `dmesg -T | grep -iE "xid|nvrm|aer|iommu"`, `nvidia-smi -q -d ECC,ROW_REMAPPER`, and NIC/switch errors. The first lower-layer event or first rank that never joined the collective is the initiator.', fallbackAction: "If the Flight Recorder is unavailable, align all rank logs by timestamp and capture the last completed collective plus the first exception on each rank. Run a small `nccl-tests all_reduce_perf` control on the same nodes to separate application collective-order/shape bugs from fabric failures.", emergencyAction: null, resumeFrom: "the last checkpoint before the timeout", sourceType: "triage", evidence: Wi.slice(0, 4), evidenceFirst: true, clarifyingQuestions: _r(e).map((F) => F.question), clarifyingChoices: _r(e), ...J });
     }
-    let O = Fr(f).length > 0 || hr(f).length > 0 || Mr(f).length > 0, k = /size mismatch for|unexpected keyword argument\s+['"]dispatch_batches|Insufficient nvidia\.com\/(?:mig-[^\s.]+|gpu)|checkpoint was saved with world_size\s*=|PytorchStreamReader failed reading zip archive|No space left on device|sample_id\s*=\s*\d+[^\n]*target_length\s*=\s*0|undefined symbol[^\n]*flash|flash_attn[^\n]*undefined symbol|Must flatten tensors with uniform dtype|gres\/gpu count reported lower|state\s*=\s*INVAL|CrashLoopBackOff|ObjectStoreFullError|BackendCompilerFailed[^\n]*inductor/i.test(f), D = /IFree\s+0\b|IUse%\s*(?:100|9\d)\b/i.test(f), x = /100%\s*used|\b0\s*avail\b/i.test(f), B = /does NOT match|checksum mismatch|sha256 mismatch|hash mismatch|sha256 matches|matches the manifest/i.test(f), z11 = /Temporary failure in name resolution|socket\.gaierror|\bSERVFAIL\b|\bNXDOMAIN\b|Name or service not known|\bEAI_AGAIN\b/i.test(f), oe = /volume node affinity conflict|node\(s\) had taint|didn't match Pod's node affinity|had volume node affinity/i.test(f), ae = /world_size\s*=\s*\d+[\s\S]*world_size\s*=\s*\d+/i.test(f) && /sha256[^\n]*match|shards are present|weights load without error|checksums? match/i.test(f), se = /NCCL\s+WARN[^\n]*CUDA graph capture mode mismatch|capture-relaxed\s+vs\s+global/i.test(f) && /graph capture replay[^\n]*(?:failed|error)|internal error[^\n]*graph capture replay/i.test(f);
-    if (!k || D || z11 || x || B || oe || ae || se || O) {
+    let S = Fr(f).length > 0 || hr(f).length > 0 || Mr(f).length > 0, C = /size mismatch for|unexpected keyword argument\s+['"]dispatch_batches|Insufficient nvidia\.com\/(?:mig-[^\s.]+|gpu)|checkpoint was saved with world_size\s*=|PytorchStreamReader failed reading zip archive|No space left on device|sample_id\s*=\s*\d+[^\n]*target_length\s*=\s*0|undefined symbol[^\n]*flash|flash_attn[^\n]*undefined symbol|Must flatten tensors with uniform dtype|gres\/gpu count reported lower|state\s*=\s*INVAL|CrashLoopBackOff|ObjectStoreFullError|BackendCompilerFailed[^\n]*inductor/i.test(f), D = /IFree\s+0\b|IUse%\s*(?:100|9\d)\b/i.test(f), x = /100%\s*used|\b0\s*avail\b/i.test(f), B = /does NOT match|checksum mismatch|sha256 mismatch|hash mismatch|sha256 matches|matches the manifest/i.test(f), z11 = /Temporary failure in name resolution|socket\.gaierror|\bSERVFAIL\b|\bNXDOMAIN\b|Name or service not known|\bEAI_AGAIN\b/i.test(f), oe = /volume node affinity conflict|node\(s\) had taint|didn't match Pod's node affinity|had volume node affinity/i.test(f), ae = /world_size\s*=\s*\d+[\s\S]*world_size\s*=\s*\d+/i.test(f) && /sha256[^\n]*match|shards are present|weights load without error|checksums? match/i.test(f), se = /NCCL\s+WARN[^\n]*CUDA graph capture mode mismatch|capture-relaxed\s+vs\s+global/i.test(f) && /graph capture replay[^\n]*(?:failed|error)|internal error[^\n]*graph capture replay/i.test(f);
+    if (!C || D || z11 || x || B || oe || ae || se || S) {
       let N = Hr(f);
       if (N.length > 0) {
         let Y = Pi(N, f, Mt, { cascade: 120, dataloader_shm_sigbus: 125, gsp_compound: 115, gpudirect_rdma_deadlock: 112, "roce-pfc-congestion": 111, ib_fabric: 110, host_kernel_fault: 108, dcgm_diag: 106, xid_router: 104, storage_stall: 102, masked_data_corruption: 100, replica_divergence: 98, sdc: 96, index_oob_mask: 94, ddp_desync: 92, oom_frag: 90, "nccl-single-rail-binding": 118, "numa-affinity-misbinding": 116, "cpu-governor-throttle": 115, "host-swap-thrash": 114, "ecc-mode-pending-reboot": 113, "thp-compaction-stall": 112, "torch-compile-recompile-thrash": 111, "pcie-link-downgrade": 109, "inode-exhaustion": 107, "nccl-algo-override": 117, "gpu-clocks-locked": 119, "ntp-clock-skew": 110, "nccl-socket-ifname-fallback": 120, "cgroup-cpu-quota-throttle": 121, "mixed-gpu-sku": 122, "nccl-version-skew": 108, "hca-error-counter-storm": 113, "ephemeral-storage-eviction": 124, "object-store-spilling": 123, "rack-power-cap": 125, "readonly-remount-checkpoint-loss": 126, "dns-resolution-failure": 128, "slurm-qos-preemption": 127, "nccl-port-blocked-by-policy": 130, "filesystem-quota-exceeded": 129, "rendezvous-endpoint-process-dead": 131, "nvlink-inactive-fabric-manager": 132, "ib-partition-pkey-mismatch": 133, "collective-timeout-below-workload": 128, "checkpoint-format-version-skew": 126, "filesystem-capacity-exhausted": 134, "nvidia-kernel-module-not-loaded": 138, "space-held-by-unlinked-files": 135, "object-store-credential-expired": 136, "checkpoint-integrity-failure": 137, "registry-pull-rate-limited": 139, "dataset-host-rate-limited": 141, "object-store-permission-revoked": 140, "cgroup-pids-limit-reached": 142, "nfs-stale-file-handle": 143, "dcgm-stress-failure": 144, "storage-backend-unreachable": 146, "host-thread-oversubscription": 145, "secret-reference-missing-after-rotation": 147, "volume-zone-affinity-conflict": 150, "ecc-page-retirement-degrading": 205, "checkpoint-write-stampede": 148, "fsdp-reduce-dtype-mismatch": 149, "cluster-gpu-capacity-exhausted": 152, "optimizer-shard-world-size-mismatch": 153, "hugepage-reservation-starves-page-cache": 154, "nccl-net-plugin-missing": 155, "gpu-bound-preprocessing-regression": 156, "container-log-write-backpressure": 157, "unstable-tsc-clocksource-demotion": 158, "mps-server-exited": 151, "no-fault-expected-configured-workload-scaling": 166, "iterable-dataset-worker-duplication": 167, "kube-topology-manager-admission-rejected": 168, "cgroup-memory-high-reclaim-throttle": 169, "tf32-disabled-by-matmul-precision": 170, "slurm-time-limit-reached": 171, "pci-bar-mmio-firmware-config": 172, "tokenizer-embedding-vocab-mismatch": 173, "no-fault-historical-log-replay": 305, "no-fault-informational-hardware-banner": 306, "k8s-memory-eviction-rank-kill": 176, "sharp-in-network-reduction-corruption": 177, "deepspeed-config-type-coercion-error": 178, "cpu-isa-mismatch-build": 179, "bf16-unsupported-gpu-architecture": 180, "pcie-aer-fatal-link-fault": 181, "pcie-aer-interrupt-storm": 181, "data-pipeline-partial-ingest": 182, "rdma-memlock-rlimit-too-low": 183, "mxfp4-block-scale-overflow": 184, "gpu-xid-unknown-vendor-escalation": 185, "carbon-aware-scheduler-preemption": 186, "nccl-invalid-algorithm-enum": 187, "nccl-graph-capture-mode-incompatibility": 190, "lustre-ost-client-eviction": 188, "gpfs-token-manager-contention": 189, preflight: 10, straggler: 5, weight_delta: 4 });
@@ -10459,13 +10468,13 @@ Paste the output here to continue.`, clarifyingQuestions: [], evidence: [], fall
     clearTimeout(b);
     let Hi = { ...Ut(e), tier: 4, resumeFrom: null, sourceType: "triage", ...$e, ...zt, ...Se };
     return { success: true, outcome: y(Hi), signature: E };
-  } catch (O) {
-    if (clearTimeout(b), O.name === "AbortError") {
+  } catch (S) {
+    if (clearTimeout(b), S.name === "AbortError") {
       console.error("[Diagnosis] Total timeout exceeded - falling back to triage playbook");
-      let k = { ...Ut(e), tier: 4, resumeFrom: null, sourceType: "timeout_fallback" };
-      return { success: true, outcome: y(k), signature: E };
+      let C = { ...Ut(e), tier: 4, resumeFrom: null, sourceType: "timeout_fallback" };
+      return { success: true, outcome: y(C), signature: E };
     }
-    throw O;
+    throw S;
   }
 }
 var Jr = 4096;
@@ -10514,77 +10523,77 @@ async function Um(e, t, r, i = "free", n, o, a, s, c, d, l) {
   e = tk(e);
   let u = "", _ = [];
   if (!Le(t) && r !== "anonymous" && t.denpex_db) try {
-    let [S, b] = await Promise.all([xt(od(t.denpex_db, r, 10, e), [], "incident-history"), xt(cd(t.denpex_db, r), [], "pattern-emergence")]);
-    _ = b.map((O) => ({ patternName: O.pattern_name, description: O.description, frequency: O.frequency, lastSeen: new Date(O.last_seen * 1e3).toISOString() })), S.length > 0 && (u = `### INCIDENT HISTORY & MEMORY
+    let [O, b] = await Promise.all([xt(od(t.denpex_db, r, 10, e), [], "incident-history"), xt(cd(t.denpex_db, r), [], "pattern-emergence")]);
+    _ = b.map((S) => ({ patternName: S.pattern_name, description: S.description, frequency: S.frequency, lastSeen: new Date(S.last_seen * 1e3).toISOString() })), O.length > 0 && (u = `### INCIDENT HISTORY & MEMORY
 
 VERIFIED RELEVANT PAST INCIDENTS:
-${S.map((k, D) => `${D + 1}. [${new Date(k.created_at * 1e3).toISOString()}] ${k.failure_type} (node: ${k.node_id || "N/A"}, outcome: ${k.fix_outcome || "unknown"})
-   Root cause: ${k.root_cause || "N/A"}
-   Fix applied: ${k.fix_applied || "N/A"}`).join(`
+${O.map((C, D) => `${D + 1}. [${new Date(C.created_at * 1e3).toISOString()}] ${C.failure_type} (node: ${C.node_id || "N/A"}, outcome: ${C.fix_outcome || "unknown"})
+   Root cause: ${C.root_cause || "N/A"}
+   Fix applied: ${C.fix_applied || "N/A"}`).join(`
 `)}`);
-  } catch (S) {
-    console.error("[Diagnosis] Memory retrieval failed:", S);
+  } catch (O) {
+    console.error("[Diagnosis] Memory retrieval failed:", O);
   }
-  let g = null, f = null, E = [e, n?.dmesg, n?.nvidia_smi].filter((S) => typeof S == "string" && S.trim().length > 0).join(`
+  let g = null, f = null, E = [e, n?.dmesg, n?.nvidia_smi].filter((O) => typeof O == "string" && O.trim().length > 0).join(`
 
-`), C = gn(E.length > e.length ? E : e);
-  if (C.filteredCount > 0) {
-    let S = gn(e);
-    e = S.filteredCount > 0 ? S.filteredLogs : e, g = C, console.log(`[Diagnosis] Patient Zero isolated. Filtered ${C.filteredCount} downstream NCCL timeout lines.`);
+`), k = gn(E.length > e.length ? E : e);
+  if (k.filteredCount > 0) {
+    let O = gn(e);
+    e = O.filteredCount > 0 ? O.filteredLogs : e, g = k, console.log(`[Diagnosis] Patient Zero isolated. Filtered ${k.filteredCount} downstream NCCL timeout lines.`);
   }
   if (n?.currentEnv && n?.lastKnownGoodEnv) try {
-    let S = JSON.parse(n.currentEnv), b = JSON.parse(n.lastKnownGoodEnv), O = dd(S, b);
-    O.driftDetected && (f = O, console.log(`[Diagnosis] Environment drift detected: ${O.changes.length} changes.`), e = `${ld(O)}
+    let O = JSON.parse(n.currentEnv), b = JSON.parse(n.lastKnownGoodEnv), S = dd(O, b);
+    S.driftDetected && (f = S, console.log(`[Diagnosis] Environment drift detected: ${S.changes.length} changes.`), e = `${ld(S)}
 
 ${e}`);
-  } catch (S) {
-    console.warn("[Diagnosis] Failed to parse env telemetry for drift detection:", S);
+  } catch (O) {
+    console.warn("[Diagnosis] Failed to parse env telemetry for drift detection:", O);
   }
   let y = await JR(e, t, r, i, u, n, o, a, s, h), T = t.CLOUD_DIAGNOSIS_REVIEW === "required" && !Le(t);
   if (T && y?.success && y.outcome && !y.cached && Number(y.outcome.tier) < 4 && y.outcome.failureType !== "NEEDS_MORE_SIGNAL") {
-    let S = y.outcome, b = null;
+    let O = y.outcome, b = null;
     try {
-      b = await Pm(e, t, { memoryContext: u || void 0, telemetry: n, codeContext: o, operatorAnswers: h, deterministicCandidate: { failureType: S.failureType, summary: S.summary, action: S.action, evidence: S.evidence, fallbackAction: S.fallbackAction, verificationCommand: S.verificationCommand } }, s);
+      b = await Pm(e, t, { memoryContext: u || void 0, telemetry: n, codeContext: o, operatorAnswers: h, deterministicCandidate: { failureType: O.failureType, summary: O.summary, action: O.action, evidence: O.evidence, fallbackAction: O.fallbackAction, verificationCommand: O.verificationCommand } }, s);
     } catch (D) {
       console.warn("[Diagnosis] Cloud model review unavailable", D instanceof Error ? D.name : "unknown");
     }
-    let O = [e, n?.dmesg, n?.nvidia_smi, n?.ibstat].filter(Boolean).join(`
+    let S = [e, n?.dmesg, n?.nvidia_smi, n?.ibstat].filter(Boolean).join(`
 
-`), k = b ? io(b.evidence || [], O) : [];
-    b && k.length && Ue(b.summary).length >= 12 && Ue(b.action).length >= 8 ? (y.outcome = { ...b, tier: 4, sourceType: "provider", summary: Ue(b.summary), action: Ue(b.action), fallbackAction: b.fallbackAction ? Ue(b.fallbackAction) : null, emergencyAction: b.emergencyAction ? Ue(b.emergencyAction) : null, verificationCommand: b.verificationCommand ? Ue(b.verificationCommand) : null, prevention: b.prevention ? Ue(b.prevention) : null, causationChain: b.causationChain || void 0, competingHypotheses: b.competingHypotheses || rt(e, b.failureType), evidence: k, resumeFrom: null }, br(y.outcome, e), Bi(y.outcome, e), y.providerCompletion = b.providerCompletion) : y.cacheEligible = false;
+`), C = b ? io(b.evidence || [], S) : [];
+    b && C.length && Ue(b.summary).length >= 12 && Ue(b.action).length >= 8 ? (y.outcome = { ...b, tier: 4, sourceType: "provider", summary: Ue(b.summary), action: Ue(b.action), fallbackAction: b.fallbackAction ? Ue(b.fallbackAction) : null, emergencyAction: b.emergencyAction ? Ue(b.emergencyAction) : null, verificationCommand: b.verificationCommand ? Ue(b.verificationCommand) : null, prevention: b.prevention ? Ue(b.prevention) : null, causationChain: b.causationChain || void 0, competingHypotheses: b.competingHypotheses || rt(e, b.failureType), evidence: C, resumeFrom: null }, br(y.outcome, e), Bi(y.outcome, e), y.providerCompletion = b.providerCompletion) : y.cacheEligible = false;
   }
   if (y?.outcome && !y.cached && /\bnan\b/i.test(e) && /\broot cause\s*:\s*(?:unknown|undetermined)\b/i.test(e) && /(?:cannot be reproduced|no NaNs reproduced|not reproducible)/i.test(e) && /(?:replay successful|replay passed|repeated replay)/i.test(e) && !/\b(?:NVRM:\s*Xid|Double Bit ECC Error|uncorrectable ECC|fatal hardware error)\b/i.test(e) && (y.outcome.failureType = "NEEDS_MORE_SIGNAL", y.outcome.confidence = Math.min(45, Number(y.outcome.confidence) || 45), y.outcome.confidenceDisplay = null, y.outcome.confidenceBand = "low", y.outcome.summary = "The NaN was observed, but repeated clean replays do not establish its root cause. The evidence is insufficient to name a specific failure mechanism.", y.outcome.rootCause = y.outcome.summary, y.outcome.action = "Keep the original log and capture the first affected layer inputs, output, RNG state, kernel versions, and per-rank telemetry if it happens again. Compare the first divergent tensor with a clean replay before changing hardware or training configuration.", y.outcome.steps = [], y.outcome.fallbackAction = null, y.outcome.emergencyAction = null, y.outcome.verificationCommand = null, y.outcome.prevention = null, y.outcome.resumeFrom = null, y.outcome.causationChain = null, y.outcome.competingHypotheses = null, y.outcome.rankedHypotheses = null, y.outcome.alternatives = [], y.outcome.forensicReport = null), y?.outcome && d && (y.outcome.id = d), y?.cached && y?.finalized && y?.outcome) {
     if (y.outcome.buildIdentity = Jn(), y.outcome.routeDiagnostics = { ...y.outcome.routeDiagnostics || {}, latencyMs: Date.now() - m, cacheHit: true }, d && !Le(t) && t.denpex_db && !ei(y.outcome)) {
-      let S = await xt(Em(y.outcome, e, t, r, _), { outcome: y.outcome, pendingRecord: void 0 }, "incident-memory-cached");
-      S.pendingRecord && (S.pendingRecord.failure_type = String(y.outcome.failureType || S.pendingRecord.failure_type), S.pendingRecord.root_cause = String(y.outcome.summary || S.pendingRecord.root_cause || ""), S.pendingRecord.fix_applied = String(y.outcome.action || S.pendingRecord.fix_applied || ""), await hn(t.denpex_db, S.pendingRecord));
+      let O = await xt(Em(y.outcome, e, t, r, _), { outcome: y.outcome, pendingRecord: void 0 }, "incident-memory-cached");
+      O.pendingRecord && (O.pendingRecord.failure_type = String(y.outcome.failureType || O.pendingRecord.failure_type), O.pendingRecord.root_cause = String(y.outcome.summary || O.pendingRecord.root_cause || ""), O.pendingRecord.fix_applied = String(y.outcome.action || O.pendingRecord.fix_applied || ""), await hn(t.denpex_db, O.pendingRecord));
     }
     return y;
   }
   if (y?.outcome && h.length) {
     Kc(y.outcome, h, e), Yr(y.outcome, e);
-    let S = Qn(y.outcome);
-    y.outcome.confidenceBand = S.band, y.outcome.confidenceDisplay = S.display, /^(?:NO_FAILURE|NO_FAULT|HEALTHY)/i.test(String(y.outcome.failureType || "")) || qn(y.outcome, e);
+    let O = Qn(y.outcome);
+    y.outcome.confidenceBand = O.band, y.outcome.confidenceDisplay = O.display, /^(?:NO_FAILURE|NO_FAULT|HEALTHY)/i.test(String(y.outcome.failureType || "")) || qn(y.outcome, e);
   }
   let A = y?.outcome ? tt(e) : e, w;
   if (y && y.success && y.outcome && !ei(y.outcome)) {
-    let S = Le(t) || !d ? { outcome: y.outcome } : await xt(Em(y.outcome, e, t, r, _), { outcome: y.outcome, pendingRecord: void 0 }, "incident-memory");
-    y.outcome = S.outcome, w = S.pendingRecord, y.outcome.forensicReport = y.outcome.forensicReport || {}, g && g.patientZeroEvent && (y.outcome.forensicReport.rootCauseReplay = y.outcome.forensicReport.rootCauseReplay || [], y.outcome.forensicReport.rootCauseReplay.unshift({ timestamp: "Patient Zero", event: g.patientZeroEvent, isInitiator: true })), f && f.driftDetected && (y.outcome.forensicReport.whatChangedAnalyzer = { previous: {}, current: {}, detectedChanges: f.changes.map((z11) => `${z11.component}: changed from ${z11.oldValue || "undefined"} to ${z11.newValue || "undefined"} (${z11.severity} severity)`) });
+    let O = Le(t) || !d ? { outcome: y.outcome } : await xt(Em(y.outcome, e, t, r, _), { outcome: y.outcome, pendingRecord: void 0 }, "incident-memory");
+    y.outcome = O.outcome, w = O.pendingRecord, y.outcome.forensicReport = y.outcome.forensicReport || {}, g && g.patientZeroEvent && (y.outcome.forensicReport.rootCauseReplay = y.outcome.forensicReport.rootCauseReplay || [], y.outcome.forensicReport.rootCauseReplay.unshift({ timestamp: "Patient Zero", event: g.patientZeroEvent, isInitiator: true })), f && f.driftDetected && (y.outcome.forensicReport.whatChangedAnalyzer = { previous: {}, current: {}, detectedChanges: f.changes.map((z11) => `${z11.component}: changed from ${z11.oldValue || "undefined"} to ${z11.newValue || "undefined"} (${z11.severity} severity)`) });
     let b = cn(e);
     b && b.score > 0 && (y.outcome.forensicReport.predictiveMaintenance = { metric: b.signals.join(", "), probabilityOfFailure: Math.min(b.score, 99), timeframe: b.level === "high" ? "24 hours" : b.level === "medium" ? "3 days" : "14 days", recommendation: b.level === "high" ? "Manual approval required before cordoning the node. Preserve telemetry and schedule field diagnostics." : "Monitor the node and schedule non-destructive field diagnostics during the next maintenance window." });
-    let O = null;
+    let S = null;
     try {
       if (n?.failedNodes) {
         let z11 = JSON.parse(n.failedNodes);
-        Array.isArray(z11) && typeof z11[0] == "string" && (O = z11[0]);
+        Array.isArray(z11) && typeof z11[0] == "string" && (S = z11[0]);
       }
     } catch {
     }
-    O || (O = ft(e));
-    let k = wn(y.outcome.summary, y.outcome.failureType, O);
-    if (k && k.actions.length > 0) {
+    S || (S = ft(e));
+    let C = wn(y.outcome.summary, y.outcome.failureType, S);
+    if (C && C.actions.length > 0) {
       y.outcome.forensicReport || (y.outcome.forensicReport = {});
       let z11 = { REBOOT_NODE: "Low", DRAIN_NODE: "Low", RECONFIGURE_FABRIC: "Medium", RESTART_JOB: "Medium", RUN_DIAGNOSTICS: "High", NOTIFY_ONCALL: "High" };
-      y.outcome.forensicReport.automationReadiness = k.actions.map((oe) => {
+      y.outcome.forensicReport.automationReadiness = C.actions.map((oe) => {
         let ae = z11[oe.action] ?? "Low", se = ae !== "High";
         return { action: oe.action, safety: ae, approval: se ? "manual-required" : "advisory", reason: `${se ? "Manual approval required. " : ""}${oe.reason} (Command: ${oe.command})` };
       });
@@ -10597,10 +10606,10 @@ ${e}`);
       Number.isFinite(ae) && ae > 1 && (y.outcome.forensicReport.costImpact = { gpusIdle: ae - 1, estimatedWastePerHour: Math.round((ae - 1) * 2.5 * 0.15), basis: `estimate: ${ae} GPUs (from your logs/metadata) \xD7 $${2.5}/GPU-hr reference rate \xD7 15% straggler penalty`, blockedJobs: 1, checkpointLostMinutes: 0 }), y.outcome.forensicReport.evidenceConfidencePerClaim = y.outcome.forensicReport.evidenceConfidencePerClaim || [], y.outcome.forensicReport.evidenceConfidencePerClaim.push({ claim: `Performance Asymmetry: ${x.evidence[0] || "Straggler Node detected"}`, overallConfidence: 95, supportedBy: x.evidence.map((se) => ({ rating: 5, evidence: se })), unsupportedBy: [] });
     }
     if (t.DB && !Le(t)) {
-      let z11 = await xt(sd(t.DB, r, y.outcome.failureType, O), null, "fleet-correlation");
+      let z11 = await xt(sd(t.DB, r, y.outcome.failureType, S), null, "fleet-correlation");
       if (z11) {
         y.outcome.forensicReport.fleetHealthCorrelation = z11;
-        let oe = [O || "unknown", ...z11.nodes.map((se) => se.node)], ae = bn(oe, a);
+        let oe = [S || "unknown", ...z11.nodes.map((se) => se.node)], ae = bn(oe, a);
         ae.isTopologyFault && ae.domain && (y.outcome.summary = `Mass-Crash Coalesced: InfiniBand/Network fabric failure detected affecting ${oe.length} nodes.`, y.outcome.rootCause = `The ${ae.domain.type.replace("_", " ")} "${ae.domain.name}" failed, causing a cascading hardware timeout across all attached nodes.`, y.outcome.forensicReport.fleetHealthCorrelation.pattern = "Systemic Network Fabric Fault (" + ae.domain.type + ")");
       }
     }
@@ -10638,9 +10647,9 @@ ${ae}${se}${ee}
     no(y.outcome), y.outcome.forensicReport?.finalization?.version !== 1 && Bn(y.outcome, { logs: A, telemetry: n, allowRmaPayload: dm(i, "team") }), y.outcome.buildIdentity = Jn();
     let b = nR(y.providerCompletion ?? y.outcome.providerCompletion);
     if (y.outcome.routeDiagnostics = { selectedTier: Number(y.outcome.tier ?? -1), sourceType: String(y.outcome.sourceType || "unknown"), latencyMs: Date.now() - m, cacheHit: !!y.cached, deepDeadlineMs: Rm, providerCompletionExpected: T || Number(y.outcome.tier ?? -1) === 4, modelReview: T ? b ? "completed" : "unavailable" : void 0, ...b ? { providerCompletion: b } : {} }, y.cacheEligible && !Le(t)) {
-      let O = t.DIAGNOSIS_CACHE?.put(`diag:${i}:${y.signature}`, JSON.stringify(y.outcome), { expirationTtl: 300 }).catch(() => {
+      let S = t.DIAGNOSIS_CACHE?.put(`diag:${i}:${y.signature}`, JSON.stringify(y.outcome), { expirationTtl: 300 }).catch(() => {
       });
-      O && (l ? l.waitUntil(O) : await O);
+      S && (l ? l.waitUntil(S) : await S);
     }
   }
   return y;
@@ -13057,7 +13066,7 @@ function registerPrompts(server) {
 
 // src/version.ts
 var SERVER_NAME = "denpex";
-var SERVER_VERSION = "0.2.30";
+var SERVER_VERSION = "0.2.31";
 
 // src/server.ts
 function createServer() {