@lawaty/lacode 2.3.0-beta.0 → 2.3.0-beta.1

package/CHANGELOG.md

@@ -9,6 +9,22 @@ commit as the changelog entry it describes.
 
 ## [Unreleased]
 
+## [2.3.0-beta.1] — provider degradation backoff
+
+### Added
+
+- Pool: a provider is now treated as degraded when two distinct models on it fail with transient, non-quota errors inside ten minutes. Every slot on that provider is benched for one flat 30-minute `provider-degraded` window (never a strike), `pool_status` names the provider, and a probe re-admits a model as soon as it answers. A single model failure keeps the existing 60-second flat outage window.
+- Pool: `npm run score` stops probing a provider after three consecutive provider-level timeouts or network errors, keeps its previous rows, and warns that the provider is degraded. The scan budget is reserved for healthy providers.
+- `oc-sync`: the local `file://.../lacode/dist/index.js` plugin entry is rewritten to the published `@lawaty/lacode@<version>` spec before syncing, so a remote host never receives a path that only exists on the development machine.
+
+### Changed
+
+- Pool: a flat outage window can now be longer than 60 seconds for a provider-wide degradation; it still never enters the strike memory, so recovery starts clean.
+
+### Fixed
+
+- Pool: repeated timeouts on a degraded provider no longer re-probe every model every few minutes or inflate each model's strike count.
+
 ## [2.3.0-beta.0] — reliable pool refresh, autonomy modes, and empty-response failover
 
 ### Added

package/dist/agents.js

@@ -35,6 +35,8 @@ var STUCK_IDLE_MS = 3 * 60 * 1e3;
 var MAX_CLAIM_AGE_MS = 30 * 60 * 1e3;
 var HANG_COOLDOWN_MS = 10 * 60 * 1e3;
 var UNAVAILABLE_COOLDOWN_MS = 60 * 1e3;
+var PROVIDER_DEGRADED_COOLDOWN_MS = 30 * 60 * 1e3;
+var PROVIDER_DEGRADED_WINDOW_MS = 10 * 60 * 1e3;
 var COOLDOWN_BASE_MS = 60 * 1e3;
 var COOLDOWN_MAX_MS = 15 * 60 * 1e3;
 var QUOTA_COOLDOWN_BASE_MS = 15 * 60 * 1e3;

package/dist/commands.js

@@ -39,6 +39,8 @@ var STUCK_IDLE_MS = 3 * 60 * 1e3;
 var MAX_CLAIM_AGE_MS = 30 * 60 * 1e3;
 var HANG_COOLDOWN_MS = 10 * 60 * 1e3;
 var UNAVAILABLE_COOLDOWN_MS = 60 * 1e3;
+var PROVIDER_DEGRADED_COOLDOWN_MS = 30 * 60 * 1e3;
+var PROVIDER_DEGRADED_WINDOW_MS = 10 * 60 * 1e3;
 var MAX_WEIGHT = 100;
 var COOLDOWN_BASE_MS = 60 * 1e3;
 var COOLDOWN_MAX_MS = 15 * 60 * 1e3;

package/dist/index.js

@@ -37,6 +37,12 @@ var MAX_CLAIM_AGE_MS = 30 * 60 * 1e3;
 var HANG_COOLDOWN_MS = 10 * 60 * 1e3;
 var UNAVAILABLE_COOLDOWN_MS = 60 * 1e3;
 var UNAVAILABLE_REASON = "endpoint-unavailable";
+var PROVIDER_DEGRADED_REASON = "provider-degraded";
+var PROVIDER_DEGRADED_COOLDOWN_MS = 30 * 60 * 1e3;
+var PROVIDER_DEGRADED_WINDOW_MS = 10 * 60 * 1e3;
+var PROVIDER_DEGRADED_MIN_MODELS = 2;
+var TRANSIENT_DEGRADATION_REASONS = /^(?:no answer|no-answer|hang|timeout|timed out|endpoint-unavailable|provider-degraded|empty response|network|server)$/i;
+var FLAT_OUTAGE_REASONS = /* @__PURE__ */ new Set([UNAVAILABLE_REASON, PROVIDER_DEGRADED_REASON]);
 var MAX_WEIGHT = 100;
 var MIN_SLOT_CONTEXT = 1e5;
 var MIN_SLOT_TPM = 2e4;
@@ -711,7 +717,7 @@ function readLimits(s) {
 }
 function recordStrike(s, model, reason, stated) {
   if (!s.dirReady) return;
-  if (reason === UNAVAILABLE_REASON) return coolUnavailable(s, model);
+  if (FLAT_OUTAGE_REASONS.has(reason)) return coolUnavailable(s, model, stated, reason);
   const now = Date.now();
   const slotsForModel = s.slots.filter((candidate) => candidate.model === model);
   const buckets = [...new Set(slotsForModel.map((slot) => slotBucket(slot)))];
@@ -742,13 +748,14 @@ function recordStrike(s, model, reason, stated) {
   }
   return strike;
 }
-function coolUnavailable(s, model) {
+function coolUnavailable(s, model, windowMs = UNAVAILABLE_COOLDOWN_MS, reason = UNAVAILABLE_REASON) {
   if (!s.dirReady) return;
   const now = Date.now();
+  const requested = Number.isFinite(windowMs) && windowMs > 0 ? Math.min(windowMs, STATED_COOLDOWN_MAX_MS) : UNAVAILABLE_COOLDOWN_MS;
   const prior = s.cooling.get(model);
-  const until = now + UNAVAILABLE_COOLDOWN_MS;
+  const until = now + requested;
   if (prior && prior.until > until) return prior;
-  const strike = { model, until, strikes: 1, reason: UNAVAILABLE_REASON, last: now };
+  const strike = { model, until, strikes: 1, reason, last: now };
   s.cooling.set(model, strike);
   const path4 = join2(s.dir, limitFile(model));
   try {
@@ -992,6 +999,24 @@ function coolProvider(s, provider, until, reason) {
   }
   return affected;
 }
+function noteProviderDegradation(s, model, reason) {
+  if (!s.dirReady) return [];
+  if (!TRANSIENT_DEGRADATION_REASONS.test(String(reason))) return [];
+  const provider = providerOf(model);
+  const slots = (s.slots.length > 0 ? s.slots : FALLBACK_SLOTS).filter((slot) => providerOf(slot.model) === provider);
+  if (slots.length < PROVIDER_DEGRADED_MIN_MODELS) return [];
+  const now = Date.now();
+  const failing = /* @__PURE__ */ new Set();
+  for (const slot of slots) {
+    const record = s.cooling.get(slot.model);
+    if (!record || record.until <= now) continue;
+    if (now - record.last > PROVIDER_DEGRADED_WINDOW_MS) continue;
+    if (!TRANSIENT_DEGRADATION_REASONS.test(String(record.reason))) continue;
+    failing.add(slot.model);
+  }
+  if (failing.size < PROVIDER_DEGRADED_MIN_MODELS) return [];
+  return coolProvider(s, provider, now + PROVIDER_DEGRADED_COOLDOWN_MS, PROVIDER_DEGRADED_REASON);
+}
 function recordFailure(s, f) {
   try {
     const path4 = join2(s.dir, "failures.jsonl");
@@ -3349,12 +3374,31 @@ var agentPoolHooks = (async ({ client, serverUrl, directory }, options) => {
     }
   };
   const MAX_ABORT_ATTEMPTS = 3;
+  const markProviderDegradation = async (log2, model, reason) => {
+    const provider = providerOf(model);
+    const slots = s.slots.length > 0 ? s.slots : FALLBACK_SLOTS;
+    const already = slots.some(
+      (slot) => providerOf(slot.model) === provider && s.cooling.get(slot.model)?.reason === PROVIDER_DEGRADED_REASON
+    );
+    const cooled = noteProviderDegradation(s, model, reason);
+    if (!already && cooled.length > 0) {
+      const until = new Date(Date.now() + PROVIDER_DEGRADED_COOLDOWN_MS).toISOString().slice(11, 19);
+      await log2("warn", `provider ${provider} degraded; ${cooled.length} slot(s) benched until ${until}Z`, {
+        provider,
+        reason,
+        models: cooled.map((record) => record.model)
+      });
+    }
+  };
   const coolForHang = async (claim, log2) => {
     const now = Date.now();
     const prior = s.cooling.get(claim.m);
     if (prior && prior.until > now) return;
     const strike = recordStrike(s, claim.m, "hang", HANG_FALLBACK_MS);
-    if (strike) await log2("warn", `cooling ${claim.m} for ${Math.round((strike.until - now) / 1e3)}s (hang)`);
+    if (strike) {
+      await log2("warn", `cooling ${claim.m} for ${Math.round((strike.until - now) / 1e3)}s (hang)`);
+      await markProviderDegradation(log2, claim.m, "hang");
+    }
   };
   const claimFor = (sessionID) => {
     if (!sessionID) return void 0;
@@ -3782,6 +3826,7 @@ var agentPoolHooks = (async ({ client, serverUrl, directory }, options) => {
               } else if (outage) {
                 const transient = coolUnavailable(s, own.claim.m);
                 if (transient) await log("warn", `cooling ${own.claim.m} for ${Math.round((transient.until - Date.now()) / 1e3)}s (${transient.reason})`);
+                await markProviderDegradation(log, own.claim.m, "endpoint-unavailable");
               } else {
                 const stated2 = statedCooldown(message);
                 const strike2 = recordStrike(s, own.claim.m, named, stated2);
@@ -3915,6 +3960,7 @@ var agentPoolHooks = (async ({ client, serverUrl, directory }, options) => {
               if (transient) {
                 await log("warn", `cooling ${own.claim.m} for ${Math.round((transient.until - Date.now()) / 1e3)}s (${transient.reason}) after a terminal error`);
               }
+              await markProviderDegradation(log, own.claim.m, "endpoint-unavailable");
             } else {
               stated2 = statedCooldown(haystack2) ?? retryAfterMs(error?.data?.responseHeaders);
               const strike2 = recordStrike(s, own.claim.m, named, stated2);
@@ -4087,6 +4133,7 @@ var agentPoolHooks = (async ({ client, serverUrl, directory }, options) => {
         if (isUnavailableText(haystack) || !limited && !auth && transientGateway) {
           const transient = coolUnavailable(s, model);
           if (transient) await log("warn", `cooling ${model} for ${Math.round((transient.until - Date.now()) / 1e3)}s (${transient.reason})`);
+          await markProviderDegradation(log, model, "endpoint-unavailable");
           await failOver(info.sessionID, model, transient?.reason ?? UNAVAILABLE_REASON, { cool: false });
           return;
         }
@@ -4166,6 +4213,7 @@ var agentPoolHooks = (async ({ client, serverUrl, directory }, options) => {
         const strike = recordStrike(s, spawn4.model, "empty response");
         const seconds = strike ? Math.max(1, Math.round((strike.until - Date.now()) / 1e3)) : 0;
         await log("warn", `empty response from ${spawn4.model} on slot ${spawn4.slot}; benched ${seconds}s`);
+        await markProviderDegradation(log, spawn4.model, "empty response");
         output.output = `Task returned an empty response from ${spawn4.model} (slot ${spawn4.slot}); the model has been benched for ${seconds}s so the next spawn uses another slot.`;
       }
       spawns.delete(key);
@@ -4242,6 +4290,7 @@ ${detail}` : detail;
           };
           const total = [...snap.counts.values()].reduce((sum, n) => sum + n, 0);
           const coolingNow = [...snap.cooling.entries()].filter(([, st]) => st.until > now);
+          const degradedProviders = [...new Set(coolingNow.filter(([, st]) => st.reason === PROVIDER_DEGRADED_REASON).map(([model]) => providerOf(model)))];
           const buckets = /* @__PURE__ */ new Map();
           for (const slot of slots) {
             const name = slotBucket(slot);
@@ -4262,6 +4311,7 @@ ${detail}` : detail;
 one ordered pass in slot order: each slot fills to its weight, then overflow goes to the least-loaded slot. Capacity is the sum of the weights; repeat a model on another line to give it more (FULL = at weight).`,
               ...next ? [] : ["next: none - no routable slot; every slot is cooling, catalog-retired, or provider-unconfigured. A spawn will fail explicitly instead of routing."],
               ...sharedBuckets.length ? [`buckets: ${sharedBuckets.map(([name, entry]) => `${name} ${entry.seats} seat(s)${entry.cooling ? " COOLING" : ""}`).join("; ")}`] : [],
+              ...degradedProviders.length ? [`provider degraded: ${degradedProviders.join(", ")} (flat window; a probe re-admits a model when it answers)`] : [],
               ...slots.map(line),
               "",
               "by process:",

package/dist/lacode.js

@@ -53,6 +53,8 @@ var STUCK_IDLE_MS = 3 * 60 * 1e3;
 var MAX_CLAIM_AGE_MS = 30 * 60 * 1e3;
 var HANG_COOLDOWN_MS = 10 * 60 * 1e3;
 var UNAVAILABLE_COOLDOWN_MS = 60 * 1e3;
+var PROVIDER_DEGRADED_COOLDOWN_MS = 30 * 60 * 1e3;
+var PROVIDER_DEGRADED_WINDOW_MS = 10 * 60 * 1e3;
 var COOLDOWN_BASE_MS = 60 * 1e3;
 var COOLDOWN_MAX_MS = 15 * 60 * 1e3;
 var QUOTA_COOLDOWN_BASE_MS = 15 * 60 * 1e3;

package/dist/pool.js

@@ -37,6 +37,12 @@ var MAX_CLAIM_AGE_MS = 30 * 60 * 1e3;
 var HANG_COOLDOWN_MS = 10 * 60 * 1e3;
 var UNAVAILABLE_COOLDOWN_MS = 60 * 1e3;
 var UNAVAILABLE_REASON = "endpoint-unavailable";
+var PROVIDER_DEGRADED_REASON = "provider-degraded";
+var PROVIDER_DEGRADED_COOLDOWN_MS = 30 * 60 * 1e3;
+var PROVIDER_DEGRADED_WINDOW_MS = 10 * 60 * 1e3;
+var PROVIDER_DEGRADED_MIN_MODELS = 2;
+var TRANSIENT_DEGRADATION_REASONS = /^(?:no answer|no-answer|hang|timeout|timed out|endpoint-unavailable|provider-degraded|empty response|network|server)$/i;
+var FLAT_OUTAGE_REASONS = /* @__PURE__ */ new Set([UNAVAILABLE_REASON, PROVIDER_DEGRADED_REASON]);
 var MAX_WEIGHT = 100;
 var MIN_SLOT_CONTEXT = 1e5;
 var MIN_SLOT_TPM = 2e4;
@@ -711,7 +717,7 @@ function readLimits(s) {
 }
 function recordStrike(s, model, reason, stated) {
   if (!s.dirReady) return;
-  if (reason === UNAVAILABLE_REASON) return coolUnavailable(s, model);
+  if (FLAT_OUTAGE_REASONS.has(reason)) return coolUnavailable(s, model, stated, reason);
   const now = Date.now();
   const slotsForModel = s.slots.filter((candidate) => candidate.model === model);
   const buckets = [...new Set(slotsForModel.map((slot) => slotBucket(slot)))];
@@ -742,13 +748,14 @@ function recordStrike(s, model, reason, stated) {
   }
   return strike;
 }
-function coolUnavailable(s, model) {
+function coolUnavailable(s, model, windowMs = UNAVAILABLE_COOLDOWN_MS, reason = UNAVAILABLE_REASON) {
   if (!s.dirReady) return;
   const now = Date.now();
+  const requested = Number.isFinite(windowMs) && windowMs > 0 ? Math.min(windowMs, STATED_COOLDOWN_MAX_MS) : UNAVAILABLE_COOLDOWN_MS;
   const prior = s.cooling.get(model);
-  const until = now + UNAVAILABLE_COOLDOWN_MS;
+  const until = now + requested;
   if (prior && prior.until > until) return prior;
-  const strike = { model, until, strikes: 1, reason: UNAVAILABLE_REASON, last: now };
+  const strike = { model, until, strikes: 1, reason, last: now };
   s.cooling.set(model, strike);
   const path = join2(s.dir, limitFile(model));
   try {
@@ -995,6 +1002,24 @@ function coolProvider(s, provider, until, reason) {
   }
   return affected;
 }
+function noteProviderDegradation(s, model, reason) {
+  if (!s.dirReady) return [];
+  if (!TRANSIENT_DEGRADATION_REASONS.test(String(reason))) return [];
+  const provider = providerOf(model);
+  const slots = (s.slots.length > 0 ? s.slots : FALLBACK_SLOTS).filter((slot) => providerOf(slot.model) === provider);
+  if (slots.length < PROVIDER_DEGRADED_MIN_MODELS) return [];
+  const now = Date.now();
+  const failing = /* @__PURE__ */ new Set();
+  for (const slot of slots) {
+    const record = s.cooling.get(slot.model);
+    if (!record || record.until <= now) continue;
+    if (now - record.last > PROVIDER_DEGRADED_WINDOW_MS) continue;
+    if (!TRANSIENT_DEGRADATION_REASONS.test(String(record.reason))) continue;
+    failing.add(slot.model);
+  }
+  if (failing.size < PROVIDER_DEGRADED_MIN_MODELS) return [];
+  return coolProvider(s, provider, now + PROVIDER_DEGRADED_COOLDOWN_MS, PROVIDER_DEGRADED_REASON);
+}
 function recordFailure(s, f) {
   try {
     const path = join2(s.dir, "failures.jsonl");
@@ -1068,6 +1093,8 @@ export {
   ORPHAN_TMP_MS,
   PACKAGE_ROOT,
   PRESETS_DIR,
+  PROVIDER_DEGRADED_COOLDOWN_MS,
+  PROVIDER_DEGRADED_REASON,
   PROVIDER_LIST_TTL_MS,
   QUOTA_COOLDOWN_BASE_MS,
   QUOTA_COOLDOWN_MAX_MS,
@@ -1112,6 +1139,7 @@ export {
   manualVariant,
   markProbe,
   noteActivity,
+  noteProviderDegradation,
   noteTpm,
   notifyDefault,
   parseRateLimitHeaders,

package/dist/tunnel.js

@@ -46,6 +46,8 @@ var STUCK_IDLE_MS = 3 * 60 * 1e3;
 var MAX_CLAIM_AGE_MS = 30 * 60 * 1e3;
 var HANG_COOLDOWN_MS = 10 * 60 * 1e3;
 var UNAVAILABLE_COOLDOWN_MS = 60 * 1e3;
+var PROVIDER_DEGRADED_COOLDOWN_MS = 30 * 60 * 1e3;
+var PROVIDER_DEGRADED_WINDOW_MS = 10 * 60 * 1e3;
 var COOLDOWN_BASE_MS = 60 * 1e3;
 var COOLDOWN_MAX_MS = 15 * 60 * 1e3;
 var QUOTA_COOLDOWN_BASE_MS = 15 * 60 * 1e3;

package/docs/USER-STORIES.md

@@ -91,6 +91,8 @@ The reconciliation with US-7 is that **acceptance and endorsement are different
 - [ ] Given a spawn fails with a rate-limit-shaped error, When the error is serialized and matched, Then the offending model is marked cooling and a cooldown file is written for it.
 - [ ] Given model A is cooling and another slot is usable, When the next spawn is routed, Then the pick is a different usable model, never the cooling one.
 - [ ] Given a pooled task completes with an empty result, When the task hook returns, Then that model is benched for `empty response`, the result explains the bench, and the next spawn routes to another slot.
+- [ ] Given two distinct models on one provider fail with transient, non-quota errors inside ten minutes, When the second failure is recorded, Then every slot on that provider is benched for one flat `provider-degraded` window and the next spawn routes to another provider.
+- [ ] Given a provider-degraded window lapses, When a probe answers on one of its models, Then that model is re-admitted without a restart, because the bench was evidence about the provider, not a strike against the model.
 - [ ] Given every model is cooling, When a spawn is routed, Then the spawn is refused explicitly with a message naming the cooling slots and their reasons, and no model is routed to.
 - [ ] Given a limit error names a reset time, When the cooldown is recorded, Then the stated reset is used instead of a guessed backoff.
 
@@ -180,6 +182,7 @@ A daily scan was killed by a reboot mid-run, and because systemd records a trigg
 - [ ] Given a scan reaches its wall-clock budget, When the scores are written, Then unscanned and truncated providers keep their previous rows, so a partial run cannot shrink the pool.
 - [ ] Given a model failed transiently (timeout, rate limit, credit), When selection runs, Then its last measured score is retained and a retry is scheduled at the stated reset or 45 minutes later.
 - [ ] Given a permanent failure (401, 403 agreement, 404), When selection runs, Then the model is not retried and does not fall back to a previous score.
+- [ ] Given three consecutive provider-level timeouts or network errors during a scan, When the breaker trips, Then the scan stops probing that provider, keeps its previous rows, and names the provider as degraded.
 - [ ] Given enough other providers to fill the pool, When selection runs, Then exactly one `opencode` slot is reserved last with weight 2.
 - [ ] Given two refreshes overlap, When the second starts, Then it skips while the lock is held, and a lock left by a dead PID is reclaimed.
 
@@ -636,6 +639,6 @@ The root-level suites under `tests/` are unit-style; the `tests/stories/` suites
 the behavioural layer, written one epic per file with every test name carrying
 its story id. Run everything with `npm test`.
 
-39/39 stories covered by 370 test case(s); thinnest story has 1.
+39/39 stories covered by 372 test case(s); thinnest story has 1.
 No story is untested. The scan covers the `tests/stories/` suites and the root-level
 `tests/*-test.mjs` suites.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lawaty/lacode",
-  "version": "2.3.0-beta.0",
+  "version": "2.3.0-beta.1",
   "description": "Ultimate opencode setup for Go users — cost-controlled pooled subagents, hang recovery, and notifications for opencode.",
   "license": "MIT",
   "type": "module",

package/scripts/pool-selection.mjs

@@ -251,6 +251,9 @@ export function selectPool({
       continue
     }
     const entryRows = Array.isArray(entry?.rows) ? entry.rows : []
+    if (entry?.degraded) {
+      warnings.push(`${entry.provider}: degraded — stopped after consecutive transient failures; previous rows kept`)
+    }
     if (entryRows.length === 0 || entryRows.some(rankable)) continue
     const kinds = new Map()
     for (const row of entryRows) {

package/scripts/score-models.mjs

@@ -68,6 +68,10 @@ const SLOW_MS = 8_000 // at or above this, speed scores 0
 const PROMPT = "Reply with the single word: ok"
 const MAX_TOKENS = 8
 const PROBE_ATTEMPTS = 2 // one retry for a timeout or a dropped connection
+// Three consecutive provider-level timeouts/network errors is enough evidence
+// that the provider is degraded. Stop spending the scan budget on the rest of
+// its catalogue; the previous rows are retained and a retry is scheduled.
+const PROVIDER_BREAKER_AFTER = 3
 /**
  * Retry the same probe only when the next attempt has a chance. A known
  * exhausted balance or a permanent 401/403/404 will not recover in 300ms; the
@@ -601,6 +605,7 @@ async function main() {
       continue
     }
     const rows = []
+    let consecutiveTransient = 0
     for (const candidate of candidates) {
       if (Date.now() >= deadline) {
         budgetHit = true
@@ -614,6 +619,9 @@ async function main() {
         await sleep(delayMs)
         outcome = await probe(candidate, entry.provider)
       }
+      const kind = outcome.kind ?? classifyFailure({ status: outcome.status, reason: outcome.reason })
+      const providerFault = !outcome.ok && ["timeout", "network", "server"].includes(kind)
+      consecutiveTransient = providerFault ? consecutiveTransient + 1 : 0
       const row = { provider: entry.provider, ...candidate, ...outcome, score: null }
       row.score = score(row)
       if (row.score && row.score.value < minScore) {
@@ -628,9 +636,17 @@ async function main() {
       rows.push(row)
       const verdict = outcome.ok ? `${outcome.ms}ms` : outcome.reason.slice(0, 60)
       console.log(`  ${pad(candidate.model.replace(`${entry.provider}/`, ""), 52)} ${padL(verdict, 16)}`)
+      if (consecutiveTransient >= PROVIDER_BREAKER_AFTER) {
+        entry.truncated = true
+        entry.degraded = true
+        console.log(
+          `  provider degraded: ${consecutiveTransient} consecutive ${kind} failures; stopped ${entry.provider} after ${rows.length}/${candidates.length}`,
+        )
+        break
+      }
       await sleep(delayMs)
     }
-    results.push({ provider: entry.provider, error: null, rows, excluded: entry.excluded, truncated: !!entry.truncated })
+    results.push({ provider: entry.provider, error: null, rows, excluded: entry.excluded, truncated: !!entry.truncated, degraded: !!entry.degraded })
   }
   if (budgetHit) {
     console.log(
@@ -733,6 +749,7 @@ async function main() {
             provider: e.provider,
             error: e.error,
             truncated: !!e.truncated,
+            degraded: !!e.degraded,
             excluded: e.excluded ?? [],
             rows: e.rows.flatMap((r) => {
               const current = {