@lawaty/lacode 2.3.0-beta.0 → 2.3.0-beta.2

package/CHANGELOG.md

@@ -9,6 +9,35 @@ commit as the changelog entry it describes.
 
 ## [Unreleased]
 
+## [2.3.0-beta.2] — degenerate refreshes never shrink the pool
+
+### Added
+
+- Pool: the refresh checks the live opencode status (`https://status.opencode.ai/api/status`) before scanning. When opencode's serving path (`inference`, `go`, `free`, `payg`) reports degraded and the current pool is intact, the scan is skipped, the pool is kept untouched, and a recheck is scheduled in 5 minutes. The signal is cached for 5 minutes and an unreachable status endpoint never blocks a refresh.
+- Pool: the refresh writer now refuses any selection below half the previous pool (minimum four slots). A degenerate scan keeps the previous `pool.txt`, logs the refusal, and schedules a 5-minute recheck. A thin or missing pool is always allowed to recover.
+- Pool: after any failed run, the recheck happens within 5 minutes even when the stated reset is hours out; a sooner provider reset is still honoured.
+
+### Changed
+
+- Pool: the transient retry cadence while degraded is 5 minutes instead of waiting out the full stated reset, so recovery is detected on the next recheck rather than the next 6-hour slot.
+- `refresh-pool --install-timer --stable` now also copies `scripts/provider-status.mjs` to the stable lib dir.
+
+## [2.3.0-beta.1] — provider degradation backoff
+
+### Added
+
+- Pool: a provider is now treated as degraded when two distinct models on it fail with transient, non-quota errors inside ten minutes. Every slot on that provider is benched for one flat 30-minute `provider-degraded` window (never a strike), `pool_status` names the provider, and a probe re-admits a model as soon as it answers. A single model failure keeps the existing 60-second flat outage window.
+- Pool: `npm run score` stops probing a provider after three consecutive provider-level timeouts or network errors, keeps its previous rows, and warns that the provider is degraded. The scan budget is reserved for healthy providers.
+- `oc-sync`: the local `file://.../lacode/dist/index.js` plugin entry is rewritten to the published `@lawaty/lacode@<version>` spec before syncing, so a remote host never receives a path that only exists on the development machine.
+
+### Changed
+
+- Pool: a flat outage window can now be longer than 60 seconds for a provider-wide degradation; it still never enters the strike memory, so recovery starts clean.
+
+### Fixed
+
+- Pool: repeated timeouts on a degraded provider no longer re-probe every model every few minutes or inflate each model's strike count.
+
 ## [2.3.0-beta.0] — reliable pool refresh, autonomy modes, and empty-response failover
 
 ### Added

package/dist/agents.js

@@ -35,6 +35,8 @@ var STUCK_IDLE_MS = 3 * 60 * 1e3;
 var MAX_CLAIM_AGE_MS = 30 * 60 * 1e3;
 var HANG_COOLDOWN_MS = 10 * 60 * 1e3;
 var UNAVAILABLE_COOLDOWN_MS = 60 * 1e3;
+var PROVIDER_DEGRADED_COOLDOWN_MS = 30 * 60 * 1e3;
+var PROVIDER_DEGRADED_WINDOW_MS = 10 * 60 * 1e3;
 var COOLDOWN_BASE_MS = 60 * 1e3;
 var COOLDOWN_MAX_MS = 15 * 60 * 1e3;
 var QUOTA_COOLDOWN_BASE_MS = 15 * 60 * 1e3;

package/dist/commands.js

@@ -39,6 +39,8 @@ var STUCK_IDLE_MS = 3 * 60 * 1e3;
 var MAX_CLAIM_AGE_MS = 30 * 60 * 1e3;
 var HANG_COOLDOWN_MS = 10 * 60 * 1e3;
 var UNAVAILABLE_COOLDOWN_MS = 60 * 1e3;
+var PROVIDER_DEGRADED_COOLDOWN_MS = 30 * 60 * 1e3;
+var PROVIDER_DEGRADED_WINDOW_MS = 10 * 60 * 1e3;
 var MAX_WEIGHT = 100;
 var COOLDOWN_BASE_MS = 60 * 1e3;
 var COOLDOWN_MAX_MS = 15 * 60 * 1e3;

package/dist/index.js

@@ -37,6 +37,12 @@ var MAX_CLAIM_AGE_MS = 30 * 60 * 1e3;
 var HANG_COOLDOWN_MS = 10 * 60 * 1e3;
 var UNAVAILABLE_COOLDOWN_MS = 60 * 1e3;
 var UNAVAILABLE_REASON = "endpoint-unavailable";
+var PROVIDER_DEGRADED_REASON = "provider-degraded";
+var PROVIDER_DEGRADED_COOLDOWN_MS = 30 * 60 * 1e3;
+var PROVIDER_DEGRADED_WINDOW_MS = 10 * 60 * 1e3;
+var PROVIDER_DEGRADED_MIN_MODELS = 2;
+var TRANSIENT_DEGRADATION_REASONS = /^(?:no answer|no-answer|hang|timeout|timed out|endpoint-unavailable|provider-degraded|empty response|network|server)$/i;
+var FLAT_OUTAGE_REASONS = /* @__PURE__ */ new Set([UNAVAILABLE_REASON, PROVIDER_DEGRADED_REASON]);
 var MAX_WEIGHT = 100;
 var MIN_SLOT_CONTEXT = 1e5;
 var MIN_SLOT_TPM = 2e4;
@@ -711,7 +717,7 @@ function readLimits(s) {
 }
 function recordStrike(s, model, reason, stated) {
   if (!s.dirReady) return;
-  if (reason === UNAVAILABLE_REASON) return coolUnavailable(s, model);
+  if (FLAT_OUTAGE_REASONS.has(reason)) return coolUnavailable(s, model, stated, reason);
   const now = Date.now();
   const slotsForModel = s.slots.filter((candidate) => candidate.model === model);
   const buckets = [...new Set(slotsForModel.map((slot) => slotBucket(slot)))];
@@ -742,13 +748,14 @@ function recordStrike(s, model, reason, stated) {
   }
   return strike;
 }
-function coolUnavailable(s, model) {
+function coolUnavailable(s, model, windowMs = UNAVAILABLE_COOLDOWN_MS, reason = UNAVAILABLE_REASON) {
   if (!s.dirReady) return;
   const now = Date.now();
+  const requested = Number.isFinite(windowMs) && windowMs > 0 ? Math.min(windowMs, STATED_COOLDOWN_MAX_MS) : UNAVAILABLE_COOLDOWN_MS;
   const prior = s.cooling.get(model);
-  const until = now + UNAVAILABLE_COOLDOWN_MS;
+  const until = now + requested;
   if (prior && prior.until > until) return prior;
-  const strike = { model, until, strikes: 1, reason: UNAVAILABLE_REASON, last: now };
+  const strike = { model, until, strikes: 1, reason, last: now };
   s.cooling.set(model, strike);
   const path4 = join2(s.dir, limitFile(model));
   try {
@@ -992,6 +999,24 @@ function coolProvider(s, provider, until, reason) {
   }
   return affected;
 }
+function noteProviderDegradation(s, model, reason) {
+  if (!s.dirReady) return [];
+  if (!TRANSIENT_DEGRADATION_REASONS.test(String(reason))) return [];
+  const provider = providerOf(model);
+  const slots = (s.slots.length > 0 ? s.slots : FALLBACK_SLOTS).filter((slot) => providerOf(slot.model) === provider);
+  if (slots.length < PROVIDER_DEGRADED_MIN_MODELS) return [];
+  const now = Date.now();
+  const failing = /* @__PURE__ */ new Set();
+  for (const slot of slots) {
+    const record = s.cooling.get(slot.model);
+    if (!record || record.until <= now) continue;
+    if (now - record.last > PROVIDER_DEGRADED_WINDOW_MS) continue;
+    if (!TRANSIENT_DEGRADATION_REASONS.test(String(record.reason))) continue;
+    failing.add(slot.model);
+  }
+  if (failing.size < PROVIDER_DEGRADED_MIN_MODELS) return [];
+  return coolProvider(s, provider, now + PROVIDER_DEGRADED_COOLDOWN_MS, PROVIDER_DEGRADED_REASON);
+}
 function recordFailure(s, f) {
   try {
     const path4 = join2(s.dir, "failures.jsonl");
@@ -3349,12 +3374,31 @@ var agentPoolHooks = (async ({ client, serverUrl, directory }, options) => {
     }
   };
   const MAX_ABORT_ATTEMPTS = 3;
+  const markProviderDegradation = async (log2, model, reason) => {
+    const provider = providerOf(model);
+    const slots = s.slots.length > 0 ? s.slots : FALLBACK_SLOTS;
+    const already = slots.some(
+      (slot) => providerOf(slot.model) === provider && s.cooling.get(slot.model)?.reason === PROVIDER_DEGRADED_REASON
+    );
+    const cooled = noteProviderDegradation(s, model, reason);
+    if (!already && cooled.length > 0) {
+      const until = new Date(Date.now() + PROVIDER_DEGRADED_COOLDOWN_MS).toISOString().slice(11, 19);
+      await log2("warn", `provider ${provider} degraded; ${cooled.length} slot(s) benched until ${until}Z`, {
+        provider,
+        reason,
+        models: cooled.map((record) => record.model)
+      });
+    }
+  };
   const coolForHang = async (claim, log2) => {
     const now = Date.now();
     const prior = s.cooling.get(claim.m);
     if (prior && prior.until > now) return;
     const strike = recordStrike(s, claim.m, "hang", HANG_FALLBACK_MS);
-    if (strike) await log2("warn", `cooling ${claim.m} for ${Math.round((strike.until - now) / 1e3)}s (hang)`);
+    if (strike) {
+      await log2("warn", `cooling ${claim.m} for ${Math.round((strike.until - now) / 1e3)}s (hang)`);
+      await markProviderDegradation(log2, claim.m, "hang");
+    }
   };
   const claimFor = (sessionID) => {
     if (!sessionID) return void 0;
@@ -3782,6 +3826,7 @@ var agentPoolHooks = (async ({ client, serverUrl, directory }, options) => {
               } else if (outage) {
                 const transient = coolUnavailable(s, own.claim.m);
                 if (transient) await log("warn", `cooling ${own.claim.m} for ${Math.round((transient.until - Date.now()) / 1e3)}s (${transient.reason})`);
+                await markProviderDegradation(log, own.claim.m, "endpoint-unavailable");
               } else {
                 const stated2 = statedCooldown(message);
                 const strike2 = recordStrike(s, own.claim.m, named, stated2);
@@ -3915,6 +3960,7 @@ var agentPoolHooks = (async ({ client, serverUrl, directory }, options) => {
               if (transient) {
                 await log("warn", `cooling ${own.claim.m} for ${Math.round((transient.until - Date.now()) / 1e3)}s (${transient.reason}) after a terminal error`);
               }
+              await markProviderDegradation(log, own.claim.m, "endpoint-unavailable");
             } else {
               stated2 = statedCooldown(haystack2) ?? retryAfterMs(error?.data?.responseHeaders);
               const strike2 = recordStrike(s, own.claim.m, named, stated2);
@@ -4087,6 +4133,7 @@ var agentPoolHooks = (async ({ client, serverUrl, directory }, options) => {
         if (isUnavailableText(haystack) || !limited && !auth && transientGateway) {
           const transient = coolUnavailable(s, model);
           if (transient) await log("warn", `cooling ${model} for ${Math.round((transient.until - Date.now()) / 1e3)}s (${transient.reason})`);
+          await markProviderDegradation(log, model, "endpoint-unavailable");
           await failOver(info.sessionID, model, transient?.reason ?? UNAVAILABLE_REASON, { cool: false });
           return;
         }
@@ -4166,6 +4213,7 @@ var agentPoolHooks = (async ({ client, serverUrl, directory }, options) => {
         const strike = recordStrike(s, spawn4.model, "empty response");
         const seconds = strike ? Math.max(1, Math.round((strike.until - Date.now()) / 1e3)) : 0;
         await log("warn", `empty response from ${spawn4.model} on slot ${spawn4.slot}; benched ${seconds}s`);
+        await markProviderDegradation(log, spawn4.model, "empty response");
         output.output = `Task returned an empty response from ${spawn4.model} (slot ${spawn4.slot}); the model has been benched for ${seconds}s so the next spawn uses another slot.`;
       }
       spawns.delete(key);
@@ -4242,6 +4290,7 @@ ${detail}` : detail;
           };
           const total = [...snap.counts.values()].reduce((sum, n) => sum + n, 0);
           const coolingNow = [...snap.cooling.entries()].filter(([, st]) => st.until > now);
+          const degradedProviders = [...new Set(coolingNow.filter(([, st]) => st.reason === PROVIDER_DEGRADED_REASON).map(([model]) => providerOf(model)))];
           const buckets = /* @__PURE__ */ new Map();
           for (const slot of slots) {
             const name = slotBucket(slot);
@@ -4262,6 +4311,7 @@ ${detail}` : detail;
 one ordered pass in slot order: each slot fills to its weight, then overflow goes to the least-loaded slot. Capacity is the sum of the weights; repeat a model on another line to give it more (FULL = at weight).`,
               ...next ? [] : ["next: none - no routable slot; every slot is cooling, catalog-retired, or provider-unconfigured. A spawn will fail explicitly instead of routing."],
               ...sharedBuckets.length ? [`buckets: ${sharedBuckets.map(([name, entry]) => `${name} ${entry.seats} seat(s)${entry.cooling ? " COOLING" : ""}`).join("; ")}`] : [],
+              ...degradedProviders.length ? [`provider degraded: ${degradedProviders.join(", ")} (flat window; a probe re-admits a model when it answers)`] : [],
               ...slots.map(line),
               "",
               "by process:",

package/dist/lacode.js

@@ -53,6 +53,8 @@ var STUCK_IDLE_MS = 3 * 60 * 1e3;
 var MAX_CLAIM_AGE_MS = 30 * 60 * 1e3;
 var HANG_COOLDOWN_MS = 10 * 60 * 1e3;
 var UNAVAILABLE_COOLDOWN_MS = 60 * 1e3;
+var PROVIDER_DEGRADED_COOLDOWN_MS = 30 * 60 * 1e3;
+var PROVIDER_DEGRADED_WINDOW_MS = 10 * 60 * 1e3;
 var COOLDOWN_BASE_MS = 60 * 1e3;
 var COOLDOWN_MAX_MS = 15 * 60 * 1e3;
 var QUOTA_COOLDOWN_BASE_MS = 15 * 60 * 1e3;

package/dist/pool.js

@@ -37,6 +37,12 @@ var MAX_CLAIM_AGE_MS = 30 * 60 * 1e3;
 var HANG_COOLDOWN_MS = 10 * 60 * 1e3;
 var UNAVAILABLE_COOLDOWN_MS = 60 * 1e3;
 var UNAVAILABLE_REASON = "endpoint-unavailable";
+var PROVIDER_DEGRADED_REASON = "provider-degraded";
+var PROVIDER_DEGRADED_COOLDOWN_MS = 30 * 60 * 1e3;
+var PROVIDER_DEGRADED_WINDOW_MS = 10 * 60 * 1e3;
+var PROVIDER_DEGRADED_MIN_MODELS = 2;
+var TRANSIENT_DEGRADATION_REASONS = /^(?:no answer|no-answer|hang|timeout|timed out|endpoint-unavailable|provider-degraded|empty response|network|server)$/i;
+var FLAT_OUTAGE_REASONS = /* @__PURE__ */ new Set([UNAVAILABLE_REASON, PROVIDER_DEGRADED_REASON]);
 var MAX_WEIGHT = 100;
 var MIN_SLOT_CONTEXT = 1e5;
 var MIN_SLOT_TPM = 2e4;
@@ -711,7 +717,7 @@ function readLimits(s) {
 }
 function recordStrike(s, model, reason, stated) {
   if (!s.dirReady) return;
-  if (reason === UNAVAILABLE_REASON) return coolUnavailable(s, model);
+  if (FLAT_OUTAGE_REASONS.has(reason)) return coolUnavailable(s, model, stated, reason);
   const now = Date.now();
   const slotsForModel = s.slots.filter((candidate) => candidate.model === model);
   const buckets = [...new Set(slotsForModel.map((slot) => slotBucket(slot)))];
@@ -742,13 +748,14 @@ function recordStrike(s, model, reason, stated) {
   }
   return strike;
 }
-function coolUnavailable(s, model) {
+function coolUnavailable(s, model, windowMs = UNAVAILABLE_COOLDOWN_MS, reason = UNAVAILABLE_REASON) {
   if (!s.dirReady) return;
   const now = Date.now();
+  const requested = Number.isFinite(windowMs) && windowMs > 0 ? Math.min(windowMs, STATED_COOLDOWN_MAX_MS) : UNAVAILABLE_COOLDOWN_MS;
   const prior = s.cooling.get(model);
-  const until = now + UNAVAILABLE_COOLDOWN_MS;
+  const until = now + requested;
   if (prior && prior.until > until) return prior;
-  const strike = { model, until, strikes: 1, reason: UNAVAILABLE_REASON, last: now };
+  const strike = { model, until, strikes: 1, reason, last: now };
   s.cooling.set(model, strike);
   const path = join2(s.dir, limitFile(model));
   try {
@@ -995,6 +1002,24 @@ function coolProvider(s, provider, until, reason) {
   }
   return affected;
 }
+function noteProviderDegradation(s, model, reason) {
+  if (!s.dirReady) return [];
+  if (!TRANSIENT_DEGRADATION_REASONS.test(String(reason))) return [];
+  const provider = providerOf(model);
+  const slots = (s.slots.length > 0 ? s.slots : FALLBACK_SLOTS).filter((slot) => providerOf(slot.model) === provider);
+  if (slots.length < PROVIDER_DEGRADED_MIN_MODELS) return [];
+  const now = Date.now();
+  const failing = /* @__PURE__ */ new Set();
+  for (const slot of slots) {
+    const record = s.cooling.get(slot.model);
+    if (!record || record.until <= now) continue;
+    if (now - record.last > PROVIDER_DEGRADED_WINDOW_MS) continue;
+    if (!TRANSIENT_DEGRADATION_REASONS.test(String(record.reason))) continue;
+    failing.add(slot.model);
+  }
+  if (failing.size < PROVIDER_DEGRADED_MIN_MODELS) return [];
+  return coolProvider(s, provider, now + PROVIDER_DEGRADED_COOLDOWN_MS, PROVIDER_DEGRADED_REASON);
+}
 function recordFailure(s, f) {
   try {
     const path = join2(s.dir, "failures.jsonl");
@@ -1068,6 +1093,8 @@ export {
   ORPHAN_TMP_MS,
   PACKAGE_ROOT,
   PRESETS_DIR,
+  PROVIDER_DEGRADED_COOLDOWN_MS,
+  PROVIDER_DEGRADED_REASON,
   PROVIDER_LIST_TTL_MS,
   QUOTA_COOLDOWN_BASE_MS,
   QUOTA_COOLDOWN_MAX_MS,
@@ -1112,6 +1139,7 @@ export {
   manualVariant,
   markProbe,
   noteActivity,
+  noteProviderDegradation,
   noteTpm,
   notifyDefault,
   parseRateLimitHeaders,

package/dist/tunnel.js

@@ -46,6 +46,8 @@ var STUCK_IDLE_MS = 3 * 60 * 1e3;
 var MAX_CLAIM_AGE_MS = 30 * 60 * 1e3;
 var HANG_COOLDOWN_MS = 10 * 60 * 1e3;
 var UNAVAILABLE_COOLDOWN_MS = 60 * 1e3;
+var PROVIDER_DEGRADED_COOLDOWN_MS = 30 * 60 * 1e3;
+var PROVIDER_DEGRADED_WINDOW_MS = 10 * 60 * 1e3;
 var COOLDOWN_BASE_MS = 60 * 1e3;
 var COOLDOWN_MAX_MS = 15 * 60 * 1e3;
 var QUOTA_COOLDOWN_BASE_MS = 15 * 60 * 1e3;

package/docs/USER-STORIES.md

@@ -91,6 +91,8 @@ The reconciliation with US-7 is that **acceptance and endorsement are different
 - [ ] Given a spawn fails with a rate-limit-shaped error, When the error is serialized and matched, Then the offending model is marked cooling and a cooldown file is written for it.
 - [ ] Given model A is cooling and another slot is usable, When the next spawn is routed, Then the pick is a different usable model, never the cooling one.
 - [ ] Given a pooled task completes with an empty result, When the task hook returns, Then that model is benched for `empty response`, the result explains the bench, and the next spawn routes to another slot.
+- [ ] Given two distinct models on one provider fail with transient, non-quota errors inside ten minutes, When the second failure is recorded, Then every slot on that provider is benched for one flat `provider-degraded` window and the next spawn routes to another provider.
+- [ ] Given a provider-degraded window lapses, When a probe answers on one of its models, Then that model is re-admitted without a restart, because the bench was evidence about the provider, not a strike against the model.
 - [ ] Given every model is cooling, When a spawn is routed, Then the spawn is refused explicitly with a message naming the cooling slots and their reasons, and no model is routed to.
 - [ ] Given a limit error names a reset time, When the cooldown is recorded, Then the stated reset is used instead of a guessed backoff.
 
@@ -178,11 +180,26 @@ A daily scan was killed by a reboot mid-run, and because systemd records a trigg
 **Acceptance criteria:**
 - [ ] Given the installed timer, When it is rendered, Then it has `OnCalendar=*-*-* 00/6:00:00`, `OnStartupSec=5min`, and `Persistent=true`, and the service has `TimeoutStartSec=infinity`.
 - [ ] Given a scan reaches its wall-clock budget, When the scores are written, Then unscanned and truncated providers keep their previous rows, so a partial run cannot shrink the pool.
-- [ ] Given a model failed transiently (timeout, rate limit, credit), When selection runs, Then its last measured score is retained and a retry is scheduled at the stated reset or 45 minutes later.
+- [ ] Given a model failed transiently (timeout, rate limit, credit), When selection runs, Then its last measured score is retained and a retry is scheduled; the refresh rechecks within 5 minutes while anything failed, even when the stated reset is hours out.
 - [ ] Given a permanent failure (401, 403 agreement, 404), When selection runs, Then the model is not retried and does not fall back to a previous score.
+- [ ] Given three consecutive provider-level timeouts or network errors during a scan, When the breaker trips, Then the scan stops probing that provider, keeps its previous rows, and names the provider as degraded.
 - [ ] Given enough other providers to fill the pool, When selection runs, Then exactly one `opencode` slot is reserved last with weight 2.
 - [ ] Given two refreshes overlap, When the second starts, Then it skips while the lock is held, and a lock left by a dead PID is reclaimed.
 
+### US-40: A degraded scan never shrinks the pool, and recovery is detected within minutes
+**As a** user whose providers degrade for minutes to hours, **I want** the refresh to keep the working pool through the outage and restore it as soon as the degradation lifts, **so that** a transient outage never strands me on two slots and I never wait a full cycle for the recovery.
+
+A refresh once replaced a healthy ten-slot pool with a two-slot scan because nothing stopped a degenerate selection from being written. The writer now refuses any selection below half the previous pool (minimum four slots) and keeps the previous file, and before scanning it checks the live opencode status: when opencode's serving path reports degraded and the pool is intact, the scan is skipped entirely and a recheck is scheduled in 5 minutes. The same 5-minute cadence applies after any failed run, so the moment the status clears — or a scan comes back full — the pool is restored on the next recheck instead of the next 6-hour slot.
+
+**Story ID(s) for tests:** US-40
+**Acceptance criteria:**
+- [ ] Given a healthy pool and a scan that returns fewer slots than half of it (minimum four), When the refresh finishes, Then the previous pool file is left untouched, the refusal is logged, and a recheck is scheduled in 5 minutes.
+- [ ] Given opencode's serving path reports degraded on the live status and the current pool is intact, When the refresh runs, Then no scan is started, the pool is kept, and a recheck is scheduled in 5 minutes.
+- [ ] Given any failed run with a stated reset hours out, When the retry is scheduled, Then the recheck still happens within 5 minutes; a sooner provider reset is honoured.
+- [ ] Given the degradation lifts, When the next 5-minute recheck runs, Then a full scan restores the pool instead of waiting for the next 6-hour slot.
+- [ ] Given a thin or missing pool, When a fuller scan completes, Then it is always written — the guard only ever blocks shrinking, never recovery.
+- [ ] Given the status endpoint is unreachable, When the refresh runs, Then the scan proceeds as before, because an unknown signal must never block a refresh and the write guard still protects the pool.
+
 ---
 
 ## Epic C — Hang detection and auto-recovery
@@ -603,6 +620,7 @@ table was filled in.
 | US-28 | B | `tests/stories/i-omniroute.test.mjs` |
 | US-31 | B | `tests/stories/b-provider-resilience.test.mjs`, `tests/stories/c-hang-recovery.test.mjs` |
 | US-39 | B | `tests/stories/l-pool-refresh.test.mjs` |
+| US-40 | B | `tests/stories/l-pool-refresh.test.mjs`, `tests/refresh-pool-test.mjs`, `tests/provider-status-test.mjs` |
 | US-8 | C | `tests/stories/c-hang-recovery.test.mjs`, `tests/pool-hang-test.mjs` |
 | US-9 | C | `tests/stories/c-hang-recovery.test.mjs`, `tests/pool-hang-test.mjs`, `tests/pool-shared-test.mjs` |
 | US-10 | C | `tests/stories/c-hang-recovery.test.mjs`, `tests/pool-hang-test.mjs`, `tests/pool-timer-test.mjs` |
@@ -636,6 +654,6 @@ The root-level suites under `tests/` are unit-style; the `tests/stories/` suites
 the behavioural layer, written one epic per file with every test name carrying
 its story id. Run everything with `npm test`.
 
-39/39 stories covered by 370 test case(s); thinnest story has 1.
+40/40 stories covered by 375 test case(s); thinnest story has 1.
 No story is untested. The scan covers the `tests/stories/` suites and the root-level
 `tests/*-test.mjs` suites.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lawaty/lacode",
-  "version": "2.3.0-beta.0",
+  "version": "2.3.0-beta.2",
   "description": "Ultimate opencode setup for Go users — cost-controlled pooled subagents, hang recovery, and notifications for opencode.",
   "license": "MIT",
   "type": "module",

package/scripts/pool-selection.mjs

@@ -251,6 +251,9 @@ export function selectPool({
       continue
     }
     const entryRows = Array.isArray(entry?.rows) ? entry.rows : []
+    if (entry?.degraded) {
+      warnings.push(`${entry.provider}: degraded — stopped after consecutive transient failures; previous rows kept`)
+    }
     if (entryRows.length === 0 || entryRows.some(rankable)) continue
     const kinds = new Map()
     for (const row of entryRows) {

package/scripts/refresh-pool.mjs

@@ -23,6 +23,7 @@ import { execFileSync } from "node:child_process"
 import { homedir, userInfo } from "node:os"
 import { dirname, join, resolve } from "node:path"
 import { fileURLToPath } from "node:url"
+import { fetchProviderStatus } from "./provider-status.mjs"
 
 const HOME = homedir()
 export const DEFAULT_SCORES_PATH = join(HOME, ".config", "lacode", "model-scores.json")
@@ -32,9 +33,21 @@ export const STABLE_LIB_DIR = (home = HOME) => join(home, ".local", "lib", "laco
 export const STABLE_SCRIPT = (home = HOME) => join(STABLE_LIB_DIR(home), "refresh-pool.mjs")
 export const STABLE_SCORER = (home = HOME) => join(STABLE_LIB_DIR(home), "score-models.mjs")
 export const STABLE_SELECTION = (home = HOME) => join(STABLE_LIB_DIR(home), "pool-selection.mjs")
+export const STABLE_STATUS = (home = HOME) => join(STABLE_LIB_DIR(home), "provider-status.mjs")
 export const REFRESH_SERVICE_NAME = "lacode-refresh-pool.service"
 export const REFRESH_TIMER_NAME = "lacode-refresh-pool.timer"
 
+/**
+ * A refresh must never replace a working pool with a degenerate one. When the
+ * new selection falls below the keep threshold, the previous pool.txt is left
+ * untouched, the failure is logged loudly, and a fast retry is scheduled.
+ * The threshold is half the previous pool (rounded down), with a floor of
+ * MIN_POOL_SLOTS so a thin-but-real pool is still protected.
+ */
+export const MIN_POOL_SLOTS = 4
+/** While degraded, recheck on this cadence instead of waiting out a long reset. */
+export const DEGRADED_RETRY_S = 5 * 60
+
 const REFRESH_LOCK_DIR = join(HOME, ".config", "lacode", "refresh.lock")
 const REFRESH_LOCK_STALE_MS = 60 * 60 * 1000
 
@@ -189,18 +202,56 @@ function runScorer(max) {
   execFileSync(process.execPath, args, { stdio: "inherit" })
 }
 
-export function scheduleRetry(retry, options = {}) {
+/**
+ * Count the usable model slots in a pool.txt text (comments and blanks ignored).
+ */
+export function poolSlotCount(text) {
+  return splitHeaderAndSlots(text).slots.filter(isSlotLine).length
+}
+
+/**
+ * Pure fail-closed guard. A refresh must never replace a working pool with a
+ * degenerate selection: when the new slot count falls below half the previous
+ * pool (floored at MIN_POOL_SLOTS), the previous pool is kept. There is no
+ * previous pool on a first-ever run, so there is nothing to protect and the
+ * guard passes.
+ */
+export function shouldKeepPool({ currentPoolText, newSlotCount }) {
+  const prev = poolSlotCount(currentPoolText ?? "")
+  if (prev <= 0) return { keep: false, prev, threshold: 0, reason: "no previous pool to protect" }
+  const threshold = Math.max(MIN_POOL_SLOTS, Math.floor(prev / 2))
+  if (newSlotCount >= threshold) return { keep: false, prev, threshold, reason: "" }
+  return {
+    keep: true,
+    prev,
+    threshold,
+    reason: `new selection has ${newSlotCount} usable slot(s), below the keep threshold of ${threshold} (previous pool had ${prev}); refusing to shrink the pool`,
+  }
+}
+
+/**
+ * How long until the next look while degraded. A provider reset in the future
+ * is honoured, but the wait never exceeds the degraded cadence: a reset hours
+ * out must not stop the 5-minute rechecks, and a reset that already passed
+ * means recheck on the cadence. No retry information at all also means the
+ * cadence.
+ */
+export function degradedRetryDelay(retry, now = Date.now()) {
   const at = Date.parse(retry?.after ?? "")
-  if (!Number.isFinite(at)) return { scheduled: false, reason: "no reset time" }
-  const now = options.now ?? Date.now
-  const nowMs = now()
-  const seconds = Math.ceil((at - nowMs) / 1000)
-  if (!Number.isFinite(seconds) || seconds <= 0) return { scheduled: false, reason: "reset is not in the future" }
-  if (seconds > 24 * 60 * 60) return { scheduled: false, reason: "reset is more than a day away" }
+  if (!Number.isFinite(at)) return DEGRADED_RETRY_S
+  const wait = Math.ceil((at - now) / 1000)
+  if (wait <= 0) return DEGRADED_RETRY_S
+  return Math.min(wait, DEGRADED_RETRY_S)
+}
+
+export function scheduleRetryIn(seconds, options = {}) {
+  if (!Number.isFinite(seconds) || seconds <= 0) return { scheduled: false, reason: "delay is not in the future" }
+  if (seconds > 24 * 60 * 60) return { scheduled: false, reason: "delay is more than a day away" }
   const exec = options.exec ?? execFileSync
   const scriptPath = options.scriptPath ?? fileURLToPath(import.meta.url)
   const nodePath = options.nodePath ?? process.execPath
   const home = options.home ?? HOME
+  const nowMs = (options.now ?? Date.now)()
   const path = `${home}/.opencode/bin:/usr/local/bin:/usr/bin:/bin`
   const unit = `lacode-refresh-pool-retry-${Math.floor(nowMs / 1000).toString(36)}`
   try {
@@ -218,12 +269,29 @@ export function scheduleRetry(retry, options = {}) {
       ],
       { stdio: "ignore" },
     )
-    return { scheduled: true, seconds, after: retry.after }
+    return { scheduled: true, seconds }
   } catch (error) {
     return { scheduled: false, reason: String(error?.message ?? error) }
   }
 }
 
+export function scheduleRetry(retry, options = {}) {
+  const at = Date.parse(retry?.after ?? "")
+  if (!Number.isFinite(at)) return { scheduled: false, reason: "no reset time" }
+  const nowMs = (options.now ?? Date.now)()
+  const seconds = Math.ceil((at - nowMs) / 1000)
+  if (!Number.isFinite(seconds) || seconds <= 0) return { scheduled: false, reason: "reset is not in the future" }
+  if (seconds > 24 * 60 * 60) return { scheduled: false, reason: "reset is more than a day away" }
+  const scheduled = scheduleRetryIn(seconds, options)
+  if (scheduled.scheduled) return { ...scheduled, after: retry.after }
+  return scheduled
+}
+
+/** Schedule a recheck on the degraded cadence (default 5 minutes). */
+export function scheduleRetrySoon(seconds = DEGRADED_RETRY_S, options = {}) {
+  return scheduleRetryIn(seconds, options)
+}
+
 export function renderTimerUnits(input = {}) {
   const nodePath = input.nodePath || process.execPath
   const scriptPath = input.scriptPath || fileURLToPath(import.meta.url)
@@ -271,6 +339,7 @@ function installTimerUnits(stable = false) {
       copyFileSync(live, scriptPath)
       copyFileSync(resolve(dirname(live), "score-models.mjs"), STABLE_SCORER(HOME))
       copyFileSync(resolve(dirname(live), "pool-selection.mjs"), STABLE_SELECTION(HOME))
+      copyFileSync(resolve(dirname(live), "provider-status.mjs"), STABLE_STATUS(HOME))
     }
     mkdirSync(dirname(units.servicePath), { recursive: true })
     writeFileAtomic(units.servicePath, units.service)
@@ -314,7 +383,15 @@ function usage() {
   --max N           models per provider passed to the scorer (default: 50)
   --scores PATH     scores file (default: ${DEFAULT_SCORES_PATH})
   --pool PATH       pool file to read/compare/write (default: ${DEFAULT_POOL_PATH})
-  -h, --help        this message`)
+  -h, --help        this message
+
+Degradation behaviour: before scanning, the refresh checks
+https://status.opencode.ai/api/status. When opencode's serving path reports
+degraded and the current pool is intact, the scan is skipped, the pool is kept,
+and a recheck is scheduled in 5 minutes. After a scan, a selection below half
+the previous pool (minimum ${MIN_POOL_SLOTS} slots) is never written; the previous pool
+is kept and a 5-minute recheck is scheduled. A retry is always rechecked
+within 5 minutes while anything failed, even when the stated reset is hours out.`)
 }
 
 async function main() {
@@ -344,6 +421,37 @@ async function main() {
     process.once("exit", releaseRefreshLock)
   }
 
+  const currentPoolText = readTextFile(poolPath)
+
+  // Live degradation signal. When opencode's own serving path reports degraded
+  // and the current pool is intact, probing now can only shrink it: keep the
+  // pool untouched and recheck in 5 minutes. A thin pool is still scanned, so
+  // a first-ever run (or a previously clobbered one) can recover, and the
+  // write guard below protects it if the scan comes back degenerate.
+  let providerStatus = { ok: false, providers: [], services: [] }
+  if (!printOnly) {
+    try {
+      providerStatus = await fetchProviderStatus()
+    } catch (error) {
+      providerStatus = { ok: false, providers: [], services: [], reason: String(error?.message ?? error) }
+    }
+    if (providerStatus.ok && providerStatus.providers.length > 0 && poolSlotCount(currentPoolText) >= MIN_POOL_SLOTS) {
+      const detail = providerStatus.services.map((entry) => `${entry.service}=${entry.status}`).join(", ")
+      console.log(
+        `opencode status degraded (${detail}); keeping the current pool of ${poolSlotCount(currentPoolText)} slots and rechecking in ${DEGRADED_RETRY_S / 60} minutes`,
+      )
+      if (!noRetry) {
+        const scheduled = scheduleRetrySoon()
+        console.log(
+          scheduled.scheduled
+            ? `scheduled degraded recheck in ${scheduled.seconds}s`
+            : `degraded recheck not scheduled: ${scheduled.reason}`,
+        )
+      }
+      return
+    }
+  }
+
   if (!fromScores) {
     try {
       runScorer(max)
@@ -363,7 +471,6 @@ async function main() {
     return
   }
 
-  const currentPoolText = readTextFile(poolPath)
   let built
   try {
     built = buildPool({ scores, currentPoolText, now: new Date() })
@@ -373,7 +480,7 @@ async function main() {
     return
   }
 
-  const oldSlotCount = splitHeaderAndSlots(currentPoolText).slots.filter(isSlotLine).length
+  const oldSlotCount = poolSlotCount(currentPoolText)
   const newSlotCount = built.slots.filter(isSlotLine).length
   const dropped = scores.providers.flatMap((entry) => (Array.isArray(entry?.excluded) ? entry.excluded : []))
   const summary = [
@@ -382,11 +489,30 @@ async function main() {
     ...(built.warnings.length ? [`warnings [${built.warnings.join("; ")}]`] : []),
   ].join(" | ")
 
-  if (!printOnly && !noRetry) {
-    const scheduled = scheduleRetry(scores.retry)
+  // Fail closed on a degenerate selection: keep the previous pool, say so
+  // loudly, and recheck in 5 minutes instead of waiting out a long reset.
+  if (!printOnly) {
+    const kept = shouldKeepPool({ currentPoolText, newSlotCount })
+    if (kept.keep) {
+      console.error(`refresh refused (fail closed): ${kept.reason}`)
+      if (!noRetry) {
+        const scheduled = scheduleRetrySoon()
+        console.log(
+          scheduled.scheduled
+            ? `scheduled degraded recheck in ${scheduled.seconds}s`
+            : `degraded recheck not scheduled: ${scheduled.reason}`,
+        )
+      }
+      process.exitCode = 1
+      return
+    }
+  }
+
+  if (!printOnly && !noRetry && scores.retry) {
+    const scheduled = scheduleRetrySoon(degradedRetryDelay(scores.retry))
     if (scheduled.scheduled) {
-      console.log(`scheduled retry in ${scheduled.seconds}s for ${scores.retry.models.length} model(s) at ${scheduled.after}`)
-    } else if (scores.retry) {
+      console.log(`scheduled retry in ${scheduled.seconds}s for ${scores.retry.models.length} model(s) (reset ${scores.retry.after})`)
+    } else {
       console.log(`retry not scheduled: ${scheduled.reason}`)
     }
   }

package/scripts/score-models.mjs

@@ -68,6 +68,10 @@ const SLOW_MS = 8_000 // at or above this, speed scores 0
 const PROMPT = "Reply with the single word: ok"
 const MAX_TOKENS = 8
 const PROBE_ATTEMPTS = 2 // one retry for a timeout or a dropped connection
+// Three consecutive provider-level timeouts/network errors is enough evidence
+// that the provider is degraded. Stop spending the scan budget on the rest of
+// its catalogue; the previous rows are retained and a retry is scheduled.
+const PROVIDER_BREAKER_AFTER = 3
 /**
  * Retry the same probe only when the next attempt has a chance. A known
  * exhausted balance or a permanent 401/403/404 will not recover in 300ms; the
@@ -601,6 +605,7 @@ async function main() {
       continue
     }
     const rows = []
+    let consecutiveTransient = 0
     for (const candidate of candidates) {
       if (Date.now() >= deadline) {
         budgetHit = true
@@ -614,6 +619,9 @@ async function main() {
         await sleep(delayMs)
         outcome = await probe(candidate, entry.provider)
       }
+      const kind = outcome.kind ?? classifyFailure({ status: outcome.status, reason: outcome.reason })
+      const providerFault = !outcome.ok && ["timeout", "network", "server"].includes(kind)
+      consecutiveTransient = providerFault ? consecutiveTransient + 1 : 0
       const row = { provider: entry.provider, ...candidate, ...outcome, score: null }
       row.score = score(row)
       if (row.score && row.score.value < minScore) {
@@ -628,9 +636,17 @@ async function main() {
       rows.push(row)
       const verdict = outcome.ok ? `${outcome.ms}ms` : outcome.reason.slice(0, 60)
       console.log(`  ${pad(candidate.model.replace(`${entry.provider}/`, ""), 52)} ${padL(verdict, 16)}`)
+      if (consecutiveTransient >= PROVIDER_BREAKER_AFTER) {
+        entry.truncated = true
+        entry.degraded = true
+        console.log(
+          `  provider degraded: ${consecutiveTransient} consecutive ${kind} failures; stopped ${entry.provider} after ${rows.length}/${candidates.length}`,
+        )
+        break
+      }
       await sleep(delayMs)
     }
-    results.push({ provider: entry.provider, error: null, rows, excluded: entry.excluded, truncated: !!entry.truncated })
+    results.push({ provider: entry.provider, error: null, rows, excluded: entry.excluded, truncated: !!entry.truncated, degraded: !!entry.degraded })
   }
   if (budgetHit) {
     console.log(
@@ -733,6 +749,7 @@ async function main() {
             provider: e.provider,
             error: e.error,
             truncated: !!e.truncated,
+            degraded: !!e.degraded,
             excluded: e.excluded ?? [],
             rows: e.rows.flatMap((r) => {
               const current = {