free-coding-models 0.5.63 → 0.5.65

package/src/core/probe-cache.js

@@ -12,7 +12,9 @@
  *   📖 Freshness rules (in order):
  *   📖   1. No entry              → due for probe
  *   📖   2. probeVersion mismatch  → due for probe (silently re-probed, entry overwritten)
- *   📖   3. status === 'broken'    → always due (allows recovery detection)
+ *   📖   3. status === 'broken'    → only due AFTER `brokenCooldownMs(consecutiveFailures)`
+ *   📖                            ✅ (issue #146: was always-due → caused quota burn on
+ *   📖                            rate-limited providers like openrouter)
  *   📖   4. now - lastProbedAt >= ttlMs → due for probe
  *   📖   5. otherwise              → fresh, skip
  *
@@ -34,11 +36,13 @@
  *   → recordProbeResults(providerKey, results, opts?) → mutates in-memory cache
  *   → getCacheStats(opts?) → { total, ok, broken, freshCount, staleCount, ... }
  *   → getCachedResultsForProvider(providerKey, opts?) → array of synthesized results
+ *   → brokenCooldownMs(failureCount)            — Backoff ladder for broken models (issue #146)
  *
  * @exports getProbeCachePath, loadCache, flushCache, clearCache,
  *          getModelsDueForProbe, isCacheFresh, recordProbeResults,
  *          getCacheStats, getCachedResultsForProvider,
- *          DEFAULT_PROBE_TTL_MS, CURRENT_PROBE_VERSION
+ *          DEFAULT_PROBE_TTL_MS, BROKEN_COOLDOWN_STEPS_MS,
+ *          brokenCooldownMs, CURRENT_PROBE_VERSION
  *
  * @see src/core/ping-loop.js — the integration point (skips fresh entries)
  * @see src/core/cache.js     — older per-session ping cache (5 min TTL, distinct concern)
@@ -55,6 +59,30 @@ import { atomicWriteJson } from './shared-helpers.js'
 /** 📖 Default probe-result freshness window — 24h. Override via opts.ttlMs. */
 export const DEFAULT_PROBE_TTL_MS = 24 * 60 * 60 * 1000
 
+/**
+ * 📖 Cooldown ladder for broken models (issue #146). After a consecutive failure,
+ * 📖 the model is treated as fresh for the duration returned by `brokenCooldownMs()`,
+ * 📖 so the ping loop skips it instead of hammering it every 2–30s.
+ * 📖 The ladder grows exponentially then plateaus: 30s → 1m → 2m → 5m.
+ * 📖 Reaching the plateau (5min) caps the probe pressure on a permanently-broken
+ * 📖 model at ~0.2 req/min instead of the previous ~30 req/min.
+ */
+export const BROKEN_COOLDOWN_STEPS_MS = [30_000, 60_000, 120_000, 300_000]
+
+/**
+ * 📖 brokenCooldownMs: Cooldown duration for a model that has failed N consecutive times.
+ * 📖 Failure count is 1-indexed (1 = first failure, 2 = second, ...).
+ * 📖 Saturates at the last entry of BROKEN_COOLDOWN_STEPS_MS.
+ *
+ * @param {number} failureCount
+ * @returns {number} milliseconds (always >= 0)
+ */
+export function brokenCooldownMs(failureCount) {
+  const n = Math.max(1, Math.floor(Number(failureCount) || 1))
+  const idx = Math.min(n - 1, BROKEN_COOLDOWN_STEPS_MS.length - 1)
+  return BROKEN_COOLDOWN_STEPS_MS[idx]
+}
+
 /**
  * 📖 Bump this number whenever ping behaviour changes (new endpoint, different prompt,
  * 📖 new provider factory from t7, etc.). On load, any entry whose probeVersion differs
@@ -270,8 +298,16 @@ export function getModelsDueForProbe(providerKey, modelIds, opts = {}) {
     if (typeof entry.probeVersion !== 'number' || entry.probeVersion !== probeVersion) {
       due.push(id); continue
     }
-    // Rule 3: broken → always due (recovery detection)
-    if (entry.status === 'broken') { due.push(id); continue }
+    // Rule 3: broken → due only AFTER brokenCooldownMs elapsed (issue #146)
+    // 📖 Previous behaviour re-pinged broken models every cycle, which burned
+    // 📖 rate-limited providers' quota (openrouter: ~1000 req/day cap).
+    // 📖 Now we honour an exponential backoff: 30s → 1m → 2m → 5m (plateau).
+    if (entry.status === 'broken') {
+      const cooldown = brokenCooldownMs(entry.consecutiveFailures ?? 1)
+      if (now - entry.lastProbedAt < cooldown) continue
+      due.push(id)
+      continue
+    }
     // Rule 4: TTL expired → due
     if (typeof entry.lastProbedAt !== 'number' || now - entry.lastProbedAt >= ttlMs) {
       due.push(id); continue
@@ -301,7 +337,13 @@ export function isCacheFresh(providerKey, modelId, opts = {}) {
   const entry = cache?.providers?.[providerKey]?.models?.[modelId]
   if (!entry) return false
   if (typeof entry.probeVersion !== 'number' || entry.probeVersion !== probeVersion) return false
-  if (entry.status === 'broken') return false
+  if (entry.status === 'broken') {
+    // 📖 Issue #146 — broken models are now FRESH for `brokenCooldownMs` after their
+    // 📖 last failure, instead of always being due. This caps the probe pressure
+    // 📖 on permanently-broken models at ~0.2 req/min instead of ~30 req/min.
+    const cooldown = brokenCooldownMs(entry.consecutiveFailures ?? 1)
+    return now - entry.lastProbedAt < cooldown
+  }
   if (typeof entry.lastProbedAt !== 'number') return false
   return now - entry.lastProbedAt < ttlMs
 }
@@ -350,12 +392,19 @@ export function recordProbeResults(providerKey, results, opts = {}) {
   for (const raw of results || []) {
     try {
       const r = normaliseResult(raw)
+      // 📖 Consecutive failure tracker (issue #146): drives the broken-cooldown ladder.
+      // 📖 `ok` resets to 0; `broken` increments from the previous value (or 1 if absent).
+      const prev = bucket[r.modelId]
+      const consecutiveFailures = r.status === 'broken'
+        ? ((prev && Number.isInteger(prev.consecutiveFailures)) ? prev.consecutiveFailures + 1 : 1)
+        : 0
       bucket[r.modelId] = {
         status: r.status,
         lastProbedAt: now,
         latencyMs: r.latencyMs,
         lastError: r.lastError,
         probeVersion: CURRENT_PROBE_VERSION,
+        consecutiveFailures,
       }
       written++
     } catch {

package/src/core/provider-cooldown.js

@@ -0,0 +1,185 @@
+/**
+ * @file provider-cooldown.js
+ * @description Per-provider quota circuit-breaker for the health-ping loop.
+ *
+ * @details
+ *   📖 Why this exists:
+ *   📖 - OpenRouter (and similar rate-limited gateways) enforce rate limits at the
+ *   📖   **provider/key** level, not per-model. A single 429 "Rate limit exceeded"
+ *   📖   response means **the whole key is paused**, often for hours.
+ *   📖 - Without this breaker, FCM keeps re-pinging every model of that provider
+ *   📖   every 2–30 seconds (because probe-cache treats `broken` as always-due),
+ *   📖   burning the user's daily quota in minutes (issue #146).
+ *   📖 - This module exposes a tiny in-memory map of paused providers so the ping
+ *   📖   loop can short-circuit them until the Retry-After window expires.
+ *
+ *   📖 Design notes:
+ *   📖 - In-memory only, intentionally. The pause is short-lived (minutes to hours)
+ *   📖   and we don't want a stale pause to outlive a server-side reset.
+ *   📖 - `pauseProviderQuota` keeps the **maximum** pause if the provider is already
+ *   📖   paused, so a rapid succession of 429s with decreasing Retry-After values
+ *   📖   never accidentally **shortens** an existing longer pause.
+ *
+ * @functions
+ *   → isProviderQuotaPaused(providerKey, now?)              — Is this provider currently paused?
+ *   → pauseProviderQuota(providerKey, ms)                    — Set a pause for ms milliseconds
+ *   → providerQuotaPauseRemaining(providerKey, now?)        — ms left in the current pause
+ *   → listPausedProviders(now?)                              — Diagnostic: {[providerKey]: remainingMs}
+ *   → clearProviderQuotaPause(providerKey)                   — Force-reset (testing/escape hatch)
+ *   → parseRetryAfterMs(value)                               — Header-only Retry-After parser
+ *   → extractRetryAfterFromResponse(resp)                    — Header + body parser (OR style message)
+ *
+ * @exports isProviderQuotaPaused, pauseProviderQuota, providerQuotaPauseRemaining,
+ *          listPausedProviders, clearProviderQuotaPause, parseRetryAfterMs,
+ *          extractRetryAfterFromResponse
+ */
+
+// ─── Module state ─────────────────────────────────────────────────────────────
+
+/**
+ * 📖 Map<providerKey, timestamp-ms> until which the provider's quota is considered
+ * 📖 paused. `now >= value` means the pause has expired and the entry is purged lazily.
+ */
+const providerQuotaPausedUntil = new Map()
+
+// ─── Pure helpers (exported, easy to unit test) ───────────────────────────────
+
+/**
+ * 📖 parseRetryAfterMs: Parse a Retry-After header value into milliseconds.
+ * 📖 Accepts either a plain-seconds integer ("120") or an HTTP-date ("Wed, 21 Oct 2026 07:28:00 GMT").
+ * 📖 Returns null if the value is missing, malformed, or already in the past.
+ *
+ * @param {string|number|null|undefined} value
+ * @returns {number|null} milliseconds, or null when not parseable
+ */
+export function parseRetryAfterMs(value) {
+  if (value == null) return null
+  if (typeof value === 'string' && value.trim() === '') return null
+  const seconds = Number(value)
+  if (Number.isFinite(seconds) && seconds >= 0) return Math.round(seconds * 1000)
+  const dateMs = Date.parse(value)
+  if (Number.isFinite(dateMs)) {
+    const delta = dateMs - Date.now()
+    return delta > 0 ? delta : null
+  }
+  return null
+}
+
+/**
+ * 📖 extractRetryAfterFromResponse: Best-effort Retry-After extraction.
+ * 📖 Tries the header first (RFC 7231), then falls back to the OpenRouter-style
+ * 📖 error body "Rate limit exceeded, please try again N seconds later.".
+ * 📖 Returns the delay in milliseconds, or 0 if no signal is found.
+ *
+ * @param {Response} resp — A Fetch Response object (uses .headers + .clone()/.text()).
+ * @returns {Promise<number>}
+ */
+export async function extractRetryAfterFromResponse(resp) {
+  if (!resp || typeof resp.headers?.get !== 'function') return 0
+
+  // 1) Standard Retry-After header (lowercase lookup matches Headers semantics)
+  const headerMs = parseRetryAfterMs(resp.headers.get('retry-after'))
+  if (headerMs && headerMs > 0) return headerMs
+
+  // 2) OpenRouter-style message body, e.g.: "Rate limit exceeded, please try again 50680 seconds later."
+  try {
+    const text = await resp.clone().text()
+    if (!text) return 0
+    const m = text.match(/try again (\d+)\s*seconds? later/i)
+    if (m) {
+      const secs = parseInt(m[1], 10)
+      if (Number.isFinite(secs) && secs > 0) return secs * 1000
+    }
+  } catch {
+    // Body unreadable — treat as no signal, don't throw.
+  }
+  return 0
+}
+
+// ─── Pause map API ────────────────────────────────────────────────────────────
+
+/**
+ * 📖 isProviderQuotaPaused: Return true while the provider's quota pause is active.
+ * 📖 Lazily purges expired pauses so the map stays bounded over long sessions.
+ *
+ * @param {string} providerKey
+ * @param {number} [now=Date.now()]
+ * @returns {boolean}
+ */
+export function isProviderQuotaPaused(providerKey, now = Date.now()) {
+  const until = providerQuotaPausedUntil.get(providerKey)
+  if (until == null) return false
+  if (now >= until) {
+    providerQuotaPausedUntil.delete(providerKey)
+    return false
+  }
+  return true
+}
+
+/**
+ * 📖 pauseProviderQuota: Mark a provider as paused for `ms` milliseconds.
+ * 📖 If already paused, keeps the longer of the existing and new windows (never shortens).
+ *
+ * @param {string} providerKey
+ * @param {number} ms — Duration in milliseconds. Non-positive values are ignored.
+ * @returns {boolean} true if the pause was applied or extended, false otherwise.
+ */
+export function pauseProviderQuota(providerKey, ms, opts = {}) {
+  if (!providerKey || typeof ms !== 'number' || !Number.isFinite(ms) || ms <= 0) return false
+  const now = opts.now ?? Date.now()
+  const until = now + ms
+  const cur = providerQuotaPausedUntil.get(providerKey) ?? 0
+  // 📖 Keep the max so a 5s pause arriving after a 14h pause never shortens it.
+  if (until > cur) providerQuotaPausedUntil.set(providerKey, until)
+  return true
+}
+
+/**
+ * 📖 providerQuotaPauseRemaining: How many ms remain on the current pause.
+ * 📖 Returns 0 when the pause has expired or the provider is not paused.
+ *
+ * @param {string} providerKey
+ * @param {number} [now=Date.now()]
+ * @returns {number} milliseconds remaining (always >= 0)
+ */
+export function providerQuotaPauseRemaining(providerKey, now = Date.now()) {
+  const until = providerQuotaPausedUntil.get(providerKey)
+  if (!until) return 0
+  const remaining = until - now
+  if (remaining <= 0) {
+    providerQuotaPausedUntil.delete(providerKey)
+    return 0
+  }
+  return remaining
+}
+
+/**
+ * 📖 listPausedProviders: Snapshot of currently paused providers + ms remaining.
+ * 📖 Intended for diagnostics (TUI footer chip, daemon /stats, error logs).
+ *
+ * @param {number} [now=Date.now()]
+ * @returns {Record<string, number>} {[providerKey]: msRemaining}
+ */
+export function listPausedProviders(now = Date.now()) {
+  const out = {}
+  for (const [providerKey, until] of providerQuotaPausedUntil) {
+    const remaining = until - now
+    if (remaining > 0) out[providerKey] = remaining
+  }
+  return out
+}
+
+/**
+ * 📖 clearProviderQuotaPause: Force-reset the pause for a provider (or all).
+ * 📖 Mainly a test/escape hatch — production code should let pauses expire naturally.
+ *
+ * @param {string} [providerKey] — omit to clear all pauses
+ * @returns {void}
+ */
+export function clearProviderQuotaPause(providerKey) {
+  if (!providerKey) {
+    providerQuotaPausedUntil.clear()
+    return
+  }
+  providerQuotaPausedUntil.delete(providerKey)
+}

package/src/tui/app.js

@@ -139,6 +139,10 @@ import {
   pruneStaleEntries as pruneProbeCacheStaleEntries,
   DEFAULT_PROBE_TTL_MS,
 } from '../core/probe-cache.js'
+import {
+  isProviderQuotaPaused,
+  listPausedProviders,
+} from '../core/provider-cooldown.js'
 import { checkConfigSecurity } from '../core/security.js'
 import { buildCliHelpText } from './cli-help.js'
 import { detectActiveTheme, THEME_BG_RGB, getTheme, patchThemeBg } from './theme.js'
@@ -1046,6 +1050,8 @@ export async function runApp(cliArgs, config, startupOptions = {}) {
       probeCacheMisses: state.probeCacheMisses || 0,
       probeCacheBrokenHidden: state.probeCacheBrokenHidden || 0,
       showBrokenMode: !!state.showBrokenMode,
+      // 📖 Provider circuit-breaker (issue #146): footer chip for paused providers
+      pausedProviders: state.pausedProviders || {},
       // 📖 Enrichment chips (t4 + t5): benchmark catalog + models.dev provenance
       // 📖 Read fresh from moduleEnrichmentStats on every render so the async
       // 📖 models.dev fetch (background, never blocks the TUI) is reflected as
@@ -1229,6 +1235,13 @@ export async function runApp(cliArgs, config, startupOptions = {}) {
         if (!state.config.favorites.includes(favKey)) return
       }
 
+      // 📖 Per-provider circuit-breaker (issue #146): if the provider's quota is
+      // 📖 paused (e.g. openrouter returned 429 "try again 50680s later"), skip
+      // 📖 every model of that provider until the Retry-After window expires.
+      // 📖 Without this, broken/overloaded models are re-pinged every cycle,
+      // 📖 burning the user's daily quota in minutes.
+      if (isProviderQuotaPaused(r.providerKey)) return
+
       // 📖 Probe-cache (t1): skip models that are still fresh + ok in the cache.
       // 📖 Broken models always pass through isCacheFresh() (it returns false for them),
       // 📖 which gives us automatic recovery detection for free.
@@ -1241,6 +1254,11 @@ export async function runApp(cliArgs, config, startupOptions = {}) {
       })
     })
 
+    // 📖 Take a snapshot of currently paused providers so the TUI footer can
+    // 📖 surface a "⏸ openrouter (14h)" chip while we wait out a Retry-After.
+    // 📖 Updates every cycle so the remaining-time ticks down live.
+    state.pausedProviders = listPausedProviders()
+
     refreshAutoPingMode()
     scheduleNextPing()
     } catch (err) {

package/src/tui/render-table.js

@@ -216,6 +216,7 @@ export function renderTable({
   probeCacheMisses = 0,
   probeCacheBrokenHidden = 0,
   showBrokenMode = false,
+  pausedProviders = {},          // 📖 issue #146: { openrouter: 50680000 } while a provider is paused
   enrichmentBenchCount = 0,
   enrichmentBenchLastUpdated = null,
   metaSourceCounts = null,
@@ -1235,6 +1236,36 @@ export function renderTable({
     probeCacheLabel = chalk.bgRgb(40, 90, 140).rgb(220, 235, 255).bold(` ${cachedTxt}${brokenTxt} `)
   }
 
+  // 📖 Provider circuit-breaker chip (issue #146): surfaces any provider that the
+  // 📖 health-ping loop has paused because it returned a 429 quota response.
+  // 📖 Format: `⏸ openrouter 14h · ⏸ mistral 5m`. The live duration auto-rolls
+  // 📖 to s/m/h/d so users see a real countdown and don't get surprised later
+  // 📖 when their requests stop working ("wait, why is everything 429-ing?").
+  let pausedProvidersLabel = ''
+  if (pausedProviders && typeof pausedProviders === 'object') {
+    const fmtRemaining = (ms) => {
+      if (!Number.isFinite(ms) || ms <= 0) return '0s'
+      const totalSec = Math.round(ms / 1000)
+      const day = Math.floor(totalSec / 86400)
+      const hr = Math.floor((totalSec % 86400) / 3600)
+      const min = Math.floor((totalSec % 3600) / 60)
+      const sec = totalSec % 60
+      if (day > 0) return `${day}d${hr}h`
+      if (hr > 0) return `${hr}h${min}m`
+      if (min > 0) return `${min}m${sec}s`
+      return `${sec}s`
+    }
+    const entries = Object.entries(pausedProviders)
+      .filter(([, ms]) => Number.isFinite(ms) && ms > 0)
+      .sort(([a], [b]) => a.localeCompare(b))
+    if (entries.length > 0) {
+      const txt = entries.map(([k, ms]) => `⏸ ${k} ${fmtRemaining(ms)}`).join(' \u00b7 ')
+      // 📖 Warm amber background — distinct from the blue probe-cache chip so the
+      // 📖 user can spot it at a glance and understand "these providers are resting".
+      pausedProvidersLabel = chalk.bgRgb(140, 80, 30).rgb(255, 230, 200).bold(` ${txt} `)
+    }
+  }
+
   // 📖 Enrichment chips (t4 + t5):
   // 📖 `📊 bench N (date) · 📡 live K / 📦 cached M`
   // 📖 Always shown when any enrichment state exists.
@@ -1277,8 +1308,8 @@ export function renderTable({
     }
   }
 
-  // 📖 Line 3: Speed Test + Global Benchmark + Probe + Probe-cache + Enrichment + Quota + Last release
-  if (releaseLabel || speedTestLabel || globalBenchmarkLabel || probeLabel || probeCacheLabel || enrichmentLabel || quotaLabel) {
+  // 📖 Line 3: Speed Test + Global Benchmark + Probe + Probe-cache + Paused-providers + Enrichment + Quota + Last release
+  if (releaseLabel || speedTestLabel || globalBenchmarkLabel || probeLabel || probeCacheLabel || pausedProvidersLabel || enrichmentLabel || quotaLabel) {
     const parts = [
       { text: '  ', key: null },
       { text: speedTestLabel, key: 'a' },
@@ -1288,6 +1319,8 @@ export function renderTable({
       { text: probeLabel, key: null },
       { text: probeCacheLabel ? '  ' : '', key: null },
       { text: probeCacheLabel, key: null },
+      { text: pausedProvidersLabel ? '  ' : '', key: null },
+      { text: pausedProvidersLabel, key: null },
       { text: enrichmentLabel ? '  ' : '', key: null },
       { text: enrichmentLabel, key: null },
       { text: quotaLabel ? '  ' : '', key: null },

package/src/tui/tui-state.js

@@ -297,6 +297,13 @@ export function createTuiState({
     probeCacheTtlMs: null,        // 📖 Set in app.js from cliArgs.probeTtlMs || DEFAULT
     showBrokenMode: false,
 
+    // 📖 Per-provider quota circuit-breaker (issue #146): snapshot of providers
+    // 📖 currently paused because they returned 429 during a health-probe.
+    // 📖 Format: { [providerKey]: remainingMs } — refreshed each ping cycle
+    // 📖 so the TUI footer can show a live "⏸ openrouter (14h)" chip.
+    // 📖 Empty when no provider is in pause.
+    pausedProviders: {},
+
     // 📖 Runtime telemetry overlay state (t3): Shift+W opens the Runtime Report
     // 📖 overlay. Data is loaded fresh from ~/.free-coding-models/runtime-telemetry.json
     // 📖 when the overlay opens — no polling, no daemon dependency.